You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by mj...@apache.org on 2016/06/23 18:45:12 UTC
[01/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Repository: incubator-joshua
Updated Branches:
refs/heads/maven-multi-module f178334de -> 1bb8a2037
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/samt/input.bn
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/samt/input.bn b/joshua-core/src/test/resources/bn-en/samt/input.bn
new file mode 100644
index 0000000..d4f7913
--- /dev/null
+++ b/joshua-core/src/test/resources/bn-en/samt/input.bn
@@ -0,0 +1,100 @@
+\u09b0\u09ac\u09c0\u09a8\u09cd\u09a6\u09cd\u09b0\u09a8\u09be\u09a5\u09c7\u09b0 \u099c\u09a8\u09cd\u09ae \u09b9\u09af\u09bc\u09c7\u099b\u09bf\u09b2 \u0995\u09b2\u0995\u09be\u09a4\u09be\u09b0 \u098f\u0995 \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 \u09aa\u09b0\u09bf\u09ac\u09be\u09b0\u09c7 \u0964
+\u09b8\u09be\u09ae\u09cd\u09aa\u09cd\u09b0\u09a4\u09bf\u0995\u0995\u09be\u09b2\u09c7 \u09ad\u09be\u09b0\u09a4\u09c7\u09b0 \u09b8\u0999\u09cd\u0997\u09c7 \u09af\u09c1\u0995\u09cd\u09a4\u09b0\u09be\u09b7\u09cd\u099f\u09cd\u09b0\u09c7\u09b0 \u09b8\u09ae\u09cd\u09aa\u09b0\u09cd\u0995\u09c7\u09b0 \u0989\u09a8\u09cd\u09a8\u09a4\u09bf \u09b9\u09af\u09bc\u09c7\u099b\u09c7 \u0964
+\u0997\u09a3\u09bf\u09a4 \u09a4\u09be\u0987 \u09ac\u09bf\u099c\u09cd\u099e\u09be\u09a8\u09c7\u09b0 \u09ad\u09be\u09b7\u09be \u0964
+\u098f \u09a5\u09c7\u0995\u09c7 \u09b8\u09b9\u099c\u09c7\u0987 \u09ac\u09cb\u099d\u09be \u09af\u09be\u09af\u09bc \u09af\u09c7 \u098f\u0987 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf \u09b9\u09ac\u09c7 \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 \u0964
+\u098f\u0995\u0987 \u09b8\u0999\u09cd\u0997\u09c7 \u09ac\u09be\u0982\u09b2\u09be\u09b0 \u09ad\u09c2\u09ae\u09bf\u0995\u09c7\u09a8\u09cd\u09a6\u09cd\u09b0\u09bf\u0995 \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 \u09aa\u09c2\u09b0\u09cd\u09ac\u09be\u09ad\u09be\u09b7\u0993 \u09ae\u09c7\u09b2\u09c7 \u098f\u0987 \u0989\u09aa\u09a8\u09cd\u09af\u09be\u09b8 \u09a5\u09c7\u0995\u09c7 \u0964
+\u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 \u09ae\u09c1\u099c\u09bf\u09ac \u0993 \u09a4\u09be\u0981\u09b0 \u09a6\u09b2 \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 \u09b8\u0982\u0996\u09cd\u09af\u09be\u0997\u09b0\u09bf\u09b7\u09cd\u09a0\u09a4\u09be \u0985\u09b0\u09cd\u099c\u09a8 \u0995\u09b0\u09c7 \u0964
+\u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 \u09a4\u09be\u09b0 \u0995\u09be\u099c \u099a\u09be\u09b2\u09bf\u09af\u09bc\u09c7 \u09af\u09c7\u09a4\u09c7 \u09a5\u09be\u0995\u09c7\u09a8 \u0964
+\u099f\u09be\u0995\u09cd\u09b8 \u099b\u09be\u09a1\u09bc\u09be\u0993 \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 \u0993 \u0986\u09b0\u0993 \u0995\u09bf\u099b\u09c1 \u09b2\u09bf\u09a8\u09be\u0995\u09cd\u09b8 \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 \u099a\u09b0\u09bf\u09a4\u09cd\u09b0 \u09b0\u09af\u09bc\u09c7\u099b\u09c7 & # 44 ; \u09a4\u09ac\u09c7 \u098f\u0997\u09c1\u09b2\u09cb \u0996\u09c1\u09ac \u09aa\u09cd\u09b0\u099a\u09b2\u09bf\u09a4 \u09a8\u09af\u09bc \u0964
+\u0987\u09b9\u09be \u09b8\u09be\u09ae\u09be\u099c\u09bf\u0995 \u0985\u09ac\u09b8\u09cd\u09a5\u09be\u09b0 \u09aa\u09cd\u09b0\u09c7\u0995\u09cd\u09b7\u09bf\u09a4\u09c7 \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 \u09a4\u09be\u09b0 \u09aa\u09cd\u09b0\u09a4\u09bf\u09aa\u0995\u09cd\u09b7\u09c7\u09b0 \u09b8\u09b9\u09bf\u09a4 \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4 \u0997\u09cd\u09b0\u09b9\u09a8\u09c7 \u09aa\u09cd\u09b0\u099a\u09b2\u09bf\u09a4 \u09a8\u09bf\u09af\u09bc\u09ae \u09ac\u09cd\u09af\u09be\u0996\u09cd\u09af\u09be \u0995\u09b0\u09c7 \u09a5\u09be\u0995\u09c7 \u0964
+\u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 \u09ac\u09be\u09ce\u09b8\u09b0\u09bf\u0995 \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf \u09ae\u09bf\u099f\u09be\u09b0
+\u09e8\u09e6\u09e6\u09ea \u09b8\u09be\u09b2\u09c7 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 \u09b6\u09b9\u09b0\u09c7 \u0985\u09a8\u09c1\u09b7\u09cd\u09a0\u09bf\u09a4 \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f \u09a6\u09b2\u09c7\u09b0 \u099c\u09be\u09a4\u09c0\u09af\u09bc \u09b8\u09ae\u09cd\u09ae\u09c7\u09b2\u09a8 \u09a4\u09bf\u09a8\u09bf \u09ae\u09c2\u09b2 \u09ac\u0995\u09cd\u09a4\u09c3\u09a4\u09be -lrb- keynote speech -rrb- \u09aa\u09cd\u09b0\u09a6\u09be\u09a8 \u0995\u09b0\u09c7\u09a8 \u0964
+\u099c\u09a8\u09b8\u0982\u0996\u09cd\u09af\u09be\u09b0 \u09ad\u09bf\u09a4\u09cd\u09a4\u09bf\u09a4\u09c7 \u0995\u09cd\u09b7\u09ae\u09a4\u09be\u09b0 \u09ac\u09a8\u09cd\u099f\u09a8 \u09aa\u09c2\u09b0\u09cd\u09ac \u09aa\u09be\u0995\u09bf\u09b8\u09cd\u09a4\u09be\u09a8\u09c7\u09b0 \u0985\u09a8\u09c1\u0995\u09c2\u09b2 \u09b9\u0993\u09af\u09bc\u09be\u09af\u09bc \u09aa\u09b6\u09cd\u099a\u09bf\u09ae \u09aa\u09be\u0995\u09bf\u09b8\u09cd\u09a4\u09be\u09a8 & quot ; \u098f\u0995 \u0987\u0989\u09a8\u09bf\u099f \u09a4\u09a4\u09cd\u09a4\u09cd\u09ac & quot ; \u09a8\u09be\u09ae\u09c7 \u098f\u0995 \u0985\u09ad\u09bf\u09a8\u09ac \u09a7\u09be\u09b0\u09a3\u09be\u09b0 \u09b8\u09c2\u09a4\u09cd\u09b0\u09aa\u09be\u09a4 \u0995\u09b0\u09c7 & # 44 ; \u09af\u09c7\u0996\u09be\u09a8\u09c7 \u09b8\u09ae\u0997\u09cd\u09b0 \u09aa\u09b6\u09cd\u099a\u09bf\u09ae \u09aa\u09be\u0995\u09bf\u09b8\u09cd\u09a4\u09be\u09a8 \u098f\u0995\u099f\u09bf \u09aa\u09cd\u09b0\u09a6\u09c7\u09b6 \u09b9\u09bf\u09b8\u09c7\u09ac\u09c7 \u09ac\u09bf\u09ac\u09c7\u099a\u09bf\u09a4 \u09b9\u09af\u09bc \u0964
+\u2022 \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 \u09a4\u09a4\u09cd\u09a4\u09cd\u09ac
+\u09ac\u09b9\u09bf\u0983\u09b8\u0982\u09af\u09cb\u0997 \u09b8\u09ae\u09c2\u09b9
+\u099f\u09be\u099f\u09be \u0995\u09ae\u09bf\u0989\u09a8\u09bf\u0995\u09c7\u09b6\u09a8\u09b8\u09c7\u09b0 \u09ac\u09bf\u09a6\u09c7\u09b6 \u09b8\u099e\u09cd\u099a\u09be\u09b0 \u09a8\u09bf\u0997\u09ae \u09b2\u09bf\u09ae\u09bf\u099f\u09c7\u09a1 \u09ad\u09ac\u09a8 & # 44 ; \u098f\u099f\u09bf \u09b6\u09b9\u09b0\u09c7\u09b0 \u099f\u09c7\u09b2\u09bf\u09af\u09cb\u0997\u09be\u09af\u09cb\u0997 \u09ac\u09cd\u09af\u09ac\u09b8\u09cd\u09a5\u09be\u09b0 \u098f\u0995\u099f\u09bf \u0985\u09a8\u09cd\u09af\u09a4\u09ae \u09aa\u09cd\u09b0\u09a7\u09be\u09a8 \u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0
+\u09a4\u09bf\u09a8\u09bf \u09b8\u09c7\u0987 \u09ac\u099b\u09b0\u09c7\u09b0 \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 \u099c\u09be\u09a4\u09c0\u09af\u09bc \u09a8\u09bf\u09b0\u09cd\u09ac\u09be\u099a\u09a8\u09c7 \u099c\u09af\u09bc\u09c0 \u09b9\u09a8 \u098f\u09ac\u0982 \u09ae\u09be\u09b0\u09cd\u0995\u09bf\u09a8 \u09af\u09c1\u0995\u09cd\u09a4\u09b0\u09be\u09b7\u09cd\u099f\u09cd\u09b0\u09c7\u09b0 \u09ea\u09ea\u09a4\u09ae \u09b0\u09be\u09b7\u09cd\u099f\u09cd\u09b0\u09aa\u09a4\u09bf \u09b9\u09bf\u09b8\u09c7\u09ac\u09c7 \u09a8\u09bf\u09b0\u09cd\u09ac\u09be\u099a\u09bf\u09a4 \u09b9\u09a8 \u0964
+\u09ac\u09b9\u09c1 \u09ad\u09be\u09b0\u09a4\u09c0\u09af\u09bc \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf \u0997\u09a8\u09cd\u09a1\u09cb\u09af\u09bc\u09be\u09a8\u09be\u09af\u09bc \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be \u09a5\u09c7\u0995\u09c7 \u0989\u09a6\u09cd\u09ad\u09c1\u09a4 \u0964
+\u09ac\u09cd\u09b0\u09bf\u099f\u09c7\u09a8\u09c7\u09b0 \u09b2\u09c7\u0996\u0995\u09a6\u09c7\u09b0 \u09b0\u099a\u09bf\u09a4 \u09a8\u09be\u099f\u0995 & # 44 ; \u0989\u09aa\u09a8\u09cd\u09af\u09be\u09b8 & # 44 ; \u0997\u09b2\u09cd\u09aa \u098f\u09ac\u0982 \u09b8\u09ae\u09cd\u09aa\u09cd\u09b0\u09a4\u09bf \u099a\u09bf\u09a4\u09cd\u09b0\u09a8\u09be\u099f\u09cd\u09af \u09ac\u09bf\u09b6\u09cd\u09ac\u09ac\u09cd\u09af\u09be\u09aa\u09c0 \u0986\u09a6\u09c3\u09a4 \u0964
+\u09e7\u09ef\u09e7\u09ef \u09b8\u09be\u09b2\u09c7\u09b0 \u09ae\u09c7 \u09ae\u09be\u09b8\u09c7 \u098f\u099f\u09bf \u09b8\u0993\u0997\u09be\u09a4 \u09aa\u09a4\u09cd\u09b0\u09bf\u0995\u09be\u09af\u09bc \u09aa\u09cd\u09b0\u0995\u09be\u09b6\u09bf\u09a4 \u09b9\u09af\u09bc \u0964
+\u09e8\u09e6\u09e6\u09eb \u09b8\u09be\u09b2\u09c7 \u0989\u0987\u09ae\u09c7\u09a8\u09b8 \u099f\u09c7\u09a8\u09bf\u09b8 \u0985\u09cd\u09af\u09be\u09b8\u09cb\u09b8\u09bf\u09af\u09bc\u09c7\u09b6\u09a8 \u099f\u09cd\u09af\u09c1\u09b0\u09c7\u09b0 \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f \u0993\u09aa\u09c7\u09a8 \u09a8\u09c7\u09a4\u09be\u099c\u09bf \u0987\u09a8\u09cd\u09a1\u09cb\u09b0 \u09b8\u09cd\u099f\u09c7\u09a1\u09bf\u09af\u09bc\u09be\u09ae\u09c7 \u0986\u09af\u09bc\u09cb\u099c\u09bf\u09a4 \u09b9\u09af\u09bc\u09c7\u099b\u09bf\u09b2 \u0964
+\u098f\u0987 \u09b8\u09ae\u09cd\u09ad\u09be\u09ac\u09a8\u09be \u09a6\u09c2\u09b0\u09c0\u0995\u09b0\u09a3\u09c7\u09b0 \u099c\u09a8\u09cd\u09af \u09a8\u09be\u09a8\u09be\u09ac\u09bf\u09a7 \u09ac\u09cd\u09af\u09ac\u09b8\u09cd\u09a5\u09be \u0997\u09cd\u09b0\u09b9\u09a3 \u0995\u09b0\u09be \u09b9\u09af\u09bc \u0964
+\u09e7\u09ef\u09ec\u09ec \u09b8\u09be\u09b2\u09c7\u09b0 \u09eb \u09ab\u09c7\u09ac\u09cd\u09b0\u09c1\u09af\u09bc\u09be\u09b0\u09bf \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 \u09ac\u09bf\u09b0\u09cb\u09a7\u09c0 \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 \u098f\u0995\u099f\u09bf \u099c\u09be\u09a4\u09c0\u09af\u09bc \u09b8\u09ae\u09cd\u09ae\u09c7\u09b2\u09a8 \u0985\u09a8\u09c1\u09b7\u09cd\u09a0\u09bf\u09a4 \u09b9\u09af\u09bc \u0964
+\u09ac\u09be\u0982\u09b2\u09be\u09a6\u09c7\u09b6 \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 \u0985\u09ab \u09a6\u09bf \u0987\u09b8\u09b2\u09be\u09ae\u09bf\u0995 \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 \u0993 \u0987\u09b8\u09b2\u09be\u09ae\u09bf\u0995 \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f \u09ac\u09cd\u09af\u09be\u0982\u0995\u09c7\u09b0 \u09b8\u09a6\u09b8\u09cd\u09af\u09aa\u09a6 \u0997\u09cd\u09b0\u09b9\u09a3 \u0995\u09b0\u09c7 \u0964
+\u09ac\u09bf\u09b7\u09af\u09bc\u09b6\u09cd\u09b0\u09c7\u09a3\u09c0 : \u09ac\u09bf\u09b6\u09cd\u09ac\u0995\u09cb\u09b7
+\u09b0\u09be\u09b6\u09bf\u09af\u09bc\u09be & # 44 ; \u09ab\u09cd\u09b0\u09be\u09a8\u09cd\u09b8 \u0993 \u0987\u09b8\u09b0\u09be\u09af\u09bc\u09c7\u09b2 \u09ad\u09be\u09b0\u09a4\u09c7\u09b0 \u09aa\u09cd\u09b0\u09a7\u09be\u09a8 \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 \u09b0\u09be\u09b7\u09cd\u099f\u09cd\u09b0 \u0993 \u09aa\u09cd\u09b0\u09a4\u09bf\u09b0\u0995\u09cd\u09b7\u09be \u09b8\u09b9\u0995\u09be\u09b0\u09c0 \u09a6\u09c7\u09b6 \u0964
+\u098f\u0987 \u09b9\u09b2 \u0986\u09ae\u09be\u09a6\u09c7\u09b0 \u09aa\u09b0\u09bf\u099a\u09bf\u09a4 \u0995\u09be\u09b2\u09cd\u09aa\u09a8\u09bf\u0995 \u098f\u0995\u0995 \u09af\u09be\u09b0 \u09b8\u09be\u09b9\u09be\u09af\u09cd\u09af\u09c7 \u0997\u09a3\u09bf\u09a4\u09c7\u09b0 \u09a5\u09bf\u0993\u09b0\u09c0 \u09b8\u09ae\u09c1\u09b9 \u09ac\u09be\u09b8\u09cd\u09a4\u09ac \u09b8\u0982\u0996\u09cd\u09af\u09be\u09b0 \u09b8\u09c7\u099f \u09a5\u09c7\u0995\u09c7 \u099c\u099f\u09bf\u09b2 \u09b8\u0982\u0996\u09cd\u09af\u09be\u09b0 \u09b8\u09c7\u099f\u09c7 \u0989\u09a8\u09cd\u09a8\u09c0\u09a4 \u09b9\u09af\u09bc \u0964
+& lt ; \u09a0\u09bf\u0995\u09be\u09a8\u09be & gt ;
+\u09b8\u09c7\u09aa\u09cd\u099f\u09c7\u09ae\u09cd\u09ac\u09b0
+\u098f\u0987 \u09a4\u09a4\u09cd\u09a4\u09cd\u09ac \u09a5\u09c7\u0995\u09c7 \u0985\u09ac\u09b6\u09cd\u09af \u09ae\u09b9\u09be \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 \u09ac\u09bf\u09b0\u09cb\u09a7\u09bf\u09a4\u09be \u0995\u09b0\u09be \u09af\u09be\u09af\u09bc \u09a8\u09be & # 44 ; \u09ac\u09b0\u0982 \u09a4\u09be \u09b8\u09ae\u09b0\u09cd\u09a5\u09a8 \u0995\u09b0\u09be \u09af\u09c7\u09a4\u09c7 \u09aa\u09be\u09b0\u09c7 \u0964
+\u0995\u09c3\u09b7\u09bf \u0989\u09ce\u09aa\u09be\u09a6\u09a8\u09c7 \u09ab\u09cd\u09b0\u09be\u09a8\u09cd\u09b8 \u0987\u0989\u09b0\u09cb\u09aa\u09c7\u09b0 \u09b8\u09ac\u099a\u09c7\u09af\u09bc\u09c7 \u0997\u09c1\u09b0\u09c1\u09a4\u09cd\u09ac\u09aa\u09c2\u09b0\u09cd\u09a3 \u09a6\u09c7\u09b6 ; \u098f\u099f\u09bf \u09ae\u09c2\u09b2\u09a4 \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af & # 44 ; \u0993\u09af\u09bc\u09be\u0987\u09a8 & # 44 ; \u09aa\u09a8\u09bf\u09b0 \u0993 \u0985\u09a8\u09cd\u09af\u09be\u09a8\u09cd\u09af \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af \u0987\u0989\u09b0\u09cb\u09aa \u0993 \u09b8\u09be\u09b0\u09be \u09ac\u09bf\u09b6\u09cd\u09ac\u09c7 \u09b0\u09aa\u09cd\u09a4\u09be\u09a8\u09bf \u0995\u09b0\u09c7 \u0964
+\u09a4\u09be\u09a6\u09c7\u09b0 \u0997\u09a3\u09bf\u09a4\u09c7 \u09aa\u09be\u099f\u09c0\u0997\u09a3\u09bf\u09a4\u09c7\u09b0 \u09aa\u09cd\u09b0\u09be\u09a7\u09be\u09a8\u09cd\u09af \u099b\u09bf\u09b2 \u0964
+\u09a6\u09c7\u09b6\u0997\u09c1\u09b2\u09cb \u09b9\u09b2\u09cb\u0983 \u09ab\u09cd\u09b0\u09be\u09a8\u09cd\u09b8 & # 44 ; \u09b9\u0982\u0995\u0982 & # 44 ; \u099a\u09c0\u09a8 & # 44 ; \u09ac\u09c7\u09b2\u099c\u09bf\u09af\u09bc\u09be\u09ae & # 44 ; \u09b8\u09c1\u0987\u099c\u09be\u09b0\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 & # 44 ; \u099c\u09be\u09b0\u09cd\u09ae\u09be\u09a8\u09c0 & # 44 ; \u09a1\u09c7\u09a8\u09ae\u09be\u09b0\u09cd\u0995 & # 44 ; \u09b8\u09c1\u0987\u09a1\u09c7\u09a8 & # 44 ; \u0985\u09b8\u09cd\u099f\u09cd\u09b0\u09bf\u09af\u09bc\u09be & # 44 ; \u099a\u09c7\u0995\u09cb\u09b6\u09cd\u09b2\u09cb\u09ad\u09be\u0995\u09bf\u09af\u09bc\u09be & # 44 ; \u0986\u09b0\u09cd\u099c\u09c7\u09a8\u09cd\u099f\u09bf\u09a8\u09be & # 44 ; \u0987\u09a4\u09be\u09b2\u09bf & # 44 ; \u09a8\u09b0\u0993\u09af\u09bc\u09c7 & # 44 ; \u09b9\u09be\u0999\u09cd\u0997\u09c7\u09b0\u09c0 & # 44 ; \u09af\u09c1\u0997\u09cb\u09b6\u09cd\u09b2\u09be\u09ad\u09bf\u09af\u09bc\u09be & # 44 ; \u09ac\u09c1\u09b2\u0997\u09c7\u09b0\u09bf\u09af\u09bc\u09be & # 44 ; \u09b0\u09c1\u09ae\u09be\u09a8\u09bf\u09af\u09bc\u09be & # 44 ; \u0997\u09cd\u09b0\u09c0\u09b8 & # 44 ; \u09ae\u09bf\u09b6\u09b0 & # 44 ; \u09b8\u09bf\u0999\u09cd\u0997\u09be\u09aa\u09c1\u09b0 & # 44 ; \u0987\u09a8\u09cd\u09a6\u09cb\u09a8\u09c7\u09b6\u09bf\u09af\u09bc\u09be & # 44 ; \u09a5\u09be\u0987\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 & # 44 ; \u099c\u09be\u09aa\u09be\u09a8 & # 44 ; \u09ac\u09be\u09b0\u09cd\u09ae\u09be & # 44 ; \u09b9\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 & # 44 ; \u09b8\u09cb\u09ad\u09bf\u09af\u09bc\u09c7\u099f \u09b0\u09be\u09b6\u09bf\u09af\u09bc\u09be & # 44 ; \u0987\u09b0\u09be\u09a8 & # 44 ; \u0987\u09b0\u09be\u0995 \u0993 \u09b6
\u09b0\u09c0\u09b2\u0982\u0995\u09be \u0964
+\u098f\u0987 \u09ac\u09cd\u09af\u09be\u09b8\u09bf\u09b2\u09bf\u0995\u09be\u09b0 \u09b8\u09cd\u09a5\u09be\u09a8\u09c7 \u098f\u0996\u09a8 \u09ac\u09cd\u09af\u09be\u0982\u0995 \u0985\u09ab \u0987\u0982\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 \u0985\u09ac\u09b8\u09cd\u09a5\u09bf\u09a4 \u0964
+\u09a6\u09c7\u09b6\u099f\u09bf\u09b0 \u0989\u09a4\u09cd\u09a4\u09b0\u09c7 \u09ac\u09bf\u09b8\u09cd\u0995\u09be\u0987 \u0989\u09aa\u09b8\u09be\u0997\u09b0 & # 44 ; \u09a6\u0995\u09cd\u09b7\u09bf\u09a3 \u09a6\u09bf\u0995\u09c7 \u099c\u09bf\u09ac\u09cd\u09b0\u09be\u09b2\u09cd\u099f\u09be\u09b0 \u09aa\u09cd\u09b0\u09a3\u09be\u09b2\u09c0 & # 44 ; \u09aa\u09cd\u09b0\u09a3\u09be\u09b2\u09c0\u09b0 \u09a6\u0995\u09cd\u09b7\u09bf\u09a3\u09c7 \u09ae\u09b0\u0995\u09cd\u0995\u09cb & # 44 ; \u09aa\u09b6\u09cd\u099a\u09bf\u09ae \u0993 \u09a6\u0995\u09cd\u09b7\u09bf\u09a3-\u09aa\u09b6\u09cd\u099a\u09bf\u09ae \u09a6\u09bf\u0995\u09c7 \u0986\u099f\u09b2\u09be\u09a8\u09cd\u099f\u09bf\u0995 \u09ae\u09b9\u09be\u09b8\u09be\u0997\u09b0 \u0964
+\u09a4\u09be\u099b\u09be\u09a1\u09bc\u09be \u098f \u09aa\u09b0\u09bf\u09b8\u09cd\u09a5\u09bf\u09a4\u09c7 \u09ac\u09cb\u099d\u09be \u09af\u09be\u09af\u09bc \u099c\u09b0\u09c1\u09b0\u09bf \u09ae\u09c1\u09b9\u09c2\u09b0\u09cd\u09a4\u09c7 \u099c\u09be\u09a4\u09bf\u09b8\u0982\u0998\u09c7\u09b0 \u09a6\u09cd\u09b0\u09c1\u09a4 \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4 \u09a8\u09c7\u09ac\u09be\u09b0 \u0985\u0995\u09cd\u09b7\u09ae\u09a4\u09be \u0964
+\u0995\u09be\u09b0\u09cd\u09b2 \u09ae\u09be\u09b0\u09cd\u0995\u09cd\u09b8\u09c7\u09b0 \u0995\u09be\u099c\u09c7\u09b0 \u09ae\u09be\u09a7\u09cd\u09af\u09ae\u09c7\u0987 \u0987\u09b9\u09be \u09aa\u09b0\u09bf\u099a\u09be\u09b2\u09bf\u09a4 \u09b9\u09af\u09bc \u0964
+\u098f\u09b0 \u09ac\u09bf\u09b7\u09af\u09bc\u09ac\u09b8\u09cd\u09a4\u09c1 \u0995\u0996\u09a8\u0993 \u09aa\u09c1\u09b0\u09be\u09a3 \u09a5\u09c7\u0995\u09c7 & # 44 ; \u0995\u0996\u09a8\u0993 \u09ae\u09a7\u09cd\u09af\u09af\u09c1\u0997\u09c0\u09af\u09bc \u09aa\u09cd\u09b0\u09c7\u09ae\u0995\u09be\u09b9\u09bf\u09a8\u09bf\u0997\u09c1\u09b2\u09bf \u09a5\u09c7\u0995\u09c7 & # 44 ; \u0995\u0996\u09a8\u0993 \u0986\u09ac\u09be\u09b0 \u098f\u0995\u09be\u09b2\u09c7\u09b0 \u09b8\u09be\u09ae\u09be\u099c\u09bf\u0995 \u0993 \u09b0\u09be\u099c\u09a8\u09c8\u09a4\u09bf\u0995 \u0998\u099f\u09a8\u09be\u09ac\u09b2\u09bf \u09a5\u09c7\u0995\u09c7 \u0997\u09c3\u09b9\u09c0\u09a4 \u0964
+\u09a4\u09bf\u09a8\u099f\u09bf \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09c7\u09b0 \u0989\u09aa\u09b0 \u09ad\u09bf\u09a4\u09cd\u09a4\u09bf \u0995\u09b0\u09c7 \u09ae\u09b9\u09be\u09ac\u09bf\u09b6\u09cd\u09ac\u09c7\u09b0 \u09af\u09c7 \u09ac\u09af\u09bc\u09b8 \u09aa\u09be\u0993\u09af\u09bc\u09be \u0997\u09c7\u099b\u09c7 \u09a4\u09be \u09b9\u09b2 \u09aa\u09cd\u09b0\u09be\u09af\u09bc \u09e7\u09e9.\u09ed � \u09e6.\u09e8 \u09ac\u09bf\u09b2\u09bf\u09af\u09bc\u09a8 \u09ac\u099b\u09b0 \u0964
+\u0995\u09be\u099b\u09c7\u0987 \u09b0\u09af\u09bc\u09c7\u099b\u09c7 \u09aa\u09c2\u09b0\u09cd\u09ac \u09b0\u09be\u09b6\u09bf\u09af\u09bc\u09be & # 44 ; \u09af\u09be \u0993\u0996\u099f\u09b8\u09cd\u0995 \u09b8\u09be\u0997\u09b0 \u0993 \u099c\u09be\u09aa\u09be\u09a8 \u09b8\u09be\u0997\u09b0\u09c7\u09b0 \u0985\u09aa\u09b0 \u09aa\u09be\u09b0\u09c7 \u0985\u09ac\u09b8\u09cd\u09a5\u09bf\u09a4 \u0964
+\u0995\u09b2\u0995\u09be\u09a4\u09be\u09af\u09bc \u0985\u09ac\u09b8\u09cd\u09a5\u09bf\u09a4 \u09ad\u09be\u09b0\u09a4\u09c7\u09b0 \u099c\u09be\u09a4\u09c0\u09af\u09bc \u0997\u09cd\u09b0\u09a8\u09cd\u09a5\u09be\u0997\u09be\u09b0 \u09a6\u09c7\u09b6\u09c7\u09b0 \u0985\u0997\u09cd\u09b0\u09a3\u09c0 \u09aa\u09be\u09ac\u09b2\u09bf\u0995 \u09b2\u09be\u0987\u09ac\u09cd\u09b0\u09c7\u09b0\u09bf \u0964
+\u099b\u09a4\u09cd\u09b0\u09be\u0995\u09ac\u09bf\u09a6\u09cd\u09af\u09be
+\u09b0\u09be\u09b7\u09cd\u099f\u09cd\u09b0\u09b8\u0999\u09cd\u0998\u09c7\u09b0 \u09ae\u09b9\u09be\u09b8\u099a\u09bf\u09ac \u09ac\u09be\u09a8 \u0995\u09bf \u09ae\u09c1\u09a8
+\u09ae\u09bf\u09a8\u09bf\u0995\u09cd\u09b8\u09c7\u09b0 \u09b0\u099a\u09af\u09bc\u09bf\u09a4\u09be \u099b\u09bf\u09b2\u09c7\u09a8 \u098f\u09a8\u09cd\u09a1\u09cd\u09b0\u09c1 \u099f\u09be\u09a8\u09c7\u09a8\u09ac\u09ae & # 44 ; \u098f\u0995 \u09aa\u09cd\u09b0\u0996\u09cd\u09af\u09be\u09a4 \u0985\u09aa\u09be\u09b0\u09c7\u099f\u09bf\u0982 \u09b8\u09bf\u09b8\u09cd\u099f\u09c7\u09ae \u09a1\u09bf\u099c\u09be\u0987\u09a8 \u09aa\u09cd\u09b0\u09b6\u09bf\u0995\u09cd\u09b7\u0995 \u0964
+\u09a6\u09cd\u09af \u099f\u09be\u0987\u09ae\u09cd \u200c \u09b8 \u0985\u09ab \u0987\u09a8\u09cd\u09a1\u09bf\u09af\u09bc\u09be-\u09a4\u09c7 \u09b2\u09c7\u0996\u09be \u09b9\u09af\u09bc \u09af\u09c7 & quot ; it is absurd to compare it with any other indian cinema ... pather panchali is pure cinema & quot ; -lrb- \u201c \u098f\u0995\u09c7 \u0985\u09a8\u09cd\u09af \u09af\u09c7\u0995\u09cb\u09a8\u0993 \u09ad\u09be\u09b0\u09a4\u09c0\u09af\u09bc \u099a\u09b2\u099a\u09cd\u099a\u09bf\u09a4\u09cd\u09b0\u09c7\u09b0 \u09b8\u09be\u09a5\u09c7 \u09a4\u09c1\u09b2\u09a8\u09be \u0995\u09b0\u09be \u0985\u09ac\u09be\u09b8\u09cd\u09a4\u09ac ... \u09aa\u09a5\u09c7\u09b0 \u09aa\u09be\u0981\u099a\u09be\u09b2\u09c0 \u09b9\u09b2 \u09ac\u09bf\u09b6\u09c1\u09a6\u09cd\u09a7 \u099a\u09b2\u099a\u09cd\u099a\u09bf\u09a4\u09cd\u09b0 \u201d -rrb- \u0964
+\u098f\u09b0\u09aa\u09b0 \u09e7\u09ef\u09eb\u09e9 \u0996\u09cd\u09b0\u09bf\u09b8\u09cd\u099f\u09be\u09ac\u09cd\u09a6\u09c7\u09b0 \u09ae\u09c7 \u09ae\u09be\u09b8\u09c7 \u09a8\u099c\u09b0\u09c1\u09b2 \u0993 \u09aa\u09cd\u09b0\u09ae\u09c0\u09b2\u09be \u09a6\u09c7\u09ac\u09c0\u0995\u09c7 \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u09be\u09b0 \u099c\u09a8\u09cd\u09af \u09b2\u09a8\u09cd\u09a1\u09a8 \u09aa\u09be\u09a0\u09be\u09a8\u09cb \u09b9\u09af\u09bc \u0964
+\u098f\u09b0 \u09a6\u0995\u09cd\u09b7\u09bf\u09a3 \u0993 \u09aa\u09c2\u09b0\u09cd\u09ac\u09c7 \u0986\u099b\u09c7 \u09ac\u09bf\u09b8\u09cd\u09a4\u09c0\u09b0\u09cd\u09a3 \u09b8\u09ae\u09ad\u09c2\u09ae\u09bf ; \u09aa\u09b6\u09cd\u099a\u09bf\u09ae \u0993 \u0989\u09a4\u09cd\u09a4\u09b0\u09c7 \u0986\u099b\u09c7 \u09b0\u09c1\u0995\u09cd\u09b7 \u09aa\u09be\u09b9\u09be\u09a1\u09bc \u0993 \u09aa\u09b0\u09cd\u09ac\u09a4 \u0964
+\u099f\u09cd\u09b0\u09c7\u09a1\u09ae\u09be\u09b0\u09cd\u0995
+\u09b2\u09b0\u09cd\u09a1 \u0993\u09af\u09bc\u09c7\u09b2\u09c7\u09b8\u09b2\u09bf\u09b0 -lrb- \u0997\u09ad\u09b0\u09cd\u09a8\u09b0-\u099c\u09c7\u09a8\u09be\u09b0\u09c7\u09b2 \u09e7\u09ed\u09ef\u09ed-\u09e7\u09ee\u09e6\u09eb -rrb- \u09b6\u09be\u09b8\u09a8\u0995\u09be\u09b2\u09c7 \u09b6\u09b9\u09b0\u09c7\u09b0 \u0989\u09b2\u09cd\u09b2\u09c7\u0996\u09af\u09cb\u0997\u09cd\u09af \u09ac\u09c3\u09a6\u09cd\u09a7\u09bf \u0998\u099f\u09c7\u099b\u09bf\u09b2 \u0964
+\u0985\u09a8\u09c7\u0995 \u0997\u09c1\u09b0\u09c1\u09a4\u09cd\u09ac\u09aa\u09c2\u09b0\u09cd\u09a8 \u0993 \u09ac\u09be\u09b8\u09cd\u09a4\u09ac \u09b8\u09ae\u09b8\u09cd\u09af\u09be \u09b8\u09ae\u09be\u09a7\u09be\u09a8\u09c7\u09b0 \u099c\u09a8\u09cd\u09af \u099c\u099f\u09bf\u09b2 \u09b8\u0982\u0996\u09cd\u09af\u09be \u0985\u09aa\u09b0\u09bf\u09b9\u09be\u09b0\u09cd\u09af
+\u09ae\u09b9\u09be \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09c7\u09b0 \u098f\u0995\u099f\u09bf \u0989\u09b2\u09cd\u09b2\u09c7\u0996\u09af\u09cb\u0997\u09cd\u09af \u09ab\u09b2\u09be\u09ab\u09b2 \u09b9\u09b2 & # 44 ; \u09ac\u09b0\u09cd\u09a4\u09ae\u09be\u09a8\u0995\u09be\u09b2\u09c7 \u09ae\u09b9\u09be\u09ac\u09bf\u09b6\u09cd\u09ac\u09c7\u09b0 \u0985\u09ac\u09b8\u09cd\u09a5\u09be \u0985\u09a4\u09c0\u09a4 \u098f\u09ac\u0982 \u09ad\u09ac\u09bf\u09b7\u09cd\u09af\u09a4\u09c7\u09b0 \u0985\u09ac\u09b8\u09cd\u09a5\u09be \u09a5\u09c7\u0995\u09c7 \u09b8\u09ae\u09cd\u09aa\u09c2\u09b0\u09cd\u09a3 \u09aa\u09c3\u09a5\u0995 \u0964
+\u0989\u0987\u09a8\u09cd\u09a1\u09cb\u099c \u09ae\u09bf\u09b2\u09bf\u09a8\u09bf\u09af\u09bc\u09be\u09ae
+\u09b0\u09ac\u09c0\u09a8\u09cd\u09a6\u09cd\u09b0\u09a8\u09be\u09a5 \u0985\u09ac\u09b6\u09cd\u09af \u098f\u0995\u09be\u09a7\u09bf\u0995 \u09b6\u09c8\u09b2\u09c0 \u0986\u09af\u09bc\u09a4\u09cd\u09a4\u09cd\u09ac \u0995\u09b0\u09c7\u099b\u09bf\u09b2\u09c7\u09a8 \u0964
+\u09b6\u09cd\u09b0\u09ae \u0985\u09b0\u09cd\u09a5\u09a8\u09c0\u09a4\u09bf
+\u09ac\u09cd\u09b0\u09bf\u099f\u09c7\u09a8 \u098f\u0995\u09b8\u09ae\u09af\u09bc \u09ac\u09bf\u09b6\u09cd\u09ac\u09c7\u09b0 \u09aa\u09cd\u09b0\u09a7\u09be\u09a8 \u0993 \u0985\u0997\u09cd\u09b0\u0997\u09be\u09ae\u09c0 \u0985\u09b0\u09cd\u09a5\u09a8\u09c8\u09a4\u09bf\u0995 \u09b6\u0995\u09cd\u09a4\u09bf \u099b\u09bf\u09b2 \u0964
+\u09aa\u09be\u0995\u09bf\u09b8\u09cd\u09a4\u09be\u09a8\u09c7\u09b0 \u09b8\u09be\u09ae\u09b0\u09bf\u0995 \u09b6\u09be\u09b8\u09a8\u09c7\u09b0 \u09ac\u09bf\u09b0\u09c1\u09a6\u09cd\u09a7\u09c7 \u0986\u09a8\u09cd\u09a6\u09cb\u09b2\u09a8 \u098f\u09ac\u0982 \u09aa\u09be\u0995\u09bf\u09b8\u09cd\u09a4\u09be\u09a8\u09c7\u09b0 \u0997\u09cb\u09b7\u09cd\u09a0\u09c0\u0997\u09a4 \u09ac\u09c8\u09b7\u09ae\u09cd\u09af\u09c7\u09b0 \u09ac\u09bf\u09b0\u09c1\u09a6\u09cd\u09a7\u09c7 \u09aa\u09cd\u09b0\u09a4\u09bf\u09ac\u09be\u09a6 \u0993 \u09ac\u09be\u0999\u09be\u09b2\u09bf\u09a6\u09c7\u09b0 \u0986\u09a8\u09cd\u09a6\u09cb\u09b2\u09a8\u0995\u09c7 \u09b8\u09cd\u09ac\u09be\u09a7\u09c0\u09a8\u09a4\u09be\u09b0 \u09aa\u09a5\u09c7 \u09a7\u09be\u09ac\u09bf\u09a4 \u0995\u09b0\u09be\u09b0 \u099c\u09a8\u09cd\u09af \u09a4\u09bf\u09a8\u09bf \u09ac\u09cd\u09af\u09be\u09aa\u0995\u09ad\u09be\u09ac\u09c7 \u09aa\u09cd\u09b0\u09b6\u0982\u09b8\u09bf\u09a4 \u0964
+\u098f\u0996\u09be\u09a8\u09c7 \u0989\u09b2\u09cd\u09b2\u09c7\u0996 \u0995\u09b0\u09be \u09aa\u09cd\u09b0\u09af\u09bc\u09cb\u099c\u09a8 \u09af\u09c7 \u0985\u09a8\u09c7\u0995\u09c7 \u0987\u09a8\u09cd\u099f\u09be\u09b0\u09a8\u09c7\u099f \u098f\u09ac\u0982 \u0993\u09af\u09bc\u09be\u09b0\u09cd\u09b2\u09cd\u09a1 \u0993\u09af\u09bc\u09be\u0987\u09a1 \u0993\u09af\u09bc\u09c7\u09ac\u0995\u09c7 \u09b8\u09ae\u09be\u09b0\u09cd\u09a5\u0995 \u09b6\u09ac\u09cd\u09a6 \u09b9\u09bf\u09b8\u09c7\u09ac\u09c7 \u0997\u09a3\u09cd\u09af \u0995\u09b0\u09b2\u09c7\u0993 \u09aa\u09cd\u09b0\u0995\u09c3\u09a4\u09aa\u0995\u09cd\u09b7\u09c7 \u09b6\u09ac\u09cd\u09a6\u09a6\u09cd\u09ac\u09af\u09bc \u09ad\u09bf\u09a8\u09cd\u09a8 \u09ac\u09bf\u09b7\u09af\u09bc \u09a8\u09bf\u09b0\u09cd\u09a6\u09c7\u09b6 \u0995\u09b0\u09c7 \u0964
+. z \u098f\u09b0 \u0986\u09a8\u09c1\u09b8\u09be\u0999\u09cd\u0997\u09c0\u0995 \u09aa\u09cb\u09b2\u09be\u09b0 \u0995\u09cb-\u0985\u09b0\u09cd\u09a1\u09bf\u09a8\u09c7\u099f \u09a6\u09c1\u099f\u09bf \u09b9\u099a\u09cd\u099b\u09c7 r = -pipe-
+\u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0
+\u09e7\u09ef\u09ed\u09e8 \u0996\u09cd\u09b0\u09c0\u09b8\u09cd\u099f\u09be\u09ac\u09cd\u09a6\u09c7 \u09a4\u09a6\u09be\u09a8\u09bf\u09a8\u09cd\u09a4\u09a8 \u0986\u09b0\u09aa\u09be\u09a8\u09c7\u099f\u09c7 \u09b8\u09b0\u09cd\u09ac\u09aa\u09cd\u09b0\u09a5\u09ae \u0987\u09b2\u09c7\u0995\u09cd\u099f\u09cd\u09b0\u09a8\u09bf\u0995 \u09ae\u09c7\u0987\u09b2 \u09aa\u09cd\u09b0\u09c7\u09b0\u09a3 \u0995\u09b0\u09be \u09b9\u09af\u09bc \u0964
+\u099c\u09c0\u09ac \u09ac\u09bf\u099c\u09cd\u099e\u09be\u09a8\u09c7\u09b0 \u09af\u09c7 \u09b6\u09be\u0996\u09be\u09af\u09bc \u099b\u09a4\u09cd\u09b0\u09be\u0995 \u0993 \u098f\u09b0 \u09ac\u09cd\u09af\u09ac\u09b9\u09be\u09b0\u09bf\u0995 \u09ac\u09bf\u09b7\u09af\u09bc \u09a8\u09bf\u09af\u09bc\u09c7 \u0986\u09b2\u09cb\u099a\u09a8\u09be \u0995\u09b0\u09be \u09b9\u09af\u09bc \u09a4\u09be\u0995\u09c7 \u099b\u09a4\u09cd\u09b0\u09be\u0995\u09ac\u09bf\u09a6\u09cd\u09af\u09be -lrb- mycology -rrb-
+\u09aa\u09be\u09a8\u09bf \u09a8\u09a6\u09c0 \u09a5\u09c7\u0995\u09c7 \u0989\u09a0\u09be\u09a8\u09cb \u09b9\u09a4\u09cb \u0995\u09af\u09bc\u09c7\u0995\u099f\u09bf \u09aa\u09c1\u09b0 \u09a6\u09bf\u09af\u09bc\u09c7- \u09a6\u09a1\u09bc\u09bf \u0993 \u09ac\u09be\u09b2\u099f\u09bf\u09b0 \u09ae\u09be\u09a7\u09cd\u09af\u09ae\u09c7 \u09aa\u09be\u09a8\u09bf \u09aa\u09b6\u09c1 \u09a6\u09cd\u09ac\u09be\u09b0\u09be \u099f\u09c7\u09a8\u09c7 \u09a4\u09cb\u09b2\u09be\u09b0 \u098f\u0995 \u09aa\u09a6\u09cd\u09a7\u09a4\u09bf \u0964
+\u098f\u09b0 \u09ae\u09a7\u09cd\u09af\u09c7 \u09b0\u09af\u09bc\u09c7\u099b\u09c7 \u0989\u09aa\u099c\u09be\u09a4\u09c0\u09af\u09bc \u09a8\u09c3\u09a4\u09cd\u09af & # 44 ; \u09b2\u09cb\u0995\u099c \u09a8\u09c3\u09a4\u09cd\u09af & # 44 ; \u09b6\u09be\u09b8\u09cd\u09a4\u09cd\u09b0\u09c0\u09af\u09bc \u09a8\u09c3\u09a4\u09cd\u09af & # 44 ; \u0987\u09a4\u09cd\u09af\u09be\u09a6\u09bf \u0964
+\u09ad\u09be\u09b0\u09a4\u09c7\u09b0 \u09aa\u09cd\u09b0\u09be\u099a\u09c0\u09a8\u09a4\u09ae \u09b8\u09be\u09b9\u09bf\u09a4\u09cd\u09af \u09aa\u09cd\u09b0\u09a5\u09ae\u09c7 \u09ae\u09cc\u0996\u09bf\u0995\u09ad\u09be\u09ac\u09c7 \u0993 \u09aa\u09b0\u09c7 \u09b2\u09bf\u0996\u09bf\u09a4 \u0986\u0995\u09be\u09b0\u09c7 \u09aa\u09cd\u09b0\u099a\u09b2\u09bf\u09a4 \u09b9\u09af\u09bc \u0964
+\u09e7\u09ef\u09ee\u09ef \u09b8\u09be\u09b2\u09c7 \u09a8\u09bf\u09b0\u09cd\u09ae\u09bf\u09a4 \u0997\u09a3\u09b6\u09a4\u09cd\u09b0\u09c1 \u099b\u09ac\u09bf\u099f\u09bf\u09a4\u09c7 \u09a4\u09be\u0981\u09b0 \u09aa\u09b0\u09bf\u099a\u09be\u09b2\u09a8\u09be \u09a4\u09c1\u09b2\u09a8\u09be\u09ae\u09c2\u09b2\u0995\u09ad\u09be\u09ac\u09c7 \u09a6\u09c1\u09b0\u09cd\u09ac\u09b2 \u098f\u09ac\u0982 \u098f\u099f\u09bf\u0995\u09c7 \u09a6\u09c0\u09b0\u09cd\u0998\u09a6\u09bf\u09a8\u09c7\u09b0 \u0985\u09b8\u09c1\u09b8\u09cd\u09a5\u09a4\u09be\u09b6\u09c7\u09b7\u09c7 \u09ab\u09bf\u09b0\u09c7 \u0986\u09b8\u09be\u09b0 \u09aa\u09b0 \u09b8\u09a4\u09cd\u09af\u099c\u09bf\u09a4\u09c7\u09b0 \u099a\u09b2\u099a\u09cd\u099a\u09bf\u09a4\u09cd\u09b0 \u09a8\u09bf\u09b0\u09cd\u09ae\u09be\u09a3\u09c7\u09b0 \u09aa\u09c1\u09a8\u09b0\u09cd\u09aa\u09cd\u09b0\u099a\u09c7\u09b7\u09cd\u099f\u09be \u09b9\u09bf\u09b8\u09c7\u09ac\u09c7\u0987 \u0997\u09a3\u09cd\u09af \u0995\u09b0\u09be \u09b9\u09af\u09bc \u0964
+\u2022 \u09ac\u09b2\u09ac\u09bf\u09a6\u09cd\u09af\u09be
+\u0985\u09a8\u09cd\u09af\u09be\u09a8\u09cd\u09af \u09b8\u09cd\u09ac\u09a4\u09cd\u09a4\u09cd\u09ac-\u09b8\u0982\u09b0\u0995\u09cd\u09b7\u09bf\u09a4 \u0985\u09aa\u09be\u09b0\u09c7\u099f\u09bf\u0982 \u09b8\u09bf\u09b8\u09cd\u099f\u09c7\u09ae \u09af\u09c7\u09ae\u09a8 \u0989\u0987\u09a8\u09cd\u09a1\u09cb\u099c \u098f\u09ac\u0982 \u09ae\u09cd\u09af\u09be\u0995 \u0993\u098f\u09b8 \u09b9\u09a4\u09c7 \u09b2\u09bf\u09a8\u09be\u0995\u09cd\u09b8 \u09ac\u09bf\u09ad\u09bf\u09a8\u09cd\u09a8\u09ad\u09be\u09ac\u09c7 \u0986\u09b2\u09be\u09a6\u09be \u0964
+\u098f\u09b6\u09bf\u09af\u09bc\u09be \u099f\u09be\u0987\u09ae\u09b8\u09c7\u09b0 \u09ad\u09be\u09b7\u09cd\u09af \u0985\u09a8\u09c1\u09af\u09be\u09af\u09bc\u09c0 & # 44 ;
+\u09ae\u09c1\u0995\u09cd\u09a4 \u09b8\u09cb\u09b0\u09cd\u09b8 \u09ac\u09be \u0993\u09aa\u09c7\u09a8 \u09b8\u09cb\u09b0\u09cd\u09b8 -lrb- open source -rrb- \u098f\u09b0 \u0985\u09b0\u09cd\u09a5 \u09b9\u09b2\u09cb \u0995\u09ae\u09cd\u09aa\u09bf\u0989\u099f\u09be\u09b0 \u09b8\u09ab\u099f\u0993\u09af\u09bc\u09cd\u09af\u09be\u09b0 \u098f\u09b0 \u09b8\u09cb\u09b0\u09cd\u09b8 \u0995\u09cb\u09a1 \u09ac\u09be \u09ae\u09c2\u09b2 \u09b8\u09be\u0982\u0995\u09c7\u09a4\u09bf\u0995 \u09ad\u09be\u09b7\u09be\u0995\u09c7 \u09ae\u09c1\u0995\u09cd\u09a4 \u09ad\u09be\u09ac\u09c7 \u09ac\u09bf\u09a4\u09b0\u09a3 \u0995\u09b0\u09be \u0964
+\u09ac\u09be\u0982\u09b2\u09be\u09a6\u09c7\u09b6 \u0985\u09a8\u09b2\u09be\u0987\u09a8\u09c7 \u09a2\u09be\u0995\u09be
+\u09aa\u09cd\u09b0\u09a5\u09ae \u09ac\u09bf\u09b6\u09cd\u09ac\u09af\u09c1\u09a6\u09cd\u09a7\u09c7 \u099c\u09be\u09b0\u09cd\u09ae\u09be\u09a8\u09bf \u09b9\u09c7\u09b0\u09c7 \u09af\u09be\u09af\u09bc \u0964
+\u09a4\u09ac\u09c7 \u098f \u09ac\u09bf\u09b7\u09af\u09bc\u099f\u09bf \u09ac\u09cb\u099d\u09be\u09b0 \u099c\u09a8\u09cd\u09af \u09ac\u09b0\u09cd\u09a4\u09ae\u09be\u09a8\u09c7\u0993 \u0997\u09ac\u09c7\u09b7\u09a3\u09be \u098f\u0997\u09bf\u09af\u09bc\u09c7 \u099a\u09b2\u099b\u09c7 \u0964
+\u09b8\u09c1\u09aa\u09be\u09b0\u098f\u0987\u099a
+\u09a4\u09be\u0995\u09c7 \u09b8\u09be\u09ae\u09b0\u09bf\u0995 \u09ac\u09be\u09b9\u09bf\u09a8\u09c0\u09b0 \u099c\u09a8\u09cd\u09af \u0986\u09a8\u09ab\u09bf\u099f \u0998\u09cb\u09b7\u09a3\u09be \u0995\u09b0\u09be \u09b9\u09af\u09bc\u09c7\u099b\u09bf\u09b2 \u0964
+\u09ad\u09c1\u099f\u09cd\u099f\u09cb \u098f\u09cd\u09af\u09be\u09b8\u09c7\u09ae\u09cd\u09ac\u09b2\u09bf \u09ac\u09af\u09bc\u0995\u099f \u0995\u09b0\u09be\u09b0 \u09b9\u09c1\u09ae\u0995\u09bf \u09a6\u09bf\u09af\u09bc\u09c7 \u0998\u09cb\u09b7\u09a3\u09be \u09a6\u09c7\u09a8 \u09af\u09c7 & # 44 ; \u0987\u09af\u09bc\u09be\u09b9\u09bf\u09af\u09bc\u09be \u0996\u09be\u09a8 \u09ae\u09c1\u099c\u09bf\u09ac\u0995\u09c7 \u09b8\u09b0\u0995\u09be\u09b0 \u0997\u09a0\u09a8\u09c7\u09b0 \u099c\u09a8\u09cd\u09af \u0986\u09b9\u09cd\u09ac\u09be\u09a8 \u099c\u09be\u09a8\u09be\u09b2\u09c7 \u09a4\u09bf\u09a8\u09bf \u09b8\u09c7 \u09b8\u09b0\u0995\u09be\u09b0\u0995\u09c7 \u09ae\u09c7\u09a8\u09c7 \u09a8\u09c7\u09ac\u09c7\u09a8 \u09a8\u09be \u0964
+\u0986\u09b0 computer \u09b6\u09ac\u09cd\u09a6\u09c7\u09b0 \u0985\u09b0\u09cd\u09a5 \u0997\u09a3\u09a8\u09be\u0995\u09be\u09b0\u09c0 \u09af\u09a8\u09cd\u09a4\u09cd\u09b0 \u0964
+\u09e7\u09ed\u09ed\u09ec \u09b8\u09be\u09b2\u09c7\u09b0 \u09ea \u099c\u09c1\u09b2\u09be\u0987 \u098f\u0987 \u0989\u09aa\u09a8\u09bf\u09ac\u09c7\u09b6\u0997\u09c1\u09b2\u09bf \u098f\u0995\u099f\u09bf \u09b8\u09cd\u09ac\u09be\u09a7\u09c0\u09a8\u09a4\u09be\u09b0 \u0998\u09cb\u09b7\u09a3\u09be\u09aa\u09a4\u09cd\u09b0 \u099c\u09be\u09b0\u09bf \u0995\u09b0\u09c7 \u0964
+\u099c\u09be\u09b0\u09cd\u09ae\u09be\u09a8\u09bf -lrb- \u099c\u09be\u09b0\u09cd\u09ae\u09be\u09a8 \u09ad\u09be\u09b7\u09be\u09af\u09bc : deutschland & # 44 ; \u09a1\u09af\u09bc\u099a\u09cd \u200c \u09b2\u09be\u09a8\u09cd\u099f\u09cd \u200c & # 44 ; \u0986-\u09a7\u09cd\u09ac-\u09ac : -lsb- d\u0254\u028ft\u0283lant -rsb- -rrb- & # 44 ; \u09ae\u09a7\u09cd\u09af \u0987\u0989\u09b0\u09cb\u09aa\u09c7\u09b0 \u098f\u0995\u099f\u09bf \u09b0\u09be\u09b7\u09cd\u099f\u09cd\u09b0 \u0964
+\u0996\u09cd\u09b0\u09bf\u09b8\u09cd\u099f\u09a7\u09b0\u09cd\u09ae \u09b0\u09be\u09b6\u09bf\u09af\u09bc\u09be\u09b0 \u09aa\u09cd\u09b0\u09a7\u09be\u09a8 \u09a7\u09b0\u09cd\u09ae \u0964
+\u0995\u09bf\u09a8\u09cd\u09a4\u09c1 \u0997\u09b2\u09a6\u09c7\u09b0 \u09b6\u09bf\u0995\u09cd\u09b7\u09be\u09b0 \u09b0\u09cb\u09ae\u09be\u09a8\u09c0\u0995\u09b0\u09a3\u09c7\u09b0 \u0997\u09a4\u09bf \u099b\u09bf\u09b2 \u09a7\u09c0\u09b0 \u0964
+\u09ac\u09bf\u09b7\u09af\u09bc\u09b6\u09cd\u09b0\u09c7\u09a3\u09c0 : \u0997\u09a8\u09c1 \u09ab\u09be\u0989\u09a8\u09cd\u09a1\u09c7\u09b6\u09a8
+\u0986\u09b0\u09cd\u09a5\u09bf\u0995 \u09a8\u09c0\u09a4\u09bf \u0993 \u09b0\u09be\u099c\u09b8\u09cd\u09ac \u09a8\u09c0\u09a4\u09bf\u0995\u09c7\u0993 \u0987\u09b9\u09be \u0985\u09a7\u09cd\u09af\u09af\u09bc\u09a8 \u0995\u09b0\u09c7 \u0964
+\u098f\u09b0 \u09ae\u09a7\u09cd\u09af\u09c7 \u09b0\u09af\u09bc\u09c7\u099b\u09c7 : \u09b9\u09af\u09bc\u09a4\u09cb \u09a4\u09cb\u09ae\u09be\u09b0 \u09aa\u09be\u09ac \u09a6\u09c7\u0996\u09be & # 44 ; \u0993\u09b0\u09c7 \u098f \u0995\u09cb\u09a8 \u09b8\u09cd\u09a8\u09c7\u09b9-\u09b8\u09c1\u09b0\u09a7\u09c1\u09a8\u09c0 \u0964
+\u09e7\u09ef\u09ef\u09e8 \u09b8\u09be\u09b2\u09c7\u09b0 \u09e8\u09e9 \u098f\u09aa\u09cd\u09b0\u09bf\u09b2 \u09b8\u09a4\u09cd\u09af\u099c\u09bf\u09ce \u09ae\u09c3\u09a4\u09cd\u09af\u09c1\u09ac\u09b0\u09a3 \u0995\u09b0\u09c7\u09a8 \u0964
+\u098f\u0987 \u09b8\u09ae\u09af\u09bc \u09a8\u099c\u09b0\u09c1\u09b2\u09c7\u09b0 \u09ae\u09c7\u09a1\u09bf\u0995\u09c7\u09b2 \u09b0\u09bf\u09aa\u09cb\u09b0\u09cd\u099f \u09ad\u09bf\u09af\u09bc\u09c7\u09a8\u09be\u09b0 \u09ac\u09bf\u0996\u09cd\u09af\u09be\u09a4 \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u0995\u09a6\u09c7\u09b0 \u0995\u09be\u099b\u09c7 \u09aa\u09be\u09a0\u09be\u09a8\u09cb \u09b9\u09af\u09bc \u0964
+\u0985\u09ad\u09bf\u09a8\u09af\u09bc \u099b\u09be\u09a1\u09bc\u09be\u0993 \u09ac\u09bf\u09ad\u09bf\u09a8\u09cd\u09a8 \u09b8\u09ae\u09af\u09bc\u09c7 \u09b0\u09be\u09a8\u09c0 \u09ae\u09c1\u0996\u09be\u09b0\u09cd\u099c\u09c0 \u09ac\u09bf\u09ad\u09bf\u09a8\u09cd\u09a8 \u09a6\u09be\u09a4\u09ac\u09cd\u09af \u09b8\u0982\u09b8\u09cd\u09a5\u09be\u09b0 \u09b8\u09be\u09a5\u09c7 \u09af\u09c1\u0995\u09cd\u09a4 \u09a5\u09c7\u0995\u09c7\u099b\u09c7\u09a8 \u0964
+\u09ac\u09be\u0982\u09b2\u09be \u09b8\u09be\u09b9\u09bf\u09a4\u09cd\u09af \u098f\u09ac\u0982 \u09b8\u0982\u09b8\u09cd\u0995\u09c3\u09a4\u09bf\u09a4\u09c7 \u09a4\u09be\u09b0 \u09ac\u09bf\u09b6\u09c7\u09b7 \u0985\u09ac\u09a6\u09be\u09a8\u09c7\u09b0 \u09b8\u09cd\u09ac\u09c0\u0995\u09c3\u09a4\u09bf\u09b8\u09cd\u09ac\u09b0\u09c1\u09aa \u09e7\u09ef\u09ed\u09ea \u0996\u09cd\u09b0\u09bf\u09b8\u09cd\u099f\u09be\u09ac\u09cd\u09a6\u09c7\u09b0 \u09ef \u09a1\u09bf\u09b8\u09c7\u09ae\u09cd\u09ac\u09b0 \u09a4\u09be\u09b0\u09bf\u0996\u09c7 \u09a2\u09be\u0995\u09be \u09ac\u09bf\u09b6\u09cd\u09ac\u09ac\u09bf\u09a6\u09cd\u09af\u09be\u09b2\u09af\u09bc \u09a4\u09be\u0995\u09c7 \u09b8\u09ae\u09cd\u09ae\u09be\u09a8\u09b8\u09c2\u099a\u0995 \u09a1\u09bf.\u09b2\u09bf\u099f \u0989\u09aa\u09be\u09a7\u09bf\u09a4\u09c7 \u09ad\u09c2\u09b7\u09bf\u09a4 \u0995\u09b0\u09c7 \u0964
+\u0995\u09b2\u0995\u09be\u09a4\u09be\u09b0 \u09a6\u09c1\u09b0\u09cd\u0997\u09be\u09aa\u09c2\u099c\u09be \u09b6\u09b9\u09b0\u09c7\u09b0 \u0985\u09a8\u09cd\u09af\u09a4\u09ae \u09aa\u09b0\u09cd\u09af\u099f\u09a8 \u0986\u0995\u09b0\u09cd\u09b7\u09a3\u0993 \u09ac\u099f\u09c7 \u0964
+\u0995\u09bf\u09a8\u09cd\u09a4\u09c1 \u09ac\u09b9\u09c1 \u09b2\u0995\u09cd\u09b7 \u09aa\u09c2\u09b0\u09cd\u09ac \u099c\u09be\u09b0\u09cd\u09ae\u09be\u09a8 \u0985\u09b0\u09cd\u09a5\u09a8\u09c8\u09a4\u09bf\u0995\u09ad\u09be\u09ac\u09c7 \u09b8\u09ae\u09c3\u09a6\u09cd\u09a7 \u0993 \u0997\u09a3\u09a4\u09be\u09a8\u09cd\u09a4\u09cd\u09b0\u09bf\u0995 \u09aa\u09b6\u09cd\u099a\u09bf\u09ae \u099c\u09be\u09b0\u09cd\u09ae\u09be\u09a8\u09bf\u09a4\u09c7 \u0985\u09ad\u09bf\u09ac\u09be\u09b8\u09c0 \u09b9\u0993\u09af\u09bc\u09be \u09b6\u09c1\u09b0\u09c1 \u0995\u09b0\u09b2\u09c7 \u09e7\u09ef\u09ec\u09e7 \u09b8\u09be\u09b2\u09c7 \u09aa\u09c2\u09b0\u09cd\u09ac \u099c\u09be\u09b0\u09cd\u09ae\u09be\u09a8\u09bf \u09b8\u09b0\u0995\u09be\u09b0 \u09ac\u09be\u09b0\u09cd\u09b2\u09bf\u09a8\u09c7 \u098f\u0995\u099f\u09bf \u09aa\u09cd\u09b0\u09be\u099a\u09c0\u09b0 \u09a4\u09c1\u09b2\u09c7 \u09a6\u09c7\u09af\u09bc \u098f\u09ac\u0982 \u09a6\u09c7\u09b6\u09c7\u09b0 \u09b8\u09c0\u09ae\u09be\u09a8\u09cd\u09a4 \u099c\u09cb\u09b0\u09a6\u09be\u09b0 \u0995\u09b0\u09c7 \u0964
+\u09aa\u09cd\u09b0\u09a5\u09ae\u099f\u09bf \u09b9\u099a\u09cd\u099b\u09c7 \u099b\u09ac\u09bf\u099f\u09bf\u09b0 \u09a8\u09bf\u09b0\u09cd\u09ac\u09be\u0995 \u09aa\u09cd\u09b0\u09a5\u09ae \u09b8\u09be\u09a4 \u09ae\u09bf\u09a8\u09bf\u099f & # 44 ; \u09af\u09be \u099a\u09be\u09b0\u09c1\u09b0 \u098f\u0995\u0998\u09c7\u09af\u09bc\u09c7\u09ae\u09bf \u099c\u09c0\u09ac\u09a8 \u09ab\u09c1\u099f\u09bf\u09af\u09bc\u09c7 \u09a4\u09cb\u09b2\u09c7 & # 44 ; \u098f\u09ac\u0982 \u09a6\u09cd\u09ac\u09bf\u09a4\u09c0\u09af\u09bc\u099f\u09bf \u09b9\u09b2 & quot ; \u09ac\u09be\u0997\u09be\u09a8\u09c7\u09b0 \u09a6\u09cb\u09b2\u09a8\u09be\u09b0 \u09a6\u09c3\u09b6\u09cd\u09af & quot ; & # 44 ; \u09af\u09c7\u0996\u09be\u09a8\u09c7 \u099a\u09be\u09b0\u09c1 \u0985\u09ae\u09b2\u09c7\u09b0 \u099c\u09a8\u09cd\u09af \u09a4\u09be\u09b0 \u09ad\u09be\u09b2\u09ac\u09be\u09b8\u09be\u09b0 \u09ae\u09c1\u0996\u09cb\u09ae\u09c1\u0996\u09bf \u09b9\u09af\u09bc \u0964
+\u09e7\u09ee \u09b6\u09a4\u0995\u09c7\u09b0 \u098f\u0995\u09a6\u09b2 \u099a\u09bf\u09a8\u09cd\u09a4\u09be\u09ac\u09bf\u09a6 \u0993 \u09b2\u09c7\u0996\u0995 \u0986\u09af\u09bc \u0993 \u0989\u09ce\u09aa\u09be\u09a6\u09a8\u09c7\u09b0 \u099a\u0995\u09cd\u09b0\u09be\u0995\u09be\u09b0 \u09aa\u09cd\u09b0\u09ac\u09be\u09b9\u09c7\u09b0 \u09ae\u09be\u09a7\u09cd\u09af\u09ae\u09c7 \u0985\u09b0\u09cd\u09a5\u09a8\u09c8\u09a4\u09bf\u0995 \u099a\u09bf\u09a8\u09cd\u09a4\u09be\u09a7\u09be\u09b0\u09be\u09b0 \u0989\u09a8\u09cd\u09a8\u09af\u09bc\u09a8 \u0998\u099f\u09be\u09a8 \u0964
+\u09af\u09cb\u09a8\u09c0\u09a4\u09c7 \u09b2\u09bf\u0999\u09cd\u0997 \u09aa\u09cd\u09b0\u09ac\u09bf\u09b7\u09cd\u099f\u0995\u09b0\u09a3\u09c7\u09b0 \u09aa\u09c2\u09b0\u09cd\u09ac\u09c7 \u0995\u09be\u09ae\u09cb\u09a6\u09cd\u09a6\u09c0\u09aa\u0995 \u0995\u09be\u09b0\u09cd\u09af\u0995\u09b2\u09be\u09aa\u0995\u09c7 \u09ac\u09b2\u09be \u09b9\u09af\u09bc \u09b6\u09c3\u0999\u09cd\u0997\u09be\u09b0 \u0964
+\u098f\u099f\u09bf \u09ae\u09c2\u09b2\u09a4 \u09b6\u09c2\u0995\u09b0\u09c7\u09b0 \u09ae\u09be\u099d\u09c7\u0987 \u09aa\u09be\u0993\u09af\u09bc\u09be \u09af\u09c7\u09a4 \u09af\u09be \u0995\u09bf\u09a8\u09be \u09b6\u09c2\u0995\u09b0\u0995\u09c7 \u0987\u09a8\u09ab\u09cd\u09b2\u09c1\u09af\u09bc\u09c7\u099e\u09cd\u099c\u09be\u09a4\u09c7 \u0986\u0995\u09cd\u09b0\u09be\u09a8\u09cd\u09a4 \u0995\u09b0\u09a4 \u0964
+\u098f\u0997\u09c1\u09b2\u09bf \u098f\u0995\u098f \u09b9\u09af\u09bc\u09c7 mycelium \u0997\u09a0\u09a8 \u0995\u09b0\u09c7 \u0964
+\u09b0\u09be\u09b6\u09bf\u09af\u09bc\u09be \u09ac\u09b0\u09cd\u09a4\u09ae\u09be\u09a8\u09c7 \u098f\u0995\u099f\u09bf \u0997\u09a3\u09a4\u09be\u09a8\u09cd\u09a4\u09cd\u09b0\u09bf\u0995 \u09b0\u09be\u09b7\u09cd\u099f\u09cd\u09b0 \u0964
+\u09b2\u09bf\u0999\u09cd\u0997
+\u098f\u0987 \u0985\u09ac\u09b8\u09cd\u09a5\u09be\u0995\u09c7 \u09ac\u09b2\u09be \u09b9\u09af\u09bc \u09b0\u09be\u0997\u09ae\u09cb\u099a\u09a8 -lrb- \u0985\u09a8\u0997\u09cd\u09af\u09be\u099c\u09ae -rrb- \u0964
+\u0987\u09a4\u09bf\u09b9\u09be\u09b8\u09c7\u09b0 \u09ac\u09bf\u09ad\u09bf\u09a8\u09cd\u09a8 \u09aa\u09b0\u09cd\u09ac\u09c7 \u098f\u0996\u09be\u09a8\u09c7\u0987 \u09b8\u09cd\u09a5\u09be\u09aa\u09bf\u09a4 \u09b9\u09af\u09bc\u09c7\u099b\u09bf\u09b2 \u09ac\u09bf\u09b6\u09be\u09b2\u09be\u0995\u09be\u09b0 \u098f\u0995\u09be\u09a7\u09bf\u0995 \u09b8\u09be\u09ae\u09cd\u09b0\u09be\u099c\u09cd\u09af \u0964
+\u09ac\u09cd\u09af\u09be\u09b8\u09cd\u099f\u09bf\u0995 \u0985\u09b0\u09cd\u09a5\u09a8\u09c0\u09a4\u09bf
+\u09ac\u09cd\u09af\u09ac\u09b9\u09be\u09b0\u0995\u09be\u09b0\u09c0\u09b0 \u0995\u09be\u099b\u09c7 \u0985\u09aa\u09be\u09b0\u09c7\u099f\u09bf\u0982 \u09b8\u09bf\u09b8\u09cd\u099f\u09c7\u09ae\u09c7\u09b0 \u09b8\u09ac\u099a\u09c7\u09af\u09bc\u09c7 \u09a6\u09c3\u09b6\u09cd\u09af\u09ae\u09be\u09a8 \u09b0\u09c2\u09aa \u09b9\u09b2 \u0995\u09ae\u09cd\u09aa\u09bf\u0989\u099f\u09be\u09b0\u09c7\u09b0 \u0987\u09a8\u09cd\u099f\u09be\u09b0\u09ab\u09c7\u09b8 \u0964
+\u09ac\u09bf\u09b8\u09cd\u09a4\u09be\u09b0\u09bf\u09a4\u0983 \u09e7\u09ef\u09ed\u09e7 \u09b8\u09be\u09b2\u09c7\u09b0 \u0985\u09b8\u09cd\u09a5\u09be\u09af\u09bc\u09c0 \u09ac\u09be\u0982\u09b2\u09be\u09a6\u09c7\u09b6 \u09b8\u09b0\u0995\u09be\u09b0
[10/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/hiero/output.gold.bleu
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/hiero/output.gold.bleu b/joshua-core/src/test/resources/bn-en/hiero/output.gold.bleu
new file mode 100644
index 0000000..596e317
--- /dev/null
+++ b/joshua-core/src/test/resources/bn-en/hiero/output.gold.bleu
@@ -0,0 +1,14 @@
+Processing 100 sentences...
+Evaluating set of 1'th candidate translations from output...
+BLEU_precision(1) = 1032 / 1474 = 0.7001
+BLEU_precision(2) = 478 / 1374 = 0.3479
+BLEU_precision(3) = 248 / 1280 = 0.1938
+BLEU_precision(4) = 141 / 1190 = 0.1185
+BLEU_precision = 0.2735
+
+Length of candidate corpus = 1474
+Effective length of reference corpus = 1446
+BLEU_BP = 1.0000
+
+ => BLEU = 0.2735
+
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/hiero/output.scores.berkeleylm.gold
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/hiero/output.scores.berkeleylm.gold b/joshua-core/src/test/resources/bn-en/hiero/output.scores.berkeleylm.gold
new file mode 100644
index 0000000..fc8eeb5
--- /dev/null
+++ b/joshua-core/src/test/resources/bn-en/hiero/output.scores.berkeleylm.gold
@@ -0,0 +1,100 @@
+ rabindranath was born in a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| -226.302
+ recently with united states with the relationship between improved . ||| -21.022
+ mathematics so science language . ||| -10.471
+ from this it it can be understood that this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf will be \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| -322.621
+ the same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match from this novel . ||| -236.836
+ \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 majority in the . ||| -217.895
+ \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his work with to that . ||| -432.357
+ other than task \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character there are but these are very is not . ||| -246.114
+ it social situation in \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the vagina . ||| -255.565
+ \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 annual \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf meters ||| -321.077
+ on the year 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in the city in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f the national he was the main speech -lrb- keynote speech -rrb- on the . ||| -664.033
+ power distribution of population on east pakistan where to west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac started with the where the whole of west pakistan was considered as a province . ||| -186.010
+ the \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 theory ||| -110.415
+ external links of ||| -4.318
+ tata communicationer " foreign sanchar nigam limited building it is the telecommunication system is one of the main providers ||| -48.733
+ he that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national assembly in election won all and was elected as the 44th president of the united states . ||| -241.090
+ many indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be from upper stage ||| -332.122
+ britain writers written drama novels and stories recently scripts in \u0986\u09a6\u09c3\u09a4 . ||| -140.670
+ on may 1919 , it was published in saogat magazine . ||| -19.086
+ 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium was arranged . ||| -452.293
+ to prevent this several measures are taken . ||| -12.088
+ on 5th february , \u09e7\u09ef\u09ec\u09ec \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 of \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 a national was held in . ||| -334.940
+ bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in in the . ||| -341.281
+ subject category : encyclopedia ||| 1.726
+ russia france and israel is the main \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 country and defense sub country . ||| -132.240
+ this is our known as an imaginary unit of mathematics formed with the help of which are set of real numbers from \u09b8\u09c7\u099f\u09c7 par with the complex number . ||| -143.878
+ <address> ||| -13.673
+ september ||| 1.148
+ from this theory though big \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 against can not be but it can be support . ||| -133.987
+ agricultural in production france is the most important country ; it is mainly \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af wine cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and export the in the whole world . ||| -252.851
+ their in mathematics \u09aa\u09be\u099f\u09c0\u0997\u09a3\u09bf\u09a4\u09c7\u09b0 person was . ||| -115.913
+ deshgulo are : france call , make noise china belgium switzerland germany denmark sweden austria chekoslovakia argentina italy norway \u09b9\u09be\u0999\u09cd\u0997\u09c7\u09b0\u09c0 yugoslavia bulgaria romania \u0997\u09cd\u09b0\u09c0\u09b8 egypt singapore indonesia \u09a5\u09be\u0987\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 japan burma \u09b9\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 soviet russia iran iraq and sri lanka . ||| -548.678
+ this \u09ac\u09cd\u09af\u09be\u09b8\u09bf\u09b2\u09bf\u0995\u09be\u09b0 places now bank of england is situated . ||| -117.393
+ on the north of the country is the \u09ac\u09bf\u09b8\u09cd\u0995\u09be\u0987 sub-sea on the south \u099c\u09bf\u09ac\u09cd\u09b0\u09be\u09b2\u09cd\u099f\u09be\u09b0 strait \u09aa\u09cd\u09b0\u09a3\u09be\u09b2\u09c0\u09b0 in the south \u09ae\u09b0\u0995\u09cd\u0995\u09cb on the west and the atlantic ocean . ||| -445.620
+ apart from this situation it can be understood that a \u09ae\u09c1\u09b9\u09c2\u09b0\u09cd\u09a4\u09c7 within \u0985\u0995\u09cd\u09b7\u09ae\u09a4\u09be after taking the decision of the united nations . ||| -248.040
+ \u0995\u09be\u09b0\u09cd\u09b2 there is work through it is controlled by . ||| -125.132
+ the subject matters sometimes puran -lrb- from sometimes in the middle age love story sometimes again from today 's social and political \u0998\u099f\u09a8\u09be\u09ac\u09b2\u09bf taken from . ||| -157.897
+ three measure based on the age of the universe is found that is almost \u09e7\u09e9.\u09ed � \u09e6.\u09e8 billion years . ||| -344.330
+ there are \u0995\u09be\u099b\u09c7\u0987 east russia which \u0993\u0996\u099f\u09b8\u09cd\u0995 sea and japan sea on the other can be is situated . ||| -235.028
+ in kolkata is located at the national library of india the leading public \u09b2\u09be\u0987\u09ac\u09cd\u09b0\u09c7\u09b0\u09bf . ||| -119.076
+ \u099b\u09a4\u09cd\u09b0\u09be\u0995\u09ac\u09bf\u09a6\u09cd\u09af\u09be ||| -103.288
+ \u09b0\u09be\u09b7\u09cd\u099f\u09cd\u09b0\u09b8\u0999\u09cd\u0998\u09c7\u09b0 general secretary \u09ac\u09be\u09a8 ki moon ||| -218.172
+ \u09ae\u09bf\u09a8\u09bf\u0995\u09cd\u09b8\u09c7\u09b0 of india was smells \u099f\u09be\u09a8\u09c7\u09a8\u09ac\u09ae a famous operating system design \u09aa\u09cd\u09b0\u09b6\u09bf\u0995\u09cd\u09b7\u0995 . ||| -337.188
+ the \u099f\u09be\u0987\u09ae\u09cd the 's of \u0987\u09a8\u09cd\u09a1\u09bf\u09af\u09bc\u09be-\u09a4\u09c7 written in the " it is absurd to compare it with any other indian cinema ... pather panchali is pure cinema " -lrb- " it other by the indian films with compared to unreal ... pather panchali are pure film " -rrb- . ||| -1714.540
+ after that , 1953 in the month of may nazrul and \u09aa\u09cd\u09b0\u09ae\u09c0\u09b2\u09be \u09a6\u09c7\u09ac\u09c0\u0995\u09c7 \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u09be\u09b0 for london sent to . ||| -336.768
+ the southern and the east there is \u09ac\u09bf\u09b8\u09cd\u09a4\u09c0\u09b0\u09cd\u09a3 land , west and in the north there are \u09b0\u09c1\u0995\u09cd\u09b7 mountain and mountain . ||| -242.245
+ \u099f\u09cd\u09b0\u09c7\u09a1\u09ae\u09be\u09b0\u09cd\u0995 ||| -103.288
+ \u09b2\u09b0\u09cd\u09a1 \u0993\u09af\u09bc\u09c7\u09b2\u09c7\u09b8\u09b2\u09bf\u09b0 -lrb- \u0997\u09ad\u09b0\u09cd\u09a8\u09b0-\u099c\u09c7\u09a8\u09be\u09b0\u09c7\u09b2 \u09e7\u09ed\u09ef\u09ed-\u09e7\u09ee\u09e6\u09eb -rrb- during his city are the ghotechilo ||| -443.070
+ many important and real extremely necessary to solve problems complex number ||| -26.515
+ the big bang is a famous result in the state of the universe so and recent situation from the separate . ||| -45.363
+ windows \u09ae\u09bf\u09b2\u09bf\u09a8\u09bf\u09af\u09bc\u09be\u09ae ||| -104.858
+ however , rabindranath more than one \u09b6\u09c8\u09b2\u09c0 \u0986\u09af\u09bc\u09a4\u09cd\u09a4\u09cd\u09ac was . ||| -219.540
+ labour economics ||| -0.994
+ britain at some point of time the was the main and his economic power in the world . ||| -20.224
+ the military rule movement against pakistan and \u0997\u09cb\u09b7\u09cd\u09a0\u09c0\u0997\u09a4 was against protest and independence movement of the bengalis , \u09a7\u09be\u09ac\u09bf\u09a4 to for he widely praised . ||| -266.155
+ here is mentioned that were internet and other name of world wide web word to be but actually \u09b6\u09ac\u09cd\u09a6\u09a6\u09cd\u09ac\u09af\u09bc different subject to the . ||| -171.397
+ . the related z polar co-ordinate two are r = . ||| -31.445
+ november ||| 1.122
+ 1972 \u0996\u09cd\u09b0\u09c0\u09b8\u09cd\u099f\u09be\u09ac\u09cd\u09a6\u09c7 then \u0986\u09b0\u09aa\u09be\u09a8\u09c7\u099f\u09c7 first \u0987\u09b2\u09c7\u0995\u09cd\u099f\u09cd\u09b0\u09a8\u09bf\u0995 mail to to . ||| -337.642
+ \u099c\u09c0\u09ac science that \u09b6\u09be\u0996\u09be\u09af\u09bc fungi and the practical about the subject to he was \u099b\u09a4\u09cd\u09b0\u09be\u0995\u09ac\u09bf\u09a6\u09cd\u09af\u09be -lrb- mycology -rrb- ||| -453.504
+ water river from \u0989\u09a0\u09be\u09a8\u09cb was some a \u09a6\u09bf\u09af\u09bc\u09c7- \u09a6\u09a1\u09bc\u09bf and \u09ac\u09be\u09b2\u099f\u09bf\u09b0 through water used by animal world a system . ||| -467.773
+ among these there are tribal dance lokuj dance classical dance etc . ||| -25.170
+ the oldest literature at first \u09ae\u09cc\u0996\u09bf\u0995\u09ad\u09be\u09ac\u09c7 and later written form is the . ||| -128.172
+ in 1989 , the \u0997\u09a3\u09b6\u09a4\u09cd\u09b0\u09c1 in this film his conducted \u09a4\u09c1\u09b2\u09a8\u09be\u09ae\u09c2\u09b2\u0995\u09ad\u09be\u09ac\u09c7 \u09a6\u09c1\u09b0\u09cd\u09ac\u09b2 and this \u09a6\u09c0\u09b0\u09cd\u0998\u09a6\u09bf\u09a8\u09c7\u09b0 \u0985\u09b8\u09c1\u09b8\u09cd\u09a5\u09a4\u09be\u09b6\u09c7\u09b7\u09c7 came back , after \u09aa\u09c1\u09a8\u09b0\u09cd\u09aa\u09cd\u09b0\u099a\u09c7\u09b7\u09cd\u099f\u09be of satyajit 's films , a is considered as . ||| -681.400
+ the \u09ac\u09b2\u09ac\u09bf\u09a6\u09cd\u09af\u09be ||| -108.280
+ other \u09b8\u09cd\u09ac\u09a4\u09cd\u09a4\u09cd\u09ac-\u09b8\u0982\u09b0\u0995\u09cd\u09b7\u09bf\u09a4 operating system like windows and mac os to linux in different different . ||| -126.793
+ asia \u099f\u09be\u0987\u09ae\u09b8\u09c7\u09b0 \u09ad\u09be\u09b7\u09cd\u09af according to , ||| -215.125
+ open source or open source -lrb- open source -rrb- the money is computer software the source code or the \u09b8\u09be\u0982\u0995\u09c7\u09a4\u09bf\u0995 language open way \u09ac\u09bf\u09a4\u09b0\u09a3 to . ||| -461.854
+ bangladesh \u0985\u09a8\u09b2\u09be\u0987\u09a8\u09c7 dhaka ||| -107.382
+ the first world war germany \u09b9\u09c7\u09b0\u09c7 can be seen . ||| -109.139
+ but this subject is to understand for even research to going on . ||| -32.160
+ \u09b8\u09c1\u09aa\u09be\u09b0\u098f\u0987\u099a ||| -103.288
+ he was military forces for \u0986\u09a8\u09ab\u09bf\u099f was declared . ||| -118.584
+ bhutto \u098f\u09cd\u09af\u09be\u09b8\u09c7\u09ae\u09cd\u09ac\u09b2\u09bf \u09ac\u09af\u09bc\u0995\u099f to 2.5 with announced that the yahya khan mujib was to form the government to \u099c\u09be\u09a8\u09be\u09b2\u09c7 he that government by \u09a8\u09c7\u09ac\u09c7\u09a8 not . ||| -478.493
+ and computer words money \u0997\u09a3\u09a8\u09be\u0995\u09be\u09b0\u09c0 machine . ||| -217.956
+ on 4th july \u09e7\u09ed\u09ed\u09ec this constituents of a independence notice \u099c\u09be\u09b0\u09bf the . ||| -236.748
+ germany -lrb- in german : deutschland \u09a1\u09af\u09bc\u099a\u09cd the \u09b2\u09be\u09a8\u09cd\u099f\u09cd of pronounced [ d\u0254\u028ft\u0283lant through ] -rrb- is a country of europe . ||| -446.865
+ the main religion \u0996\u09cd\u09b0\u09bf\u09b8\u09cd\u099f\u09a7\u09b0\u09cd\u09ae from russia . ||| -110.479
+ but \u0997\u09b2\u09a6\u09c7\u09b0 education \u09b0\u09cb\u09ae\u09be\u09a8\u09c0\u0995\u09b0\u09a3\u09c7\u09b0 \u0997\u09a4\u09bf was slow down . ||| -321.499
+ subject category : gnu foundation ||| -4.645
+ economic policy and revenue \u09a8\u09c0\u09a4\u09bf\u0995\u09c7\u0993 it study the . ||| -124.215
+ among these there are : may be \u09a4\u09cb\u09ae\u09be\u09b0 get seen \u0993\u09b0\u09c7 this any \u09b8\u09cd\u09a8\u09c7\u09b9-\u09b8\u09c1\u09b0\u09a7\u09c1\u09a8\u09c0 . ||| -335.374
+ on 23rd april 1992 satyajit died . ||| -13.054
+ at this time nazrul 's medical \u09b0\u09bf\u09aa\u09cb\u09b0\u09cd\u099f stay famous \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u0995\u09a6\u09c7\u09b0 to send to . ||| -232.362
+ other than acted at different times rani \u09ae\u09c1\u0996\u09be\u09b0\u09cd\u099c\u09c0 different \u09a6\u09be\u09a4\u09ac\u09cd\u09af are connected with \u09a5\u09c7\u0995\u09c7\u099b\u09c7\u09a8 . ||| -338.434
+ bengali literature and culture a special contribution on \u09b8\u09cd\u09ac\u09c0\u0995\u09c3\u09a4\u09bf\u09b8\u09cd\u09ac\u09b0\u09c1\u09aa year on 9th december , dhaka university he was honorary \u09a1\u09bf.\u09b2\u09bf\u099f \u0989\u09aa\u09be\u09a7\u09bf\u09a4\u09c7 awarded with the . ||| -358.420
+ in kolkata durga puja tourism \u0986\u0995\u09b0\u09cd\u09b7\u09a3\u0993 of the city is also a reason ||| -131.010
+ but many people of east germany started living in economically reach and democratic west germany , east germany government built a wall in berlin in 1961 and reinforced its boundaries . ||| -38.023
+ the first one is the film \u09a8\u09bf\u09b0\u09cd\u09ac\u09be\u0995 first seven \u09ae\u09bf\u09a8\u09bf\u099f which \u099a\u09be\u09b0\u09c1\u09b0 \u098f\u0995\u0998\u09c7\u09af\u09bc\u09c7\u09ae\u09bf life shown level and the second one is " the \u09a6\u09cb\u09b2\u09a8\u09be\u09b0 scene " where \u099a\u09be\u09b0\u09c1 \u0985\u09ae\u09b2\u09c7\u09b0 for his love \u09ae\u09c1\u0996\u09cb\u09ae\u09c1\u0996\u09bf in . ||| -900.700
+ a group of 18th century the \u099a\u09bf\u09a8\u09cd\u09a4\u09be\u09ac\u09bf\u09a6 and writer income and production \u099a\u0995\u09cd\u09b0\u09be\u0995\u09be\u09b0 current through economic thoughts development it . ||| -251.701
+ the arousal activities before penetrating male organ into vagina is called foreplay . ||| -16.182
+ it is mainly \u09b6\u09c2\u0995\u09b0\u09c7\u09b0 in in was which can \u09b6\u09c2\u0995\u09b0\u0995\u09c7 \u0987\u09a8\u09ab\u09cd\u09b2\u09c1\u09af\u09bc\u09c7\u099e\u09cd\u099c\u09be\u09a4\u09c7 affected by british . ||| -341.900
+ these \u098f\u0995\u098f the mycelium formed the . ||| -217.337
+ russia at present a democratic country . ||| -6.462
+ penis ||| -3.566
+ this state is called orgasm . ||| -3.009
+ history different period here established the royal more than one empire . ||| -34.107
+ micro economics ||| -0.625
+ the user to operating system the visible form of the computer interface . ||| -29.010
+ \u09ac\u09bf\u09b8\u09cd\u09a4\u09be\u09b0\u09bf\u09a4\u0983 , 1971 temporary bangladesh government ||| -110.935
[25/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/tools/GrammarPacker.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/tools/GrammarPacker.java b/joshua-core/src/main/java/org/apache/joshua/tools/GrammarPacker.java
new file mode 100644
index 0000000..b9208d2
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/tools/GrammarPacker.java
@@ -0,0 +1,959 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.tools;
+
+import static org.apache.joshua.decoder.ff.tm.packed.PackedGrammar.VOCABULARY_FILENAME;
+
+import java.io.BufferedOutputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Queue;
+import java.util.TreeMap;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.ff.tm.format.HieroFormatReader;
+import org.apache.joshua.decoder.ff.tm.format.MosesFormatReader;
+import org.apache.joshua.util.FormatUtils;
+import org.apache.joshua.util.encoding.EncoderConfiguration;
+import org.apache.joshua.util.encoding.FeatureTypeAnalyzer;
+import org.apache.joshua.util.encoding.IntEncoder;
+import org.apache.joshua.util.io.LineReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+
+public class GrammarPacker {
+
+ private static final Logger LOG = LoggerFactory.getLogger(GrammarPacker.class);
+
+ /**
+ * The packed grammar version number. Increment this any time you add new features, and update
+ * the documentation.
+ *
+ * Version history:
+ *
+ * - 3 (May 2016). This was the first version that was marked. It removed the special phrase-
+ * table packing that packed phrases without the [X,1] on the source and target sides, which
+ * then required special handling in the decoder to use for phrase-based decoding.
+ *
+ *
+ */
+ public static final int VERSION = 3;
+
+ // Size limit for slice in bytes.
+ private static int DATA_SIZE_LIMIT = (int) (Integer.MAX_VALUE * 0.8);
+ // Estimated average number of feature entries for one rule.
+ private static int DATA_SIZE_ESTIMATE = 20;
+
+ private static final String SOURCE_WORDS_SEPARATOR = " ||| ";
+
+ // Output directory name.
+ private String output;
+
+ // Input grammar to be packed.
+ private String grammar;
+
+ public String getGrammar() {
+ return grammar;
+ }
+
+ public String getOutputDirectory() {
+ return output;
+ }
+
+ // Approximate maximum size of a slice in number of rules
+ private int approximateMaximumSliceSize;
+
+ private boolean labeled;
+
+ private boolean packAlignments;
+ private boolean grammarAlignments;
+ private String alignments;
+
+ private FeatureTypeAnalyzer types;
+ private EncoderConfiguration encoderConfig;
+
+ private String dump;
+
+ private int max_source_len;
+
+ public GrammarPacker(String grammar_filename, String config_filename, String output_filename,
+ String alignments_filename, String featuredump_filename, boolean grammar_alignments,
+ int approximateMaximumSliceSize)
+ throws IOException {
+ this.labeled = true;
+ this.grammar = grammar_filename;
+ this.output = output_filename;
+ this.dump = featuredump_filename;
+ this.grammarAlignments = grammar_alignments;
+ this.approximateMaximumSliceSize = approximateMaximumSliceSize;
+ this.max_source_len = 0;
+
+ // TODO: Always open encoder config? This is debatable.
+ this.types = new FeatureTypeAnalyzer(true);
+
+ this.alignments = alignments_filename;
+ packAlignments = grammarAlignments || (alignments != null);
+ if (!packAlignments) {
+ LOG.info("No alignments file or grammar specified, skipping.");
+ } else if (alignments != null && !new File(alignments_filename).exists()) {
+ throw new RuntimeException("Alignments file does not exist: " + alignments);
+ }
+
+ if (config_filename != null) {
+ readConfig(config_filename);
+ types.readConfig(config_filename);
+ } else {
+ LOG.info("No config specified. Attempting auto-detection of feature types.");
+ }
+ LOG.info("Approximate maximum slice size (in # of rules) set to {}", approximateMaximumSliceSize);
+
+ File working_dir = new File(output);
+ working_dir.mkdir();
+ if (!working_dir.exists()) {
+ throw new RuntimeException("Failed creating output directory.");
+ }
+ }
+
+ private void readConfig(String config_filename) throws IOException {
+ LineReader reader = new LineReader(config_filename);
+ while (reader.hasNext()) {
+ // Clean up line, chop comments off and skip if the result is empty.
+ String line = reader.next().trim();
+ if (line.indexOf('#') != -1)
+ line = line.substring(0, line.indexOf('#'));
+ if (line.isEmpty())
+ continue;
+ String[] fields = line.split("[\\s]+");
+
+ if (fields.length < 2) {
+ throw new RuntimeException("Incomplete line in config.");
+ }
+ if ("slice_size".equals(fields[0])) {
+ // Number of records to concurrently load into memory for sorting.
+ approximateMaximumSliceSize = Integer.parseInt(fields[1]);
+ }
+ }
+ reader.close();
+ }
+
+ /**
+ * Executes the packing.
+ *
+ * @throws IOException if there is an error reading the grammar
+ */
+ public void pack() throws IOException {
+ LOG.info("Beginning exploration pass.");
+
+ // Explore pass. Learn vocabulary and feature value histograms.
+ LOG.info("Exploring: {}", grammar);
+
+ HieroFormatReader grammarReader = getGrammarReader();
+ explore(grammarReader);
+
+ LOG.info("Exploration pass complete. Freezing vocabulary and finalizing encoders.");
+ if (dump != null) {
+ PrintWriter dump_writer = new PrintWriter(dump);
+ dump_writer.println(types.toString());
+ dump_writer.close();
+ }
+
+ types.inferTypes(this.labeled);
+ LOG.info("Type inference complete.");
+
+ LOG.info("Finalizing encoding.");
+
+ LOG.info("Writing encoding.");
+ types.write(output + File.separator + "encoding");
+
+ writeVocabulary();
+
+ String configFile = output + File.separator + "config";
+ LOG.info("Writing config to '{}'", configFile);
+ // Write config options
+ FileWriter config = new FileWriter(configFile);
+ config.write(String.format("version = %d\n", VERSION));
+ config.write(String.format("max-source-len = %d\n", max_source_len));
+ config.close();
+
+ // Read previously written encoder configuration to match up to changed
+ // vocabulary id's.
+ LOG.info("Reading encoding.");
+ encoderConfig = new EncoderConfiguration();
+ encoderConfig.load(output + File.separator + "encoding");
+
+ LOG.info("Beginning packing pass.");
+ // Actual binarization pass. Slice and pack source, target and data.
+ grammarReader = getGrammarReader();
+ LineReader alignment_reader = null;
+ if (packAlignments && !grammarAlignments)
+ alignment_reader = new LineReader(alignments);
+ binarize(grammarReader, alignment_reader);
+ LOG.info("Packing complete.");
+
+ LOG.info("Packed grammar in: {}", output);
+ LOG.info("Done.");
+ }
+
+ /**
+ * Returns a reader that turns whatever file format is found into Hiero grammar rules.
+ *
+ * @param grammarFile
+ * @return
+ * @throws IOException
+ */
+ private HieroFormatReader getGrammarReader() throws IOException {
+ LineReader reader = new LineReader(grammar);
+ String line = reader.next();
+ if (line.startsWith("[")) {
+ return new HieroFormatReader(grammar);
+ } else {
+ return new MosesFormatReader(grammar);
+ }
+ }
+
+ /**
+ * This first pass over the grammar
+ * @param reader
+ */
+ private void explore(HieroFormatReader reader) {
+
+ // We always assume a labeled grammar. Unlabeled features are assumed to be dense and to always
+ // appear in the same order. They are assigned numeric names in order of appearance.
+ this.types.setLabeled(true);
+
+ for (Rule rule: reader) {
+
+ max_source_len = Math.max(max_source_len, rule.getFrench().length);
+
+ /* Add symbols to vocabulary.
+ * NOTE: In case of nonterminals, we add both stripped versions ("[X]")
+ * and "[X,1]" to the vocabulary.
+ *
+ * TODO: MJP May 2016: Is it necessary to add [X,1]? This is currently being done in
+ * {@link HieroFormatReader}, which is called by {@link MosesFormatReader}.
+ */
+
+ // Add feature names to vocabulary and pass the value through the
+ // appropriate encoder.
+ int feature_counter = 0;
+ String[] features = rule.getFeatureString().split("\\s+");
+ for (int f = 0; f < features.length; ++f) {
+ if (features[f].contains("=")) {
+ String[] fe = features[f].split("=");
+ if (fe[0].equals("Alignment"))
+ continue;
+ types.observe(Vocabulary.id(fe[0]), Float.parseFloat(fe[1]));
+ } else {
+ types.observe(Vocabulary.id(String.valueOf(feature_counter++)),
+ Float.parseFloat(features[f]));
+ }
+ }
+ }
+ }
+
+ /**
+ * Returns a String encoding the first two source words.
+ * If there is only one source word, use empty string for the second.
+ */
+ private String getFirstTwoSourceWords(final String[] source_words) {
+ return source_words[0] + SOURCE_WORDS_SEPARATOR + ((source_words.length > 1) ? source_words[1] : "");
+ }
+
+ private void binarize(HieroFormatReader grammarReader, LineReader alignment_reader) throws IOException {
+ int counter = 0;
+ int slice_counter = 0;
+ int num_slices = 0;
+
+ boolean ready_to_flush = false;
+ // to determine when flushing is possible
+ String prev_first_two_source_words = null;
+
+ PackingTrie<SourceValue> source_trie = new PackingTrie<SourceValue>();
+ PackingTrie<TargetValue> target_trie = new PackingTrie<TargetValue>();
+ FeatureBuffer feature_buffer = new FeatureBuffer();
+
+ AlignmentBuffer alignment_buffer = null;
+ if (packAlignments)
+ alignment_buffer = new AlignmentBuffer();
+
+ TreeMap<Integer, Float> features = new TreeMap<Integer, Float>();
+ for (Rule rule: grammarReader) {
+ counter++;
+ slice_counter++;
+
+ String lhs_word = Vocabulary.word(rule.getLHS());
+ String[] source_words = rule.getFrenchWords().split("\\s+");
+ String[] target_words = rule.getEnglishWords().split("\\s+");
+ String[] feature_entries = rule.getFeatureString().split("\\s+");
+
+ // Reached slice limit size, indicate that we're closing up.
+ if (!ready_to_flush
+ && (slice_counter > approximateMaximumSliceSize
+ || feature_buffer.overflowing()
+ || (packAlignments && alignment_buffer.overflowing()))) {
+ ready_to_flush = true;
+ // store the first two source words when slice size limit was reached
+ prev_first_two_source_words = getFirstTwoSourceWords(source_words);
+ }
+ // ready to flush
+ if (ready_to_flush) {
+ final String first_two_source_words = getFirstTwoSourceWords(source_words);
+ // the grammar can only be partitioned at the level of first two source word changes.
+ // Thus, we can only flush if the current first two source words differ from the ones
+ // when the slice size limit was reached.
+ if (!first_two_source_words.equals(prev_first_two_source_words)) {
+ LOG.warn("ready to flush and first two words have changed ({} vs. {})",
+ prev_first_two_source_words, first_two_source_words);
+ LOG.info("flushing {} rules to slice.", slice_counter);
+ flush(source_trie, target_trie, feature_buffer, alignment_buffer, num_slices);
+ source_trie.clear();
+ target_trie.clear();
+ feature_buffer.clear();
+ if (packAlignments)
+ alignment_buffer.clear();
+
+ num_slices++;
+ slice_counter = 0;
+ ready_to_flush = false;
+ }
+ }
+
+ int alignment_index = -1;
+ // If present, process alignments.
+ if (packAlignments) {
+ String alignment_line;
+ if (grammarAlignments) {
+ alignment_line = rule.getAlignmentString();
+ } else {
+ if (!alignment_reader.hasNext()) {
+ LOG.error("No more alignments starting in line {}", counter);
+ throw new RuntimeException("No more alignments starting in line " + counter);
+ }
+ alignment_line = alignment_reader.next().trim();
+ }
+ String[] alignment_entries = alignment_line.split("\\s");
+ byte[] alignments = new byte[alignment_entries.length * 2];
+ if (alignment_line.length() > 0) {
+ for (int i = 0; i < alignment_entries.length; i++) {
+ String[] parts = alignment_entries[i].split("-");
+ alignments[2 * i] = Byte.parseByte(parts[0]);
+ alignments[2 * i + 1] = Byte.parseByte(parts[1]);
+ }
+ }
+ alignment_index = alignment_buffer.add(alignments);
+ }
+
+ // Process features.
+ // Implicitly sort via TreeMap, write to data buffer, remember position
+ // to pass on to the source trie node.
+ features.clear();
+ int feature_count = 0;
+ for (int f = 0; f < feature_entries.length; ++f) {
+ String feature_entry = feature_entries[f];
+ int feature_id;
+ float feature_value;
+ if (feature_entry.contains("=")) {
+ String[] parts = feature_entry.split("=");
+ if (parts[0].equals("Alignment"))
+ continue;
+ feature_id = Vocabulary.id(parts[0]);
+ feature_value = Float.parseFloat(parts[1]);
+ } else {
+ feature_id = Vocabulary.id(String.valueOf(feature_count++));
+ feature_value = Float.parseFloat(feature_entry);
+ }
+ if (feature_value != 0)
+ features.put(encoderConfig.innerId(feature_id), feature_value);
+ }
+ int features_index = feature_buffer.add(features);
+
+ // Sanity check on the data block index.
+ if (packAlignments && features_index != alignment_index) {
+ LOG.error("Block index mismatch between features ({}) and alignments ({}).",
+ features_index, alignment_index);
+ throw new RuntimeException("Data block index mismatch.");
+ }
+
+ // Process source side.
+ SourceValue sv = new SourceValue(Vocabulary.id(lhs_word), features_index);
+ int[] source = new int[source_words.length];
+ for (int i = 0; i < source_words.length; i++) {
+ if (FormatUtils.isNonterminal(source_words[i]))
+ source[i] = Vocabulary.id(FormatUtils.stripNonTerminalIndex(source_words[i]));
+ else
+ source[i] = Vocabulary.id(source_words[i]);
+ }
+ source_trie.add(source, sv);
+
+ // Process target side.
+ TargetValue tv = new TargetValue(sv);
+ int[] target = new int[target_words.length];
+ for (int i = 0; i < target_words.length; i++) {
+ if (FormatUtils.isNonterminal(target_words[i])) {
+ target[target_words.length - (i + 1)] = -FormatUtils.getNonterminalIndex(target_words[i]);
+ } else {
+ target[target_words.length - (i + 1)] = Vocabulary.id(target_words[i]);
+ }
+ }
+ target_trie.add(target, tv);
+ }
+ // flush last slice and clear buffers
+ flush(source_trie, target_trie, feature_buffer, alignment_buffer, num_slices);
+ }
+
+ /**
+ * Serializes the source, target and feature data structures into interlinked binary files. Target
+ * is written first, into a skeletal (node don't carry any data) upward-pointing trie, updating
+ * the linking source trie nodes with the position once it is known. Source and feature data are
+ * written simultaneously. The source structure is written into a downward-pointing trie and
+ * stores the rule's lhs as well as links to the target and feature stream. The feature stream is
+ * prompted to write out a block
+ *
+ * @param source_trie
+ * @param target_trie
+ * @param feature_buffer
+ * @param id
+ * @throws IOException
+ */
+ private void flush(PackingTrie<SourceValue> source_trie,
+ PackingTrie<TargetValue> target_trie, FeatureBuffer feature_buffer,
+ AlignmentBuffer alignment_buffer, int id) throws IOException {
+ // Make a slice object for this piece of the grammar.
+ PackingFileTuple slice = new PackingFileTuple("slice_" + String.format("%05d", id));
+ // Pull out the streams for source, target and data output.
+ DataOutputStream source_stream = slice.getSourceOutput();
+ DataOutputStream target_stream = slice.getTargetOutput();
+ DataOutputStream target_lookup_stream = slice.getTargetLookupOutput();
+ DataOutputStream feature_stream = slice.getFeatureOutput();
+ DataOutputStream alignment_stream = slice.getAlignmentOutput();
+
+ Queue<PackingTrie<TargetValue>> target_queue;
+ Queue<PackingTrie<SourceValue>> source_queue;
+
+ // The number of bytes both written into the source stream and
+ // buffered in the source queue.
+ int source_position;
+ // The number of bytes written into the target stream.
+ int target_position;
+
+ // Add trie root into queue, set target position to 0 and set cumulated
+ // size to size of trie root.
+ target_queue = new LinkedList<PackingTrie<TargetValue>>();
+ target_queue.add(target_trie);
+ target_position = 0;
+
+ // Target lookup table for trie levels.
+ int current_level_size = 1;
+ int next_level_size = 0;
+ ArrayList<Integer> target_lookup = new ArrayList<Integer>();
+
+ // Packing loop for upwards-pointing target trie.
+ while (!target_queue.isEmpty()) {
+ // Pop top of queue.
+ PackingTrie<TargetValue> node = target_queue.poll();
+ // Register that this is where we're writing the node to.
+ node.address = target_position;
+ // Tell source nodes that we're writing to this position in the file.
+ for (TargetValue tv : node.values)
+ tv.parent.target = node.address;
+ // Write link to parent.
+ if (node.parent != null)
+ target_stream.writeInt(node.parent.address);
+ else
+ target_stream.writeInt(-1);
+ target_stream.writeInt(node.symbol);
+ // Enqueue children.
+ for (int k : node.children.descendingKeySet()) {
+ PackingTrie<TargetValue> child = node.children.get(k);
+ target_queue.add(child);
+ }
+ target_position += node.size(false, true);
+ next_level_size += node.children.descendingKeySet().size();
+
+ current_level_size--;
+ if (current_level_size == 0) {
+ target_lookup.add(target_position);
+ current_level_size = next_level_size;
+ next_level_size = 0;
+ }
+ }
+ target_lookup_stream.writeInt(target_lookup.size());
+ for (int i : target_lookup)
+ target_lookup_stream.writeInt(i);
+ target_lookup_stream.close();
+
+ // Setting up for source and data writing.
+ source_queue = new LinkedList<PackingTrie<SourceValue>>();
+ source_queue.add(source_trie);
+ source_position = source_trie.size(true, false);
+ source_trie.address = target_position;
+
+ // Ready data buffers for writing.
+ feature_buffer.initialize();
+ if (packAlignments)
+ alignment_buffer.initialize();
+
+ // Packing loop for downwards-pointing source trie.
+ while (!source_queue.isEmpty()) {
+ // Pop top of queue.
+ PackingTrie<SourceValue> node = source_queue.poll();
+ // Write number of children.
+ source_stream.writeInt(node.children.size());
+ // Write links to children.
+ for (int k : node.children.descendingKeySet()) {
+ PackingTrie<SourceValue> child = node.children.get(k);
+ // Enqueue child.
+ source_queue.add(child);
+ // Child's address will be at the current end of the queue.
+ child.address = source_position;
+ // Advance cumulated size by child's size.
+ source_position += child.size(true, false);
+ // Write the link.
+ source_stream.writeInt(k);
+ source_stream.writeInt(child.address);
+ }
+ // Write number of data items.
+ source_stream.writeInt(node.values.size());
+ // Write lhs and links to target and data.
+ for (SourceValue sv : node.values) {
+ int feature_block_index = feature_buffer.write(sv.data);
+ if (packAlignments) {
+ int alignment_block_index = alignment_buffer.write(sv.data);
+ if (alignment_block_index != feature_block_index) {
+ LOG.error("Block index mismatch.");
+ throw new RuntimeException("Block index mismatch: alignment (" + alignment_block_index
+ + ") and features (" + feature_block_index + ") don't match.");
+ }
+ }
+ source_stream.writeInt(sv.lhs);
+ source_stream.writeInt(sv.target);
+ source_stream.writeInt(feature_block_index);
+ }
+ }
+ // Flush the data stream.
+ feature_buffer.flush(feature_stream);
+ if (packAlignments)
+ alignment_buffer.flush(alignment_stream);
+
+ target_stream.close();
+ source_stream.close();
+ feature_stream.close();
+ if (packAlignments)
+ alignment_stream.close();
+ }
+
+ public void writeVocabulary() throws IOException {
+ final String vocabularyFilename = output + File.separator + VOCABULARY_FILENAME;
+ LOG.info("Writing vocabulary to {}", vocabularyFilename);
+ Vocabulary.write(vocabularyFilename);
+ }
+
+ /**
+ * Integer-labeled, doubly-linked trie with some provisions for packing.
+ *
+ * @author Juri Ganitkevitch
+ *
+ * @param <D> The trie's value type.
+ */
+ class PackingTrie<D extends PackingTrieValue> {
+ int symbol;
+ PackingTrie<D> parent;
+
+ TreeMap<Integer, PackingTrie<D>> children;
+ List<D> values;
+
+ int address;
+
+ PackingTrie() {
+ address = -1;
+
+ symbol = 0;
+ parent = null;
+
+ children = new TreeMap<Integer, PackingTrie<D>>();
+ values = new ArrayList<D>();
+ }
+
+ PackingTrie(PackingTrie<D> parent, int symbol) {
+ this();
+ this.parent = parent;
+ this.symbol = symbol;
+ }
+
+ void add(int[] path, D value) {
+ add(path, 0, value);
+ }
+
+ private void add(int[] path, int index, D value) {
+ if (index == path.length)
+ this.values.add(value);
+ else {
+ PackingTrie<D> child = children.get(path[index]);
+ if (child == null) {
+ child = new PackingTrie<D>(this, path[index]);
+ children.put(path[index], child);
+ }
+ child.add(path, index + 1, value);
+ }
+ }
+
+ /**
+ * Calculate the size (in ints) of a packed trie node. Distinguishes downwards pointing (parent
+ * points to children) from upwards pointing (children point to parent) tries, as well as
+ * skeletal (no data, just the labeled links) and non-skeletal (nodes have a data block)
+ * packing.
+ *
+ * @param downwards Are we packing into a downwards-pointing trie?
+ * @param skeletal Are we packing into a skeletal trie?
+ *
+ * @return Number of bytes the trie node would occupy.
+ */
+ int size(boolean downwards, boolean skeletal) {
+ int size = 0;
+ if (downwards) {
+ // Number of children and links to children.
+ size = 1 + 2 * children.size();
+ } else {
+ // Link to parent.
+ size += 2;
+ }
+ // Non-skeletal packing: number of data items.
+ if (!skeletal)
+ size += 1;
+ // Non-skeletal packing: write size taken up by data items.
+ if (!skeletal && !values.isEmpty())
+ size += values.size() * values.get(0).size();
+
+ return size;
+ }
+
+ void clear() {
+ children.clear();
+ values.clear();
+ }
+ }
+
+ interface PackingTrieValue {
+ int size();
+ }
+
+ class SourceValue implements PackingTrieValue {
+ int lhs;
+ int data;
+ int target;
+
+ public SourceValue() {
+ }
+
+ SourceValue(int lhs, int data) {
+ this.lhs = lhs;
+ this.data = data;
+ }
+
+ void setTarget(int target) {
+ this.target = target;
+ }
+
+ public int size() {
+ return 3;
+ }
+ }
+
+ class TargetValue implements PackingTrieValue {
+ SourceValue parent;
+
+ TargetValue(SourceValue parent) {
+ this.parent = parent;
+ }
+
+ public int size() {
+ return 0;
+ }
+ }
+
+ abstract class PackingBuffer<T> {
+ private byte[] backing;
+ protected ByteBuffer buffer;
+
+ protected ArrayList<Integer> memoryLookup;
+ protected int totalSize;
+ protected ArrayList<Integer> onDiskOrder;
+
+ PackingBuffer() throws IOException {
+ allocate();
+ memoryLookup = new ArrayList<Integer>();
+ onDiskOrder = new ArrayList<Integer>();
+ totalSize = 0;
+ }
+
+ abstract int add(T item);
+
+ // Allocate a reasonably-sized buffer for the feature data.
+ private void allocate() {
+ backing = new byte[approximateMaximumSliceSize * DATA_SIZE_ESTIMATE];
+ buffer = ByteBuffer.wrap(backing);
+ }
+
+ // Reallocate the backing array and buffer, copies data over.
+ protected void reallocate() {
+ if (backing.length == Integer.MAX_VALUE)
+ return;
+ long attempted_length = backing.length * 2l;
+ int new_length;
+ // Detect overflow.
+ if (attempted_length >= Integer.MAX_VALUE)
+ new_length = Integer.MAX_VALUE;
+ else
+ new_length = (int) attempted_length;
+ byte[] new_backing = new byte[new_length];
+ System.arraycopy(backing, 0, new_backing, 0, backing.length);
+ int old_position = buffer.position();
+ ByteBuffer new_buffer = ByteBuffer.wrap(new_backing);
+ new_buffer.position(old_position);
+ buffer = new_buffer;
+ backing = new_backing;
+ }
+
+ /**
+ * Prepare the data buffer for disk writing.
+ */
+ void initialize() {
+ onDiskOrder.clear();
+ }
+
+ /**
+ * Enqueue a data block for later writing.
+ *
+ * @param block_index The index of the data block to add to writing queue.
+ * @return The to-be-written block's output index.
+ */
+ int write(int block_index) {
+ onDiskOrder.add(block_index);
+ return onDiskOrder.size() - 1;
+ }
+
+ /**
+ * Performs the actual writing to disk in the order specified by calls to write() since the last
+ * call to initialize().
+ *
+ * @param out
+ * @throws IOException
+ */
+ void flush(DataOutputStream out) throws IOException {
+ writeHeader(out);
+ int size;
+ int block_address;
+ for (int block_index : onDiskOrder) {
+ block_address = memoryLookup.get(block_index);
+ size = blockSize(block_index);
+ out.write(backing, block_address, size);
+ }
+ }
+
+ void clear() {
+ buffer.clear();
+ memoryLookup.clear();
+ onDiskOrder.clear();
+ }
+
+ boolean overflowing() {
+ return (buffer.position() >= DATA_SIZE_LIMIT);
+ }
+
+ private void writeHeader(DataOutputStream out) throws IOException {
+ if (out.size() == 0) {
+ out.writeInt(onDiskOrder.size());
+ out.writeInt(totalSize);
+ int disk_position = headerSize();
+ for (int block_index : onDiskOrder) {
+ out.writeInt(disk_position);
+ disk_position += blockSize(block_index);
+ }
+ } else {
+ throw new RuntimeException("Got a used stream for header writing.");
+ }
+ }
+
+ private int headerSize() {
+ // One integer for each data block, plus number of blocks and total size.
+ return 4 * (onDiskOrder.size() + 2);
+ }
+
+ private int blockSize(int block_index) {
+ int block_address = memoryLookup.get(block_index);
+ return (block_index < memoryLookup.size() - 1 ? memoryLookup.get(block_index + 1) : totalSize)
+ - block_address;
+ }
+ }
+
+ class FeatureBuffer extends PackingBuffer<TreeMap<Integer, Float>> {
+
+ private IntEncoder idEncoder;
+
+ FeatureBuffer() throws IOException {
+ super();
+ idEncoder = types.getIdEncoder();
+ LOG.info("Encoding feature ids in: {}", idEncoder.getKey());
+ }
+
+ /**
+ * Add a block of features to the buffer.
+ *
+ * @param features TreeMap with the features for one rule.
+ * @return The index of the resulting data block.
+ */
+ int add(TreeMap<Integer, Float> features) {
+ int data_position = buffer.position();
+
+ // Over-estimate how much room this addition will need: for each
+ // feature (ID_SIZE for label, "upper bound" of 4 for the value), plus ID_SIZE for
+ // the number of features. If this won't fit, reallocate the buffer.
+ int size_estimate = (4 + EncoderConfiguration.ID_SIZE) * features.size()
+ + EncoderConfiguration.ID_SIZE;
+ if (buffer.capacity() - buffer.position() <= size_estimate)
+ reallocate();
+
+ // Write features to buffer.
+ idEncoder.write(buffer, features.size());
+ for (Integer k : features.descendingKeySet()) {
+ float v = features.get(k);
+ // Sparse features.
+ if (v != 0.0) {
+ idEncoder.write(buffer, k);
+ encoderConfig.encoder(k).write(buffer, v);
+ }
+ }
+ // Store position the block was written to.
+ memoryLookup.add(data_position);
+ // Update total size (in bytes).
+ totalSize = buffer.position();
+
+ // Return block index.
+ return memoryLookup.size() - 1;
+ }
+ }
+
+ class AlignmentBuffer extends PackingBuffer<byte[]> {
+
+ AlignmentBuffer() throws IOException {
+ super();
+ }
+
+ /**
+ * Add a rule alignments to the buffer.
+ *
+ * @param alignments a byte array with the alignment points for one rule.
+ * @return The index of the resulting data block.
+ */
+ int add(byte[] alignments) {
+ int data_position = buffer.position();
+ int size_estimate = alignments.length + 1;
+ if (buffer.capacity() - buffer.position() <= size_estimate)
+ reallocate();
+
+ // Write alignment points to buffer.
+ buffer.put((byte) (alignments.length / 2));
+ buffer.put(alignments);
+
+ // Store position the block was written to.
+ memoryLookup.add(data_position);
+ // Update total size (in bytes).
+ totalSize = buffer.position();
+ // Return block index.
+ return memoryLookup.size() - 1;
+ }
+ }
+
+ class PackingFileTuple implements Comparable<PackingFileTuple> {
+ private File sourceFile;
+ private File targetLookupFile;
+ private File targetFile;
+
+ private File featureFile;
+ private File alignmentFile;
+
+ PackingFileTuple(String prefix) {
+ sourceFile = new File(output + File.separator + prefix + ".source");
+ targetFile = new File(output + File.separator + prefix + ".target");
+ targetLookupFile = new File(output + File.separator + prefix + ".target.lookup");
+ featureFile = new File(output + File.separator + prefix + ".features");
+
+ alignmentFile = null;
+ if (packAlignments)
+ alignmentFile = new File(output + File.separator + prefix + ".alignments");
+
+ LOG.info("Allocated slice: {}", sourceFile.getAbsolutePath());
+ }
+
+ DataOutputStream getSourceOutput() throws IOException {
+ return getOutput(sourceFile);
+ }
+
+ DataOutputStream getTargetOutput() throws IOException {
+ return getOutput(targetFile);
+ }
+
+ DataOutputStream getTargetLookupOutput() throws IOException {
+ return getOutput(targetLookupFile);
+ }
+
+ DataOutputStream getFeatureOutput() throws IOException {
+ return getOutput(featureFile);
+ }
+
+ DataOutputStream getAlignmentOutput() throws IOException {
+ if (alignmentFile != null)
+ return getOutput(alignmentFile);
+ return null;
+ }
+
+ private DataOutputStream getOutput(File file) throws IOException {
+ if (file.createNewFile()) {
+ return new DataOutputStream(new BufferedOutputStream(new FileOutputStream(file)));
+ } else {
+ throw new RuntimeException("File doesn't exist: " + file.getName());
+ }
+ }
+
+ long getSize() {
+ return sourceFile.length() + targetFile.length() + featureFile.length();
+ }
+
+ @Override
+ public int compareTo(PackingFileTuple o) {
+ if (getSize() > o.getSize()) {
+ return -1;
+ } else if (getSize() < o.getSize()) {
+ return 1;
+ } else {
+ return 0;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/tools/GrammarPackerCli.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/tools/GrammarPackerCli.java b/joshua-core/src/main/java/org/apache/joshua/tools/GrammarPackerCli.java
new file mode 100644
index 0000000..3cd4d0c
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/tools/GrammarPackerCli.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.tools;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.kohsuke.args4j.CmdLineException;
+import org.kohsuke.args4j.CmdLineParser;
+import org.kohsuke.args4j.Option;
+import org.kohsuke.args4j.spi.StringArrayOptionHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class GrammarPackerCli {
+
+ private static final Logger LOG = LoggerFactory.getLogger(GrammarPackerCli.class);
+
+ // Input grammars to be packed (with a joint vocabulary)
+ @Option(name = "--grammars", aliases = {"-g", "-i"}, handler = StringArrayOptionHandler.class, required = true, usage = "list of grammars to pack (jointly, i.e. they share the same vocabulary)")
+ private List<String> grammars = new ArrayList<>();
+
+ // Output grammars
+ @Option(name = "--outputs", aliases = {"-p", "-o"}, handler = StringArrayOptionHandler.class, required = true, usage = "output directories of packed grammars.")
+ private List<String> outputs = new ArrayList<>();
+
+ // Output grammars
+ @Option(name = "--alignments", aliases = {"-a", "--fa"}, handler = StringArrayOptionHandler.class, required = false, usage = "alignment files")
+ private List<String> alignments_filenames = new ArrayList<>();
+
+ // Config filename
+ @Option(name = "--config_file", aliases = {"-c"}, required = false, usage = "(optional) packing configuration file")
+ private String config_filename;
+
+ @Option(name = "--dump_files", aliases = {"-d"}, handler = StringArrayOptionHandler.class, usage = "(optional) dump feature stats to file")
+ private List<String> featuredump_filenames = new ArrayList<>();
+
+ @Option(name = "--ga", usage = "whether alignments are present in the grammar")
+ private boolean grammar_alignments = false;
+
+ @Option(name = "--slice_size", aliases = {"-s"}, required = false, usage = "approximate slice size in # of rules (default=1000000)")
+ private int slice_size = 1000000;
+
+
+ private void run() throws IOException {
+
+ final List<String> missingFilenames = new ArrayList<>(grammars.size());
+ for (final String g : grammars) {
+ if (!new File(g).exists()) {
+ missingFilenames.add(g);
+ }
+ }
+ if (!missingFilenames.isEmpty()) {
+ throw new IOException("Input grammar files not found: " + missingFilenames.toString());
+ }
+
+ if (config_filename != null && !new File(config_filename).exists()) {
+ throw new IOException("Config file not found: " + config_filename);
+ }
+
+ if (!outputs.isEmpty()) {
+ if (outputs.size() != grammars.size()) {
+ throw new IOException("Must provide an output directory for each grammar");
+ }
+ final List<String> existingOutputs = new ArrayList<>(outputs.size());
+ for (final String o : outputs) {
+ if (new File(o).exists()) {
+ existingOutputs.add(o);
+ }
+ }
+ if (!existingOutputs.isEmpty()) {
+ throw new IOException("These output directories already exist (will not overwrite): " + existingOutputs.toString());
+ }
+ }
+ if (outputs.isEmpty()) {
+ for (final String g : grammars) {
+ outputs.add(g + ".packed");
+ }
+ }
+
+ if (!alignments_filenames.isEmpty()) {
+ final List<String> missingAlignmentFiles = new ArrayList<>(alignments_filenames.size());
+ for (final String a : alignments_filenames) {
+ if (!new File(a).exists()) {
+ missingAlignmentFiles.add(a);
+ }
+ }
+ if (!missingAlignmentFiles.isEmpty()) {
+ throw new IOException("Alignment files not found: " + missingAlignmentFiles.toString());
+ }
+ }
+
+ // create Packer instances for each grammar
+ final List<GrammarPacker> packers = new ArrayList<>(grammars.size());
+ for (int i = 0; i < grammars.size(); i++) {
+ LOG.info("Starting GrammarPacker for {}", grammars.get(i));
+ final String alignment_filename = alignments_filenames.isEmpty() ? null : alignments_filenames.get(i);
+ final String featuredump_filename = featuredump_filenames.isEmpty() ? null : featuredump_filenames.get(i);
+ final GrammarPacker packer = new GrammarPacker(
+ grammars.get(i),
+ config_filename,
+ outputs.get(i),
+ alignment_filename,
+ featuredump_filename,
+ grammar_alignments,
+ slice_size);
+ packers.add(packer);
+ }
+
+ // run all packers in sequence, accumulating vocabulary items
+ for (final GrammarPacker packer : packers) {
+ LOG.info("Starting GrammarPacker for {}", packer.getGrammar());
+ packer.pack();
+ LOG.info("PackedGrammar located at {}", packer.getOutputDirectory());
+ }
+
+ // for each packed grammar, overwrite the internally serialized vocabulary with the current global one.
+ for (final GrammarPacker packer : packers) {
+ LOG.info("Writing final common Vocabulary to {}", packer.getOutputDirectory());
+ packer.writeVocabulary();
+ }
+ }
+
+ public static void main(String[] args) throws IOException {
+ final GrammarPackerCli cli = new GrammarPackerCli();
+ final CmdLineParser parser = new CmdLineParser(cli);
+
+ try {
+ parser.parseArgument(args);
+ cli.run();
+ } catch (CmdLineException e) {
+ LOG.error(e.getMessage(), e);
+ parser.printUsage(System.err);
+ System.exit(1);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/tools/LabelPhrases.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/tools/LabelPhrases.java b/joshua-core/src/main/java/org/apache/joshua/tools/LabelPhrases.java
new file mode 100644
index 0000000..2fd2b3f
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/tools/LabelPhrases.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.tools;
+
+import java.io.IOException;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.corpus.syntax.ArraySyntaxTree;
+import org.apache.joshua.util.io.LineReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Finds labeling for a set of phrases.
+ *
+ * @author Juri Ganitkevitch
+ */
+public class LabelPhrases {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LabelPhrases.class);
+
+ /**
+ * Main method.
+ *
+ * @param args names of the two grammars to be compared
+ * @throws IOException if there is an error reading the input grammars
+ */
+ public static void main(String[] args) throws IOException {
+
+ if (args.length < 1 || args[0].equals("-h")) {
+ System.err.println("Usage: " + LabelPhrases.class.toString());
+ System.err.println(" -p phrase_file phrase-sentence file to process");
+ System.err.println();
+ System.exit(-1);
+ }
+
+ String phrase_file_name = null;
+
+ for (int i = 0; i < args.length; i++) {
+ if ("-p".equals(args[i])) phrase_file_name = args[++i];
+ }
+ if (phrase_file_name == null) {
+ LOG.error("a phrase file is required for operation");
+ System.exit(-1);
+ }
+
+ LineReader phrase_reader = new LineReader(phrase_file_name);
+
+ while (phrase_reader.ready()) {
+ String line = phrase_reader.readLine();
+
+ String[] fields = line.split("\\t");
+ if (fields.length != 3 || fields[2].equals("()")) {
+ System.err.println("[FAIL] Empty parse in line:\t" + line);
+ continue;
+ }
+
+ String[] phrase_strings = fields[0].split("\\s");
+ int[] phrase_ids = new int[phrase_strings.length];
+ for (int i = 0; i < phrase_strings.length; i++)
+ phrase_ids[i] = Vocabulary.id(phrase_strings[i]);
+
+ ArraySyntaxTree syntax = new ArraySyntaxTree(fields[2]);
+ int[] sentence_ids = syntax.getTerminals();
+
+ int match_start = -1;
+ int match_end = -1;
+ for (int i = 0; i < sentence_ids.length; i++) {
+ if (phrase_ids[0] == sentence_ids[i]) {
+ match_start = i;
+ int j = 0;
+ while (j < phrase_ids.length && phrase_ids[j] == sentence_ids[i + j]) {
+ j++;
+ }
+ if (j == phrase_ids.length) {
+ match_end = i + j;
+ break;
+ }
+ }
+ }
+
+ int label = syntax.getOneConstituent(match_start, match_end);
+ if (label == 0) label = syntax.getOneSingleConcatenation(match_start, match_end);
+ if (label == 0) label = syntax.getOneRightSideCCG(match_start, match_end);
+ if (label == 0) label = syntax.getOneLeftSideCCG(match_start, match_end);
+ if (label == 0) label = syntax.getOneDoubleConcatenation(match_start, match_end);
+ if (label == 0) {
+ System.err.println("[FAIL] No label found in line:\t" + line);
+ continue;
+ }
+
+ System.out.println(Vocabulary.word(label) + "\t" + line);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/tools/TestSetFilter.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/tools/TestSetFilter.java b/joshua-core/src/main/java/org/apache/joshua/tools/TestSetFilter.java
new file mode 100644
index 0000000..ecb2e6e
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/tools/TestSetFilter.java
@@ -0,0 +1,383 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.tools;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+import org.apache.joshua.util.io.LineReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestSetFilter {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TestSetFilter.class);
+
+ private Filter filter = null;
+
+ // for caching of accepted rules
+ private String lastSourceSide;
+ private boolean acceptedLastSourceSide;
+
+ public int cached = 0;
+ public int RULE_LENGTH = 12;
+ public boolean verbose = false;
+ public boolean parallel = false;
+
+ private static final String DELIMITER = "|||";
+ private static final String DELIMITER_REGEX = " \\|\\|\\| ";
+ public static final String DELIM = String.format(" %s ", DELIMITER);
+ public static final Pattern P_DELIM = Pattern.compile(DELIMITER_REGEX);
+ private final String NT_REGEX = "\\[[^\\]]+?\\]";
+
+ public TestSetFilter() {
+ acceptedLastSourceSide = false;
+ lastSourceSide = null;
+ }
+
+ public String getFilterName() {
+ if (filter != null)
+ if (filter instanceof FastFilter)
+ return "fast";
+ else if (filter instanceof LooseFilter)
+ return "loose";
+ else
+ return "exact";
+ return "null";
+ }
+
+ public void setVerbose(boolean value) {
+ verbose = value;
+ }
+
+ public void setParallel(boolean value) {
+ parallel = value;
+ }
+
+ public void setFilter(String type) {
+ if (type.equals("fast"))
+ filter = new FastFilter();
+ else if (type.equals("exact"))
+ filter = new ExactFilter();
+ else if (type.equals("loose"))
+ filter = new LooseFilter();
+ else
+ throw new RuntimeException(String.format("Invalid filter type '%s'", type));
+ }
+
+ public void setRuleLength(int value) {
+ RULE_LENGTH = value;
+ }
+
+ private void loadTestSentences(String filename) throws IOException {
+ int count = 0;
+
+ try {
+ for (String line: new LineReader(filename)) {
+ filter.addSentence(line);
+ count++;
+ }
+ } catch (FileNotFoundException e) {
+ LOG.error(e.getMessage(), e);
+ }
+
+ if (verbose)
+ System.err.println(String.format("Added %d sentences.\n", count));
+ }
+
+ /**
+ * Top-level filter, responsible for calling the fast or exact version. Takes the source side
+ * of a rule and determines whether there is any sentence in the test set that can match it.
+ * @param sourceSide an input source sentence
+ * @return true if is any sentence in the test set can match the source input
+ */
+ public boolean inTestSet(String sourceSide) {
+ if (!sourceSide.equals(lastSourceSide)) {
+ lastSourceSide = sourceSide;
+ acceptedLastSourceSide = filter.permits(sourceSide);
+ } else {
+ cached++;
+ }
+
+ return acceptedLastSourceSide;
+ }
+
+ /**
+ * Determines whether a rule is an abstract rule. An abstract rule is one that has no terminals on
+ * its source side.
+ *
+ * If the rule is abstract, the rule's arity is returned. Otherwise, 0 is returned.
+ */
+ private boolean isAbstract(String source) {
+ int nonterminalCount = 0;
+ for (String t : source.split("\\s+")) {
+ if (!t.matches(NT_REGEX))
+ return false;
+ nonterminalCount++;
+ }
+ return nonterminalCount != 0;
+ }
+
+ private interface Filter {
+ /* Tell the filter about a sentence in the test set being filtered to */
+ public void addSentence(String sentence);
+
+ /* Returns true if the filter permits the specified source side */
+ public boolean permits(String sourceSide);
+ }
+
+ private class FastFilter implements Filter {
+ private Set<String> ngrams = null;
+
+ public FastFilter() {
+ ngrams = new HashSet<String>();
+ }
+
+ @Override
+ public boolean permits(String source) {
+ for (String chunk : source.split(NT_REGEX)) {
+ chunk = chunk.trim();
+ /* Important: you need to make sure the string isn't empty. */
+ if (!chunk.equals("") && !ngrams.contains(chunk))
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public void addSentence(String sentence) {
+ String[] tokens = sentence.trim().split("\\s+");
+ int maxOrder = RULE_LENGTH < tokens.length ? RULE_LENGTH : tokens.length;
+ for (int order = 1; order <= maxOrder; order++) {
+ for (int start = 0; start < tokens.length - order + 1; start++)
+ ngrams.add(createNGram(tokens, start, order));
+ }
+ }
+
+ private String createNGram(String[] tokens, int start, int order) {
+ if (order < 1 || start + order > tokens.length) {
+ return "";
+ }
+ String result = tokens[start];
+ for (int i = 1; i < order; i++)
+ result += " " + tokens[start + i];
+ return result;
+ }
+ }
+
+ private class LooseFilter implements Filter {
+ List<String> testSentences = null;
+
+ public LooseFilter() {
+ testSentences = new ArrayList<String>();
+ }
+
+ @Override
+ public void addSentence(String source) {
+ testSentences.add(source);
+ }
+
+ @Override
+ public boolean permits(String source) {
+ Pattern pattern = getPattern(source);
+ for (String testSentence : testSentences) {
+ if (pattern.matcher(testSentence).find()) {
+ return true;
+ }
+ }
+ return isAbstract(source);
+ }
+
+ protected Pattern getPattern(String source) {
+ String pattern = source;
+ pattern = pattern.replaceAll(String.format("\\s*%s\\s*", NT_REGEX), ".+");
+ pattern = pattern.replaceAll("\\s+", ".*");
+// System.err.println(String.format("PATTERN(%s) = %s", source, pattern));
+ return Pattern.compile(pattern);
+ }
+ }
+
+ /**
+ * This class is the same as LooseFilter except with a tighter regex for matching rules.
+ */
+ private class ExactFilter implements Filter {
+ private FastFilter fastFilter = null;
+ private Map<String, Set<Integer>> sentencesByWord;
+ List<String> testSentences = null;
+
+ public ExactFilter() {
+ fastFilter = new FastFilter();
+ sentencesByWord = new HashMap<String, Set<Integer>>();
+ testSentences = new ArrayList<String>();
+ }
+
+ @Override
+ public void addSentence(String source) {
+ fastFilter.addSentence(source);
+ addSentenceToWordHash(source, testSentences.size());
+ testSentences.add(source);
+ }
+
+ /**
+ * Always permit abstract rules. Otherwise, query the fast filter, and if that passes, apply
+ *
+ */
+ @Override
+ public boolean permits(String sourceSide) {
+ if (isAbstract(sourceSide))
+ return true;
+
+ if (fastFilter.permits(sourceSide)) {
+ Pattern pattern = getPattern(sourceSide);
+ for (int i : getSentencesForRule(sourceSide)) {
+ if (pattern.matcher(testSentences.get(i)).find()) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ protected Pattern getPattern(String source) {
+ String pattern = Pattern.quote(source);
+ pattern = pattern.replaceAll(NT_REGEX, "\\\\E.+\\\\Q");
+ pattern = pattern.replaceAll("\\\\Q\\\\E", "");
+ pattern = "(?:^|\\s)" + pattern + "(?:$|\\s)";
+ return Pattern.compile(pattern);
+ }
+
+ /*
+ * Map words to all the sentences they appear in.
+ */
+ private void addSentenceToWordHash(String sentence, int index) {
+ String[] tokens = sentence.split("\\s+");
+ for (String t : tokens) {
+ if (! sentencesByWord.containsKey(t))
+ sentencesByWord.put(t, new HashSet<Integer>());
+ sentencesByWord.get(t).add(index);
+ }
+ }
+
+ private Set<Integer> getSentencesForRule(String source) {
+ Set<Integer> sentences = null;
+ for (String token : source.split("\\s+")) {
+ if (!token.matches(NT_REGEX)) {
+ if (sentencesByWord.containsKey(token)) {
+ if (sentences == null)
+ sentences = new HashSet<Integer>(sentencesByWord.get(token));
+ else
+ sentences.retainAll(sentencesByWord.get(token));
+ }
+ }
+ }
+
+ return sentences;
+ }
+ }
+
+ public static void main(String[] argv) throws IOException {
+ // do some setup
+ if (argv.length < 1) {
+ System.err.println("usage: TestSetFilter [-v|-p|-f|-e|-l|-n N|-g grammar] test_set1 [test_set2 ...]");
+ System.err.println(" -g grammar file (can also be on STDIN)");
+ System.err.println(" -v verbose output");
+ System.err.println(" -p parallel compatibility");
+ System.err.println(" -f fast mode (default)");
+ System.err.println(" -e exact mode (slower)");
+ System.err.println(" -l loose mode");
+ System.err.println(" -n max n-gram to compare to (default 12)");
+ return;
+ }
+
+ String grammarFile = null;
+
+ TestSetFilter filter = new TestSetFilter();
+
+ for (int i = 0; i < argv.length; i++) {
+ if (argv[i].equals("-v")) {
+ filter.setVerbose(true);
+ continue;
+ } else if (argv[i].equals("-p")) {
+ filter.setParallel(true);
+ continue;
+ } else if (argv[i].equals("-g")) {
+ grammarFile = argv[++i];
+ continue;
+ } else if (argv[i].equals("-f")) {
+ filter.setFilter("fast");
+ continue;
+ } else if (argv[i].equals("-e")) {
+ filter.setFilter("exact");
+ continue;
+ } else if (argv[i].equals("-l")) {
+ filter.setFilter("loose");
+ continue;
+ } else if (argv[i].equals("-n")) {
+ filter.setRuleLength(Integer.parseInt(argv[i + 1]));
+ i++;
+ continue;
+ }
+
+ filter.loadTestSentences(argv[i]);
+ }
+
+ int rulesIn = 0;
+ int rulesOut = 0;
+ if (filter.verbose) {
+ System.err.println(String.format("Filtering rules with the %s filter...", filter.getFilterName()));
+// System.err.println("Using at max " + filter.RULE_LENGTH + " n-grams...");
+ }
+ LineReader reader = (grammarFile != null)
+ ? new LineReader(grammarFile, filter.verbose)
+ : new LineReader(System.in);
+ for (String rule: reader) {
+ rulesIn++;
+
+ String[] parts = P_DELIM.split(rule);
+ if (parts.length >= 4) {
+ // the source is the second field for thrax grammars, first field for phrasal ones
+ String source = rule.startsWith("[") ? parts[1].trim() : parts[0].trim();
+ if (filter.inTestSet(source)) {
+ System.out.println(rule);
+ if (filter.parallel)
+ System.out.flush();
+ rulesOut++;
+ } else if (filter.parallel) {
+ System.out.println("");
+ System.out.flush();
+ }
+ }
+ }
+ if (filter.verbose) {
+ System.err.println("[INFO] Total rules read: " + rulesIn);
+ System.err.println("[INFO] Rules kept: " + rulesOut);
+ System.err.println("[INFO] Rules dropped: " + (rulesIn - rulesOut));
+ System.err.println("[INFO] cached queries: " + filter.cached);
+ }
+
+ return;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/ui/Orientation.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/ui/Orientation.java b/joshua-core/src/main/java/org/apache/joshua/ui/Orientation.java
new file mode 100644
index 0000000..4c536ce
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/ui/Orientation.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.ui;
+
+public enum Orientation {
+ HORIZONTAL, VERTICAL
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/ui/StartupWindow.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/ui/StartupWindow.java b/joshua-core/src/main/java/org/apache/joshua/ui/StartupWindow.java
new file mode 100644
index 0000000..cccdd80
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/ui/StartupWindow.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.ui;
+
+import java.awt.BorderLayout;
+import java.awt.Color;
+import java.awt.Font;
+import java.awt.GraphicsEnvironment;
+import java.awt.Image;
+import java.awt.Point;
+
+import javax.swing.BorderFactory;
+import javax.swing.ImageIcon;
+import javax.swing.JLabel;
+import javax.swing.JPanel;
+import javax.swing.JWindow;
+
+/**
+ * Startup window for Joshua programs.
+ *
+ * @author Lane Schwartz
+ * @author Aaron Phillips
+ */
+public class StartupWindow extends JWindow {
+
+ /** Serialization identifier. */
+ private static final long serialVersionUID = 1L;
+
+ /**
+ * Constructs a splash screen.
+ *
+ * @param title Title to be displayed
+ */
+ public StartupWindow(String title) {
+ this(title, "Joshua Developers", "2010", Color.BLACK, 5);
+ }
+
+ public StartupWindow(String title, String author, String year, Image image, Color borderColor,
+ int borderWidth) {
+ JPanel content = (JPanel) getContentPane();
+ content.setBackground(Color.WHITE);
+
+ int width = 250;
+ int height = 100;
+
+ Point center = GraphicsEnvironment.getLocalGraphicsEnvironment().getCenterPoint();
+ setBounds(center.x - width / 2, center.y - height / 2, width, height);
+
+ JLabel titleLabel = new JLabel(title, JLabel.CENTER);
+ titleLabel.setFont(new Font("Sans-Serif", Font.BOLD, 24));
+ content.add(titleLabel, BorderLayout.NORTH);
+
+ JLabel copyright = new JLabel("\u24D2 " + year + " - " + author, JLabel.CENTER);
+ copyright.setFont(new Font("Sans-Serif", Font.PLAIN, 8));
+ content.add(copyright, BorderLayout.SOUTH);
+
+ if (image != null) {
+ content.add(new JLabel(new ImageIcon(image)));
+ }
+
+ content.setBorder(BorderFactory.createLineBorder(borderColor, borderWidth));
+
+ // Display it
+ setVisible(true);
+ }
+
+ public StartupWindow(String title, String author, String year, Color borderColor, int borderWidth) {
+ this(title, author, year, null, borderColor, borderWidth);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/ui/package-info.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/ui/package-info.java b/joshua-core/src/main/java/org/apache/joshua/ui/package-info.java
new file mode 100644
index 0000000..1d69516
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/ui/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * Provides classes for visualizing parts of the translation process.
+ */
+package org.apache.joshua.ui;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/DerivationTree.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/DerivationTree.java b/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/DerivationTree.java
new file mode 100644
index 0000000..f09a40a
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/DerivationTree.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.ui.tree_visualizer;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Collections;
+
+import org.apache.joshua.ui.tree_visualizer.tree.Tree;
+
+import edu.uci.ics.jung.graph.DirectedOrderedSparseMultigraph;
+import edu.uci.ics.jung.graph.util.EdgeType;
+import edu.uci.ics.jung.graph.util.Pair;
+
+public class DerivationTree extends DirectedOrderedSparseMultigraph<Node, DerivationTreeEdge> {
+ /**
+ * Eclipse thinks this is necessary.
+ */
+ private static final long serialVersionUID = 2914449263979566324L;
+
+ public final Node root;
+ public final Node sourceRoot;
+
+ public DerivationTree(Tree t, String source) {
+ final Tree.Node treeRoot = t.root();
+ final String rootLabel = treeRoot.label();
+ root = new Node(rootLabel, false);
+ sourceRoot = new Node(rootLabel, true);
+ addVertex(root);
+ addVertex(sourceRoot);
+ addSubtreeRootedAt(root, treeRoot);
+ final String[] sourceWords = source.split("\\s+");
+ addSourceSubtreeRootedAt(sourceRoot, treeRoot, 0, sourceWords.length, sourceWords);
+ }
+
+ private void addSubtreeRootedAt(Node n, Tree.Node tn) {
+ for (Tree.Node child : tn.children()) {
+ Node childNode = new Node(child.label(), false);
+ addVertex(childNode);
+ addEdge(new DerivationTreeEdge(false), new Pair<Node>(n, childNode), EdgeType.DIRECTED);
+ addSubtreeRootedAt(childNode, child);
+ }
+ }
+
+ private void addSourceSubtreeRootedAt(Node n, Tree.Node tn, int firstIndex, int lastIndex,
+ String[] sourceWords) {
+ int nextUncoveredIndex = firstIndex;
+ Tree.NodeSourceStartComparator cmp = new Tree.NodeSourceStartComparator();
+ List<Tree.Node> children = tn.children();
+ Collections.sort(children, cmp);
+ for (Tree.Node child : children) {
+ if (child.isLeaf()) {
+ continue;
+ }
+ int sourceStartIndex = child.sourceStartIndex();
+ int sourceEndIndex = child.sourceEndIndex();
+ if (sourceStartIndex > nextUncoveredIndex) {
+ insertSourceLeaf(n, sourceWords, nextUncoveredIndex, sourceStartIndex);
+ }
+ Node childNode = new Node(child.label(), true);
+ addEdge(new DerivationTreeEdge(true), new Pair<Node>(n, childNode), EdgeType.DIRECTED);
+ nextUncoveredIndex = sourceEndIndex;
+ addSourceSubtreeRootedAt(childNode, child, sourceStartIndex, sourceEndIndex, sourceWords);
+ }
+ if (nextUncoveredIndex < lastIndex) {
+ insertSourceLeaf(n, sourceWords, nextUncoveredIndex, lastIndex);
+ }
+ }
+
+ private void insertSourceLeaf(Node n, String[] words, int start, int end) {
+ final String[] leafWords = Arrays.copyOfRange(words, start, end);
+ String label = leafWords[0];
+ for (int i = 1; i < leafWords.length; i++) {
+ label += " " + leafWords[i];
+ }
+ Node childNode = new Node(label, true);
+ addEdge(new DerivationTreeEdge(true), new Pair<Node>(n, childNode), EdgeType.DIRECTED);
+ }
+
+ public void setSubtreeHighlight(Node n, boolean b) {
+ n.isHighlighted = b;
+ for (Node s : getSuccessors(n)) {
+ setSubtreeHighlight(s, b);
+ }
+ return;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/DerivationTreeEdge.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/DerivationTreeEdge.java b/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/DerivationTreeEdge.java
new file mode 100644
index 0000000..33b6b22
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/DerivationTreeEdge.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.ui.tree_visualizer;
+
+public class DerivationTreeEdge {
+ public final boolean pointsToSource;
+
+ public DerivationTreeEdge(boolean pts) {
+ pointsToSource = pts;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/DerivationTreeTransformer.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/DerivationTreeTransformer.java b/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/DerivationTreeTransformer.java
new file mode 100644
index 0000000..3e4010f
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/DerivationTreeTransformer.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.ui.tree_visualizer;
+
+import java.awt.Dimension;
+import java.awt.geom.Point2D;
+
+import org.apache.commons.collections15.Transformer;
+
+import edu.uci.ics.jung.algorithms.layout.TreeLayout;
+import edu.uci.ics.jung.graph.DelegateForest;
+
+public class DerivationTreeTransformer implements Transformer<Node, Point2D> {
+ private TreeLayout<Node, DerivationTreeEdge> treeLayout;
+ private DerivationTree graph;
+ private Node root;
+ private Node sourceRoot;
+
+ private boolean isAnchored;
+ private Point2D anchorPoint;
+
+ private double Y_DIST;
+ private double X_DIST;
+
+
+ public DerivationTreeTransformer(DerivationTree t, Dimension d, boolean isAnchored) {
+ this.isAnchored = isAnchored;
+ anchorPoint = new Point2D.Double(0, 0);
+ graph = t;
+ DelegateForest<Node, DerivationTreeEdge> del = new DelegateForest<Node, DerivationTreeEdge>(t);
+ del.setRoot(t.root);
+ del.setRoot(t.sourceRoot);
+ root = t.root;
+ sourceRoot = t.sourceRoot;
+ Y_DIST = d.getHeight() / (2 * (1 + distanceToLeaf(root)));
+ int leafCount = 0;
+ for (Node n : t.getVertices()) {
+ if (t.outDegree(n) == 0) leafCount++;
+ }
+ X_DIST = d.getWidth() / leafCount;
+
+ treeLayout = new TreeLayout<Node, DerivationTreeEdge>(del, (int) Math.round(X_DIST));
+ }
+
+ public Point2D transform(Node n) {
+ double x, y;
+ Point2D t = treeLayout.transform(n);
+ if (n.isSource) {
+ x =
+ /* treeLayout.transform(root).getX() + */(t.getX()
+ - treeLayout.transform(sourceRoot).getX() + treeLayout.transform(root).getX());
+ y = Y_DIST * (distanceToLeaf(n) + 1);
+ } else {
+ x = t.getX();
+ y = Y_DIST * (-1) * distanceToLeaf(n);
+ }
+ if (isAnchored) {
+ x += anchorPoint.getX();
+ y += anchorPoint.getY();
+ }
+ return new Point2D.Double(x, y + Y_DIST * (1 + distanceToLeaf(root)));
+ }
+
+ private int distanceToLeaf(Node n) {
+ if (graph.getSuccessors(n).isEmpty()) return 0;
+ int result = 0;
+ for (Object x : graph.getSuccessors(n)) {
+ int tmp = distanceToLeaf((Node) x);
+ if (tmp > result) result = tmp;
+ }
+ return 1 + result;
+ }
+
+ public Dimension getSize() {
+ int height = (int) Math.round(2 * Y_DIST * (1 + distanceToLeaf(root)));
+ int width = (int) Math.round(2 * treeLayout.transform(root).getX());
+ Dimension ret = new Dimension(width, height);
+ return ret;
+ }
+
+ public Point2D getAnchorPosition(DerivationViewer.AnchorType type) {
+ switch (type) {
+ case ANCHOR_ROOT:
+ return transform(root);
+ case ANCHOR_LEFTMOST_LEAF:
+ Node n = root;
+ while (graph.getSuccessorCount(n) != 0)
+ n = (Node) graph.getSuccessors(n).toArray()[0];
+ return transform(n);
+ default:
+ return new Point2D.Double(0, 0);
+ }
+ }
+
+ public void setAnchorPoint(DerivationViewer.AnchorType type, Point2D viewerAnchor) {
+ Point2D oldAnchor = getAnchorPosition(type);
+ double x = viewerAnchor.getX() - oldAnchor.getX();
+ double y = viewerAnchor.getY() - oldAnchor.getY();
+ anchorPoint = new Point2D.Double(x, y);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/DerivationViewer.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/DerivationViewer.java b/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/DerivationViewer.java
new file mode 100644
index 0000000..8c6151d
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/DerivationViewer.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.ui.tree_visualizer;
+
+import java.awt.BasicStroke;
+import java.awt.Color;
+import java.awt.Dimension;
+import java.awt.Paint;
+import java.awt.Shape;
+import java.awt.Stroke;
+import java.awt.geom.Point2D;
+import java.awt.geom.Rectangle2D;
+
+import javax.swing.JLabel;
+
+import org.apache.commons.collections15.Transformer;
+
+import edu.uci.ics.jung.algorithms.layout.CircleLayout;
+import edu.uci.ics.jung.algorithms.layout.StaticLayout;
+import edu.uci.ics.jung.visualization.VisualizationViewer;
+import edu.uci.ics.jung.visualization.control.DefaultModalGraphMouse;
+import edu.uci.ics.jung.visualization.control.LayoutScalingControl;
+import edu.uci.ics.jung.visualization.control.ModalGraphMouse;
+import edu.uci.ics.jung.visualization.decorators.ToStringLabeller;
+import edu.uci.ics.jung.visualization.renderers.Renderer.VertexLabel.Position;
+
+@SuppressWarnings("serial")
+public class DerivationViewer extends VisualizationViewer<Node, DerivationTreeEdge> {
+ public static final int DEFAULT_HEIGHT = 500;
+ public static final int DEFAULT_WIDTH = 500;
+ public static final Color SRC = Color.WHITE;
+ private Color TGT;
+
+ public static final Color HIGHLIGHT = Color.pink;
+
+ public static enum AnchorType {
+ ANCHOR_ROOT, ANCHOR_LEFTMOST_LEAF
+ };
+
+ private AnchorType anchorStyle;
+ private Point2D anchorPoint;
+
+ public DerivationViewer(DerivationTree g, Dimension d, Color targetColor, AnchorType anchor) {
+ super(new CircleLayout<Node, DerivationTreeEdge>(g));
+ anchorStyle = anchor;
+ DerivationTreeTransformer dtt = new DerivationTreeTransformer(g, d, false);
+ StaticLayout<Node, DerivationTreeEdge> derivationLayout =
+ new StaticLayout<Node, DerivationTreeEdge>(g, dtt);
+ // derivationLayout.setSize(dtt.getSize());
+ setGraphLayout(derivationLayout);
+ scaleToLayout(new LayoutScalingControl());
+ // g.addCorrespondences();
+ setPreferredSize(new Dimension(DEFAULT_HEIGHT, DEFAULT_WIDTH));
+ getRenderContext().setVertexLabelTransformer(new ToStringLabeller<Node>());
+
+ DefaultModalGraphMouse<Node, DerivationTreeEdge> graphMouse =
+ new DefaultModalGraphMouse<Node, DerivationTreeEdge>();
+ graphMouse.setMode(ModalGraphMouse.Mode.TRANSFORMING);
+ setGraphMouse(graphMouse);
+ addKeyListener(graphMouse.getModeKeyListener());
+ // this.setPickedVertexState(new DerivationTreePickedState(g));
+
+ getRenderContext().setVertexFillPaintTransformer(vp);
+ getRenderContext().setEdgeStrokeTransformer(es);
+ getRenderContext().setVertexShapeTransformer(ns);
+ getRenderer().getVertexLabelRenderer().setPosition(Position.CNTR);
+
+ TGT = targetColor;
+ anchorPoint = dtt.getAnchorPosition(anchorStyle);
+ }
+
+ public void setGraph(DerivationTree tree) {
+ DerivationTreeTransformer dtt = new DerivationTreeTransformer(tree, getSize(), true);
+ dtt.setAnchorPoint(anchorStyle, anchorPoint);
+ setGraphLayout(new StaticLayout<Node, DerivationTreeEdge>(tree, dtt));
+ }
+
+ private Transformer<Node, Paint> vp = new Transformer<Node, Paint>() {
+ public Paint transform(Node n) {
+ if (n.isHighlighted) return HIGHLIGHT;
+ if (n.isSource)
+ return SRC;
+ else
+ return TGT;
+ }
+ };
+
+ private static Transformer<DerivationTreeEdge, Stroke> es =
+ new Transformer<DerivationTreeEdge, Stroke>() {
+ public Stroke transform(DerivationTreeEdge e) {
+ if (e.pointsToSource) {
+ return new BasicStroke(1.0f,
+ BasicStroke.CAP_BUTT,
+ BasicStroke.JOIN_MITER,
+ 10.0f,
+ new float[] {10.0f},
+ 0.0f);
+ } else {
+ return new BasicStroke(1.0f);
+ }
+ }
+ };
+
+ private static Transformer<Node, Shape> ns = new Transformer<Node, Shape>() {
+ public Shape transform(Node n) {
+ JLabel x = new JLabel();
+ double len = x.getFontMetrics(x.getFont()).stringWidth(n.toString());
+ double margin = 5.0;
+ return new Rectangle2D.Double((len + margin) / (-2), 0, len + 2 * margin, 20);
+ }
+ };
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/DerivationViewerApplet.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/DerivationViewerApplet.java b/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/DerivationViewerApplet.java
new file mode 100644
index 0000000..d6e7a35
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/DerivationViewerApplet.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.ui.tree_visualizer;
+
+import java.awt.Color;
+
+import javax.swing.JApplet;
+
+import org.apache.joshua.ui.tree_visualizer.tree.Tree;
+
+/**
+ * An applet for viewing DerivationTrees. It consists of a DerivationViewer inside of the applet's
+ * Panel.
+ *
+ * @author Jonathan Weese
+ *
+ */
+@SuppressWarnings("serial")
+public class DerivationViewerApplet extends JApplet {
+ /**
+ * Initializes the applet by getting the source sentence and the tree representation from the
+ * applet tag in a web page.
+ */
+ public void init() {
+ String source = getParameter("sourceSentence");
+ String derivation = getParameter("derivationTree");
+ Tree tree = new Tree(derivation);
+
+ add(new DerivationViewer(new DerivationTree(tree, source),
+ getSize(),
+ Color.red,
+ DerivationViewer.AnchorType.ANCHOR_ROOT));
+ return;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/Node.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/Node.java b/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/Node.java
new file mode 100644
index 0000000..2ffeb06
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/Node.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.ui.tree_visualizer;
+
+/**
+ * A representation of a node in a derivation tree. The derivation tree class itself is
+ * parameterized in terms of this class and the <code>DerivationEdge</code> class. A
+ * <code>Node</code> may represent either a non-terminal symbol or one or more terminal symbols of
+ * the derivation.
+ */
+public class Node {
+ /**
+ * The label to be shown on the node. If the node is a non-terminal symbol, it is the name of the
+ * symbol. Otherwise, it is terminal symbols joined with spaces.
+ */
+ public final String label;
+
+ /**
+ * Indicates whether this node is part of the source-side of target- side derivation tree.
+ */
+ public final boolean isSource;
+
+ /**
+ * A boolean to let the renderer know whether this vertex is highlighted.
+ */
+ public boolean isHighlighted = false;
+
+ /**
+ * Constructor used for root nodes or nodes whose parent is not given.
+ *
+ * @param label a <code>String</code> that represents the symbols at this node
+ * @param isSource a boolean saying whether this is a source-side node
+ */
+ public Node(String label, boolean isSource) {
+ this.label = label;
+ this.isSource = isSource;
+ }
+
+ @Override
+ public String toString() {
+ return label;
+ }
+}
[26/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/server/ServerThread.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/server/ServerThread.java b/joshua-core/src/main/java/org/apache/joshua/server/ServerThread.java
new file mode 100644
index 0000000..d054515
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/server/ServerThread.java
@@ -0,0 +1,294 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.server;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.StringReader;
+import java.net.Socket;
+import java.net.SocketException;
+import java.net.URLDecoder;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+
+import com.sun.net.httpserver.HttpExchange;
+import com.sun.net.httpserver.HttpHandler;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.Decoder;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.Translation;
+import org.apache.joshua.decoder.Translations;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.ff.tm.Trie;
+import org.apache.joshua.decoder.ff.tm.format.HieroFormatReader;
+import org.apache.joshua.decoder.io.JSONMessage;
+import org.apache.joshua.decoder.io.TranslationRequestStream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class handles a concurrent request for translations from a newly opened socket, for
+ * both raw TCP/IP connections and for HTTP connections.
+ *
+ */
+public class ServerThread extends Thread implements HttpHandler {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ServerThread.class);
+ private static final Charset FILE_ENCODING = Charset.forName("UTF-8");
+
+ private final JoshuaConfiguration joshuaConfiguration;
+ private Socket socket = null;
+ private final Decoder decoder;
+
+ /**
+ * Creates a new TcpServerThread that can run a set of translations.
+ *
+ * @param socket the socket representing the input/output streams
+ * @param decoder the configured decoder that handles performing translations
+ * @param joshuaConfiguration a populated {@link org.apache.joshua.decoder.JoshuaConfiguration}
+ */
+ public ServerThread(Socket socket, Decoder decoder, JoshuaConfiguration joshuaConfiguration) {
+ this.joshuaConfiguration = joshuaConfiguration;
+ this.socket = socket;
+ this.decoder = decoder;
+ }
+
+ /**
+ * Reads the input from the socket, submits the input to the decoder, transforms the resulting
+ * translations into the required output format, writes out the formatted output, then closes the
+ * socket.
+ */
+ @Override
+ public void run() {
+
+ //TODO: use try-with-resources block
+ try {
+ BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream(), FILE_ENCODING));
+
+ TranslationRequestStream request = new TranslationRequestStream(reader, joshuaConfiguration);
+
+ try {
+ Translations translations = decoder.decodeAll(request);
+
+ OutputStream out = socket.getOutputStream();
+
+ for (Translation translation: translations) {
+ out.write(translation.toString().getBytes());
+ }
+
+ } catch (SocketException e) {
+ LOG.error(" Socket interrupted", e);
+ request.shutdown();
+ } finally {
+ reader.close();
+ socket.close();
+ }
+ } catch (IOException e) {
+ LOG.error(e.getMessage(), e);
+ }
+ }
+
+ public HashMap<String, String> queryToMap(String query){
+ HashMap<String, String> result = new HashMap<String, String>();
+ for (String param : query.split("&")) {
+ String pair[] = param.split("=");
+ if (pair.length > 1) {
+ result.put(pair[0], pair[1]);
+ } else {
+ result.put(pair[0], "");
+ }
+ }
+ return result;
+ }
+
+ private class HttpWriter extends OutputStream {
+
+ private HttpExchange client = null;
+ private OutputStream out = null;
+
+ public HttpWriter(HttpExchange client) {
+ this.client = client;
+ client.getResponseHeaders().add("Access-Control-Allow-Origin", "*");
+ }
+
+ @Override
+ public void write(byte[] response) throws IOException {
+ client.sendResponseHeaders(200, response.length);
+ out = client.getResponseBody();
+ out.write(response);
+ out.close();
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ out.write(b);
+ }
+ }
+
+ /**
+ * Called to handle an HTTP connection. This looks for metadata in the URL string, which is processed
+ * if present. It also then handles returning a JSON-formatted object to the caller.
+ *
+ * @param client the client connection
+ */
+ @Override
+ public synchronized void handle(HttpExchange client) throws IOException {
+
+ HashMap<String, String> params = queryToMap(URLDecoder.decode(client.getRequestURI().getQuery(), "UTF-8"));
+ String query = params.get("q");
+ String meta = params.get("meta");
+
+ BufferedReader reader = new BufferedReader(new StringReader(query));
+ TranslationRequestStream request = new TranslationRequestStream(reader, joshuaConfiguration);
+
+ Translations translations = decoder.decodeAll(request);
+ JSONMessage message = new JSONMessage();
+ if (meta != null && ! meta.isEmpty())
+ handleMetadata(meta, message);
+
+ for (Translation translation: translations) {
+ LOG.info("TRANSLATION: '{}' with {} k-best items", translation, translation.getStructuredTranslations().size());
+ message.addTranslation(translation);
+ }
+
+ OutputStream out = new HttpWriter(client);
+ out.write(message.toString().getBytes());
+ if (LOG.isDebugEnabled())
+ LOG.debug(message.toString());
+ out.close();
+
+ reader.close();
+ }
+
+ /**
+ * Processes metadata commands received in the HTTP request. Some commands result in sending data back.
+ *
+ * @param meta the metadata request
+ * @return result string (for some commands)
+ */
+ private void handleMetadata(String meta, JSONMessage message) {
+ String[] tokens = meta.split("\\s+", 2);
+ String type = tokens[0];
+ String args = tokens.length > 1 ? tokens[1] : "";
+
+ if (type.equals("get_weight")) {
+ String weight = tokens[1];
+ LOG.info("WEIGHT: %s = %.3f", weight, Decoder.weights.getWeight(weight));
+
+ } else if (type.equals("set_weights")) {
+ // Change a decoder weight
+ String[] argTokens = args.split("\\s+");
+ for (int i = 0; i < argTokens.length; i += 2) {
+ String feature = argTokens[i];
+ String newValue = argTokens[i+1];
+ float old_weight = Decoder.weights.getWeight(feature);
+ Decoder.weights.set(feature, Float.parseFloat(newValue));
+ LOG.info("set_weights: {} {} -> {}", feature, old_weight, Decoder.weights.getWeight(feature));
+ }
+
+ message.addMetaData("weights " + Decoder.weights.toString());
+
+ } else if (type.equals("get_weights")) {
+ message.addMetaData("weights " + Decoder.weights.toString());
+
+ } else if (type.equals("add_rule")) {
+ String argTokens[] = args.split(" \\|\\|\\| ");
+
+ if (argTokens.length < 3) {
+ LOG.error("* INVALID RULE '{}'", meta);
+ return;
+ }
+
+ String lhs = argTokens[0];
+ String source = argTokens[1];
+ String target = argTokens[2];
+ String featureStr = "";
+ if (argTokens.length > 3)
+ featureStr = argTokens[3];
+
+ /* Prepend source and target side nonterminals for phrase-based decoding. Probably better
+ * handled in each grammar type's addRule() function.
+ */
+ String ruleString = (joshuaConfiguration.search_algorithm.equals("stack"))
+ ? String.format("%s ||| [X,1] %s ||| [X,1] %s ||| custom=1 %s", lhs, source, target, featureStr)
+ : String.format("%s ||| %s ||| %s ||| custom=1 %s", lhs, source, target, featureStr);
+
+ Rule rule = new HieroFormatReader().parseLine(ruleString);
+ decoder.addCustomRule(rule);
+
+ LOG.info("Added custom rule {}", rule.toString());
+
+ } else if (type.equals("list_rules")) {
+
+ LOG.info("list_rules");
+
+ // Walk the the grammar trie
+ ArrayList<Trie> nodes = new ArrayList<Trie>();
+ nodes.add(decoder.getCustomPhraseTable().getTrieRoot());
+
+ while (nodes.size() > 0) {
+ Trie trie = nodes.remove(0);
+
+ if (trie == null)
+ continue;
+
+ if (trie.hasRules()) {
+ for (Rule rule: trie.getRuleCollection().getRules()) {
+ message.addRule(rule.toString());
+ LOG.debug("Found rule: " + rule);
+ }
+ }
+
+ if (trie.getExtensions() != null)
+ nodes.addAll(trie.getExtensions());
+ }
+
+ } else if (type.equals("remove_rule")) {
+
+ Rule rule = new HieroFormatReader().parseLine(args);
+
+ LOG.info("remove_rule " + rule);
+
+ Trie trie = decoder.getCustomPhraseTable().getTrieRoot();
+ int[] sourceTokens = rule.getFrench();
+ for (int i = 0; i < sourceTokens.length; i++) {
+ Trie nextTrie = trie.match(sourceTokens[i]);
+ if (nextTrie == null)
+ return;
+
+ trie = nextTrie;
+ }
+
+ if (trie.hasRules()) {
+ for (Rule ruleCand: trie.getRuleCollection().getRules()) {
+ if (Arrays.equals(rule.getEnglish(), ruleCand.getEnglish())) {
+ trie.getRuleCollection().getRules().remove(ruleCand);
+ break;
+ }
+ }
+ return;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/server/TcpServer.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/server/TcpServer.java b/joshua-core/src/main/java/org/apache/joshua/server/TcpServer.java
new file mode 100644
index 0000000..e054186
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/server/TcpServer.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.server;
+
+import java.io.IOException;
+import java.net.ServerSocket;
+
+import org.apache.joshua.decoder.Decoder;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * TCP/IP server. Accepts newline-separated input sentences written to the socket, translates them
+ * all, and writes the resulting translations back out to the socket.
+ */
+public class TcpServer {
+ private static final Logger LOG = LoggerFactory.getLogger(TcpServer.class);
+ private final JoshuaConfiguration joshuaConfiguration;
+ private Decoder decoder;
+ private int port;
+
+ public TcpServer(Decoder decoder, int port,JoshuaConfiguration joshuaConfiguration) {
+ this.joshuaConfiguration = joshuaConfiguration;
+ this.decoder = decoder;
+ this.port = port;
+ }
+
+ /**
+ * Listens on a port for new socket connections. Concurrently handles multiple socket connections.
+ */
+ public void start() {
+
+ try {
+ ServerSocket serverSocket = new ServerSocket(joshuaConfiguration.server_port);
+ LOG.info("** TCP Server running and listening on port {}.", port);
+
+ boolean listening = true;
+ while (listening)
+ new ServerThread(serverSocket.accept(), decoder, joshuaConfiguration).start();
+
+ serverSocket.close();
+
+ } catch (IOException e) {
+ throw new RuntimeException(String.format("Could not listen on port: %d.",
+ joshuaConfiguration.server_port));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/subsample/AlignedSubsampler.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/subsample/AlignedSubsampler.java b/joshua-core/src/main/java/org/apache/joshua/subsample/AlignedSubsampler.java
new file mode 100644
index 0000000..2915685
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/subsample/AlignedSubsampler.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.subsample;
+
+import java.io.BufferedWriter;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+
+
+/**
+ * A subsampler which takes in word-alignments as well as the F and E files. To remove redundant
+ * code, this class uses callback techniques in order to "override" the superclass methods.
+ *
+ * @see org.apache.joshua.subsample.Subsampler
+ * @author wren ng thornton wren@users.sourceforge.net
+ * @version $LastChangedDate$
+ */
+public class AlignedSubsampler extends Subsampler {
+
+ public AlignedSubsampler(String[] testFiles, int maxN, int targetCount) throws IOException {
+ super(testFiles, maxN, targetCount);
+ }
+
+
+ /**
+ * @param filelist list of source files to subsample from
+ * @param targetFtoERatio goal for ratio of output F length to output E length
+ * @param extf extension of F files
+ * @param exte extension of E files
+ * @param exta extension of alignment files
+ * @param fpath path to source F files
+ * @param epath path to source E files
+ * @param apath path to source alignment files
+ * @param output basename for output files (will append extensions)
+ * @throws IOException if there is an error reading the input file(s)
+ */
+ public void subsample(String filelist, float targetFtoERatio, String extf, String exte,
+ String exta, String fpath, String epath, String apath, String output) throws IOException {
+ this.subsample(filelist, targetFtoERatio, new PhraseWriter(new BufferedWriter(
+ new OutputStreamWriter(new FileOutputStream(output + "." + extf), "UTF8")),
+ new BufferedWriter(
+ new OutputStreamWriter(new FileOutputStream(output + "." + exte), "UTF8")),
+ new BufferedWriter(
+ new OutputStreamWriter(new FileOutputStream(output + "." + exta), "UTF8"))),
+ new BiCorpusFactory(fpath, epath, apath, extf, exte, exta) { /* Local class definition */
+ public BiCorpus fromFiles(String f) throws IOException {
+ return this.alignedFromFiles(f);
+ }
+ });
+ }
+
+
+ @SuppressWarnings("static-access")
+ public static void main(String[] args) {
+ new SubsamplerCLI() { /* Local class definition */
+
+ // TODO hasArg is a static method. It should be accessed as OptionBuilder.hasArg()
+ protected final Option oa = OptionBuilder.withArgName("lang").hasArg()
+ .withDescription("Word alignment extension").isRequired().create("a");
+
+ // TODO hasArg is a static method. It should be accessed as OptionBuilder.hasArg()
+ protected final Option oapath = OptionBuilder.withArgName("path").hasArg()
+ .withDescription("Directory containing word alignment files").create("apath");
+
+ public Options getCliOptions() {
+ return super.getCliOptions().addOption(oa).addOption(oapath);
+ }
+
+ public String getClassName() {
+ return AlignedSubsampler.class.getName();
+ }
+
+ public void runSubsampler(String[] testFiles, int maxN, int targetCount, float ratio)
+ throws IOException {
+ new AlignedSubsampler(testFiles, maxN, targetCount).subsample(ot.getValue(), ratio,
+ of.getValue(), oe.getValue(), oa.getValue(), ofpath.getValue(), oepath.getValue(),
+ oapath.getValue(), ooutput.getValue());
+ }
+
+ }.runMain(args);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/subsample/Alignment.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/subsample/Alignment.java b/joshua-core/src/main/java/org/apache/joshua/subsample/Alignment.java
new file mode 100644
index 0000000..073eb5c
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/subsample/Alignment.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.subsample;
+
+/**
+ * A set of word alignments between an F phrase and an E phrase. The implementation uses a
+ * two-dimensional bit vector, though for our purposes we could just keep the original string around
+ * (which would save lots of time parsing and reconstructing the string).
+ *
+ * @author UMD (Jimmy Lin, Chris Dyer, et al.)
+ * @author wren ng thornton wren@users.sourceforge.net
+ * @version $LastChangedDate$
+ */
+public class Alignment {
+ private short eLength;
+ private short fLength;
+ private M2 aligned;
+
+ public Alignment(short fLength, short eLength, String alignments) {
+ this.eLength = eLength;
+ this.fLength = fLength;
+ this.aligned = new M2(fLength, eLength);
+
+ if (alignments == null || alignments.length() == 0) {
+ return;
+ }
+ String[] als = alignments.split("\\s+"); // TODO: joshua.util.Regex
+ for (String al : als) {
+ String[] pair = al.split("-");
+ if (pair.length != 2)
+ throw new IllegalArgumentException("Malformed alignment string: " + alignments);
+ short f = Short.parseShort(pair[0]);
+ short e = Short.parseShort(pair[1]);
+ if (f >= fLength || e >= eLength)
+ throw new IndexOutOfBoundsException("out of bounds: " + f + "," + e);
+ aligned.set(f, e);
+ }
+ }
+
+
+ public String toString() {
+ StringBuffer sb = new StringBuffer();
+ for (short i = 0; i < fLength; i++)
+ for (short j = 0; j < eLength; j++)
+ if (aligned.get(i, j)) sb.append(i).append('-').append(j).append(' ');
+
+ // Remove trailing space
+ if (sb.length() > 0) sb.delete(sb.length() - 1, sb.length());
+
+ return sb.toString();
+ }
+
+
+ /** A (short,short)->boolean map for storing alignments. */
+ private final static class M2 {
+ private short width;
+ private boolean[] bits;
+
+ public M2(short f, short e) {
+ width = f;
+ bits = new boolean[f * e];
+ }
+
+ public boolean get(short f, short e) {
+ return bits[width * e + f];
+ }
+
+ public void set(short f, short e) {
+ try {
+ bits[width * e + f] = true;
+ } catch (ArrayIndexOutOfBoundsException ee) {
+ throw new RuntimeException("Set(" + f + ", " + e + "): caught " + ee);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/subsample/BiCorpus.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/subsample/BiCorpus.java b/joshua-core/src/main/java/org/apache/joshua/subsample/BiCorpus.java
new file mode 100644
index 0000000..06ec0e9
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/subsample/BiCorpus.java
@@ -0,0 +1,186 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.subsample;
+
+import java.io.BufferedReader;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+import org.apache.joshua.corpus.Phrase;
+
+/**
+ * Class for representing a sentence-aligned bi-corpus (with optional word-alignments).
+ * <p>
+ * In order to avoid memory crashes we no longer extend an ArrayList, which tries to cache the
+ * entire file in memory at once. This means we'll re-read through each file (1 +
+ * {@link Subsampler#MAX_SENTENCE_LENGTH} / binsize) times where binsize is determined by the
+ * <code>subsample(String, float, PhraseWriter, BiCorpusFactory)</code> method.
+ *
+ * @author UMD (Jimmy Lin, Chris Dyer, et al.)
+ * @author wren ng thornton wren@users.sourceforge.net
+ * @version $LastChangedDate$
+ */
+public class BiCorpus implements Iterable<PhrasePair> {
+ protected final String foreignFileName;
+ protected final String nativeFileName;
+ protected final String alignmentFileName;
+
+ /**
+ * Constructor for unaligned BiCorpus.
+ * @param foreignFileName todo
+ * @param nativeFileName todo
+ * @throws IOException todo
+ */
+ public BiCorpus(String foreignFileName, String nativeFileName) throws IOException {
+ this(foreignFileName, nativeFileName, null);
+ }
+
+ /**
+ * Constructor for word-aligned BiCorpus.
+ * @param foreignFileName todo
+ * @param nativeFileName todo
+ * @param alignmentFileName todo
+ * @throws IOException todo
+ * @throws IllegalArgumentException todo
+ * @throws IndexOutOfBoundsException todo
+ */
+ public BiCorpus(String foreignFileName, String nativeFileName, String alignmentFileName)
+ throws IOException, IllegalArgumentException, IndexOutOfBoundsException {
+ this.foreignFileName = foreignFileName;
+ this.nativeFileName = nativeFileName;
+ this.alignmentFileName = alignmentFileName;
+
+ // Check for fileLengthMismatchException
+ // Of course, that will be checked for in each iteration
+ //
+ // We write it this way to avoid warnings from the foreach style loop
+ Iterator<PhrasePair> it = iterator();
+ while (it.hasNext()) {
+ it.next();
+ }
+ }
+
+
+ // ===============================================================
+ // Methods
+ // ===============================================================
+ // BUG: We don't close file handles. The other reader classes apparently have finalizers to handle
+ // this well enough for our purposes, but we should migrate to using joshua.util.io.LineReader and
+ // be sure to close it in the end.
+
+ // We're not allowed to throw exceptions from Iterator/Iterable
+ // so we have evil boilerplate to crash the system
+ /**
+ * Iterate through the files represented by this <code>BiCorpus</code>, returning a
+ * {@link PhrasePair} for each pair (or triple) of lines.
+ */
+ @SuppressWarnings("resource")
+ public Iterator<PhrasePair> iterator() {
+ PhraseReader closureRF = null;
+ PhraseReader closureRE = null;
+ BufferedReader closureRA = null;
+ try {
+ closureRF = new PhraseReader(new FileReader(this.foreignFileName), (byte) 1);
+ closureRE = new PhraseReader(new FileReader(this.nativeFileName), (byte) 0);
+ closureRA =
+ (null == this.alignmentFileName ? null : new BufferedReader(new FileReader(
+ this.alignmentFileName)));
+ } catch (FileNotFoundException e) {
+ throw new RuntimeException("File not found", e);
+ }
+ // Making final for closure capturing in the local class definition
+ final PhraseReader rf = closureRF;
+ final PhraseReader re = closureRE;
+ final BufferedReader ra = closureRA;
+
+ return new Iterator<PhrasePair>() { /* Local class definition */
+ private Phrase nextForeignPhrase = null;
+
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ public boolean hasNext() {
+ if (null == this.nextForeignPhrase) {
+ try {
+ this.nextForeignPhrase = rf.readPhrase();
+ } catch (IOException e) {
+ throw new RuntimeException("IOException", e);
+ }
+ }
+ return null != this.nextForeignPhrase;
+ }
+
+ public PhrasePair next() {
+ if (this.hasNext()) {
+ Phrase f = this.nextForeignPhrase;
+
+ Phrase e = null;
+ try {
+ e = re.readPhrase();
+ } catch (IOException ioe) {
+ throw new RuntimeException("IOException", ioe);
+ }
+ if (null == e) {
+ fileLengthMismatchException();
+ return null; // Needed to make javac happy
+ } else {
+ if (e.size() != 0 && f.size() != 0) {
+ if (null != ra) {
+ String line = null;
+ try {
+ line = ra.readLine();
+ } catch (IOException ioe) {
+ throw new RuntimeException("IOException", ioe);
+ }
+
+ if (null == line) {
+ fileLengthMismatchException();
+ return null; // Needed to make javac happy
+ } else {
+ Alignment a = new Alignment((short) f.size(), (short) e.size(), line);
+
+ this.nextForeignPhrase = null;
+ return new PhrasePair(f, e, a);
+ }
+ } else {
+ this.nextForeignPhrase = null;
+ return new PhrasePair(f, e);
+ }
+ } else {
+ // Inverted while loop
+ this.nextForeignPhrase = null;
+ return this.next();
+ }
+ }
+ } else {
+ throw new NoSuchElementException();
+ }
+ }
+ }; /* End local class definition */
+ } /* end iterator() */
+
+
+ private static void fileLengthMismatchException() throws RuntimeException {
+ throw new RuntimeException("Mismatched file lengths!");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/subsample/BiCorpusFactory.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/subsample/BiCorpusFactory.java b/joshua-core/src/main/java/org/apache/joshua/subsample/BiCorpusFactory.java
new file mode 100644
index 0000000..eda3bf5
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/subsample/BiCorpusFactory.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.subsample;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * A callback closure for <code>Subsampler.subsample</code>. This class is used by
+ * {@link AlignedSubsampler} in order to "override" methods of {@link Subsampler}, minimizing code
+ * duplication.
+ *
+ * @author wren ng thornton wren@users.sourceforge.net
+ * @version $LastChangedDate$
+ */
+public class BiCorpusFactory {
+ // Making these final requires Java6, doesn't work in Java5
+ protected final String fpath;
+ protected final String epath;
+ protected final String apath;
+ protected final String extf;
+ protected final String exte;
+ protected final String exta;
+
+ public BiCorpusFactory(String fpath, String epath, String apath, String extf, String exte,
+ String exta) {
+ // The various concatenation has been moved up here
+ // to get it out of the loops where fromFiles is called.
+ this.fpath = (fpath == null ? "." : fpath) + File.separator;
+ this.epath = (epath == null ? "." : epath) + File.separator;
+ this.apath = (apath == null ? "." : apath) + File.separator;
+ this.extf = "." + extf;
+ this.exte = "." + exte;
+ this.exta = (exta == null ? null : "." + exta);
+ }
+
+
+ /**
+ * Generate unaligned {@link org.apache.joshua.subsample.BiCorpus} by default.
+ * @param f todo
+ * @return an unaligned {@link org.apache.joshua.subsample.BiCorpus}
+ * @throws IOException if there is an error reading input file
+ */
+ public BiCorpus fromFiles(String f) throws IOException {
+ return this.unalignedFromFiles(f);
+ }
+
+ /**
+ * Generate unaligned BiCorpus.
+ * @param f todo
+ * @return an unaligned {@link org.apache.joshua.subsample.BiCorpus}
+ * @throws IOException if there is an error reading input file
+ */
+ public BiCorpus unalignedFromFiles(String f) throws IOException {
+ return new BiCorpus(fpath + f + extf, epath + f + exte);
+ }
+
+ /**
+ * Generate aligned BiCorpus.
+ * @param f todo
+ * @return an aligned {@link org.apache.joshua.subsample.BiCorpus}
+ * @throws IOException if there is an error reading input file
+ */
+ public BiCorpus alignedFromFiles(String f) throws IOException {
+ return new BiCorpus(fpath + f + extf, epath + f + exte, apath + f + exta);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/subsample/PhrasePair.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/subsample/PhrasePair.java b/joshua-core/src/main/java/org/apache/joshua/subsample/PhrasePair.java
new file mode 100644
index 0000000..41c05d3
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/subsample/PhrasePair.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.subsample;
+
+import org.apache.joshua.corpus.Phrase;
+
+
+/**
+ * Phrase-aligned tuple class associating an F phrase, E phrase, and (possibly null)
+ * word-alignments. This is primarily for maintaining sentence-alignment.
+ *
+ * @author UMD (Jimmy Lin, Chris Dyer, et al.)
+ * @author wren ng thornton wren@users.sourceforge.net
+ * @version $LastChangedDate$
+ */
+public class PhrasePair {
+ // Making these final requires Java6, not Java5
+ private final Phrase f;
+ private final Phrase e;
+ private final Alignment a;
+
+ // ===============================================================
+ // Constructors
+ // ===============================================================
+ public PhrasePair(Phrase f_, Phrase e_) {
+ this(f_, e_, null);
+ }
+
+ public PhrasePair(Phrase f, Phrase e, Alignment a) {
+ this.f = f;
+ this.e = e;
+ this.a = a;
+ }
+
+ // ===============================================================
+ // Attributes
+ // ===============================================================
+ public Phrase getF() {
+ return f;
+ }
+
+ public Phrase getE() {
+ return e;
+ }
+
+ public Alignment getAlignment() {
+ return a;
+ }
+
+ // ===============================================================
+ // Methods
+ // ===============================================================
+ public float ratioFtoE() {
+ return ((float) this.f.size()) / ((float) this.e.size());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/subsample/PhraseReader.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/subsample/PhraseReader.java b/joshua-core/src/main/java/org/apache/joshua/subsample/PhraseReader.java
new file mode 100644
index 0000000..6db216f
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/subsample/PhraseReader.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.subsample;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.Reader;
+
+import org.apache.joshua.corpus.BasicPhrase;
+
+/**
+ * Wrapper class to read in each line as a BasicPhrase.
+ *
+ * @author UMD (Jimmy Lin, Chris Dyer, et al.)
+ * @author wren ng thornton wren@users.sourceforge.net
+ * @version $LastChangedDate$
+ */
+public class PhraseReader extends BufferedReader {
+ private byte language;
+
+ public PhraseReader(Reader r, byte language) {
+ super(r);
+ this.language = language;
+ }
+
+ public BasicPhrase readPhrase() throws IOException {
+ String line = super.readLine();
+ return (line == null ? null : new BasicPhrase(this.language, line));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/subsample/PhraseWriter.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/subsample/PhraseWriter.java b/joshua-core/src/main/java/org/apache/joshua/subsample/PhraseWriter.java
new file mode 100644
index 0000000..11bbf08
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/subsample/PhraseWriter.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.subsample;
+
+import java.io.BufferedWriter;
+import java.io.IOException;
+
+
+/**
+ * A PhrasePair-parallel BufferedWriter. In an ideal world we could get the compiler to inline all
+ * of this, to have zero-overhead while not duplicating code. Alas, Java's not that cool. The
+ * "final" could help on JIT at least.
+ *
+ * @author wren ng thornton wren@users.sourceforge.net
+ * @version $LastChangedDate$
+ */
+final public class PhraseWriter {
+ // Making these final requires Java6, not Java5
+ private final BufferedWriter wf;
+ private final BufferedWriter we;
+ private final BufferedWriter wa;
+
+ // ===============================================================
+ // Constructors
+ // ===============================================================
+ public PhraseWriter(BufferedWriter wf_, BufferedWriter we_) {
+ this(wf_, we_, null);
+ }
+
+ public PhraseWriter(BufferedWriter wf, BufferedWriter we, BufferedWriter wa) {
+ this.wf = wf;
+ this.we = we;
+ this.wa = wa;
+ }
+
+
+ // ===============================================================
+ // Methods
+ // ===============================================================
+ public void write(PhrasePair pp) throws IOException {
+ this.wf.write(pp.getF().toString());
+ this.we.write(pp.getE().toString());
+ if (null != this.wa) this.wa.write(pp.getAlignment().toString());
+ }
+
+ public void newLine() throws IOException {
+ this.wf.newLine();
+ this.we.newLine();
+ if (null != this.wa) this.wa.newLine();
+ }
+
+ public void flush() throws IOException {
+ this.wf.flush();
+ this.we.flush();
+ if (null != this.wa) this.wa.flush();
+ }
+
+ public void close() throws IOException {
+ this.wf.close();
+ this.we.close();
+ if (null != this.wa) this.wa.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/subsample/Subsampler.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/subsample/Subsampler.java b/joshua-core/src/main/java/org/apache/joshua/subsample/Subsampler.java
new file mode 100644
index 0000000..36e1925
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/subsample/Subsampler.java
@@ -0,0 +1,246 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.subsample;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.joshua.corpus.BasicPhrase;
+import org.apache.joshua.corpus.Phrase;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A class for subsampling a large (F,E)-parallel sentence-aligned corpus to generate a smaller
+ * corpus whose N-grams are relevant to some seed corpus. The idea of subsampling owes to Kishore
+ * Papineni.
+ *
+ * @author UMD (Jimmy Lin, Chris Dyer, et al.)
+ * @author wren ng thornton wren@users.sourceforge.net
+ * @version $LastChangedDate$
+ */
+public class Subsampler {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Subsampler.class);
+
+ protected Map<Phrase, Integer> ngramCounts;
+ protected int maxN;
+ protected int targetCount;
+ protected int maxSubsample = 1500000;
+
+ protected static final int MAX_SENTENCE_LENGTH = 100;
+ protected static final int MIN_RATIO_LENGTH = 10;
+
+
+ public Subsampler(String[] testFiles, int maxN, int targetCount) throws IOException {
+ this.maxN = maxN;
+ this.targetCount = targetCount;
+ this.ngramCounts = loadNgrams(testFiles);
+ }
+
+ private HashMap<Phrase, Integer> loadNgrams(String[] files) throws IOException {
+ HashMap<Phrase, Integer> map = new HashMap<Phrase, Integer>();
+ for (String fn : files) {
+ LOG.debug("Loading test set from {}", fn);
+
+ PhraseReader reader = new PhraseReader(new FileReader(fn), (byte) 1);
+ Phrase phrase;
+ int lineCount = 0;
+ try {
+ while ((phrase = reader.readPhrase()) != null) {
+ lineCount++;
+ List<Phrase> ngrams = phrase.getSubPhrases(this.maxN);
+ for (Phrase ngram : ngrams)
+ map.put(ngram, 0);
+ }
+ } finally {
+ reader.close();
+ }
+ LOG.debug("Processed {} lines in {}", lineCount, fn);
+ }
+ LOG.debug("Test set: {} ngrams", map.size());
+ return map;
+ }
+
+
+ /**
+ * The general subsampler function for external use.
+ *
+ * @param filelist list of source files to subsample from
+ * @param targetFtoERatio goal for ratio of output F length to output E length
+ * @param extf extension of F files
+ * @param exte extension of E files
+ * @param fpath path to source F files
+ * @param epath path to source E files
+ * @param output basename for output files (will append extensions)
+ * @throws IOException if there is an issue reading one of the input files
+ */
+ public void subsample(String filelist, float targetFtoERatio, String extf, String exte,
+ String fpath, String epath, String output) throws IOException {
+ this.subsample(filelist, targetFtoERatio, new PhraseWriter(new BufferedWriter(
+ new OutputStreamWriter(new FileOutputStream(output + "." + extf), "UTF8")),
+ new BufferedWriter(
+ new OutputStreamWriter(new FileOutputStream(output + "." + exte), "UTF8"))),
+ new BiCorpusFactory(fpath, epath, null, extf, exte, null));
+ }
+
+ /**
+ * The main wrapper for the subsample worker. Closes the
+ * {@link org.apache.joshua.subsample.PhraseWriter} before exiting.
+ * @param filelist list of source files to subsample from
+ * @param targetFtoERatio goal for ratio of output F length to output E length
+ * @param out a {@link org.apache.joshua.subsample.PhraseWriter} to flush data to
+ * @param bcFactory used to generate a sentence-aligned {@link org.apache.joshua.subsample.BiCorpus}
+ * @throws IOException if there is an issue reading one of the input files
+ */
+ protected void subsample(String filelist, float targetFtoERatio, PhraseWriter out,
+ BiCorpusFactory bcFactory) throws IOException {
+ try {
+ // Read filenames into a list
+ List<String> files = new ArrayList<String>();
+ {
+ FileReader fr = null;
+ BufferedReader br = null;
+ try {
+ fr = new FileReader(filelist);
+ br = new BufferedReader(fr);
+ String file;
+ while ((file = br.readLine()) != null) {
+ files.add(file);
+ }
+ } finally {
+ // Maybe redundant, but UMD's FixBugs says to
+ // close br (and close is idempotent anyways)
+ if (null != fr) fr.close();
+ if (null != br) br.close();
+ }
+ }
+
+ int totalSubsampled = 0;
+ // Iterating on files in order biases towards files
+ // earlier in the list
+ for (String f : files) {
+ LOG.info("Loading training data: {}", f);
+
+ BiCorpus bc = bcFactory.fromFiles(f);
+
+ HashMap<PhrasePair, PhrasePair> set = new HashMap<PhrasePair, PhrasePair>();
+
+ int binsize = 10; // BUG: Magic-Number
+ int max_k = MAX_SENTENCE_LENGTH / binsize;
+ LOG.debug("Looking in length range");
+ // Iterating bins from small to large biases
+ // towards short sentences
+ for (int k = 0; k < max_k; k++) {
+ LOG.debug(" [{}, {}]", (k * binsize + 1), ((k + 1) * binsize));
+ this.subsample(set, bc, k * binsize + 1, (k + 1) * binsize, targetFtoERatio);
+
+ if (set.size() + totalSubsampled > maxSubsample) break;
+ }
+
+ float ff = 0.0f;
+ float ef = 0.0f;
+ for (PhrasePair pp : set.keySet()) {
+ // Get pp.ratioFtoE() for all pp
+ ff += pp.getF().size();
+ ef += pp.getE().size();
+
+ out.write(set.get(pp));
+ out.newLine();
+ }
+ out.flush();
+
+ totalSubsampled += set.size();
+ LOG.info("current={} [total={}] currentRatio={}", set.size(), totalSubsampled, (ff / ef));
+
+ // TODO: is this gc actually dubious? Or
+ // does profiling show it helps? We only
+ // do it once per file, so it's not a
+ // performance blackhole.
+ set = null;
+ bc = null;
+ System.gc();
+ }
+ } finally {
+ out.close();
+ }
+ }
+
+ /**
+ * The worker function for subsampling.
+ *
+ * @param set The set to put selected sentences into
+ * @param bc The sentence-aligned corpus to read from
+ * @param minLength The minimum F sentence length
+ * @param maxLength The maximum F sentence length
+ * @param targetFtoERatio The desired ratio of F length to E length
+ */
+ private void subsample(HashMap<PhrasePair, PhrasePair> set, BiCorpus bc, int minLength,
+ int maxLength, float targetFtoERatio) {
+ for (PhrasePair pp : bc) {
+ PhrasePair lowercase_pp =
+ new PhrasePair(new BasicPhrase((byte) 1, pp.getF().toString().toLowerCase()),
+ new BasicPhrase((byte) 1, pp.getE().toString().toLowerCase()), pp.getAlignment());
+
+ {
+ int eLength = pp.getE().size();
+ if (eLength == 0 || eLength > MAX_SENTENCE_LENGTH) continue;
+ }
+
+ int fLength = pp.getF().size();
+ if (fLength == 0 || fLength < minLength || fLength > maxLength
+ || fLength > MAX_SENTENCE_LENGTH) continue;
+ if (fLength > 10 && targetFtoERatio != 0.0f) {
+ float ratio = pp.ratioFtoE();
+ if (fLength >= MIN_RATIO_LENGTH
+ && (ratio > 1.3f * targetFtoERatio || ratio * 1.3f < targetFtoERatio)) continue;
+ }
+ if (set.containsKey(lowercase_pp)) continue;
+
+ // at this point, length checks out and the sentence hasn't
+ // been selected yet
+
+ List<Phrase> ngrams = pp.getF().getSubPhrases(this.maxN);
+ boolean useSentence = false;
+ for (Phrase ng : ngrams) {
+ Integer count = this.ngramCounts.get(ng);
+ if (count == null) continue;
+ if (count < targetCount) {
+ useSentence = true;
+ count++;
+ this.ngramCounts.put(ng, count);
+ }
+ }
+ if (useSentence) set.put(lowercase_pp, pp);
+ }
+ }
+
+
+ public static void main(String[] args) {
+ new SubsamplerCLI().runMain(args);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/subsample/SubsamplerCLI.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/subsample/SubsamplerCLI.java b/joshua-core/src/main/java/org/apache/joshua/subsample/SubsamplerCLI.java
new file mode 100644
index 0000000..5a287c3
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/subsample/SubsamplerCLI.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.subsample;
+
+import java.io.IOException;
+
+import org.apache.commons.cli.GnuParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+
+
+/**
+ * This class defines a callback closure to allow "overriding" the main function in subclasses of
+ * {@link Subsampler}, without duplicating code. For all subclasses, CLI <code>Options</code> should
+ * be members of the class (so they're visible to <code>runSubsampler</code> as well as
+ * <code>getCliOptions</code>), the <code>getCliOptions</code> method should be overridden to add
+ * the additional options (via <code>super</code> to keep the old options), and the
+ * <code>runSubsampler</code> method should be overridden to do the primary work for main. The
+ * <code>runMain</code> method ties everything together and should not need modification. Due to the
+ * one-use nature of subclasses of <code>SubsampleCLI</code>, they generally should be implemented
+ * as anonymous local classes.
+ *
+ * @author wren ng thornton wren@users.sourceforge.net
+ * @version $LastChangedDate$
+ */
+@SuppressWarnings("static-access")
+public class SubsamplerCLI {
+ // TODO hasArg is a static method. It should be accessed as OptionBuilder.hasArg()
+ protected final Option ot = OptionBuilder.withArgName("listfile").hasArg()
+ .withDescription("A file containing a list of training file basenames (what to sample from)")
+ .isRequired().create("training");
+
+ // TODO hasArg is a static method. It should be accessed as OptionBuilder.hasArg()
+ protected final Option otest = OptionBuilder.withArgName("file").hasArgs()
+ .withDescription("The test file (what to sample for)").isRequired().create("test");
+
+ // TODO hasArg is a static method. It should be accessed as OptionBuilder.hasArg()
+ protected final Option ooutput = OptionBuilder.withArgName("basename").hasArgs()
+ .withDescription("File basename for output training corpus").isRequired().create("output");
+
+ // TODO hasArg is a static method. It should be accessed as OptionBuilder.hasArg()
+ protected final Option of = OptionBuilder.withArgName("lang").hasArg()
+ .withDescription("Foreign language extension").isRequired().create("f");
+
+ // TODO hasArg is a static method. It should be accessed as OptionBuilder.hasArg()
+ protected final Option oe = OptionBuilder.withArgName("lang").hasArg()
+ .withDescription("Native language extension").isRequired().create("e");
+
+ // TODO hasArg is a static method. It should be accessed as OptionBuilder.hasArg()
+ protected final Option ofpath = OptionBuilder.withArgName("path").hasArg()
+ .withDescription("Directory containing foreign language files").create("fpath");
+
+ // TODO hasArg is a static method. It should be accessed as OptionBuilder.hasArg()
+ protected final Option oepath = OptionBuilder.withArgName("path").hasArg()
+ .withDescription("Directory containing native language files").create("epath");
+
+ // TODO hasArg is a static method. It should be accessed as OptionBuilder.hasArg()
+ protected final Option oratio = OptionBuilder.withArgName("ratio").hasArg()
+ .withDescription("Target F/E ratio").create("ratio");
+
+ /**
+ * Return all Options. The HelpFormatter will print them in sorted order, so it doesn't matter
+ * when we add them. Subclasses should override this method by adding more options.
+ * @return all of the {@link org.apache.commons.cli.Options}
+ */
+ public Options getCliOptions() {
+ return new Options().addOption(ot).addOption(otest).addOption(of).addOption(oe)
+ .addOption(ofpath).addOption(oepath).addOption(oratio).addOption(ooutput);
+ }
+
+ /**
+ * This method should be overridden to return the class used in
+ * {@link org.apache.joshua.subsample.SubsamplerCLI#runSubsampler(String[], int, int, float)}.
+ * @return the {@link org.apache.joshua.subsample.Subsampler} implementation
+ */
+ public String getClassName() {
+ return Subsampler.class.getName();
+ }
+
+ /**
+ * Callback to run the subsampler. This function needs access to the variables holding each
+ * Option, thus all this closure nonsense.
+ * @param testFiles a String array of test files
+ * @param maxN todo
+ * @param targetCount todo
+ * @param ratio todo
+ * @throws IOException if there is an issue whilst reading input files
+ */
+ public void runSubsampler(String[] testFiles, int maxN, int targetCount, float ratio)
+ throws IOException {
+ new Subsampler(testFiles, maxN, targetCount).subsample(ot.getValue(), ratio, of.getValue(),
+ oe.getValue(), ofpath.getValue(), oepath.getValue(), ooutput.getValue());
+ }
+
+ /**
+ * Non-static version of main so that we can define anonymous local classes to override or extend
+ * the above.
+ * @param args a String array of input options
+ */
+ public void runMain(String[] args) {
+ Options o = this.getCliOptions();
+ try {
+ new GnuParser().parse(o, args);
+ } catch (ParseException pe) {
+ // The message from pe is ugly, so we omit it.
+ System.err.println("Error parsing command line");
+ new HelpFormatter().printHelp(this.getClassName(), o);
+ System.exit(1);
+ }
+
+ try {
+ float ratio = 0.8f;
+ if (this.oratio.getValue() != null) {
+ ratio = Float.parseFloat(this.oratio.getValue());
+ }
+ this.runSubsampler(this.otest.getValues(), 12, 20, ratio);
+ } catch (Exception e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/subsample/package-info.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/subsample/package-info.java b/joshua-core/src/main/java/org/apache/joshua/subsample/package-info.java
new file mode 100644
index 0000000..b7fe744
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/subsample/package-info.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * Provides executables Subsampler and AlignedSubsampler,
+ * for subsampling from large training corpora based on a
+ * test corpus.
+ */
+package org.apache.joshua.subsample;
+
[23/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/ExtractTopCand.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/ExtractTopCand.java b/joshua-core/src/main/java/org/apache/joshua/util/ExtractTopCand.java
new file mode 100644
index 0000000..8f9b575
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/ExtractTopCand.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util;
+
+import java.io.BufferedWriter;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+
+import org.apache.joshua.util.io.IndexedReader;
+import org.apache.joshua.util.io.LineReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This program extracts the 1-best output translations from the n-best output translations
+ * generated by {@link org.apache.joshua.decoder.Decoder}.
+ *
+ * @author wren ng thornton wren@users.sourceforge.net
+ * @version $LastChangedDate: 2009-03-26 15:06:57 -0400 (Thu, 26 Mar 2009) $
+ */
+/*
+ * TODO: This class should be renamed, something like ExtractBestCandidates or
+ * ExtractBestTranslations. Saying "top" implies more than one (the top how many?) and "cand" is
+ * unnecessary abbreviation (also, who cares about candidacy?). Once we rename this, the
+ * ./example2/decode_example2.sh script will need updating (as will the end-to-end code)
+ */
+public class ExtractTopCand {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ExtractTopCand.class);
+
+ /**
+ * Usage: <code>java ExtractTopCand nbestInputFile 1bestOutputFile</code>.
+ * <p>
+ * If the input file name is "-" then input is read from <code>System.in</code>. If the output
+ * file name is "-" then output is directed to <code>System.out</code>. If a file already exists
+ * with the output file name, it is truncated before writing. The bulk of this program is
+ * implemented by {@link org.apache.joshua.util.ExtractTopCand#extractOneBest(IndexedReader, BufferedWriter, int)}.
+ * @param args input arguments for the tool
+ */
+ public static void main(String[] args) {
+ String inFile = "-";
+ String outFile = "-";
+ int field = 1;
+ if (args.length == 1) {
+ inFile = args[0];
+ } else if (args.length == 2) {
+ inFile = args[0];
+ outFile = args[1];
+ } else if (args.length == 3) {
+ inFile = args[0];
+ outFile = args[1];
+ field = Integer.parseInt(args[2]);
+ } else {
+ System.err
+ .println("Usage: ExtractTopCand [nbestInputFile [1bestOutputFile]]\n (default to stdin/stdout)");
+ System.exit(1);
+ }
+
+ try {
+ // TODO: see documentation for extractOneBest
+ // regarding using an n-best SegmentFileParser.
+ IndexedReader<String> nbestReader =
+ new IndexedReader<String>("line", "-".equals(inFile)
+ ? new LineReader(System.in)
+ : new LineReader(inFile));
+
+ /*
+ * TODO: This duplicates FileUtility.getWriteFileStream but with the addition of defaulting to
+ * System.out; should fix that (without breaking other clients of that method). We ultimately
+ * want something which autochecks for errors (like Writer); has a newLine method (like
+ * BufferedWriter); can wrap System.out; can autoflush; and it'd be handy to have the
+ * print/println methods of PrintStream/PrintWriter to boot. PrintWriter *almost* gives us all
+ * this, but it swallows errors and gives no way to retrieve them >:(
+ */
+ BufferedWriter onebestWriter =
+ new BufferedWriter(new OutputStreamWriter(("-".equals(outFile)
+ ? System.out
+ : new FileOutputStream(outFile, false)), "UTF-8"));
+
+ extractOneBest(nbestReader, onebestWriter, field);
+
+ } catch (IOException ioe) {
+ // NOTE: if our onebest was System.out, then that
+ // will already have been closed by the finally
+ // block. Printing to a closed PrintStream generates
+ // no exceptions. We should be printing to System.err
+ // anyways, but this something subtle to be aware of.
+ LOG.error(ioe.getMessage(), ioe);
+ }
+ }
+
+
+ /**
+ * Prints the one-best translation for each segment ID from the reader as a line on the writer,
+ * and closes both before exiting. The translations for a segment are printed in the order of the
+ * first occurance of the segment ID. Any information about the segment other than the translation
+ * (including segment ID) is not printed to the writer.
+ *
+ * <b>Developer Notes</b> This implementation assumes:
+ * <ol>
+ * <li>all translations for a segment are contiguous</li>
+ * <li>the 1-best translation is the first one encountered.</li>
+ * </ol>
+ * We will need to alter the implementation if these assumptions no longer hold for the output of
+ * JoshuaDecoder (or any sensible n-best format passed to this method).
+ * <p>
+ * TODO We should switch to using an n-best SegmentFileParser to
+ * ensure future compatibility with being able to configure the output format of the decoder. The
+ * MERT code needs such a SegmentFileParser anyways, so that will reduce the code duplication
+ * between these two classes.
+ *
+ * @param nbestReader todo
+ * @param onebestWriter todo
+ * @param field todo
+ * @throws IOException if there is an issue reading or writing input/output data
+ */
+ protected static void extractOneBest(IndexedReader<String> nbestReader,
+ BufferedWriter onebestWriter, int field) throws IOException {
+
+ try {
+ String prevID = null;
+ for (String line : nbestReader) {
+
+ // pass empty lines through
+ if (Regex.commentOrEmptyLine.matches(line)) {
+ onebestWriter.newLine();
+ continue;
+ }
+
+ String[] columns = Regex.threeBarsWithSpace.split(line);
+
+ // We allow non-integer segment IDs because the
+ // Segment interface does, and we have no reason
+ // to add new restrictions.
+ String newID = columns[0].trim();
+
+ // We want to give the same error message
+ // regardless of whether there's a leading space
+ // or not. And, we don't want to accidentally
+ // accept lines with lots and lots of columns.
+ if ("".equals(newID) || newID.startsWith("|||")) {
+ throw nbestReader.wrapIOException(new IOException("Malformed line, missing segment ID:\n"
+ + line));
+ }
+
+ // Make sure there's a translation there too
+ // TODO: good error message for when the second
+ // "|||" doesn't have a following field, m/\|{3}\s*$/
+ if (3 > columns.length) {
+ throw nbestReader.wrapIOException(new IOException(
+ "Malformed line, should have at least two \" ||| \":\n" + line));
+ }
+
+
+ if (null == prevID || !prevID.equals(newID)) {
+ onebestWriter.write(columns[field], 0, columns[field].length());
+ onebestWriter.newLine();
+ onebestWriter.flush();
+
+ prevID = newID;
+ }
+ }
+ } finally {
+ try {
+ nbestReader.close();
+ } finally {
+ onebestWriter.close();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/FileUtility.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/FileUtility.java b/joshua-core/src/main/java/org/apache/joshua/util/FileUtility.java
new file mode 100644
index 0000000..a36b07f
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/FileUtility.java
@@ -0,0 +1,318 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.Closeable;
+import java.io.File;
+import java.io.FileDescriptor;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.nio.charset.Charset;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Scanner;
+
+/**
+ * utility functions for file operations
+ *
+ * @author Zhifei Li, zhifei.work@gmail.com
+ * @author wren ng thornton wren@users.sourceforge.net
+ * @since 28 February 2009
+ */
+public class FileUtility {
+ public static String DEFAULT_ENCODING = "UTF-8";
+
+ /*
+ * Note: charset name is case-agnostic "UTF-8" is the canonical name "UTF8", "unicode-1-1-utf-8"
+ * are aliases Java doesn't distinguish utf8 vs UTF-8 like Perl does
+ */
+ private static final Charset FILE_ENCODING = Charset.forName(DEFAULT_ENCODING);
+
+ /**
+ * Warning, will truncate/overwrite existing files
+ * @param filename a file for which to obtain a writer
+ * @return the buffered writer object
+ * @throws IOException if there is a problem reading the inout file
+ */
+ public static BufferedWriter getWriteFileStream(String filename) throws IOException {
+ return new BufferedWriter(new OutputStreamWriter(
+ // TODO: add GZIP
+ filename.equals("-") ? new FileOutputStream(FileDescriptor.out) : new FileOutputStream(
+ filename, false), FILE_ENCODING));
+ }
+
+ /**
+ * Recursively delete the specified file or directory.
+ *
+ * @param f File or directory to delete
+ * @return <code>true</code> if the specified file or directory was deleted, <code>false</code>
+ * otherwise
+ */
+ public static boolean deleteRecursively(File f) {
+ if (null != f) {
+ if (f.isDirectory())
+ for (File child : f.listFiles())
+ deleteRecursively(child);
+ return f.delete();
+ } else {
+ return false;
+ }
+ }
+
+ /**
+ * Writes data from the integer array to disk as raw bytes, overwriting the old file if present.
+ *
+ * @param data The integer array to write to disk.
+ * @param filename The filename where the data should be written.
+ * @throws IOException if there is a problem writing to the output file
+ * @return the FileOutputStream on which the bytes were written
+ */
+ public static FileOutputStream writeBytes(int[] data, String filename) throws IOException {
+ FileOutputStream out = new FileOutputStream(filename, false);
+ writeBytes(data, out);
+ return out;
+ }
+
+ /**
+ * Writes data from the integer array to disk as raw bytes.
+ *
+ * @param data The integer array to write to disk.
+ * @param out The output stream where the data should be written.
+ * @throws IOException if there is a problem writing bytes
+ */
+ public static void writeBytes(int[] data, OutputStream out) throws IOException {
+
+ byte[] b = new byte[4];
+
+ for (int word : data) {
+ b[0] = (byte) ((word >>> 24) & 0xFF);
+ b[1] = (byte) ((word >>> 16) & 0xFF);
+ b[2] = (byte) ((word >>> 8) & 0xFF);
+ b[3] = (byte) ((word >>> 0) & 0xFF);
+
+ out.write(b);
+ }
+ }
+
+ public static void copyFile(String srFile, String dtFile) throws IOException {
+ try {
+ File f1 = new File(srFile);
+ File f2 = new File(dtFile);
+ copyFile(f1, f2);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public static void copyFile(File srFile, File dtFile) throws IOException {
+ try {
+
+ InputStream in = new FileInputStream(srFile);
+
+ // For Append the file.
+ // OutputStream out = new FileOutputStream(f2,true);
+
+ // For Overwrite the file.
+ OutputStream out = new FileOutputStream(dtFile);
+
+ byte[] buf = new byte[1024];
+ int len;
+ while ((len = in.read(buf)) > 0) {
+ out.write(buf, 0, len);
+ }
+ in.close();
+ out.close();
+ System.out.println("File copied.");
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ static public boolean deleteFile(String fileName) {
+
+ File f = new File(fileName);
+
+ // Make sure the file or directory exists and isn't write protected
+ if (!f.exists())
+ System.out.println("Delete: no such file or directory: " + fileName);
+
+ if (!f.canWrite())
+ System.out.println("Delete: write protected: " + fileName);
+
+ // If it is a directory, make sure it is empty
+ if (f.isDirectory()) {
+ String[] files = f.list();
+ if (files.length > 0)
+ System.out.println("Delete: directory not empty: " + fileName);
+ }
+
+ // Attempt to delete it
+ boolean success = f.delete();
+
+ if (!success)
+ System.out.println("Delete: deletion failed");
+
+ return success;
+
+ }
+
+ /**
+ * Returns the base directory of the file. For example, dirname('/usr/local/bin/emacs') ->
+ * '/usr/local/bin'
+ * @param fileName the input path
+ * @return the parent path
+ */
+ static public String dirname(String fileName) {
+ if (fileName.indexOf(File.separator) != -1)
+ return fileName.substring(0, fileName.lastIndexOf(File.separator));
+
+ return ".";
+ }
+
+ public static void createFolderIfNotExisting(String folderName) {
+ File f = new File(folderName);
+ if (!f.isDirectory()) {
+ System.out.println(" createFolderIfNotExisting -- Making directory: " + folderName);
+ f.mkdirs();
+ } else {
+ System.out.println(" createFolderIfNotExisting -- Directory: " + folderName
+ + " already existed");
+ }
+ }
+
+ public static void closeCloseableIfNotNull(Closeable fileWriter) {
+ if (fileWriter != null) {
+ try {
+ fileWriter.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ /**
+ * Returns the directory were the program has been started,
+ * the base directory you will implicitly get when specifying no
+ * full path when e.g. opening a file
+ * @return the current 'user.dir'
+ */
+ public static String getWorkingDirectory() {
+ return System.getProperty("user.dir");
+ }
+
+ /**
+ * Method to handle standard IO exceptions. catch (Exception e) {Utility.handleIO_exception(e);}
+ * @param e an input {@link java.lang.Exception}
+ */
+ public static void handleExceptions(Exception e) {
+ throw new RuntimeException(e);
+ }
+
+ /**
+ * Convenience method to get a full file as a String
+ * @param file the input {@link java.io.File}
+ * @return The file as a String. Lines are separated by newline character.
+ */
+ public static String getFileAsString(File file) {
+ String result = "";
+ List<String> lines = getLines(file, true);
+ for (int i = 0; i < lines.size() - 1; i++) {
+ result += lines.get(i) + "\n";
+ }
+ if (!lines.isEmpty()) {
+ result += lines.get(lines.size() - 1);
+ }
+ return result;
+ }
+
+ /**
+ * This method returns a List of String. Each element of the list corresponds to a line from the
+ * input file. The boolean keepDuplicates in the input determines if duplicate lines are allowed
+ * in the output LinkedList or not.
+ * @param file the input file
+ * @param keepDuplicates whether to retain duplicate lines
+ * @return a {@link java.util.List} of lines
+ */
+ static public List<String> getLines(File file, boolean keepDuplicates) {
+ LinkedList<String> list = new LinkedList<String>();
+ String line = "";
+ try {
+ BufferedReader InputReader = new BufferedReader(new FileReader(file));
+ for (;;) { // this loop writes writes in a Sting each sentence of
+ // the file and process it
+ int current = InputReader.read();
+ if (current == -1 || current == '\n') {
+ if (keepDuplicates || !list.contains(line))
+ list.add(line);
+ line = "";
+ if (current == -1)
+ break; // EOF
+ } else
+ line += (char) current;
+ }
+ InputReader.close();
+ } catch (Exception e) {
+ handleExceptions(e);
+ }
+ return list;
+ }
+
+ /**
+ * Returns a Scanner of the inputFile using a specific encoding
+ *
+ * @param inputFile the file for which to get a {@link java.util.Scanner} object
+ * @param encoding the encoding to use within the Scanner
+ * @return a {@link java.util.Scanner} object for a given file
+ */
+ public static Scanner getScanner(File inputFile, String encoding) {
+ Scanner scan = null;
+ try {
+ scan = new Scanner(inputFile, encoding);
+ } catch (IOException e) {
+ FileUtility.handleExceptions(e);
+ }
+ return scan;
+ }
+
+ /**
+ * Returns a Scanner of the inputFile using default encoding
+ *
+ * @param inputFile the file for which to get a {@link java.util.Scanner} object
+ * @return a {@link java.util.Scanner} object for a given file
+ */
+ public static Scanner getScanner(File inputFile) {
+ return getScanner(inputFile, DEFAULT_ENCODING);
+ }
+
+ static public String getFirstLineInFile(File inputFile) {
+ Scanner scan = FileUtility.getScanner(inputFile);
+ if (!scan.hasNextLine())
+ return null;
+ String line = scan.nextLine();
+ scan.close();
+ return line;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/FormatUtils.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/FormatUtils.java b/joshua-core/src/main/java/org/apache/joshua/util/FormatUtils.java
new file mode 100644
index 0000000..ead2db7
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/FormatUtils.java
@@ -0,0 +1,245 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util;
+
+import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
+import java.util.regex.Pattern;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Utility class for format issues.
+ *
+ * @author Juri Ganitkevitch
+ * @author Lane Schwartz
+ */
+public class FormatUtils {
+
+ private static final Logger LOG = LoggerFactory.getLogger(FormatUtils.class);
+
+ private static final String INDEX_SEPARATOR = ",";
+
+ /**
+ * Determines whether the string is a nonterminal by checking that the first character is [
+ * and the last character is ].
+ *
+ * @param token input string
+ * @return true if it's a nonterminal symbol, false otherwise
+ */
+ public static boolean isNonterminal(String token) {
+ return (token.length() >=3 && token.charAt(0) == '[') && (token.charAt(token.length() - 1) == ']');
+ }
+
+ /**
+ * Determines whether the ID represents a nonterminal. This is a trivial check, since nonterminal
+ * IDs are simply negative ones.
+ *
+ * @param id the vocabulary ID
+ * @return true if a nonterminal ID, false otherwise
+ */
+ public static boolean isNonterminal(int id) {
+ return id < 0;
+ }
+
+ /**
+ * Nonterminals are stored in the vocabulary in square brackets. This removes them when you
+ * just want the raw nonterminal word.
+ * Supports indexed and non-indexed nonTerminals:
+ * [GOAL] -> GOAL
+ * [X,1] -> [X]
+ *
+ * @param nt the nonterminal, e.g., "[GOAL]"
+ * @return the cleaned nonterminal, e.g., "GOAL"
+ */
+ public static String cleanNonTerminal(String nt) {
+ if (isNonterminal(nt)) {
+ if (isIndexedNonTerminal(nt)) {
+ // strip ",.*]"
+ return nt.substring(1, nt.indexOf(INDEX_SEPARATOR));
+ }
+ // strip "]"
+ return nt.substring(1, nt.length() - 1);
+ }
+ return nt;
+ }
+
+ private static boolean isIndexedNonTerminal(String nt) {
+ return nt.contains(INDEX_SEPARATOR);
+ }
+
+ /**
+ * Removes the index from a nonTerminal: [X,1] -> [X].
+ * @param nt an input non-terminal string
+ * @return the stripped non terminal string
+ */
+ public static String stripNonTerminalIndex(String nt) {
+ return ensureNonTerminalBrackets(cleanNonTerminal(nt));
+ }
+
+ /**
+ * Nonterminals on source and target sides are represented as [X,1], where 1 is an integer
+ * that links the two sides. This function extracts the index, e.g.,
+ *
+ * getNonterminalIndex("[X,7]") produces 7
+ *
+ * @param nt the nonterminal string
+ * @return the index
+ */
+ public static int getNonterminalIndex(String nt) {
+ return Integer.parseInt(nt.substring(nt.indexOf(INDEX_SEPARATOR) + 1, nt.length() - 1));
+ }
+
+ /**
+ * Ensures that a string looks like what the system considers a nonterminal to be.
+ *
+ * @param nt the nonterminal string
+ * @return the nonterminal string surrounded in square brackets (if not already)
+ */
+ public static String ensureNonTerminalBrackets(String nt) {
+ if (isNonterminal(nt))
+ return nt;
+ else
+ return "[" + nt + "]";
+ }
+
+ public static String escapeSpecialSymbols(String s) {
+ return s.replaceAll("\\[", "-lsb-")
+ .replaceAll("\\]", "-rsb-")
+ .replaceAll("\\|", "-pipe-");
+ }
+
+ public static String unescapeSpecialSymbols(String s) {
+ return s.replaceAll("-lsb-", "[")
+ .replaceAll("-rsb-", "]")
+ .replaceAll("-pipe-", "|");
+ }
+
+ /**
+ * wrap sentence with sentence start/stop markers
+ * as defined by Vocabulary; separated by a single whitespace.
+ * @param s an input sentence
+ * @return the wrapped sentence
+ */
+ public static String addSentenceMarkers(String s) {
+ return Vocabulary.START_SYM + " " + s + " " + Vocabulary.STOP_SYM;
+ }
+
+ /**
+ * strip sentence markers (and whitespaces) from string
+ * @param s the sentence to strip of markers (and whitespaces)
+ * @return the stripped string
+ */
+ public static String removeSentenceMarkers(String s) {
+ return s.replaceAll("<s> ", "").replace(" </s>", "");
+ }
+
+ /**
+ * Returns true if the String parameter represents a valid number.
+ * <p>
+ * The body of this method is taken from the Javadoc documentation for the Java Double class.
+ *
+ * @param string an input string
+ * @see java.lang.Double
+ * @return <code>true</code> if the string represents a valid number, <code>false</code> otherwise
+ */
+ public static boolean isNumber(String string) {
+ final String Digits = "(\\p{Digit}+)";
+ final String HexDigits = "(\\p{XDigit}+)";
+ // an exponent is 'e' or 'E' followed by an optionally
+ // signed decimal integer.
+ final String Exp = "[eE][+-]?" + Digits;
+ final String fpRegex = ("[\\x00-\\x20]*" + // Optional leading "whitespace"
+ "[+-]?(" + // Optional sign character
+ "NaN|" + // "NaN" string
+ "Infinity|" + // "Infinity" string
+
+ // A decimal floating-point string representing a finite positive
+ // number without a leading sign has at most five basic pieces:
+ // Digits . Digits ExponentPart FloatTypeSuffix
+ //
+ // Since this method allows integer-only strings as input
+ // in addition to strings of floating-point literals, the
+ // two sub-patterns below are simplifications of the grammar
+ // productions from the Java Language Specification, 2nd
+ // edition, section 3.10.2.
+
+ // Digits ._opt Digits_opt ExponentPart_opt FloatTypeSuffix_opt
+ "(((" + Digits + "(\\.)?(" + Digits + "?)(" + Exp + ")?)|" +
+
+ // . Digits ExponentPart_opt FloatTypeSuffix_opt
+ "(\\.(" + Digits + ")(" + Exp + ")?)|" +
+
+ // Hexadecimal strings
+ "((" +
+ // 0[xX] HexDigits ._opt BinaryExponent FloatTypeSuffix_opt
+ "(0[xX]" + HexDigits + "(\\.)?)|" +
+
+ // 0[xX] HexDigits_opt . HexDigits BinaryExponent FloatTypeSuffix_opt
+ "(0[xX]" + HexDigits + "?(\\.)" + HexDigits + ")" +
+
+ ")[pP][+-]?" + Digits + "))" + "[fFdD]?))" + "[\\x00-\\x20]*");// Optional
+ // trailing
+ // "whitespace"
+
+ return Pattern.matches(fpRegex, string);
+ }
+
+ /**
+ * Set System.out and System.err to use the UTF8 character encoding.
+ *
+ * @return <code>true</code> if both System.out and System.err were successfully set to use UTF8,
+ * <code>false</code> otherwise.
+ */
+ public static boolean useUTF8() {
+
+ try {
+ System.setOut(new PrintStream(System.out, true, "UTF8"));
+ System.setErr(new PrintStream(System.err, true, "UTF8"));
+ return true;
+ } catch (UnsupportedEncodingException e1) {
+ LOG.warn("UTF8 is not a valid encoding; using system default encoding for System.out and System.err.");
+ return false;
+ } catch (SecurityException e2) {
+ LOG.warn("Security manager is configured to disallow changes to System.out or System.err; using system default encoding.");
+ return false;
+ }
+ }
+
+ /**
+ * Determines if a string contains ALL CAPS
+ *
+ * @param token an input token
+ * @return true if the string is all in uppercase, false otherwise
+ */
+ public static boolean ISALLUPPERCASE(String token) {
+ for (int i = 0; i < token.length(); i++)
+ if (! Character.isUpperCase(token.charAt(i)))
+ return false;
+ return true;
+ }
+
+ public static String capitalize(String word) {
+ if (word == null || word.length() == 0)
+ return word;
+ return word.substring(0, 1).toUpperCase() + word.substring(1);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/IntegerPair.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/IntegerPair.java b/joshua-core/src/main/java/org/apache/joshua/util/IntegerPair.java
new file mode 100644
index 0000000..bfbfa23
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/IntegerPair.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util;
+
+/**
+ * Memory-efficient implementation of an integer tuple.
+ *
+ * @author Lane Schwartz
+ */
+public final class IntegerPair {
+
+ public final int first;
+ public final int second;
+
+ public IntegerPair(final int first, final int second) {
+ this.first = first;
+ this.second = second;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/JoshuaEval.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/JoshuaEval.java b/joshua-core/src/main/java/org/apache/joshua/util/JoshuaEval.java
new file mode 100644
index 0000000..79473ee
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/JoshuaEval.java
@@ -0,0 +1,624 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.text.DecimalFormat;
+import java.util.TreeSet;
+
+import org.apache.joshua.metrics.EvaluationMetric;
+
+public class JoshuaEval {
+ final static DecimalFormat f4 = new DecimalFormat("###0.0000");
+
+ // if true, evaluation is performed for each candidate translation as
+ // well as on the entire candidate set
+ static boolean verbose;
+
+ // number of candidate translations
+ static int numSentences;
+
+ // number of reference translations per sentence
+ static int refsPerSen;
+
+ // 0: no normalization, 1: "NIST-style" tokenization, and also rejoin 'm, 're, *'s, 've, 'll, 'd,
+ // and n't,
+ // 2: apply 1 and also rejoin dashes between letters, 3: apply 1 and also drop non-ASCII
+ // characters
+ // 4: apply 1+2+3
+ static private int textNormMethod;
+
+ // refSentences[i][r] is the rth reference translation of the ith sentence
+ static String[][] refSentences;
+
+ // name of evaluation metric
+ static String metricName;
+
+ // options for the evaluation metric (e.g. for BLEU, maxGramLength and effLengthMethod)
+ static String[] metricOptions;
+
+ // the scorer
+ static EvaluationMetric evalMetric;
+
+ // if true, the reference set(s) is (are) evaluated
+ static boolean evaluateRefs;
+
+ // file names for input files. When refsPerSen > 1, refFileName can be
+ // the name of a single file, or a file name prefix.
+ static String refFileName;
+ static String candFileName;
+
+ // format of the candidate file: "plain" if one candidate per sentence, and "nbest" if a decoder
+ // output
+ static String candFileFormat;
+
+ // if format is nbest, evaluate the r'th candidate of each sentence
+ static int candRank;
+
+
+ private static void evaluateCands_plain(String inFileName) {
+ evaluate(candFileName, "plain", 1, 1);
+ }
+
+
+ private static void evaluateCands_nbest(String inFileName, int testIndex) {
+ evaluate(candFileName, "nbest", -1, testIndex);
+ }
+
+
+ private static void evaluateRefSet(int r) {
+ evaluate(refFileName, "plain", refsPerSen, r);
+ }
+
+
+ private static void evaluate(String inFileName, String inFileFormat, int candPerSen, int testIndex) {
+ // candPerSen: how many candidates are provided per sentence?
+ // (if inFileFormat is nbest, then candPerSen is ignored, since it is variable)
+ // testIndex: which of the candidates (for each sentence) should be tested?
+ // e.g. testIndex=1 means first candidate should be evaluated
+ // testIndex=candPerSen means last candidate should be evaluated
+
+ if (inFileFormat.equals("plain") && candPerSen < 1) {
+ throw new RuntimeException("candPerSen must be positive for a file in plain format.");
+ }
+
+ if (inFileFormat.equals("plain") && (testIndex < 1 || testIndex > candPerSen)) {
+ throw new RuntimeException("For the plain format, testIndex must be in [1,candPerSen]");
+ }
+
+
+ String[] topCand_str = new String[numSentences];
+
+ // BUG: all of this needs to be replaced with the SegmentFileParser and related interfaces.
+ try (InputStream inStream = new FileInputStream(new File(inFileName));
+ BufferedReader inFile = new BufferedReader(new InputStreamReader(inStream, "utf8"))) {
+
+ // read the candidates
+
+ String line, candidate_str;
+
+ if (inFileFormat.equals("plain")) {
+
+ for (int i = 0; i < numSentences; ++i) {
+
+ // skip candidates 1 through testIndex-1
+ for (int n = 1; n < testIndex; ++n) {
+ line = inFile.readLine();
+ }
+
+ // read testIndex'th candidate
+ candidate_str = inFile.readLine();
+
+ topCand_str[i] = normalize(candidate_str, textNormMethod);
+
+ for (int n = testIndex + 1; n <= candPerSen; ++n) {
+ // skip candidates testIndex+1 through candPerSen-1
+ // (this probably only applies when evaluating a combined reference file)
+ line = inFile.readLine();
+ }
+
+ } // for (i)
+
+ } else { // nbest format
+
+ int i = 0;
+ int n = 1;
+ line = inFile.readLine();
+
+ while (line != null && i < numSentences) {
+
+ /*
+ * line format:
+ *
+ * .* ||| words of candidate translation . ||| feat-1_val feat-2_val ...
+ * feat-numParams_val .*
+ */
+
+ while (n < candRank) {
+ line = inFile.readLine();
+ ++n;
+ }
+
+ // at the moment, line stores the candRank'th candidate (1-indexed) of the i'th sentence
+ // (0-indexed)
+
+ if (line == null) {
+ println("Not enough candidates in " + inFileName + " to extract the " + candRank
+ + "'th candidate for each sentence.");
+ throw new RuntimeException("(Failed to extract one for the " + i + "'th sentence (0-indexed).)");
+ }
+
+ int read_i = Integer.parseInt(line.substring(0, line.indexOf(" |||")).trim());
+ if (read_i == i) {
+ line = line.substring(line.indexOf("||| ") + 4); // get rid of initial text
+ candidate_str = line.substring(0, line.indexOf(" |||"));
+ topCand_str[i] = normalize(candidate_str, textNormMethod);
+ if (i < numSentences - 1) {
+ while (read_i == i) {
+ line = inFile.readLine();
+ read_i = Integer.parseInt(line.substring(0, line.indexOf(" |||")).trim());
+ }
+ }
+ n = 1;
+ i += 1;
+ } else {
+ String msg = "Not enough candidates in " + inFileName + " to extract the " + candRank
+ + "'th candidate for each sentence. (Failed to extract one for the "
+ + i + "'th sentence (0-indexed).)";
+ throw new RuntimeException(msg);
+ }
+
+ } // while (line != null)
+
+ if (i != numSentences) {
+ throw new RuntimeException("Not enough candidates were found (i = " + i + "; was expecting " + numSentences
+ + ")");
+ }
+
+ } // nbest format
+
+ inFile.close();
+
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+
+ int[] IA = new int[numSentences];
+ for (int i = 0; i < numSentences; ++i) {
+ IA[i] = i;
+ }
+ int[][] SS = evalMetric.suffStats(topCand_str, IA);
+
+ int suffStatsCount = evalMetric.get_suffStatsCount();
+
+ int[] totStats = new int[suffStatsCount];
+ for (int s = 0; s < suffStatsCount; ++s) {
+ totStats[s] = 0;
+ for (int i = 0; i < numSentences; ++i) {
+ totStats[s] += SS[i][s];
+ }
+ }
+
+ evalMetric.printDetailedScore_fromStats(totStats, false);
+
+ if (verbose) {
+ println("");
+ println("Printing detailed scores for individual sentences...");
+ for (int i = 0; i < numSentences; ++i) {
+ print("Sentence #" + i + ": ");
+ int[] stats = new int[suffStatsCount];
+ for (int s = 0; s < suffStatsCount; ++s) {
+ stats[s] = SS[i][s];
+ }
+ evalMetric.printDetailedScore_fromStats(stats, true);
+ // already prints a \n
+ }
+ }
+
+ } // void evaluate(...)
+
+
+ private static void printUsage(int argsLen) {
+ println("Oops, you provided " + argsLen + " args!");
+ println("");
+ println("Usage:");
+ println(" JoshuaEval [-cand candFile] [-format candFileformat] [-rank r]\n [-ref refFile] [-rps refsPerSen] [-m metricName metric options]\n [-evr evalRefs] [-v verbose]");
+ println("");
+ println(" (*) -cand candFile: candidate translations\n [[default: candidates.txt]]");
+ println(" (*) -format candFileFormat: is the candidate file a plain file (one candidate\n per sentence) or does it contain multiple candidates per sentence as\n a decoder's output)? For the first, use \"plain\". For the second,\n use \"nbest\".\n [[default: plain]]");
+ println(" (*) -rank r: if format=nbest, evaluate the set of r'th candidates.\n [[default: 1]]");
+ println(" (*) -ref refFile: reference translations (or file name prefix)\n [[default: references.txt]]");
+ println(" (*) -rps refsPerSen: number of reference translations per sentence\n [[default: 1]]");
+ println(" (*) -txtNrm textNormMethod: how should text be normalized?\n (0) don't normalize text,\n or (1) \"NIST-style\", and also rejoin 're, *'s, n't, etc,\n or (2) apply 1 and also rejoin dashes between letters,\n or (3) apply 1 and also drop non-ASCII characters,\n or (4) apply 1+2+3\n [[default: 1]]");
+ println(" (*) -m metricName metric options: name of evaluation metric and its options\n [[default: BLEU 4 closest]]");
+ println(" (*) -evr evalRefs: evaluate references (1) or not (0) (sanity check)\n [[default: 0]]");
+ println(" (*) -v verbose: evaluate individual sentences (1) or not (0)\n [[default: 0]]");
+ println("");
+ println("Ex.: java JoshuaEval -cand nbest.out -ref ref.all -rps 4 -m BLEU 4 shortest");
+ }
+
+
+ private static void processArgsAndInitialize(String[] args) {
+ EvaluationMetric.set_knownMetrics();
+
+ // set default values
+ candFileName = "candidates.txt";
+ candFileFormat = "plain";
+ candRank = 1;
+ refFileName = "references.txt";
+ refsPerSen = 1;
+ textNormMethod = 1;
+ metricName = "BLEU";
+ metricOptions = new String[2];
+ metricOptions[0] = "4";
+ metricOptions[1] = "closest";
+ evaluateRefs = false;
+ verbose = false;
+
+ int argno = 0;
+
+ while (argno < args.length) {
+ String option = args[argno];
+ if (option.equals("-cand")) {
+ candFileName = args[argno + 1];
+ } else if (option.equals("-format")) {
+ candFileFormat = args[argno + 1];
+ if (!candFileFormat.equals("plain") && !candFileFormat.equals("nbest")) {
+ throw new RuntimeException("candFileFormat must be either plain or nbest.");
+ }
+ } else if (option.equals("-rank")) {
+ candRank = Integer.parseInt(args[argno + 1]);
+ if (refsPerSen < 1) {
+ throw new RuntimeException("Argument for -rank must be positive.");
+ }
+ } else if (option.equals("-ref")) {
+ refFileName = args[argno + 1];
+ } else if (option.equals("-rps")) {
+ refsPerSen = Integer.parseInt(args[argno + 1]);
+ if (refsPerSen < 1) {
+ throw new RuntimeException("refsPerSen must be positive.");
+ }
+ } else if (option.equals("-txtNrm")) {
+ textNormMethod = Integer.parseInt(args[argno + 1]);
+ if (textNormMethod < 0 || textNormMethod > 4) {
+ throw new RuntimeException("textNormMethod should be between 0 and 4");
+ }
+ } else if (option.equals("-m")) {
+ metricName = args[argno + 1];
+ if (EvaluationMetric.knownMetricName(metricName)) {
+ int optionCount = EvaluationMetric.metricOptionCount(metricName);
+ metricOptions = new String[optionCount];
+ for (int opt = 0; opt < optionCount; ++opt) {
+ metricOptions[opt] = args[argno + opt + 2];
+ }
+ argno += optionCount;
+ } else {
+ throw new RuntimeException("Unknown metric name " + metricName + ".");
+ }
+ } else if (option.equals("-evr")) {
+ int evr = Integer.parseInt(args[argno + 1]);
+ if (evr == 1) {
+ evaluateRefs = true;
+ } else if (evr == 0) {
+ evaluateRefs = false;
+ } else {
+ throw new RuntimeException("evalRefs must be either 0 or 1.");
+ }
+ } else if (option.equals("-v")) {
+ int v = Integer.parseInt(args[argno + 1]);
+ if (v == 1) {
+ verbose = true;
+ } else if (v == 0) {
+ verbose = false;
+ } else {
+ throw new RuntimeException("verbose must be either 0 or 1.");
+ }
+ } else {
+ throw new RuntimeException("Unknown option " + option);
+ }
+
+ argno += 2;
+
+ } // while (argno)
+
+ if (refsPerSen > 1) {
+ String refFile = refFileName + "0";
+ if (! new File(refFile).exists())
+ refFile = refFileName + ".0";
+ if (! new File(refFile).exists()) {
+ throw new RuntimeException(String.format("* FATAL: can't find first reference file '%s{0,.0}'", refFileName));
+ }
+
+ numSentences = countLines(refFile);
+ } else {
+ numSentences = countLines(refFileName);
+ }
+
+ // read in reference sentences
+ refSentences = new String[numSentences][refsPerSen];
+
+ try {
+
+ // read in reference sentences
+ BufferedReader reference_readers[] = new BufferedReader[refsPerSen];
+ if (refsPerSen == 1) {
+ reference_readers[0] = new BufferedReader(new InputStreamReader(new FileInputStream(new File(refFileName)), "utf8"));
+ } else {
+ for (int i = 0; i < refsPerSen; i++) {
+ String refFile = refFileName + i;
+ if (! new File(refFile).exists())
+ refFile = refFileName + "." + i;
+ if (! new File(refFile).exists()) {
+ throw new RuntimeException(String.format("* FATAL: can't find reference file '%s'", refFile));
+ }
+
+ reference_readers[i] = new BufferedReader(new InputStreamReader(new FileInputStream(new File(refFile)), "utf8"));
+ }
+ }
+
+ for (int i = 0; i < numSentences; ++i) {
+ for (int r = 0; r < refsPerSen; ++r) {
+ // read the rth reference translation for the ith sentence
+ refSentences[i][r] = normalize(reference_readers[r].readLine(), textNormMethod);
+ }
+ }
+
+ // close all the reference files
+ for (int i = 0; i < refsPerSen; i++)
+ reference_readers[i].close();
+
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ // set static data members for the EvaluationMetric class
+ EvaluationMetric.set_numSentences(numSentences);
+ EvaluationMetric.set_refsPerSen(refsPerSen);
+ EvaluationMetric.set_refSentences(refSentences);
+
+ // do necessary initialization for the evaluation metric
+ evalMetric = EvaluationMetric.getMetric(metricName, metricOptions);
+
+ println("Processing " + numSentences + " sentences...");
+
+ } // processArgsAndInitialize(String[] args)
+
+ private static String normalize(String str, int normMethod) {
+ if (normMethod == 0) return str;
+
+ // replace HTML/SGML
+ str = str.replaceAll(""", "\"");
+ str = str.replaceAll("&", "&");
+ str = str.replaceAll("<", "<");
+ str = str.replaceAll(">", ">");
+ str = str.replaceAll("'", "'");
+
+
+
+ // split on these characters:
+ // ! " # $ % & ( ) * + / : ; < = > ? @ [ \ ] ^ _ ` { | } ~
+ // i.e. ASCII 33-126, except alphanumeric, and except "," "-" "." "'"
+
+ // ! "# $%& ( ) * +/:;<=> ?@ [ \ ] ^_` { | }~
+ String split_on = "!\"#\\$%&\\(\\)\\*\\+/:;<=>\\?@\\[\\\\\\]\\^_`\\{\\|\\}~";
+
+ // println("split_on: " + split_on);
+
+ for (int k = 0; k < split_on.length(); ++k) {
+ // for each split character, reprocess the string
+ String regex = "" + split_on.charAt(k);
+ if (regex.equals("\\")) {
+ ++k;
+ regex += split_on.charAt(k);
+ }
+ str = str.replaceAll(regex, " " + regex + " ");
+ }
+
+
+
+ // split on "." and "," and "-", conditioned on proper context
+
+ str = " " + str + " ";
+ str = str.replaceAll("\\s+", " ");
+
+ TreeSet<Integer> splitIndices = new TreeSet<Integer>();
+
+ for (int i = 0; i < str.length(); ++i) {
+ char ch = str.charAt(i);
+ if (ch == '.' || ch == ',') {
+ // split if either of the previous or next characters is a non-digit
+ char prev_ch = str.charAt(i - 1);
+ char next_ch = str.charAt(i + 1);
+ if (prev_ch < '0' || prev_ch > '9' || next_ch < '0' || next_ch > '9') {
+ splitIndices.add(i);
+ }
+ } else if (ch == '-') {
+ // split if preceded by a digit
+ char prev_ch = str.charAt(i - 1);
+ if (prev_ch >= '0' && prev_ch <= '9') {
+ splitIndices.add(i);
+ }
+ }
+ }
+
+ String str0 = str;
+ str = "";
+
+ for (int i = 0; i < str0.length(); ++i) {
+ if (splitIndices.contains(i)) {
+ str += " " + str0.charAt(i) + " ";
+ } else {
+ str += str0.charAt(i);
+ }
+ }
+
+
+
+ // rejoin i'm, we're, *'s, won't, don't, etc
+
+ str = " " + str + " ";
+ str = str.replaceAll("\\s+", " ");
+
+ str = str.replaceAll(" i 'm ", " i'm ");
+ str = str.replaceAll(" we 're ", " we're ");
+ str = str.replaceAll(" 's ", "'s ");
+ str = str.replaceAll(" 've ", "'ve ");
+ str = str.replaceAll(" 'll ", "'ll ");
+ str = str.replaceAll(" 'd ", "'d ");
+ str = str.replaceAll(" n't ", "n't ");
+
+
+
+ // remove spaces around dashes
+ if (normMethod == 2 || normMethod == 4) {
+
+ TreeSet<Integer> skipIndices = new TreeSet<Integer>();
+ str = " " + str + " ";
+
+ for (int i = 0; i < str.length(); ++i) {
+ char ch = str.charAt(i);
+ if (ch == '-') {
+ // rejoin if surrounded by spaces, and then letters
+ if (str.charAt(i - 1) == ' ' && str.charAt(i + 1) == ' ') {
+ if (Character.isLetter(str.charAt(i - 2)) && Character.isLetter(str.charAt(i + 2))) {
+ skipIndices.add(i - 1);
+ skipIndices.add(i + 1);
+ }
+ }
+ }
+ }
+
+ str0 = str;
+ str = "";
+
+ for (int i = 0; i < str0.length(); ++i) {
+ if (!skipIndices.contains(i)) {
+ str += str0.charAt(i);
+ }
+ }
+ }
+
+
+
+ // drop non-ASCII characters
+ if (normMethod == 3 || normMethod == 4) {
+
+ str0 = str;
+ str = "";
+
+ for (int i = 0; i < str0.length(); ++i) {
+ char ch = str0.charAt(i);
+ if (ch <= 127) { // i.e. if ASCII
+ str += ch;
+ }
+ }
+ }
+
+
+
+ str = str.replaceAll("\\s+", " ");
+
+ str = str.trim();
+
+ return str;
+ }
+
+ // TODO: we should handle errors properly for the three use sites of this function, and should
+ // remove the function.
+ // OK, but we don't want it to use LineReader, so it can function within the standalone release of
+ // Z-MERT. -- O.Z.
+ private static int countLines(String fileName) {
+ int count = 0;
+
+ try {
+ BufferedReader inFile = new BufferedReader(new FileReader(fileName));
+
+ String line;
+ do {
+ line = inFile.readLine();
+ if (line != null) ++count;
+ } while (line != null);
+
+ inFile.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ return count;
+ }
+
+
+ private static void println(Object obj) {
+ System.out.println(obj);
+ }
+
+ private static void print(Object obj) {
+ System.out.print(obj);
+ }
+
+ public static void main(String[] args) {
+ if (args.length == 0) {
+ printUsage(args.length);
+ System.exit(0);
+ } else {
+ processArgsAndInitialize(args);
+ }
+ // non-specified args will be set to default values in processArgsAndInitialize
+
+ if (candFileFormat.equals("plain")) {
+ println("Evaluating candidate translations in plain file " + candFileName + "...");
+ evaluateCands_plain(candFileName);
+ } else if (candFileFormat.equals("nbest")) {
+ println("Evaluating set of " + candRank + "'th candidate translations from " + candFileName
+ + "...");
+ evaluateCands_nbest(candFileName, candRank);
+ }
+ println("");
+
+ if (evaluateRefs) {
+ // evaluate the references themselves; useful if developing a new evaluation metric
+
+ println("");
+ println("PERFORMING SANITY CHECK:");
+ println("------------------------");
+ println("");
+ println("This metric's scores range from " + evalMetric.worstPossibleScore() + " (worst) to "
+ + evalMetric.bestPossibleScore() + " (best).");
+
+ for (int r = 1; r <= refsPerSen; ++r) {
+ println("");
+ println("(*) Evaluating reference set " + r + ":");
+ println("");
+ evaluateRefSet(r);
+ println("");
+ }
+ }
+
+ // System.exit(0);
+
+ } // main(String[] args)
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/ListUtil.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/ListUtil.java b/joshua-core/src/main/java/org/apache/joshua/util/ListUtil.java
new file mode 100644
index 0000000..afb5af1
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/ListUtil.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util;
+
+import java.util.List;
+
+public class ListUtil {
+
+ /**
+ * Static method to generate a list representation for an ArrayList of Strings S1,...,Sn
+ *
+ * @param list A list of Strings
+ * @return A String consisting of the original list of strings concatenated and separated by
+ * commas, and enclosed by square brackets i.e. '[S1,S2,...,Sn]'
+ */
+ public static String stringListString(List<String> list) {
+
+ String result = "[";
+ for (int i = 0; i < list.size() - 1; i++) {
+ result += list.get(i) + ",";
+ }
+
+ if (list.size() > 0) {
+ // get the generated word for the last target position
+ result += list.get(list.size() - 1);
+ }
+
+ result += "]";
+
+ return result;
+
+ }
+
+ public static <E> String objectListString(List<E> list) {
+ String result = "[";
+ for (int i = 0; i < list.size() - 1; i++) {
+ result += list.get(i) + ",";
+ }
+ if (list.size() > 0) {
+ // get the generated word for the last target position
+ result += list.get(list.size() - 1);
+ }
+ result += "]";
+ return result;
+ }
+
+ /**
+ * Static method to generate a simple concatenated representation for an ArrayList of Strings
+ * S1,...,Sn
+ *
+ * @param list A list of Strings
+ * @return todo
+ */
+ public static String stringListStringWithoutBrackets(List<String> list) {
+ return stringListStringWithoutBracketsWithSpecifiedSeparator(list, " ");
+ }
+
+ public static String stringListStringWithoutBracketsCommaSeparated(List<String> list) {
+ return stringListStringWithoutBracketsWithSpecifiedSeparator(list, ",");
+ }
+
+ public static String stringListStringWithoutBracketsWithSpecifiedSeparator(List<String> list,
+ String separator) {
+
+ String result = "";
+ for (int i = 0; i < list.size() - 1; i++) {
+ result += list.get(i) + separator;
+ }
+
+ if (list.size() > 0) {
+ // get the generated word for the last target position
+ result += list.get(list.size() - 1);
+ }
+
+ return result;
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/Lists.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/Lists.java b/joshua-core/src/main/java/org/apache/joshua/util/Lists.java
new file mode 100644
index 0000000..d62d1aa
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/Lists.java
@@ -0,0 +1,567 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util;
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+/**
+ *
+ *
+ * @author Lane Schwartz
+ */
+public class Lists {
+
+ // public static void main(String[] args) {
+ //
+ // int[] list = {100, 200, 300, 400, 500};
+ //
+ // for (IndexedInt i : eachWithIndex(list)) {
+ //
+ // System.out.println(i.getIndex() + " " + i.getValue());
+ //
+ // }
+ //
+ // Integer[] list2 = new Integer[]{10, 20, 30, 40};
+ // for (Indexed<Integer> i : eachWithIndex(list2)) {
+ //
+ // System.out.println(i.getIndex() + " " + i.getValue());
+ //
+ // }
+ //
+ // java.util.List<Integer> list3 = new java.util.ArrayList<Integer>();
+ // for (int i : list2) { list3.add(i); }
+ //
+ // for (Indexed<Integer> i : eachWithIndex(list3)) {
+ //
+ // System.out.println(i.getIndex() + " " + i.getValue());
+ //
+ // }
+ // }
+
+
+ public static Iterable<Integer> upto(final int exclusiveUpperBound) {
+ return new Iterable<Integer>() {
+ public Iterator<Integer> iterator() {
+ return new Iterator<Integer>() {
+ int next = 0;
+
+ public boolean hasNext() {
+ return next < exclusiveUpperBound;
+ }
+
+ public Integer next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ int result = next;
+ next += 1;
+ return result;
+ }
+
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ };
+ }
+
+ };
+ }
+
+ public static Iterable<IndexedByte> eachWithIndex(final byte[] list) {
+
+ return new Iterable<IndexedByte>() {
+
+ public Iterator<IndexedByte> iterator() {
+ return new Iterator<IndexedByte>() {
+
+ int nextIndex = -1;
+ IndexedByte indexedValue;
+
+ public boolean hasNext() {
+ return (nextIndex < list.length);
+ }
+
+ public IndexedByte next() {
+ if (nextIndex >= list.length) {
+ throw new NoSuchElementException();
+ } else if (nextIndex < 0) {
+ nextIndex = 0;
+ indexedValue = new IndexedByte(list[nextIndex], nextIndex);
+ } else {
+ indexedValue.value = list[nextIndex];
+ indexedValue.index = nextIndex;
+ }
+
+ nextIndex += 1;
+ return indexedValue;
+ }
+
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ };
+ }
+
+ };
+ }
+
+ public static Iterable<IndexedShort> eachWithIndex(final short[] list) {
+
+ return new Iterable<IndexedShort>() {
+
+ public Iterator<IndexedShort> iterator() {
+ return new Iterator<IndexedShort>() {
+
+ int nextIndex = -1;
+ IndexedShort indexedValue;
+
+ public boolean hasNext() {
+ return (nextIndex < list.length);
+ }
+
+ public IndexedShort next() {
+ if (nextIndex >= list.length) {
+ throw new NoSuchElementException();
+ } else if (nextIndex < 0) {
+ nextIndex = 0;
+ indexedValue = new IndexedShort(list[nextIndex], nextIndex);
+ } else {
+ indexedValue.value = list[nextIndex];
+ indexedValue.index = nextIndex;
+ }
+
+ nextIndex += 1;
+ return indexedValue;
+ }
+
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ };
+ }
+
+ };
+ }
+
+ public static Iterable<IndexedInt> eachWithIndex(final int[] list) {
+
+ return new Iterable<IndexedInt>() {
+
+ public Iterator<IndexedInt> iterator() {
+ return new Iterator<IndexedInt>() {
+
+ int nextIndex = -1;
+ IndexedInt indexedValue;
+
+ public boolean hasNext() {
+ return (nextIndex < list.length);
+ }
+
+ public IndexedInt next() {
+ if (nextIndex >= list.length) {
+ throw new NoSuchElementException();
+ } else if (nextIndex < 0) {
+ nextIndex = 0;
+ indexedValue = new IndexedInt(list[nextIndex], nextIndex);
+ } else {
+ indexedValue.value = list[nextIndex];
+ indexedValue.index = nextIndex;
+ }
+
+ nextIndex += 1;
+ return indexedValue;
+ }
+
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ };
+ }
+
+ };
+ }
+
+ public static Iterable<IndexedLong> eachWithIndex(final long[] list) {
+
+ return new Iterable<IndexedLong>() {
+
+ public Iterator<IndexedLong> iterator() {
+ return new Iterator<IndexedLong>() {
+
+ int nextIndex = -1;
+ IndexedLong indexedValue;
+
+ public boolean hasNext() {
+ return (nextIndex < list.length);
+ }
+
+ public IndexedLong next() {
+ if (nextIndex >= list.length) {
+ throw new NoSuchElementException();
+ } else if (nextIndex < 0) {
+ nextIndex = 0;
+ indexedValue = new IndexedLong(list[nextIndex], nextIndex);
+ } else {
+ indexedValue.value = list[nextIndex];
+ indexedValue.index = nextIndex;
+ }
+
+ nextIndex += 1;
+ return indexedValue;
+ }
+
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ };
+ }
+
+ };
+ }
+
+ public static Iterable<IndexedFloat> eachWithIndex(final float[] list) {
+
+ return new Iterable<IndexedFloat>() {
+
+ public Iterator<IndexedFloat> iterator() {
+ return new Iterator<IndexedFloat>() {
+
+ int nextIndex = -1;
+ IndexedFloat indexedValue;
+
+ public boolean hasNext() {
+ return (nextIndex < list.length);
+ }
+
+ public IndexedFloat next() {
+ if (nextIndex >= list.length) {
+ throw new NoSuchElementException();
+ } else if (nextIndex < 0) {
+ nextIndex = 0;
+ indexedValue = new IndexedFloat(list[nextIndex], nextIndex);
+ } else {
+ indexedValue.value = list[nextIndex];
+ indexedValue.index = nextIndex;
+ }
+
+ nextIndex += 1;
+ return indexedValue;
+ }
+
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ };
+ }
+
+ };
+ }
+
+ public static Iterable<IndexedDouble> eachWithIndex(final double[] list) {
+
+ return new Iterable<IndexedDouble>() {
+
+ public Iterator<IndexedDouble> iterator() {
+ return new Iterator<IndexedDouble>() {
+
+ int nextIndex = -1;
+ IndexedDouble indexedValue;
+
+ public boolean hasNext() {
+ return (nextIndex < list.length);
+ }
+
+ public IndexedDouble next() {
+ if (nextIndex >= list.length) {
+ throw new NoSuchElementException();
+ } else if (nextIndex < 0) {
+ nextIndex = 0;
+ indexedValue = new IndexedDouble(list[nextIndex], nextIndex);
+ } else {
+ indexedValue.value = list[nextIndex];
+ indexedValue.index = nextIndex;
+ }
+
+ nextIndex += 1;
+ return indexedValue;
+ }
+
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ };
+ }
+
+ };
+ }
+
+ public static <V> Iterable<Indexed<V>> eachWithIndex(final V[] list) {
+ return new Iterable<Indexed<V>>() {
+
+ public Iterator<Indexed<V>> iterator() {
+ return new Iterator<Indexed<V>>() {
+
+ int nextIndex = -1;
+ Indexed<V> indexedValue;
+
+ public boolean hasNext() {
+ return (nextIndex < list.length);
+ }
+
+ public Indexed<V> next() {
+ if (nextIndex >= list.length) {
+ throw new NoSuchElementException();
+ } else if (nextIndex < 0) {
+ nextIndex = 0;
+ indexedValue = new Indexed<V>(list[nextIndex], nextIndex);
+ } else {
+ indexedValue.value = list[nextIndex];
+ indexedValue.index = nextIndex;
+ }
+
+ nextIndex += 1;
+ return indexedValue;
+ }
+
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ };
+ }
+
+ };
+ }
+
+ public static <V> Iterable<Indexed<V>> eachWithIndex(final Iterator<V> valueIterator) {
+ return new Iterable<Indexed<V>>() {
+
+ public Iterator<Indexed<V>> iterator() {
+ return new Iterator<Indexed<V>>() {
+
+ int nextIndex = -1;
+ Indexed<V> indexedValue;
+
+ public boolean hasNext() {
+ return valueIterator.hasNext();
+ }
+
+ public Indexed<V> next() {
+ if (!valueIterator.hasNext()) {
+ throw new NoSuchElementException();
+ } else if (nextIndex < 0) {
+ nextIndex = 0;
+ indexedValue = new Indexed<V>(valueIterator.next(), nextIndex);
+ } else {
+ indexedValue.value = valueIterator.next();
+ indexedValue.index = nextIndex;
+ }
+
+ nextIndex += 1;
+ return indexedValue;
+ }
+
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ };
+ }
+
+ };
+ }
+
+ public static <V> Iterable<Indexed<V>> eachWithIndex(final Iterable<V> iterable) {
+ return eachWithIndex(iterable.iterator());
+ }
+
+
+ public static class Index {
+
+ int index;
+
+ Index(int index) {
+ this.index = index;
+ }
+
+ public int getIndex() {
+ return this.index;
+ }
+
+ void setIndex(int index) {
+ this.index = index;
+ }
+ }
+
+ public static class IndexedBoolean extends Index {
+
+ boolean value;
+
+ IndexedBoolean(boolean value, int index) {
+ super(index);
+ this.value = value;
+ }
+
+ public boolean getValue() {
+ return this.value;
+ }
+
+ void setValue(boolean value) {
+ this.value = value;
+ this.index += 1;
+ }
+ }
+
+ public static class IndexedByte extends Index {
+
+ byte value;
+
+ IndexedByte(byte value, int index) {
+ super(index);
+ this.value = value;
+ }
+
+ public byte getValue() {
+ return this.value;
+ }
+
+ void setValue(byte value) {
+ this.value = value;
+ this.index += 1;
+ }
+ }
+
+ public static class IndexedShort extends Index {
+
+ short value;
+
+ IndexedShort(short value, int index) {
+ super(index);
+ this.value = value;
+ }
+
+ public short getValue() {
+ return this.value;
+ }
+
+ void setValue(short value) {
+ this.value = value;
+ this.index += 1;
+ }
+ }
+
+ public static class IndexedInt extends Index {
+
+ int value;
+
+ IndexedInt(int value, int index) {
+ super(index);
+ this.value = value;
+ }
+
+ public int getValue() {
+ return this.value;
+ }
+
+ void setValue(int value) {
+ this.value = value;
+ this.index += 1;
+ }
+ }
+
+ public static class IndexedLong extends Index {
+
+ long value;
+
+ IndexedLong(long value, int index) {
+ super(index);
+ this.value = value;
+ }
+
+ public long getValue() {
+ return this.value;
+ }
+
+ void setValue(long value) {
+ this.value = value;
+ this.index += 1;
+ }
+ }
+
+ public static class IndexedFloat extends Index {
+
+ float value;
+
+ IndexedFloat(float value, int index) {
+ super(index);
+ this.value = value;
+ }
+
+ public float getValue() {
+ return this.value;
+ }
+
+ void setValue(float value) {
+ this.value = value;
+ this.index += 1;
+ }
+ }
+
+ public static class IndexedDouble extends Index {
+
+ double value;
+
+ IndexedDouble(double value, int index) {
+ super(index);
+ this.value = value;
+ }
+
+ public double getValue() {
+ return this.value;
+ }
+
+ void setValue(double value) {
+ this.value = value;
+ this.index += 1;
+ }
+ }
+
+
+ public static class Indexed<V> extends Index {
+
+ V value;
+
+ Indexed(V value, int index) {
+ super(index);
+ this.value = value;
+ }
+
+ public V getValue() {
+ return this.value;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/NBestListUtility.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/NBestListUtility.java b/joshua-core/src/main/java/org/apache/joshua/util/NBestListUtility.java
new file mode 100644
index 0000000..08c85ba
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/NBestListUtility.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Methods for extracting information from an NBest List
+ *
+ * @author Gideon Maillette de Buy Wenniger
+ *
+ */
+public class NBestListUtility {
+ private static final String JOSHUA_SEPARATOR = "|||";
+
+ // See : http://www.regular-expressions.info/lookaround.html
+ public static String featureFunctionMatchingRegularExpression(String featureFunctionName) {
+ String result = featureFunctionName + ".+?" + "(?=\\=)";
+ return result;
+ }
+
+ public static List<String> findAllFeatureOccurences(String contentsString,
+ String featureFunctionPrefix) {
+ List<String> allMatches = findAllMatches(
+ featureFunctionMatchingRegularExpression(featureFunctionPrefix), contentsString);
+ return allMatches;
+ }
+
+ public static List<String> findAllMatches(String regularExpression, String contentsString) {
+ List<String> allMatches = new ArrayList<String>();
+ Matcher m = Pattern.compile(regularExpression).matcher(contentsString);
+ while (m.find()) {
+ allMatches.add(m.group());
+ }
+ return allMatches;
+ }
+
+ public static Double getTotalWeightFromNBestLine(String nBestLine) {
+ int firstIndexWeightSubstring = nBestLine.lastIndexOf(JOSHUA_SEPARATOR)
+ + JOSHUA_SEPARATOR.length();
+ String weightSubstring = nBestLine.substring(firstIndexWeightSubstring);
+ return Double.parseDouble(weightSubstring);
+ }
+
+ public static List<Double> getTotalWeightsFromNBestListString(String nBestListAsString) {
+ List<Double> result = new ArrayList<Double>();
+ String[] lines = nBestListAsString.split("\n");
+ for (String line : lines) {
+ result.add(getTotalWeightFromNBestLine(line));
+ }
+ return result;
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/Ngram.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/Ngram.java b/joshua-core/src/main/java/org/apache/joshua/util/Ngram.java
new file mode 100644
index 0000000..73909ce
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/Ngram.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.joshua.corpus.Vocabulary;
+
+/**
+ * Provides convenience functions for extracting all ngrams from a sentence, represented as an array
+ * of words.
+ */
+public class Ngram {
+
+ public static void getNgrams(Map<String, Integer> tbl, int startOrder, int endOrder,
+ final int[] wrds) {
+
+ for (int i = 0; i < wrds.length; i++)
+ for (int j = startOrder - 1; j < endOrder && j + i < wrds.length; j++) {// ngram: [i,i+j]
+ StringBuffer ngram = new StringBuffer();
+ for (int k = i; k <= i + j; k++) {
+ int t_wrd = wrds[k];
+ ngram.append(Vocabulary.word(t_wrd));
+ if (k < i + j)
+ ngram.append(" ");
+ }
+ String ngramStr = ngram.toString();
+ increaseCount(tbl, ngramStr, 1);
+ }
+ }
+
+ /**
+ * If symbolTbl!=null, then convert interger to String
+ * @param tbl todo
+ * @param startOrder todo
+ * @param endOrder todo
+ * @param wrds todo
+ */
+ public static void getNgrams(Map<String, Integer> tbl, int startOrder, int endOrder,
+ final List<Integer> wrds) {
+
+ for (int i = 0; i < wrds.size(); i++)
+ for (int j = startOrder - 1; j < endOrder && j + i < wrds.size(); j++) {// ngram: [i,i+j]
+ StringBuffer ngram = new StringBuffer();
+ for (int k = i; k <= i + j; k++) {
+ int t_wrd = wrds.get(k);
+ ngram.append(Vocabulary.word(t_wrd));
+ if (k < i + j)
+ ngram.append(" ");
+ }
+ String ngramStr = ngram.toString();
+ increaseCount(tbl, ngramStr, 1);
+ }
+ }
+
+ /**
+ * If symbolTbl!=null, then convert string to integer
+ * @param tbl todo
+ * @param startOrder todo
+ * @param endOrder todo
+ * @param wrds todo
+ */
+ public static void getNgrams(Map<String, Integer> tbl, int startOrder, int endOrder,
+ final String[] wrds) {
+
+ for (int i = 0; i < wrds.length; i++)
+ for (int j = startOrder - 1; j < endOrder && j + i < wrds.length; j++) {// ngram: [i,i+j]
+ StringBuffer ngram = new StringBuffer();
+ for (int k = i; k <= i + j; k++) {
+ String t_wrd = wrds[k];
+ ngram.append(t_wrd);
+ if (k < i + j)
+ ngram.append(" ");
+ }
+ String ngramStr = ngram.toString();
+ increaseCount(tbl, ngramStr, 1);
+ }
+ }
+
+ static private void increaseCount(Map<String, Integer> tbl, String feat, int increment) {
+ Integer oldCount = tbl.get(feat);
+ if (oldCount != null)
+ tbl.put(feat, oldCount + increment);
+ else
+ tbl.put(feat, increment);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/NullIterator.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/NullIterator.java b/joshua-core/src/main/java/org/apache/joshua/util/NullIterator.java
new file mode 100644
index 0000000..c6e4b46
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/NullIterator.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util;
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+
+/**
+ * This class provides a null-object Iterator. That is, an iterator over an empty collection.
+ *
+ * @author wren ng thornton wren@users.sourceforge.net
+ * @version $LastChangedDate: 2009-03-26 15:06:57 -0400 (Thu, 26 Mar 2009) $
+ */
+public class NullIterator<E> implements Iterable<E>, Iterator<E> {
+
+ // ===============================================================
+ // Iterable -- for foreach loops, because sometimes Java can be very stupid
+ // ===============================================================
+
+ /**
+ * Return self as an iterator. We restrict the return type because some code is written to accept
+ * both Iterable and Iterator, and the fact that we are both confuses Java. So this is just an
+ * upcast, but more succinct to type.
+ */
+ public Iterator<E> iterator() {
+ return this;
+ }
+
+
+ // ===============================================================
+ // Iterator
+ // ===============================================================
+
+ /** Always returns false. */
+ public boolean hasNext() {
+ return false;
+ }
+
+ /** Always throws {@link NoSuchElementException}. */
+ public E next() throws NoSuchElementException {
+ throw new NoSuchElementException();
+ }
+
+ /** Unsupported. */
+ public void remove() throws UnsupportedOperationException {
+ throw new UnsupportedOperationException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/PackedGrammarServer.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/PackedGrammarServer.java b/joshua-core/src/main/java/org/apache/joshua/util/PackedGrammarServer.java
new file mode 100644
index 0000000..74c8e4a
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/PackedGrammarServer.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.ff.tm.Trie;
+import org.apache.joshua.decoder.ff.tm.packed.PackedGrammar;
+import org.apache.joshua.util.io.LineReader;
+
+public class PackedGrammarServer {
+
+ private PackedGrammar grammar;
+
+ public PackedGrammarServer(String packed_directory,JoshuaConfiguration joshuaConfiguration) throws FileNotFoundException, IOException {
+ grammar = new PackedGrammar(packed_directory, -1, "owner", "thrax", joshuaConfiguration);
+ }
+
+ public List<Rule> get(String source) {
+ return get(source.trim().split("\\s+"));
+ }
+
+ public List<Rule> get(String[] source) {
+ int[] src = Vocabulary.addAll(source);
+ Trie walker = grammar.getTrieRoot();
+ for (int s : src) {
+ walker = walker.match(s);
+ if (walker == null)
+ return null;
+ }
+ return walker.getRuleCollection().getRules();
+ }
+
+ public Map<String, Float> scores(String source, String target) {
+ return scores(source.trim().split("\\s+"), target.trim().split("\\s+"));
+ }
+
+ public Map<String, Float> scores(String[] source, String[] target) {
+ List<Rule> rules = get(source);
+
+ if (rules == null)
+ return null;
+
+ int[] tgt = Vocabulary.addAll(target);
+ for (Rule r : rules)
+ if (Arrays.equals(tgt, r.getEnglish()))
+ return r.getFeatureVector().getMap();
+
+ return null;
+ }
+
+
+ public static void main(String[] args) throws FileNotFoundException, IOException {
+ JoshuaConfiguration joshuaConfiguration = new JoshuaConfiguration();
+ PackedGrammarServer pgs = new PackedGrammarServer(args[0], joshuaConfiguration);
+
+ for (String line: new LineReader(System.in)) {
+ List<Rule> rules = pgs.get(line);
+ if (rules == null) continue;
+ for (Rule r : rules)
+ System.out.println(r.toString());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/Pair.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/Pair.java b/joshua-core/src/main/java/org/apache/joshua/util/Pair.java
new file mode 100644
index 0000000..2dd536d
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/Pair.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util;
+
+/**
+ * Represents a pair of elements.
+ *
+ * @author Lane Schwartz
+ * @version $LastChangedDate$
+ *
+ * @param <First> Type of the first element in the pair.
+ * @param <Second> Type of the second element in the pair.
+ */
+public class Pair<First, Second> {
+
+ /** The first element of the pair. */
+ public First first;
+
+ /** The second element of the pair. */
+ public Second second;
+
+ private Integer hashCode = null;
+
+ /**
+ * Constructs a pair of elements.
+ *
+ * @param first the first element in the pair
+ * @param second the second element in the pair
+ */
+ public Pair(First first, Second second) {
+ this.first = first;
+ this.second = second;
+ }
+
+ /**
+ * Gets the second element in the pair
+ *
+ * @return the first element in the pair
+ */
+ public First getFirst() {
+ return first;
+ }
+
+ /**
+ * Sets the first element in the pair.
+ *
+ * @param first the new value for the first element in the pair
+ */
+ public void setFirst(First first) {
+ this.first = first;
+ }
+
+ /**
+ * Gets the second element in the pair.
+ *
+ * @return the second element in the pair
+ */
+ public Second getSecond() {
+ return second;
+ }
+
+ /**
+ * Sets the second element in the pair.
+ *
+ * @param second the new value for the second element in the pair
+ */
+ public void setSecond(Second second) {
+ this.second = second;
+ }
+
+
+ public int hashCode() {
+
+ if (hashCode == null) {
+ if (first == null) {
+ if (second == null) {
+ hashCode = 0;
+ } else {
+ hashCode = second.hashCode();
+ }
+ } else if (second == null) {
+ hashCode = first.hashCode();
+ } else {
+ hashCode = first.hashCode() + 37 * second.hashCode();
+ }
+ }
+
+ return hashCode;
+ }
+
+ @SuppressWarnings("unchecked")
+ public boolean equals(Object o) {
+ if (o instanceof Pair<?, ?>) {
+
+ Pair<First, Second> other = (Pair<First, Second>) o;
+
+ if (first == null) {
+ if (second == null) {
+ return other.first == null && other.second == null;
+ } else {
+ return other.first == null && second.equals(other.second);
+ }
+ } else if (second == null) {
+ return first.equals(other.first) && other.second == null;
+ } else {
+ return first.equals(other.first) && second.equals(other.second);
+ }
+
+ } else {
+ return false;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/Platform.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/Platform.java b/joshua-core/src/main/java/org/apache/joshua/util/Platform.java
new file mode 100644
index 0000000..22089da
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/Platform.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util;
+
+public class Platform {
+
+ public static boolean isMac() {
+ return System.getProperties().getProperty("os.name").toLowerCase().indexOf("mac") != -1;
+ }
+
+}
[09/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/hiero/output.scores.gold
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/hiero/output.scores.gold b/joshua-core/src/test/resources/bn-en/hiero/output.scores.gold
new file mode 100644
index 0000000..a73761a
--- /dev/null
+++ b/joshua-core/src/test/resources/bn-en/hiero/output.scores.gold
@@ -0,0 +1,805 @@
+ rabindranath was born in kolkata a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| -227.217
+ rabindranath was born in kolkata in a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| -228.133
+ rabindranath was born in kolkata one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| -228.160
+ rabindranath born in the a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| -228.388
+ rabindranath born in the one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| -228.823
+ rabindranath 's birth was the a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| -228.862
+ rabindranath was born in kolkata in \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| -228.892
+ rabindranath 's birth in the a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| -229.004
+ rabindranath was born in kolkata a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 \u09aa\u09b0\u09bf\u09ac\u09be\u09b0\u09c7 . ||| -321.208
+ rabindranath was born in kolkata in a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 \u09aa\u09b0\u09bf\u09ac\u09be\u09b0\u09c7 . ||| -322.124
+ recently with united states with the relationship between improved . ||| -21.022
+ recently with united states with the relation between improved . ||| -21.210
+ in recent times india with united states relationship between improved . ||| -22.055
+ recently india with united states relationship between improved . ||| -22.129
+ in recent times with united states with the relationship between improved . ||| -22.439
+ recently with united states with the relation improved . ||| -22.538
+ in recent times in india with united states relationship between improved . ||| -22.811
+ mathematics so science language . ||| -10.471
+ mathematics is science language . ||| -10.543
+ mathematics that science language . ||| -11.587
+ science mathematics that language . ||| -12.065
+ from this it can be understood easily that this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf will be \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| -322.982
+ it can be understood from this easily that this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf will be \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| -322.995
+ from this it can be understood easily that in this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf will be \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| -322.996
+ it can be understood from this easily that in this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf will be \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| -323.010
+ it can be understood from this can easily that this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf will be \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| -323.095
+ it can be understood from this can easily that in this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf will be \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| -323.109
+ from this it will be it can be understood that in this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| -323.442
+ from this it it can be understood that this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf will be \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| -323.453
+ from this it will be it can be understood that the \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| -323.498
+ from this it it can be understood that in this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf will be \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| -323.521
+ the same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match from this novel . ||| -236.836
+ the same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 presage match from this novel . ||| -236.965
+ in the same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match from this novel . ||| -237.005
+ in the same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 presage match from this novel . ||| -237.135
+ in the same along with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match from this novel . ||| -237.997
+ in the same along with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 presage match from this novel . ||| -238.127
+ the same with of bengal earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match from this novel . ||| -238.469
+ the same with of bengal earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 presage match from this novel . ||| -238.599
+ in the same with of bengal earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match from this novel . ||| -238.638
+ the same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match from this novels . ||| -238.721
+ \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 majority in the . ||| -217.895
+ \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 majority that the . ||| -220.097
+ \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 majority where the . ||| -220.171
+ \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 single absolute majority in the . ||| -220.174
+ \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 the . majority in ||| -224.035
+ \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his work with to that . ||| -432.357
+ \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his work with can and pajama . ||| -433.284
+ \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his work with to and pajama . ||| -433.453
+ \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his work with can that . ||| -433.520
+ \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his work with that can . ||| -433.577
+ \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his work with a that . ||| -433.974
+ \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his work with a and pajama . ||| -434.091
+ \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 in his work with to that . ||| -434.368
+ \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 in his work with can and pajama . ||| -435.295
+ \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 in his work with to and pajama . ||| -435.464
+ other than task \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character there are but these are very is not . ||| -246.114
+ other than task \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and some more linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character there are but these are very is not . ||| -246.373
+ other than task \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character is but these are very is not . ||| -246.508
+ other than task \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and some more linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character is but these are very is not . ||| -246.767
+ other than task \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character there is but these are very is not . ||| -246.771
+ other than task \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and some more linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character there is but these are very is not . ||| -247.030
+ other than task \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and more some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character there are but these are very is not . ||| -247.032
+ other than task \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and more some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character is but these are very is not . ||| -247.426
+ other than task \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character there are but these are very not common . ||| -247.458
+ other than task \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character there are but these very is not . ||| -247.572
+ it social situation in \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the fingers . ||| -256.668
+ it social situation in \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the is . ||| -257.068
+ this social situation in \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the fingers . ||| -257.080
+ it social situation in \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the in the . ||| -257.139
+ this social situation in \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the is . ||| -257.480
+ it social situation in \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the the . ||| -257.520
+ this social situation in \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the in the . ||| -257.551
+ it social situation in \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rules of the fingers . ||| -257.705
+ it social situation in \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with the \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the fingers . ||| -257.756
+ this social situation in \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the the . ||| -257.932
+ \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 annual \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf meters ||| -321.077
+ \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 annual \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf meter ||| -321.092
+ \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 arrange \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf meters ||| -321.673
+ \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 arrange \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf meter ||| -321.689
+ \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 annual \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf metres ||| -322.768
+ \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 arrange \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf metres ||| -323.365
+ on the year 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in the city in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f the national he was the main speech -lrb- keynote speech -rrb- on the . ||| -664.033
+ on the year 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in the city in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f the national he was the main speech -lrb- keynote speech -rrb- to the . ||| -664.104
+ on the year 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in the city in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f the national he was the main speech -lrb- keynote speech -rrb- , the . ||| -664.430
+ on the year 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in the city in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f as the national he was the main speech -lrb- keynote speech -rrb- on the . ||| -664.633
+ on the year 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in the city in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f as the national he was the main speech -lrb- keynote speech -rrb- to the . ||| -664.703
+ on the year 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in the city in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f as the national he was the main speech -lrb- keynote speech -rrb- , the . ||| -665.029
+ on the year 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 held in the city \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f as the national he was the main speech -lrb- keynote speech -rrb- on the . ||| -665.105
+ on the year 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 held in the city as the national \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f he was the main speech -lrb- keynote speech -rrb- on the . ||| -665.144
+ on the year 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in the city in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f the national was he main speech -lrb- keynote speech -rrb- on the . ||| -665.170
+ on the year 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 held in the city \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f as the national he was the main speech -lrb- keynote speech -rrb- to the . ||| -665.175
+ based on power distribution of population east pakistan where to west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac they started the where the whole of west pakistan a province was considered as . ||| -187.213
+ based on power distribution of population east pakistan where to west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac they started the where the whole of west pakistan a province was considered as a . ||| -187.302
+ based on power distribution of population east pakistan where to west pakistan " one unit theory , " is a \u0985\u09ad\u09bf\u09a8\u09ac they started the where the whole of west pakistan a province was considered as . ||| -187.326
+ based on power distribution of population east pakistan where to west pakistan " one unit theory , " is a \u0985\u09ad\u09bf\u09a8\u09ac they started the where the whole of west pakistan a province was considered as a . ||| -187.415
+ based on power distribution of population east pakistan where to west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac started with the where the whole of west pakistan a province was considered as . ||| -187.500
+ based on power distribution of population east pakistan where to west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac started with the where the whole of west pakistan a province was considered as a . ||| -187.589
+ based on power distribution of population east pakistan where to west pakistan " one unit theory , " is a \u0985\u09ad\u09bf\u09a8\u09ac started with the where the whole of west pakistan a province was considered as . ||| -187.612
+ based on power distribution of population east pakistan where to west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac they started to the where the whole of west pakistan a province was considered as . ||| -187.644
+ based on power distribution of population east pakistan where to west pakistan " one unit theory , " is a \u0985\u09ad\u09bf\u09a8\u09ac started with the where the whole of west pakistan a province was considered as a . ||| -187.701
+ based on power distribution of population east pakistan where to west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac they started to the where the whole of west pakistan a province was considered as a . ||| -187.733
+ the \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 theory ||| -110.415
+ the theory of \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 ||| -110.598
+ mathematical \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 theory ||| -110.707
+ \u2022 \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 theory ||| -111.160
+ . \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 theory ||| -111.550
+ mathematical theory of \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 ||| -112.488
+ external links of ||| -4.318
+ external link of ||| -5.587
+ outer link of ||| -5.674
+ external communication of ||| -5.747
+ outer communication of ||| -6.128
+ tata communicationer " foreign sanchar nigam limited building this is one of the main providers in telecommunication system of the city ||| -49.965
+ tata communicationer " foreign sanchar nigam limited building it is the telecommunication system is a one of the main providers ||| -50.012
+ tata communication " foreign sanchar nigam limited building this is one of the main providers in telecommunication system of the city ||| -50.098
+ tata communicationer " foreign sanchar nigam limited building it in the city telecommunication system is a one of the main providers ||| -50.441
+ tata communicationer " foreign sanchar nigam limited building this is one of the main providers in telecommunication system in the city ||| -50.456
+ tata communication " foreign sanchar nigam limited building this is one of the main providers in telecommunication system in the city ||| -50.589
+ tata communicationer " foreign sanchar nigam limited building it telecommunication system of the city is a one of the main providers ||| -50.709
+ tata communicationer " foreign sanchar nigam limited building it is the telecommunication system a one of the main providers ||| -50.735
+ tata communicationer " foreign sanchar nigam limited building this is one of the main providers in telecommunication system of the city is ||| -50.780
+ tata communicationer " foreign sanchar nigam limited building it is in the city telecommunication system is a one of the main providers ||| -50.893
+ he that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national assembly in election won all and was elected as the 44th president of the united states . ||| -241.090
+ he that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national assembly in the won all and was elected as the 44th president of the united states . ||| -241.262
+ in the same year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national assembly in election won all and was elected as the 44th president of the united states . ||| -241.368
+ in the same year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national assembly in the won all and was elected as the 44th president of the united states . ||| -241.540
+ he that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 assembly in election of national won all and was elected as the 44th president of the united states . ||| -241.586
+ he that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 assembly in national election won all and was elected as the 44th president of the united states . ||| -241.820
+ in the same year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 assembly in election of national won all and was elected as the 44th president of the united states . ||| -241.865
+ he that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 assembly the national election won all and was elected as the 44th president of the united states . ||| -241.881
+ in the year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national assembly in election won all and was elected as the 44th president of the united states . ||| -241.915
+ in that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national assembly in election won all and was elected as the 44th president of the united states . ||| -241.944
+ many indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be from upper stage ||| -332.122
+ many indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be from the upper stage ||| -332.706
+ many indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 from \u099f\u09c7\u0995\u09cd\u09b8\u09be upper stage ||| -333.394
+ many indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be to upper stage ||| -333.411
+ many of the indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be from upper stage ||| -333.551
+ many of indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be from upper stage ||| -333.558
+ many the indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be from upper stage ||| -333.877
+ of the indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be from upper stage ||| -334.019
+ many indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be upper stage from ||| -334.071
+ britain writers written drama novels stories and recently scripts in \u0986\u09a6\u09c3\u09a4 . ||| -140.876
+ britain writers written drama novels stories and in the recent scripts in \u0986\u09a6\u09c3\u09a4 . ||| -141.370
+ britain writers written drama novel stories and recently scripts in \u0986\u09a6\u09c3\u09a4 . ||| -141.436
+ britain writers written drama novels story and recently scripts in \u0986\u09a6\u09c3\u09a4 . ||| -141.720
+ britain writers written drama novels stories and recently script in \u0986\u09a6\u09c3\u09a4 . ||| -141.838
+ britain writers written drama novel stories and in the recent scripts in \u0986\u09a6\u09c3\u09a4 . ||| -141.930
+ britain writers written drama novels stories and recently scripts in the \u0986\u09a6\u09c3\u09a4 . ||| -141.937
+ britain writers the drama novels stories and recently scripts in \u0986\u09a6\u09c3\u09a4 . ||| -142.192
+ britain writers written drama novels story and in the recent scripts in \u0986\u09a6\u09c3\u09a4 . ||| -142.214
+ britain writers written drama novel story and recently scripts in \u0986\u09a6\u09c3\u09a4 . ||| -142.280
+ 1919 , in the month of may it was published in saogat magazine . ||| -19.117
+ in the month of may , 1919 it was published in saogat magazine . ||| -19.140
+ 1919 , it was published in saogat magazine in the month of may . ||| -19.348
+ 1919 on it was published in saogat magazine in the month of may . ||| -19.883
+ in 1919 it was published in saogat magazine in the month of may . ||| -20.253
+ in 1919 in the month of may it was published in saogat magazine . ||| -20.330
+ 1919 , it was published in saogat magazine during the month of may . ||| -20.762
+ 1919 on it was published in saogat magazine during the month of may . ||| -21.296
+ in 1919 it was published in saogat magazine during the month of may . ||| -21.667
+ 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium was arranged . ||| -452.293
+ in year 2005 \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium was arranged . ||| -452.393
+ 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium was organized . ||| -452.714
+ in year 2005 \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium was organized . ||| -452.814
+ 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium was arranged in . ||| -453.595
+ in year 2005 \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium was arranged in . ||| -453.695
+ 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium arranged in . ||| -453.728
+ 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association of the tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium was arranged . ||| -453.979
+ in year 2005 \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association of the tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium was arranged . ||| -454.079
+ 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association of the tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium was organized . ||| -454.400
+ to prevent this several measures are taken . ||| -12.088
+ to avoid this possibility several measures are taken . ||| -13.100
+ to prevent this several measures are the . ||| -15.767
+ to prevent this several measures are in the . ||| -16.171
+ to prevent this several measures are in . ||| -16.336
+ on 5th february , \u09e7\u09ef\u09ec\u09ec \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 of \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 a national was held in . ||| -334.940
+ on 5th february , \u09e7\u09ef\u09ec\u09ec \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 against a national was held in . ||| -335.202
+ on 5th february , \u09e7\u09ef\u09ec\u09ec \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 against \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 a national was held in . ||| -335.223
+ on 5th february , \u09e7\u09ef\u09ec\u09ec \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 opposition \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 a national was held in . ||| -335.450
+ bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in in the . ||| -341.281
+ bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in took the . ||| -341.957
+ bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank took secured its place in the . ||| -342.020
+ bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f took bank secured its place in the . ||| -342.985
+ bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in adopted the . ||| -343.243
+ bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in in the goods . ||| -343.618
+ bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and the islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in in the . ||| -344.062
+ bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and the islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in took the . ||| -344.738
+ bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and had \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in in the . ||| -345.143
+ bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and had \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in took the . ||| -345.819
+ subject category : encyclopedia ||| 1.726
+ subject class : encyclopedia ||| 0.624
+ subject matter : encyclopedia ||| 0.479
+ subject : encyclopedia ||| 0.305
+ category : encyclopedia ||| -0.153
+ russia france and israel the main \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 state of india and defense sub country . ||| -132.586
+ russia france and israel is the main \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 country and defense sub country . ||| -132.906
+ russia france and israel the main \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 country of india and defense sub country . ||| -133.257
+ russia france and israel is the main \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 state and defense sub country . ||| -133.294
+ russia france and israel the main \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 state of india and defence sub country . ||| -133.416
+ russia france and israel is the main \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 country and defence sub country . ||| -133.736
+ russia france and israel is the main \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 countries and defense sub country . ||| -133.777
+ russia france and israel india 's main \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 country and defense sub country . ||| -133.794
+ russia france and israel the main \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 country and defense sub country . ||| -133.804
+ russia france and israel india 's main \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 country and defence sub country . ||| -134.624
+ this is our known as an imaginary unit which with the help of mathematics formed the set of real numbers from \u09b8\u09c7\u099f\u09c7 par with the complex number . ||| -144.029
+ this is our known as imaginary unit which with the help of mathematics formed the set of real numbers from \u09b8\u09c7\u099f\u09c7 par with the complex number . ||| -144.652
+ this is our known as an imaginary unit which with the help of mathematics formed the set of real number from \u09b8\u09c7\u099f\u09c7 par with the complex number . ||| -144.865
+ this is our known as an imaginary unit which with the help of mathematics formed are set of real numbers from \u09b8\u09c7\u099f\u09c7 par with the complex number . ||| -145.042
+ this is our known as an imaginary unit which with the help of mathematics formed the set of real numbers to \u09b8\u09c7\u099f\u09c7 par with the complex number . ||| -145.094
+ this is our known as an imaginary unit which with the help of mathematics formed the real number set from \u09b8\u09c7\u099f\u09c7 par with the complex number . ||| -145.309
+ this is our known as imaginary unit which with the help of mathematics formed the set of real number from \u09b8\u09c7\u099f\u09c7 par with the complex number . ||| -145.488
+ this is our known as imaginary unit which with the help of mathematics formed are set of real numbers from \u09b8\u09c7\u099f\u09c7 par with the complex number . ||| -145.665
+ this is our known as imaginary unit which with the help of mathematics formed the set of real numbers to \u09b8\u09c7\u099f\u09c7 par with the complex number . ||| -145.717
+ this is our known as an imaginary unit which with the help of mathematics formed the set of real number to \u09b8\u09c7\u099f\u09c7 par with the complex number . ||| -145.832
+ <address> ||| -13.673
+ < \u09a0\u09bf\u0995\u09be\u09a8\u09be > ||| -120.820
+ < \u09a0\u09bf\u0995\u09be\u09a8\u09be , gt , ||| -219.955
+ , lt , \u09a0\u09bf\u0995\u09be\u09a8\u09be > ||| -220.429
+ < \u09a0\u09bf\u0995\u09be\u09a8\u09be , gt ; ||| -220.805
+ < \u09a0\u09bf\u0995\u09be\u09a8\u09be , gt and ||| -221.814
+ september ||| 1.148
+ september . ||| -3.006
+ \u09b8\u09c7\u09aa\u09cd\u099f\u09c7\u09ae\u09cd\u09ac\u09b0 ||| -103.288
+ from this theory though big \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 against can not be but it can be support . ||| -133.987
+ from this theory though big \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 against can not be rather it can be support . ||| -134.257
+ from this theory though big \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 against can not be but it can be supported . ||| -134.405
+ from this theory though big \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 against can not be rather it can be supported . ||| -134.676
+ from this theory though the big \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 against can not be but it can be support . ||| -134.713
+ this theory from though big \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 against can not be but it can be support . ||| -134.731
+ from this theory though the big \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 against can not be rather it can be support . ||| -134.984
+ this theory from though big \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 against can not be rather it can be support . ||| -135.001
+ from this theory though the big \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 against can not be but it can be supported . ||| -135.132
+ however , from this theory the big \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 against can not be but it can be support . ||| -135.265
+ agricultural in production france country ; it is the most important mainly \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af wine cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and export in the whole world . ||| -254.285
+ agricultural in production france is the most important country , it is mainly \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af wine cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and export in the whole world . ||| -254.323
+ agricultural in production france is the most important country ; it is mainly \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af wine cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and export in the whole world . ||| -254.426
+ agricultural in production france country ; it is the most important mainly \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af wine cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and in the whole world export the . ||| -254.434
+ agricultural in production france is the most important country , it is mainly \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af wine cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and in the whole world export the . ||| -254.473
+ agricultural in production france is the most important country ; it is mainly \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af wine cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and in the whole world export the . ||| -254.575
+ agricultural in production france country ; it is the most important mainly \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af wine cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and export the in the whole world . ||| -254.907
+ agricultural in production france is the most important country , it is mainly \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af wine cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and export the in the whole world . ||| -254.945
+ agricultural in production france country ; it is the most important mainly \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af wine cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and in the whole world export . ||| -255.035
+ agricultural in production france is the most important country ; it is mainly \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af wine cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and export the in the whole world . ||| -255.048
+ their in mathematics \u09aa\u09be\u099f\u09c0\u0997\u09a3\u09bf\u09a4\u09c7\u09b0 person was . ||| -115.913
+ their mathematics explains \u09aa\u09be\u099f\u09c0\u0997\u09a3\u09bf\u09a4\u09c7\u09b0 person was . ||| -117.241
+ their in mathematics \u09aa\u09be\u099f\u09c0\u0997\u09a3\u09bf\u09a4\u09c7\u09b0 were was . ||| -117.692
+ in mathematics their \u09aa\u09be\u099f\u09c0\u0997\u09a3\u09bf\u09a4\u09c7\u09b0 person was . ||| -117.711
+ they in mathematics \u09aa\u09be\u099f\u09c0\u0997\u09a3\u09bf\u09a4\u09c7\u09b0 person was . ||| -117.775
+ in mathematics of their \u09aa\u09be\u099f\u09c0\u0997\u09a3\u09bf\u09a4\u09c7\u09b0 person was . ||| -117.823
+ their in mathematics \u09aa\u09be\u099f\u09c0\u0997\u09a3\u09bf\u09a4\u09c7\u09b0 are was . ||| -117.940
+ their in mathematics was \u09aa\u09be\u099f\u09c0\u0997\u09a3\u09bf\u09a4\u09c7\u09b0 were . ||| -118.017
+ deshgulo are : france call , make noise china belgium switzerland germany denmark sweden austria chekoslovakia argentina italy norway \u09b9\u09be\u0999\u09cd\u0997\u09c7\u09b0\u09c0 yugoslavia bulgaria romania \u0997\u09cd\u09b0\u09c0\u09b8 egypt singapore indonesia \u09a5\u09be\u0987\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 japan burma \u09b9\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 soviet russia iran iraq and sri lanka . ||| -548.678
+ deshgulo are : france call , make noise china belgium switzerland germany denmark sweden austria chekoslovakia argentina italy norway \u09b9\u09be\u0999\u09cd\u0997\u09c7\u09b0\u09c0 yugoslavia bulgaria rumania \u0997\u09cd\u09b0\u09c0\u09b8 egypt singapore indonesia \u09a5\u09be\u0987\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 japan burma \u09b9\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 soviet russia iran iraq and sri lanka . ||| -549.105
+ deshgulo are : france call , make noise china belgium switzerland germany denmark sweden austria chekoslovakia argentina italy norway \u09b9\u09be\u0999\u09cd\u0997\u09c7\u09b0\u09c0 yugoslavia bulgaria romania \u0997\u09cd\u09b0\u09c0\u09b8 egypt singapore indonesia \u09a5\u09be\u0987\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 japan burma \u09b9\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 the soviet russia iran iraq and sri lanka . ||| -549.276
+ deshgulo are : france call , make noise china belgium switzerland germany denmark sweden austria chekoslovakia argentina italy norway \u09b9\u09be\u0999\u09cd\u0997\u09c7\u09b0\u09c0 yugoslavia bulgaria romania \u0997\u09cd\u09b0\u09c0\u09b8 egypt singapore indonesia \u09a5\u09be\u0987\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 japan burma \u09b9\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 soviet russia iran iraq and after visiting srilanka . ||| -549.440
+ deshgulo are : france call , make noise china belgium switzerland germany denmark sweden austria chekoslovakia argentina italy norway \u09b9\u09be\u0999\u09cd\u0997\u09c7\u09b0\u09c0 yugoslavia bulgaria rumania \u0997\u09cd\u09b0\u09c0\u09b8 egypt singapore indonesia \u09a5\u09be\u0987\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 japan burma \u09b9\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 the soviet russia iran iraq and sri lanka . ||| -549.702
+ deshgulo are : france call , make noise china belgium switzerland germany denmark sweden austria chekoslovakia argentina italy norway \u09b9\u09be\u0999\u09cd\u0997\u09c7\u09b0\u09c0 yugoslavia bulgaria rumania \u0997\u09cd\u09b0\u09c0\u09b8 egypt singapore indonesia \u09a5\u09be\u0987\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 japan burma \u09b9\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 soviet russia iran iraq and after visiting srilanka . ||| -549.867
+ deshgulo are : france call , make noise china belgium switzerland germany denmark sweden austria chekoslovakia argentina italy norway \u09b9\u09be\u0999\u09cd\u0997\u09c7\u09b0\u09c0 yugoslavia bulgaria romania \u0997\u09cd\u09b0\u09c0\u09b8 egypt singapore indonesia \u09a5\u09be\u0987\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 japan burma \u09b9\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 the soviet russia iran iraq and after visiting srilanka . ||| -550.038
+ deshgulo france are : call , make noise china belgium switzerland germany denmark sweden austria chekoslovakia argentina italy norway \u09b9\u09be\u0999\u09cd\u0997\u09c7\u09b0\u09c0 yugoslavia bulgaria romania \u0997\u09cd\u09b0\u09c0\u09b8 egypt singapore indonesia \u09a5\u09be\u0987\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 japan burma \u09b9\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 soviet russia iran iraq and sri lanka . ||| -550.040
+ deshgulo are : france call , make noise china belgium switzerland germany denmark sweden austria chekoslovakia argentina italy norway \u09b9\u09be\u0999\u09cd\u0997\u09c7\u09b0\u09c0 yugoslavia bulgaria rumania \u0997\u09cd\u09b0\u09c0\u09b8 egypt singapore indonesia \u09a5\u09be\u0987\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 japan burma \u09b9\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 the soviet russia iran iraq and after visiting srilanka . ||| -550.464
+ deshgulo france are : call , make noise china belgium switzerland germany denmark sweden austria chekoslovakia argentina italy norway \u09b9\u09be\u0999\u09cd\u0997\u09c7\u09b0\u09c0 yugoslavia bulgaria rumania \u0997\u09cd\u09b0\u09c0\u09b8 egypt singapore indonesia \u09a5\u09be\u0987\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 japan burma \u09b9\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 soviet russia iran iraq and sri lanka . ||| -550.466
+ this \u09ac\u09cd\u09af\u09be\u09b8\u09bf\u09b2\u09bf\u0995\u09be\u09b0 places now bank of england is situated . ||| -117.393
+ in this \u09ac\u09cd\u09af\u09be\u09b8\u09bf\u09b2\u09bf\u0995\u09be\u09b0 places now bank of england is situated . ||| -117.961
+ this \u09ac\u09cd\u09af\u09be\u09b8\u09bf\u09b2\u09bf\u0995\u09be\u09b0 places now bank of england is situated at . ||| -118.373
+ this \u09ac\u09cd\u09af\u09be\u09b8\u09bf\u09b2\u09bf\u0995\u09be\u09b0 places now bank of england is situated in . ||| -118.438
+ this \u09ac\u09cd\u09af\u09be\u09b8\u09bf\u09b2\u09bf\u0995\u09be\u09b0 places is now bank of england is situated . ||| -118.475
+ this \u09ac\u09cd\u09af\u09be\u09b8\u09bf\u09b2\u09bf\u0995\u09be\u09b0 places now bank of england is located in . ||| -118.654
+ this \u09ac\u09cd\u09af\u09be\u09b8\u09bf\u09b2\u09bf\u0995\u09be\u09b0 places now bank of england is located at . ||| -118.668
+ in this \u09ac\u09cd\u09af\u09be\u09b8\u09bf\u09b2\u09bf\u0995\u09be\u09b0 places now bank of england is situated at . ||| -118.942
+ in this \u09ac\u09cd\u09af\u09be\u09b8\u09bf\u09b2\u09bf\u0995\u09be\u09b0 places now bank of england is situated in . ||| -119.007
+ in this \u09ac\u09cd\u09af\u09be\u09b8\u09bf\u09b2\u09bf\u0995\u09be\u09b0 places now bank of england is located in . ||| -119.223
+ on the north of the country is the \u09ac\u09bf\u09b8\u09cd\u0995\u09be\u0987 sub-sea on the south \u099c\u09bf\u09ac\u09cd\u09b0\u09be\u09b2\u09cd\u099f\u09be\u09b0 strait \u09aa\u09cd\u09b0\u09a3\u09be\u09b2\u09c0\u09b0 in the south \u09ae\u09b0\u0995\u09cd\u0995\u09cb on the west and the atlantic ocean . ||| -445.790
+ on the north of the country is the \u09ac\u09bf\u09b8\u09cd\u0995\u09be\u0987 sub-sea on the south \u099c\u09bf\u09ac\u09cd\u09b0\u09be\u09b2\u09cd\u099f\u09be\u09b0 strait \u09aa\u09cd\u09b0\u09a3\u09be\u09b2\u09c0\u09b0 in the south \u09ae\u09b0\u0995\u09cd\u0995\u09cb to the west and the atlantic ocean . ||| -445.962
+ on the north of the country is the \u09ac\u09bf\u09b8\u09cd\u0995\u09be\u0987 sub-sea to the south \u099c\u09bf\u09ac\u09cd\u09b0\u09be\u09b2\u09cd\u099f\u09be\u09b0 strait \u09aa\u09cd\u09b0\u09a3\u09be\u09b2\u09c0\u09b0 in the south \u09ae\u09b0\u0995\u09cd\u0995\u09cb on the west and the atlantic ocean . ||| -446.269
+ on the north of the country is the \u09ac\u09bf\u09b8\u09cd\u0995\u09be\u0987 sub-sea to the south \u099c\u09bf\u09ac\u09cd\u09b0\u09be\u09b2\u09cd\u099f\u09be\u09b0 strait \u09aa\u09cd\u09b0\u09a3\u09be\u09b2\u09c0\u09b0 in the south \u09ae\u09b0\u0995\u09cd\u0995\u09cb to the west and the atlantic ocean . ||| -446.441
+ on the north of the country is the \u09ac\u09bf\u09b8\u09cd\u0995\u09be\u0987 sub-sea on the south \u099c\u09bf\u09ac\u09cd\u09b0\u09be\u09b2\u09cd\u099f\u09be\u09b0 strait \u09aa\u09cd\u09b0\u09a3\u09be\u09b2\u09c0\u09b0 in the south of the \u09ae\u09b0\u0995\u09cd\u0995\u09cb on the west and the atlantic ocean . ||| -446.505
+ on the north of the country is the \u09ac\u09bf\u09b8\u09cd\u0995\u09be\u0987 sub-sea on the south \u099c\u09bf\u09ac\u09cd\u09b0\u09be\u09b2\u09cd\u099f\u09be\u09b0 strait \u09aa\u09cd\u09b0\u09a3\u09be\u09b2\u09c0\u09b0 in \u09ae\u09b0\u0995\u09cd\u0995\u09cb on the west and the atlantic ocean . ||| -446.556
+ in the north of this country \u09ac\u09bf\u09b8\u09cd\u0995\u09be\u0987 sub-sea on the south \u099c\u09bf\u09ac\u09cd\u09b0\u09be\u09b2\u09cd\u099f\u09be\u09b0 strait \u09aa\u09cd\u09b0\u09a3\u09be\u09b2\u09c0\u09b0 in the south \u09ae\u09b0\u0995\u09cd\u0995\u09cb on the west and the atlantic ocean . ||| -446.604
+ on the north of the country is the \u09ac\u09bf\u09b8\u09cd\u0995\u09be\u0987 sub-sea on the south \u099c\u09bf\u09ac\u09cd\u09b0\u09be\u09b2\u09cd\u099f\u09be\u09b0 strait \u09aa\u09cd\u09b0\u09a3\u09be\u09b2\u09c0\u09b0 the \u09ae\u09b0\u0995\u09cd\u0995\u09cb on the west and the atlantic ocean . ||| -446.613
+ on the north of the country is the \u09ac\u09bf\u09b8\u09cd\u0995\u09be\u0987 sub-sea on the south \u099c\u09bf\u09ac\u09cd\u09b0\u09be\u09b2\u09cd\u099f\u09be\u09b0 strait \u09aa\u09cd\u09b0\u09a3\u09be\u09b2\u09c0\u09b0 in the south of the \u09ae\u09b0\u0995\u09cd\u0995\u09cb to the west and the atlantic ocean . ||| -446.677
+ on the north of the country is the \u09ac\u09bf\u09b8\u09cd\u0995\u09be\u0987 sub-sea on the south \u099c\u09bf\u09ac\u09cd\u09b0\u09be\u09b2\u09cd\u099f\u09be\u09b0 strait \u09aa\u09cd\u09b0\u09a3\u09be\u09b2\u09c0\u09b0 south \u09ae\u09b0\u0995\u09cd\u0995\u09cb on the west and the atlantic ocean . ||| -446.722
+ apart from this situation it can be understood that a \u09ae\u09c1\u09b9\u09c2\u09b0\u09cd\u09a4\u09c7 within \u0985\u0995\u09cd\u09b7\u09ae\u09a4\u09be after taking the decision of the united nations . ||| -248.040
+ other than this situation it can be understood that a \u09ae\u09c1\u09b9\u09c2\u09b0\u09cd\u09a4\u09c7 within \u0985\u0995\u09cd\u09b7\u09ae\u09a4\u09be after taking the decision of the united nations . ||| -248.164
+ apart from this situation it can be understood that a \u09ae\u09c1\u09b9\u09c2\u09b0\u09cd\u09a4\u09c7 fast within \u0985\u0995\u09cd\u09b7\u09ae\u09a4\u09be after taking decision of the united nations . ||| -248.648
+ other than this situation it can be understood that a \u09ae\u09c1\u09b9\u09c2\u09b0\u09cd\u09a4\u09c7 fast within \u0985\u0995\u09cd\u09b7\u09ae\u09a4\u09be after taking decision of the united nations . ||| -248.772
+ apart from this situation it can be understood that a \u09ae\u09c1\u09b9\u09c2\u09b0\u09cd\u09a4\u09c7 of the united nations the decision within \u0985\u0995\u09cd\u09b7\u09ae\u09a4\u09be after taking . ||| -248.791
+ apart from this situation it can be understood that a \u09ae\u09c1\u09b9\u09c2\u09b0\u09cd\u09a4\u09c7 the decision of the united nations within \u0985\u0995\u09cd\u09b7\u09ae\u09a4\u09be after taking . ||| -248.825
+ other than this situation it can be understood that a \u09ae\u09c1\u09b9\u09c2\u09b0\u09cd\u09a4\u09c7 of the united nations the decision within \u0985\u0995\u09cd\u09b7\u09ae\u09a4\u09be after taking . ||| -248.916
+ other than this situation it can be understood that a \u09ae\u09c1\u09b9\u09c2\u09b0\u09cd\u09a4\u09c7 the decision of the united nations within \u0985\u0995\u09cd\u09b7\u09ae\u09a4\u09be after taking . ||| -248.949
+ apart from this situation it can be understood that a \u09ae\u09c1\u09b9\u09c2\u09b0\u09cd\u09a4\u09c7 of the united nations fast within \u0985\u0995\u09cd\u09b7\u09ae\u09a4\u09be after taking decision . ||| -248.954
+ apart from this situation it can be understood that a \u09ae\u09c1\u09b9\u09c2\u09b0\u09cd\u09a4\u09c7 of the united nations the within \u0985\u0995\u09cd\u09b7\u09ae\u09a4\u09be after taking decision . ||| -249.057
+ \u0995\u09be\u09b0\u09cd\u09b2 there is work through it is controlled by . ||| -125.132
+ \u0995\u09be\u09b0\u09cd\u09b2 there is work through this are conducted by the . ||| -125.984
+ \u0995\u09be\u09b0\u09cd\u09b2 there is works that it is controlled by . ||| -125.986
+ \u0995\u09be\u09b0\u09cd\u09b2 there is work that it is controlled by . ||| -126.112
+ \u0995\u09be\u09b0\u09cd\u09b2 there is work through this is controlled by . ||| -126.220
+ \u0995\u09be\u09b0\u09cd\u09b2 there is work through it are conducted by the . ||| -126.432
+ \u0995\u09be\u09b0\u09cd\u09b2 there is only through it is controlled by . ||| -126.447
+ the subject matters sometimes puran -lrb- from sometimes in the middle age love story sometimes again from today 's social and political \u0998\u099f\u09a8\u09be\u09ac\u09b2\u09bf taken from . ||| -157.897
+ the subject matters sometimes puran -lrb- from sometimes in the middle age love story sometimes again from today 's social and political \u0998\u099f\u09a8\u09be\u09ac\u09b2\u09bf from accepted . ||| -158.142
+ the subject matters sometimes puran -lrb- from sometimes in the middle age love story from sometimes again today 's social and political \u0998\u099f\u09a8\u09be\u09ac\u09b2\u09bf taken from . ||| -158.179
+ the subject matters sometimes puran -lrb- from sometimes in the middle age love story from sometimes again today 's social and political \u0998\u099f\u09a8\u09be\u09ac\u09b2\u09bf from accepted . ||| -158.424
+ the subject sometimes puran -lrb- from sometimes in the middle age love story sometimes again from today 's social and political \u0998\u099f\u09a8\u09be\u09ac\u09b2\u09bf taken from . ||| -158.455
+ the subject sometimes puran -lrb- from sometimes in the middle age love story sometimes again from today 's social and political \u0998\u099f\u09a8\u09be\u09ac\u09b2\u09bf from accepted . ||| -158.701
+ the subject sometimes puran -lrb- from sometimes in the middle age love story from sometimes again today 's social and political \u0998\u099f\u09a8\u09be\u09ac\u09b2\u09bf taken from . ||| -158.738
+ the subject sometimes puran -lrb- from sometimes in the middle age love story from sometimes again today 's social and political \u0998\u099f\u09a8\u09be\u09ac\u09b2\u09bf from accepted . ||| -158.983
+ the subject matters sometimes from puran -lrb- sometimes in the middle age love story sometimes again from today 's social and political \u0998\u099f\u09a8\u09be\u09ac\u09b2\u09bf taken from . ||| -159.231
+ the subject matters sometimes puran -lrb- sometimes from in the middle age love story sometimes again from today 's social and political \u0998\u099f\u09a8\u09be\u09ac\u09b2\u09bf taken from . ||| -159.296
+ three measure based on the age of the universe is found that is almost \u09e7\u09e9.\u09ed � \u09e6.\u09e8 billion years . ||| -344.330
+ three measure based on the age of the universe is found that is about \u09e7\u09e9.\u09ed � \u09e6.\u09e8 billion years . ||| -345.865
+ three measure based on the age of the universe is found that are almost \u09e7\u09e9.\u09ed � \u09e6.\u09e8 billion years . ||| -346.107
+ three measure based on the age of the universe is found that is around \u09e7\u09e9.\u09ed � \u09e6.\u09e8 billion years . ||| -346.305
+ three measure based on that the age of the universe is found that is almost \u09e7\u09e9.\u09ed � \u09e6.\u09e8 billion years . ||| -346.572
+ three measure based on that the age of the universe is found that is about \u09e7\u09e9.\u09ed � \u09e6.\u09e8 billion years . ||| -348.107
+ three measure based on that the age of the universe is found that are almost \u09e7\u09e9.\u09ed � \u09e6.\u09e8 billion years . ||| -348.349
+ three measure based on that the age of the universe is found that is around \u09e7\u09e9.\u09ed � \u09e6.\u09e8 billion years . ||| -348.547
+ three measure based on the age of the universe is found that is almost \u09e7\u09e9.\u09ed � \u09e6.\u09e8 billion year . ||| -349.266
+ three measure based on the age of the universe is found that is about \u09e7\u09e9.\u09ed � \u09e6.\u09e8 billion year . ||| -350.801
+ \u0995\u09be\u099b\u09c7\u0987 there are east russia which \u0993\u0996\u099f\u09b8\u09cd\u0995 sea and japan sea on the other can be is situated . ||| -235.123
+ \u0995\u09be\u099b\u09c7\u0987 there are east russia which \u0993\u0996\u099f\u09b8\u09cd\u0995 sea and japan sea on the other can be situated . ||| -235.484
+ \u0995\u09be\u099b\u09c7\u0987 there are east russia which \u0993\u0996\u099f\u09b8\u09cd\u0995 sea and japan sea on the other can be located . ||| -235.808
+ \u0995\u09be\u099b\u09c7\u0987 there are east russia which \u0993\u0996\u099f\u09b8\u09cd\u0995 sea and japan sea on the other can be is situated at . ||| -236.103
+ \u0995\u09be\u099b\u09c7\u0987 there is east russia which \u0993\u0996\u099f\u09b8\u09cd\u0995 sea and japan sea on the other can be situated . ||| -236.141
+ \u0995\u09be\u099b\u09c7\u0987 there are east russia which \u0993\u0996\u099f\u09b8\u09cd\u0995 sea and japan sea on the other can be is situated in . ||| -236.168
+ \u0995\u09be\u099b\u09c7\u0987 are east russia which \u0993\u0996\u099f\u09b8\u09cd\u0995 sea and japan sea on the other can be situated . ||| -236.282
+ \u0995\u09be\u099b\u09c7\u0987 there are east russia which \u0993\u0996\u099f\u09b8\u09cd\u0995 sea and japan sea another can be situated . ||| -236.371
+ \u0995\u09be\u099b\u09c7\u0987 there are east russia which \u0993\u0996\u099f\u09b8\u09cd\u0995 sea and japan sea on the other can be is located in . ||| -236.384
+ \u0995\u09be\u099b\u09c7\u0987 there are eastern russia which \u0993\u0996\u099f\u09b8\u09cd\u0995 sea and japan sea on the other can be is situated . ||| -236.499
+ in kolkata is located at the national library of india the leading public \u09b2\u09be\u0987\u09ac\u09cd\u09b0\u09c7\u09b0\u09bf . ||| -119.076
+ in kolkata is located in the national library of india the leading public \u09b2\u09be\u0987\u09ac\u09cd\u09b0\u09c7\u09b0\u09bf . ||| -119.169
+ in kolkata is situated at the national library of india the leading public \u09b2\u09be\u0987\u09ac\u09cd\u09b0\u09c7\u09b0\u09bf . ||| -119.263
+ in kolkata is situated in the national library of india the leading public \u09b2\u09be\u0987\u09ac\u09cd\u09b0\u09c7\u09b0\u09bf . ||| -119.665
+ in kolkata is located at the national library of india the country leading public \u09b2\u09be\u0987\u09ac\u09cd\u09b0\u09c7\u09b0\u09bf . ||| -120.880
+ in kolkata is located in the national library of the leading public \u09b2\u09be\u0987\u09ac\u09cd\u09b0\u09c7\u09b0\u09bf . ||| -120.887
+ in kolkata is located at the national library of india the country was public \u09b2\u09be\u0987\u09ac\u09cd\u09b0\u09c7\u09b0\u09bf . ||| -120.912
+ in kolkata is located in the national library of india the country leading public \u09b2\u09be\u0987\u09ac\u09cd\u09b0\u09c7\u09b0\u09bf . ||| -120.973
+ in kolkata is located in the national library of india the country was public \u09b2\u09be\u0987\u09ac\u09cd\u09b0\u09c7\u09b0\u09bf . ||| -121.004
+ in kolkata is situated at the national library of india the country leading public \u09b2\u09be\u0987\u09ac\u09cd\u09b0\u09c7\u09b0\u09bf . ||| -121.067
+ \u099b\u09a4\u09cd\u09b0\u09be\u0995\u09ac\u09bf\u09a6\u09cd\u09af\u09be ||| -103.288
+ \u09b0\u09be\u09b7\u09cd\u099f\u09cd\u09b0\u09b8\u0999\u09cd\u0998\u09c7\u09b0 general secretary \u09ac\u09be\u09a8 ki moon ||| -218.172
+ \u09b0\u09be\u09b7\u09cd\u099f\u09cd\u09b0\u09b8\u0999\u09cd\u0998\u09c7\u09b0 secretary general \u09ac\u09be\u09a8 ki moon ||| -218.226
+ \u09b0\u09be\u09b7\u09cd\u099f\u09cd\u09b0\u09b8\u0999\u09cd\u0998\u09c7\u09b0 chief secretary \u09ac\u09be\u09a8 ki moon ||| -220.109
+ \u09ac\u09be\u09a8 \u09b0\u09be\u09b7\u09cd\u099f\u09cd\u09b0\u09b8\u0999\u09cd\u0998\u09c7\u09b0 general secretary ki moon ||| -221.235
+ \u09b0\u09be\u09b7\u09cd\u099f\u09cd\u09b0\u09b8\u0999\u09cd\u0998\u09c7\u09b0 general secretary \u09ac\u09be\u09a8 what moon ||| -221.542
+ \u09ae\u09bf\u09a8\u09bf\u0995\u09cd\u09b8\u09c7\u09b0 of india was smells \u099f\u09be\u09a8\u09c7\u09a8\u09ac\u09ae a famous operating system design \u09aa\u09cd\u09b0\u09b6\u09bf\u0995\u09cd\u09b7\u0995 . ||| -337.188
+ \u09ae\u09bf\u09a8\u09bf\u0995\u09cd\u09b8\u09c7\u09b0 of india was smells \u099f\u09be\u09a8\u09c7\u09a8\u09ac\u09ae a famous operating system designing \u09aa\u09cd\u09b0\u09b6\u09bf\u0995\u09cd\u09b7\u0995 . ||| -337.335
+ \u09ae\u09bf\u09a8\u09bf\u0995\u09cd\u09b8\u09c7\u09b0 of was smells \u099f\u09be\u09a8\u09c7\u09a8\u09ac\u09ae a famous operating system design \u09aa\u09cd\u09b0\u09b6\u09bf\u0995\u09cd\u09b7\u0995 . ||| -337.851
+ \u09ae\u09bf\u09a8\u09bf\u0995\u09cd\u09b8\u09c7\u09b0 of was smells \u099f\u09be\u09a8\u09c7\u09a8\u09ac\u09ae a famous operating system designing \u09aa\u09cd\u09b0\u09b6\u09bf\u0995\u09cd\u09b7\u0995 . ||| -337.998
+ \u09ae\u09bf\u09a8\u09bf\u0995\u09cd\u09b8\u09c7\u09b0 of india was smells \u099f\u09be\u09a8\u09c7\u09a8\u09ac\u09ae in a famous operating system design \u09aa\u09cd\u09b0\u09b6\u09bf\u0995\u09cd\u09b7\u0995 . ||| -338.299
+ \u09ae\u09bf\u09a8\u09bf\u0995\u09cd\u09b8\u09c7\u09b0 of india was smells \u099f\u09be\u09a8\u09c7\u09a8\u09ac\u09ae in a famous operating system designing \u09aa\u09cd\u09b0\u09b6\u09bf\u0995\u09cd\u09b7\u0995 . ||| -338.446
+ the \u099f\u09be\u0987\u09ae\u09cd the 's of \u0987\u09a8\u09cd\u09a1\u09bf\u09af\u09bc\u09be-\u09a4\u09c7 written in the " it is absurd to compare it with any other indian cinema ... pather panchali is pure cinema " -lrb- " it other by the indian films with compared to unreal ... pather panchali are pure film " -rrb- . ||| -1714.540
+ the \u099f\u09be\u0987\u09ae\u09cd of 's of \u0987\u09a8\u09cd\u09a1\u09bf\u09af\u09bc\u09be-\u09a4\u09c7 written in the " it is absurd to compare it with any other indian cinema ... pather panchali is pure cinema " -lrb- " it other by the indian films with compared to unreal ... pather panchali are pure film " -rrb- . ||| -1714.622
+ the \u099f\u09be\u0987\u09ae\u09cd the 's of \u0987\u09a8\u09cd\u09a1\u09bf\u09af\u09bc\u09be-\u09a4\u09c7 written in that " it is absurd to compare it with any other indian cinema ... pather panchali is pure cinema " -lrb- " it other by the indian films with compared to unreal ... pather panchali are pure film " -rrb- . ||| -1714.748
+ the \u099f\u09be\u0987\u09ae\u09cd the 's of \u0987\u09a8\u09cd\u09a1\u09bf\u09af\u09bc\u09be-\u09a4\u09c7 written in the " it is absurd to compare it with any other indian cinema ... pather panchali is pure cinema " -lrb- " it other by indian films with compared to unreal ... pather panchali are pure film " -rrb- . ||| -1714.769
+ the \u099f\u09be\u0987\u09ae\u09cd of 's of \u0987\u09a8\u09cd\u09a1\u09bf\u09af\u09bc\u09be-\u09a4\u09c7 written in that " it is absurd to compare it with any other indian cinema ... pather panchali is pure cinema " -lrb- " it other by the indian films with compared to unreal ... pather panchali are pure film " -rrb- . ||| -1714.830
+ the \u099f\u09be\u0987\u09ae\u09cd of 's of \u0987\u09a8\u09cd\u09a1\u09bf\u09af\u09bc\u09be-\u09a4\u09c7 written in the " it is absurd to compare it with any other indian cinema ... pather panchali is pure cinema " -lrb- " it other by indian films with compared to unreal ... pather panchali are pure film " -rrb- . ||| -1714.851
+ the \u099f\u09be\u0987\u09ae\u09cd the 's of \u0987\u09a8\u09cd\u09a1\u09bf\u09af\u09bc\u09be-\u09a4\u09c7 written in that " it is absurd to compare it with any other indian cinema ... pather panchali is pure cinema " -lrb- " it other by indian films with compared to unreal ... pather panchali are pure film " -rrb- . ||| -1714.978
+ the \u099f\u09be\u0987\u09ae\u09cd the 's of \u0987\u09a8\u09cd\u09a1\u09bf\u09af\u09bc\u09be-\u09a4\u09c7 written in that , " , it is absurd to compare it with any other indian cinema ... pather panchali is pure cinema " -lrb- " it other by the indian films with compared to unreal ... pather panchali are pure film " -rrb- . ||| -1715.032
+ the \u099f\u09be\u0987\u09ae\u09cd of 's of \u0987\u09a8\u09cd\u09a1\u09bf\u09af\u09bc\u09be-\u09a4\u09c7 written in that " it is absurd to compare it with any other indian cinema ... pather panchali is pure cinema " -lrb- " it other by indian films with compared to unreal ... pather panchali are pure film " -rrb- . ||| -1715.060
+ the \u099f\u09be\u0987\u09ae\u09cd of 's of \u0987\u09a8\u09cd\u09a1\u09bf\u09af\u09bc\u09be-\u09a4\u09c7 written in that , " , it is absurd to compare it with any other indian cinema ... pather panchali is pure cinema " -lrb- " it other by the indian films with compared to unreal ... pather panchali are pure film " -rrb- . ||| -1715.114
+ after that , 1953 in the month of may nazrul and \u09aa\u09cd\u09b0\u09ae\u09c0\u09b2\u09be \u09a6\u09c7\u09ac\u09c0\u0995\u09c7 \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u09be\u09b0 for london sent to . ||| -336.768
+ after that in the month of may , 1953 nazrul and \u09aa\u09cd\u09b0\u09ae\u09c0\u09b2\u09be \u09a6\u09c7\u09ac\u09c0\u0995\u09c7 \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u09be\u09b0 for london sent to . ||| -336.929
+ after that 1953 , in the month of may nazrul and \u09aa\u09cd\u09b0\u09ae\u09c0\u09b2\u09be \u09a6\u09c7\u09ac\u09c0\u0995\u09c7 \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u09be\u09b0 for london sent to . ||| -337.044
+ after that , 1953 during the month of may nazrul and \u09aa\u09cd\u09b0\u09ae\u09c0\u09b2\u09be \u09a6\u09c7\u09ac\u09c0\u0995\u09c7 \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u09be\u09b0 for london sent to . ||| -337.100
+ after that , 1953 in the month of may nazrul and \u09aa\u09cd\u09b0\u09ae\u09c0\u09b2\u09be \u09a6\u09c7\u09ac\u09c0\u0995\u09c7 \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u09be\u09b0 for london sent to the . ||| -337.124
+ after that in the month of may , 1953 nazrul and \u09aa\u09cd\u09b0\u09ae\u09c0\u09b2\u09be \u09a6\u09c7\u09ac\u09c0\u0995\u09c7 \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u09be\u09b0 for london sent to the . ||| -337.286
+ after that , 1953 in the month of may najrul and \u09aa\u09cd\u09b0\u09ae\u09c0\u09b2\u09be \u09a6\u09c7\u09ac\u09c0\u0995\u09c7 \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u09be\u09b0 for london sent to . ||| -337.301
+ after this 1953 , in the month of may nazrul and \u09aa\u09cd\u09b0\u09ae\u09c0\u09b2\u09be \u09a6\u09c7\u09ac\u09c0\u0995\u09c7 \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u09be\u09b0 for london sent to . ||| -337.367
+ after that 1953 , in the month of may nazrul and \u09aa\u09cd\u09b0\u09ae\u09c0\u09b2\u09be \u09a6\u09c7\u09ac\u09c0\u0995\u09c7 \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u09be\u09b0 for london sent to the . ||| -337.401
+ after that , 1953 during the month of may nazrul and \u09aa\u09cd\u09b0\u09ae\u09c0\u09b2\u09be \u09a6\u09c7\u09ac\u09c0\u0995\u09c7 \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u09be\u09b0 for london sent to the . ||| -337.457
+ the southern and the east there is \u09ac\u09bf\u09b8\u09cd\u09a4\u09c0\u09b0\u09cd\u09a3 land , west and in the north there are \u09b0\u09c1\u0995\u09cd\u09b7 mountain and mountain . ||| -242.245
+ the southern and the east there is \u09ac\u09bf\u09b8\u09cd\u09a4\u09c0\u09b0\u09cd\u09a3 land , west and on the north there are \u09b0\u09c1\u0995\u09cd\u09b7 mountain and mountain . ||| -242.290
+ the southern and the east there is \u09ac\u09bf\u09b8\u09cd\u09a4\u09c0\u09b0\u09cd\u09a3 land , west and in the north there is \u09b0\u09c1\u0995\u09cd\u09b7 mountain and mountain . ||| -242.774
+ the southern and the east there is \u09ac\u09bf\u09b8\u09cd\u09a4\u09c0\u09b0\u09cd\u09a3 plain , west and in the north there are \u09b0\u09c1\u0995\u09cd\u09b7 mountain and mountain . ||| -242.788
+ the southern and the east there is \u09ac\u09bf\u09b8\u09cd\u09a4\u09c0\u09b0\u09cd\u09a3 land , west and on the north there is \u09b0\u09c1\u0995\u09cd\u09b7 mountain and mountain . ||| -242.819
+ the southern and the east there is \u09ac\u09bf\u09b8\u09cd\u09a4\u09c0\u09b0\u09cd\u09a3 plain , west and on the north there are \u09b0\u09c1\u0995\u09cd\u09b7 mountain and mountain . ||| -242.833
+ the south and the east there is \u09ac\u09bf\u09b8\u09cd\u09a4\u09c0\u09b0\u09cd\u09a3 land , west and in the north there are \u09b0\u09c1\u0995\u09cd\u09b7 mountain and mountain . ||| -242.915
+ the south and the east there is \u09ac\u09bf\u09b8\u09cd\u09a4\u09c0\u09b0\u09cd\u09a3 land , west and on the north there are \u09b0\u09c1\u0995\u09cd\u09b7 mountain and mountain . ||| -242.960
+ the southern and the east there is \u09ac\u09bf\u09b8\u09cd\u09a4\u09c0\u09b0\u09cd\u09a3 land , west and in the north there are \u09b0\u09c1\u0995\u09cd\u09b7 hill and mountain . ||| -242.973
+ the southern and the east there is \u09ac\u09bf\u09b8\u09cd\u09a4\u09c0\u09b0\u09cd\u09a3 land , west and on the north there are \u09b0\u09c1\u0995\u09cd\u09b7 hill and mountain . ||| -243.018
+ \u099f\u09cd\u09b0\u09c7\u09a1\u09ae\u09be\u09b0\u09cd\u0995 ||| -103.288
+ \u09b2\u09b0\u09cd\u09a1 \u0993\u09af\u09bc\u09c7\u09b2\u09c7\u09b8\u09b2\u09bf\u09b0 -lrb- \u0997\u09ad\u09b0\u09cd\u09a8\u09b0-\u099c\u09c7\u09a8\u09be\u09b0\u09c7\u09b2 \u09e7\u09ed\u09ef\u09ed-\u09e7\u09ee\u09e6\u09eb -rrb- during his city are the ghotechilo ||| -443.070
+ \u09b2\u09b0\u09cd\u09a1 \u0993\u09af\u09bc\u09c7\u09b2\u09c7\u09b8\u09b2\u09bf\u09b0 -lrb- \u0997\u09ad\u09b0\u09cd\u09a8\u09b0-\u099c\u09c7\u09a8\u09be\u09b0\u09c7\u09b2 \u09e7\u09ed\u09ef\u09ed-\u09e7\u09ee\u09e6\u09eb -rrb- during his city are the already happened ||| -443.197
+ \u09b2\u09b0\u09cd\u09a1 \u0993\u09af\u09bc\u09c7\u09b2\u09c7\u09b8\u09b2\u09bf\u09b0 -lrb- \u0997\u09ad\u09b0\u09cd\u09a8\u09b0-\u099c\u09c7\u09a8\u09be\u09b0\u09c7\u09b2 \u09e7\u09ed\u09ef\u09ed-\u09e7\u09ee\u09e6\u09eb -rrb- during his city are increased ghotechilo ||| -444.124
+ \u09b2\u09b0\u09cd\u09a1 \u0993\u09af\u09bc\u09c7\u09b2\u09c7\u09b8\u09b2\u09bf\u09b0 -lrb- \u0997\u09ad\u09b0\u09cd\u09a8\u09b0-\u099c\u09c7\u09a8\u09be\u09b0\u09c7\u09b2 \u09e7\u09ed\u09ef\u09ed-\u09e7\u09ee\u09e6\u09eb -rrb- during his city are increased already happened ||| -444.714
+ \u09b2\u09b0\u09cd\u09a1 \u0993\u09af\u09bc\u09c7\u09b2\u09c7\u09b8\u09b2\u09bf\u09b0 -lrb- \u0997\u09ad\u09b0\u09cd\u09a8\u09b0-\u099c\u09c7\u09a8\u09be\u09b0\u09c7\u09b2 \u09e7\u09ed\u09ef\u09ed-\u09e7\u09ee\u09e6\u09eb -rrb- during his city are mentionable the already happened ||| -444.808
+ \u09b2\u09b0\u09cd\u09a1 \u0993\u09af\u09bc\u09c7\u09b2\u09c7\u09b8\u09b2\u09bf\u09b0 -lrb- \u0997\u09ad\u09b0\u09cd\u09a8\u09b0-\u099c\u09c7\u09a8\u09be\u09b0\u09c7\u09b2 \u09e7\u09ed\u09ef\u09ed-\u09e7\u09ee\u09e6\u09eb -rrb- during his in the city are the already happened ||| -445.096
+ \u09b2\u09b0\u09cd\u09a1 \u0993\u09af\u09bc\u09c7\u09b2\u09c7\u09b8\u09b2\u09bf\u09b0 -lrb- \u0997\u09ad\u09b0\u09cd\u09a8\u09b0-\u099c\u09c7\u09a8\u09be\u09b0\u09c7\u09b2 \u09e7\u09ed\u09ef\u09ed-\u09e7\u09ee\u09e6\u09eb -rrb- during his city are of the already happened ||| -445.116
+ \u09b2\u09b0\u09cd\u09a1 \u0993\u09af\u09bc\u09c7\u09b2\u09c7\u09b8\u09b2\u09bf\u09b0 -lrb- \u0997\u09ad\u09b0\u09cd\u09a8\u09b0-\u099c\u09c7\u09a8\u09be\u09b0\u09c7\u09b2 \u09e7\u09ed\u09ef\u09ed-\u09e7\u09ee\u09e6\u09eb -rrb- during his are in the city of the already happened ||| -445.119
+ \u09b2\u09b0\u09cd\u09a1 \u0993\u09af\u09bc\u09c7\u09b2\u09c7\u09b8\u09b2\u09bf\u09b0 -lrb- \u0997\u09ad\u09b0\u09cd\u09a8\u09b0-\u099c\u09c7\u09a8\u09be\u09b0\u09c7\u09b2 \u09e7\u09ed\u09ef\u09ed-\u09e7\u09ee\u09e6\u09eb -rrb- during his the are the already happened ||| -445.235
+ \u09b2\u09b0\u09cd\u09a1 \u0993\u09af\u09bc\u09c7\u09b2\u09c7\u09b8\u09b2\u09bf\u09b0 -lrb- \u0997\u09ad\u09b0\u09cd\u09a8\u09b0-\u099c\u09c7\u09a8\u09be\u09b0\u09c7\u09b2 \u09e7\u09ed\u09ef\u09ed-\u09e7\u09ee\u09e6\u09eb -rrb- during his the city are the already happened ||| -445.283
+ many important and real extremely necessary to solve problems complex number ||| -26.515
+ many important and real to solve problems complex number apariharza ||| -26.733
+ many important and real to solve problems extremely necessary complex number ||| -26.848
+ many important and real to solve problems complex number inevitable ||| -27.273
+ many important and real extremely necessary to solve problems complex numbers ||| -27.283
+ many important and real problem extremely necessary to solve complex number ||| -27.298
+ many important and real to solve problems complex number extremely necessary ||| -27.308
+ many important and real problem to solve complex number inevitable ||| -27.422
+ many important and real problem to solve complex number extremely necessary ||| -27.457
+ many important and real to solve problems extremely necessary complex numbers ||| -27.756
+ the big bang is a important result are in the state of the universe so and recent situation from the separate . ||| -46.594
+ the big bang is a important result is in the state of the universe so and recent situation from the separate . ||| -46.767
+ the big bang is a important result are in the state of the universe so and recent situation from the the . ||| -47.097
+ the big bang is a important result are in the state of the universe so and recent situation from the different . ||| -47.151
+ the big bang is a important result is in the state of the universe so and recent situation from the the . ||| -47.270
+ the big bang is a important result is in the state of the universe so and recent situation from the different . ||| -47.324
+ the big bang is a important result are in the state of the universe so and recent state from the separate . ||| -47.472
+ the big bang is a important result is in the state of the universe so and recent state from the separate . ||| -47.646
+ the big bang is a important result are in the universe so state and recent situation from the separate . ||| -47.814
+ the big bang is a important result are in the state of the universe so and recent situation from completely separate . ||| -47.853
+ windows \u09ae\u09bf\u09b2\u09bf\u09a8\u09bf\u09af\u09bc\u09be\u09ae ||| -104.858
+ \u0989\u0987\u09a8\u09cd\u09a1\u09cb\u099c \u09ae\u09bf\u09b2\u09bf\u09a8\u09bf\u09af\u09bc\u09be\u09ae ||| -207.084
+ however , rabindranath more than one \u09b6\u09c8\u09b2\u09c0 \u0986\u09af\u09bc\u09a4\u09cd\u09a4\u09cd\u09ac was . ||| -219.540
+ rabindranath , more than one \u09b6\u09c8\u09b2\u09c0 \u0986\u09af\u09bc\u09a4\u09cd\u09a4\u09cd\u09ac was . ||| -219.826
+ rabindranath however , more than one \u09b6\u09c8\u09b2\u09c0 \u0986\u09af\u09bc\u09a4\u09cd\u09a4\u09cd\u09ac was . ||| -219.926
+ rabindranath , in more than one \u09b6\u09c8\u09b2\u09c0 \u0986\u09af\u09bc\u09a4\u09cd\u09a4\u09cd\u09ac was . ||| -219.999
+ however , rabindranath more than one \u09b6\u09c8\u09b2\u09c0 \u0986\u09af\u09bc\u09a4\u09cd\u09a4\u09cd\u09ac a . ||| -220.052
+ however , rabindranath more than one \u09b6\u09c8\u09b2\u09c0 \u0986\u09af\u09bc\u09a4\u09cd\u09a4\u09cd\u09ac by . ||| -220.167
+ rabindranath , more than one \u09b6\u09c8\u09b2\u09c0 \u0986\u09af\u09bc\u09a4\u09cd\u09a4\u09cd\u09ac a . ||| -220.337
+ rabindranath however , more than one \u09b6\u09c8\u09b2\u09c0 \u0986\u09af\u09bc\u09a4\u09cd\u09a4\u09cd\u09ac a . ||| -220.438
+ rabindranath , more than one \u09b6\u09c8\u09b2\u09c0 \u0986\u09af\u09bc\u09a4\u09cd\u09a4\u09cd\u09ac by . ||| -220.452
+ labour economics ||| -0.994
+ labor economy ||| -1.238
+ labor economy . ||| -2.085
+ labor economics ||| -3.592
+ labour economy ||| -4.189
+ the labor economy ||| -4.211
+ britain was once upon a time the main and his economic power in the world . ||| -20.876
+ britain was once upon a time the and his economic power in the world . ||| -21.159
+ britain was once upon a time his and the main economic power in the world . ||| -21.293
+ britain at some point of time was the main and his economic power in the world . ||| -21.391
+ britain at the same time was the main and his economic power in the world . ||| -21.710
+ britain at one point of time was the main and his economic power in the world . ||| -21.899
+ britain once upon a time was the main and his economic power in the world . ||| -22.348
+ britain was once upon a time main and his economic power in the world . ||| -22.745
+ britain at some point of time in the world was the main economic power and his . ||| -22.813
+ britain at some point of time was his and the main economic power in the world . ||| -22.850
+ movement against the military rule and \u0997\u09cb\u09b7\u09cd\u09a0\u09c0\u0997\u09a4 of pakistan was against protest and independence movement of the bengalis , \u09a7\u09be\u09ac\u09bf\u09a4 to for he widely praised . ||| -266.275
+ movement against the military rule and pakistan \u0997\u09cb\u09b7\u09cd\u09a0\u09c0\u0997\u09a4 was against protest and independence movement of the bengalis , \u09a7\u09be\u09ac\u09bf\u09a4 to for he widely praised . ||| -266.393
+ movement against the pakistani military rule and \u0997\u09cb\u09b7\u09cd\u09a0\u09c0\u0997\u09a4 of pakistan was against protest and independence movement of the bengalis , \u09a7\u09be\u09ac\u09bf\u09a4 to for he widely praised . ||| -266.487
+ movement against the military rule and \u0997\u09cb\u09b7\u09cd\u09a0\u09c0\u0997\u09a4 of pakistan was against protest and independence movement of the bengalis from \u09a7\u09be\u09ac\u09bf\u09a4 to for he widely praised . ||| -266.505
+ movement against the military rule and the \u0997\u09cb\u09b7\u09cd\u09a0\u09c0\u0997\u09a4 was against protest and independence movement of the bengalis , \u09a7\u09be\u09ac\u09bf\u09a4 to for he widely praised . ||| -266.536
+ movement against the military rule and \u0997\u09cb\u09b7\u09cd\u09a0\u09c0\u0997\u09a4 of pakistan was against protest and movement of the bengalis independence , \u09a7\u09be\u09ac\u09bf\u09a4 to for he widely praised . ||| -266.570
+ movement against the pakistani military rule and pakistan \u0997\u09cb\u09b7\u09cd\u09a0\u09c0\u0997\u09a4 was against protest and independence movement of the bengalis , \u09a7\u09be\u09ac\u09bf\u09a4 to for he widely praised . ||| -266.604
+ movement against the military rule and pakistan \u0997\u09cb\u09b7\u09cd\u09a0\u09c0\u0997\u09a4 was against protest and independence movement of the bengalis from \u09a7\u09be\u09ac\u09bf\u09a4 to for he widely praised . ||| -266.622
+ movement against the military rule and pakistan \u0997\u09cb\u09b7\u09cd\u09a0\u09c0\u0997\u09a4 was against protest and movement of the bengalis independence , \u09a7\u09be\u09ac\u09bf\u09a4 to for he widely praised . ||| -266.687
+ movement against the pakistani military rule and \u0997\u09cb\u09b7\u09cd\u09a0\u09c0\u0997\u09a4 of pakistan was against protest and independence movement of the bengalis from \u09a7\u09be\u09ac\u09bf\u09a4 to for he widely praised . ||| -266.716
+ here is mentioned that were internet and other name of world wide web word to be but actually \u09b6\u09ac\u09cd\u09a6\u09a6\u09cd\u09ac\u09af\u09bc different subject to the . ||| -171.397
+ here is mentioned that were internet and other name of world wide web word to be , but actually \u09b6\u09ac\u09cd\u09a6\u09a6\u09cd\u09ac\u09af\u09bc different subject to the . ||| -171.639
+ here is mentioned that were internet and other name of world wide web to be word , but actually \u09b6\u09ac\u09cd\u09a6\u09a6\u09cd\u09ac\u09af\u09bc different subject to the . ||| -171.721
+ here is mentioned that were internet and world wide web \u2019 s synonymous to be word , but actually \u09b6\u09ac\u09cd\u09a6\u09a6\u09cd\u09ac\u09af\u09bc different subject to the . ||| -172.036
+ here is mentioned that were internet and other name of world wide web word to be though actually \u09b6\u09ac\u09cd\u09a6\u09a6\u09cd\u09ac\u09af\u09bc different subject to the . ||| -172.203
+ here is mentioned that many internet and other name of world wide web word to be but actually \u09b6\u09ac\u09cd\u09a6\u09a6\u09cd\u09ac\u09af\u09bc different subject to the . ||| -172.747
+ here is mentioned that many internet and other name of world wide web word to be , but actually \u09b6\u09ac\u09cd\u09a6\u09a6\u09cd\u09ac\u09af\u09bc different subject to the . ||| -172.989
+ here is mentioned that were internet and other name of world wide web word to be but actually \u09b6\u09ac\u09cd\u09a6\u09a6\u09cd\u09ac\u09af\u09bc different subject indicates the . ||| -172.999
+ here is mentioned that many internet and other name of world wide web to be word , but actually \u09b6\u09ac\u09cd\u09a6\u09a6\u09cd\u09ac\u09af\u09bc different subject to the . ||| -173.072
+ here is mentioned that were internet and other name of world wide web word to be , but actually \u09b6\u09ac\u09cd\u09a6\u09a6\u09cd\u09ac\u09af\u09bc different subject indicates the . ||| -173.241
+ . the related z polar co-ordinate two are r = . ||| -31.445
+ . the related z polar co-ordinate two is r = . ||| -31.908
+ . the related of z polar co-ordinate two are r = . ||| -32.274
+ . z its related polar co-ordinate two are r = . ||| -32.722
+ . the related of z polar co-ordinate two is r = . ||| -32.737
+ in the related z polar co-ordinate two are r = . ||| -32.749
+ . z the related polar co-ordinate two are r = . ||| -32.809
+ november ||| 1.122
+ november . ||| -4.248
+ \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0 ||| -103.288
+ 1972 \u0996\u09cd\u09b0\u09c0\u09b8\u09cd\u099f\u09be\u09ac\u09cd\u09a6\u09c7 then \u0986\u09b0\u09aa\u09be\u09a8\u09c7\u099f\u09c7 first \u0987\u09b2\u09c7\u0995\u09cd\u099f\u09cd\u09b0\u09a8\u09bf\u0995 mail to to . ||| -337.642
+ 1972 \u0996\u09cd\u09b0\u09c0\u09b8\u09cd\u099f\u09be\u09ac\u09cd\u09a6\u09c7 " \u0986\u09b0\u09aa\u09be\u09a8\u09c7\u099f\u09c7 first \u0987\u09b2\u09c7\u0995\u09cd\u099f\u09cd\u09b0\u09a8\u09bf\u0995 mail to to . ||| -338.450
+ 1972 \u0996\u09cd\u09b0\u09c0\u09b8\u09cd\u099f\u09be\u09ac\u09cd\u09a6\u09c7 then \u0986\u09b0\u09aa\u09be\u09a8\u09c7\u099f\u09c7 first ever \u0987\u09b2\u09c7\u0995\u09cd\u099f\u09cd\u09b0\u09a8\u09bf\u0995 mail to to . ||| -338.529
+ 1972 \u0996\u09cd\u09b0\u09c0\u09b8\u09cd\u099f\u09be\u09ac\u09cd\u09a6\u09c7 then \u0986\u09b0\u09aa\u09be\u09a8\u09c7\u099f\u09c7 first \u0987\u09b2\u09c7\u0995\u09cd\u099f\u09cd\u09b0\u09a8\u09bf\u0995 mail to used . ||| -338.826
+ 1972 \u0996\u09cd\u09b0\u09c0\u09b8\u09cd\u099f\u09be\u09ac\u09cd\u09a6\u09c7 then \u0986\u09b0\u09aa\u09be\u09a8\u09c7\u099f\u09c7 first \u0987\u09b2\u09c7\u0995\u09cd\u099f\u09cd\u09b0\u09a8\u09bf\u0995 to mail to . ||| -339.004
+ \u0996\u09cd\u09b0\u09c0\u09b8\u09cd\u099f\u09be\u09ac\u09cd\u09a6\u09c7 1972 then \u0986\u09b0\u09aa\u09be\u09a8\u09c7\u099f\u09c7 first \u0987\u09b2\u09c7\u0995\u09cd\u099f\u09cd\u09b0\u09a8\u09bf\u0995 mail to to . ||| -339.066
+ 1972 \u0996\u09cd\u09b0\u09c0\u09b8\u09cd\u099f\u09be\u09ac\u09cd\u09a6\u09c7 then professor \u0986\u09b0\u09aa\u09be\u09a8\u09c7\u099f\u09c7 first \u0987\u09b2\u09c7\u0995\u09cd\u099f\u09cd\u09b0\u09a8\u09bf\u0995 mail to to . ||| -339.213
+ 1972 \u0996\u09cd\u09b0\u09c0\u09b8\u09cd\u099f\u09be\u09ac\u09cd\u09a6\u09c7 then \u0986\u09b0\u09aa\u09be\u09a8\u09c7\u099f\u09c7 first \u0987\u09b2\u09c7\u0995\u09cd\u099f\u09cd\u09b0\u09a8\u09bf\u0995 mail to arrested . ||| -339.272
+ 1972 \u0996\u09cd\u09b0\u09c0\u09b8\u09cd\u099f\u09be\u09ac\u09cd\u09a6\u09c7 then first \u0987\u09b2\u09c7\u0995\u09cd\u099f\u09cd\u09b0\u09a8\u09bf\u0995 \u0986\u09b0\u09aa\u09be\u09a8\u09c7\u099f\u09c7 mail to to . ||| -339.300
+ 1972 \u0996\u09cd\u09b0\u09c0\u09b8\u09cd\u099f\u09be\u09ac\u09cd\u09a6\u09c7 first \u0987\u09b2\u09c7\u0995\u09cd\u099f\u09cd\u09b0\u09a8\u09bf\u0995 then \u0986\u09b0\u09aa\u09be\u09a8\u09c7\u099f\u09c7 mail to to . ||| -339.300
+ \u099c\u09c0\u09ac science that \u09b6\u09be\u0996\u09be\u09af\u09bc fungi and the practical about the subject to he was \u099b\u09a4\u09cd\u09b0\u09be\u0995\u09ac\u09bf\u09a6\u09cd\u09af\u09be -lrb- mycology -rrb- ||| -453.504
+ \u099c\u09c0\u09ac science that \u09b6\u09be\u0996\u09be\u09af\u09bc fungi and its practical about the subject to he was \u099b\u09a4\u09cd\u09b0\u09be\u0995\u09ac\u09bf\u09a6\u09cd\u09af\u09be -lrb- mycology -rrb- ||| -453.963
+ \u099c\u09c0\u09ac science that \u09b6\u09be\u0996\u09be\u09af\u09bc fungi and the practical about the subject to he \u099b\u09a4\u09cd\u09b0\u09be\u0995\u09ac\u09bf\u09a6\u09cd\u09af\u09be -lrb- mycology -rrb- ||| -454.443
+ \u099c\u09c0\u09ac science that \u09b6\u09be\u0996\u09be\u09af\u09bc fungus and the practical about the subject to he was \u099b\u09a4\u09cd\u09b0\u09be\u0995\u09ac\u09bf\u09a6\u09cd\u09af\u09be -lrb- mycology -rrb- ||| -454.447
+ \u099c\u09c0\u09ac science the \u09b6\u09be\u0996\u09be\u09af\u09bc fungi and the practical about the subject to he was \u099b\u09a4\u09cd\u09b0\u09be\u0995\u09ac\u09bf\u09a6\u09cd\u09af\u09be -lrb- mycology -rrb- ||| -454.737
+ \u099c\u09c0\u09ac science that \u09b6\u09be\u0996\u09be\u09af\u09bc fungi and the about the practical subject to he was \u099b\u09a4\u09cd\u09b0\u09be\u0995\u09ac\u09bf\u09a6\u09cd\u09af\u09be -lrb- mycology -rrb- ||| -454.823
+ \u099c\u09c0\u09ac science that \u09b6\u09be\u0996\u09be\u09af\u09bc fungi and the practical about the issue to he was \u099b\u09a4\u09cd\u09b0\u09be\u0995\u09ac\u09bf\u09a6\u09cd\u09af\u09be -lrb- mycology -rrb- ||| -454.872
+ \u099c\u09c0\u09ac science that \u09b6\u09be\u0996\u09be\u09af\u09bc fungi and its practical about the subject to he \u099b\u09a4\u09cd\u09b0\u09be\u0995\u09ac\u09bf\u09a6\u09cd\u09af\u09be -lrb- mycology -rrb- ||| -454.901
+ \u099c\u09c0\u09ac science that the \u09b6\u09be\u0996\u09be\u09af\u09bc fungi and the practical about the subject to he was \u099b\u09a4\u09cd\u09b0\u09be\u0995\u09ac\u09bf\u09a6\u09cd\u09af\u09be -lrb- mycology -rrb- ||| -454.903
+ \u099c\u09c0\u09ac science that \u09b6\u09be\u0996\u09be\u09af\u09bc fungi and its practical about the issue to he was \u099b\u09a4\u09cd\u09b0\u09be\u0995\u09ac\u09bf\u09a6\u09cd\u09af\u09be -lrb- mycology -rrb- ||| -455.330
+ water river from \u0989\u09a0\u09be\u09a8\u09cb was some a \u09a6\u09bf\u09af\u09bc\u09c7- \u09a6\u09a1\u09bc\u09bf and \u09ac\u09be\u09b2\u099f\u09bf\u09b0 through water used by animal world a system . ||| -467.773
+ water from the river \u0989\u09a0\u09be\u09a8\u09cb was some a \u09a6\u09bf\u09af\u09bc\u09c7- \u09a6\u09a1\u09bc\u09bf and \u09ac\u09be\u09b2\u099f\u09bf\u09b0 through water used by animal world a system . ||| -467.834
+ water river from \u0989\u09a0\u09be\u09a8\u09cb was some a \u09a6\u09bf\u09af\u09bc\u09c7- \u09a6\u09a1\u09bc\u09bf and \u09ac\u09be\u09b2\u099f\u09bf\u09b0 through water used by animal world a method . ||| -468.060
+ water from the river \u0989\u09a0\u09be\u09a8\u09cb was some a \u09a6\u09bf\u09af\u09bc\u09c7- \u09a6\u09a1\u09bc\u09bf and \u09ac\u09be\u09b2\u099f\u09bf\u09b0 through water used by animal world a method . ||| -468.122
+ water river from \u0989\u09a0\u09be\u09a8\u09cb was some purs \u09a6\u09bf\u09af\u09bc\u09c7- \u09a6\u09a1\u09bc\u09bf and \u09ac\u09be\u09b2\u099f\u09bf\u09b0 through water used by animal world a system . ||| -468.162
+ water from the river \u0989\u09a0\u09be\u09a8\u09cb was some purs \u09a6\u09bf\u09af\u09bc\u09c7- \u09a6\u09a1\u09bc\u09bf and \u09ac\u09be\u09b2\u099f\u09bf\u09b0 through water used by animal world a system . ||| -468.224
+ water river from \u0989\u09a0\u09be\u09a8\u09cb it was some a \u09a6\u09bf\u09af\u09bc\u09c7- \u09a6\u09a1\u09bc\u09bf and \u09ac\u09be\u09b2\u099f\u09bf\u09b0 through water used by animal world a system . ||| -468.293
+ water from the river \u0989\u09a0\u09be\u09a8\u09cb it was some a \u09a6\u09bf\u09af\u09bc\u09c7- \u09a6\u09a1\u09bc\u09bf and \u09ac\u09be\u09b2\u099f\u09bf\u09b0 through water used by animal world a system . ||| -468.355
+ water river from \u0989\u09a0\u09be\u09a8\u09cb was some purs \u09a6\u09bf\u09af\u09bc\u09c7- \u09a6\u09a1\u09bc\u09bf and \u09ac\u09be\u09b2\u099f\u09bf\u09b0 through water used by animal world a method . ||| -468.450
+ water from the river \u0989\u09a0\u09be\u09a8\u09cb was some purs \u09a6\u09bf\u09af\u09bc\u09c7- \u09a6\u09a1\u09bc\u09bf and \u09ac\u09be\u09b2\u099f\u09bf\u09b0 through water used by animal world a method . ||| -468.512
+ among these there are tribal dance lokuj dance classical dance etc . ||| -25.170
+ among these there are tribal dance influences impact dance classical dance etc . ||| -25.620
+ among them are tribal dance lokuj dance classical dance etc . ||| -26.831
+ among these there is tribal dance lokuj dance classical dance etc . ||| -27.046
+ among these there is tribal dance influences impact dance classical dance etc . ||| -27.496
+ the oldest literature at first \u09ae\u09cc\u0996\u09bf\u0995\u09ad\u09be\u09ac\u09c7 and later written form is the . ||| -128.172
+ the oldest literature at first \u09ae\u09cc\u0996\u09bf\u0995\u09ad\u09be\u09ac\u09c7 and later written form is in the . ||| -128.707
+ the oldest literature at first \u09ae\u09cc\u0996\u09bf\u0995\u09ad\u09be\u09ac\u09c7 and later written form is in . ||| -128.982
+ the oldest literature at first \u09ae\u09cc\u0996\u09bf\u0995\u09ad\u09be\u09ac\u09c7 and later written in the in the . ||| -129.377
+ the oldest literature at first \u09ae\u09cc\u0996\u09bf\u0995\u09ad\u09be\u09ac\u09c7 and later written in the the . ||| -129.578
+ the oldest literature first \u09ae\u09cc\u0996\u09bf\u0995\u09ad\u09be\u09ac\u09c7 and later written form is the . ||| -129.651
+ the oldest literature at first \u09ae\u09cc\u0996\u09bf\u0995\u09ad\u09be\u09ac\u09c7 and later written form the in the . ||| -129.915
+ the oldest literature at first \u09ae\u09cc\u0996\u09bf\u0995\u09ad\u09be\u09ac\u09c7 and later written in the in . ||| -129.925
+ the oldest literature at first \u09ae\u09cc\u0996\u09bf\u0995\u09ad\u09be\u09ac\u09c7 and later written form the the . ||| -129.969
+ the oldest literature at first \u09ae\u09cc\u0996\u09bf\u0995\u09ad\u09be\u09ac\u09c7 and later written form the in . ||| -130.156
+ in 1989 , the \u0997\u09a3\u09b6\u09a4\u09cd\u09b0\u09c1 in this film his conducted \u09a4\u09c1\u09b2\u09a8\u09be\u09ae\u09c2\u09b2\u0995\u09ad\u09be\u09ac\u09c7 \u09a6\u09c1\u09b0\u09cd\u09ac\u09b2 and this \u09a6\u09c0\u09b0\u09cd\u0998\u09a6\u09bf\u09a8\u09c7\u09b0 \u0985\u09b8\u09c1\u09b8\u09cd\u09a5\u09a4\u09be\u09b6\u09c7\u09b7\u09c7 came back , after \u09aa\u09c1\u09a8\u09b0\u09cd\u09aa\u09cd\u09b0\u099a\u09c7\u09b7\u09cd\u099f\u09be of satyajit 's films , a is considered as . ||| -681.400
+ in 1989 , the \u0997\u09a3\u09b6\u09a4\u09cd\u09b0\u09c1 in this film his conducted \u09a4\u09c1\u09b2\u09a8\u09be\u09ae\u09c2\u09b2\u0995\u09ad\u09be\u09ac\u09c7 \u09a6\u09c1\u09b0\u09cd\u09ac\u09b2 and this \u09a6\u09c0\u09b0\u09cd\u0998\u09a6\u09bf\u09a8\u09c7\u09b0 \u0985\u09b8\u09c1\u09b8\u09cd\u09a5\u09a4\u09be\u09b6\u09c7\u09b7\u09c7 came back , after \u09aa\u09c1\u09a8\u09b0\u09cd\u09aa\u09cd\u09b0\u099a\u09c7\u09b7\u09cd\u099f\u09be of satyajit \u2019 s films , a is considered as . ||| -681.602
+ in 1989 , the \u0997\u09a3\u09b6\u09a4\u09cd\u09b0\u09c1 in this film his between \u09a4\u09c1\u09b2\u09a8\u09be\u09ae\u09c2\u09b2\u0995\u09ad\u09be\u09ac\u09c7 \u09a6\u09c1\u09b0\u09cd\u09ac\u09b2 and this \u09a6\u09c0\u09b0\u09cd\u0998\u09a6\u09bf\u09a8\u09c7\u09b0 \u0985\u09b8\u09c1\u09b8\u09cd\u09a5\u09a4\u09be\u09b6\u09c7\u09b7\u09c7 came back , after \u09aa\u09c1\u09a8\u09b0\u09cd\u09aa\u09cd\u09b0\u099a\u09c7\u09b7\u09cd\u099f\u09be of satyajit 's films , a is considered as . ||| -681.620
+ in the year 1989 the \u0997\u09a3\u09b6\u09a4\u09cd\u09b0\u09c1 in this film his conducted \u09a4\u09c1\u09b2\u09a8\u09be\u09ae\u09c2\u09b2\u0995\u09ad\u09be\u09ac\u09c7 \u09a6\u09c1\u09b0\u09cd\u09ac\u09b2 and this \u09a6\u09c0\u09b0\u09cd\u0998\u09a6\u09bf\u09a8\u09c7\u09b0 \u0985\u09b8\u09c1\u09b8\u09cd\u09a5\u09a4\u09be\u09b6\u09c7\u09b7\u09c7 came back , after \u09aa\u09c1\u09a8\u09b0\u09cd\u09aa\u09cd\u09b0\u099a\u09c7\u09b7\u09cd\u099f\u09be of satyajit 's films , a is considered as . ||| -681.675
+ in 1989 , the \u0997\u09a3\u09b6\u09a4\u09cd\u09b0\u09c1 film his conducted \u09a4\u09c1\u09b2\u09a8\u09be\u09ae\u09c2\u09b2\u0995\u09ad\u09be\u09ac\u09c7 \u09a6\u09c1\u09b0\u09cd\u09ac\u09b2 and this \u09a6\u09c0\u09b0\u09cd\u0998\u09a6\u09bf\u09a8\u09c7\u09b0 \u0985\u09b8\u09c1\u09b8\u09cd\u09a5\u09a4\u09be\u09b6\u09c7\u09b7\u09c7 came back , after \u09aa\u09c1\u09a8\u09b0\u09cd\u09aa\u09cd\u09b0\u099a\u09c7\u09b7\u09cd\u099f\u09be of satyajit 's films , a is considered as . ||| -681.684
+ in 1989 , the \u0997\u09a3\u09b6\u09a4\u09cd\u09b0\u09c1 in this film his between \u09a4\u09c1\u09b2\u09a8\u09be\u09ae\u09c2\u09b2\u0995\u09ad\u09be\u09ac\u09c7 \u09a6\u09c1\u09b0\u09cd\u09ac\u09b2 and this \u09a6\u09c0\u09b0\u09cd\u0998\u09a6\u09bf\u09a8\u09c7\u09b0 \u0985\u09b8\u09c1\u09b8\u09cd\u09a5\u09a4\u09be\u09b6\u09c7\u09b7\u09c7 came back , after \u09aa\u09c1\u09a8\u09b0\u09cd\u09aa\u09cd\u09b0\u099a\u09c7\u09b7\u09cd\u099f\u09be of satyajit \u2019 s films , a is considered as . ||| -681.822
+ on the year 1989 the \u0997\u09a3\u09b6\u09a4\u09cd\u09b0\u09c1 in this film his conducted \u09a4\u09c1\u09b2\u09a8\u09be\u09ae\u09c2\u09b2\u0995\u09ad\u09be\u09ac\u09c7 \u09a6\u09c1\u09b0\u09cd\u09ac\u09b2 and this \u09a6\u09c0\u09b0\u09cd\u0998\u09a6\u09bf\u09a8\u09c7\u09b0 \u0985\u09b8\u09c1\u09b8\u09cd\u09a5\u09a4\u09be\u09b6\u09c7\u09b7\u09c7 came back , after \u09aa\u09c1\u09a8\u09b0\u09cd\u09aa\u09cd\u09b0\u099a\u09c7\u09b7\u09cd\u099f\u09be of satyajit 's films , a is considered as . ||| -681.844
+ in 1989 , the \u0997\u09a3\u09b6\u09a4\u09cd\u09b0\u09c1 in this film his , \u09a4\u09c1\u09b2\u09a8\u09be\u09ae\u09c2\u09b2\u0995\u09ad\u09be\u09ac\u09c7 \u09a6\u09c1\u09b0\u09cd\u09ac\u09b2 and this \u09a6\u09c0\u09b0\u09cd\u0998\u09a6\u09bf\u09a8\u09c7\u09b0 \u0985\u09b8\u09c1\u09b8\u09cd\u09a5\u09a4\u09be\u09b6\u09c7\u09b7\u09c7 came back , after \u09aa\u09c1\u09a8\u09b0\u09cd\u09aa\u09cd\u09b0\u099a\u09c7\u09b7\u09cd\u099f\u09be of satyajit 's films , a is considered as . ||| -681.861
+ in the year 1989 the \u0997\u09a3\u09b6\u09a4\u09cd\u09b0\u09c1 in this film his conducted \u09a4\u09c1\u09b2\u09a8\u09be\u09ae\u09c2\u09b2\u0995\u09ad\u09be\u09ac\u09c7 \u09a6\u09c1\u09b0\u09cd\u09ac\u09b2 and this \u09a6\u09c0\u09b0\u09cd\u0998\u09a6\u09bf\u09a8\u09c7\u09b0 \u0985\u09b8\u09c1\u09b8\u09cd\u09a5\u09a4\u09be\u09b6\u09c7\u09b7\u09c7 came back , after \u09aa\u09c1\u09a8\u09b0\u09cd\u09aa\u09cd\u09b0\u099a\u09c7\u09b7\u09cd\u099f\u09be of satyajit \u2019 s films , a is considered as . ||| -681.877
+ in 1989 , the \u0997\u09a3\u09b6\u09a4\u09cd\u09b0\u09c1 film his conducted \u09a4\u09c1\u09b2\u09a8\u09be\u09ae\u09c2\u09b2\u0995\u09ad\u09be\u09ac\u09c7 \u09a6\u09c1\u09b0\u09cd\u09ac\u09b2 and this \u09a6\u09c0\u09b0\u09cd\u0998\u09a6\u09bf\u09a8\u09c7\u09b0 \u0985\u09b8\u09c1\u09b8\u09cd\u09a5\u09a4\u09be\u09b6\u09c7\u09b7\u09c7 came back , after \u09aa\u09c1\u09a8\u09b0\u09cd\u09aa\u09cd\u09b0\u099a\u09c7\u09b7\u09cd\u099f\u09be of satyajit \u2019 s films , a is considered as . ||| -681.886
+ the \u09ac\u09b2\u09ac\u09bf\u09a6\u09cd\u09af\u09be ||| -108.280
+ mathematical \u09ac\u09b2\u09ac\u09bf\u09a6\u09cd\u09af\u09be ||| -108.571
+ \u2022 \u09ac\u09b2\u09ac\u09bf\u09a6\u09cd\u09af\u09be ||| -109.025
+ . \u09ac\u09b2\u09ac\u09bf\u09a6\u09cd\u09af\u09be ||| -109.415
+ \u09ac\u09b2\u09ac\u09bf\u09a6\u09cd\u09af\u09be mathematical theory . ||| -110.926
+ mathematical theory . \u09ac\u09b2\u09ac\u09bf\u09a6\u09cd\u09af\u09be ||| -113.178
+ other \u09b8\u09cd\u09ac\u09a4\u09cd\u09a4\u09cd\u09ac-\u09b8\u0982\u09b0\u0995\u09cd\u09b7\u09bf\u09a4 operating system like windows and mac os to linux in different different . ||| -126.793
+ other \u09b8\u09cd\u09ac\u09a4\u09cd\u09a4\u09cd\u09ac-\u09b8\u0982\u09b0\u0995\u09cd\u09b7\u09bf\u09a4 operating systems like windows and mac os to linux in different different . ||| -126.966
+ other \u09b8\u09cd\u09ac\u09a4\u09cd\u09a4\u09cd\u09ac-\u09b8\u0982\u09b0\u0995\u09cd\u09b7\u09bf\u09a4 operating system such as windows and mac os to linux in different different . ||| -127.419
+ other \u09b8\u09cd\u09ac\u09a4\u09cd\u09a4\u09cd\u09ac-\u09b8\u0982\u09b0\u0995\u09cd\u09b7\u09bf\u09a4 operating system like the windows and mac os to linux in different different . ||| -127.754
+ other \u09b8\u09cd\u09ac\u09a4\u09cd\u09a4\u09cd\u09ac-\u09b8\u0982\u09b0\u0995\u09cd\u09b7\u09bf\u09a4 operating system like windows and mac os from linux in different different . ||| -127.852
+ other \u09b8\u09cd\u09ac\u09a4\u09cd\u09a4\u09cd\u09ac-\u09b8\u0982\u09b0\u0995\u09cd\u09b7\u09bf\u09a4 operating systems like the windows and mac os to linux in different different . ||| -127.927
+ other \u09b8\u09cd\u09ac\u09a4\u09cd\u09a4\u09cd\u09ac-\u09b8\u0982\u09b0\u0995\u09cd\u09b7\u09bf\u09a4 operating systems like windows and mac os from linux in different different . ||| -128.024
+ other \u09b8\u09cd\u09ac\u09a4\u09cd\u09a4\u09cd\u09ac-\u09b8\u0982\u09b0\u0995\u09cd\u09b7\u09bf\u09a4 linux operating system like the windows and mac os to do their different . ||| -128.239
+ other \u09b8\u09cd\u09ac\u09a4\u09cd\u09a4\u09cd\u09ac-\u09b8\u0982\u09b0\u0995\u09cd\u09b7\u09bf\u09a4 operating system like windows and mac os to linux in different separate . ||| -128.265
+ other \u09b8\u09cd\u09ac\u09a4\u09cd\u09a4\u09cd\u09ac-\u09b8\u0982\u09b0\u0995\u09cd\u09b7\u09bf\u09a4 linux operating system like the windows and mac os to in different different . ||| -128.309
+ asia \u099f\u09be\u0987\u09ae\u09b8\u09c7\u09b0 \u09ad\u09be\u09b7\u09cd\u09af according to , ||| -215.125
+ asia \u099f\u09be\u0987\u09ae\u09b8\u09c7\u09b0 according to the \u09ad\u09be\u09b7\u09cd\u09af , ||| -215.700
+ asia \u099f\u09be\u0987\u09ae\u09b8\u09c7\u09b0 according to \u09ad\u09be\u09b7\u09cd\u09af , ||| -216.133
+ asia \u099f\u09be\u0987\u09ae\u09b8\u09c7\u09b0 \u09ad\u09be\u09b7\u09cd\u09af to the , ||| -217.133
+ asia \u099f\u09be\u0987\u09ae\u09b8\u09c7\u09b0 \u09ad\u09be\u09b7\u09cd\u09af to , ||| -217.215
+ asia \u099f\u09be\u0987\u09ae\u09b8\u09c7\u09b0 \u09ad\u09be\u09b7\u09cd\u09af according to the , ||| -217.239
+ the \u099f\u09be\u0987\u09ae\u09b8\u09c7\u09b0 \u09ad\u09be\u09b7\u09cd\u09af according to , ||| -217.435
+ open source or open source -lrb- open source -rrb- the money is computer software the source code or the \u09b8\u09be\u0982\u0995\u09c7\u09a4\u09bf\u0995 language open way \u09ac\u09bf\u09a4\u09b0\u09a3 to . ||| -461.854
+ open source or open source -lrb- open source -rrb- the money is computer software the source code or main \u09b8\u09be\u0982\u0995\u09c7\u09a4\u09bf\u0995 language open way \u09ac\u09bf\u09a4\u09b0\u09a3 to . ||| -461.897
+ open source or open source -lrb- open source -rrb- the money is computer software the source code or the \u09b8\u09be\u0982\u0995\u09c7\u09a4\u09bf\u0995 language open way \u09ac\u09bf\u09a4\u09b0\u09a3 is . ||| -462.229
+ open source or open source -lrb- open source -rrb- the money is computer software the source code or the \u09b8\u09be\u0982\u0995\u09c7\u09a4\u09bf\u0995 language \u09ac\u09bf\u09a4\u09b0\u09a3 to open way . ||| -462.237
+ open source or open source -lrb- open source -rrb- the money is computer software the source code or main \u09b8\u09be\u0982\u0995\u09c7\u09a4\u09bf\u0995 language open way \u09ac\u09bf\u09a4\u09b0\u09a3 is . ||| -462.272
+ open source or open source -lrb- open source -rrb- the money is computer software the source code or main \u09b8\u09be\u0982\u0995\u09c7\u09a4\u09bf\u0995 language \u09ac\u09bf\u09a4\u09b0\u09a3 to open way . ||| -462.279
+ open source or open source -lrb- open source -rrb- the money is computer software the source code or the \u09b8\u09be\u0982\u0995\u09c7\u09a4\u09bf\u0995 language free way \u09ac\u09bf\u09a4\u09b0\u09a3 to . ||| -462.350
+ open source or open source -lrb- open source -rrb- the money is computer software the source code or main \u09b8\u09be\u0982\u0995\u09c7\u09a4\u09bf\u0995 language free way \u09ac\u09bf\u09a4\u09b0\u09a3 to . ||| -462.392
+ open source or open source -lrb- open source -rrb- the money is computer software the source code or the \u09b8\u09be\u0982\u0995\u09c7\u09a4\u09bf\u0995 language open in \u09ac\u09bf\u09a4\u09b0\u09a3 to . ||| -462.512
+ open source or open source -lrb- open source -rrb- the money is computer software the source code or main \u09b8\u09be\u0982\u0995\u09c7\u09a4\u09bf\u0995 language open in \u09ac\u09bf\u09a4\u09b0\u09a3 to . ||| -462.554
+ bangladesh \u0985\u09a8\u09b2\u09be\u0987\u09a8\u09c7 dhaka ||| -107.382
+ bangladesh \u0985\u09a8\u09b2\u09be\u0987\u09a8\u09c7 of dhaka ||| -108.858
+ bangladesh \u0985\u09a8\u09b2\u09be\u0987\u09a8\u09c7 the ||| -110.883
+ \u0985\u09a8\u09b2\u09be\u0987\u09a8\u09c7 of bangladesh dhaka ||| -111.027
+ bangladesh dhaka \u0985\u09a8\u09b2\u09be\u0987\u09a8\u09c7 ||| -111.126
+ \u0985\u09a8\u09b2\u09be\u0987\u09a8\u09c7 bangladesh dhaka ||| -111.514
+ the first world war germany \u09b9\u09c7\u09b0\u09c7 can be seen . ||| -109.139
+ the first world war germany \u09b9\u09c7\u09b0\u09c7 can be . ||| -109.520
+ the first world war germany can be \u09b9\u09c7\u09b0\u09c7 . ||| -110.263
+ the first world war german \u09b9\u09c7\u09b0\u09c7 can be seen . ||| -110.711
+ the first world war german \u09b9\u09c7\u09b0\u09c7 can be . ||| -111.092
+ germany the first world war \u09b9\u09c7\u09b0\u09c7 can be seen . ||| -111.122
+ but this subject is to understand for even research to going on . ||| -32.160
+ but this subject is to understand for even research progress going on . ||| -32.389
+ but this subject is to understand even for research to going on . ||| -32.514
+ but this subject is to understand for presently research to going on . ||| -32.599
+ but this subject is to understand presently for research to going on . ||| -32.608
+ but this subject is to understand even for research progress going on . ||| -32.743
+ but this subject is to understand for presently research progress going on . ||| -32.828
+ but this subject is to understand presently for research progress going on . ||| -32.836
+ but this subject is to for even research to going on . ||| -33.085
+ \u09b8\u09c1\u09aa\u09be\u09b0\u098f\u0987\u099a ||| -103.288
+ he was military forces for \u0986\u09a8\u09ab\u09bf\u099f was declared . ||| -118.584
+ he was military forces for \u0986\u09a8\u09ab\u09bf\u099f declared was . ||| -118.970
+ he was military forces for \u0986\u09a8\u09ab\u09bf\u099f declared was made . ||| -119.007
+ he was for military forces \u0986\u09a8\u09ab\u09bf\u099f was declared . ||| -119.052
+ he was military forces to \u0986\u09a8\u09ab\u09bf\u099f was declared . ||| -119.195
+ he was military forces for \u0986\u09a8\u09ab\u09bf\u099f declared the . ||| -119.315
+ he was military forces for \u0986\u09a8\u09ab\u09bf\u099f declared was created . ||| -119.424
+ he was for military forces \u0986\u09a8\u09ab\u09bf\u099f declared was . ||| -119.437
+ he was for military forces \u0986\u09a8\u09ab\u09bf\u099f declared was made . ||| -119.475
+ he was for the military forces \u0986\u09a8\u09ab\u09bf\u099f declared was . ||| -120.208
+ bhutto \u098f\u09cd\u09af\u09be\u09b8\u09c7\u09ae\u09cd\u09ac\u09b2\u09bf \u09ac\u09af\u09bc\u0995\u099f to 2.5 with announced that the yahya khan mujib was to form the government to \u099c\u09be\u09a8\u09be\u09b2\u09c7 he that government by \u09a8\u09c7\u09ac\u09c7\u09a8 not . ||| -478.493
+ bhutto \u098f\u09cd\u09af\u09be\u09b8\u09c7\u09ae\u09cd\u09ac\u09b2\u09bf \u09ac\u09af\u09bc\u0995\u099f to 2.5 with announced that the yahya khan mujib was to form the government for \u099c\u09be\u09a8\u09be\u09b2\u09c7 he that government by \u09a8\u09c7\u09ac\u09c7\u09a8 not . ||| -478.680
+ bhutto \u098f\u09cd\u09af\u09be\u09b8\u09c7\u09ae\u09cd\u09ac\u09b2\u09bf \u09ac\u09af\u09bc\u0995\u099f to 2.5 with announced that the yahya khan mujib was to form the government to \u099c\u09be\u09a8\u09be\u09b2\u09c7 he that to by \u09a8\u09c7\u09ac\u09c7\u09a8 not . ||| -478.753
+ bhutto \u098f\u09cd\u09af\u09be\u09b8\u09c7\u09ae\u09cd\u09ac\u09b2\u09bf \u09ac\u09af\u09bc\u0995\u099f to 2.5 with announced that the yahya khan mujib was to form the government for \u099c\u09be\u09a8\u09be\u09b2\u09c7 he that to by \u09a8\u09c7\u09ac\u09c7\u09a8 not . ||| -478.940
+ bhutto \u098f\u09cd\u09af\u09be\u09b8\u09c7\u09ae\u09cd\u09ac\u09b2\u09bf \u09ac\u09af\u09bc\u0995\u099f to 2.5 with announced that the yahya khan mujib was to form the government to \u099c\u09be\u09a8\u09be\u09b2\u09c7 he that government with \u09a8\u09c7\u09ac\u09c7\u09a8 not . ||| -478.984
+ bhutto \u098f\u09cd\u09af\u09be\u09b8\u09c7\u09ae\u09cd\u09ac\u09b2\u09bf \u09ac\u09af\u09bc\u0995\u099f to 2.5 with announced that the yahya khan mujib for to form the government \u099c\u09be\u09a8\u09be\u09b2\u09c7 he that government by \u09a8\u09c7\u09ac\u09c7\u09a8 not . ||| -478.985
+ bhutto \u098f\u09cd\u09af\u09be\u09b8\u09c7\u09ae\u09cd\u09ac\u09b2\u09bf \u09ac\u09af\u09bc\u0995\u099f to 2.5 with declared in the yahya khan mujib was to form the government to \u099c\u09be\u09a8\u09be\u09b2\u09c7 he that government by \u09a8\u09c7\u09ac\u09c7\u09a8 not . ||| -479.051
+ bhutto \u098f\u09cd\u09af\u09be\u09b8\u09c7\u09ae\u09cd\u09ac\u09b2\u09bf \u09ac\u09af\u09bc\u0995\u099f to 2.5 with announced that the yahya khan was to form the government to \u099c\u09be\u09a8\u09be\u09b2\u09c7 he that government by \u09a8\u09c7\u09ac\u09c7\u09a8 not . ||| -479.071
+ bhutto \u098f\u09cd\u09af\u09be\u09b8\u09c7\u09ae\u09cd\u09ac\u09b2\u09bf \u09ac\u09af\u09bc\u0995\u099f to 2.5 with announced that the yahya khan mujib was for to form the government \u099c\u09be\u09a8\u09be\u09b2\u09c7 he that government by \u09a8\u09c7\u09ac\u09c7\u09a8 not . ||| -479.145
+ bhutto \u098f\u09cd\u09af\u09be\u09b8\u09c7\u09ae\u09cd\u09ac\u09b2\u09bf \u09ac\u09af\u09bc\u0995\u099f to 2.5 with announced that the yahya khan mujib to form the government to \u099c\u09be\u09a8\u09be\u09b2\u09c7 he that government by \u09a8\u09c7\u09ac\u09c7\u09a8 not . ||| -479.157
+ and computer words money \u0997\u09a3\u09a8\u09be\u0995\u09be\u09b0\u09c0 machine . ||| -217.956
+ the computer words money \u0997\u09a3\u09a8\u09be\u0995\u09be\u09b0\u09c0 machine . ||| -218.249
+ and the computer words money \u0997\u09a3\u09a8\u09be\u0995\u09be\u09b0\u09c0 machine . ||| -218.435
+ and computer words of the \u0997\u09a3\u09a8\u09be\u0995\u09be\u09b0\u09c0 machine . ||| -218.670
+ and computer words means \u0997\u09a3\u09a8\u09be\u0995\u09be\u09b0\u09c0 machine . ||| -218.855
+ the computer words of the \u0997\u09a3\u09a8\u09be\u0995\u09be\u09b0\u09c0 machine . ||| -218.963
+ and computer words meaning \u0997\u09a3\u09a8\u09be\u0995\u09be\u09b0\u09c0 machine . ||| -219.032
+ and computer words the \u0997\u09a3\u09a8\u09be\u0995\u09be\u09b0\u09c0 machine . ||| -219.069
+ and the computer words of the \u0997\u09a3\u09a8\u09be\u0995\u09be\u09b0\u09c0 machine . ||| -219.149
+ on 4th july \u09e7\u09ed\u09ed\u09ec this constituents of a independence notice \u099c\u09be\u09b0\u09bf the . ||| -236.748
+ on 4th july \u09e7\u09ed\u09ed\u09ec this colonies are a independence notice \u099c\u09be\u09b0\u09bf the . ||| -236.808
+ \u09e7\u09ed\u09ed\u09ec on 4th july this constituents of a independence notice \u099c\u09be\u09b0\u09bf the . ||| -236.813
+ on 4th july \u09e7\u09ed\u09ed\u09ec this constituents a independence notice \u099c\u09be\u09b0\u09bf the . ||| -236.833
+ \u09e7\u09ed\u09ed\u09ec on 4th july this colonies are a independence notice \u099c\u09be\u09b0\u09bf the . ||| -236.873
+ \u09e7\u09ed\u09ed\u09ec on 4th july this constituents a independence notice \u099c\u09be\u09b0\u09bf the . ||| -236.899
+ on 4th july \u09e7\u09ed\u09ed\u09ec this constituents of a independence written proclamation \u099c\u09be\u09b0\u09bf the . ||| -237.042
+ on 4th july \u09e7\u09ed\u09ed\u09ec this colonies are a independence written proclamation \u099c\u09be\u09b0\u09bf the . ||| -237.102
+ \u09e7\u09ed\u09ed\u09ec on 4th july this constituents of a independence written proclamation \u099c\u09be\u09b0\u09bf the . ||| -237.108
+ on 4th july \u09e7\u09ed\u09ed\u09ec this constituents a independence written proclamation \u099c\u09be\u09b0\u09bf the . ||| -237.127
+ germany -lrb- in german : deutschland \u09a1\u09af\u09bc\u099a\u09cd the \u09b2\u09be\u09a8\u09cd\u099f\u09cd of pronounced [ d\u0254\u028ft\u0283lant ] -rrb- through a country of europe . ||| -447.278
+ germany -lrb- in german : deutschland \u09a1\u09af\u09bc\u099a\u09cd the \u09b2\u09be\u09a8\u09cd\u099f\u09cd of pronounced [ d\u0254\u028ft\u0283lant ] -rrb- the is a country of europe . ||| -447.969
+ germany -lrb- in german : deutschland \u09a1\u09af\u09bc\u099a\u09cd the \u09b2\u09be\u09a8\u09cd\u099f\u09cd of pronounced [ d\u0254\u028ft\u0283lant ] -rrb- through is a country of europe . ||| -447.976
+ germany -lrb- in german : deutschland \u09a1\u09af\u09bc\u099a\u09cd the \u09b2\u09be\u09a8\u09cd\u099f\u09cd of pronounced [ d\u0254\u028ft\u0283lant ] -rrb- middle is a country of europe . ||| -448.142
+ germany -lrb- in german : deutschland \u09a1\u09af\u09bc\u099a\u09cd the \u09b2\u09be\u09a8\u09cd\u099f\u09cd of pronounced [ d\u0254\u028ft\u0283lant ] -rrb- central is a country of europe . ||| -448.203
+ germany -lrb- in german : deutschland \u09a1\u09af\u09bc\u099a\u09cd the \u09b2\u09be\u09a8\u09cd\u099f\u09cd of pronounced [ d\u0254\u028ft\u0283lant ] -rrb- the a country of europe . ||| -448.948
+ germany -lrb- in german : deutschland \u09a1\u09af\u09bc\u099a\u09cd the \u09b2\u09be\u09a8\u09cd\u099f\u09cd of pronounced [ d\u0254\u028ft\u0283lant ] -rrb- in a country of europe . ||| -449.132
+ germany -lrb- in german : deutschland \u09a1\u09af\u09bc\u099a\u09cd the \u09b2\u09be\u09a8\u09cd\u099f\u09cd of pronounced [ d\u0254\u028ft\u0283lant ] -rrb- in is a country of europe . ||| -449.136
+ germany -lrb- in german : deutschland \u09a1\u09af\u09bc\u099a\u09cd the \u09b2\u09be\u09a8\u09cd\u099f\u09cd of pronounced [ d\u0254\u028ft\u0283lant ] -rrb- middle a country of europe . ||| -449.138
+ germany -lrb- in german : deutschland \u09a1\u09af\u09bc\u099a\u09cd the \u09b2\u09be\u09a8\u09cd\u099f\u09cd of pronounced [ d\u0254\u028ft\u0283lant ] -rrb- the middle is a country of europe . ||| -449.331
+ the main religion \u0996\u09cd\u09b0\u09bf\u09b8\u09cd\u099f\u09a7\u09b0\u09cd\u09ae from russia . ||| -110.479
+ \u0996\u09cd\u09b0\u09bf\u09b8\u09cd\u099f\u09a7\u09b0\u09cd\u09ae the main religion . ||| -110.595
+ \u0996\u09cd\u09b0\u09bf\u09b8\u09cd\u099f\u09a7\u09b0\u09cd\u09ae russia the main religion . ||| -110.737
+ \u0996\u09cd\u09b0\u09bf\u09b8\u09cd\u099f\u09a7\u09b0\u09cd\u09ae russia main religion . ||| -111.472
+ \u0996\u09cd\u09b0\u09bf\u09b8\u09cd\u099f\u09a7\u09b0\u09cd\u09ae the religion of russia . ||| -111.696
+ \u0996\u09cd\u09b0\u09bf\u09b8\u09cd\u099f\u09a7\u09b0\u09cd\u09ae russia of the main religion . ||| -112.120
+ \u0996\u09cd\u09b0\u09bf\u09b8\u09cd\u099f\u09a7\u09b0\u09cd\u09ae russia the religion . ||| -112.320
+ but \u0997\u09b2\u09a6\u09c7\u09b0 education \u09b0\u09cb\u09ae\u09be\u09a8\u09c0\u0995\u09b0\u09a3\u09c7\u09b0 \u0997\u09a4\u09bf was slow down . ||| -321.499
+ but \u0997\u09b2\u09a6\u09c7\u09b0 the \u09b0\u09cb\u09ae\u09be\u09a8\u09c0\u0995\u09b0\u09a3\u09c7\u09b0 \u0997\u09a4\u09bf was slow down . ||| -322.246
+ but \u0997\u09b2\u09a6\u09c7\u09b0 the educational \u09b0\u09cb\u09ae\u09be\u09a8\u09c0\u0995\u09b0\u09a3\u09c7\u09b0 \u0997\u09a4\u09bf was slow down . ||| -323.577
+ but \u0997\u09b2\u09a6\u09c7\u09b0 the two \u09b0\u09cb\u09ae\u09be\u09a8\u09c0\u0995\u09b0\u09a3\u09c7\u09b0 \u0997\u09a4\u09bf was slow down . ||| -323.606
+ but \u0997\u09b2\u09a6\u09c7\u09b0 are \u09b0\u09cb\u09ae\u09be\u09a8\u09c0\u0995\u09b0\u09a3\u09c7\u09b0 \u0997\u09a4\u09bf was slow down . ||| -323.608
+ but \u0997\u09b2\u09a6\u09c7\u09b0 education \u09b0\u09cb\u09ae\u09be\u09a8\u09c0\u0995\u09b0\u09a3\u09c7\u09b0 \u0997\u09a4\u09bf was too slow . ||| -323.760
+ subject category : gnu foundation ||| -4.645
+ subject class : gnu foundation ||| -5.952
+ subject : gnu foundation ||| -6.057
+ subject matter : gnu foundation ||| -6.097
+ subject category : gonu foundation ||| -6.361
+ category : gnu foundation ||| -6.519
+ economic policy and revenue \u09a8\u09c0\u09a4\u09bf\u0995\u09c7\u0993 it study the . ||| -124.215
+ economic policy and tax \u09a8\u09c0\u09a4\u09bf\u0995\u09c7\u0993 it study the . ||| -124.324
+ economic policy and tax revenue \u09a8\u09c0\u09a4\u09bf\u0995\u09c7\u0993 it study the . ||| -124.637
+ economic policy and revenue \u09a8\u09c0\u09a4\u09bf\u0995\u09c7\u0993 it study by the . ||| -125.398
+ economic policy and revenue \u09a8\u09c0\u09a4\u09bf\u0995\u09c7\u0993 it study . ||| -125.403
+ economic policy and revenue \u09a8\u09c0\u09a4\u09bf\u0995\u09c7\u0993 it study in the . ||| -125.507
+ economic policy and tax \u09a8\u09c0\u09a4\u09bf\u0995\u09c7\u0993 it study by the . ||| -125.508
+ economic policy and tax \u09a8\u09c0\u09a4\u09bf\u0995\u09c7\u0993 it study . ||| -125.513
+ economic policy and revenue \u09a8\u09c0\u09a4\u09bf\u0995\u09c7\u0993 it study of the . ||| -125.533
+ economic policy and tax \u09a8\u09c0\u09a4\u09bf\u0995\u09c7\u0993 it study in the . ||| -125.617
+ among these there are : may be \u09a4\u09cb\u09ae\u09be\u09b0 get seen \u0993\u09b0\u09c7 this any \u09b8\u09cd\u09a8\u09c7\u09b9-\u09b8\u09c1\u09b0\u09a7\u09c1\u09a8\u09c0 . ||| -335.374
+ among these there are : may be \u09a4\u09cb\u09ae\u09be\u09b0 get is \u0993\u09b0\u09c7 this any \u09b8\u09cd\u09a8\u09c7\u09b9-\u09b8\u09c1\u09b0\u09a7\u09c1\u09a8\u09c0 . ||| -335.614
+ among these there are : may be \u09a4\u09cb\u09ae\u09be\u09b0 will get seen \u0993\u09b0\u09c7 this any \u09b8\u09cd\u09a8\u09c7\u09b9-\u09b8\u09c1\u09b0\u09a7\u09c1\u09a8\u09c0 . ||| -335.670
+ among these there are : would have \u09a4\u09cb\u09ae\u09be\u09b0 get seen \u0993\u09b0\u09c7 this any \u09b8\u09cd\u09a8\u09c7\u09b9-\u09b8\u09c1\u09b0\u09a7\u09c1\u09a8\u09c0 . ||| -335.763
+ among these there are : may be \u09a4\u09cb\u09ae\u09be\u09b0 numbers is \u0993\u09b0\u09c7 this any \u09b8\u09cd\u09a8\u09c7\u09b9-\u09b8\u09c1\u09b0\u09a7\u09c1\u09a8\u09c0 . ||| -335.855
+ among these there are : may be \u09a4\u09cb\u09ae\u09be\u09b0 will get is \u0993\u09b0\u09c7 this any \u09b8\u09cd\u09a8\u09c7\u09b9-\u09b8\u09c1\u09b0\u09a7\u09c1\u09a8\u09c0 . ||| -335.910
+ among these there are : would have \u09a4\u09cb\u09ae\u09be\u09b0 get is \u0993\u09b0\u09c7 this any \u09b8\u09cd\u09a8\u09c7\u09b9-\u09b8\u09c1\u09b0\u09a7\u09c1\u09a8\u09c0 . ||| -336.003
+ among these there are : may be \u09a4\u09cb\u09ae\u09be\u09b0 get seen \u0993\u09b0\u09c7 this no \u09b8\u09cd\u09a8\u09c7\u09b9-\u09b8\u09c1\u09b0\u09a7\u09c1\u09a8\u09c0 . ||| -336.003
+ among these there are : may be \u09a4\u09cb\u09ae\u09be\u09b0 get is \u0993\u09b0\u09c7 this no \u09b8\u09cd\u09a8\u09c7\u09b9-\u09b8\u09c1\u09b0\u09a7\u09c1\u09a8\u09c0 . ||| -336.243
+ among these there are : may be \u09a4\u09cb\u09ae\u09be\u09b0 will get seen \u0993\u09b0\u09c7 this no \u09b8\u09cd\u09a8\u09c7\u09b9-\u09b8\u09c1\u09b0\u09a7\u09c1\u09a8\u09c0 . ||| -336.299
+ on 23rd april 1992 satyajit died . ||| -13.547
+ on 23rd april 1992 satyajit expired . ||| -13.865
+ on 23rd april 1992 satyajit passed away . ||| -14.349
+ , on 23rd april 1992 satyajit died . ||| -15.479
+ at this time nazrul 's medical \u09b0\u09bf\u09aa\u09cb\u09b0\u09cd\u099f stay famous \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u0995\u09a6\u09c7\u09b0 to send to . ||| -232.362
+ during this time nazrul 's medical \u09b0\u09bf\u09aa\u09cb\u09b0\u09cd\u099f stay famous \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u0995\u09a6\u09c7\u09b0 to send to . ||| -232.364
+ at this time nazrul medical \u09b0\u09bf\u09aa\u09cb\u09b0\u09cd\u099f stay famous \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u0995\u09a6\u09c7\u09b0 to send to . ||| -232.423
+ during this time nazrul medical \u09b0\u09bf\u09aa\u09cb\u09b0\u09cd\u099f stay famous \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u0995\u09a6\u09c7\u09b0 to send to . ||| -232.424
+ at this time nazrul 's medical \u09b0\u09bf\u09aa\u09cb\u09b0\u09cd\u099f stay famous \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u0995\u09a6\u09c7\u09b0 to sent to . ||| -232.483
+ during this time nazrul 's medical \u09b0\u09bf\u09aa\u09cb\u09b0\u09cd\u099f stay famous \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u0995\u09a6\u09c7\u09b0 to sent to . ||| -232.485
+ at this time nazrul medical \u09b0\u09bf\u09aa\u09cb\u09b0\u09cd\u099f stay famous \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u0995\u09a6\u09c7\u09b0 to sent to . ||| -232.544
+ during this time nazrul medical \u09b0\u09bf\u09aa\u09cb\u09b0\u09cd\u099f stay famous \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u0995\u09a6\u09c7\u09b0 to sent to . ||| -232.545
+ at this time nazrul 's medical \u09b0\u09bf\u09aa\u09cb\u09b0\u09cd\u099f earning famous \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u0995\u09a6\u09c7\u09b0 to send to . ||| -232.659
+ during this time nazrul 's medical \u09b0\u09bf\u09aa\u09cb\u09b0\u09cd\u099f earning famous \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u0995\u09a6\u09c7\u09b0 to send to . ||| -232.661
+ other than acted at different times rani \u09ae\u09c1\u0996\u09be\u09b0\u09cd\u099c\u09c0 different \u09a6\u09be\u09a4\u09ac\u09cd\u09af are connected with \u09a5\u09c7\u0995\u09c7\u099b\u09c7\u09a8 . ||| -338.434
+ apart from acted at different times rani \u09ae\u09c1\u0996\u09be\u09b0\u09cd\u099c\u09c0 different \u09a6\u09be\u09a4\u09ac\u09cd\u09af are connected with \u09a5\u09c7\u0995\u09c7\u099b\u09c7\u09a8 . ||| -338.715
+ acted other than at different times rani \u09ae\u09c1\u0996\u09be\u09b0\u09cd\u099c\u09c0 different \u09a6\u09be\u09a4\u09ac\u09cd\u09af are connected with \u09a5\u09c7\u0995\u09c7\u099b\u09c7\u09a8 . ||| -339.387
+ other than acted rani \u09ae\u09c1\u0996\u09be\u09b0\u09cd\u099c\u09c0 at different times different \u09a6\u09be\u09a4\u09ac\u09cd\u09af are connected with \u09a5\u09c7\u0995\u09c7\u099b\u09c7\u09a8 . ||| -339.774
+ other than acted at different times rani \u09ae\u09c1\u0996\u09be\u09b0\u09cd\u099c\u09c0 various \u09a6\u09be\u09a4\u09ac\u09cd\u09af are connected with \u09a5\u09c7\u0995\u09c7\u099b\u09c7\u09a8 . ||| -339.815
+ apart from acted rani \u09ae\u09c1\u0996\u09be\u09b0\u09cd\u099c\u09c0 at different times different \u09a6\u09be\u09a4\u09ac\u09cd\u09af are connected with \u09a5\u09c7\u0995\u09c7\u099b\u09c7\u09a8 . ||| -340.056
+ other than acted at different times rani \u09ae\u09c1\u0996\u09be\u09b0\u09cd\u099c\u09c0 different \u09a6\u09be\u09a4\u09ac\u09cd\u09af organizations connected with \u09a5\u09c7\u0995\u09c7\u099b\u09c7\u09a8 . ||| -341.428
+ other than acted at different times rani \u09ae\u09c1\u0996\u09be\u09b0\u09cd\u099c\u09c0 different \u09a6\u09be\u09a4\u09ac\u09cd\u09af connected with are \u09a5\u09c7\u0995\u09c7\u099b\u09c7\u09a8 . ||| -341.691
+ apart from acted at different times rani \u09ae\u09c1\u0996\u09be\u09b0\u09cd\u099c\u09c0 different \u09a6\u09be\u09a4\u09ac\u09cd\u09af organizations connected with \u09a5\u09c7\u0995\u09c7\u099b\u09c7\u09a8 . ||| -341.709
+ apart from acted at different times rani \u09ae\u09c1\u0996\u09be\u09b0\u09cd\u099c\u09c0 different \u09a6\u09be\u09a4\u09ac\u09cd\u09af connected with are \u09a5\u09c7\u0995\u09c7\u099b\u09c7\u09a8 . ||| -341.972
+ bengali literature and culture a special contribution on \u09b8\u09cd\u09ac\u09c0\u0995\u09c3\u09a4\u09bf\u09b8\u09cd\u09ac\u09b0\u09c1\u09aa year on 9th december , dhaka university he was honorary \u09a1\u09bf.\u09b2\u09bf\u099f \u0989\u09aa\u09be\u09a7\u09bf\u09a4\u09c7 awarded with the . ||| -358.420
+ bengali literature and culture his special contribution on \u09b8\u09cd\u09ac\u09c0\u0995\u09c3\u09a4\u09bf\u09b8\u09cd\u09ac\u09b0\u09c1\u09aa year on 9th december , dhaka university he was honorary \u09a1\u09bf.\u09b2\u09bf\u099f \u0989\u09aa\u09be\u09a7\u09bf\u09a4\u09c7 awarded with the . ||| -358.507
+ bengali literature and culture a special contribution on \u09b8\u09cd\u09ac\u09c0\u0995\u09c3\u09a4\u09bf\u09b8\u09cd\u09ac\u09b0\u09c1\u09aa on 9th december of year on dhaka university he was honorary \u09a1\u09bf.\u09b2\u09bf\u099f \u0989\u09aa\u09be\u09a7\u09bf\u09a4\u09c7 awarded with the . ||| -359.274
+ bengali literature and culture a special contribution on \u09b8\u09cd\u09ac\u09c0\u0995\u09c3\u09a4\u09bf\u09b8\u09cd\u09ac\u09b0\u09c1\u09aa year on 9th december , dhaka university he honorary \u09a1\u09bf.\u09b2\u09bf\u099f \u0989\u09aa\u09be\u09a7\u09bf\u09a4\u09c7 awarded with the . ||| -359.359
+ bengali literature and culture his special contribution on \u09b8\u09cd\u09ac\u09c0\u0995\u09c3\u09a4\u09bf\u09b8\u09cd\u09ac\u09b0\u09c1\u09aa on 9th december of year on dhaka university he was honorary \u09a1\u09bf.\u09b2\u09bf\u099f \u0989\u09aa\u09be\u09a7\u09bf\u09a4\u09c7 awarded with the . ||| -359.360
+ bengali literature and culture a special contribution on \u09b8\u09cd\u09ac\u09c0\u0995\u09c3\u09a4\u09bf\u09b8\u09cd\u09ac\u09b0\u09c1\u09aa year on 9th december , dhaka university was honorary \u09a1\u09bf.\u09b2\u09bf\u099f \u0989\u09aa\u09be\u09a7\u09bf\u09a4\u09c7 awarded with the . ||| -359.371
+ bengali literature and culture his special contribution on \u09b8\u09cd\u09ac\u09c0\u0995\u09c3\u09a4\u09bf\u09b8\u09cd\u09ac\u09b0\u09c1\u09aa year on 9th december , dhaka university he honorary \u09a1\u09bf.\u09b2\u09bf\u099f \u0989\u09aa\u09be\u09a7\u09bf\u09a4\u09c7 awarded with the . ||| -359.445
+ bengali literature and culture in his special contribution on \u09b8\u09cd\u09ac\u09c0\u0995\u09c3\u09a4\u09bf\u09b8\u09cd\u09ac\u09b0\u09c1\u09aa year on 9th december , dhaka university he was honorary \u09a1\u09bf.\u09b2\u09bf\u099f \u0989\u09aa\u09be\u09a7\u09bf\u09a4\u09c7 awarded with the . ||| -359.454
+ bengali literature and culture his special contribution on \u09b8\u09cd\u09ac\u09c0\u0995\u09c3\u09a4\u09bf\u09b8\u09cd\u09ac\u09b0\u09c1\u09aa year on 9th december , dhaka university was honorary \u09a1\u09bf.\u09b2\u09bf\u099f \u0989\u09aa\u09be\u09a7\u09bf\u09a4\u09c7 awarded with the . ||| -359.457
+ bengali literature and culture the special contribution on \u09b8\u09cd\u09ac\u09c0\u0995\u09c3\u09a4\u09bf\u09b8\u09cd\u09ac\u09b0\u09c1\u09aa year on 9th december , dhaka university he was honorary \u09a1\u09bf.\u09b2\u09bf\u099f \u0989\u09aa\u09be\u09a7\u09bf\u09a4\u09c7 awarded with the . ||| -359.600
+ in kolkata durga puja tourism \u0986\u0995\u09b0\u09cd\u09b7\u09a3\u0993 of the city is also a reason ||| -131.010
+ kolkata durga puja tourism \u0986\u0995\u09b0\u09cd\u09b7\u09a3\u0993 of the city is also a reason ||| -131.047
+ in kolkata durga puja tourism of the city \u0986\u0995\u09b0\u09cd\u09b7\u09a3\u0993 is also a reason ||| -132.052
+ kolkata durga puja tourism of the city \u0986\u0995\u09b0\u09cd\u09b7\u09a3\u0993 is also a reason ||| -132.088
+ in kolkata durga puja tourists spot of the city \u0986\u0995\u09b0\u09cd\u09b7\u09a3\u0993 is also a reason ||| -132.288
+ kolkata durga puja tourists spot of the city \u0986\u0995\u09b0\u09cd\u09b7\u09a3\u0993 is also a reason ||| -132.325
+ in kolkata durga puja tourism of the \u0986\u0995\u09b0\u09cd\u09b7\u09a3\u0993 is also a reason ||| -132.364
+ kolkata durga puja tourism of the \u0986\u0995\u09b0\u09cd\u09b7\u09a3\u0993 is also a reason ||| -132.401
+ kolkata 's durga puja tourism of the city \u0986\u0995\u09b0\u09cd\u09b7\u09a3\u0993 is also a reason ||| -132.643
+ the durga puja tourism of the city \u0986\u0995\u09b0\u09cd\u09b7\u09a3\u0993 is also a reason ||| -132.694
+ but many people of east germany started living in economically reach and democratic west germany , east germany government built a wall in berlin in 1961 and reinforced its boundaries . ||| -38.023
+ but many of east german started living in economically reach and democratic west germany , east germany government built a wall in berlin in 1961 and reinforced its boundaries . ||| -38.234
+ but many people of east german started living in economically reach and democratic west germany , east germany government built a wall in berlin in 1961 and reinforced its boundaries . ||| -38.458
+ but when millions of east german started living in economically reach and democratic west germany , east germany government built a wall in berlin in 1961 and reinforced its boundaries . ||| -39.049
+ but many people of east germany started living in economically reach and democratic west germany , east germany government built a wall in berlin in 1961 of the and its boundaries . ||| -42.727
+ but many people of east german started living in economically reach and democratic west germany , east germany government built a wall in berlin in 1961 of the and its boundaries . ||| -43.162
+ the first one is the film \u09a8\u09bf\u09b0\u09cd\u09ac\u09be\u0995 first seven \u09ae\u09bf\u09a8\u09bf\u099f which \u099a\u09be\u09b0\u09c1\u09b0 \u098f\u0995\u0998\u09c7\u09af\u09bc\u09c7\u09ae\u09bf life shown level and the second one is " the \u09a6\u09cb\u09b2\u09a8\u09be\u09b0 scene " where \u099a\u09be\u09b0\u09c1 \u0985\u09ae\u09b2\u09c7\u09b0 for his love \u09ae\u09c1\u0996\u09cb\u09ae\u09c1\u0996\u09bf in . ||| -901.665
+ the first one is the film \u09a8\u09bf\u09b0\u09cd\u09ac\u09be\u0995 first seven \u09ae\u09bf\u09a8\u09bf\u099f which \u099a\u09be\u09b0\u09c1\u09b0 \u098f\u0995\u0998\u09c7\u09af\u09bc\u09c7\u09ae\u09bf life shown level and second one is " the \u09a6\u09cb\u09b2\u09a8\u09be\u09b0 scene " where \u099a\u09be\u09b0\u09c1 \u0985\u09ae\u09b2\u09c7\u09b0 for his love \u09ae\u09c1\u0996\u09cb\u09ae\u09c1\u0996\u09bf in . ||| -901.686
+ the first one is the film \u09a8\u09bf\u09b0\u09cd\u09ac\u09be\u0995 first seven \u09ae\u09bf\u09a8\u09bf\u099f which \u099a\u09be\u09b0\u09c1\u09b0 \u098f\u0995\u0998\u09c7\u09af\u09bc\u09c7\u09ae\u09bf life shown level and the second one is , " the \u09a6\u09cb\u09b2\u09a8\u09be\u09b0 scene " where \u099a\u09be\u09b0\u09c1 \u0985\u09ae\u09b2\u09c7\u09b0 for his love \u09ae\u09c1\u0996\u09cb\u09ae\u09c1\u0996\u09bf in . ||| -901.706
+ the first one is the film \u09a8\u09bf\u09b0\u09cd\u09ac\u09be\u0995 first seven \u09ae\u09bf\u09a8\u09bf\u099f which \u099a\u09be\u09b0\u09c1\u09b0 \u098f\u0995\u0998\u09c7\u09af\u09bc\u09c7\u09ae\u09bf life shown level and the second one is , " , the \u09a6\u09cb\u09b2\u09a8\u09be\u09b0 scene " where \u099a\u09be\u09b0\u09c1 \u0985\u09ae\u09b2\u09c7\u09b0 for his love \u09ae\u09c1\u0996\u09cb\u09ae\u09c1\u0996\u09bf in . ||| -901.789
+ the first one is the film \u09a8\u09bf\u09b0\u09cd\u09ac\u09be\u0995 first seven \u09ae\u09bf\u09a8\u09bf\u099f which \u099a\u09be\u09b0\u09c1\u09b0 \u098f\u0995\u0998\u09c7\u09af\u09bc\u09c7\u09ae\u09bf life shown level and the second one is " the \u09a6\u09cb\u09b2\u09a8\u09be\u09b0 scene " where \u099a\u09be\u09b0\u09c1 \u0985\u09ae\u09b2\u09c7\u09b0 for his love \u09ae\u09c1\u0996\u09cb\u09ae\u09c1\u0996\u09bf the . ||| -901.834
+ the first one is the film \u09a8\u09bf\u09b0\u09cd\u09ac\u09be\u0995 first seven \u09ae\u09bf\u09a8\u09bf\u099f which \u099a\u09be\u09b0\u09c1\u09b0 \u098f\u0995\u0998\u09c7\u09af\u09bc\u09c7\u09ae\u09bf life shown level and second one is " the \u09a6\u09cb\u09b2\u09a8\u09be\u09b0 scene " where \u099a\u09be\u09b0\u09c1 \u0985\u09ae\u09b2\u09c7\u09b0 for his love \u09ae\u09c1\u0996\u09cb\u09ae\u09c1\u0996\u09bf the . ||| -901.855
+ the first one is the film \u09a8\u09bf\u09b0\u09cd\u09ac\u09be\u0995 first seven \u09ae\u09bf\u09a8\u09bf\u099f which \u099a\u09be\u09b0\u09c1\u09b0 \u098f\u0995\u0998\u09c7\u09af\u09bc\u09c7\u09ae\u09bf life shown level and the second one is , " the \u09a6\u09cb\u09b2\u09a8\u09be\u09b0 scene " where \u099a\u09be\u09b0\u09c1 \u0985\u09ae\u09b2\u09c7\u09b0 for his love \u09ae\u09c1\u0996\u09cb\u09ae\u09c1\u0996\u09bf the . ||| -901.875
+ the first one is the film \u09a8\u09bf\u09b0\u09cd\u09ac\u09be\u0995 first seven \u09ae\u09bf\u09a8\u09bf\u099f which \u099a\u09be\u09b0\u09c1\u09b0 \u098f\u0995\u0998\u09c7\u09af\u09bc\u09c7\u09ae\u09bf life shown them and the second one is " the \u09a6\u09cb\u09b2\u09a8\u09be\u09b0 scene " where \u099a\u09be\u09b0\u09c1 \u0985\u09ae\u09b2\u09c7\u09b0 for his love \u09ae\u09c1\u0996\u09cb\u09ae\u09c1\u0996\u09bf in . ||| -901.906
+ the first one is the film \u09a8\u09bf\u09b0\u09cd\u09ac\u09be\u0995 first seven \u09ae\u09bf\u09a8\u09bf\u099f which \u099a\u09be\u09b0\u09c1\u09b0 \u098f\u0995\u0998\u09c7\u09af\u09bc\u09c7\u09ae\u09bf life shown them and second one is " the \u09a6\u09cb\u09b2\u09a8\u09be\u09b0 scene " where \u099a\u09be\u09b0\u09c1 \u0985\u09ae\u09b2\u09c7\u09b0 for his love \u09ae\u09c1\u0996\u09cb\u09ae\u09c1\u0996\u09bf in . ||| -901.927
+ the first one is the film \u09a8\u09bf\u09b0\u09cd\u09ac\u09be\u0995 first seven \u09ae\u09bf\u09a8\u09bf\u099f which \u099a\u09be\u09b0\u09c1\u09b0 \u098f\u0995\u0998\u09c7\u09af\u09bc\u09c7\u09ae\u09bf life shown level and the second one is " the \u09a6\u09cb\u09b2\u09a8\u09be\u09b0 scene , " , where \u099a\u09be\u09b0\u09c1 \u0985\u09ae\u09b2\u09c7\u09b0 for his love \u09ae\u09c1\u0996\u09cb\u09ae\u09c1\u0996\u09bf in . ||| -901.927
+ a group of 18th century the \u099a\u09bf\u09a8\u09cd\u09a4\u09be\u09ac\u09bf\u09a6 and writer income and production \u099a\u0995\u09cd\u09b0\u09be\u0995\u09be\u09b0 current through economic thoughts development it . ||| -251.701
+ a group of 18th century the \u099a\u09bf\u09a8\u09cd\u09a4\u09be\u09ac\u09bf\u09a6 and writer income and production \u099a\u0995\u09cd\u09b0\u09be\u0995\u09be\u09b0 current through economic thoughts development people . ||| -251.890
+ a group of 18th century the \u099a\u09bf\u09a8\u09cd\u09a4\u09be\u09ac\u09bf\u09a6 and writer income and production \u099a\u0995\u09cd\u09b0\u09be\u0995\u09be\u09b0 current through the economic thoughts development it . ||| -252.122
+ a group of 18th century the \u099a\u09bf\u09a8\u09cd\u09a4\u09be\u09ac\u09bf\u09a6 and writer income and production \u099a\u0995\u09cd\u09b0\u09be\u0995\u09be\u09b0 current through the economic thoughts development people . ||| -252.311
+ a group of 18th century the writer of \u099a\u09bf\u09a8\u09cd\u09a4\u09be\u09ac\u09bf\u09a6 and income and production \u099a\u0995\u09cd\u09b0\u09be\u0995\u09be\u09b0 current through economic thoughts development it . ||| -252.494
+ a group of 18th century the \u099a\u09bf\u09a8\u09cd\u09a4\u09be\u09ac\u09bf\u09a6 and writer income and \u099a\u0995\u09cd\u09b0\u09be\u0995\u09be\u09b0 of production current through economic thoughts development it . ||| -252.561
+ a group of 18th century the writer of \u099a\u09bf\u09a8\u09cd\u09a4\u09be\u09ac\u09bf\u09a6 and income and production \u099a\u0995\u09cd\u09b0\u09be\u0995\u09be\u09b0 current through economic thoughts development people . ||| -252.684
+ a group of 18th century the \u099a\u09bf\u09a8\u09cd\u09a4\u09be\u09ac\u09bf\u09a6 and writer income and \u099a\u0995\u09cd\u09b0\u09be\u0995\u09be\u09b0 of production current through economic thoughts development people . ||| -252.750
+ a group of 18th century the \u099a\u09bf\u09a8\u09cd\u09a4\u09be\u09ac\u09bf\u09a6 and writer income and production \u099a\u0995\u09cd\u09b0\u09be\u0995\u09be\u09b0 current through economic thoughts development emperor . ||| -252.881
+ a group of 18th century the writer of \u099a\u09bf\u09a8\u09cd\u09a4\u09be\u09ac\u09bf\u09a6 and income and production \u099a\u0995\u09cd\u09b0\u09be\u0995\u09be\u09b0 current through the economic thoughts development it . ||| -252.916
+ the arousal activities before penetrating male organ into vagina is called foreplay . ||| -16.182
+ excitement before entering penis in to vagina is called as shringar . ||| -18.301
+ the arousal activities before penetrating male organ into vagina is called the foreplay . ||| -18.399
+ the arousal activities before penetrating male organ into vagina is called as the foreplay . ||| -18.916
+ the arousal activities before penetrating male organ into vagina is called stimulation . ||| -18.991
+ it is mainly \u09b6\u09c2\u0995\u09b0\u09c7\u09b0 in in was which can \u09b6\u09c2\u0995\u09b0\u0995\u09c7 \u0987\u09a8\u09ab\u09cd\u09b2\u09c1\u09af\u09bc\u09c7\u099e\u09cd\u099c\u09be\u09a4\u09c7 affected by british . ||| -341.900
+ it is mainly \u09b6\u09c2\u0995\u09b0\u09c7\u09b0 in in was which can \u09b6\u09c2\u0995\u09b0\u0995\u09c7 \u0987\u09a8\u09ab\u09cd\u09b2\u09c1\u09af\u09bc\u09c7\u099e\u09cd\u099c\u09be\u09a4\u09c7 by british . ||| -342.098
+ it is mainly \u09b6\u09c2\u0995\u09b0\u09c7\u09b0 in in was which can \u09b6\u09c2\u0995\u09b0\u0995\u09c7 \u0987\u09a8\u09ab\u09cd\u09b2\u09c1\u09af\u09bc\u09c7\u099e\u09cd\u099c\u09be\u09a4\u09c7 infected by british . ||| -342.401
+ it is basically \u09b6\u09c2\u0995\u09b0\u09c7\u09b0 in in was which can \u09b6\u09c2\u0995\u09b0\u0995\u09c7 \u0987\u09a8\u09ab\u09cd\u09b2\u09c1\u09af\u09bc\u09c7\u099e\u09cd\u099c\u09be\u09a4\u09c7 affected by british . ||| -342.474
+ it is mainly \u09b6\u09c2\u0995\u09b0\u09c7\u09b0 in in which was which can \u09b6\u09c2\u0995\u09b0\u0995\u09c7 \u0987\u09a8\u09ab\u09cd\u09b2\u09c1\u09af\u09bc\u09c7\u099e\u09cd\u099c\u09be\u09a4\u09c7 affected by british . ||| -342.625
+ it is basically \u09b6\u09c2\u0995\u09b0\u09c7\u09b0 in in was which can \u09b6\u09c2\u0995\u09b0\u0995\u09c7 \u0987\u09a8\u09ab\u09cd\u09b2\u09c1\u09af\u09bc\u09c7\u099e\u09cd\u099c\u09be\u09a4\u09c7 by british . ||| -342.672
+ it is mainly \u09b6\u09c2\u0995\u09b0\u09c7\u09b0 in in which was which can \u09b6\u09c2\u0995\u09b0\u0995\u09c7 \u0987\u09a8\u09ab\u09cd\u09b2\u09c1\u09af\u09bc\u09c7\u099e\u09cd\u099c\u09be\u09a4\u09c7 by british . ||| -342.823
+ it is basically \u09b6\u09c2\u0995\u09b0\u09c7\u09b0 in in was which can \u09b6\u09c2\u0995\u09b0\u0995\u09c7 \u0987\u09a8\u09ab\u09cd\u09b2\u09c1\u09af\u09bc\u09c7\u099e\u09cd\u099c\u09be\u09a4\u09c7 infected by british . ||| -342.975
+ it is mainly \u09b6\u09c2\u0995\u09b0\u09c7\u09b0 in to was which can \u09b6\u09c2\u0995\u09b0\u0995\u09c7 \u0987\u09a8\u09ab\u09cd\u09b2\u09c1\u09af\u09bc\u09c7\u099e\u09cd\u099c\u09be\u09a4\u09c7 affected by british . ||| -342.986
+ it is mainly \u09b6\u09c2\u0995\u09b0\u09c7\u09b0 in is was which can \u09b6\u09c2\u0995\u09b0\u0995\u09c7 \u0987\u09a8\u09ab\u09cd\u09b2\u09c1\u09af\u09bc\u09c7\u099e\u09cd\u099c\u09be\u09a4\u09c7 affected by british . ||| -343.116
+ these \u098f\u0995\u098f the mycelium formed the . ||| -217.337
+ these \u098f\u0995\u098f to mycelium formed the . ||| -217.640
+ these are \u098f\u0995\u098f the mycelium formed the . ||| -217.706
+ these are \u098f\u0995\u098f to mycelium formed the . ||| -218.008
+ these \u098f\u0995\u098f by mycelium formed the . ||| -218.513
+ these \u098f\u0995\u098f by the mycelium formed the . ||| -218.612
+ these \u098f\u0995\u098f the mycelium was formed by the . ||| -218.675
+ russia at present a democratic country . ||| -6.462
+ russia now a democratic country . ||| -7.292
+ russia at present is a democratic country . ||| -7.364
+ penis ||| -3.566
+ gender ||| -3.988
+ sex ||| -4.385
+ the ||| -5.681
+ sexual organs ||| -5.746
+ \u09b2\u09bf\u0999\u09cd\u0997 ||| -103.288
+ this state is called orgasm . ||| -3.009
+ in this state is called orgasm . ||| -7.491
+ this state is called the orgasm . ||| -8.316
+ this situation is called orgasm . ||| -8.507
+ history different period here was established royal more than one empire . ||| -34.264
+ history different period this was established royal more than one empire . ||| -34.343
+ the different period here was established royal more than one empire . ||| -34.344
+ history different period was established this royal more than one empire . ||| -34.400
+ the different period this was established royal more than one empire . ||| -34.423
+ history different period here itself was established royal more than one empire . ||| -34.705
+ history different period here only was established royal more than one empire . ||| -34.842
+ the different period here only was established royal more than one empire . ||| -34.922
+ history different period here was established royal more than one kingdom . ||| -35.172
+ history different period this was established royal more than one kingdom . ||| -35.251
+ micro economics ||| -0.625
+ macro economics ||| -2.243
+ macro economy ||| -4.426
+ macro economics . ||| -4.763
+ macro economy . ||| -5.267
+ micro economy ||| -5.717
+ microeconomics ||| -5.724
+ the user to operating system the visible form of the computer interface . ||| -29.010
+ the user to operating system most visible form of the computer interface . ||| -29.437
+ the user to operating system the most visible form of the computer interface . ||| -29.636
+ user to operating system the visible form of the computer interface . ||| -29.674
+ the user to operating system the visible are the form of computer interface . ||| -29.793
+ the user to operating system the visible is the form of computer interface . ||| -30.125
+ the user to operating system the visible form of the computers interface . ||| -30.297
+ the user to operating system are the form of the visible computer interface . ||| -30.497
+ the user to operating system the visible form of are computer interface . ||| -30.536
+ the user to operating system most visible form of the computers interface . ||| -30.724
+ \u09ac\u09bf\u09b8\u09cd\u09a4\u09be\u09b0\u09bf\u09a4\u0983 , 1971 temporary bangladesh government ||| -110.935
+ \u09ac\u09bf\u09b8\u09cd\u09a4\u09be\u09b0\u09bf\u09a4\u0983 temporary bangladesh government in 1971 ||| -111.050
+ \u09ac\u09bf\u09b8\u09cd\u09a4\u09be\u09b0\u09bf\u09a4\u0983 in 1971 temporary bangladesh government ||| -111.126
+ \u09ac\u09bf\u09b8\u09cd\u09a4\u09be\u09b0\u09bf\u09a4\u0983 temporary bangladesh government , 1971 ||| -111.220
+ \u09ac\u09bf\u09b8\u09cd\u09a4\u09be\u09b0\u09bf\u09a4\u0983 , 1971 temporary government of bangladesh ||| -111.288
+ \u09ac\u09bf\u09b8\u09cd\u09a4\u09be\u09b0\u09bf\u09a4\u0983 in 1971 temporary government of bangladesh ||| -111.480
[44/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/corpus/syntax/ArraySyntaxTree.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/corpus/syntax/ArraySyntaxTree.java b/joshua-core/src/main/java/org/apache/joshua/corpus/syntax/ArraySyntaxTree.java
new file mode 100644
index 0000000..f374279
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/corpus/syntax/ArraySyntaxTree.java
@@ -0,0 +1,411 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.corpus.syntax;
+
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+import java.util.Stack;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.util.io.LineReader;
+
+public class ArraySyntaxTree implements SyntaxTree, Externalizable {
+
+ /**
+ * Note that index stores the indices of lattice node positions, i.e. the last element of index is
+ * the terminal node, pointing to lattice.size()
+ */
+ private ArrayList<Integer> forwardIndex;
+ private ArrayList<Integer> forwardLattice;
+ private ArrayList<Integer> backwardIndex;
+ private ArrayList<Integer> backwardLattice;
+
+ private ArrayList<Integer> terminals;
+
+ private boolean useBackwardLattice = true;
+
+ private static final int MAX_CONCATENATIONS = 3;
+ private static final int MAX_LABELS = 100;
+
+ public ArraySyntaxTree() {
+ forwardIndex = null;
+ forwardLattice = null;
+ backwardIndex = null;
+ backwardLattice = null;
+
+ terminals = null;
+ }
+
+
+ public ArraySyntaxTree(String parsed_line) {
+ initialize();
+ appendFromPennFormat(parsed_line);
+ }
+
+
+ /**
+ * Returns a collection of single-non-terminal labels that exactly cover the specified span in the
+ * lattice.
+ */
+ public Collection<Integer> getConstituentLabels(int from, int to) {
+ Collection<Integer> labels = new HashSet<Integer>();
+ int span_length = to - from;
+ for (int i = forwardIndex.get(from); i < forwardIndex.get(from + 1); i += 2) {
+ int current_span = forwardLattice.get(i + 1);
+ if (current_span == span_length)
+ labels.add(forwardLattice.get(i));
+ else if (current_span < span_length) break;
+ }
+ return labels;
+ }
+
+
+ public int getOneConstituent(int from, int to) {
+ int spanLength = to - from;
+ Stack<Integer> stack = new Stack<Integer>();
+
+ for (int i = forwardIndex.get(from); i < forwardIndex.get(from + 1); i += 2) {
+ int currentSpan = forwardLattice.get(i + 1);
+ if (currentSpan == spanLength) {
+ return forwardLattice.get(i);
+ } else if (currentSpan < spanLength) break;
+ }
+ if (stack.isEmpty()) return 0;
+ StringBuilder sb = new StringBuilder();
+ while (!stack.isEmpty()) {
+ String w = Vocabulary.word(stack.pop());
+ if (sb.length() != 0) sb.append(":");
+ sb.append(w);
+ }
+ String label = sb.toString();
+ return Vocabulary.id(adjustMarkup(label));
+ }
+
+
+ public int getOneSingleConcatenation(int from, int to) {
+ for (int midpt = from + 1; midpt < to; midpt++) {
+ int x = getOneConstituent(from, midpt);
+ if (x == 0) continue;
+ int y = getOneConstituent(midpt, to);
+ if (y == 0) continue;
+ String label = Vocabulary.word(x) + "+" + Vocabulary.word(y);
+ return Vocabulary.id(adjustMarkup(label));
+ }
+ return 0;
+ }
+
+
+ public int getOneDoubleConcatenation(int from, int to) {
+ for (int a = from + 1; a < to - 1; a++) {
+ for (int b = a + 1; b < to; b++) {
+ int x = getOneConstituent(from, a);
+ if (x == 0) continue;
+ int y = getOneConstituent(a, b);
+ if (y == 0) continue;
+ int z = getOneConstituent(b, to);
+ if (z == 0) continue;
+ String label = Vocabulary.word(x) + "+" + Vocabulary.word(y) + "+" + Vocabulary.word(z);
+ return Vocabulary.id(adjustMarkup(label));
+ }
+ }
+ return 0;
+ }
+
+
+ public int getOneRightSideCCG(int from, int to) {
+ for (int end = to + 1; end <= forwardLattice.size(); end++) {
+ int x = getOneConstituent(from, end);
+ if (x == 0) continue;
+ int y = getOneConstituent(to, end);
+ if (y == 0) continue;
+ String label = Vocabulary.word(x) + "/" + Vocabulary.word(y);
+ return Vocabulary.id(adjustMarkup(label));
+ }
+ return 0;
+ }
+
+
+ public int getOneLeftSideCCG(int from, int to) {
+ for (int start = from - 1; start >= 0; start--) {
+ int x = getOneConstituent(start, to);
+ if (x == 0) continue;
+ int y = getOneConstituent(start, from);
+ if (y == 0) continue;
+ String label = Vocabulary.word(y) + "\\" + Vocabulary.word(x);
+ return Vocabulary.id(adjustMarkup(label));
+ }
+ return 0;
+ }
+
+
+ /**
+ * Returns a collection of concatenated non-terminal labels that exactly cover the specified span
+ * in the lattice. The number of non-terminals concatenated is limited by MAX_CONCATENATIONS and
+ * the total number of labels returned is bounded by MAX_LABELS.
+ */
+ public Collection<Integer> getConcatenatedLabels(int from, int to) {
+ Collection<Integer> labels = new HashSet<Integer>();
+
+ int span_length = to - from;
+ Stack<Integer> nt_stack = new Stack<Integer>();
+ Stack<Integer> pos_stack = new Stack<Integer>();
+ Stack<Integer> depth_stack = new Stack<Integer>();
+
+ // seed stacks (reverse order to save on iterations, longer spans)
+ for (int i = forwardIndex.get(from + 1) - 2; i >= forwardIndex.get(from); i -= 2) {
+ int current_span = forwardLattice.get(i + 1);
+ if (current_span < span_length) {
+ nt_stack.push(forwardLattice.get(i));
+ pos_stack.push(from + current_span);
+ depth_stack.push(1);
+ } else if (current_span >= span_length) break;
+ }
+
+ while (!nt_stack.isEmpty() && labels.size() < MAX_LABELS) {
+ int nt = nt_stack.pop();
+ int pos = pos_stack.pop();
+ int depth = depth_stack.pop();
+
+ // maximum depth reached without filling span
+ if (depth == MAX_CONCATENATIONS) continue;
+
+ int remaining_span = to - pos;
+ for (int i = forwardIndex.get(pos + 1) - 2; i >= forwardIndex.get(pos); i -= 2) {
+ int current_span = forwardLattice.get(i + 1);
+ if (current_span > remaining_span) break;
+
+ // create and look up concatenated label
+ int concatenated_nt =
+ Vocabulary.id(adjustMarkup(Vocabulary.word(nt) + "+"
+ + Vocabulary.word(forwardLattice.get(i))));
+ if (current_span < remaining_span) {
+ nt_stack.push(concatenated_nt);
+ pos_stack.push(pos + current_span);
+ depth_stack.push(depth + 1);
+ } else if (current_span == remaining_span) {
+ labels.add(concatenated_nt);
+ }
+ }
+ }
+
+ return labels;
+ }
+
+ // TODO: can pre-comupute all that in top-down fashion.
+ public Collection<Integer> getCcgLabels(int from, int to) {
+ Collection<Integer> labels = new HashSet<Integer>();
+
+ int span_length = to - from;
+ // TODO: range checks on the to and from
+
+ boolean is_prefix = (forwardLattice.get(forwardIndex.get(from) + 1) > span_length);
+ if (is_prefix) {
+ Map<Integer, Set<Integer>> main_constituents = new HashMap<Integer, Set<Integer>>();
+ // find missing to the right
+ for (int i = forwardIndex.get(from); i < forwardIndex.get(from + 1); i += 2) {
+ int current_span = forwardLattice.get(i + 1);
+ if (current_span <= span_length)
+ break;
+ else {
+ int end_pos = forwardLattice.get(i + 1) + from;
+ Set<Integer> nts = main_constituents.get(end_pos);
+ if (nts == null) main_constituents.put(end_pos, new HashSet<Integer>());
+ main_constituents.get(end_pos).add(forwardLattice.get(i));
+ }
+ }
+ for (int i = forwardIndex.get(to); i < forwardIndex.get(to + 1); i += 2) {
+ Set<Integer> main_set = main_constituents.get(to + forwardLattice.get(i + 1));
+ if (main_set != null) {
+ for (int main : main_set)
+ labels.add(Vocabulary.id(adjustMarkup(Vocabulary.word(main) + "/"
+ + Vocabulary.word(forwardLattice.get(i)))));
+ }
+ }
+ }
+
+ if (!is_prefix) {
+ if (useBackwardLattice) {
+ // check if there is any possible higher-level constituent overlapping
+ int to_end =
+ (to == backwardIndex.size() - 1) ? backwardLattice.size() : backwardIndex.get(to + 1);
+ // check longest span ending in to..
+ if (backwardLattice.get(to_end - 1) <= span_length) return labels;
+
+ Map<Integer, Set<Integer>> main_constituents = new HashMap<Integer, Set<Integer>>();
+ // find missing to the left
+ for (int i = to_end - 2; i >= backwardIndex.get(to); i -= 2) {
+ int current_span = backwardLattice.get(i + 1);
+ if (current_span <= span_length)
+ break;
+ else {
+ int start_pos = to - backwardLattice.get(i + 1);
+ Set<Integer> nts = main_constituents.get(start_pos);
+ if (nts == null) main_constituents.put(start_pos, new HashSet<Integer>());
+ main_constituents.get(start_pos).add(backwardLattice.get(i));
+ }
+ }
+ for (int i = backwardIndex.get(from); i < backwardIndex.get(from + 1); i += 2) {
+ Set<Integer> main_set = main_constituents.get(from - backwardLattice.get(i + 1));
+ if (main_set != null) {
+ for (int main : main_set)
+ labels.add(Vocabulary.id(adjustMarkup(Vocabulary.word(main) + "\\"
+ + Vocabulary.word(backwardLattice.get(i)))));
+ }
+ }
+ } else {
+ // TODO: bothersome no-backwards-arrays method.
+ }
+ }
+ return labels;
+ }
+
+ @Override
+ public int[] getTerminals() {
+ return getTerminals(0, terminals.size());
+ }
+
+ @Override
+ public int[] getTerminals(int from, int to) {
+ int[] span = new int[to - from];
+ for (int i = from; i < to; i++)
+ span[i - from] = terminals.get(i);
+ return span;
+ }
+
+ public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ // TODO Auto-generated method stub
+ }
+
+ public void writeExternal(ObjectOutput out) throws IOException {
+ // TODO Auto-generated method stub
+ }
+
+ /**
+ * Reads Penn Treebank format file
+ * @param file_name the string path of the Penn Treebank file
+ * @throws IOException if the file does not exist
+ */
+ public void readExternalText(String file_name) throws IOException {
+ LineReader reader = new LineReader(file_name);
+ initialize();
+ for (String line : reader) {
+ if (line.trim().equals("")) continue;
+ appendFromPennFormat(line);
+ }
+ }
+
+ public void writeExternalText(String file_name) throws IOException {
+ // TODO Auto-generated method stub
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < forwardIndex.size(); i++)
+ sb.append("FI[" + i + "] =\t" + forwardIndex.get(i) + "\n");
+ sb.append("\n");
+ for (int i = 0; i < forwardLattice.size(); i += 2)
+ sb.append("F[" + i + "] =\t" + Vocabulary.word(forwardLattice.get(i)) + " , "
+ + forwardLattice.get(i + 1) + "\n");
+
+ sb.append("\n");
+ for (int i = 0; i < terminals.size(); i += 1)
+ sb.append("T[" + i + "] =\t" + Vocabulary.word(terminals.get(i)) + " , 1 \n");
+
+ if (this.useBackwardLattice) {
+ sb.append("\n");
+ for (int i = 0; i < backwardIndex.size(); i++)
+ sb.append("BI[" + i + "] =\t" + backwardIndex.get(i) + "\n");
+ sb.append("\n");
+ for (int i = 0; i < backwardLattice.size(); i += 2)
+ sb.append("B[" + i + "] =\t" + Vocabulary.word(backwardLattice.get(i)) + " , "
+ + backwardLattice.get(i + 1) + "\n");
+ }
+ return sb.toString();
+ }
+
+
+ private void initialize() {
+ forwardIndex = new ArrayList<Integer>();
+ forwardIndex.add(0);
+ forwardLattice = new ArrayList<Integer>();
+ if (this.useBackwardLattice) {
+ backwardIndex = new ArrayList<Integer>();
+ backwardIndex.add(0);
+ backwardLattice = new ArrayList<Integer>();
+ }
+
+ terminals = new ArrayList<Integer>();
+ }
+
+
+ // TODO: could make this way more efficient
+ private void appendFromPennFormat(String line) {
+ String[] tokens = line.replaceAll("\\(", " ( ").replaceAll("\\)", " ) ").trim().split("\\s+");
+
+ boolean next_nt = false;
+ int current_id = 0;
+ Stack<Integer> stack = new Stack<Integer>();
+
+ for (String token : tokens) {
+ if ("(".equals(token)) {
+ next_nt = true;
+ continue;
+ }
+ if (")".equals(token)) {
+ int closing_pos = stack.pop();
+ forwardLattice.set(closing_pos, forwardIndex.size() - forwardLattice.get(closing_pos));
+ if (this.useBackwardLattice) {
+ backwardLattice.add(forwardLattice.get(closing_pos - 1));
+ backwardLattice.add(forwardLattice.get(closing_pos));
+ }
+ continue;
+ }
+ if (next_nt) {
+ // get NT id
+ current_id = Vocabulary.id(adjustMarkup(token));
+ // add into lattice
+ forwardLattice.add(current_id);
+ // push NT span field onto stack (added hereafter, we're just saving the "- 1")
+ stack.push(forwardLattice.size());
+ // add NT span field
+ forwardLattice.add(forwardIndex.size());
+ } else {
+ current_id = Vocabulary.id(token);
+ terminals.add(current_id);
+
+ forwardIndex.add(forwardLattice.size());
+ if (this.useBackwardLattice) backwardIndex.add(backwardLattice.size());
+ }
+ next_nt = false;
+ }
+ }
+
+ private String adjustMarkup(String nt) {
+ return "[" + nt.replaceAll("[\\[\\]]", "") + "]";
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/corpus/syntax/SyntaxTree.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/corpus/syntax/SyntaxTree.java b/joshua-core/src/main/java/org/apache/joshua/corpus/syntax/SyntaxTree.java
new file mode 100644
index 0000000..6bb4c0b
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/corpus/syntax/SyntaxTree.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.corpus.syntax;
+
+import java.util.Collection;
+
+public interface SyntaxTree {
+
+ public Collection<Integer> getConstituentLabels(int from, int to);
+
+ public Collection<Integer> getConcatenatedLabels(int from, int to);
+
+ public Collection<Integer> getCcgLabels(int from, int to);
+
+ public int[] getTerminals();
+
+ public int[] getTerminals(int from, int to);
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ArgsParser.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ArgsParser.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ArgsParser.java
new file mode 100644
index 0000000..5af6d11
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ArgsParser.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+
+import org.apache.joshua.util.io.LineReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * @author orluke
+ *
+ */
+public class ArgsParser {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ArgsParser.class);
+
+ private String configFile = null;
+
+ /**
+ * Parse the arguments passed from the command line when the JoshuaDecoder application was
+ * executed from the command line.
+ *
+ * @param args string array of input arguments
+ * @param config the {@link org.apache.joshua.decoder.JoshuaConfiguration}
+ * @throws IOException if there is an error wit the input arguments
+ */
+ public ArgsParser(String[] args, JoshuaConfiguration config) throws IOException {
+
+ /*
+ * Look for a verbose flag, -v.
+ *
+ * Look for an argument to the "-config" flag to find the config file, if any.
+ */
+ if (args.length >= 1) {
+ // Search for a verbose flag
+ for (int i = 0; i < args.length; i++) {
+ if (args[i].equals("-v")) {
+ Decoder.VERBOSE = Integer.parseInt(args[i + 1].trim());
+ config.setVerbosity(Decoder.VERBOSE);
+ }
+
+ if (args[i].equals("-version")) {
+ LineReader reader = new LineReader(String.format("%s/VERSION", System.getenv("JOSHUA")));
+ reader.readLine();
+ String version = reader.readLine().split("\\s+")[2];
+ System.out.println(String.format("The Apache Joshua machine translator, version %s", version));
+ System.out.println("joshua.incubator.apache.org");
+ System.exit(0);
+
+ } else if (args[i].equals("-license")) {
+ try {
+ for (String line: Files.readAllLines(Paths.get(String.format("%s/../LICENSE",
+ JoshuaConfiguration.class.getProtectionDomain().getCodeSource().getLocation().getPath())),
+ Charset.defaultCharset())) {
+ System.out.println(line);
+ }
+ } catch (IOException e) {
+ throw new RuntimeException("FATAL: missing license file!", e);
+ }
+ System.exit(0);
+ }
+ }
+
+ // Search for the configuration file from the end (so as to take the last one)
+ for (int i = args.length-1; i >= 0; i--) {
+ if (args[i].equals("-c") || args[i].equals("-config")) {
+
+ setConfigFile(args[i + 1].trim());
+ try {
+ LOG.info("Parameters read from configuration file: {}", getConfigFile());
+ config.readConfigFile(getConfigFile());
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ break;
+ }
+ }
+
+ // Now process all the command-line args
+ config.processCommandLineOptions(args);
+ }
+ }
+
+ /**
+ * @return the configFile
+ */
+ public String getConfigFile() {
+ return configFile;
+ }
+
+ /**
+ * @param configFile the configFile to set
+ */
+ public void setConfigFile(String configFile) {
+ this.configFile = configFile;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/BLEU.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/BLEU.java b/joshua-core/src/main/java/org/apache/joshua/decoder/BLEU.java
new file mode 100644
index 0000000..8b51403
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/BLEU.java
@@ -0,0 +1,562 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.ff.state_maintenance.NgramDPState;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.hypergraph.HyperEdge;
+import org.apache.joshua.util.Ngram;
+import org.apache.joshua.util.Regex;
+
+/**
+ * this class implements: (1) sentence-level bleu, with smoothing
+ *
+ * @author Zhifei Li, zhifei.work@gmail.com
+ */
+public class BLEU {
+ // do_ngram_clip: consider global n-gram clip
+
+ public static float computeSentenceBleu(String[] refSents, String hypSent) {
+ return computeSentenceBleu(refSents, hypSent, true, 4, false);
+ }
+
+ // ====================multiple references
+ /**
+ *
+ * @param refSents todo
+ * @param hypSent todo
+ * @param doNgramClip Should usually be true
+ * @param bleuOrder Should usually be 4
+ * @param useShortestRef Probably use false
+ * @return todo
+ */
+ public static float computeSentenceBleu(String[] refSents, String hypSent, boolean doNgramClip,
+ int bleuOrder, boolean useShortestRef) {
+ // === ref tbl
+ HashMap<String, Integer> maxRefCountTbl = constructMaxRefCountTable(refSents, bleuOrder);
+
+ // == ref len
+ int[] refLens = new int[refSents.length];
+ for (int i = 0; i < refSents.length; i++) {
+ String[] refWords = Regex.spaces.split(refSents[i]);
+ refLens[i] = refWords.length;
+ }
+
+ float effectiveRefLen = computeEffectiveLen(refLens, useShortestRef);
+
+ // === hyp tbl
+ String[] hypWrds = Regex.spaces.split(hypSent);
+ HashMap<String, Integer> hypNgramTbl = new HashMap<String, Integer>();
+ Ngram.getNgrams(hypNgramTbl, 1, bleuOrder, hypWrds);
+ return computeSentenceBleu(effectiveRefLen, maxRefCountTbl, hypWrds.length, hypNgramTbl,
+ doNgramClip, bleuOrder);
+ }
+
+ public static float computeEffectiveLen(int[] refLens, boolean useShortestRef) {
+ if (useShortestRef) {
+ int res = Integer.MAX_VALUE;
+ for (int i = 0; i < refLens.length; i++)
+ if (refLens[i] < res)
+ res = refLens[i];
+ return res;
+ } else {// default is average length
+ float res = 0;
+ for (int i = 0; i < refLens.length; i++)
+ res += refLens[i];
+ return res * 1.0f / refLens.length;
+ }
+ }
+
+ /**
+ * words in the ngrams are using integer symbol ID
+ * @param refSents todo
+ * @param bleuOrder todo
+ * @return todo
+ * */
+ public static HashMap<String, Integer> constructMaxRefCountTable(String[] refSents, int bleuOrder) {
+
+ List<HashMap<String, Integer>> listRefNgramTbl = new ArrayList<HashMap<String, Integer>>();
+ for (int i = 0; i < refSents.length; i++) {
+ // if(refSents[i]==null){System.out.println("null ref sent"); System.exit(1);}
+ // String[] refWords = refSents[i].split("\\s+");
+ String[] refWords = Regex.spaces.split(refSents[i]);
+
+ HashMap<String, Integer> refNgramTbl = new HashMap<String, Integer>();
+ Ngram.getNgrams(refNgramTbl, 1, bleuOrder, refWords);
+ listRefNgramTbl.add(refNgramTbl);
+ }
+
+ return computeMaxRefCountTbl(listRefNgramTbl);
+ }
+
+ /**
+ * compute max_ref_count for each ngram in the reference sentences
+ * @param listRefNgramTbl todo
+ * @return todo
+ * */
+ public static HashMap<String, Integer> computeMaxRefCountTbl(
+ List<HashMap<String, Integer>> listRefNgramTbl) {
+
+ HashMap<String, Integer> merged = new HashMap<String, Integer>();
+
+ // == get merged key set
+ for (HashMap<String, Integer> tbl : listRefNgramTbl) {
+ for (String ngram : tbl.keySet()) {
+ merged.put(ngram, 0);
+ }
+ }
+
+ // == get max ref count
+ for (String ngram : merged.keySet()) {
+ int max = 0;
+ for (HashMap<String, Integer> tbl : listRefNgramTbl) {
+ Integer val = tbl.get(ngram);
+ if (val != null && val > max)
+ max = val;
+ }
+
+ merged.put(ngram, max);
+ }
+ return merged;
+ }
+
+ public static float computeSentenceBleu(float effectiveRefLen,
+ HashMap<String, Integer> maxRefCountTbl, int hypLen, HashMap<String, Integer> hypNgramTbl,
+ boolean doNgramClip, int bleuOrder) {
+
+ float resBleu = 0.0f;
+
+ int[] numNgramMatch = new int[bleuOrder];
+ for (Map.Entry<String, Integer> entry : hypNgramTbl.entrySet()) {// each ngram in hyp
+ String ngram = entry.getKey();
+ if (maxRefCountTbl.containsKey(ngram)) {
+ int hypNgramCount = entry.getValue();
+
+ int effectiveNumMatch = hypNgramCount;
+
+ if (doNgramClip) {// min{hypNgramCount, maxRefCount}
+ int maxRefCount = maxRefCountTbl.get(ngram);
+ effectiveNumMatch = (int) Support.findMin(hypNgramCount, maxRefCount); // ngram clip;
+ }
+
+ numNgramMatch[Regex.spaces.split(ngram).length - 1] += effectiveNumMatch;
+ }
+ }
+
+ resBleu = computeBleu(hypLen, effectiveRefLen, numNgramMatch, bleuOrder);
+ // System.out.println("hyp_len: " + hyp_sent.length + "; ref_len:" + ref_sent.length +
+ // "; bleu: " + res_bleu +" num_ngram_matches: " + num_ngram_match[0] + " " +num_ngram_match[1]+
+ // " " + num_ngram_match[2] + " " +num_ngram_match[3]);
+ // System.out.println("Blue is " + res_bleu);
+ return resBleu;
+ }
+
+ // ==============================multiple references end
+
+ public static float computeSentenceBleu(String refSent, String hypSent, boolean doNgramClip,
+ int bleuOrder) {
+ String[] refWrds = Regex.spaces.split(refSent);
+ String[] hypWrds = Regex.spaces.split(hypSent);
+ HashMap<String, Integer> refNgramTbl = new HashMap<String, Integer>();
+ Ngram.getNgrams(refNgramTbl, 1, bleuOrder, refWrds);
+ HashMap<String, Integer> hypNgramTbl = new HashMap<String, Integer>();
+ Ngram.getNgrams(hypNgramTbl, 1, bleuOrder, hypWrds);
+ return computeSentenceBleu(refWrds.length, refNgramTbl, hypWrds.length, hypNgramTbl,
+ doNgramClip, bleuOrder);
+ }
+
+ public static float computeSentenceBleu(int refLen, HashMap<String, Integer> refNgramTbl,
+ int hypLen, HashMap<String, Integer> hypNgramTbl, boolean doNgramClip, int bleuOrder) {
+ float resBleu = 0;
+
+ int[] numNgramMatch = new int[bleuOrder];
+ for (Map.Entry<String, Integer> entry : hypNgramTbl.entrySet()) {
+ String ngram = entry.getKey();
+ if (refNgramTbl.containsKey(ngram)) {
+ if (doNgramClip) {
+ numNgramMatch[Regex.spaces.split(ngram).length - 1] += Support.findMin(
+ refNgramTbl.get(ngram), entry.getValue()); // ngram clip
+ } else {
+ numNgramMatch[Regex.spaces.split(ngram).length - 1] += entry.getValue();// without ngram count clipping
+ }
+ }
+ }
+ resBleu = computeBleu(hypLen, refLen, numNgramMatch, bleuOrder);
+ // System.out.println("hyp_len: " + hyp_sent.length + "; ref_len:" + ref_sent.length +
+ // "; bleu: " + res_bleu +" num_ngram_matches: " + num_ngram_match[0] + " " +num_ngram_match[1]+
+ // " " + num_ngram_match[2] + " " +num_ngram_match[3]);
+ // System.out.println("Blue is " + res_bleu);
+ return resBleu;
+ }
+
+ // sentence-bleu: BLEU= bp * prec; where prec = exp (sum 1/4 * log(prec[order]))
+ public static float computeBleu(int hypLen, float refLen, int[] numNgramMatch, int bleuOrder) {
+ if (hypLen <= 0 || refLen <= 0) {
+ throw new RuntimeException("error: ref or hyp is zero len");
+ }
+ float res = 0;
+ float wt = 1.0f / bleuOrder;
+ float prec = 0;
+ float smooth_factor = 1.0f;
+ for (int t = 0; t < bleuOrder && t < hypLen; t++) {
+ if (numNgramMatch[t] > 0) {
+ prec += wt * Math.log(numNgramMatch[t] * 1.0 / (hypLen - t));
+ } else {
+ smooth_factor *= 0.5;// TODO
+ prec += wt * Math.log(smooth_factor / (hypLen - t));
+ }
+ }
+ float bp = (hypLen >= refLen) ? 1.0f : (float) Math.exp(1 - refLen / hypLen);
+ res = bp * (float) Math.exp(prec);
+ // System.out.println("hyp_len: " + hyp_len + "; ref_len:" + ref_len + "prec: " + Math.exp(prec)
+ // + "; bp: " + bp + "; bleu: " + res);
+ return res;
+ }
+
+ public static HashMap<String, Integer> constructNgramTable(String sentence, int bleuOrder) {
+ HashMap<String, Integer> ngramTable = new HashMap<String, Integer>();
+ String[] refWrds = Regex.spaces.split(sentence);
+ Ngram.getNgrams(ngramTable, 1, bleuOrder, refWrds);
+ return ngramTable;
+ }
+
+ // ================================ Google linear corpus gain
+ // ============================================
+ public static float computeLinearCorpusGain(float[] linearCorpusGainThetas, String[] refSents,
+ String hypSent) {
+ int bleuOrder = 4;
+ int hypLength = Regex.spaces.split(hypSent).length;
+ HashMap<String, Integer> refereceNgramTable = BLEU.constructMaxRefCountTable(refSents,
+ bleuOrder);
+ HashMap<String, Integer> hypNgramTable = BLEU.constructNgramTable(hypSent, bleuOrder);
+ return computeLinearCorpusGain(linearCorpusGainThetas, hypLength, hypNgramTable,
+ refereceNgramTable);
+ }
+
+ /**
+ * speed consideration: assume hypNgramTable has a smaller size than referenceNgramTable does
+ * @param linearCorpusGainThetas todo
+ * @param hypLength todo
+ * @param hypNgramTable todo
+ * @param referenceNgramTable todo
+ * @return todo
+ */
+ public static float computeLinearCorpusGain(float[] linearCorpusGainThetas, int hypLength,
+ Map<String, Integer> hypNgramTable, Map<String, Integer> referenceNgramTable) {
+ float res = 0;
+ res += linearCorpusGainThetas[0] * hypLength;
+ for (Entry<String, Integer> entry : hypNgramTable.entrySet()) {
+ String ngram = entry.getKey();
+ if (referenceNgramTable.containsKey(ngram)) {// delta function
+ int ngramOrder = Regex.spaces.split(ngram).length;
+ res += entry.getValue() * linearCorpusGainThetas[ngramOrder];
+ }
+ }
+ return res;
+ }
+
+ /* Convenience function */
+ public static int[] computeNgramMatches(String[] refSents, String hypSent) {
+ int bleuOrder = 4;
+ int hypLength = Regex.spaces.split(hypSent).length;
+ HashMap<String, Integer> refereceNgramTable = BLEU.constructMaxRefCountTable(refSents,
+ bleuOrder);
+ HashMap<String, Integer> hypNgramTable = BLEU.constructNgramTable(hypSent, bleuOrder);
+ return computeNgramMatches(hypLength, hypNgramTable, refereceNgramTable, bleuOrder);
+ }
+
+ public static int[] computeNgramMatches(int hypLength, Map<String, Integer> hypNgramTable,
+ Map<String, Integer> referenceNgramTable, int highestOrder) {
+ int[] res = new int[highestOrder + 1];
+ res[0] = hypLength;
+ for (Entry<String, Integer> entry : hypNgramTable.entrySet()) {
+ String ngram = entry.getKey();
+ if (referenceNgramTable.containsKey(ngram)) {// delta function
+ int ngramOrder = Regex.spaces.split(ngram).length;
+ res[ngramOrder] += entry.getValue();
+ }
+ }
+
+ /*
+ System.err.print("NGRAMS:");
+ for (String ngram: hypNgramTable.keySet())
+ System.err.print(" | " + ngram);
+ System.err.println();
+ System.err.print("REF:");
+ for (String ngram: referenceNgramTable.keySet())
+ System.err.print(" | " + ngram);
+ System.err.println();
+ System.err.print("COUNTS:");
+ for (int i = 1; i <= 4; i++)
+ System.err.print(" " + res[i]);
+ System.err.println();
+ */
+
+ return res;
+ }
+
+ static public float[] computeLinearCorpusThetas(int numUnigramTokens, float unigramPrecision,
+ float decayRatio) {
+ float[] res = new float[5];
+ res[0] = -1.0f / numUnigramTokens;
+ for (int i = 1; i < 5; i++)
+ res[i] = (1.0f / (4.0f * numUnigramTokens * unigramPrecision * (float) Math.pow(decayRatio,
+ i - 1)));
+
+ float firstWeight = res[0];
+ for (int i = 0; i < 5; i++)
+ res[i] /= Math.abs(firstWeight);// normalize by first one
+
+ System.out.print("Normalized Thetas are: ");
+ for (int i = 0; i < 5; i++)
+ System.out.print(res[i] + " ");
+ System.out.print("\n");
+
+ return res;
+ }
+
+ public static final int maxOrder = 4;
+
+ /**
+ * Computes BLEU statistics incurred by a rule. This is (a) all ngram (n <= 4) for terminal rules
+ * and (b) all ngrams overlying boundary points between terminals in the rule and ngram state from
+ * tail nodes.
+ *
+ * There are four cases to handle:
+ * <ul>
+ * <li>only words
+ * <li>a number of words followed by a nonterminal (left context of tail tail node)
+ * <li>a nonterminal (right context of tail node) followed by one or more words
+ * <li>two nonterminals (right context of tail node 1, left context of tail node 2)
+ * </ul>
+ *
+ * Of these, all but the first have a boundary point to consider.
+ *
+ * @param edge todo
+ * @param spanPct todo
+ * @param references the reference to compute statistics against
+ * @return todo
+ */
+ public static Stats compute(HyperEdge edge, float spanPct, References references) {
+ Stats stats = new Stats();
+ // TODO: this should not be the span width, but the real ref scaled to the span percentage
+ stats.reflen = (int) (spanPct * references.reflen);
+
+ Rule rule = edge.getRule();
+ if (rule != null) {
+ int[] symbols = rule.getEnglish();
+
+// System.err.println(String.format("compute(%s)", rule));
+
+ ArrayList<Integer> currentNgram = new ArrayList<Integer>();
+ int boundary = -1;
+ int tailIndex = -1;
+ for (int i = 0; i < symbols.length; i++) {
+ if (symbols[i] < 0) {
+ tailIndex++;
+
+ NgramDPState ngramState = null;
+ try {
+ ngramState = (NgramDPState) edge.getTailNodes().get(tailIndex).getDPState(0);
+ } catch (ClassCastException e) {
+ throw new RuntimeException(String.format(
+ "* FATAL: first state needs to be NgramDPState (found %s)", edge.getTailNodes()
+ .get(tailIndex).getDPState(0).getClass()));
+ }
+
+ // Compute ngrams overlapping with left context of tail node
+ if (currentNgram.size() > 0) {
+ boundary = currentNgram.size();
+ for (int id : ngramState.getLeftLMStateWords())
+ currentNgram.add(id);
+
+ // Compute the BLEU statistics
+ BLEU.Stats partStats = computeOverDivide(currentNgram, references, boundary);
+ stats.add(partStats);
+
+// System.err.println(" " + Vocabulary.getWords(ngramState.getLeftLMStateWords()));
+
+ currentNgram.clear();
+ }
+
+// System.err.println(" " + Vocabulary.getWords(ngramState.getRightLMStateWords()));
+
+ // Accumulate ngrams from right context of tail node
+ for (int id : ngramState.getRightLMStateWords())
+ currentNgram.add(id);
+
+ boundary = currentNgram.size();
+
+ } else { // terminal symbol
+ currentNgram.add(symbols[i]);
+ stats.len++;
+
+// System.err.println(" " + Vocabulary.word(symbols[i]));
+
+ if (boundary != -1) {
+ BLEU.Stats partStats = computeOverDivide(currentNgram, references, boundary);
+ stats.add(partStats);
+
+ // Shift off the context from the nonterminal's righthand side
+ for (int j = 0; j < boundary; j++)
+ currentNgram.remove(0);
+ boundary = -1;
+ }
+ }
+
+ /*
+ * At the end, we might have (a) nothing, (b) a sequence of words from a nonterminal's
+ * righthand side, (c) a sequence of words from the rule, or (d) a sequence of words from a
+ * nonterminal's righthand context and from the rule
+ */
+ if (currentNgram.size() > 0 && currentNgram.size() != boundary) { // skip cases (a) and (b)
+ BLEU.Stats partStats = computeOverDivide(currentNgram, references, boundary);
+ stats.add(partStats);
+ }
+ }
+ }
+ return stats;
+ }
+
+ /**
+ * When computing BLEU statistics over a rule, we need to avoid adding in ngrams that are
+ * exclusively contained inside tail nodes. This function accumulates all the eligible ngrams from
+ * a string respective of an optional boundary point, and then calls computeNgramMatches().
+ *
+ * @param ngram the current set of ngrams
+ * @param references contains the set of ngrams to compare against
+ * @param boundary the boundary over which all ngrams must fall (-1 means ignore boundary)
+ * @return
+ */
+ private static Stats computeOverDivide(ArrayList<Integer> ngram, References references,
+ int boundary) {
+
+// System.err.print(String.format(" BOUNDARY(%s, %d)", Vocabulary.getWords(ngram), boundary));
+
+ HashMap<String, Integer> boundaryNgrams = new HashMap<String, Integer>();
+ for (int width = 1; width <= Math.min(maxOrder, ngram.size()); width++) {
+ for (int i = 0; i < ngram.size() - width + 1; i++) {
+ int j = i + width;
+
+ final List<Integer> piece = ngram.subList(i, j);
+ if (boundary == -1 || (boundary > i && boundary < j)) {
+ String ngramStr = Vocabulary.getWords(piece);
+ if (!boundaryNgrams.containsKey(ngramStr))
+ boundaryNgrams.put(ngramStr, 1);
+ else
+ boundaryNgrams.put(ngramStr, boundaryNgrams.get(ngramStr));
+ }
+ }
+ }
+
+ /*
+ System.err.print(" FOUND");
+ for (String phr: boundaryNgrams.keySet())
+ System.err.print(" | " + phr);
+ System.err.println();
+ */
+
+ BLEU.Stats result = new BLEU.Stats();
+ int[] stats = BLEU.computeNgramMatches(0, boundaryNgrams, references.ngramCounts, maxOrder);
+ System.arraycopy(stats, 1, result.counts, 0, maxOrder);
+
+ return result;
+ }
+
+ public static class References {
+ HashMap<String, Integer> ngramCounts;
+ float reflen;
+
+ public References(String reference) {
+ String[] refs = new String[1];
+ refs[0] = reference;
+ fill(refs);
+ }
+
+ public References(String[] references) {
+ fill(references);
+ }
+
+ private void fill(String[] references) {
+ ngramCounts = new HashMap<String, Integer>();
+ reflen = 0.0f;
+ for (int i = 0; i < references.length; i++) {
+ String[] ref = references[i].split(" ");
+ Ngram.getNgrams(ngramCounts, 1, maxOrder, ref);
+ reflen += ref.length;
+ }
+ reflen /= references.length;
+ }
+ }
+
+ public static float score(Stats stats) {
+ float score = 0f;
+ float wt = 1.0f / maxOrder;
+ float prec = 0;
+ float smooth_factor = 1.0f;
+ for (int t = 0; t < maxOrder && t < stats.len; t++) {
+ if (stats.counts[t] > 0) {
+ prec += wt * Math.log(stats.counts[t] * 1.0 / (stats.len - t));
+ } else {
+ smooth_factor *= 0.5;// TODO
+ prec += wt * Math.log(smooth_factor / (stats.len - t));
+ }
+ }
+ float bp = (stats.len >= stats.reflen) ? 1.0f : (float) Math.exp(1 - stats.reflen / stats.len);
+ score = bp * (float) Math.exp(prec);
+
+// System.err.println(String.format("BLEU(%d %d %d %d / BP=%f) = %f", stats.counts[0], stats.counts[1], stats.counts[2], stats.counts[3], bp, score));
+ return score;
+ }
+
+ /**
+ * Accumulated sufficient statistics for computing BLEU.
+ */
+ public static class Stats {
+ public int[] counts;
+ public float len;
+ public float reflen;
+
+ public Stats() {
+ counts = new int[4];
+ len = 0.0f;
+ reflen = 0.0f;
+ }
+
+ public Stats(int[] counts, float len, float reflen) {
+ this.counts = counts;
+ this.len = len;
+ this.reflen = reflen;
+ }
+
+ public void add(Stats otherStats) {
+ for (int i = 0; i < counts.length; i++)
+ counts[i] += otherStats.counts[i];
+
+ len += otherStats.len;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/Decoder.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/Decoder.java b/joshua-core/src/main/java/org/apache/joshua/decoder/Decoder.java
new file mode 100644
index 0000000..097ce59
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/Decoder.java
@@ -0,0 +1,813 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder;
+
+import static org.apache.joshua.decoder.ff.FeatureVector.DENSE_FEATURE_NAMES;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.IOException;
+import java.io.FileNotFoundException;
+import java.lang.reflect.Constructor;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.BlockingQueue;
+
+import com.google.common.base.Strings;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.ff.FeatureVector;
+import org.apache.joshua.decoder.ff.FeatureFunction;
+import org.apache.joshua.decoder.ff.PhraseModel;
+import org.apache.joshua.decoder.ff.StatefulFF;
+import org.apache.joshua.decoder.ff.lm.LanguageModelFF;
+import org.apache.joshua.decoder.ff.tm.Grammar;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.ff.tm.format.HieroFormatReader;
+import org.apache.joshua.decoder.ff.tm.hash_based.MemoryBasedBatchGrammar;
+import org.apache.joshua.decoder.ff.tm.packed.PackedGrammar;
+import org.apache.joshua.decoder.io.TranslationRequestStream;
+import org.apache.joshua.decoder.phrase.PhraseTable;
+import org.apache.joshua.decoder.segment_file.Sentence;
+import org.apache.joshua.util.FileUtility;
+import org.apache.joshua.util.FormatUtils;
+import org.apache.joshua.util.Regex;
+import org.apache.joshua.util.io.LineReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class handles decoder initialization and the complication introduced by multithreading.
+ *
+ * After initialization, the main entry point to the Decoder object is
+ * decodeAll(TranslationRequest), which returns a set of Translation objects wrapped in an iterable
+ * Translations object. It is important that we support multithreading both (a) across the sentences
+ * within a request and (b) across requests, in a round-robin fashion. This is done by maintaining a
+ * fixed sized concurrent thread pool. When a new request comes in, a RequestParallelizer thread is
+ * launched. This object iterates over the request's sentences, obtaining a thread from the
+ * thread pool, and using that thread to decode the sentence. If a decoding thread is not available,
+ * it will block until one is in a fair (FIFO) manner. RequestParallelizer thereby permits intra-request
+ * parallelization by separating out reading the input stream from processing the translated sentences,
+ * but also ensures that round-robin parallelization occurs, since RequestParallelizer uses the
+ * thread pool before translating each request.
+ *
+ * A decoding thread is handled by DecoderThread and launched from DecoderThreadRunner. The purpose
+ * of the runner is to record where to place the translated sentence when it is done (i.e., which
+ * Translations object). Translations itself is an iterator whose next() call blocks until the next
+ * translation is available.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ * @author Zhifei Li, zhifei.work@gmail.com
+ * @author wren ng thornton wren@users.sourceforge.net
+ * @author Lane Schwartz dowobeha@users.sourceforge.net
+ */
+public class Decoder {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Decoder.class);
+
+ private final JoshuaConfiguration joshuaConfiguration;
+
+ public JoshuaConfiguration getJoshuaConfiguration() {
+ return joshuaConfiguration;
+ }
+
+ /*
+ * Many of these objects themselves are global objects. We pass them in when constructing other
+ * objects, so that they all share pointers to the same object. This is good because it reduces
+ * overhead, but it can be problematic because of unseen dependencies (for example, in the
+ * Vocabulary shared by language model, translation grammar, etc).
+ */
+ private List<Grammar> grammars;
+ private ArrayList<FeatureFunction> featureFunctions;
+ private Grammar customPhraseTable;
+
+ /* The feature weights. */
+ public static FeatureVector weights;
+
+ public static int VERBOSE = 1;
+
+ private BlockingQueue<DecoderThread> threadPool = null;
+
+ // ===============================================================
+ // Constructors
+ // ===============================================================
+
+ /**
+ * Constructor method that creates a new decoder using the specified configuration file.
+ *
+ * @param joshuaConfiguration a populated {@link org.apache.joshua.decoder.JoshuaConfiguration}
+ * @param configFile name of configuration file.
+ */
+ public Decoder(JoshuaConfiguration joshuaConfiguration, String configFile) {
+ this(joshuaConfiguration);
+ this.initialize(configFile);
+ }
+
+ /**
+ * Factory method that creates a new decoder using the specified configuration file.
+ *
+ * @param configFile Name of configuration file.
+ * @return a configured {@link org.apache.joshua.decoder.Decoder}
+ */
+ public static Decoder createDecoder(String configFile) {
+ JoshuaConfiguration joshuaConfiguration = new JoshuaConfiguration();
+ return new Decoder(joshuaConfiguration, configFile);
+ }
+
+ /**
+ * Constructs an uninitialized decoder for use in testing.
+ * <p>
+ * This method is private because it should only ever be called by the
+ * {@link #getUninitalizedDecoder()} method to provide an uninitialized decoder for use in
+ * testing.
+ */
+ private Decoder(JoshuaConfiguration joshuaConfiguration) {
+ this.joshuaConfiguration = joshuaConfiguration;
+ this.grammars = new ArrayList<Grammar>();
+ this.threadPool = new ArrayBlockingQueue<DecoderThread>(
+ this.joshuaConfiguration.num_parallel_decoders, true);
+ this.customPhraseTable = null;
+ }
+
+ /**
+ * Gets an uninitialized decoder for use in testing.
+ * <p>
+ * This method is called by unit tests or any outside packages (e.g., MERT) relying on the
+ * decoder.
+ * @param joshuaConfiguration a {@link org.apache.joshua.decoder.JoshuaConfiguration} object
+ * @return an uninitialized decoder for use in testing
+ */
+ static public Decoder getUninitalizedDecoder(JoshuaConfiguration joshuaConfiguration) {
+ return new Decoder(joshuaConfiguration);
+ }
+
+ // ===============================================================
+ // Public Methods
+ // ===============================================================
+
+ /**
+ * This class is responsible for getting sentences from the TranslationRequest and procuring a
+ * DecoderThreadRunner to translate it. Each call to decodeAll(TranslationRequest) launches a
+ * thread that will read the request's sentences, obtain a DecoderThread to translate them, and
+ * then place the Translation in the appropriate place.
+ *
+ * @author Matt Post <po...@cs.jhu.edu>
+ *
+ */
+ private class RequestParallelizer extends Thread {
+ /* Source of sentences to translate. */
+ private final TranslationRequestStream request;
+
+ /* Where to put translated sentences. */
+ private final Translations response;
+
+ RequestParallelizer(TranslationRequestStream request, Translations response) {
+ this.request = request;
+ this.response = response;
+ }
+
+ @Override
+ public void run() {
+ /*
+ * Repeatedly get an input sentence, wait for a DecoderThread, and then start a new thread to
+ * translate the sentence. We start a new thread (via DecoderRunnerThread) as opposed to
+ * blocking, so that the RequestHandler can go on to the next sentence in this request, which
+ * allows parallelization across the sentences of the request.
+ */
+ for (;;) {
+ Sentence sentence = request.next();
+
+ if (sentence == null) {
+ response.finish();
+ break;
+ }
+
+ // This will block until a DecoderThread becomes available.
+ DecoderThread thread = Decoder.this.getThread();
+ new DecoderThreadRunner(thread, sentence, response).start();
+ }
+ }
+
+ /**
+ * Strips the nonterminals from the lefthand side of the rule.
+ *
+ * @param rule
+ * @return
+ */
+ private String formatRule(Rule rule) {
+ String ruleString = "";
+ boolean first = true;
+ for (int word: rule.getFrench()) {
+ if (!first)
+ ruleString += " " + Vocabulary.word(word);
+ first = false;
+ }
+
+ ruleString += " |||"; // space will get added with first English word
+ first = true;
+ for (int word: rule.getEnglish()) {
+ if (!first)
+ ruleString += " " + Vocabulary.word(word);
+ first = false;
+ }
+
+ // strip of the leading space
+ return ruleString.substring(1);
+ }
+ }
+
+ /**
+ * Retrieve a thread from the thread pool, blocking until one is available. The blocking occurs in
+ * a fair fashion (i.e,. FIFO across requests).
+ *
+ * @return a thread that can be used for decoding.
+ */
+ public DecoderThread getThread() {
+ try {
+ return threadPool.take();
+ } catch (InterruptedException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ return null;
+ }
+
+ /**
+ * This class handles running a DecoderThread (which takes care of the actual translation of an
+ * input Sentence, returning a Translation object when its done). This is done in a thread so as
+ * not to tie up the RequestHandler that launched it, freeing it to go on to the next sentence in
+ * the TranslationRequest, in turn permitting parallelization across the sentences of a request.
+ *
+ * When the decoder thread is finshed, the Translation object is placed in the correct place in
+ * the corresponding Translations object that was returned to the caller of
+ * Decoder.decodeAll(TranslationRequest).
+ *
+ * @author Matt Post <po...@cs.jhu.edu>
+ */
+ private class DecoderThreadRunner extends Thread {
+
+ private final DecoderThread decoderThread;
+ private final Sentence sentence;
+ private final Translations translations;
+
+ DecoderThreadRunner(DecoderThread thread, Sentence sentence, Translations translations) {
+ this.decoderThread = thread;
+ this.sentence = sentence;
+ this.translations = translations;
+ }
+
+ @Override
+ public void run() {
+ /*
+ * Process any found metadata.
+ */
+
+ /*
+ * Use the thread to translate the sentence. Then record the translation with the
+ * corresponding Translations object, and return the thread to the pool.
+ */
+ try {
+ Translation translation = decoderThread.translate(this.sentence);
+ translations.record(translation);
+
+ /*
+ * This is crucial! It's what makes the thread available for the next sentence to be
+ * translated.
+ */
+ threadPool.put(decoderThread);
+ } catch (Exception e) {
+ throw new RuntimeException(String.format(
+ "Input %d: FATAL UNCAUGHT EXCEPTION: %s", sentence.id(), e.getMessage()), e);
+ // translations.record(new Translation(sentence, null, featureFunctions, joshuaConfiguration));
+ }
+ }
+ }
+
+ /**
+ * This function is the main entry point into the decoder. It translates all the sentences in a
+ * (possibly boundless) set of input sentences. Each request launches its own thread to read the
+ * sentences of the request.
+ *
+ * @param request the populated {@link org.apache.joshua.decoder.io.TranslationRequestStream}
+ * @throws IOException if there is an error with the input stream or writing the output
+ * @return an iterable, asynchronously-filled list of Translations
+ */
+ public Translations decodeAll(TranslationRequestStream request) throws IOException {
+ Translations translations = new Translations(request);
+
+ /* Start a thread to handle requests on the input stream */
+ new RequestParallelizer(request, translations).start();
+
+ return translations;
+ }
+
+
+ /**
+ * We can also just decode a single sentence.
+ *
+ * @param sentence {@link org.apache.joshua.lattice.Lattice} input
+ * @return the sentence {@link org.apache.joshua.decoder.Translation}
+ */
+ public Translation decode(Sentence sentence) {
+ // Get a thread.
+
+ try {
+ DecoderThread thread = threadPool.take();
+ Translation translation = thread.translate(sentence);
+ threadPool.put(thread);
+
+ return translation;
+
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+
+ return null;
+ }
+
+ /**
+ * Clean shutdown of Decoder, resetting all
+ * static variables, such that any other instance of Decoder
+ * afterwards gets a fresh start.
+ */
+ public void cleanUp() {
+ // shut down DecoderThreads
+ for (DecoderThread thread : threadPool) {
+ try {
+ thread.join();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ resetGlobalState();
+ }
+
+ public static void resetGlobalState() {
+ // clear/reset static variables
+ DENSE_FEATURE_NAMES.clear();
+ Vocabulary.clear();
+ Vocabulary.unregisterLanguageModels();
+ LanguageModelFF.resetLmIndex();
+ StatefulFF.resetGlobalStateIndex();
+ }
+
+ public static void writeConfigFile(double[] newWeights, String template, String outputFile,
+ String newDiscriminativeModel) {
+ try {
+ int columnID = 0;
+
+ BufferedWriter writer = FileUtility.getWriteFileStream(outputFile);
+ LineReader reader = new LineReader(template);
+ try {
+ for (String line : reader) {
+ line = line.trim();
+ if (Regex.commentOrEmptyLine.matches(line) || line.indexOf("=") != -1) {
+ // comment, empty line, or parameter lines: just copy
+ writer.write(line);
+ writer.newLine();
+
+ } else { // models: replace the weight
+ String[] fds = Regex.spaces.split(line);
+ StringBuffer newSent = new StringBuffer();
+ if (!Regex.floatingNumber.matches(fds[fds.length - 1])) {
+ throw new IllegalArgumentException("last field is not a number; the field is: "
+ + fds[fds.length - 1]);
+ }
+
+ if (newDiscriminativeModel != null && "discriminative".equals(fds[0])) {
+ newSent.append(fds[0]).append(' ');
+ newSent.append(newDiscriminativeModel).append(' ');// change the
+ // file name
+ for (int i = 2; i < fds.length - 1; i++) {
+ newSent.append(fds[i]).append(' ');
+ }
+ } else {// regular
+ for (int i = 0; i < fds.length - 1; i++) {
+ newSent.append(fds[i]).append(' ');
+ }
+ }
+ if (newWeights != null)
+ newSent.append(newWeights[columnID++]);// change the weight
+ else
+ newSent.append(fds[fds.length - 1]);// do not change
+
+ writer.write(newSent.toString());
+ writer.newLine();
+ }
+ }
+ } finally {
+ reader.close();
+ writer.close();
+ }
+
+ if (newWeights != null && columnID != newWeights.length) {
+ throw new IllegalArgumentException("number of models does not match number of weights");
+ }
+
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ // ===============================================================
+ // Initialization Methods
+ // ===============================================================
+
+ /**
+ * Moses requires the pattern .*_.* for sparse features, and prohibits underscores in dense features.
+ * This conforms to that pattern. We assume non-conforming dense features start with tm_ or lm_,
+ * and the only sparse feature that needs converting is OOVPenalty.
+ *
+ * @param feature
+ * @return the feature in Moses format
+ */
+ private String mosesize(String feature) {
+ if (joshuaConfiguration.moses) {
+ if (feature.startsWith("tm_") || feature.startsWith("lm_"))
+ return feature.replace("_", "-");
+ }
+
+ return feature;
+ }
+
+ /**
+ * Initialize all parts of the JoshuaDecoder.
+ *
+ * @param configFile File containing configuration options
+ * @return An initialized decoder
+ */
+ public Decoder initialize(String configFile) {
+ try {
+
+ long pre_load_time = System.currentTimeMillis();
+
+ /* Weights can be listed in a separate file (denoted by parameter "weights-file") or directly
+ * in the Joshua config file. Config file values take precedent.
+ */
+ this.readWeights(joshuaConfiguration.weights_file);
+
+
+ /* Add command-line-passed weights to the weights array for processing below */
+ if (!Strings.isNullOrEmpty(joshuaConfiguration.weight_overwrite)) {
+ String[] tokens = joshuaConfiguration.weight_overwrite.split("\\s+");
+ for (int i = 0; i < tokens.length; i += 2) {
+ String feature = tokens[i];
+ float value = Float.parseFloat(tokens[i+1]);
+
+ if (joshuaConfiguration.moses)
+ feature = demoses(feature);
+
+ joshuaConfiguration.weights.add(String.format("%s %s", feature, tokens[i+1]));
+ LOG.info("COMMAND LINE WEIGHT: {} -> {}", feature, value);
+ }
+ }
+
+ /* Read the weights found in the config file */
+ for (String pairStr: joshuaConfiguration.weights) {
+ String pair[] = pairStr.split("\\s+");
+
+ /* Sanity check for old-style unsupported feature invocations. */
+ if (pair.length != 2) {
+ StringBuilder errMsg = new StringBuilder();
+ errMsg.append("FATAL: Invalid feature weight line found in config file.\n");
+ errMsg.append(String.format("The line was '%s'\n", pairStr));
+ errMsg.append("You might be using an old version of the config file that is no longer supported\n");
+ errMsg.append("Check joshua-decoder.org or email joshua_support@googlegroups.com for help\n");
+ errMsg.append("Code = " + 17);
+ throw new RuntimeException(errMsg.toString());
+ }
+
+ weights.set(pair[0], Float.parseFloat(pair[1]));
+ }
+
+ LOG.info("Read {} weights ({} of them dense)", weights.size(), DENSE_FEATURE_NAMES.size());
+
+ // Do this before loading the grammars and the LM.
+ this.featureFunctions = new ArrayList<FeatureFunction>();
+
+ // Initialize and load grammars. This must happen first, since the vocab gets defined by
+ // the packed grammar (if any)
+ this.initializeTranslationGrammars();
+ LOG.info("Grammar loading took: {} seconds.",
+ (System.currentTimeMillis() - pre_load_time) / 1000);
+
+ // Initialize the features: requires that LM model has been initialized.
+ this.initializeFeatureFunctions();
+
+ // This is mostly for compatibility with the Moses tuning script
+ if (joshuaConfiguration.show_weights_and_quit) {
+ for (int i = 0; i < DENSE_FEATURE_NAMES.size(); i++) {
+ String name = DENSE_FEATURE_NAMES.get(i);
+ if (joshuaConfiguration.moses)
+ System.out.println(String.format("%s= %.5f", mosesize(name), weights.getDense(i)));
+ else
+ System.out.println(String.format("%s %.5f", name, weights.getDense(i)));
+ }
+ System.exit(0);
+ }
+
+ // Sort the TM grammars (needed to do cube pruning)
+ if (joshuaConfiguration.amortized_sorting) {
+ LOG.info("Grammar sorting happening lazily on-demand.");
+ } else {
+ long pre_sort_time = System.currentTimeMillis();
+ for (Grammar grammar : this.grammars) {
+ grammar.sortGrammar(this.featureFunctions);
+ }
+ LOG.info("Grammar sorting took {} seconds.",
+ (System.currentTimeMillis() - pre_sort_time) / 1000);
+ }
+
+ // Create the threads
+ for (int i = 0; i < joshuaConfiguration.num_parallel_decoders; i++) {
+ this.threadPool.put(new DecoderThread(this.grammars, Decoder.weights,
+ this.featureFunctions, joshuaConfiguration));
+ }
+ } catch (IOException | InterruptedException e) {
+ LOG.warn(e.getMessage(), e);
+ }
+
+ return this;
+ }
+
+ /**
+ * Initializes translation grammars Retained for backward compatibility
+ *
+ * @param ownersSeen Records which PhraseModelFF's have been instantiated (one is needed for each
+ * owner)
+ * @throws IOException
+ */
+ private void initializeTranslationGrammars() throws IOException {
+
+ if (joshuaConfiguration.tms.size() > 0) {
+
+ // collect packedGrammars to check if they use a shared vocabulary
+ final List<PackedGrammar> packed_grammars = new ArrayList<>();
+
+ // tm = {thrax/hiero,packed,samt,moses} OWNER LIMIT FILE
+ for (String tmLine : joshuaConfiguration.tms) {
+
+ String type = tmLine.substring(0, tmLine.indexOf(' '));
+ String[] args = tmLine.substring(tmLine.indexOf(' ')).trim().split("\\s+");
+ HashMap<String, String> parsedArgs = FeatureFunction.parseArgs(args);
+
+ String owner = parsedArgs.get("owner");
+ int span_limit = Integer.parseInt(parsedArgs.get("maxspan"));
+ String path = parsedArgs.get("path");
+
+ Grammar grammar = null;
+ if (! type.equals("moses") && ! type.equals("phrase")) {
+ if (new File(path).isDirectory()) {
+ try {
+ PackedGrammar packed_grammar = new PackedGrammar(path, span_limit, owner, type, joshuaConfiguration);
+ packed_grammars.add(packed_grammar);
+ grammar = packed_grammar;
+ } catch (FileNotFoundException e) {
+ String msg = String.format("Couldn't load packed grammar from '%s'", path)
+ + "Perhaps it doesn't exist, or it may be an old packed file format.";
+ throw new RuntimeException(msg);
+ }
+ } else {
+ // thrax, hiero, samt
+ grammar = new MemoryBasedBatchGrammar(type, path, owner,
+ joshuaConfiguration.default_non_terminal, span_limit, joshuaConfiguration);
+ }
+
+ } else {
+
+ int maxSourceLen = parsedArgs.containsKey("max-source-len")
+ ? Integer.parseInt(parsedArgs.get("max-source-len"))
+ : -1;
+
+ joshuaConfiguration.search_algorithm = "stack";
+ grammar = new PhraseTable(path, owner, type, joshuaConfiguration);
+ }
+
+ this.grammars.add(grammar);
+ }
+
+ checkSharedVocabularyChecksumsForPackedGrammars(packed_grammars);
+
+ } else {
+ LOG.warn("no grammars supplied! Supplying dummy glue grammar.");
+ MemoryBasedBatchGrammar glueGrammar = new MemoryBasedBatchGrammar("glue", joshuaConfiguration);
+ glueGrammar.setSpanLimit(-1);
+ glueGrammar.addGlueRules(featureFunctions);
+ this.grammars.add(glueGrammar);
+ }
+
+ /* Add the grammar for custom entries */
+ if (joshuaConfiguration.search_algorithm.equals("stack"))
+ this.customPhraseTable = new PhraseTable(null, "custom", "phrase", joshuaConfiguration);
+ else
+ this.customPhraseTable = new MemoryBasedBatchGrammar("custom", joshuaConfiguration);
+ this.grammars.add(this.customPhraseTable);
+
+ /* Create an epsilon-deleting grammar */
+ if (joshuaConfiguration.lattice_decoding) {
+ LOG.info("Creating an epsilon-deleting grammar");
+ MemoryBasedBatchGrammar latticeGrammar = new MemoryBasedBatchGrammar("lattice", joshuaConfiguration);
+ latticeGrammar.setSpanLimit(-1);
+ HieroFormatReader reader = new HieroFormatReader();
+
+ String goalNT = FormatUtils.cleanNonTerminal(joshuaConfiguration.goal_symbol);
+ String defaultNT = FormatUtils.cleanNonTerminal(joshuaConfiguration.default_non_terminal);
+
+ //FIXME: too many arguments
+ String ruleString = String.format("[%s] ||| [%s,1] <eps> ||| [%s,1] ||| ", goalNT, goalNT, defaultNT,
+ goalNT, defaultNT);
+
+ Rule rule = reader.parseLine(ruleString);
+ latticeGrammar.addRule(rule);
+ rule.estimateRuleCost(featureFunctions);
+
+ this.grammars.add(latticeGrammar);
+ }
+
+ /* Now create a feature function for each owner */
+ HashSet<String> ownersSeen = new HashSet<String>();
+
+ for (Grammar grammar: this.grammars) {
+ String owner = Vocabulary.word(grammar.getOwner());
+ if (! ownersSeen.contains(owner)) {
+ this.featureFunctions.add(new PhraseModel(weights, new String[] { "tm", "-owner", owner },
+ joshuaConfiguration, grammar));
+ ownersSeen.add(owner);
+ }
+ }
+
+ LOG.info("Memory used {} MB",
+ ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1000000.0));
+ }
+
+ /**
+ * Checks if multiple packedGrammars have the same vocabulary by comparing their vocabulary file checksums.
+ */
+ private static void checkSharedVocabularyChecksumsForPackedGrammars(final List<PackedGrammar> packed_grammars) {
+ String previous_checksum = "";
+ for (PackedGrammar grammar : packed_grammars) {
+ final String checksum = grammar.computeVocabularyChecksum();
+ if (previous_checksum.isEmpty()) {
+ previous_checksum = checksum;
+ } else {
+ if (!checksum.equals(previous_checksum)) {
+ throw new RuntimeException(
+ "Trying to load multiple packed grammars with different vocabularies!" +
+ "Have you packed them jointly?");
+ }
+ previous_checksum = checksum;
+ }
+ }
+ }
+
+ /*
+ * This function reads the weights for the model. Feature names and their weights are listed one
+ * per line in the following format:
+ *
+ * FEATURE_NAME WEIGHT
+ */
+ private void readWeights(String fileName) {
+ Decoder.weights = new FeatureVector();
+
+ if (fileName.equals(""))
+ return;
+
+ try {
+ LineReader lineReader = new LineReader(fileName);
+
+ for (String line : lineReader) {
+ line = line.replaceAll("\\s+", " ");
+
+ if (line.equals("") || line.startsWith("#") || line.startsWith("//")
+ || line.indexOf(' ') == -1)
+ continue;
+
+ String tokens[] = line.split("\\s+");
+ String feature = tokens[0];
+ Float value = Float.parseFloat(tokens[1]);
+
+ // Kludge for compatibility with Moses tuners
+ if (joshuaConfiguration.moses) {
+ feature = demoses(feature);
+ }
+
+ weights.increment(feature, value);
+ }
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+ LOG.info("Read {} weights from file '{}'", weights.size(), fileName);
+ }
+
+ private String demoses(String feature) {
+ if (feature.endsWith("="))
+ feature = feature.replace("=", "");
+ if (feature.equals("OOV_Penalty"))
+ feature = "OOVPenalty";
+ else if (feature.startsWith("tm-") || feature.startsWith("lm-"))
+ feature = feature.replace("-", "_");
+ return feature;
+ }
+
+ /**
+ * Feature functions are instantiated with a line of the form
+ *
+ * <pre>
+ * FEATURE OPTIONS
+ * </pre>
+ *
+ * Weights for features are listed separately.
+ *
+ * @throws IOException
+ *
+ */
+ private void initializeFeatureFunctions() throws IOException {
+
+ for (String featureLine : joshuaConfiguration.features) {
+ // line starts with NAME, followed by args
+ // 1. create new class named NAME, pass it config, weights, and the args
+
+ String fields[] = featureLine.split("\\s+");
+ String featureName = fields[0];
+
+ try {
+
+ Class<?> clas = getFeatureFunctionClass(featureName);
+ Constructor<?> constructor = clas.getConstructor(FeatureVector.class,
+ String[].class, JoshuaConfiguration.class);
+ FeatureFunction feature = (FeatureFunction) constructor.newInstance(weights, fields, joshuaConfiguration);
+ this.featureFunctions.add(feature);
+
+ } catch (Exception e) {
+ throw new RuntimeException(String.format("Unable to instantiate feature function '%s'!", featureLine), e);
+ }
+ }
+
+ for (FeatureFunction feature : featureFunctions) {
+ LOG.info("FEATURE: {}", feature.logString());
+ }
+
+ weights.registerDenseFeatures(featureFunctions);
+ }
+
+ /**
+ * Searches a list of predefined paths for classes, and returns the first one found. Meant for
+ * instantiating feature functions.
+ *
+ * @param name
+ * @return the class, found in one of the search paths
+ * @throws ClassNotFoundException
+ */
+ private Class<?> getFeatureFunctionClass(String featureName) {
+ Class<?> clas = null;
+
+ String[] packages = { "org.apache.joshua.decoder.ff", "org.apache.joshua.decoder.ff.lm", "org.apache.joshua.decoder.ff.phrase" };
+ for (String path : packages) {
+ try {
+ clas = Class.forName(String.format("%s.%s", path, featureName));
+ break;
+ } catch (ClassNotFoundException e) {
+ try {
+ clas = Class.forName(String.format("%s.%sFF", path, featureName));
+ break;
+ } catch (ClassNotFoundException e2) {
+ // do nothing
+ }
+ }
+ }
+ return clas;
+ }
+
+ /**
+ * Adds a rule to the custom grammar.
+ *
+ * @param rule the rule to add
+ */
+ public void addCustomRule(Rule rule) {
+ customPhraseTable.addRule(rule);
+ rule.estimateRuleCost(featureFunctions);
+ }
+
+ public Grammar getCustomPhraseTable() {
+ return customPhraseTable;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/DecoderThread.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/DecoderThread.java b/joshua-core/src/main/java/org/apache/joshua/decoder/DecoderThread.java
new file mode 100644
index 0000000..d6f5233
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/DecoderThread.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.joshua.decoder.chart_parser.Chart;
+import org.apache.joshua.decoder.ff.FeatureFunction;
+import org.apache.joshua.decoder.ff.FeatureVector;
+import org.apache.joshua.decoder.ff.SourceDependentFF;
+import org.apache.joshua.decoder.ff.tm.Grammar;
+import org.apache.joshua.decoder.hypergraph.ForestWalker;
+import org.apache.joshua.decoder.hypergraph.GrammarBuilderWalkerFunction;
+import org.apache.joshua.decoder.hypergraph.HyperGraph;
+import org.apache.joshua.decoder.phrase.Stacks;
+import org.apache.joshua.decoder.segment_file.Sentence;
+import org.apache.joshua.corpus.Vocabulary;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class handles decoding of individual Sentence objects (which can represent plain sentences
+ * or lattices). A single sentence can be decoded by a call to translate() and, if an InputHandler
+ * is used, many sentences can be decoded in a thread-safe manner via a single call to
+ * translateAll(), which continually queries the InputHandler for sentences until they have all been
+ * consumed and translated.
+ *
+ * The DecoderFactory class is responsible for launching the threads.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ * @author Zhifei Li, zhifei.work@gmail.com
+ */
+
+public class DecoderThread extends Thread {
+ private static final Logger LOG = LoggerFactory.getLogger(DecoderThread.class);
+
+ private final JoshuaConfiguration joshuaConfiguration;
+ /*
+ * these variables may be the same across all threads (e.g., just copy from DecoderFactory), or
+ * differ from thread to thread
+ */
+ private final List<Grammar> allGrammars;
+ private final List<FeatureFunction> featureFunctions;
+
+
+ // ===============================================================
+ // Constructor
+ // ===============================================================
+ public DecoderThread(List<Grammar> grammars, FeatureVector weights,
+ List<FeatureFunction> featureFunctions, JoshuaConfiguration joshuaConfiguration) throws IOException {
+
+ this.joshuaConfiguration = joshuaConfiguration;
+ this.allGrammars = grammars;
+
+ this.featureFunctions = new ArrayList<FeatureFunction>();
+ for (FeatureFunction ff : featureFunctions) {
+ if (ff instanceof SourceDependentFF) {
+ this.featureFunctions.add(((SourceDependentFF) ff).clone());
+ } else {
+ this.featureFunctions.add(ff);
+ }
+ }
+ }
+
+ // ===============================================================
+ // Methods
+ // ===============================================================
+
+ @Override
+ public void run() {
+ // Nothing to do but wait.
+ }
+
+ /**
+ * Translate a sentence.
+ *
+ * @param sentence The sentence to be translated.
+ * @return the sentence {@link org.apache.joshua.decoder.Translation}
+ */
+ public Translation translate(Sentence sentence) {
+
+ LOG.info("Input {}: {}", sentence.id(), sentence.fullSource());
+
+ if (sentence.target() != null)
+ LOG.info("Input {}: Constraining to target sentence '{}'",
+ sentence.id(), sentence.target());
+
+ // skip blank sentences
+ if (sentence.isEmpty()) {
+ LOG.info("Translation {}: Translation took 0 seconds", sentence.id());
+ return new Translation(sentence, null, featureFunctions, joshuaConfiguration);
+ }
+
+ long startTime = System.currentTimeMillis();
+
+ int numGrammars = allGrammars.size();
+ Grammar[] grammars = new Grammar[numGrammars];
+
+ for (int i = 0; i < allGrammars.size(); i++)
+ grammars[i] = allGrammars.get(i);
+
+ if (joshuaConfiguration.segment_oovs)
+ sentence.segmentOOVs(grammars);
+
+ /**
+ * Joshua supports (as of September 2014) both phrase-based and hierarchical decoding. Here
+ * we build the appropriate chart. The output of both systems is a hypergraph, which is then
+ * used for further processing (e.g., k-best extraction).
+ */
+ HyperGraph hypergraph = null;
+ try {
+
+ if (joshuaConfiguration.search_algorithm.equals("stack")) {
+ Stacks stacks = new Stacks(sentence, this.featureFunctions, grammars, joshuaConfiguration);
+
+ hypergraph = stacks.search();
+ } else {
+ /* Seeding: the chart only sees the grammars, not the factories */
+ Chart chart = new Chart(sentence, this.featureFunctions, grammars,
+ joshuaConfiguration.goal_symbol, joshuaConfiguration);
+
+ hypergraph = (joshuaConfiguration.use_dot_chart)
+ ? chart.expand()
+ : chart.expandSansDotChart();
+ }
+
+ } catch (java.lang.OutOfMemoryError e) {
+ LOG.error("Input {}: out of memory", sentence.id());
+ hypergraph = null;
+ }
+
+ float seconds = (System.currentTimeMillis() - startTime) / 1000.0f;
+ LOG.info("Input {}: Translation took {} seconds", sentence.id(), seconds);
+ LOG.info("Input {}: Memory used is {} MB", sentence.id(), (Runtime
+ .getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1000000.0);
+
+ /* Return the translation unless we're doing synchronous parsing. */
+ if (!joshuaConfiguration.parse || hypergraph == null) {
+ return new Translation(sentence, hypergraph, featureFunctions, joshuaConfiguration);
+ }
+
+ /*****************************************************************************************/
+
+ /*
+ * Synchronous parsing.
+ *
+ * Step 1. Traverse the hypergraph to create a grammar for the second-pass parse.
+ */
+ Grammar newGrammar = getGrammarFromHyperGraph(joshuaConfiguration.goal_symbol, hypergraph);
+ newGrammar.sortGrammar(this.featureFunctions);
+ long sortTime = System.currentTimeMillis();
+ LOG.info("Sentence {}: New grammar has {} rules.", sentence.id(),
+ newGrammar.getNumRules());
+
+ /* Step 2. Create a new chart and parse with the instantiated grammar. */
+ Grammar[] newGrammarArray = new Grammar[] { newGrammar };
+ Sentence targetSentence = new Sentence(sentence.target(), sentence.id(), joshuaConfiguration);
+ Chart chart = new Chart(targetSentence, featureFunctions, newGrammarArray, "GOAL",joshuaConfiguration);
+ int goalSymbol = GrammarBuilderWalkerFunction.goalSymbol(hypergraph);
+ String goalSymbolString = Vocabulary.word(goalSymbol);
+ LOG.info("Sentence {}: goal symbol is {} ({}).", sentence.id(),
+ goalSymbolString, goalSymbol);
+ chart.setGoalSymbolID(goalSymbol);
+
+ /* Parsing */
+ HyperGraph englishParse = chart.expand();
+ long secondParseTime = System.currentTimeMillis();
+ LOG.info("Sentence {}: Finished second chart expansion ({} seconds).",
+ sentence.id(), (secondParseTime - sortTime) / 1000);
+ LOG.info("Sentence {} total time: {} seconds.\n", sentence.id(),
+ (secondParseTime - startTime) / 1000);
+ LOG.info("Memory used after sentence {} is {} MB", sentence.id(), (Runtime
+ .getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1000000.0);
+ return new Translation(sentence, englishParse, featureFunctions, joshuaConfiguration); // or do something else
+ }
+
+ private Grammar getGrammarFromHyperGraph(String goal, HyperGraph hg) {
+ GrammarBuilderWalkerFunction f = new GrammarBuilderWalkerFunction(goal,joshuaConfiguration);
+ ForestWalker walker = new ForestWalker();
+ walker.walk(hg.goalNode, f);
+ return f.getGrammar();
+ }
+}
[51/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
maven multi-module layout 1st commit: moving files into joshua-core
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/e2734396
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/e2734396
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/e2734396
Branch: refs/heads/maven-multi-module
Commit: e27343960822424d71c54f8911eba56a252f281c
Parents: 55e88d1
Author: Pavel Danchenko <da...@amazon.com>
Authored: Tue Jun 21 17:11:37 2016 +0200
Committer: Pavel Danchenko <da...@amazon.com>
Committed: Tue Jun 21 17:42:29 2016 +0200
----------------------------------------------------------------------
joshua-core/.gitignore | 1 +
joshua-core/pom.xml | 145 +
joshua-core/resources/berkeley_lm/lm | 16 +
joshua-core/resources/berkeley_lm/lm.berkeleylm | Bin 0 -> 4294 bytes
.../resources/berkeley_lm/lm.berkeleylm.gz | Bin 0 -> 1786 bytes
joshua-core/resources/berkeley_lm/lm.gz | Bin 0 -> 162 bytes
joshua-core/resources/grammar.glue | 4 +
.../resources/kbest_extraction/glue-grammar | 3 +
joshua-core/resources/kbest_extraction/grammar | 25 +
.../resources/kbest_extraction/joshua.config | 27 +
joshua-core/resources/kbest_extraction/lm.gz | Bin 0 -> 2466496 bytes
.../resources/kbest_extraction/output.gold | 3126 +++
.../kbest_extraction/output.scores.gold | 3126 +++
joshua-core/resources/phrase_decoder/config | 29 +
.../resources/phrase_decoder/constrained.config | 28 +
.../phrase_decoder/constrained.output.gold | 5 +
joshua-core/resources/phrase_decoder/lm.1.gz | Bin 0 -> 2235 bytes
.../resources/phrase_decoder/output.gold | 1 +
joshua-core/resources/phrase_decoder/rules.1.gz | Bin 0 -> 2998042 bytes
joshua-core/resources/wa_grammar | 3 +
joshua-core/resources/wa_grammar.packed/config | 1 +
.../resources/wa_grammar.packed/encoding | Bin 0 -> 154 bytes
.../wa_grammar.packed/slice_00000.alignments | Bin 0 -> 45 bytes
.../wa_grammar.packed/slice_00000.features | Bin 0 -> 47 bytes
.../wa_grammar.packed/slice_00000.source | Bin 0 -> 204 bytes
.../wa_grammar.packed/slice_00000.target | Bin 0 -> 128 bytes
.../wa_grammar.packed/slice_00000.target.lookup | Bin 0 -> 32 bytes
.../resources/wa_grammar.packed/vocabulary | Bin 0 -> 238 bytes
.../java/org/apache/joshua/adagrad/AdaGrad.java | 160 +
.../org/apache/joshua/adagrad/AdaGradCore.java | 3127 +++
.../org/apache/joshua/adagrad/Optimizer.java | 728 +
.../apache/joshua/corpus/AbstractPhrase.java | 133 +
.../org/apache/joshua/corpus/BasicPhrase.java | 97 +
.../apache/joshua/corpus/ContiguousPhrase.java | 127 +
.../java/org/apache/joshua/corpus/Corpus.java | 160 +
.../java/org/apache/joshua/corpus/Phrase.java | 117 +
.../java/org/apache/joshua/corpus/Span.java | 175 +
.../org/apache/joshua/corpus/SymbolTable.java | 327 +
.../apache/joshua/corpus/TerminalIterator.java | 85 +
.../org/apache/joshua/corpus/Vocabulary.java | 301 +
.../joshua/corpus/syntax/ArraySyntaxTree.java | 411 +
.../apache/joshua/corpus/syntax/SyntaxTree.java | 34 +
.../org/apache/joshua/decoder/ArgsParser.java | 118 +
.../java/org/apache/joshua/decoder/BLEU.java | 562 +
.../java/org/apache/joshua/decoder/Decoder.java | 813 +
.../apache/joshua/decoder/DecoderThread.java | 201 +
.../joshua/decoder/JoshuaConfiguration.java | 729 +
.../apache/joshua/decoder/JoshuaDecoder.java | 148 +
.../joshua/decoder/NbestMinRiskReranker.java | 446 +
.../joshua/decoder/StructuredTranslation.java | 164 +
.../decoder/StructuredTranslationFactory.java | 117 +
.../java/org/apache/joshua/decoder/Support.java | 86 +
.../org/apache/joshua/decoder/Translation.java | 239 +
.../org/apache/joshua/decoder/Translations.java | 158 +
.../joshua/decoder/chart_parser/Cell.java | 294 +
.../joshua/decoder/chart_parser/Chart.java | 746 +
.../decoder/chart_parser/ComputeNodeResult.java | 225 +
.../decoder/chart_parser/CubePruneState.java | 114 +
.../joshua/decoder/chart_parser/DotChart.java | 476 +
.../joshua/decoder/chart_parser/SourcePath.java | 63 +
.../decoder/chart_parser/StateConstraint.java | 75 +
.../joshua/decoder/chart_parser/SuperNode.java | 62 +
.../decoder/chart_parser/package-info.java | 24 +
.../joshua/decoder/ff/ArityPhrasePenalty.java | 72 +
.../joshua/decoder/ff/FeatureFunction.java | 364 +
.../apache/joshua/decoder/ff/FeatureVector.java | 385 +
.../joshua/decoder/ff/LabelCombinationFF.java | 62 +
.../joshua/decoder/ff/LabelSubstitutionFF.java | 131 +
.../joshua/decoder/ff/LexicalFeatures.java | 150 +
.../apache/joshua/decoder/ff/OOVPenalty.java | 106 +
.../apache/joshua/decoder/ff/PhraseModel.java | 135 +
.../apache/joshua/decoder/ff/PhrasePenalty.java | 86 +
.../apache/joshua/decoder/ff/RuleCountBin.java | 74 +
.../org/apache/joshua/decoder/ff/RuleFF.java | 123 +
.../apache/joshua/decoder/ff/RuleLength.java | 52 +
.../decoder/ff/RulePropertiesQuerying.java | 49 +
.../org/apache/joshua/decoder/ff/RuleShape.java | 99 +
.../joshua/decoder/ff/SourceDependentFF.java | 29 +
.../apache/joshua/decoder/ff/SourcePathFF.java | 63 +
.../apache/joshua/decoder/ff/StatefulFF.java | 88 +
.../apache/joshua/decoder/ff/StatelessFF.java | 79 +
.../apache/joshua/decoder/ff/TargetBigram.java | 216 +
.../apache/joshua/decoder/ff/WordPenalty.java | 80 +
.../ff/fragmentlm/ConcatenationIterator.java | 93 +
.../decoder/ff/fragmentlm/FragmentLMFF.java | 365 +
.../ff/fragmentlm/PennTreebankReader.java | 135 +
.../joshua/decoder/ff/fragmentlm/Tree.java | 779 +
.../joshua/decoder/ff/fragmentlm/Trees.java | 270 +
.../apache/joshua/decoder/ff/lm/AbstractLM.java | 122 +
.../apache/joshua/decoder/ff/lm/ArpaFile.java | 328 +
.../apache/joshua/decoder/ff/lm/ArpaNgram.java | 73 +
.../ff/lm/DefaultNGramLanguageModel.java | 134 +
.../org/apache/joshua/decoder/ff/lm/KenLM.java | 233 +
.../joshua/decoder/ff/lm/LanguageModelFF.java | 527 +
.../decoder/ff/lm/NGramLanguageModel.java | 73 +
.../ff/lm/StateMinimizingLanguageModel.java | 202 +
.../joshua/decoder/ff/lm/berkeley_lm/LICENSE | 13 +
.../ff/lm/berkeley_lm/LMGrammarBerkeley.java | 205 +
.../joshua/decoder/ff/lm/berkeley_lm/README | 5 +
.../ff/lm/berkeley_lm/SymbolTableWrapper.java | 102 +
.../ff/lm/bloomfilter_lm/BloomFilter.java | 215 +
.../BloomFilterLanguageModel.java | 560 +
.../ff/lm/bloomfilter_lm/package-info.java | 25 +
.../joshua/decoder/ff/lm/buildin_lm/TrieLM.java | 331 +
.../decoder/ff/lm/buildin_lm/package-info.java | 19 +
.../joshua/decoder/ff/lm/package-info.java | 42 +
.../apache/joshua/decoder/ff/package-info.java | 42 +
.../joshua/decoder/ff/phrase/Distortion.java | 71 +
.../ff/similarity/EdgePhraseSimilarityFF.java | 279 +
.../decoder/ff/state_maintenance/DPState.java | 34 +
.../ff/state_maintenance/KenLMState.java | 56 +
.../ff/state_maintenance/NgramDPState.java | 100 +
.../joshua/decoder/ff/tm/AbstractGrammar.java | 228 +
.../decoder/ff/tm/BasicRuleCollection.java | 101 +
.../joshua/decoder/ff/tm/CreateGlueGrammar.java | 126 +
.../apache/joshua/decoder/ff/tm/Grammar.java | 120 +
.../joshua/decoder/ff/tm/GrammarReader.java | 158 +
.../org/apache/joshua/decoder/ff/tm/Rule.java | 633 +
.../joshua/decoder/ff/tm/RuleCollection.java | 76 +
.../decoder/ff/tm/SentenceFilteredGrammar.java | 366 +
.../org/apache/joshua/decoder/ff/tm/Trie.java | 108 +
.../ff/tm/UnsortedRuleCollectionException.java | 40 +
.../decoder/ff/tm/format/HieroFormatReader.java | 106 +
.../decoder/ff/tm/format/MosesFormatReader.java | 108 +
.../ff/tm/hash_based/ExtensionIterator.java | 73 +
.../tm/hash_based/MemoryBasedBatchGrammar.java | 292 +
.../ff/tm/hash_based/MemoryBasedRuleBin.java | 59 +
.../ff/tm/hash_based/MemoryBasedTrie.java | 88 +
.../decoder/ff/tm/hash_based/package-info.java | 23 +
.../joshua/decoder/ff/tm/package-info.java | 25 +
.../decoder/ff/tm/packed/PackedGrammar.java | 1066 +
.../ff/tm/packed/SliceAggregatingTrie.java | 236 +
.../decoder/hypergraph/AlignedSourceTokens.java | 112 +
.../decoder/hypergraph/AllSpansWalker.java | 63 +
.../hypergraph/DefaultInsideOutside.java | 407 +
.../hypergraph/FeatureVectorExtractor.java | 80 +
.../joshua/decoder/hypergraph/ForestWalker.java | 79 +
.../GrammarBuilderWalkerFunction.java | 180 +
.../joshua/decoder/hypergraph/HGNode.java | 331 +
.../joshua/decoder/hypergraph/HyperEdge.java | 101 +
.../joshua/decoder/hypergraph/HyperGraph.java | 163 +
.../decoder/hypergraph/HyperGraphPruning.java | 176 +
.../decoder/hypergraph/KBestExtractor.java | 1052 +
.../hypergraph/OutputStringExtractor.java | 195 +
.../hypergraph/StringToTreeConverter.java | 74 +
.../hypergraph/TrivialInsideOutside.java | 31 +
.../decoder/hypergraph/ViterbiExtractor.java | 178 +
.../decoder/hypergraph/WalkerFunction.java | 37 +
.../hypergraph/WordAlignmentExtractor.java | 134 +
.../decoder/hypergraph/WordAlignmentState.java | 192 +
.../joshua/decoder/hypergraph/package-info.java | 25 +
.../apache/joshua/decoder/io/DeNormalize.java | 203 +
.../apache/joshua/decoder/io/JSONMessage.java | 159 +
.../decoder/io/TranslationRequestStream.java | 179 +
.../org/apache/joshua/decoder/package-info.java | 26 +
.../apache/joshua/decoder/phrase/Candidate.java | 241 +
.../decoder/phrase/CandidateComparator.java | 28 +
.../apache/joshua/decoder/phrase/Coverage.java | 235 +
.../apache/joshua/decoder/phrase/Future.java | 119 +
.../apache/joshua/decoder/phrase/Header.java | 87 +
.../joshua/decoder/phrase/Hypothesis.java | 154 +
.../org/apache/joshua/decoder/phrase/Note.java | 44 +
.../joshua/decoder/phrase/PhraseChart.java | 197 +
.../joshua/decoder/phrase/PhraseTable.java | 181 +
.../org/apache/joshua/decoder/phrase/Stack.java | 229 +
.../apache/joshua/decoder/phrase/Stacks.java | 269 +
.../joshua/decoder/phrase/TargetPhrases.java | 80 +
.../decoder/segment_file/ConstraintRule.java | 100 +
.../decoder/segment_file/ConstraintSpan.java | 80 +
.../decoder/segment_file/ParseTreeInput.java | 40 +
.../decoder/segment_file/ParsedSentence.java | 56 +
.../joshua/decoder/segment_file/Sentence.java | 450 +
.../joshua/decoder/segment_file/Token.java | 158 +
.../decoder/segment_file/package-info.java | 25 +
.../java/org/apache/joshua/lattice/Arc.java | 117 +
.../java/org/apache/joshua/lattice/Lattice.java | 587 +
.../java/org/apache/joshua/lattice/Node.java | 159 +
.../lattice/NodeIdentifierComparator.java | 41 +
.../org/apache/joshua/lattice/package-info.java | 22 +
.../java/org/apache/joshua/metrics/BLEU.java | 575 +
.../org/apache/joshua/metrics/BLEU_SBP.java | 63 +
.../apache/joshua/metrics/EvaluationMetric.java | 405 +
.../apache/joshua/metrics/GradeLevelBLEU.java | 280 +
.../java/org/apache/joshua/metrics/METEOR.java | 235 +
.../joshua/metrics/MinimumChangeBLEU.java | 223 +
.../joshua/metrics/NewMetric.java.template | 134 +
.../java/org/apache/joshua/metrics/Precis.java | 334 +
.../joshua/metrics/PrecisMinusSourceBLEU.java | 182 +
.../java/org/apache/joshua/metrics/SARI.java | 681 +
.../org/apache/joshua/metrics/SourceBLEU.java | 107 +
.../java/org/apache/joshua/metrics/TER.java | 460 +
.../org/apache/joshua/metrics/TERMinusBLEU.java | 194 +
.../org/apache/joshua/metrics/TercomRunner.java | 115 +
.../org/apache/joshua/metrics/ZeroOneLoss.java | 88 +
.../main/java/org/apache/joshua/mira/MIRA.java | 160 +
.../java/org/apache/joshua/mira/MIRACore.java | 3112 +++
.../java/org/apache/joshua/mira/Optimizer.java | 643 +
.../joshua/oracle/OracleExtractionHG.java | 797 +
.../apache/joshua/oracle/OracleExtractor.java | 58 +
.../java/org/apache/joshua/oracle/SplitHg.java | 300 +
.../org/apache/joshua/oracle/package-info.java | 26 +
.../apache/joshua/pro/ClassifierInterface.java | 41 +
.../org/apache/joshua/pro/ClassifierMegaM.java | 121 +
.../apache/joshua/pro/ClassifierPerceptron.java | 114 +
.../org/apache/joshua/pro/ClassifierSVM.java | 140 +
.../java/org/apache/joshua/pro/Optimizer.java | 454 +
.../main/java/org/apache/joshua/pro/PRO.java | 159 +
.../java/org/apache/joshua/pro/PROCore.java | 3027 +++
.../org/apache/joshua/server/ServerThread.java | 294 +
.../org/apache/joshua/server/TcpServer.java | 65 +
.../joshua/subsample/AlignedSubsampler.java | 103 +
.../org/apache/joshua/subsample/Alignment.java | 92 +
.../org/apache/joshua/subsample/BiCorpus.java | 186 +
.../joshua/subsample/BiCorpusFactory.java | 83 +
.../org/apache/joshua/subsample/PhrasePair.java | 72 +
.../apache/joshua/subsample/PhraseReader.java | 46 +
.../apache/joshua/subsample/PhraseWriter.java | 79 +
.../org/apache/joshua/subsample/Subsampler.java | 246 +
.../apache/joshua/subsample/SubsamplerCLI.java | 141 +
.../apache/joshua/subsample/package-info.java | 25 +
.../org/apache/joshua/tools/GrammarPacker.java | 959 +
.../apache/joshua/tools/GrammarPackerCli.java | 156 +
.../org/apache/joshua/tools/LabelPhrases.java | 111 +
.../org/apache/joshua/tools/TestSetFilter.java | 383 +
.../java/org/apache/joshua/ui/Orientation.java | 23 +
.../org/apache/joshua/ui/StartupWindow.java | 87 +
.../java/org/apache/joshua/ui/package-info.java | 22 +
.../ui/tree_visualizer/DerivationTree.java | 103 +
.../ui/tree_visualizer/DerivationTreeEdge.java | 27 +
.../DerivationTreeTransformer.java | 117 +
.../ui/tree_visualizer/DerivationViewer.java | 128 +
.../tree_visualizer/DerivationViewerApplet.java | 51 +
.../apache/joshua/ui/tree_visualizer/Node.java | 59 +
.../ui/tree_visualizer/browser/Browser.java | 237 +
.../browser/DerivationTreeFrame.java | 253 +
.../browser/TranslationInfo.java | 56 +
.../joshua/ui/tree_visualizer/tree/Tree.java | 283 +
.../java/org/apache/joshua/util/Algorithms.java | 85 +
.../main/java/org/apache/joshua/util/Bits.java | 128 +
.../java/org/apache/joshua/util/BotMap.java | 94 +
.../main/java/org/apache/joshua/util/Cache.java | 176 +
.../java/org/apache/joshua/util/ChartSpan.java | 88 +
.../apache/joshua/util/CommandLineParser.java | 738 +
.../java/org/apache/joshua/util/Constants.java | 36 +
.../java/org/apache/joshua/util/Counted.java | 92 +
.../java/org/apache/joshua/util/Counts.java | 306 +
.../org/apache/joshua/util/ExtractTopCand.java | 188 +
.../org/apache/joshua/util/FileUtility.java | 318 +
.../org/apache/joshua/util/FormatUtils.java | 245 +
.../org/apache/joshua/util/IntegerPair.java | 36 +
.../java/org/apache/joshua/util/JoshuaEval.java | 624 +
.../java/org/apache/joshua/util/ListUtil.java | 95 +
.../main/java/org/apache/joshua/util/Lists.java | 567 +
.../apache/joshua/util/NBestListUtility.java | 74 +
.../main/java/org/apache/joshua/util/Ngram.java | 105 +
.../org/apache/joshua/util/NullIterator.java | 65 +
.../apache/joshua/util/PackedGrammarServer.java | 87 +
.../main/java/org/apache/joshua/util/Pair.java | 130 +
.../java/org/apache/joshua/util/Platform.java | 27 +
.../org/apache/joshua/util/QuietFormatter.java | 36 +
.../main/java/org/apache/joshua/util/Regex.java | 143 +
.../org/apache/joshua/util/ReverseOrder.java | 39 +
.../org/apache/joshua/util/SampledList.java | 69 +
.../org/apache/joshua/util/SocketUtility.java | 144 +
.../org/apache/joshua/util/StreamGobbler.java | 50 +
.../joshua/util/UnicodeCharacterName.java | 22466 +++++++++++++++++
.../apache/joshua/util/encoding/Analyzer.java | 235 +
.../joshua/util/encoding/EightBitQuantizer.java | 92 +
.../util/encoding/EncoderConfiguration.java | 160 +
.../joshua/util/encoding/EncoderFactory.java | 42 +
.../util/encoding/FeatureTypeAnalyzer.java | 254 +
.../joshua/util/encoding/FloatEncoder.java | 39 +
.../apache/joshua/util/encoding/IntEncoder.java | 39 +
.../util/encoding/PrimitiveFloatEncoder.java | 129 +
.../util/encoding/PrimitiveIntEncoder.java | 111 +
.../joshua/util/encoding/VariableQuantizer.java | 106 +
.../org/apache/joshua/util/io/BinaryIn.java | 91 +
.../org/apache/joshua/util/io/BinaryOut.java | 505 +
.../apache/joshua/util/io/IndexedReader.java | 155 +
.../org/apache/joshua/util/io/LineReader.java | 368 +
.../org/apache/joshua/util/io/NullReader.java | 63 +
.../joshua/util/io/ProgressInputStream.java | 82 +
.../java/org/apache/joshua/util/io/Reader.java | 51 +
.../org/apache/joshua/util/io/package-info.java | 22 +
.../org/apache/joshua/util/package-info.java | 22 +
.../util/quantization/BooleanQuantizer.java | 45 +
.../joshua/util/quantization/Quantizer.java | 45 +
.../quantization/QuantizerConfiguration.java | 119 +
.../util/quantization/QuantizerFactory.java | 50 +
.../util/quantization/StatelessQuantizer.java | 38 +
.../joshua/util/quantization/package-info.java | 19 +
.../joshua/zmert/IntermediateOptimizer.java | 991 +
.../java/org/apache/joshua/zmert/MertCore.java | 3191 +++
.../java/org/apache/joshua/zmert/ZMERT.java | 156 +
.../org/apache/joshua/zmert/package-info.java | 24 +
joshua-core/src/main/resources/log4j.properties | 20 +
joshua-core/src/overview.html | 41 +
.../apache/joshua/corpus/CorpusArrayTest.java | 177 +
.../java/org/apache/joshua/corpus/SpanTest.java | 47 +
.../apache/joshua/corpus/VocabularyTest.java | 135 +
.../joshua/corpus/vocab/VocabularyTest.java | 180 +
.../ArtificialGrammarAndCorpusCreater.java | 130 +
.../joshua/decoder/DecoderThreadTest.java | 172 +
.../joshua/decoder/JoshuaDecoderTest.java | 83 +
.../joshua/decoder/TestConfigFileCreater.java | 184 +
.../apache/joshua/decoder/TranslationsTest.java | 87 +
.../decoder/ff/ArityPhrasePenaltyFFTest.java | 64 +
.../joshua/decoder/ff/lm/ArpaFileTest.java | 226 +
.../decoder/ff/lm/LanguageModelFFTest.java | 95 +
.../LMBerkeleySentenceProbablityTest.java | 47 +
.../lm/berkeley_lm/LMGrammarBerkeleyTest.java | 80 +
.../joshua/decoder/io/DeNormalizeTest.java | 273 +
.../decoder/io/TranslationRequestTest.java | 149 +
.../kbest_extraction/KBestExtractionTest.java | 80 +
.../joshua/decoder/phrase/CoverageTest.java | 140 +
.../ConstrainedPhraseDecodingTest.java | 77 +
.../phrase/decode/PhraseDecodingTest.java | 77 +
.../segment_file/AlmostTooLongSentenceTest.java | 96 +
.../decoder/segment_file/SentenceTest.java | 109 +
.../java/org/apache/joshua/lattice/ArcTest.java | 86 +
.../org/apache/joshua/lattice/LatticeTest.java | 192 +
.../org/apache/joshua/lattice/NodeTest.java | 105 +
.../org/apache/joshua/packed/Benchmark.java | 126 +
.../org/apache/joshua/packed/CountRules.java | 110 +
.../org/apache/joshua/packed/PrintRules.java | 199 +
.../test/java/org/apache/joshua/packed/README | 6 +
.../org/apache/joshua/packed/VocabTest.java | 58 +
.../java/org/apache/joshua/packed/packer.config | 6 +
.../java/org/apache/joshua/packed/small_grammar | 20000 +++++++++++++++
.../test/java/org/apache/joshua/packed/test.sh | 20 +
.../apache/joshua/system/AlignmentMapTest.java | 72 +
.../org/apache/joshua/system/KenLmTest.java | 95 +
.../system/MultithreadedTranslationTests.java | 156 +
.../joshua/system/StructuredOutputTest.java | 118 +
.../system/StructuredTranslationTest.java | 274 +
.../ui/tree_visualizer/tree/TreeTest.java | 111 +
.../java/org/apache/joshua/util/BitsTest.java | 187 +
.../java/org/apache/joshua/util/CacheTest.java | 53 +
.../java/org/apache/joshua/util/CountsTest.java | 98 +
.../org/apache/joshua/util/FormatUtilsTest.java | 74 +
.../org/apache/joshua/util/io/BinaryTest.java | 74 +
.../java/org/apache/joshua/zmert/BLEUTest.java | 132 +
.../src/test/resources/bn-en/hiero/.gitignore | 4 +
.../src/test/resources/bn-en/hiero/class.map | 5140 ++++
.../resources/bn-en/hiero/class_lm_2gram.gz | Bin 0 -> 18052 bytes
.../resources/bn-en/hiero/class_lm_9gram.gz | Bin 0 -> 12733137 bytes
.../src/test/resources/bn-en/hiero/glue-grammar | 3 +
.../src/test/resources/bn-en/hiero/grammar.gz | Bin 0 -> 518164 bytes
.../src/test/resources/bn-en/hiero/input.bn | 100 +
.../bn-en/hiero/joshua-berkeleylm.config | 46 +
.../resources/bn-en/hiero/joshua-classlm.config | 51 +
.../test/resources/bn-en/hiero/joshua.config | 50 +
.../src/test/resources/bn-en/hiero/lm.gz | Bin 0 -> 2466496 bytes
.../resources/bn-en/hiero/output-classlm.gold | 678 +
.../src/test/resources/bn-en/hiero/output.gold | 805 +
.../test/resources/bn-en/hiero/output.gold.bleu | 14 +
.../bn-en/hiero/output.scores.berkeleylm.gold | 100 +
.../resources/bn-en/hiero/output.scores.gold | 805 +
.../test/resources/bn-en/hiero/reference.en.0 | 100 +
.../test/resources/bn-en/hiero/reference.en.1 | 100 +
.../test/resources/bn-en/hiero/reference.en.2 | 100 +
.../test/resources/bn-en/hiero/reference.en.3 | 100 +
.../resources/bn-en/hiero/test-berkeleylm.sh | 33 +
.../test/resources/bn-en/hiero/test-classlm.sh | 32 +
.../test/resources/bn-en/hiero/test-filter.sh | 35 +
.../src/test/resources/bn-en/hiero/test.sh | 35 +
.../src/test/resources/bn-en/hiero/topN.pl | 18 +
.../src/test/resources/bn-en/packed/.gitignore | 3 +
.../test/resources/bn-en/packed/grammar.glue | 5673 +++++
.../src/test/resources/bn-en/packed/grammar.gz | Bin 0 -> 3540984 bytes
.../bn-en/packed/grammar.packed/encoding | Bin 0 -> 767 bytes
.../packed/grammar.packed/slice_00000.features | Bin 0 -> 4631480 bytes
.../packed/grammar.packed/slice_00000.source | Bin 0 -> 4240012 bytes
.../packed/grammar.packed/slice_00000.target | Bin 0 -> 162776 bytes
.../grammar.packed/slice_00000.target.lookup | Bin 0 -> 56 bytes
.../bn-en/packed/grammar.packed/vocabulary | Bin 0 -> 136522 bytes
.../src/test/resources/bn-en/packed/input.bn | 100 +
.../test/resources/bn-en/packed/joshua.config | 47 +
.../src/test/resources/bn-en/packed/lm.gz | Bin 0 -> 2466496 bytes
.../src/test/resources/bn-en/packed/output.gold | 862 +
.../resources/bn-en/packed/output.scores.gold | 862 +
.../test/resources/bn-en/packed/reference.en.0 | 100 +
.../test/resources/bn-en/packed/reference.en.1 | 100 +
.../test/resources/bn-en/packed/reference.en.2 | 100 +
.../test/resources/bn-en/packed/reference.en.3 | 100 +
.../resources/bn-en/packed/reference.en.all | 400 +
.../src/test/resources/bn-en/packed/test.sh | 20 +
.../src/test/resources/bn-en/samt/grammar.glue | 5673 +++++
.../src/test/resources/bn-en/samt/grammar.gz | Bin 0 -> 3847934 bytes
.../src/test/resources/bn-en/samt/input.bn | 100 +
.../src/test/resources/bn-en/samt/joshua.config | 47 +
joshua-core/src/test/resources/bn-en/samt/lm.gz | Bin 0 -> 2466496 bytes
.../src/test/resources/bn-en/samt/output.gold | 0
.../test/resources/bn-en/samt/output.gold.bleu | 14 +
.../resources/bn-en/samt/output.scores.gold | 862 +
.../test/resources/bn-en/samt/reference.en.0 | 100 +
.../test/resources/bn-en/samt/reference.en.1 | 100 +
.../test/resources/bn-en/samt/reference.en.2 | 100 +
.../test/resources/bn-en/samt/reference.en.3 | 100 +
.../src/test/resources/bn-en/samt/test.sh | 35 +
joshua-core/src/test/resources/data/tiny.en | 5 +
.../resources/decoder/constrained/.gitignore | 4 +
.../resources/decoder/constrained/glue-grammar | 3 +
.../resources/decoder/constrained/gold.scores | 27 +
.../resources/decoder/constrained/grammar.gz | Bin 0 -> 518164 bytes
.../test/resources/decoder/constrained/input.bn | 8 +
.../resources/decoder/constrained/joshua.config | 45 +
.../test/resources/decoder/constrained/lm.gz | Bin 0 -> 2466496 bytes
.../resources/decoder/constrained/output.bleu | 0
.../resources/decoder/constrained/output.gold | 30 +
.../test/resources/decoder/constrained/test.sh | 30 +
.../test/resources/decoder/constrained/weights | 22 +
.../resources/decoder/denormalization/input.txt | 1 +
.../decoder/denormalization/output.expected | 1 +
.../resources/decoder/denormalization/test.sh | 30 +
.../src/test/resources/decoder/dont-crash/input | 10 +
.../test/resources/decoder/dont-crash/test.sh | 29 +
.../resources/decoder/empty-test/.gitignore | 3 +
.../src/test/resources/decoder/empty-test/input | 1 +
.../resources/decoder/empty-test/output.gold | 1 +
.../test/resources/decoder/empty-test/test.sh | 29 +
.../resources/decoder/fragmentlm/fragments.txt | 7 +
.../src/test/resources/decoder/fragmentlm/glue | 1 +
.../test/resources/decoder/fragmentlm/grammar | 4 +
.../src/test/resources/decoder/fragmentlm/input | 1 +
.../resources/decoder/fragmentlm/joshua.config | 109 +
.../resources/decoder/fragmentlm/mapping.txt | 4 +
.../test/resources/decoder/fragmentlm/test.sh | 30 +
.../decoder/k-best-extraction/glue-grammar | 3 +
.../resources/decoder/k-best-extraction/grammar | 25 +
.../decoder/k-best-extraction/input.txt | 1 +
.../decoder/k-best-extraction/joshua.config | 27 +
.../resources/decoder/k-best-extraction/lm.gz | Bin 0 -> 2466496 bytes
.../decoder/k-best-extraction/output.gold | 3126 +++
.../k-best-extraction/output.scores.gold | 3126 +++
.../resources/decoder/k-best-extraction/test.sh | 33 +
.../resources/decoder/left-state/glue-grammar | 3 +
.../resources/decoder/left-state/grammar.gz | Bin 0 -> 518164 bytes
.../test/resources/decoder/left-state/input.bn | 2 +
.../resources/decoder/left-state/joshua.config | 44 +
.../src/test/resources/decoder/left-state/lm.gz | Bin 0 -> 2466496 bytes
.../resources/decoder/left-state/output.gold | 600 +
.../decoder/left-state/output.scores.gold | 600 +
.../test/resources/decoder/left-state/test.sh | 33 +
.../test/resources/decoder/lowercaser/config | 140 +
.../resources/decoder/lowercaser/grammar.glue | 4 +
.../resources/decoder/lowercaser/grammar.test | 1 +
.../resources/decoder/lowercaser/output.gold | 5 +
.../test/resources/decoder/lowercaser/test.sh | 40 +
.../decoder/metadata/add_rule/output.gold | 4 +
.../resources/decoder/metadata/add_rule/test.sh | 32 +
.../resources/decoder/moses-compat/n-best.txt | 0
.../decoder/moses-compat/output.expected | 6 +
.../test/resources/decoder/moses-compat/test.sh | 40 +
.../test/resources/decoder/n-ary/glue-grammar | 3 +
.../test/resources/decoder/n-ary/gold.scores | 2 +
.../src/test/resources/decoder/n-ary/grammar | 9 +
.../src/test/resources/decoder/n-ary/input.txt | 2 +
.../test/resources/decoder/n-ary/joshua.config | 22 +
.../src/test/resources/decoder/n-ary/lm.gz | Bin 0 -> 2466496 bytes
.../test/resources/decoder/n-ary/output.bleu | 0
.../test/resources/decoder/n-ary/output.gold | 2 +
.../src/test/resources/decoder/n-ary/test.sh | 33 +
.../src/test/resources/decoder/n-ary/weights | 6 +
.../decoder/num_translation_options/README | 1 +
.../num_translation_options/glue-grammar | 3 +
.../decoder/num_translation_options/grammar.gz | Bin 0 -> 119 bytes
.../grammar.packed/encoding | Bin 0 -> 32 bytes
.../grammar.packed/slice_00000.features | Bin 0 -> 43 bytes
.../grammar.packed/slice_00000.source | Bin 0 -> 132 bytes
.../grammar.packed/slice_00000.target | Bin 0 -> 120 bytes
.../grammar.packed/slice_00000.target.lookup | Bin 0 -> 32 bytes
.../grammar.packed/vocabulary | Bin 0 -> 144 bytes
.../decoder/num_translation_options/input | 1 +
.../num_translation_options/joshua.config | 30 +
.../joshua.config.packed | 30 +
.../decoder/num_translation_options/lm.gz | Bin 0 -> 2466496 bytes
.../decoder/num_translation_options/output.gold | 12 +
.../decoder/num_translation_options/test.sh | 17 +
.../src/test/resources/decoder/oov-list/config | 29 +
.../resources/decoder/oov-list/glue-grammar | 3 +
.../src/test/resources/decoder/oov-list/grammar | 11 +
.../test/resources/decoder/oov-list/input.txt | 3 +
.../test/resources/decoder/oov-list/output.gold | 3 +
.../src/test/resources/decoder/oov-list/test.sh | 30 +
.../resources/decoder/phrase/constrained/config | 29 +
.../decoder/phrase/constrained/corpus.es | 1 +
.../decoder/phrase/constrained/glue.grammar | 3 +
.../decoder/phrase/constrained/output.gold | 5 +
.../decoder/phrase/constrained/test.sh | 32 +
.../test/resources/decoder/phrase/decode/config | 29 +
.../decoder/phrase/decode/config.packed | 29 +
.../resources/decoder/phrase/decode/corpus.es | 1 +
.../resources/decoder/phrase/decode/lm.1.gz | Bin 0 -> 2235 bytes
.../resources/decoder/phrase/decode/output.gold | 1 +
.../resources/decoder/phrase/decode/rules.1.gz | Bin 0 -> 2998042 bytes
.../decoder/phrase/decode/rules.packed/config | 2 +
.../decoder/phrase/decode/rules.packed/encoding | Bin 0 -> 87 bytes
.../decode/rules.packed/slice_00000.features | Bin 0 -> 4128858 bytes
.../decode/rules.packed/slice_00000.source | Bin 0 -> 1982244 bytes
.../decode/rules.packed/slice_00000.target | Bin 0 -> 2652936 bytes
.../rules.packed/slice_00000.target.lookup | Bin 0 -> 32 bytes
.../phrase/decode/rules.packed/vocabulary | Bin 0 -> 169236 bytes
.../decoder/phrase/decode/test-packed.sh | 32 +
.../resources/decoder/phrase/decode/test.sh | 17 +
.../decoder/phrase/include-align-index/README | 2 +
.../decoder/phrase/include-align-index/config | 29 +
.../phrase/include-align-index/corpus.es | 1 +
.../decoder/phrase/include-align-index/lm.1.gz | Bin 0 -> 2235 bytes
.../decoder/phrase/include-align-index/log | 50 +
.../decoder/phrase/include-align-index/output | 1 +
.../phrase/include-align-index/output.gold | 1 +
.../phrase/include-align-index/rules.1.gz | Bin 0 -> 2998042 bytes
.../decoder/phrase/include-align-index/test.sh | 32 +
.../decoder/phrase/unique-hypotheses/README | 1 +
.../decoder/phrase/unique-hypotheses/corpus.es | 1 +
.../phrase/unique-hypotheses/joshua.config | 23 +
.../decoder/phrase/unique-hypotheses/lm.1.gz | 1 +
.../phrase/unique-hypotheses/output.gold | 300 +
.../decoder/phrase/unique-hypotheses/rules.1.gz | 1 +
.../decoder/phrase/unique-hypotheses/test.sh | 32 +
.../resources/decoder/rescoring/glue-grammar | 3 +
.../test/resources/decoder/rescoring/grammar.gz | Bin 0 -> 177 bytes
.../test/resources/decoder/rescoring/input.txt | 2 +
.../resources/decoder/rescoring/joshua.config | 31 +
.../resources/decoder/rescoring/output.gold | 12 +
.../test/resources/decoder/rescoring/test.sh | 30 +
.../test/resources/decoder/segment-oovs/config | 41 +
.../resources/decoder/segment-oovs/input.txt | 1 +
.../decoder/segment-oovs/output.expected | 82 +
.../test/resources/decoder/segment-oovs/test.sh | 31 +
.../decoder/source-annotations/grammar | 5 +
.../decoder/source-annotations/grammar.glue | 3 +
.../decoder/source-annotations/input.txt | 1 +
.../decoder/source-annotations/joshua.config | 140 +
.../decoder/source-annotations/lm.kenlm | Bin 0 -> 25355958 bytes
.../decoder/source-annotations/output.gold | 2 +
.../decoder/source-annotations/test.sh | 36 +
.../resources/decoder/target-bigram/out.gold | 3 +
.../resources/decoder/target-bigram/test.sh | 32 +
.../test/resources/decoder/target-bigram/vocab | 4 +
.../test/resources/decoder/too-long/output.gold | 4 +
.../src/test/resources/decoder/too-long/test.sh | 36 +
.../decoder/tree-output/fragment-map.txt | 2 +
.../resources/decoder/tree-output/glue-grammar | 6 +
.../resources/decoder/tree-output/grammar.gz | Bin 0 -> 134 bytes
.../test/resources/decoder/tree-output/input | 5 +
.../resources/decoder/tree-output/joshua.config | 45 +
.../test/resources/decoder/tree-output/lm.gz | Bin 0 -> 2466496 bytes
.../resources/decoder/tree-output/output.gold | 5 +
.../test/resources/decoder/tree-output/test.sh | 30 +
.../resources/grammar/sparse-features/grammar | 1 +
.../grammar/sparse-features/grammar.glue | 3 +
.../sparse-features/grammar.packed/encoding | Bin 0 -> 118 bytes
.../grammar.packed/slice_00000.features | Bin 0 -> 18 bytes
.../grammar.packed/slice_00000.source | Bin 0 -> 52 bytes
.../grammar.packed/slice_00000.target | Bin 0 -> 24 bytes
.../grammar.packed/slice_00000.target.lookup | Bin 0 -> 16 bytes
.../sparse-features/grammar.packed/vocabulary | Bin 0 -> 104 bytes
.../sparse-features/joshua-packed.config | 12 +
.../grammar/sparse-features/joshua.config | 12 +
.../grammar/sparse-features/output.gold | 1 +
.../grammar/sparse-features/test-packed.sh | 32 +
.../resources/grammar/sparse-features/test.sh | 32 +
.../src/test/resources/joshua/README.broken | 1 +
.../src/test/resources/lattice-short/README | 3 +
.../test/resources/lattice-short/glue-grammar | 3 +
.../test/resources/lattice-short/grammar.test | 3 +
.../src/test/resources/lattice-short/input | 5 +
.../test/resources/lattice-short/joshua.config | 39 +
.../resources/lattice-short/output.expected | 18 +
.../src/test/resources/lattice-short/test.lm | 113 +
.../src/test/resources/lattice-short/test.sh | 31 +
.../src/test/resources/lattice/.gitignore | 3 +
joshua-core/src/test/resources/lattice/README | 4 +
.../src/test/resources/lattice/glue-grammar | 3 +
.../src/test/resources/lattice/grammar.test | 204 +
.../src/test/resources/lattice/joshua.config | 47 +
.../src/test/resources/lattice/output.expected | 33 +
.../src/test/resources/lattice/test-lattice.pdf | Bin 0 -> 10943 bytes
joshua-core/src/test/resources/lattice/test.lm | 113 +
joshua-core/src/test/resources/lattice/test.plf | 4 +
joshua-core/src/test/resources/lattice/test.sh | 37 +
joshua-core/src/test/resources/lm/berkeley/lm | 16 +
.../test/resources/lm/berkeley/lm.berkeleylm | Bin 0 -> 4294 bytes
.../test/resources/lm/berkeley/lm.berkeleylm.gz | Bin 0 -> 1786 bytes
.../src/test/resources/lm/berkeley/lm.gz | Bin 0 -> 162 bytes
.../src/test/resources/lm/berkeley/output.gold | 4 +
.../src/test/resources/lm/berkeley/test.sh | 30 +
.../test/resources/packed-grammar/.gitignore | 8 +
.../src/test/resources/packed-grammar/README | 2 +
.../test/resources/packed-grammar/grammar.gz | Bin 0 -> 576901 bytes
.../src/test/resources/packed-grammar/input.bn | 100 +
.../test/resources/packed-grammar/joshua.config | 46 +
.../src/test/resources/packed-grammar/lm.gz | Bin 0 -> 2466496 bytes
.../test/resources/packed-grammar/output.gold | 100 +
.../resources/packed-grammar/reference.en.0 | 100 +
.../resources/packed-grammar/reference.en.1 | 100 +
.../resources/packed-grammar/reference.en.2 | 100 +
.../resources/packed-grammar/reference.en.3 | 100 +
.../resources/packed-grammar/test-multiple.sh | 31 +
.../src/test/resources/packed-grammar/test.sh | 38 +
joshua-core/src/test/resources/parser/grammar | 11 +
.../src/test/resources/parser/grammar.glue | 1 +
joshua-core/src/test/resources/parser/input | 4 +
.../src/test/resources/parser/output.gold | 4 +
.../src/test/resources/parser/parse.config | 18 +
joshua-core/src/test/resources/parser/test.sh | 29 +
joshua-core/src/test/resources/parser/weights | 4 +
.../src/test/resources/pipeline/.gitignore | 2 +
.../src/test/resources/pipeline/Makefile | 10 +
.../src/test/resources/pipeline/final-bleu.gold | 1 +
.../test/resources/pipeline/input/devtest.en.0 | 100 +
.../test/resources/pipeline/input/devtest.en.1 | 100 +
.../test/resources/pipeline/input/devtest.en.2 | 100 +
.../test/resources/pipeline/input/devtest.en.3 | 100 +
.../test/resources/pipeline/input/devtest.ur | 100 +
.../src/test/resources/pipeline/input/train.en | 1000 +
.../src/test/resources/pipeline/input/train.ur | 1000 +
.../src/test/resources/pipeline/input/tune.en.0 | 100 +
.../src/test/resources/pipeline/input/tune.en.1 | 100 +
.../src/test/resources/pipeline/input/tune.en.2 | 100 +
.../src/test/resources/pipeline/input/tune.en.3 | 100 +
.../src/test/resources/pipeline/input/tune.ur | 100 +
.../src/test/resources/pipeline/test-ghkm.sh | 43 +
joshua-core/src/test/resources/pipeline/test.sh | 39 +
.../resources/prune-equivalent-translations.py | 47 +
joshua-core/src/test/resources/run-all-tests.sh | 55 +
.../src/test/resources/scripts/.gitignore | 1 +
.../test/resources/scripts/merge_lms_test.py | 53 +
.../resources/scripts/normalization/.gitignore | 2 +
.../scripts/normalization/data/train.en | 21 +
.../scripts/normalization/data/train.en.norm | 21 +
.../resources/scripts/normalization/test.sh | 29 +
.../test/resources/scripts/run_bundler_test.py | 378 +
.../scripts/support/moses_grammar/input | 10 +
.../support/moses_grammar/output.expected | 10 +
.../scripts/support/moses_grammar/test.sh | 30 +
.../src/test/resources/server/http/expected | 15 +
.../src/test/resources/server/http/test.sh | 36 +
.../src/test/resources/server/tcp-text/expected | 9 +
.../src/test/resources/server/tcp-text/test.sh | 47 +
joshua-core/src/test/resources/testng.xml | 30 +
joshua-core/src/test/resources/thrax/.gitignore | 5 +
.../resources/thrax/extraction/input/thrax.conf | 71 +
.../resources/thrax/extraction/input/train.a | 100 +
.../resources/thrax/extraction/input/train.en | 100 +
.../resources/thrax/extraction/input/train.ps | 100 +
.../src/test/resources/thrax/extraction/test.sh | 36 +
.../resources/thrax/filtering/dev.hi-en.hi.1 | 1 +
.../test/resources/thrax/filtering/exact.gold | 993 +
.../resources/thrax/filtering/exact.log.gold | 17 +
.../test/resources/thrax/filtering/fast.gold | 1087 +
.../resources/thrax/filtering/fast.log.gold | 17 +
.../test/resources/thrax/filtering/grammar.de | 4 +
.../thrax/filtering/grammar.filtered.gz | Bin 0 -> 134958 bytes
.../src/test/resources/thrax/filtering/input.de | 3 +
.../resources/thrax/filtering/loose.log.gold | 16 +
.../resources/thrax/filtering/test-exact.sh | 34 +
.../test/resources/thrax/filtering/test-fast.sh | 34 +
.../resources/thrax/filtering/test-loose.sh | 34 +
resources/berkeley_lm/lm | 16 -
resources/berkeley_lm/lm.berkeleylm | Bin 4294 -> 0 bytes
resources/berkeley_lm/lm.berkeleylm.gz | Bin 1786 -> 0 bytes
resources/berkeley_lm/lm.gz | Bin 162 -> 0 bytes
resources/grammar.glue | 4 -
resources/kbest_extraction/glue-grammar | 3 -
resources/kbest_extraction/grammar | 25 -
resources/kbest_extraction/joshua.config | 27 -
resources/kbest_extraction/lm.gz | Bin 2466496 -> 0 bytes
resources/kbest_extraction/output.gold | 3126 ---
resources/kbest_extraction/output.scores.gold | 3126 ---
resources/phrase_decoder/config | 29 -
resources/phrase_decoder/constrained.config | 28 -
.../phrase_decoder/constrained.output.gold | 5 -
resources/phrase_decoder/lm.1.gz | Bin 2235 -> 0 bytes
resources/phrase_decoder/output.gold | 1 -
resources/phrase_decoder/rules.1.gz | Bin 2998042 -> 0 bytes
resources/wa_grammar | 3 -
resources/wa_grammar.packed/config | 1 -
resources/wa_grammar.packed/encoding | Bin 154 -> 0 bytes
.../wa_grammar.packed/slice_00000.alignments | Bin 45 -> 0 bytes
.../wa_grammar.packed/slice_00000.features | Bin 47 -> 0 bytes
resources/wa_grammar.packed/slice_00000.source | Bin 204 -> 0 bytes
resources/wa_grammar.packed/slice_00000.target | Bin 128 -> 0 bytes
.../wa_grammar.packed/slice_00000.target.lookup | Bin 32 -> 0 bytes
resources/wa_grammar.packed/vocabulary | Bin 238 -> 0 bytes
.../java/org/apache/joshua/adagrad/AdaGrad.java | 160 -
.../org/apache/joshua/adagrad/AdaGradCore.java | 3127 ---
.../org/apache/joshua/adagrad/Optimizer.java | 728 -
.../apache/joshua/corpus/AbstractPhrase.java | 133 -
.../org/apache/joshua/corpus/BasicPhrase.java | 97 -
.../apache/joshua/corpus/ContiguousPhrase.java | 127 -
.../java/org/apache/joshua/corpus/Corpus.java | 160 -
.../java/org/apache/joshua/corpus/Phrase.java | 117 -
.../java/org/apache/joshua/corpus/Span.java | 175 -
.../org/apache/joshua/corpus/SymbolTable.java | 327 -
.../apache/joshua/corpus/TerminalIterator.java | 85 -
.../org/apache/joshua/corpus/Vocabulary.java | 301 -
.../joshua/corpus/syntax/ArraySyntaxTree.java | 411 -
.../apache/joshua/corpus/syntax/SyntaxTree.java | 34 -
.../org/apache/joshua/decoder/ArgsParser.java | 118 -
.../java/org/apache/joshua/decoder/BLEU.java | 562 -
.../java/org/apache/joshua/decoder/Decoder.java | 813 -
.../apache/joshua/decoder/DecoderThread.java | 201 -
.../joshua/decoder/JoshuaConfiguration.java | 729 -
.../apache/joshua/decoder/JoshuaDecoder.java | 148 -
.../joshua/decoder/NbestMinRiskReranker.java | 446 -
.../joshua/decoder/StructuredTranslation.java | 164 -
.../decoder/StructuredTranslationFactory.java | 117 -
.../java/org/apache/joshua/decoder/Support.java | 86 -
.../org/apache/joshua/decoder/Translation.java | 239 -
.../org/apache/joshua/decoder/Translations.java | 158 -
.../joshua/decoder/chart_parser/Cell.java | 294 -
.../joshua/decoder/chart_parser/Chart.java | 746 -
.../decoder/chart_parser/ComputeNodeResult.java | 225 -
.../decoder/chart_parser/CubePruneState.java | 114 -
.../joshua/decoder/chart_parser/DotChart.java | 476 -
.../joshua/decoder/chart_parser/SourcePath.java | 63 -
.../decoder/chart_parser/StateConstraint.java | 75 -
.../joshua/decoder/chart_parser/SuperNode.java | 62 -
.../decoder/chart_parser/package-info.java | 24 -
.../joshua/decoder/ff/ArityPhrasePenalty.java | 72 -
.../joshua/decoder/ff/FeatureFunction.java | 364 -
.../apache/joshua/decoder/ff/FeatureVector.java | 385 -
.../joshua/decoder/ff/LabelCombinationFF.java | 62 -
.../joshua/decoder/ff/LabelSubstitutionFF.java | 131 -
.../joshua/decoder/ff/LexicalFeatures.java | 150 -
.../apache/joshua/decoder/ff/OOVPenalty.java | 106 -
.../apache/joshua/decoder/ff/PhraseModel.java | 135 -
.../apache/joshua/decoder/ff/PhrasePenalty.java | 86 -
.../apache/joshua/decoder/ff/RuleCountBin.java | 74 -
.../org/apache/joshua/decoder/ff/RuleFF.java | 123 -
.../apache/joshua/decoder/ff/RuleLength.java | 52 -
.../decoder/ff/RulePropertiesQuerying.java | 49 -
.../org/apache/joshua/decoder/ff/RuleShape.java | 99 -
.../joshua/decoder/ff/SourceDependentFF.java | 29 -
.../apache/joshua/decoder/ff/SourcePathFF.java | 63 -
.../apache/joshua/decoder/ff/StatefulFF.java | 88 -
.../apache/joshua/decoder/ff/StatelessFF.java | 79 -
.../apache/joshua/decoder/ff/TargetBigram.java | 216 -
.../apache/joshua/decoder/ff/WordPenalty.java | 80 -
.../ff/fragmentlm/ConcatenationIterator.java | 93 -
.../decoder/ff/fragmentlm/FragmentLMFF.java | 365 -
.../ff/fragmentlm/PennTreebankReader.java | 135 -
.../joshua/decoder/ff/fragmentlm/Tree.java | 779 -
.../joshua/decoder/ff/fragmentlm/Trees.java | 270 -
.../apache/joshua/decoder/ff/lm/AbstractLM.java | 122 -
.../apache/joshua/decoder/ff/lm/ArpaFile.java | 328 -
.../apache/joshua/decoder/ff/lm/ArpaNgram.java | 73 -
.../ff/lm/DefaultNGramLanguageModel.java | 134 -
.../org/apache/joshua/decoder/ff/lm/KenLM.java | 233 -
.../joshua/decoder/ff/lm/LanguageModelFF.java | 527 -
.../decoder/ff/lm/NGramLanguageModel.java | 73 -
.../ff/lm/StateMinimizingLanguageModel.java | 202 -
.../joshua/decoder/ff/lm/berkeley_lm/LICENSE | 13 -
.../ff/lm/berkeley_lm/LMGrammarBerkeley.java | 205 -
.../joshua/decoder/ff/lm/berkeley_lm/README | 5 -
.../ff/lm/berkeley_lm/SymbolTableWrapper.java | 102 -
.../ff/lm/bloomfilter_lm/BloomFilter.java | 215 -
.../BloomFilterLanguageModel.java | 560 -
.../ff/lm/bloomfilter_lm/package-info.java | 25 -
.../joshua/decoder/ff/lm/buildin_lm/TrieLM.java | 331 -
.../decoder/ff/lm/buildin_lm/package-info.java | 19 -
.../joshua/decoder/ff/lm/package-info.java | 42 -
.../apache/joshua/decoder/ff/package-info.java | 42 -
.../joshua/decoder/ff/phrase/Distortion.java | 71 -
.../ff/similarity/EdgePhraseSimilarityFF.java | 279 -
.../decoder/ff/state_maintenance/DPState.java | 34 -
.../ff/state_maintenance/KenLMState.java | 56 -
.../ff/state_maintenance/NgramDPState.java | 100 -
.../joshua/decoder/ff/tm/AbstractGrammar.java | 228 -
.../decoder/ff/tm/BasicRuleCollection.java | 101 -
.../joshua/decoder/ff/tm/CreateGlueGrammar.java | 126 -
.../apache/joshua/decoder/ff/tm/Grammar.java | 120 -
.../joshua/decoder/ff/tm/GrammarReader.java | 158 -
.../org/apache/joshua/decoder/ff/tm/Rule.java | 633 -
.../joshua/decoder/ff/tm/RuleCollection.java | 76 -
.../decoder/ff/tm/SentenceFilteredGrammar.java | 366 -
.../org/apache/joshua/decoder/ff/tm/Trie.java | 108 -
.../ff/tm/UnsortedRuleCollectionException.java | 40 -
.../decoder/ff/tm/format/HieroFormatReader.java | 106 -
.../decoder/ff/tm/format/MosesFormatReader.java | 108 -
.../ff/tm/hash_based/ExtensionIterator.java | 73 -
.../tm/hash_based/MemoryBasedBatchGrammar.java | 292 -
.../ff/tm/hash_based/MemoryBasedRuleBin.java | 59 -
.../ff/tm/hash_based/MemoryBasedTrie.java | 88 -
.../decoder/ff/tm/hash_based/package-info.java | 23 -
.../joshua/decoder/ff/tm/package-info.java | 25 -
.../decoder/ff/tm/packed/PackedGrammar.java | 1066 -
.../ff/tm/packed/SliceAggregatingTrie.java | 236 -
.../decoder/hypergraph/AlignedSourceTokens.java | 112 -
.../decoder/hypergraph/AllSpansWalker.java | 63 -
.../hypergraph/DefaultInsideOutside.java | 407 -
.../hypergraph/FeatureVectorExtractor.java | 80 -
.../joshua/decoder/hypergraph/ForestWalker.java | 79 -
.../GrammarBuilderWalkerFunction.java | 180 -
.../joshua/decoder/hypergraph/HGNode.java | 331 -
.../joshua/decoder/hypergraph/HyperEdge.java | 101 -
.../joshua/decoder/hypergraph/HyperGraph.java | 163 -
.../decoder/hypergraph/HyperGraphPruning.java | 176 -
.../decoder/hypergraph/KBestExtractor.java | 1052 -
.../hypergraph/OutputStringExtractor.java | 195 -
.../hypergraph/StringToTreeConverter.java | 74 -
.../hypergraph/TrivialInsideOutside.java | 31 -
.../decoder/hypergraph/ViterbiExtractor.java | 178 -
.../decoder/hypergraph/WalkerFunction.java | 37 -
.../hypergraph/WordAlignmentExtractor.java | 134 -
.../decoder/hypergraph/WordAlignmentState.java | 192 -
.../joshua/decoder/hypergraph/package-info.java | 25 -
.../apache/joshua/decoder/io/DeNormalize.java | 203 -
.../apache/joshua/decoder/io/JSONMessage.java | 159 -
.../decoder/io/TranslationRequestStream.java | 179 -
.../org/apache/joshua/decoder/package-info.java | 26 -
.../apache/joshua/decoder/phrase/Candidate.java | 241 -
.../decoder/phrase/CandidateComparator.java | 28 -
.../apache/joshua/decoder/phrase/Coverage.java | 235 -
.../apache/joshua/decoder/phrase/Future.java | 119 -
.../apache/joshua/decoder/phrase/Header.java | 87 -
.../joshua/decoder/phrase/Hypothesis.java | 154 -
.../org/apache/joshua/decoder/phrase/Note.java | 44 -
.../joshua/decoder/phrase/PhraseChart.java | 197 -
.../joshua/decoder/phrase/PhraseTable.java | 181 -
.../org/apache/joshua/decoder/phrase/Stack.java | 229 -
.../apache/joshua/decoder/phrase/Stacks.java | 269 -
.../joshua/decoder/phrase/TargetPhrases.java | 80 -
.../decoder/segment_file/ConstraintRule.java | 100 -
.../decoder/segment_file/ConstraintSpan.java | 80 -
.../decoder/segment_file/ParseTreeInput.java | 40 -
.../decoder/segment_file/ParsedSentence.java | 56 -
.../joshua/decoder/segment_file/Sentence.java | 450 -
.../joshua/decoder/segment_file/Token.java | 158 -
.../decoder/segment_file/package-info.java | 25 -
.../java/org/apache/joshua/lattice/Arc.java | 117 -
.../java/org/apache/joshua/lattice/Lattice.java | 587 -
.../java/org/apache/joshua/lattice/Node.java | 159 -
.../lattice/NodeIdentifierComparator.java | 41 -
.../org/apache/joshua/lattice/package-info.java | 22 -
.../java/org/apache/joshua/metrics/BLEU.java | 575 -
.../org/apache/joshua/metrics/BLEU_SBP.java | 63 -
.../apache/joshua/metrics/EvaluationMetric.java | 405 -
.../apache/joshua/metrics/GradeLevelBLEU.java | 280 -
.../java/org/apache/joshua/metrics/METEOR.java | 235 -
.../joshua/metrics/MinimumChangeBLEU.java | 223 -
.../joshua/metrics/NewMetric.java.template | 134 -
.../java/org/apache/joshua/metrics/Precis.java | 334 -
.../joshua/metrics/PrecisMinusSourceBLEU.java | 182 -
.../java/org/apache/joshua/metrics/SARI.java | 681 -
.../org/apache/joshua/metrics/SourceBLEU.java | 107 -
.../java/org/apache/joshua/metrics/TER.java | 460 -
.../org/apache/joshua/metrics/TERMinusBLEU.java | 194 -
.../org/apache/joshua/metrics/TercomRunner.java | 115 -
.../org/apache/joshua/metrics/ZeroOneLoss.java | 88 -
src/main/java/org/apache/joshua/mira/MIRA.java | 160 -
.../java/org/apache/joshua/mira/MIRACore.java | 3112 ---
.../java/org/apache/joshua/mira/Optimizer.java | 643 -
.../joshua/oracle/OracleExtractionHG.java | 797 -
.../apache/joshua/oracle/OracleExtractor.java | 58 -
.../java/org/apache/joshua/oracle/SplitHg.java | 300 -
.../org/apache/joshua/oracle/package-info.java | 26 -
.../apache/joshua/pro/ClassifierInterface.java | 41 -
.../org/apache/joshua/pro/ClassifierMegaM.java | 121 -
.../apache/joshua/pro/ClassifierPerceptron.java | 114 -
.../org/apache/joshua/pro/ClassifierSVM.java | 140 -
.../java/org/apache/joshua/pro/Optimizer.java | 454 -
src/main/java/org/apache/joshua/pro/PRO.java | 159 -
.../java/org/apache/joshua/pro/PROCore.java | 3027 ---
.../org/apache/joshua/server/ServerThread.java | 294 -
.../org/apache/joshua/server/TcpServer.java | 65 -
.../joshua/subsample/AlignedSubsampler.java | 103 -
.../org/apache/joshua/subsample/Alignment.java | 92 -
.../org/apache/joshua/subsample/BiCorpus.java | 186 -
.../joshua/subsample/BiCorpusFactory.java | 83 -
.../org/apache/joshua/subsample/PhrasePair.java | 72 -
.../apache/joshua/subsample/PhraseReader.java | 46 -
.../apache/joshua/subsample/PhraseWriter.java | 79 -
.../org/apache/joshua/subsample/Subsampler.java | 246 -
.../apache/joshua/subsample/SubsamplerCLI.java | 141 -
.../apache/joshua/subsample/package-info.java | 25 -
.../org/apache/joshua/tools/GrammarPacker.java | 959 -
.../apache/joshua/tools/GrammarPackerCli.java | 156 -
.../org/apache/joshua/tools/LabelPhrases.java | 111 -
.../org/apache/joshua/tools/TestSetFilter.java | 383 -
.../java/org/apache/joshua/ui/Orientation.java | 23 -
.../org/apache/joshua/ui/StartupWindow.java | 87 -
.../java/org/apache/joshua/ui/package-info.java | 22 -
.../ui/tree_visualizer/DerivationTree.java | 103 -
.../ui/tree_visualizer/DerivationTreeEdge.java | 27 -
.../DerivationTreeTransformer.java | 117 -
.../ui/tree_visualizer/DerivationViewer.java | 128 -
.../tree_visualizer/DerivationViewerApplet.java | 51 -
.../apache/joshua/ui/tree_visualizer/Node.java | 59 -
.../ui/tree_visualizer/browser/Browser.java | 237 -
.../browser/DerivationTreeFrame.java | 253 -
.../browser/TranslationInfo.java | 56 -
.../joshua/ui/tree_visualizer/tree/Tree.java | 283 -
.../java/org/apache/joshua/util/Algorithms.java | 85 -
src/main/java/org/apache/joshua/util/Bits.java | 128 -
.../java/org/apache/joshua/util/BotMap.java | 94 -
src/main/java/org/apache/joshua/util/Cache.java | 176 -
.../java/org/apache/joshua/util/ChartSpan.java | 88 -
.../apache/joshua/util/CommandLineParser.java | 738 -
.../java/org/apache/joshua/util/Constants.java | 36 -
.../java/org/apache/joshua/util/Counted.java | 92 -
.../java/org/apache/joshua/util/Counts.java | 306 -
.../org/apache/joshua/util/ExtractTopCand.java | 188 -
.../org/apache/joshua/util/FileUtility.java | 318 -
.../org/apache/joshua/util/FormatUtils.java | 245 -
.../org/apache/joshua/util/IntegerPair.java | 36 -
.../java/org/apache/joshua/util/JoshuaEval.java | 624 -
.../java/org/apache/joshua/util/ListUtil.java | 95 -
src/main/java/org/apache/joshua/util/Lists.java | 567 -
.../apache/joshua/util/NBestListUtility.java | 74 -
src/main/java/org/apache/joshua/util/Ngram.java | 105 -
.../org/apache/joshua/util/NullIterator.java | 65 -
.../apache/joshua/util/PackedGrammarServer.java | 87 -
src/main/java/org/apache/joshua/util/Pair.java | 130 -
.../java/org/apache/joshua/util/Platform.java | 27 -
.../org/apache/joshua/util/QuietFormatter.java | 36 -
src/main/java/org/apache/joshua/util/Regex.java | 143 -
.../org/apache/joshua/util/ReverseOrder.java | 39 -
.../org/apache/joshua/util/SampledList.java | 69 -
.../org/apache/joshua/util/SocketUtility.java | 144 -
.../org/apache/joshua/util/StreamGobbler.java | 50 -
.../joshua/util/UnicodeCharacterName.java | 22466 -----------------
.../apache/joshua/util/encoding/Analyzer.java | 235 -
.../joshua/util/encoding/EightBitQuantizer.java | 92 -
.../util/encoding/EncoderConfiguration.java | 160 -
.../joshua/util/encoding/EncoderFactory.java | 42 -
.../util/encoding/FeatureTypeAnalyzer.java | 254 -
.../joshua/util/encoding/FloatEncoder.java | 39 -
.../apache/joshua/util/encoding/IntEncoder.java | 39 -
.../util/encoding/PrimitiveFloatEncoder.java | 129 -
.../util/encoding/PrimitiveIntEncoder.java | 111 -
.../joshua/util/encoding/VariableQuantizer.java | 106 -
.../org/apache/joshua/util/io/BinaryIn.java | 91 -
.../org/apache/joshua/util/io/BinaryOut.java | 505 -
.../apache/joshua/util/io/IndexedReader.java | 155 -
.../org/apache/joshua/util/io/LineReader.java | 368 -
.../org/apache/joshua/util/io/NullReader.java | 63 -
.../joshua/util/io/ProgressInputStream.java | 82 -
.../java/org/apache/joshua/util/io/Reader.java | 51 -
.../org/apache/joshua/util/io/package-info.java | 22 -
.../org/apache/joshua/util/package-info.java | 22 -
.../util/quantization/BooleanQuantizer.java | 45 -
.../joshua/util/quantization/Quantizer.java | 45 -
.../quantization/QuantizerConfiguration.java | 119 -
.../util/quantization/QuantizerFactory.java | 50 -
.../util/quantization/StatelessQuantizer.java | 38 -
.../joshua/util/quantization/package-info.java | 19 -
.../joshua/zmert/IntermediateOptimizer.java | 991 -
.../java/org/apache/joshua/zmert/MertCore.java | 3191 ---
.../java/org/apache/joshua/zmert/ZMERT.java | 156 -
.../org/apache/joshua/zmert/package-info.java | 24 -
src/main/resources/log4j.properties | 20 -
src/overview.html | 41 -
.../apache/joshua/corpus/CorpusArrayTest.java | 177 -
.../java/org/apache/joshua/corpus/SpanTest.java | 47 -
.../apache/joshua/corpus/VocabularyTest.java | 135 -
.../joshua/corpus/vocab/VocabularyTest.java | 180 -
.../ArtificialGrammarAndCorpusCreater.java | 130 -
.../joshua/decoder/DecoderThreadTest.java | 172 -
.../joshua/decoder/JoshuaDecoderTest.java | 83 -
.../joshua/decoder/TestConfigFileCreater.java | 184 -
.../apache/joshua/decoder/TranslationsTest.java | 87 -
.../decoder/ff/ArityPhrasePenaltyFFTest.java | 64 -
.../joshua/decoder/ff/lm/ArpaFileTest.java | 226 -
.../decoder/ff/lm/LanguageModelFFTest.java | 95 -
.../LMBerkeleySentenceProbablityTest.java | 47 -
.../lm/berkeley_lm/LMGrammarBerkeleyTest.java | 80 -
.../joshua/decoder/io/DeNormalizeTest.java | 273 -
.../decoder/io/TranslationRequestTest.java | 149 -
.../kbest_extraction/KBestExtractionTest.java | 80 -
.../joshua/decoder/phrase/CoverageTest.java | 140 -
.../ConstrainedPhraseDecodingTest.java | 77 -
.../phrase/decode/PhraseDecodingTest.java | 77 -
.../segment_file/AlmostTooLongSentenceTest.java | 96 -
.../decoder/segment_file/SentenceTest.java | 109 -
.../java/org/apache/joshua/lattice/ArcTest.java | 86 -
.../org/apache/joshua/lattice/LatticeTest.java | 192 -
.../org/apache/joshua/lattice/NodeTest.java | 105 -
.../org/apache/joshua/packed/Benchmark.java | 126 -
.../org/apache/joshua/packed/CountRules.java | 110 -
.../org/apache/joshua/packed/PrintRules.java | 199 -
src/test/java/org/apache/joshua/packed/README | 6 -
.../org/apache/joshua/packed/VocabTest.java | 58 -
.../java/org/apache/joshua/packed/packer.config | 6 -
.../java/org/apache/joshua/packed/small_grammar | 20000 ---------------
src/test/java/org/apache/joshua/packed/test.sh | 20 -
.../apache/joshua/system/AlignmentMapTest.java | 72 -
.../org/apache/joshua/system/KenLmTest.java | 95 -
.../system/MultithreadedTranslationTests.java | 156 -
.../joshua/system/StructuredOutputTest.java | 118 -
.../system/StructuredTranslationTest.java | 274 -
.../ui/tree_visualizer/tree/TreeTest.java | 111 -
.../java/org/apache/joshua/util/BitsTest.java | 187 -
.../java/org/apache/joshua/util/CacheTest.java | 53 -
.../java/org/apache/joshua/util/CountsTest.java | 98 -
.../org/apache/joshua/util/FormatUtilsTest.java | 74 -
.../org/apache/joshua/util/io/BinaryTest.java | 74 -
.../java/org/apache/joshua/zmert/BLEUTest.java | 132 -
src/test/resources/bn-en/hiero/.gitignore | 4 -
src/test/resources/bn-en/hiero/class.map | 5140 ----
.../resources/bn-en/hiero/class_lm_2gram.gz | Bin 18052 -> 0 bytes
.../resources/bn-en/hiero/class_lm_9gram.gz | Bin 12733137 -> 0 bytes
src/test/resources/bn-en/hiero/glue-grammar | 3 -
src/test/resources/bn-en/hiero/grammar.gz | Bin 518164 -> 0 bytes
src/test/resources/bn-en/hiero/input.bn | 100 -
.../bn-en/hiero/joshua-berkeleylm.config | 46 -
.../resources/bn-en/hiero/joshua-classlm.config | 51 -
src/test/resources/bn-en/hiero/joshua.config | 50 -
src/test/resources/bn-en/hiero/lm.gz | Bin 2466496 -> 0 bytes
.../resources/bn-en/hiero/output-classlm.gold | 678 -
src/test/resources/bn-en/hiero/output.gold | 805 -
src/test/resources/bn-en/hiero/output.gold.bleu | 14 -
.../bn-en/hiero/output.scores.berkeleylm.gold | 100 -
.../resources/bn-en/hiero/output.scores.gold | 805 -
src/test/resources/bn-en/hiero/reference.en.0 | 100 -
src/test/resources/bn-en/hiero/reference.en.1 | 100 -
src/test/resources/bn-en/hiero/reference.en.2 | 100 -
src/test/resources/bn-en/hiero/reference.en.3 | 100 -
.../resources/bn-en/hiero/test-berkeleylm.sh | 33 -
src/test/resources/bn-en/hiero/test-classlm.sh | 32 -
src/test/resources/bn-en/hiero/test-filter.sh | 35 -
src/test/resources/bn-en/hiero/test.sh | 35 -
src/test/resources/bn-en/hiero/topN.pl | 18 -
src/test/resources/bn-en/packed/.gitignore | 3 -
src/test/resources/bn-en/packed/grammar.glue | 5673 -----
src/test/resources/bn-en/packed/grammar.gz | Bin 3540984 -> 0 bytes
.../bn-en/packed/grammar.packed/encoding | Bin 767 -> 0 bytes
.../packed/grammar.packed/slice_00000.features | Bin 4631480 -> 0 bytes
.../packed/grammar.packed/slice_00000.source | Bin 4240012 -> 0 bytes
.../packed/grammar.packed/slice_00000.target | Bin 162776 -> 0 bytes
.../grammar.packed/slice_00000.target.lookup | Bin 56 -> 0 bytes
.../bn-en/packed/grammar.packed/vocabulary | Bin 136522 -> 0 bytes
src/test/resources/bn-en/packed/input.bn | 100 -
src/test/resources/bn-en/packed/joshua.config | 47 -
src/test/resources/bn-en/packed/lm.gz | Bin 2466496 -> 0 bytes
src/test/resources/bn-en/packed/output.gold | 862 -
.../resources/bn-en/packed/output.scores.gold | 862 -
src/test/resources/bn-en/packed/reference.en.0 | 100 -
src/test/resources/bn-en/packed/reference.en.1 | 100 -
src/test/resources/bn-en/packed/reference.en.2 | 100 -
src/test/resources/bn-en/packed/reference.en.3 | 100 -
.../resources/bn-en/packed/reference.en.all | 400 -
src/test/resources/bn-en/packed/test.sh | 20 -
src/test/resources/bn-en/samt/grammar.glue | 5673 -----
src/test/resources/bn-en/samt/grammar.gz | Bin 3847934 -> 0 bytes
src/test/resources/bn-en/samt/input.bn | 100 -
src/test/resources/bn-en/samt/joshua.config | 47 -
src/test/resources/bn-en/samt/lm.gz | Bin 2466496 -> 0 bytes
src/test/resources/bn-en/samt/output.gold | 0
src/test/resources/bn-en/samt/output.gold.bleu | 14 -
.../resources/bn-en/samt/output.scores.gold | 862 -
src/test/resources/bn-en/samt/reference.en.0 | 100 -
src/test/resources/bn-en/samt/reference.en.1 | 100 -
src/test/resources/bn-en/samt/reference.en.2 | 100 -
src/test/resources/bn-en/samt/reference.en.3 | 100 -
src/test/resources/bn-en/samt/test.sh | 35 -
src/test/resources/data/tiny.en | 5 -
.../resources/decoder/constrained/.gitignore | 4 -
.../resources/decoder/constrained/glue-grammar | 3 -
.../resources/decoder/constrained/gold.scores | 27 -
.../resources/decoder/constrained/grammar.gz | Bin 518164 -> 0 bytes
src/test/resources/decoder/constrained/input.bn | 8 -
.../resources/decoder/constrained/joshua.config | 45 -
src/test/resources/decoder/constrained/lm.gz | Bin 2466496 -> 0 bytes
.../resources/decoder/constrained/output.bleu | 0
.../resources/decoder/constrained/output.gold | 30 -
src/test/resources/decoder/constrained/test.sh | 30 -
src/test/resources/decoder/constrained/weights | 22 -
.../resources/decoder/denormalization/input.txt | 1 -
.../decoder/denormalization/output.expected | 1 -
.../resources/decoder/denormalization/test.sh | 30 -
src/test/resources/decoder/dont-crash/input | 10 -
src/test/resources/decoder/dont-crash/test.sh | 29 -
.../resources/decoder/empty-test/.gitignore | 3 -
src/test/resources/decoder/empty-test/input | 1 -
.../resources/decoder/empty-test/output.gold | 1 -
src/test/resources/decoder/empty-test/test.sh | 29 -
.../resources/decoder/fragmentlm/fragments.txt | 7 -
src/test/resources/decoder/fragmentlm/glue | 1 -
src/test/resources/decoder/fragmentlm/grammar | 4 -
src/test/resources/decoder/fragmentlm/input | 1 -
.../resources/decoder/fragmentlm/joshua.config | 109 -
.../resources/decoder/fragmentlm/mapping.txt | 4 -
src/test/resources/decoder/fragmentlm/test.sh | 30 -
.../decoder/k-best-extraction/glue-grammar | 3 -
.../resources/decoder/k-best-extraction/grammar | 25 -
.../decoder/k-best-extraction/input.txt | 1 -
.../decoder/k-best-extraction/joshua.config | 27 -
.../resources/decoder/k-best-extraction/lm.gz | Bin 2466496 -> 0 bytes
.../decoder/k-best-extraction/output.gold | 3126 ---
.../k-best-extraction/output.scores.gold | 3126 ---
.../resources/decoder/k-best-extraction/test.sh | 33 -
.../resources/decoder/left-state/glue-grammar | 3 -
.../resources/decoder/left-state/grammar.gz | Bin 518164 -> 0 bytes
src/test/resources/decoder/left-state/input.bn | 2 -
.../resources/decoder/left-state/joshua.config | 44 -
src/test/resources/decoder/left-state/lm.gz | Bin 2466496 -> 0 bytes
.../resources/decoder/left-state/output.gold | 600 -
.../decoder/left-state/output.scores.gold | 600 -
src/test/resources/decoder/left-state/test.sh | 33 -
src/test/resources/decoder/lowercaser/config | 140 -
.../resources/decoder/lowercaser/grammar.glue | 4 -
.../resources/decoder/lowercaser/grammar.test | 1 -
.../resources/decoder/lowercaser/output.gold | 5 -
src/test/resources/decoder/lowercaser/test.sh | 40 -
.../decoder/metadata/add_rule/output.gold | 4 -
.../resources/decoder/metadata/add_rule/test.sh | 32 -
.../resources/decoder/moses-compat/n-best.txt | 0
.../decoder/moses-compat/output.expected | 6 -
src/test/resources/decoder/moses-compat/test.sh | 40 -
src/test/resources/decoder/n-ary/glue-grammar | 3 -
src/test/resources/decoder/n-ary/gold.scores | 2 -
src/test/resources/decoder/n-ary/grammar | 9 -
src/test/resources/decoder/n-ary/input.txt | 2 -
src/test/resources/decoder/n-ary/joshua.config | 22 -
src/test/resources/decoder/n-ary/lm.gz | Bin 2466496 -> 0 bytes
src/test/resources/decoder/n-ary/output.bleu | 0
src/test/resources/decoder/n-ary/output.gold | 2 -
src/test/resources/decoder/n-ary/test.sh | 33 -
src/test/resources/decoder/n-ary/weights | 6 -
.../decoder/num_translation_options/README | 1 -
.../num_translation_options/glue-grammar | 3 -
.../decoder/num_translation_options/grammar.gz | Bin 119 -> 0 bytes
.../grammar.packed/encoding | Bin 32 -> 0 bytes
.../grammar.packed/slice_00000.features | Bin 43 -> 0 bytes
.../grammar.packed/slice_00000.source | Bin 132 -> 0 bytes
.../grammar.packed/slice_00000.target | Bin 120 -> 0 bytes
.../grammar.packed/slice_00000.target.lookup | Bin 32 -> 0 bytes
.../grammar.packed/vocabulary | Bin 144 -> 0 bytes
.../decoder/num_translation_options/input | 1 -
.../num_translation_options/joshua.config | 30 -
.../joshua.config.packed | 30 -
.../decoder/num_translation_options/lm.gz | Bin 2466496 -> 0 bytes
.../decoder/num_translation_options/output.gold | 12 -
.../decoder/num_translation_options/test.sh | 17 -
src/test/resources/decoder/oov-list/config | 29 -
.../resources/decoder/oov-list/glue-grammar | 3 -
src/test/resources/decoder/oov-list/grammar | 11 -
src/test/resources/decoder/oov-list/input.txt | 3 -
src/test/resources/decoder/oov-list/output.gold | 3 -
src/test/resources/decoder/oov-list/test.sh | 30 -
.../resources/decoder/phrase/constrained/config | 29 -
.../decoder/phrase/constrained/corpus.es | 1 -
.../decoder/phrase/constrained/glue.grammar | 3 -
.../decoder/phrase/constrained/output.gold | 5 -
.../decoder/phrase/constrained/test.sh | 32 -
src/test/resources/decoder/phrase/decode/config | 29 -
.../decoder/phrase/decode/config.packed | 29 -
.../resources/decoder/phrase/decode/corpus.es | 1 -
.../resources/decoder/phrase/decode/lm.1.gz | Bin 2235 -> 0 bytes
.../resources/decoder/phrase/decode/output.gold | 1 -
.../resources/decoder/phrase/decode/rules.1.gz | Bin 2998042 -> 0 bytes
.../decoder/phrase/decode/rules.packed/config | 2 -
.../decoder/phrase/decode/rules.packed/encoding | Bin 87 -> 0 bytes
.../decode/rules.packed/slice_00000.features | Bin 4128858 -> 0 bytes
.../decode/rules.packed/slice_00000.source | Bin 1982244 -> 0 bytes
.../decode/rules.packed/slice_00000.target | Bin 2652936 -> 0 bytes
.../rules.packed/slice_00000.target.lookup | Bin 32 -> 0 bytes
.../phrase/decode/rules.packed/vocabulary | Bin 169236 -> 0 bytes
.../decoder/phrase/decode/test-packed.sh | 32 -
.../resources/decoder/phrase/decode/test.sh | 17 -
.../decoder/phrase/include-align-index/README | 2 -
.../decoder/phrase/include-align-index/config | 29 -
.../phrase/include-align-index/corpus.es | 1 -
.../decoder/phrase/include-align-index/lm.1.gz | Bin 2235 -> 0 bytes
.../decoder/phrase/include-align-index/log | 50 -
.../decoder/phrase/include-align-index/output | 1 -
.../phrase/include-align-index/output.gold | 1 -
.../phrase/include-align-index/rules.1.gz | Bin 2998042 -> 0 bytes
.../decoder/phrase/include-align-index/test.sh | 32 -
.../decoder/phrase/unique-hypotheses/README | 1 -
.../decoder/phrase/unique-hypotheses/corpus.es | 1 -
.../phrase/unique-hypotheses/joshua.config | 23 -
.../decoder/phrase/unique-hypotheses/lm.1.gz | 1 -
.../phrase/unique-hypotheses/output.gold | 300 -
.../decoder/phrase/unique-hypotheses/rules.1.gz | 1 -
.../decoder/phrase/unique-hypotheses/test.sh | 32 -
.../resources/decoder/rescoring/glue-grammar | 3 -
src/test/resources/decoder/rescoring/grammar.gz | Bin 177 -> 0 bytes
src/test/resources/decoder/rescoring/input.txt | 2 -
.../resources/decoder/rescoring/joshua.config | 31 -
.../resources/decoder/rescoring/output.gold | 12 -
src/test/resources/decoder/rescoring/test.sh | 30 -
src/test/resources/decoder/segment-oovs/config | 41 -
.../resources/decoder/segment-oovs/input.txt | 1 -
.../decoder/segment-oovs/output.expected | 82 -
src/test/resources/decoder/segment-oovs/test.sh | 31 -
.../decoder/source-annotations/grammar | 5 -
.../decoder/source-annotations/grammar.glue | 3 -
.../decoder/source-annotations/input.txt | 1 -
.../decoder/source-annotations/joshua.config | 140 -
.../decoder/source-annotations/lm.kenlm | Bin 25355958 -> 0 bytes
.../decoder/source-annotations/output.gold | 2 -
.../decoder/source-annotations/test.sh | 36 -
.../resources/decoder/target-bigram/out.gold | 3 -
.../resources/decoder/target-bigram/test.sh | 32 -
src/test/resources/decoder/target-bigram/vocab | 4 -
src/test/resources/decoder/too-long/output.gold | 4 -
src/test/resources/decoder/too-long/test.sh | 36 -
.../decoder/tree-output/fragment-map.txt | 2 -
.../resources/decoder/tree-output/glue-grammar | 6 -
.../resources/decoder/tree-output/grammar.gz | Bin 134 -> 0 bytes
src/test/resources/decoder/tree-output/input | 5 -
.../resources/decoder/tree-output/joshua.config | 45 -
src/test/resources/decoder/tree-output/lm.gz | Bin 2466496 -> 0 bytes
.../resources/decoder/tree-output/output.gold | 5 -
src/test/resources/decoder/tree-output/test.sh | 30 -
.../resources/grammar/sparse-features/grammar | 1 -
.../grammar/sparse-features/grammar.glue | 3 -
.../sparse-features/grammar.packed/encoding | Bin 118 -> 0 bytes
.../grammar.packed/slice_00000.features | Bin 18 -> 0 bytes
.../grammar.packed/slice_00000.source | Bin 52 -> 0 bytes
.../grammar.packed/slice_00000.target | Bin 24 -> 0 bytes
.../grammar.packed/slice_00000.target.lookup | Bin 16 -> 0 bytes
.../sparse-features/grammar.packed/vocabulary | Bin 104 -> 0 bytes
.../sparse-features/joshua-packed.config | 12 -
.../grammar/sparse-features/joshua.config | 12 -
.../grammar/sparse-features/output.gold | 1 -
.../grammar/sparse-features/test-packed.sh | 32 -
.../resources/grammar/sparse-features/test.sh | 32 -
src/test/resources/joshua/README.broken | 1 -
src/test/resources/lattice-short/README | 3 -
src/test/resources/lattice-short/glue-grammar | 3 -
src/test/resources/lattice-short/grammar.test | 3 -
src/test/resources/lattice-short/input | 5 -
src/test/resources/lattice-short/joshua.config | 39 -
.../resources/lattice-short/output.expected | 18 -
src/test/resources/lattice-short/test.lm | 113 -
src/test/resources/lattice-short/test.sh | 31 -
src/test/resources/lattice/.gitignore | 3 -
src/test/resources/lattice/README | 4 -
src/test/resources/lattice/glue-grammar | 3 -
src/test/resources/lattice/grammar.test | 204 -
src/test/resources/lattice/joshua.config | 47 -
src/test/resources/lattice/output.expected | 33 -
src/test/resources/lattice/test-lattice.pdf | Bin 10943 -> 0 bytes
src/test/resources/lattice/test.lm | 113 -
src/test/resources/lattice/test.plf | 4 -
src/test/resources/lattice/test.sh | 37 -
src/test/resources/lm/berkeley/lm | 16 -
src/test/resources/lm/berkeley/lm.berkeleylm | Bin 4294 -> 0 bytes
src/test/resources/lm/berkeley/lm.berkeleylm.gz | Bin 1786 -> 0 bytes
src/test/resources/lm/berkeley/lm.gz | Bin 162 -> 0 bytes
src/test/resources/lm/berkeley/output.gold | 4 -
src/test/resources/lm/berkeley/test.sh | 30 -
src/test/resources/packed-grammar/.gitignore | 8 -
src/test/resources/packed-grammar/README | 2 -
src/test/resources/packed-grammar/grammar.gz | Bin 576901 -> 0 bytes
src/test/resources/packed-grammar/input.bn | 100 -
src/test/resources/packed-grammar/joshua.config | 46 -
src/test/resources/packed-grammar/lm.gz | Bin 2466496 -> 0 bytes
src/test/resources/packed-grammar/output.gold | 100 -
.../resources/packed-grammar/reference.en.0 | 100 -
.../resources/packed-grammar/reference.en.1 | 100 -
.../resources/packed-grammar/reference.en.2 | 100 -
.../resources/packed-grammar/reference.en.3 | 100 -
.../resources/packed-grammar/test-multiple.sh | 31 -
src/test/resources/packed-grammar/test.sh | 38 -
src/test/resources/parser/grammar | 11 -
src/test/resources/parser/grammar.glue | 1 -
src/test/resources/parser/input | 4 -
src/test/resources/parser/output.gold | 4 -
src/test/resources/parser/parse.config | 18 -
src/test/resources/parser/test.sh | 29 -
src/test/resources/parser/weights | 4 -
src/test/resources/pipeline/.gitignore | 2 -
src/test/resources/pipeline/Makefile | 10 -
src/test/resources/pipeline/final-bleu.gold | 1 -
src/test/resources/pipeline/input/devtest.en.0 | 100 -
src/test/resources/pipeline/input/devtest.en.1 | 100 -
src/test/resources/pipeline/input/devtest.en.2 | 100 -
src/test/resources/pipeline/input/devtest.en.3 | 100 -
src/test/resources/pipeline/input/devtest.ur | 100 -
src/test/resources/pipeline/input/train.en | 1000 -
src/test/resources/pipeline/input/train.ur | 1000 -
src/test/resources/pipeline/input/tune.en.0 | 100 -
src/test/resources/pipeline/input/tune.en.1 | 100 -
src/test/resources/pipeline/input/tune.en.2 | 100 -
src/test/resources/pipeline/input/tune.en.3 | 100 -
src/test/resources/pipeline/input/tune.ur | 100 -
src/test/resources/pipeline/test-ghkm.sh | 43 -
src/test/resources/pipeline/test.sh | 39 -
.../resources/prune-equivalent-translations.py | 47 -
src/test/resources/run-all-tests.sh | 55 -
src/test/resources/scripts/.gitignore | 1 -
src/test/resources/scripts/merge_lms_test.py | 53 -
.../resources/scripts/normalization/.gitignore | 2 -
.../scripts/normalization/data/train.en | 21 -
.../scripts/normalization/data/train.en.norm | 21 -
.../resources/scripts/normalization/test.sh | 29 -
src/test/resources/scripts/run_bundler_test.py | 378 -
.../scripts/support/moses_grammar/input | 10 -
.../support/moses_grammar/output.expected | 10 -
.../scripts/support/moses_grammar/test.sh | 30 -
src/test/resources/server/http/expected | 15 -
src/test/resources/server/http/test.sh | 36 -
src/test/resources/server/tcp-text/expected | 9 -
src/test/resources/server/tcp-text/test.sh | 47 -
src/test/resources/testng.xml | 30 -
src/test/resources/thrax/.gitignore | 5 -
.../resources/thrax/extraction/input/thrax.conf | 71 -
.../resources/thrax/extraction/input/train.a | 100 -
.../resources/thrax/extraction/input/train.en | 100 -
.../resources/thrax/extraction/input/train.ps | 100 -
src/test/resources/thrax/extraction/test.sh | 36 -
.../resources/thrax/filtering/dev.hi-en.hi.1 | 1 -
src/test/resources/thrax/filtering/exact.gold | 993 -
.../resources/thrax/filtering/exact.log.gold | 17 -
src/test/resources/thrax/filtering/fast.gold | 1087 -
.../resources/thrax/filtering/fast.log.gold | 17 -
src/test/resources/thrax/filtering/grammar.de | 4 -
.../thrax/filtering/grammar.filtered.gz | Bin 134958 -> 0 bytes
src/test/resources/thrax/filtering/input.de | 3 -
.../resources/thrax/filtering/loose.log.gold | 16 -
.../resources/thrax/filtering/test-exact.sh | 34 -
src/test/resources/thrax/filtering/test-fast.sh | 34 -
.../resources/thrax/filtering/test-loose.sh | 34 -
1320 files changed, 157318 insertions(+), 157172 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/.gitignore
----------------------------------------------------------------------
diff --git a/joshua-core/.gitignore b/joshua-core/.gitignore
new file mode 100644
index 0000000..b83d222
--- /dev/null
+++ b/joshua-core/.gitignore
@@ -0,0 +1 @@
+/target/
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/pom.xml
----------------------------------------------------------------------
diff --git a/joshua-core/pom.xml b/joshua-core/pom.xml
new file mode 100644
index 0000000..fa9b721
--- /dev/null
+++ b/joshua-core/pom.xml
@@ -0,0 +1,145 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache</groupId>
+ <artifactId>apache</artifactId>
+ <version>10</version>
+ </parent>
+ <groupId>org.apache.joshua</groupId>
+ <artifactId>joshua-parent</artifactId>
+ <packaging>pom</packaging>
+ <version>6.0.6-SNAPSHOT</version>
+ <name>Apache Joshua Machine Translation Toolkit - Parent</name>
+ <description>Joshua is an open-source statistical machine
+ translation decoder for phrase-based, hierarchical,
+ and syntax-based machine translation, written in Java.
+ </description>
+ <url>http://joshua.incubator.apache.org</url>
+ <inceptionYear>2016</inceptionYear>
+
+ <properties>
+ <slf4j.version>1.7.21</slf4j.version>
+ </properties>
+ <licenses>
+ <license>
+ <name>The Apache Software License, Version 2.0</name>
+ <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
+ </license>
+ </licenses>
+
+ <organization>
+ <name>The Apache Software Foundation</name>
+ <url>http://www.apache.org/</url>
+ </organization>
+
+ <developers>
+ <developer>
+ <id>lewismc</id>
+ <name>Lewis John McGibbney</name>
+ <email>lewismc [at] apache [dot] org</email>
+ <roles>
+ <role>Committer</role>
+ <role>PMC Member</role>
+ </roles>
+ </developer>
+ <developer>
+ <id>mjpost</id>
+ <name>Matt Post</name>
+ <email>post [at] cs [dot] jhu [dot] edu</email>
+ <roles>
+ <role>Committer</role>
+ <role>PMC Member</role>
+ </roles>
+ </developer>
+ </developers>
+
+ <mailingLists>
+ <mailingList>
+ <name>Dev Mailing List</name>
+ <post>dev[at]joshua[dot]incubator[dot]apache[dot]org</post>
+ <subscribe>dev-subscribe[at]joshua[dot]incubator[dot]apache[dot]org</subscribe>
+ <unsubscribe>dev-unsubscribe[at]joshua[dot]incubator[dot]apache[dot]org</unsubscribe>
+ <archive>http://mail-archives.apache.org/mod_mbox/incubator-joshua-dev/</archive>
+ </mailingList>
+
+ <mailingList>
+ <name>User Mailing List</name>
+ <post>user[at]joshua[dot]incubator[dot]apache[dot]org</post>
+ <subscribe>user-subscribe[at]joshua[dot]incubator[dot]apache[dot]org</subscribe>
+ <unsubscribe>user-unsubscribe[at]joshua[dot]incubator[dot]apache[dot]org</unsubscribe>
+ <archive>http://mail-archives.apache.org/mod_mbox/incubator-joshua-user/</archive>
+ </mailingList>
+
+ <mailingList>
+ <name>Commits Mailing List</name>
+ <post>commits[at]joshua[dot]incubator[dot]apache[dot]org</post>
+ <subscribe>commits-subscribe[at]joshua[dot]incubator[dot]apache[dot]org</subscribe>
+ <unsubscribe>commits-unsubscribe[at]joshua[dot]incubator[dot]apache[dot]org</unsubscribe>
+ <archive>http://mail-archives.apache.org/mod_mbox/incubator-joshua-commits/</archive>
+ </mailingList>
+ </mailingLists>
+
+ <scm>
+ <connection>scm:git:http://git-wip-us.apache.org/repos/asf/incubator-joshua.git</connection>
+ <developerConnection>scm:git:http://git-wip-us.apache.org/repos/asf/incubator-joshua.git</developerConnection>
+ <url>https://git-wip-us.apache.org/repos/asf/incubator-joshua.git</url>
+ <tag>HEAD</tag>
+ </scm>
+ <issueManagement>
+ <system>JIRA</system>
+ <url>https://issues.apache.org/jira/browse/JOSHUA</url>
+ </issueManagement>
+ <ciManagement>
+ <system>Jenkins</system>
+ <url>https://builds.apache.org/job/joshua_master/</url>
+ </ciManagement>
+
+ <modules>
+ <module>joshua-core</module>
+ <module>joshua-servlet</module>
+ </modules>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.12</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
+ <build>
+ <defaultGoal>install</defaultGoal>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>1.8</source>
+ <target>1.8</target>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ </build>
+
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/resources/berkeley_lm/lm
----------------------------------------------------------------------
diff --git a/joshua-core/resources/berkeley_lm/lm b/joshua-core/resources/berkeley_lm/lm
new file mode 100644
index 0000000..05b4e6b
--- /dev/null
+++ b/joshua-core/resources/berkeley_lm/lm
@@ -0,0 +1,16 @@
+
+\data\
+ngram 1=5
+ngram 2=3
+
+\1-grams:
+-99.000000 <unk>
+-99.000000 <s> -1.752754
+-2.034158 the -0.800943
+-5.318589 chat-rooms -0.151088
+-1.495702 </s>
+
+\2-grams:
+-1.773970 <s> the
+-4.878868 the chat-rooms
+-0.499794 chat-rooms </s>
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/resources/berkeley_lm/lm.berkeleylm
----------------------------------------------------------------------
diff --git a/joshua-core/resources/berkeley_lm/lm.berkeleylm b/joshua-core/resources/berkeley_lm/lm.berkeleylm
new file mode 100644
index 0000000..c048464
Binary files /dev/null and b/joshua-core/resources/berkeley_lm/lm.berkeleylm differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/resources/berkeley_lm/lm.berkeleylm.gz
----------------------------------------------------------------------
diff --git a/joshua-core/resources/berkeley_lm/lm.berkeleylm.gz b/joshua-core/resources/berkeley_lm/lm.berkeleylm.gz
new file mode 100644
index 0000000..f9f8d16
Binary files /dev/null and b/joshua-core/resources/berkeley_lm/lm.berkeleylm.gz differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/resources/berkeley_lm/lm.gz
----------------------------------------------------------------------
diff --git a/joshua-core/resources/berkeley_lm/lm.gz b/joshua-core/resources/berkeley_lm/lm.gz
new file mode 100644
index 0000000..ae47266
Binary files /dev/null and b/joshua-core/resources/berkeley_lm/lm.gz differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/resources/grammar.glue
----------------------------------------------------------------------
diff --git a/joshua-core/resources/grammar.glue b/joshua-core/resources/grammar.glue
new file mode 100644
index 0000000..69e1520
--- /dev/null
+++ b/joshua-core/resources/grammar.glue
@@ -0,0 +1,4 @@
+[GOAL] ||| <s> ||| <s> ||| 0
+[GOAL] ||| [GOAL,1] [X,2] ||| [GOAL,1] [X,2] ||| -1
+[GOAL] ||| [GOAL,1] </s> ||| [GOAL,1] </s> ||| 0
+[GOAL] ||| <s> [X,1] </s> ||| <s> [X,1] </s> ||| 0
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/resources/kbest_extraction/glue-grammar
----------------------------------------------------------------------
diff --git a/joshua-core/resources/kbest_extraction/glue-grammar b/joshua-core/resources/kbest_extraction/glue-grammar
new file mode 100644
index 0000000..6a1162f
--- /dev/null
+++ b/joshua-core/resources/kbest_extraction/glue-grammar
@@ -0,0 +1,3 @@
+[GOAL] ||| <s> ||| <s> ||| 0
+[GOAL] ||| [GOAL,1] [X,2] ||| [GOAL,1] [X,2] ||| -1
+[GOAL] ||| [GOAL,1] </s> ||| [GOAL,1] </s> ||| 0
[32/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/lattice/Node.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/lattice/Node.java b/joshua-core/src/main/java/org/apache/joshua/lattice/Node.java
new file mode 100644
index 0000000..ecff22e
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/lattice/Node.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.lattice;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * A node in a directed graph.
+ *
+ * @author Lane Schwartz
+ * @since 2008-07-08
+ *
+ * @param <Label> Type of label associated with an arc.
+ */
+public class Node<Label> {
+
+ // ===============================================================
+ // Member variables
+ // ===============================================================
+
+ /**
+ * Numeric integer identifier of this node. Package-private scope so that Lattice can quickly
+ * access this variable.
+ */
+ private Integer id;
+
+ /**
+ * Arcs which begin at this node. Package-private scope so that Lattice can quickly access this
+ * variable.
+ */
+ private List<Arc<Label>> outgoingArcs;
+
+
+ // ===============================================================
+ // Constructor(s)
+ // ===============================================================
+
+ /**
+ * Constructs a new node with the specified numeric identifier.
+ * @param id the new node identifier
+ */
+ public Node(int id) {
+ this.id = id;
+ this.outgoingArcs = new ArrayList<Arc<Label>>();
+ }
+
+
+ // ===========================================================
+ // Accessor methods (set/get)
+ // ===========================================================
+
+ /**
+ * Gets the numeric integer identifier of this node.
+ *
+ * @return Numeric integer identifier of this node.
+ */
+ public int getNumber() {
+ return id;
+
+ }
+
+ public int id() {
+ return id;
+ }
+
+ public void setID(int i) {
+ this.id = i;
+ }
+
+ /**
+ * Gets the arcs that begin at this node.
+ *
+ * @return The arcs that begin at this node.
+ */
+ public List<Arc<Label>> getOutgoingArcs() {
+ return outgoingArcs;
+ }
+
+ public void setOutgoingArcs(List<Arc<Label>> arcs) {
+ outgoingArcs = arcs;
+ }
+
+ /**
+ * Gets an iterable object capable of iterating over all nodes directly reachable from this node.
+ * This will be all nodes which are the target of an outgoing arc from this node.
+ *
+ * @return An iterable object capable of iterating over all nodes directly reachable from this
+ * node.
+ */
+ public Iterable<Node<Label>> reachableNodes() {
+ final Iterator<Arc<Label>> arcIterator = outgoingArcs.iterator();
+
+ return new Iterable<Node<Label>>() {
+ public Iterator<Node<Label>> iterator() {
+ return new Iterator<Node<Label>>() {
+
+ public boolean hasNext() {
+ return arcIterator.hasNext();
+ }
+
+ public Node<Label> next() {
+ return arcIterator.next().getHead();
+ }
+
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ };
+ }
+ };
+ }
+
+
+ /**
+ * Adds a new outgoing arc to this node that points to the specified destination. The new arc will
+ * have the specified weight and specified label.
+ *
+ * @param destination Destination node of the new outgoing arc.
+ * @param weight Weight of the new outgoing arc.
+ * @param label Label of the new outgoing arc.
+ */
+ public void addArc(Node<Label> destination, float weight, Label label) {
+ outgoingArcs.add(new Arc<Label>(this, destination, weight, label));
+ }
+
+
+ /**
+ * Gets the number of outgoing arcs that begin at this node.
+ *
+ * @return The number of outgoing arcs that begin at this node.
+ */
+ public int size() {
+ return outgoingArcs.size();
+ }
+
+ @Override
+ public String toString() {
+ return "Node-" + id;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/lattice/NodeIdentifierComparator.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/lattice/NodeIdentifierComparator.java b/joshua-core/src/main/java/org/apache/joshua/lattice/NodeIdentifierComparator.java
new file mode 100644
index 0000000..aca5526
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/lattice/NodeIdentifierComparator.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.lattice;
+
+import java.io.Serializable;
+import java.util.Comparator;
+
+/**
+ * Compares nodes based only on the natural order of their integer identifiers.
+ *
+ * @author Lane Schwartz
+ */
+public class NodeIdentifierComparator implements Comparator<Node<?>>, Serializable {
+
+ private static final long serialVersionUID = 1L;
+
+ /* See Javadoc for java.util.Comparator#compare */
+ public int compare(Node<?> o1, Node<?> o2) {
+ if (o1.id() < o2.id())
+ return -1;
+ else if (o1.id() == o2.id())
+ return 0;
+ return 1;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/lattice/package-info.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/lattice/package-info.java b/joshua-core/src/main/java/org/apache/joshua/lattice/package-info.java
new file mode 100644
index 0000000..6b44542
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/lattice/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * Provides implementations of lattice and related data structures.
+ */
+package org.apache.joshua.lattice;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/metrics/BLEU.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/metrics/BLEU.java b/joshua-core/src/main/java/org/apache/joshua/metrics/BLEU.java
new file mode 100644
index 0000000..f1b50e7
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/metrics/BLEU.java
@@ -0,0 +1,575 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.metrics;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+public class BLEU extends EvaluationMetric {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BLEU.class);
+
+ // The maximum n-gram we care about
+ private int maxGramLength;
+ private EffectiveLengthMethod effLengthMethod;
+ // 1: closest, 2: shortest, 3: average
+ // protected HashMap[][] maxNgramCounts;
+
+ protected HashMap<String, Integer>[] maxNgramCounts;
+ protected int[][] refWordCount;
+ protected double[] weights;
+
+ public BLEU() {
+ this(4, "closest");
+ }
+
+ public BLEU(String[] BLEU_options) {
+ this(Integer.parseInt(BLEU_options[0]), BLEU_options[1]);
+ }
+
+ public BLEU(int mxGrmLn, String methodStr) {
+ if (mxGrmLn >= 1) {
+ setMaxGramLength(mxGrmLn);
+ } else {
+ LOG.error("Maximum gram length must be positive");
+ throw new RuntimeException("Maximum gram length must be positive");
+ }
+
+ if (methodStr.equals("closest")) {
+ setEffLengthMethod(EffectiveLengthMethod.CLOSEST);
+ } else if (methodStr.equals("shortest")) {
+ setEffLengthMethod(EffectiveLengthMethod.SHORTEST);
+ // } else if (methodStr.equals("average")) {
+ // effLengthMethod = EffectiveLengthMethod.AVERAGE;
+ } else {
+ LOG.error("Unknown effective length method string {}", methodStr);
+ // System.out.println("Should be one of closest, shortest, or average.");
+ LOG.error("Should be one of closest or shortest.");
+ throw new RuntimeException("Should be one of closest or shortest.");
+ }
+
+ initialize();
+ }
+
+ protected void initialize() {
+ metricName = "BLEU";
+ toBeMinimized = false;
+ suffStatsCount = 2 * getMaxGramLength() + 2;
+ // 2 per gram length for its precision, and 2 for length info
+ set_weightsArray();
+ set_maxNgramCounts();
+ }
+
+ @Override
+ public double bestPossibleScore() {
+ return 1.0;
+ }
+
+ @Override
+ public double worstPossibleScore() {
+ return 0.0;
+ }
+
+ /**
+ * Sets the BLEU weights for each n-gram level to uniform.
+ */
+ protected void set_weightsArray() {
+ weights = new double[1 + getMaxGramLength()];
+ for (int n = 1; n <= getMaxGramLength(); ++n) {
+ weights[n] = 1.0 / getMaxGramLength();
+ }
+ }
+
+ /**
+ * Computes the maximum ngram counts for each sentence (storing them in
+ * <code>maxNgramCounts</code>), which are used for clipping n-gram counts.
+ */
+ protected void set_maxNgramCounts() {
+ @SuppressWarnings("unchecked")
+ HashMap<String, Integer>[] temp_HMA = new HashMap[numSentences];
+ maxNgramCounts = temp_HMA;
+
+ String gram = "";
+ int oldCount = 0, nextCount = 0;
+
+ for (int i = 0; i < numSentences; ++i) {
+ maxNgramCounts[i] = getNgramCountsAll(refSentences[i][0]);
+ // initialize to ngramCounts[n] of the first reference translation...
+
+ // ...and update as necessary from the other reference translations
+ for (int r = 1; r < refsPerSen; ++r) {
+ HashMap<String, Integer> nextNgramCounts = getNgramCountsAll(refSentences[i][r]);
+ for (Map.Entry<String, Integer> entry : nextNgramCounts.entrySet()) {
+ gram = entry.getKey();
+ nextCount = entry.getValue();
+
+ if (maxNgramCounts[i].containsKey(gram)) { // update if necessary
+ oldCount = maxNgramCounts[i].get(gram);
+ if (nextCount > oldCount) {
+ maxNgramCounts[i].put(gram, nextCount);
+ }
+ } else { // add it
+ maxNgramCounts[i].put(gram, nextCount);
+ }
+
+ }
+
+ } // for (r)
+
+ } // for (i)
+
+ // For efficiency, calculate the reference lenghts, which will be used in effLength...
+
+ refWordCount = new int[numSentences][refsPerSen];
+ for (int i = 0; i < numSentences; ++i) {
+ for (int r = 0; r < refsPerSen; ++r) {
+ refWordCount[i][r] = wordCount(refSentences[i][r]);
+ }
+ }
+ }
+
+ /**
+ * Computes the BLEU sufficient statistics on a hypothesis.
+ * @param cand_str todo
+ * @param i todo
+ * @return int[] representing statistics on a hypothesis
+ */
+ public int[] suffStats(String cand_str, int i) {
+ int[] stats = new int[suffStatsCount];
+
+ // int wordCount = words.length;
+ // for (int j = 0; j < wordCount; ++j) { words[j] = words[j].intern(); }
+
+ if (!cand_str.equals("")) {
+ String[] words = cand_str.split("\\s+");
+ set_prec_suffStats(stats, words, i);
+ stats[suffStatsCount - 2] = words.length;
+ stats[suffStatsCount - 1] = effLength(words.length, i);
+ } else {
+ String[] words = new String[0];
+ set_prec_suffStats(stats, words, i);
+ stats[suffStatsCount - 2] = 0;
+ stats[suffStatsCount - 1] = effLength(0, i);
+ }
+
+ return stats;
+ }
+
+ /**
+ * Computes the precision sufficient statistics, clipping counts.
+ *
+ * @param stats int[] representing statistics on a hypothesis.
+ * @param words String[] of input terms
+ * @param i todo
+ */
+ public void set_prec_suffStats(int[] stats, String[] words, int i) {
+ HashMap<String, Integer>[] candCountsArray = getNgramCountsArray(words);
+
+ for (int n = 1; n <= getMaxGramLength(); ++n) {
+
+ int correctGramCount = 0;
+ String gram = "";
+ int candGramCount = 0, maxRefGramCount = 0, clippedCount = 0;
+
+ Iterator<String> it = (candCountsArray[n].keySet()).iterator();
+
+ while (it.hasNext()) {
+ // for each n-gram type in the candidate
+ gram = it.next();
+ candGramCount = candCountsArray[n].get(gram);
+ // if (maxNgramCounts[i][n].containsKey(gram)) {
+ // maxRefGramCount = maxNgramCounts[i][n].get(gram);
+ if (maxNgramCounts[i].containsKey(gram)) {
+ maxRefGramCount = maxNgramCounts[i].get(gram);
+ } else {
+ maxRefGramCount = 0;
+ }
+
+ clippedCount = Math.min(candGramCount, maxRefGramCount);
+ correctGramCount += clippedCount;
+ }
+
+ stats[2 * (n - 1)] = correctGramCount;
+ stats[2 * (n - 1) + 1] = Math.max(words.length - (n - 1), 0); // total gram count
+
+ } // for (n)
+ }
+
+ public int effLength(int candLength, int i) {
+ if (getEffLengthMethod() == EffectiveLengthMethod.CLOSEST) { // closest
+
+ int closestRefLength = refWordCount[i][0];
+ int minDiff = Math.abs(candLength - closestRefLength);
+
+ for (int r = 1; r < refsPerSen; ++r) {
+ int nextRefLength = refWordCount[i][r];
+ int nextDiff = Math.abs(candLength - nextRefLength);
+
+ if (nextDiff < minDiff) {
+ closestRefLength = nextRefLength;
+ minDiff = nextDiff;
+ } else if (nextDiff == minDiff && nextRefLength < closestRefLength) {
+ closestRefLength = nextRefLength;
+ minDiff = nextDiff;
+ }
+ }
+
+ return closestRefLength;
+
+ } else if (getEffLengthMethod() == EffectiveLengthMethod.SHORTEST) { // shortest
+
+ int shortestRefLength = refWordCount[i][0];
+
+ for (int r = 1; r < refsPerSen; ++r) {
+ int nextRefLength = refWordCount[i][r];
+ if (nextRefLength < shortestRefLength) {
+ shortestRefLength = nextRefLength;
+ }
+ }
+
+ return shortestRefLength;
+
+ }
+ /*
+ * // commented out because it needs sufficient statistics to be doubles else { // average
+ *
+ * int totalRefLength = refWordCount[i][0];
+ *
+ * for (int r = 1; r < refsPerSen; ++r) { totalRefLength += refWordCount[i][r]; }
+ *
+ * return totalRefLength/(double)refsPerSen;
+ *
+ * }
+ */
+ return candLength; // should never get here anyway
+
+ }
+
+ public double score(int[] stats) {
+ if (stats.length != suffStatsCount) {
+ String msg = "Mismatch between stats.length and suffStatsCount (" + stats.length + " vs. "
+ + suffStatsCount + ") in BLEU.score(int[])";
+ LOG.error(msg);
+ throw new RuntimeException(msg);
+ }
+
+ double BLEUsum = 0.0;
+ double smooth_addition = 1.0; // following bleu-1.04.pl
+ double c_len = stats[suffStatsCount - 2];
+ double r_len = stats[suffStatsCount - 1];
+
+ double correctGramCount, totalGramCount;
+
+ for (int n = 1; n <= getMaxGramLength(); ++n) {
+ correctGramCount = stats[2 * (n - 1)];
+ totalGramCount = stats[2 * (n - 1) + 1];
+
+ double prec_n;
+ if (totalGramCount > 0) {
+ prec_n = correctGramCount / totalGramCount;
+ } else {
+ prec_n = 1; // following bleu-1.04.pl ???????
+ }
+
+ if (prec_n == 0) {
+ smooth_addition *= 0.5;
+ prec_n = smooth_addition / (c_len - n + 1);
+ // isn't c_len-n+1 just totalGramCount ???????
+ }
+
+ BLEUsum += weights[n] * Math.log(prec_n);
+
+ }
+
+ double BP = 1.0;
+ if (c_len < r_len)
+ BP = Math.exp(1 - (r_len / c_len));
+ // if c_len > r_len, no penalty applies
+
+ return BP * Math.exp(BLEUsum);
+
+ }
+
+ public void printDetailedScore_fromStats(int[] stats, boolean oneLiner) {
+ double BLEUsum = 0.0;
+ double smooth_addition = 1.0; // following bleu-1.04.pl
+ double c_len = stats[suffStatsCount - 2];
+ double r_len = stats[suffStatsCount - 1];
+
+ double correctGramCount, totalGramCount;
+
+ if (oneLiner) {
+ System.out.print("Precisions: ");
+ }
+
+ for (int n = 1; n <= getMaxGramLength(); ++n) {
+ correctGramCount = stats[2 * (n - 1)];
+ totalGramCount = stats[2 * (n - 1) + 1];
+
+ double prec_n;
+ if (totalGramCount > 0) {
+ prec_n = correctGramCount / totalGramCount;
+ } else {
+ prec_n = 1; // following bleu-1.04.pl ???????
+ }
+
+ if (prec_n > 0) {
+ if (totalGramCount > 0) {
+ if (oneLiner) {
+ System.out.print(n + "=" + f4.format(prec_n) + ", ");
+ } else {
+ System.out.println("BLEU_precision(" + n + ") = " + (int) correctGramCount + " / "
+ + (int) totalGramCount + " = " + f4.format(prec_n));
+ }
+ } else {
+ if (oneLiner) {
+ System.out.print(n + "=N/A, ");
+ } else {
+ System.out
+ .println("BLEU_precision(" + n + ") = N/A (candidate has no " + n + "-grams)");
+ }
+ }
+ } else {
+ smooth_addition *= 0.5;
+ prec_n = smooth_addition / (c_len - n + 1);
+ // isn't c_len-n+1 just totalGramCount ???????
+
+ if (oneLiner) {
+ System.out.print(n + "~" + f4.format(prec_n) + ", ");
+ } else {
+ System.out.println("BLEU_precision(" + n + ") = " + (int) correctGramCount + " / "
+ + (int) totalGramCount + " ==smoothed==> " + f4.format(prec_n));
+ }
+ }
+
+ BLEUsum += weights[n] * Math.log(prec_n);
+
+ }
+
+ if (oneLiner) {
+ System.out.print("(overall=" + f4.format(Math.exp(BLEUsum)) + "), ");
+ } else {
+ System.out.println("BLEU_precision = " + f4.format(Math.exp(BLEUsum)));
+ System.out.println("");
+ }
+
+ double BP = 1.0;
+ if (c_len < r_len)
+ BP = Math.exp(1 - (r_len / c_len));
+ // if c_len > r_len, no penalty applies
+
+ if (oneLiner) {
+ System.out.print("BP=" + f4.format(BP) + ", ");
+ } else {
+ System.out.println("Length of candidate corpus = " + (int) c_len);
+ System.out.println("Effective length of reference corpus = " + (int) r_len);
+ System.out.println("BLEU_BP = " + f4.format(BP));
+ System.out.println("");
+ }
+
+ System.out.println(" => BLEU = " + f4.format(BP * Math.exp(BLEUsum)));
+ }
+
+ protected int wordCount(String cand_str) {
+ if (!cand_str.equals("")) {
+ return cand_str.split("\\s+").length;
+ } else {
+ return 0;
+ }
+ }
+
+ public HashMap<String, Integer>[] getNgramCountsArray(String cand_str) {
+ if (!cand_str.equals("")) {
+ return getNgramCountsArray(cand_str.split("\\s+"));
+ } else {
+ return getNgramCountsArray(new String[0]);
+ }
+ }
+
+ public HashMap<String, Integer>[] getNgramCountsArray(String[] words) {
+ @SuppressWarnings("unchecked")
+ HashMap<String, Integer>[] ngramCountsArray = new HashMap[1 + getMaxGramLength()];
+ ngramCountsArray[0] = null;
+ for (int n = 1; n <= getMaxGramLength(); ++n) {
+ ngramCountsArray[n] = new HashMap<String, Integer>();
+ }
+
+ int len = words.length;
+ String gram;
+ int st = 0;
+
+ for (; st <= len - getMaxGramLength(); ++st) {
+
+ gram = words[st];
+ if (ngramCountsArray[1].containsKey(gram)) {
+ int oldCount = ngramCountsArray[1].get(gram);
+ ngramCountsArray[1].put(gram, oldCount + 1);
+ } else {
+ ngramCountsArray[1].put(gram, 1);
+ }
+
+ for (int n = 2; n <= getMaxGramLength(); ++n) {
+ gram = gram + " " + words[st + n - 1];
+ if (ngramCountsArray[n].containsKey(gram)) {
+ int oldCount = ngramCountsArray[n].get(gram);
+ ngramCountsArray[n].put(gram, oldCount + 1);
+ } else {
+ ngramCountsArray[n].put(gram, 1);
+ }
+ } // for (n)
+
+ } // for (st)
+
+ // now st is either len-maxGramLength+1 or zero (if above loop never entered, which
+ // happens with sentences that have fewer than maxGramLength words)
+
+ for (; st < len; ++st) {
+
+ gram = words[st];
+ if (ngramCountsArray[1].containsKey(gram)) {
+ int oldCount = ngramCountsArray[1].get(gram);
+ ngramCountsArray[1].put(gram, oldCount + 1);
+ } else {
+ ngramCountsArray[1].put(gram, 1);
+ }
+
+ int n = 2;
+ for (int fin = st + 1; fin < len; ++fin) {
+ gram = gram + " " + words[st + n - 1];
+
+ if (ngramCountsArray[n].containsKey(gram)) {
+ int oldCount = ngramCountsArray[n].get(gram);
+ ngramCountsArray[n].put(gram, oldCount + 1);
+ } else {
+ ngramCountsArray[n].put(gram, 1);
+ }
+ ++n;
+ } // for (fin)
+
+ } // for (st)
+
+ return ngramCountsArray;
+
+ }
+
+ public HashMap<String, Integer> getNgramCountsAll(String cand_str) {
+ if (!cand_str.equals("")) {
+ return getNgramCountsAll(cand_str.split("\\s+"));
+ } else {
+ return getNgramCountsAll(new String[0]);
+ }
+ }
+
+ public HashMap<String, Integer> getNgramCountsAll(String[] words) {
+ HashMap<String, Integer> ngramCountsAll = new HashMap<String, Integer>();
+
+ int len = words.length;
+ String gram;
+ int st = 0;
+
+ for (; st <= len - getMaxGramLength(); ++st) {
+
+ gram = words[st];
+ if (ngramCountsAll.containsKey(gram)) {
+ int oldCount = ngramCountsAll.get(gram);
+ ngramCountsAll.put(gram, oldCount + 1);
+ } else {
+ ngramCountsAll.put(gram, 1);
+ }
+
+ for (int n = 2; n <= getMaxGramLength(); ++n) {
+ gram = gram + " " + words[st + n - 1];
+ if (ngramCountsAll.containsKey(gram)) {
+ int oldCount = ngramCountsAll.get(gram);
+ ngramCountsAll.put(gram, oldCount + 1);
+ } else {
+ ngramCountsAll.put(gram, 1);
+ }
+ } // for (n)
+
+ } // for (st)
+
+ // now st is either len-maxGramLength+1 or zero (if above loop never entered, which
+ // happens with sentences that have fewer than maxGramLength words)
+
+ for (; st < len; ++st) {
+
+ gram = words[st];
+ if (ngramCountsAll.containsKey(gram)) {
+ int oldCount = ngramCountsAll.get(gram);
+ ngramCountsAll.put(gram, oldCount + 1);
+ } else {
+ ngramCountsAll.put(gram, 1);
+ }
+
+ int n = 2;
+ for (int fin = st + 1; fin < len; ++fin) {
+ gram = gram + " " + words[st + n - 1];
+
+ if (ngramCountsAll.containsKey(gram)) {
+ int oldCount = ngramCountsAll.get(gram);
+ ngramCountsAll.put(gram, oldCount + 1);
+ } else {
+ ngramCountsAll.put(gram, 1);
+ }
+ ++n;
+ } // for (fin)
+
+ } // for (st)
+
+ return ngramCountsAll;
+
+ }
+
+ /**
+ * @return the maxGramLength
+ */
+ public int getMaxGramLength() {
+ return maxGramLength;
+ }
+
+ /**
+ * @param maxGramLength the maxGramLength to set
+ */
+ public void setMaxGramLength(int maxGramLength) {
+ this.maxGramLength = maxGramLength;
+ }
+
+ /**
+ * @return the effLengthMethod
+ */
+ public EffectiveLengthMethod getEffLengthMethod() {
+ return effLengthMethod;
+ }
+
+ /**
+ * @param effLengthMethod the effLengthMethod to set
+ */
+ public void setEffLengthMethod(EffectiveLengthMethod effLengthMethod) {
+ this.effLengthMethod = effLengthMethod;
+ }
+
+ public enum EffectiveLengthMethod {
+ CLOSEST, SHORTEST, AVERAGE
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/metrics/BLEU_SBP.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/metrics/BLEU_SBP.java b/joshua-core/src/main/java/org/apache/joshua/metrics/BLEU_SBP.java
new file mode 100644
index 0000000..6b97ff4
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/metrics/BLEU_SBP.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.metrics;
+
+public class BLEU_SBP extends BLEU {
+ // constructors
+ public BLEU_SBP() {
+ super();
+ }
+
+ public BLEU_SBP(String[] BLEU_SBP_options) {
+ super(BLEU_SBP_options);
+ }
+
+ public BLEU_SBP(int mxGrmLn, String methodStr) {
+ super(mxGrmLn, methodStr);
+ }
+
+
+
+ public int[] suffStats(String cand_str, int i) {
+ int[] stats = new int[suffStatsCount];
+ stats[0] = 1;
+
+ String[] words = cand_str.split("\\s+");
+
+ // int wordCount = words.length;
+ // for (int j = 0; j < wordCount; ++j) { words[j] = words[j].intern(); }
+
+ set_prec_suffStats(stats, words, i);
+
+ // the only place where BLEU_SBP differs from BLEU /* ~~~ */
+ /* ~~~ */
+ // stats[maxGramLength+1] = words.length;
+ // stats[maxGramLength+2] = effLength(words.length,i);
+ /* ~~~ */
+
+ /* ~~~ */
+ int effectiveLength = effLength(words.length, i);
+ stats[getMaxGramLength() + 1] = Math.min(words.length, effectiveLength);
+ stats[getMaxGramLength() + 2] = effectiveLength;
+ /* ~~~ */
+
+ return stats;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/metrics/EvaluationMetric.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/metrics/EvaluationMetric.java b/joshua-core/src/main/java/org/apache/joshua/metrics/EvaluationMetric.java
new file mode 100644
index 0000000..9a8786c
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/metrics/EvaluationMetric.java
@@ -0,0 +1,405 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.metrics;
+
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.text.DecimalFormat;
+import java.util.Arrays;
+import java.util.TreeMap;
+
+/***
+ * @author Omar Zaidan
+ */
+public abstract class EvaluationMetric {
+ /* static data members */
+ private static TreeMap<String, Integer> metricOptionCount; // maps metric names -> number of
+ // options for that metric
+ protected static int numSentences; // number of sentences in the MERT set
+ protected static int numDocuments; // number of documents in the MERT set
+ protected static int refsPerSen;
+ protected static String[][] refSentences;
+ protected final static DecimalFormat f0 = new DecimalFormat("###0");
+ protected final static DecimalFormat f4 = new DecimalFormat("###0.0000");
+ protected static String tmpDirPrefix;
+
+ /* non-static data members */
+ protected int suffStatsCount; // number of sufficient statistics
+ protected String metricName; // number of metric
+ protected boolean toBeMinimized;
+
+ // is this a metric that should be minimized?
+ // e.g. toBeMinimized = true for 01LOSS, WER, TER
+ // toBeMinimized = false for BLEU
+
+ /* static (=> also non-abstract) methods */
+ public static void set_knownMetrics() {
+ metricOptionCount = new TreeMap<String, Integer>();
+
+ metricOptionCount.put("BLEU", 2);
+ // the "BLEU" metric expects an options array of length 2
+ metricOptionCount.put("BLEU_SBP", 2);
+ // the "BLEU_SBP" metric expects an options array of length 2
+ metricOptionCount.put("01LOSS", 0);
+ // the "01LOSS" metric expects an options array of length 0
+ metricOptionCount.put("TER", 6);
+ // the "TER" metric expects an options array of length 5
+ // metricOptionCount.put("METEOR",4);
+ // the "METEOR" metric expects an options array of length 4
+ // metricOptionCount.put("RYPT",5);
+ // the "RYPT" metric expects an options array of length 5
+ metricOptionCount.put("TER-BLEU", 8);
+ // the "TER-BLEU" metric expects an options array of length 7
+ // metricOptionCount.put("WER",0);
+ // the "WER" metric expects an options array of length 0
+ metricOptionCount.put("MC_BLEU", 4);
+ metricOptionCount.put("PRECIS", 6);
+ metricOptionCount.put("SRC_BLEU", 4);
+ metricOptionCount.put("PRECIS-SRC_BLEU", 6);
+ metricOptionCount.put("GL_BLEU", 3);
+ metricOptionCount.put("SARI", 2); // options: max-ngram source-path
+ }
+
+ public static EvaluationMetric getMetric(String metricName, String[] metricOptions) {
+ EvaluationMetric retMetric = null;
+
+ if (metricName.equals("BLEU")) {
+ retMetric = new BLEU(metricOptions); // the "BLEU" metric corresponds to the BLEU class
+ } else if (metricName.equals("BLEU_SBP")) {
+ retMetric = new BLEU_SBP(metricOptions); // the "BLEU_SBP" metric corresponds to the BLEU_SBP
+ // class
+ } else if (metricName.equals("01LOSS")) {
+ retMetric = new ZeroOneLoss(metricOptions); // the "01LOSS" metric corresponds to the
+ // ZeroOneLoss class
+ } else if (metricName.equals("TER")) {
+ retMetric = new TER(metricOptions); // the "TER" metric corresponds to the TER class
+ // } else if (metricName.equals("METEOR")) {
+ // retMetric = new METEOR(metricOptions); // the "METEOR" metric corresponds to the METEOR
+ // class
+ // } else if (metricName.equals("RYPT")) {
+ // retMetric = new RYPT(metricOptions); // the "RYPT" metric corresponds to the RYPT class
+ } else if (metricName.equals("TER-BLEU")) {
+ retMetric = new TERMinusBLEU(metricOptions); // the "TER-BLEU" metric corresponds to the
+ // TERMinusBLEU class
+ // } else if (metricName.equals("WER")) {
+ // retMetric = new WordErrorRate(metricOptions); // the "WER" metric corresponds to the
+ // WordErrorRate class
+ } else if (metricName.equals("MC_BLEU")) {
+ retMetric = new MinimumChangeBLEU(metricOptions); // the "MC_BLEU" metric corresponds to the
+ // ParaphraseBLEU class
+ } else if (metricName.equals("PRECIS")) {
+ retMetric = new Precis(metricOptions);
+ } else if (metricName.equals("SRC_BLEU")) {
+ retMetric = new SourceBLEU(metricOptions);
+ } else if (metricName.equals("PRECIS-SRC_BLEU")) {
+ retMetric = new PrecisMinusSourceBLEU(metricOptions);
+ } else if (metricName.equals("GL_BLEU")) {
+ retMetric = new GradeLevelBLEU(metricOptions); // the "GL_BLEU" metric corresponds to the
+ // GradeLevelBLEU class
+ } else if (metricName.equals("SARI")) {
+ retMetric = new SARI(metricOptions);
+ }
+
+ return retMetric;
+ }
+
+ public static void set_numSentences(int x) {
+ numSentences = x;
+ }
+
+ public static void set_numDocuments(int x) {
+ numDocuments = x;
+ }
+
+ public static void set_refsPerSen(int x) {
+ refsPerSen = x;
+ }
+
+ public static void set_tmpDirPrefix(String S) {
+ tmpDirPrefix = S;
+ }
+
+ public static void set_refSentences(String[][] refs) {
+ refSentences = new String[numSentences][refsPerSen];
+ for (int i = 0; i < numSentences; ++i) {
+ for (int r = 0; r < refsPerSen; ++r) {
+ refSentences[i][r] = refs[i][r];
+ }
+ }
+ }
+
+ public static boolean knownMetricName(String name) {
+ return metricOptionCount.containsKey(name);
+ }
+
+ public static int metricOptionCount(String name) {
+ return metricOptionCount.get(name);
+ }
+
+ /* non-abstract, non-static methods */
+ public int get_suffStatsCount() {
+ return suffStatsCount;
+ }
+
+ public String get_metricName() {
+ return metricName;
+ }
+
+ public boolean getToBeMinimized() {
+ return toBeMinimized;
+ }
+
+ public boolean isBetter(double x, double y) {
+ // return true if x is better than y
+ if (toBeMinimized) {
+ return (x < y);
+ } else {
+ return (x > y);
+ }
+ }
+
+ public double score(String cand_str, int i) {
+ String[] SA = new String[1];
+ SA[0] = cand_str;
+ int[] IA = new int[1];
+ IA[0] = i;
+
+ int[][] SS = suffStats(SA, IA);
+
+ int[] stats = new int[suffStatsCount];
+ for (int s = 0; s < suffStatsCount; ++s) {
+ stats[s] = SS[0][s];
+ }
+
+ return score(stats);
+ }
+
+ public double score(String[] topCand_str) {
+ int[] stats = suffStats(topCand_str);
+ return score(stats);
+ }
+
+ public int[] suffStats(String[] topCand_str) {
+ int[] IA = new int[numSentences];
+ for (int i = 0; i < numSentences; ++i) {
+ IA[i] = i;
+ }
+
+ int[][] SS = suffStats(topCand_str, IA);
+
+ int[] totStats = new int[suffStatsCount];
+ for (int s = 0; s < suffStatsCount; ++s) {
+ totStats[s] = 0;
+ for (int i = 0; i < numSentences; ++i) {
+ totStats[s] += SS[i][s];
+ }
+ }
+
+ return totStats;
+ }
+
+ /**
+ * Calculates sufficient statistics on each sentence in the corpus, returning them as arrays.
+ *
+ * @param cand_strings todo
+ * @param cand_indices todo
+ * @return todo
+ */
+ public int[][] suffStats(String[] cand_strings, int[] cand_indices) {
+
+ int candCount = cand_strings.length;
+ if (cand_indices.length != candCount) {
+ System.out.println("Array lengths mismatch in suffStats(String[],int[]); returning null.");
+ return null;
+ }
+
+ int[][] stats = new int[candCount][suffStatsCount];
+
+ for (int d = 0; d < candCount; ++d) {
+ int[] currStats = suffStats(cand_strings[d], cand_indices[d]);
+
+ for (int s = 0; s < suffStatsCount; ++s) {
+ stats[d][s] = currStats[s];
+ }
+ } // for (d)
+
+ return stats;
+ }
+
+ public void createSuffStatsFile(String cand_strings_fileName, String cand_indices_fileName,
+ String outputFileName, int maxBatchSize) {
+ // similar to the above suffStats(String[], int[])
+
+ try {
+ FileInputStream inStream_cands = new FileInputStream(cand_strings_fileName);
+ BufferedReader inFile_cands =
+ new BufferedReader(new InputStreamReader(inStream_cands, "utf8"));
+
+ FileInputStream inStream_indices = new FileInputStream(cand_indices_fileName);
+ BufferedReader inFile_indices =
+ new BufferedReader(new InputStreamReader(inStream_indices, "utf8"));
+
+ PrintWriter outFile = new PrintWriter(outputFileName);
+
+ String[] cand_strings = new String[maxBatchSize];
+ int[] cand_indices = new int[maxBatchSize];
+
+ String line_cand = inFile_cands.readLine();
+ String line_index = inFile_indices.readLine();
+
+ while (line_cand != null) {
+ int size = 0;
+ while (line_cand != null) {
+ cand_strings[size] = line_cand;
+ cand_indices[size] = Integer.parseInt(line_index);
+ ++size; // now size is how many were read for this currnet batch
+ if (size == maxBatchSize) break;
+
+ line_cand = inFile_cands.readLine();
+ line_index = inFile_indices.readLine();
+ }
+
+ if (size < maxBatchSize) { // last batch, and smaller than maxBatchSize
+ String[] cand_strings_temp = new String[size];
+ int[] cand_indices_temp = new int[size];
+ for (int d = 0; d < size; ++d) {
+ cand_strings_temp[d] = cand_strings[d];
+ cand_indices_temp[d] = cand_indices[d];
+ }
+ cand_strings = cand_strings_temp;
+ cand_indices = cand_indices_temp;
+ }
+
+ int[][] SS = suffStats(cand_strings, cand_indices);
+ for (int d = 0; d < size; ++d) {
+ StringBuilder stats_str = new StringBuilder();
+
+ for (int s = 0; s < suffStatsCount - 1; ++s) {
+ stats_str.append(SS[d][s]).append(" ");
+ }
+ stats_str.append(SS[d][suffStatsCount - 1]);
+
+ outFile.println(stats_str);
+ }
+
+ line_cand = inFile_cands.readLine();
+ line_index = inFile_indices.readLine();
+ }
+
+ inFile_cands.close();
+ inFile_indices.close();
+ outFile.close();
+
+ } catch (IOException e) {
+ throw new RuntimeException("IOException in EvaluationMetric.createSuffStatsFile(...): "
+ + e.getMessage(), e);
+ }
+
+ }
+
+ public void printDetailedScore(String[] topCand_str, boolean oneLiner) {
+ int[] stats = suffStats(topCand_str);
+ printDetailedScore_fromStats(stats, oneLiner);
+ }
+
+ public double score(int[][] stats) {
+ // returns an average of document scores (aka the document-level score, as opposed to
+ // corpus-level score)
+ // stats[][] is indexed [doc][s]
+
+ double retVal = 0.0;
+ for (int doc = 0; doc < numDocuments; ++doc) {
+ retVal += score(stats[doc]);
+ }
+ return retVal / numDocuments;
+ }
+
+ public double score(int[][] stats, int firstRank, int lastRank) {
+ // returns an average of document scores, restricted to the documents
+ // ranked firstRank-lastRank, inclusive (ranks are 1-indexed, even though the docs are
+ // 0-indexed)
+
+ double[] scores = docScores(stats);
+
+ Arrays.sort(scores);
+ // sorts into ascending order
+
+ double retVal = 0.0;
+
+ if (toBeMinimized) {
+ // scores[0] is rank 1, scores[numDocuments-1] is rank numDocuments
+ // => scores[j] is rank j+1
+ // => rank r is scores[r-1]
+ for (int j = firstRank - 1; j < lastRank; ++j) {
+ retVal += scores[j];
+ }
+ } else {
+ // scores[numDocuments-1] is rank 1, scores[0] is rank numDocuments
+ // => scores[j] is rank numDocuments-j
+ // => rank r is scores[numDocuments-r]
+ for (int j = numDocuments - firstRank; j >= numDocuments - lastRank; --j) {
+ retVal += scores[j];
+ }
+ }
+
+ return retVal / (lastRank - firstRank + 1);
+
+ }
+
+ public double[] docScores(int[][] stats) {
+ // returns an array of document scores
+ // stats[][] is indexed [doc][s]
+
+ double[] scores = new double[numDocuments];
+ for (int doc = 0; doc < numDocuments; ++doc) {
+ scores[doc] = score(stats[doc]);
+ }
+ return scores;
+ }
+
+ public void printDetailedScore_fromStats(int[][] stats, String[] docNames) {
+ // prints individual document scores
+ // stats[][] is indexed [doc][s]
+
+ for (int doc = 0; doc < numDocuments; ++doc) {
+ if (docNames == null) {
+ System.out.print("Document #" + doc + ": ");
+ } else {
+ System.out.print(docNames[doc] + ": ");
+ }
+ printDetailedScore_fromStats(stats[doc], true);
+ }
+ }
+
+ /* abstract (=> also non-static) methods */
+ protected abstract void initialize();
+
+ public abstract double bestPossibleScore();
+
+ public abstract double worstPossibleScore();
+
+ public abstract int[] suffStats(String cand_str, int i);
+
+ public abstract double score(int[] stats);
+
+ public abstract void printDetailedScore_fromStats(int[] stats, boolean oneLiner);
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/metrics/GradeLevelBLEU.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/metrics/GradeLevelBLEU.java b/joshua-core/src/main/java/org/apache/joshua/metrics/GradeLevelBLEU.java
new file mode 100644
index 0000000..e143572
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/metrics/GradeLevelBLEU.java
@@ -0,0 +1,280 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.metrics;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.text.DecimalFormat;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public class GradeLevelBLEU extends BLEU {
+
+ private static final Logger LOG = LoggerFactory.getLogger(GradeLevelBLEU.class);
+
+ // syllable pattern matches /C*V+/
+ private static final Pattern syllable = Pattern.compile("([^aeiouy]*[aeiouy]+)");
+ private static final Pattern silentE = Pattern.compile("[^aeiou]e$");
+ private static final int SOURCE = 0, CANDIDATE = 1, REFERENCE = 2;
+ private int srcIndex = 1, sentCountIndex;
+ private SourceBLEU srcBLEU;
+ private double targetGL = 9.87; // tune.simp avg GL = 9.8704 (tune.en =
+ // 14.0785
+ private double alpha = 0.9;
+ private boolean useTarget = true;
+ private boolean useBLEUplus = true;
+
+ public GradeLevelBLEU() {
+ super();
+ }
+
+ // target == 0 : use the default target
+ // target > 0 : use that target
+ // target < 0 : use source GL for target
+ public GradeLevelBLEU(String[] options) {
+ super();
+ // there are 3 arguments: target GL, alpha, and source path
+ // the BLEU options are assumed to be "4 closest"
+ if (Double.parseDouble(options[0]) > 0)
+ targetGL = Double.parseDouble(options[0]);
+ else if (Double.parseDouble(options[0]) < 0) useTarget = false;
+ if (Double.parseDouble(options[1]) > 0) alpha = Double.parseDouble(options[1]);
+ try {
+ loadSources(options[2]);
+ } catch (IOException e) {
+ throw new RuntimeException("Error loading the source sentences from " + options[2], e);
+ }
+ if (useBLEUplus) srcBLEU = new SourceBLEU(4, "closest", srcIndex, true);
+ initialize();
+ }
+
+ // hacky way to add the source sentence as the last reference sentence (in
+ // accordance with SourceBLEU)
+ public void loadSources(String filepath) throws IOException {
+ String[][] newRefSentences = new String[numSentences][refsPerSen + 1];
+ BufferedReader br = new BufferedReader(new FileReader(filepath));
+ String line;
+ int i = 0;
+ while (i < numSentences && (line = br.readLine()) != null) {
+ for (int r = 0; r < refsPerSen; ++r) {
+ newRefSentences[i][r] = refSentences[i][r];
+ }
+ newRefSentences[i][refsPerSen] = line.trim();
+ i++;
+ }
+ br.close();
+ }
+
+ public void initialize() {
+ metricName = "GL_BLEU";
+ setEffLengthMethod(EffectiveLengthMethod.SHORTEST);
+ toBeMinimized = false;
+ suffStatsCount = 4 * getMaxGramLength() + 7;
+ sentCountIndex = 4 * getMaxGramLength();
+ set_weightsArray();
+ set_maxNgramCounts();
+ }
+
+ public int[] suffStats(String cand_str, int i) {
+ int[] stats = new int[suffStatsCount];
+
+ String[] candidate_tokens = null;
+
+ if (!cand_str.equals("")) {
+ candidate_tokens = cand_str.split("\\s+");
+ } else {
+ candidate_tokens = new String[0];
+ stats[tokenLength(CANDIDATE)] = 0;
+ stats[tokenLength(REFERENCE)] = effLength(0, i);
+ }
+ // set the BLEU stats
+ set_prec_suffStats(stats, candidate_tokens, i);
+
+ // set source BLEU stats
+ if (useBLEUplus) {
+ int[] src_prec_suffStats = srcBLEU.suffStats(cand_str, i);
+ for (int j = 0; j < src_prec_suffStats.length; j++) {
+ stats[2 * getMaxGramLength() + j] = src_prec_suffStats[j];
+ }
+ }
+
+ // now set the readability stats
+ String[] reference_tokens = refSentences[i][0].split("\\s+");
+ String[] source_tokens = refSentences[i][srcIndex].split("\\s+");
+
+ // set the number of sentences (necessary to calculate GL)
+ stats[sentCountIndex] = 1;
+ // token length
+ stats[tokenLength(CANDIDATE)] = candidate_tokens.length;
+ stats[tokenLength(REFERENCE)] = reference_tokens.length;
+ stats[tokenLength(SOURCE)] = source_tokens.length;
+
+ // syllable length
+ stats[syllableLength(CANDIDATE)] = countTotalSyllables(candidate_tokens);
+ stats[syllableLength(REFERENCE)] = countTotalSyllables(reference_tokens);
+ stats[syllableLength(SOURCE)] = countTotalSyllables(source_tokens);
+
+ return stats;
+ }
+
+ // create methods for accessing the indices to reduce possible human error
+ private int tokenLength(int whichSentence) {
+ return suffStatsCount - 3 + whichSentence;
+ }
+
+ private int syllableLength(int whichSentence) {
+ return suffStatsCount - 6 + whichSentence;
+ }
+
+ // count syllables in a "sentence" (ss.length >= 1)
+ public int countTotalSyllables(String[] ss) {
+ int count = 0;
+ for (String s : ss) {
+ int i = countSyllables(s);
+ count += i;
+ }
+ return count;
+ }
+
+ // count syllables in a "word"
+ // add a syllable for punctuation, etc., so it isn't free
+ public int countSyllables(String s) {
+ if (s.equals("-")) {
+ return 1;
+ }
+ // if the word is hyphenated, split at the hyphen before counting
+ // syllables
+ if (s.contains("-")) {
+ int count = 0;
+ String[] temp = s.split("-");
+ for (String t : temp)
+ count += countSyllables(t);
+ return count;
+ }
+
+ int count = 0;
+ Matcher m = syllable.matcher(s);
+ while (m.find())
+ count++;
+ // subtract 1 if the word ends in a silent e
+ m = silentE.matcher(s);
+ if (m.find()) count--;
+ if (count <= 0) count = 1;
+ return count;
+ }
+
+ public double score(int[] stats) {
+ if (stats.length != suffStatsCount) {
+ String msg = "Mismatch between stats.length and suffStatsCount (" + stats.length + " vs. "
+ + suffStatsCount + ") in BLEU.score(int[])";
+ LOG.error(msg);
+ throw new RuntimeException(msg);
+ }
+ double BLEUscore = super.score(stats);
+ double candGL =
+ gradeLevel(stats[tokenLength(CANDIDATE)], stats[syllableLength(CANDIDATE)],
+ stats[sentCountIndex]);
+ double readabilityPenalty = 1;
+
+ if (useTarget) {
+ readabilityPenalty = getReadabilityPenalty(candGL, targetGL);
+ } else {
+ double srcGL =
+ gradeLevel(stats[tokenLength(SOURCE)], stats[syllableLength(SOURCE)],
+ stats[sentCountIndex]);
+ readabilityPenalty = getReadabilityPenalty(candGL, srcGL);
+ }
+
+ if (useBLEUplus) {
+ int[] srcStats = new int[2 * getMaxGramLength()];
+ for (int i = 0; i < 2 * getMaxGramLength(); i++) {
+ srcStats[i] = stats[2 * getMaxGramLength() + i];
+ }
+ srcStats[2 * getMaxGramLength()] = stats[tokenLength(CANDIDATE)];
+ srcStats[2 * getMaxGramLength()] = stats[tokenLength(SOURCE)];
+ double srcBLEUscore = srcBLEU.score(stats);
+ BLEUscore = BLEU_plus(BLEUscore, srcBLEUscore);
+ }
+ return readabilityPenalty * BLEUscore;
+ }
+
+ // Flesch-Kincaid Grade Level
+ // (http://en.wikipedia.org/wiki/Flesch-Kincaid_readability_test)
+ public double gradeLevel(int numWords, int numSyllables, int numSentences) {
+ double d = 0.39 * numWords / numSentences + 11.8 * numSyllables / numWords - 15.19;
+ if (d < 0) d = 0;
+ return d;
+ }
+
+ // calculate BLEU+ (per submitted paper CCB reviewed)
+ private double BLEU_plus(double bleu_ref, double bleu_src) {
+ return alpha * bleu_ref - (1 - alpha) * bleu_src;
+ }
+
+ private double getReadabilityPenalty(double this_gl, double target_gl) {
+ if (this_gl < target_gl) return 1.0;
+ return 0.0;
+ }
+
+ public void printDetailedScore_fromStats(int[] stats, boolean oneLiner) {
+ DecimalFormat df = new DecimalFormat("#.###");
+ double source_gl =
+ gradeLevel(stats[tokenLength(SOURCE)], stats[syllableLength(SOURCE)], stats[sentCountIndex]);
+ double cand_gl =
+ gradeLevel(stats[tokenLength(CANDIDATE)], stats[syllableLength(CANDIDATE)],
+ stats[sentCountIndex]);
+ double ref_gl =
+ gradeLevel(stats[tokenLength(REFERENCE)], stats[syllableLength(REFERENCE)],
+ stats[sentCountIndex]);
+ double penalty = 1;
+ double bleu_ref = super.score(stats);
+ double bleu_src = srcBLEU.score(stats);
+ double bleu_plus = BLEU_plus(bleu_ref, bleu_src);
+
+ if (useTarget)
+ penalty = getReadabilityPenalty(cand_gl, targetGL);
+ else
+ penalty = getReadabilityPenalty(cand_gl, source_gl);
+
+ if (oneLiner) {
+ System.out.print("GL_BLEU=" + df.format(score(stats)));
+ System.out.print(" BLEU=" + df.format(bleu_ref));
+ System.out.print(" BLEU_src=" + df.format(bleu_src));
+ System.out.print(" iBLEU=" + df.format(bleu_plus));
+ System.out.print(" GL_cand=" + df.format(cand_gl));
+ System.out.print(" GL_src=" + df.format(source_gl));
+ System.out.print(" GL_ref=" + df.format(ref_gl));
+ System.out.print(" Read_penalty=" + df.format(penalty));
+ System.out.println();
+ } else {
+ System.out.println("GL_BLEU = " + df.format(score(stats)));
+ System.out.println("BLEU = " + df.format(bleu_ref));
+ System.out.println("BLEU_src = " + df.format(bleu_src));
+ System.out.println("iBLEU = " + df.format(bleu_plus));
+ System.out.println("GL_cand = " + df.format(cand_gl));
+ System.out.println("GL_src = " + df.format(source_gl));
+ System.out.println("GL_ref = " + df.format(ref_gl));
+ System.out.println("Read penalty = " + df.format(penalty));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/metrics/METEOR.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/metrics/METEOR.java b/joshua-core/src/main/java/org/apache/joshua/metrics/METEOR.java
new file mode 100644
index 0000000..33232db
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/metrics/METEOR.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.metrics;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+
+import org.apache.joshua.util.StreamGobbler;
+
+public class METEOR extends EvaluationMetric {
+ protected String targetLanguage;
+ protected boolean normalize;
+ protected boolean keepPunctuation;
+ private int maxComputations;
+
+ public METEOR(String[] Metric_options) {
+ // M_o[0]: -l language, one of {en,cz,fr,de,es}
+ // M_o[1]: -normalize, one of {norm_yes,norm_no}
+ // M_o[2]: -keepPunctuation, one of {keepPunc,removePunc}
+ // M_o[3]: maxComputations, positive integer
+
+ // default in meteor v0.8: en, norm_no, removePunc
+
+ if (Metric_options[0].equals("en")) {
+ targetLanguage = "en";
+ } else if (Metric_options[0].equals("cz")) {
+ targetLanguage = "cz";
+ } else if (Metric_options[0].equals("fr")) {
+ targetLanguage = "fr";
+ } else if (Metric_options[0].equals("de")) {
+ targetLanguage = "de";
+ } else if (Metric_options[0].equals("es")) {
+ targetLanguage = "es";
+ } else {
+ String msg = "Unknown language string " + Metric_options[0]
+ + ". Should be one of {en,cz,fr,de,es}.";
+ throw new RuntimeException(msg);
+ }
+
+ if (Metric_options[1].equals("norm_yes")) {
+ normalize = true;
+ } else if (Metric_options[1].equals("norm_no")) {
+ normalize = false;
+ } else {
+ String msg = "Unknown normalize string " + Metric_options[1]
+ + ". Should be one of norm_yes or norm_no.";
+ throw new RuntimeException(msg);
+ }
+
+ if (Metric_options[2].equals("keepPunc")) {
+ keepPunctuation = true;
+ } else if (Metric_options[1].equals("removePunk")) {
+ keepPunctuation = false;
+ } else {
+ String msg = "Unknown keepPunctuation string " + Metric_options[1]
+ + ". Should be one of keepPunc or removePunk.";
+ throw new RuntimeException(msg);
+ }
+
+ maxComputations = Integer.parseInt(Metric_options[3]);
+ if (maxComputations < 1) {
+ throw new RuntimeException("Maximum computations must be positive");
+ }
+
+ initialize(); // set the data members of the metric
+ }
+
+ protected void initialize() {
+ metricName = "METEOR";
+ toBeMinimized = false;
+ suffStatsCount = 5;
+ }
+
+ public double bestPossibleScore() {
+ return 1.0;
+ }
+
+ public double worstPossibleScore() {
+ return 0.0;
+ }
+
+ public int[] suffStats(String cand_str, int i) {
+ // this method should never be used when the metric is METEOR,
+ // because METEOR.java overrides suffStats(String[],int[]) below,
+ // which is the only method that calls suffStats(Sting,int).
+ return null;
+ }
+
+ public int[][] suffStats(String[] cand_strings, int[] cand_indices) {
+ // calculate sufficient statistics for each sentence in an arbitrary set of candidates
+
+ int candCount = cand_strings.length;
+ if (cand_indices.length != candCount) {
+ System.out.println("Array lengths mismatch in suffStats(String[],int[]); returning null.");
+ return null;
+ }
+
+ int[][] stats = new int[candCount][suffStatsCount];
+
+ try {
+
+ // 1) Create input files for meteor
+
+ // 1a) Create hypothesis file
+ FileOutputStream outStream = new FileOutputStream("hyp.txt.METEOR", false); // false: don't
+ // append
+ OutputStreamWriter outStreamWriter = new OutputStreamWriter(outStream, "utf8");
+ BufferedWriter outFile = new BufferedWriter(outStreamWriter);
+
+ for (int d = 0; d < candCount; ++d) {
+ writeLine(cand_strings[d], outFile);
+ }
+
+ outFile.close();
+
+ // 1b) Create reference file
+ outStream = new FileOutputStream("ref.txt.METEOR", false); // false: don't append
+ outStreamWriter = new OutputStreamWriter(outStream, "utf8");
+ outFile = new BufferedWriter(outStreamWriter);
+
+ for (int d = 0; d < candCount; ++d) {
+ for (int r = 0; r < refsPerSen; ++r) {
+ writeLine(refSentences[cand_indices[d]][r], outFile);
+ }
+ }
+
+ outFile.close();
+
+ // 2) Launch meteor as an external process
+
+ String cmd_str = "./meteor hyp.txt.METEOR ref.txt.METEOR";
+ cmd_str += " -l " + targetLanguage;
+ cmd_str += " -r " + refsPerSen;
+ if (normalize) {
+ cmd_str += " -normalize";
+ }
+ if (keepPunctuation) {
+ cmd_str += " -keepPunctuation";
+ }
+ cmd_str += " -ssOut";
+
+ Runtime rt = Runtime.getRuntime();
+ Process p = rt.exec(cmd_str);
+
+ StreamGobbler errorGobbler = new StreamGobbler(p.getErrorStream(), 0);
+ StreamGobbler outputGobbler = new StreamGobbler(p.getInputStream(), 0);
+
+ errorGobbler.start();
+ outputGobbler.start();
+
+ @SuppressWarnings("unused")
+ int exitValue = p.waitFor();
+
+
+ // 3) Read SS from output file produced by meteor
+
+ BufferedReader inFile = new BufferedReader(new FileReader("TER_out.ter"));
+ String line = "";
+
+ line = inFile.readLine(); // skip hyp line
+ line = inFile.readLine(); // skip ref line
+
+ for (int d = 0; d < candCount; ++d) {
+ line = inFile.readLine(); // read info
+ String[] strA = line.split("\\s+");
+
+ stats[d][0] = (int) Double.parseDouble(strA[0]);
+ stats[d][1] = (int) Double.parseDouble(strA[1]);
+ stats[d][2] = (int) Double.parseDouble(strA[2]);
+ stats[d][3] = (int) Double.parseDouble(strA[3]);
+ stats[d][4] = (int) Double.parseDouble(strA[4]);
+ }
+
+ inFile.close();
+ } catch (IOException | InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+
+ return stats;
+ }
+
+ public double score(int[] stats) {
+ if (stats.length != suffStatsCount) {
+ throw new RuntimeException("Mismatch between stats.length and suffStatsCount (" + stats.length
+ + " vs. " + suffStatsCount + ") in METEOR.score(int[])");
+ }
+
+ double sc = 0.0;
+
+ // sc = ???
+
+ return sc;
+ }
+
+ public void printDetailedScore_fromStats(int[] stats, boolean oneLiner) {
+ if (oneLiner) {
+ System.out.println("METEOR = METEOR(" + stats[0] + "," + stats[1] + "," + stats[2] + ","
+ + stats[3] + "," + stats[4] + " = " + score(stats));
+ } else {
+ System.out.println("# matches = " + stats[0]);
+ System.out.println("test length = " + stats[1]);
+ System.out.println("ref length = " + stats[2]);
+ System.out.println("# chunks = " + stats[3]);
+ System.out.println("length cost = " + stats[4]);
+ System.out.println("METEOR = " + score(stats));
+ }
+ }
+
+ private void writeLine(String line, BufferedWriter writer) throws IOException {
+ writer.write(line, 0, line.length());
+ writer.newLine();
+ writer.flush();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/metrics/MinimumChangeBLEU.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/metrics/MinimumChangeBLEU.java b/joshua-core/src/main/java/org/apache/joshua/metrics/MinimumChangeBLEU.java
new file mode 100644
index 0000000..6d19ba5
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/metrics/MinimumChangeBLEU.java
@@ -0,0 +1,223 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.metrics;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.joshua.util.Algorithms;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class MinimumChangeBLEU extends BLEU {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MinimumChangeBLEU.class);
+
+ // we assume that the source for the paraphrasing run is
+ // part of the set of references
+ private int sourceReferenceIndex;
+ private double thresholdWER;
+
+
+ public MinimumChangeBLEU() {
+ super();
+ this.sourceReferenceIndex = 0;
+ this.thresholdWER = 0.3;
+ initialize();
+ }
+
+
+ public MinimumChangeBLEU(String[] options) {
+ super(options);
+ this.sourceReferenceIndex = Integer.parseInt(options[2]);
+ this.thresholdWER = Double.parseDouble(options[3]);
+ initialize();
+ }
+
+
+ protected void initialize() {
+ metricName = "MC_BLEU";
+ toBeMinimized = false;
+ // adding 1 to the sufficient stats for regular BLEU
+ suffStatsCount = 2 * getMaxGramLength() + 3;
+
+ set_weightsArray();
+ set_maxNgramCounts();
+ }
+
+
+ protected void set_maxNgramCounts() {
+ @SuppressWarnings("unchecked")
+ HashMap<String, Integer>[] temp_HMA = new HashMap[numSentences];
+ maxNgramCounts = temp_HMA;
+
+ String gram = "";
+ int oldCount = 0, nextCount = 0;
+
+ for (int i = 0; i < numSentences; ++i) {
+ // update counts as necessary from the reference translations
+ for (int r = 0; r < refsPerSen; ++r) {
+ // skip source reference
+ if (r == this.sourceReferenceIndex) continue;
+ if (maxNgramCounts[i] == null) {
+ maxNgramCounts[i] = getNgramCountsAll(refSentences[i][r]);
+ } else {
+ HashMap<String, Integer> nextNgramCounts = getNgramCountsAll(refSentences[i][r]);
+ for (Map.Entry<String, Integer> entry : nextNgramCounts.entrySet()) {
+ gram = entry.getKey();
+ nextCount = entry.getValue();
+
+ if (maxNgramCounts[i].containsKey(gram)) {
+ oldCount = maxNgramCounts[i].get(gram);
+ if (nextCount > oldCount) {
+ maxNgramCounts[i].put(gram, nextCount);
+ }
+ } else { // add it
+ maxNgramCounts[i].put(gram, nextCount);
+ }
+ }
+ }
+ } // for (r)
+ } // for (i)
+
+ // for efficiency, calculate the reference lenghts, which will be used
+ // in effLength...
+ refWordCount = new int[numSentences][refsPerSen];
+ for (int i = 0; i < numSentences; ++i) {
+ for (int r = 0; r < refsPerSen; ++r) {
+ if (r == this.sourceReferenceIndex) continue;
+ refWordCount[i][r] = wordCount(refSentences[i][r]);
+ }
+ }
+ }
+
+
+ public int[] suffStats(String cand_str, int i) {
+ int[] stats = new int[suffStatsCount];
+
+ String[] candidate_words;
+ if (!cand_str.equals(""))
+ candidate_words = cand_str.split("\\s+");
+ else
+ candidate_words = new String[0];
+
+ // dropping "_OOV" marker
+ for (int j = 0; j < candidate_words.length; j++) {
+ if (candidate_words[j].endsWith("_OOV"))
+ candidate_words[j] = candidate_words[j].substring(0, candidate_words[j].length() - 4);
+ }
+
+ set_prec_suffStats(stats, candidate_words, i);
+ String[] source_words = refSentences[i][sourceReferenceIndex].split("\\s+");
+ stats[suffStatsCount - 1] = Algorithms.levenshtein(candidate_words, source_words);
+ stats[suffStatsCount - 2] = effLength(candidate_words.length, i);
+ stats[suffStatsCount - 3] = candidate_words.length;
+
+ return stats;
+ }
+
+
+ public int effLength(int candLength, int i) {
+ if (getEffLengthMethod() == EffectiveLengthMethod.CLOSEST) {
+ int closestRefLength = Integer.MIN_VALUE;
+ int minDiff = Math.abs(candLength - closestRefLength);
+
+ for (int r = 0; r < refsPerSen; ++r) {
+ if (r == this.sourceReferenceIndex) continue;
+ int nextRefLength = refWordCount[i][r];
+ int nextDiff = Math.abs(candLength - nextRefLength);
+
+ if (nextDiff < minDiff) {
+ closestRefLength = nextRefLength;
+ minDiff = nextDiff;
+ } else if (nextDiff == minDiff && nextRefLength < closestRefLength) {
+ closestRefLength = nextRefLength;
+ minDiff = nextDiff;
+ }
+ }
+ return closestRefLength;
+ } else if (getEffLengthMethod() == EffectiveLengthMethod.SHORTEST) {
+ int shortestRefLength = Integer.MAX_VALUE;
+
+ for (int r = 0; r < refsPerSen; ++r) {
+ if (r == this.sourceReferenceIndex) continue;
+
+ int nextRefLength = refWordCount[i][r];
+ if (nextRefLength < shortestRefLength) {
+ shortestRefLength = nextRefLength;
+ }
+ }
+ return shortestRefLength;
+ }
+
+ return candLength; // should never get here anyway
+ }
+
+
+ public double score(int[] stats) {
+ if (stats.length != suffStatsCount) {
+ String msg ="Mismatch between stats.length and " + "suffStatsCount (" + stats.length
+ + " vs. " + suffStatsCount + ") in BLEU.score(int[])";
+ LOG.error(msg);
+ throw new RuntimeException(msg);
+ }
+
+ double accuracy = 0.0;
+ double smooth_addition = 1.0; // following bleu-1.04.pl
+ double c_len = stats[suffStatsCount - 3];
+ double r_len = stats[suffStatsCount - 2];
+
+ double wer = stats[suffStatsCount - 1] / c_len;
+ double wer_penalty = (wer >= thresholdWER) ? 1.0 : (wer / thresholdWER);
+
+ double correctGramCount, totalGramCount;
+
+ for (int n = 1; n <= getMaxGramLength(); ++n) {
+ correctGramCount = stats[2 * (n - 1)];
+ totalGramCount = stats[2 * (n - 1) + 1];
+
+ double prec_n;
+ if (totalGramCount > 0) {
+ prec_n = correctGramCount / totalGramCount;
+ } else {
+ prec_n = 1; // following bleu-1.04.pl ???????
+ }
+
+ if (prec_n == 0) {
+ smooth_addition *= 0.5;
+ prec_n = smooth_addition / (c_len - n + 1);
+ // isn't c_len-n+1 just totalGramCount ???????
+ }
+ accuracy += weights[n] * Math.log(prec_n);
+ }
+ double brevity_penalty = 1.0;
+ if (c_len < r_len) brevity_penalty = Math.exp(1 - (r_len / c_len));
+
+ return wer_penalty * brevity_penalty * Math.exp(accuracy);
+ }
+
+
+ public void printDetailedScore_fromStats(int[] stats, boolean oneLiner) {
+ double wer = stats[suffStatsCount - 1] / stats[suffStatsCount - 3];
+ double wer_penalty = (wer >= thresholdWER) ? 1.0d : (wer / thresholdWER);
+
+ System.out.println("WER_penalty = " + wer_penalty);
+ System.out.println("MC_BLEU= " + score(stats));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/metrics/NewMetric.java.template
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/metrics/NewMetric.java.template b/joshua-core/src/main/java/org/apache/joshua/metrics/NewMetric.java.template
new file mode 100644
index 0000000..3b8ed83
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/metrics/NewMetric.java.template
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package joshua.zmert;
+import java.math.*;
+import java.util.*;
+import java.io.*;
+
+***(1)***
+public class __new_metric_CLASS_name__ extends EvaluationMetric
+{
+ /********************************************
+ private data members for this error metric
+ ********************************************/
+
+ ***(2)***
+ private ;
+ private ;
+ private ;
+
+ /*
+ You already have access to these data members of the parent
+ class (EvaluationMetric):
+ int numSentences;
+ number of sentences in the MERT set
+ int refsPerSen;
+ number of references per sentence
+ String[][] refSentences;
+ refSentences[i][r] stores the r'th reference of the i'th
+ source sentence (both indices are 0-based)
+ */
+ /********************************************
+ ********************************************/
+
+ public constructorNameMustMatchClassName(String[] Metric_options)
+ {
+
+ ***(3)***
+
+ //
+ //
+ // process the Metric_options array
+ //
+ //
+
+ initialize(); // set the data members of the metric
+ }
+
+ protected void initialize()
+ {
+ ***(4)***
+ metricName = "XXXXXXXX"; <- pick a metric name
+ toBeMinimized = true/false; <- should it be minimized?
+ suffStatsCount = ???; <- how many SS does the metric need?
+
+ ***(5)***
+ /* here you make calls to any methods that set the data members */
+ /* here you make calls to any methods that set the data members */
+ /* here you make calls to any methods that set the data members */
+ }
+
+ ***(6)***
+ public double bestPossibleScore() { return ???; }
+ --> what's the best score of the metric? <--
+ public double worstPossibleScore() { return ???; }
+ --> what's the worst score of the metric? <--
+
+ ***(7)***
+ /* here you define any methods that set the data members */
+ /* here you define any methods that set the data members */
+ /* here you define any methods that set the data members */
+
+ ***(8)***
+ public int[] suffStats(String cand_str, int i) throws Exception
+ {
+ int[] stats = new int[suffStatsCount];
+
+ //
+ //
+ // set contents of stats[] here!
+ //
+ //
+
+ return stats;
+ }
+
+ ***(9a)***
+ public double score(int[] stats)
+ {
+ if (stats.length != suffStatsCount) {
+ System.out.println("Mismatch between stats.length and suffStatsCount (" + stats.length + " vs. " + suffStatsCount + ") in NewMetric.score(int[])");
+ System.exit(1);
+ }
+
+ double sc = 0.0;
+
+ //
+ //
+ // set sc here!
+ //
+ //
+
+ return sc;
+ }
+
+ ***(9b)***
+ public void printDetailedScore_fromStats(int[] stats, boolean oneLiner)
+ {
+ System.out.println(metricName + " = " + score(stats));
+
+ //
+ //
+ // optional (for debugging purposes)
+ //
+ //
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/metrics/Precis.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/metrics/Precis.java b/joshua-core/src/main/java/org/apache/joshua/metrics/Precis.java
new file mode 100644
index 0000000..b2d852b
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/metrics/Precis.java
@@ -0,0 +1,334 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.metrics;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.joshua.util.Algorithms;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+// The metric re-uses most of the BLEU code
+public class Precis extends BLEU {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Precis.class);
+
+ private static final double REF_CR = -1.0;
+
+ // We assume that the source for the paraphrasing run is
+ // part of the set of references, this is its index.
+ private int sourceReferenceIndex;
+
+ // A global target compression rate to achieve
+ // if negative, we default to locally aiming for the compression
+ // rate given by the (closest) reference compression?
+ private double targetCompressionRate;
+
+ // Are we optimizing for character-based compression (as opposed
+ // to token-based)?
+ private boolean characterBased;
+
+ // Weight for factoring in Levenshtein distance to source as a penalty for
+ // insufficient change.
+ private double similarityWeight;
+
+ public Precis() {
+ super();
+ this.sourceReferenceIndex = 0;
+ this.targetCompressionRate = 0;
+ this.characterBased = false;
+ this.similarityWeight = 0;
+ initialize();
+ }
+
+ // We require the BLEU arguments (that's 2) plus
+ // 3 of our own (see above) - the total is registered with
+ // ZMERT in EvaluationMetric, line ~66
+ public Precis(String[] options) {
+ super(options);
+ this.sourceReferenceIndex = Integer.parseInt(options[2]);
+
+ if ("ref".equals(options[3])) {
+ targetCompressionRate = REF_CR;
+ } else {
+ targetCompressionRate = Double.parseDouble(options[3]);
+ if (targetCompressionRate > 1 || targetCompressionRate < 0)
+ throw new RuntimeException("Invalid compression ratio requested: " + options[3]);
+ }
+
+ if ("chars".equals(options[4]))
+ this.characterBased = true;
+ else if ("words".equals(options[4]))
+ this.characterBased = false;
+ else
+ throw new RuntimeException("Unknown compression style: " + options[4]);
+
+ similarityWeight = Double.parseDouble(options[5]);
+ if (similarityWeight < 0 || similarityWeight > 1)
+ throw new RuntimeException("Source penalty out of bounds: " + options[5]);
+
+ initialize();
+ }
+
+ // in addition to BLEU's statistics, we store some length info;
+ // for character-based compression we need to store more (for token-based
+ // BLEU already has us partially covered by storing some num_of_words)
+ //
+ // here's where you'd make additional room for statistics of your own
+ protected void initialize() {
+ metricName = "PRECIS";
+ toBeMinimized = false;
+ // Adding 3 to the sufficient stats for regular BLEU - character-based
+ // compression requires extra stats. We additionally store the Levenshtein
+ // distance to the source, the source length in tokens and the source
+ // length relevant
+ suffStatsCount = 2 * getMaxGramLength() + 4 + (this.characterBased ? 3 : 0);
+
+ set_weightsArray();
+ set_maxNgramCounts();
+ }
+
+ // The only difference to BLEU here is that we're excluding the input from
+ // the collection of ngram statistics - that's actually up for debate
+ protected void set_maxNgramCounts() {
+ @SuppressWarnings("unchecked")
+ HashMap<String, Integer>[] temp_HMA = new HashMap[numSentences];
+ maxNgramCounts = temp_HMA;
+
+ String gram = "";
+ int oldCount = 0, nextCount = 0;
+
+ for (int i = 0; i < numSentences; ++i) {
+ // update counts as necessary from the reference translations
+ for (int r = 0; r < refsPerSen; ++r) {
+ // skip source reference
+ if (r == this.sourceReferenceIndex) continue;
+ if (maxNgramCounts[i] == null) {
+ maxNgramCounts[i] = getNgramCountsAll(refSentences[i][r]);
+ } else {
+ HashMap<String, Integer> nextNgramCounts = getNgramCountsAll(refSentences[i][r]);
+ for ( Map.Entry<String, Integer> entry : nextNgramCounts.entrySet() ) {
+ gram = entry.getKey();
+ nextCount = entry.getValue();
+
+ if (maxNgramCounts[i].containsKey(gram)) {
+ oldCount = maxNgramCounts[i].get(gram);
+ if (nextCount > oldCount) {
+ maxNgramCounts[i].put(gram, nextCount);
+ }
+ } else { // add it
+ maxNgramCounts[i].put(gram, nextCount);
+ }
+ }
+ }
+ } // for (r)
+ } // for (i)
+
+ // for efficiency, calculate the reference lengths, which will be used
+ // in effLength...
+ refWordCount = new int[numSentences][refsPerSen];
+ for (int i = 0; i < numSentences; ++i) {
+ for (int r = 0; r < refsPerSen; ++r) {
+ refWordCount[i][r] = wordCount(refSentences[i][r]);
+ }
+ }
+ }
+
+ // computation of statistics
+ public int[] suffStats(String cand_str, int i) {
+ int[] stats = new int[suffStatsCount];
+
+ String[] candidate_words;
+ if (!cand_str.equals(""))
+ candidate_words = cand_str.split("\\s+");
+ else
+ candidate_words = new String[0];
+
+ // Set n-gram precision stats.
+ set_prec_suffStats(stats, candidate_words, i);
+
+ // Same as BLEU.
+ stats[2 * getMaxGramLength()] = candidate_words.length;
+ stats[2 * getMaxGramLength() + 1] = effLength(candidate_words.length, i);
+
+ // Source length in tokens.
+ stats[2 * getMaxGramLength() + 2] = refWordCount[i][sourceReferenceIndex];
+
+ // Character-based compression requires stats in character counts.
+ if (this.characterBased) {
+ // Candidate length in characters.
+ stats[suffStatsCount - 4] = cand_str.length() - candidate_words.length + 1;
+ // Reference length in characters.
+ stats[suffStatsCount - 3] = effLength(stats[suffStatsCount - 4], i, true);
+ // Source length in characters.
+ stats[suffStatsCount - 2] =
+ refSentences[i][sourceReferenceIndex].length() - refWordCount[i][sourceReferenceIndex]
+ + 1;
+ }
+
+ // Levenshtein distance to source.
+ if (this.similarityWeight > 0)
+ stats[suffStatsCount - 1] =
+ Algorithms.levenshtein(candidate_words,
+ refSentences[i][sourceReferenceIndex].split("\\s+"));
+
+ return stats;
+ }
+
+ public int effLength(int candLength, int i) {
+ return effLength(candLength, i, false);
+ }
+
+ // hacked to be able to return character length upon request
+ public int effLength(int candLength, int i, boolean character_length) {
+ if (getEffLengthMethod() == EffectiveLengthMethod.CLOSEST) {
+ int closestRefLength = Integer.MIN_VALUE;
+ int minDiff = Math.abs(candLength - closestRefLength);
+
+ for (int r = 0; r < refsPerSen; ++r) {
+ if (r == this.sourceReferenceIndex) continue;
+ int nextRefLength =
+ (character_length
+ ? refSentences[i][r].length() - refWordCount[i][r] + 1
+ : refWordCount[i][r]);
+ int nextDiff = Math.abs(candLength - nextRefLength);
+
+ if (nextDiff < minDiff) {
+ closestRefLength = nextRefLength;
+ minDiff = nextDiff;
+ } else if (nextDiff == minDiff && nextRefLength < closestRefLength) {
+ closestRefLength = nextRefLength;
+ minDiff = nextDiff;
+ }
+ }
+ return closestRefLength;
+ } else if (getEffLengthMethod() == EffectiveLengthMethod.SHORTEST) {
+ int shortestRefLength = Integer.MAX_VALUE;
+
+ for (int r = 0; r < refsPerSen; ++r) {
+ if (r == this.sourceReferenceIndex) continue;
+
+ int nextRefLength =
+ (character_length
+ ? refSentences[i][r].length() - refWordCount[i][r] + 1
+ : refWordCount[i][r]);
+ if (nextRefLength < shortestRefLength) {
+ shortestRefLength = nextRefLength;
+ }
+ }
+ return shortestRefLength;
+ }
+
+ return candLength; // should never get here anyway
+ }
+
+ // calculate the actual score from the statistics
+ public double score(int[] stats) {
+ if (stats.length != suffStatsCount) {
+ String msg = "Mismatch between stats.length and suffStatsCount (" + stats.length + " vs. "
+ + suffStatsCount + ") in Precis.score(int[])";
+ LOG.error(msg);
+ throw new RuntimeException(msg);
+ }
+
+ double accuracy = 0.0;
+ double smooth_addition = 1.0; // following bleu-1.04.pl
+
+ double cnd_len = stats[2 * getMaxGramLength()];
+ double ref_len = stats[2 * getMaxGramLength() + 1];
+ double src_len = stats[2 * getMaxGramLength() + 2];
+ double compression_cnd_len = stats[suffStatsCount - 4];
+ double compression_ref_len = stats[suffStatsCount - 3];
+ double compression_src_len = stats[suffStatsCount - 2];
+ double src_lev = stats[suffStatsCount - 1];
+
+ double compression_ratio = compression_cnd_len / compression_src_len;
+
+ double verbosity_penalty =
+ getVerbosityPenalty(compression_ratio, (targetCompressionRate == REF_CR
+ ? compression_ref_len / compression_src_len
+ : targetCompressionRate));
+
+ // this part matches BLEU
+ double correctGramCount, totalGramCount;
+ for (int n = 1; n <= getMaxGramLength(); ++n) {
+ correctGramCount = stats[2 * (n - 1)];
+ totalGramCount = stats[2 * (n - 1) + 1];
+ double prec_n;
+ if (totalGramCount > 0) {
+ prec_n = correctGramCount / totalGramCount;
+ } else {
+ prec_n = 1;
+ }
+ if (prec_n == 0) {
+ smooth_addition *= 0.5;
+ prec_n = smooth_addition / (cnd_len - n + 1);
+ }
+ accuracy += weights[n] * Math.log(prec_n);
+ }
+ double brevity_penalty = 1.0;
+ double similarity_penalty = similarityWeight * Math.max(0, 1 - src_lev / src_len);
+
+ if (cnd_len < ref_len) brevity_penalty = Math.exp(1 - (ref_len / cnd_len));
+
+ // We add on our penalties on top of BLEU.
+ return verbosity_penalty * brevity_penalty * Math.exp(accuracy) - similarity_penalty;
+ }
+
+ // Somewhat not-so-detailed, this is used in the JoshuaEval tool.
+ public void printDetailedScore_fromStats(int[] stats, boolean oneLiner) {
+ double cnd_len = stats[2 * getMaxGramLength()];
+ double ref_len = stats[2 * getMaxGramLength() + 1];
+ double src_len = stats[2 * getMaxGramLength() + 2];
+ double compression_cnd_len = stats[suffStatsCount - 4];
+ double compression_ref_len = stats[suffStatsCount - 3];
+ double compression_src_len = stats[suffStatsCount - 2];
+ double src_lev = stats[suffStatsCount - 1];
+
+ double brevity_penalty = 1;
+ if (cnd_len < ref_len) brevity_penalty = Math.exp(1 - (ref_len / cnd_len));
+
+ double cr = compression_cnd_len / compression_src_len;
+ double similarity_penalty = Math.max(0, 1 - src_lev / src_len);
+
+ double verbosity_penalty =
+ getVerbosityPenalty(cr, (targetCompressionRate == REF_CR ? compression_ref_len
+ / compression_src_len : targetCompressionRate));
+
+ System.out.println(String.format("Similarity Penalty = %.2f * %.4f", similarityWeight,
+ similarity_penalty));
+ System.out.println(String.format("Verbosity Penalty = %.4f", verbosity_penalty));
+ System.out.println(String.format("Brevity Penalty = %.4f", brevity_penalty));
+ System.out.println(String.format("Precis = %.4f", score(stats)));
+ }
+
+ // Returns the score penalty as a function of the achieved and target
+ // compression rates currently an exponential fall-off to make sure the not
+ // compressing enough is costly.
+ protected static double getVerbosityPenalty(double cr, double target_rate) {
+ if (cr <= target_rate)
+ return 1.0;
+ else {
+ // linear option: (1 - cr) / (1 - compressionRate);
+ // doesn't penalize insufficient compressions hard enough
+ return Math.exp(5 * (target_rate - cr));
+ }
+ }
+}
[59/60] incubator-joshua git commit: Merge branch
'maven-multi-module' of https://github.com/logogin/incubator-joshua into
maven-multi-module
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/main/java/org/apache/joshua/decoder/Decoder.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/Decoder.java
index 0000000,097ce59..f319e40
mode 000000,100644..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/Decoder.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/Decoder.java
@@@ -1,0 -1,813 +1,813 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.decoder;
+
+ import static org.apache.joshua.decoder.ff.FeatureVector.DENSE_FEATURE_NAMES;
++import static org.apache.joshua.decoder.ff.tm.OwnerMap.getOwner;
+
+ import java.io.BufferedWriter;
+ import java.io.File;
+ import java.io.IOException;
+ import java.io.FileNotFoundException;
+ import java.lang.reflect.Constructor;
+ import java.util.ArrayList;
+ import java.util.HashMap;
+ import java.util.HashSet;
+ import java.util.List;
++import java.util.Set;
+ import java.util.concurrent.ArrayBlockingQueue;
+ import java.util.concurrent.BlockingQueue;
+
+ import com.google.common.base.Strings;
+
+ import org.apache.joshua.corpus.Vocabulary;
+ import org.apache.joshua.decoder.ff.FeatureVector;
+ import org.apache.joshua.decoder.ff.FeatureFunction;
+ import org.apache.joshua.decoder.ff.PhraseModel;
+ import org.apache.joshua.decoder.ff.StatefulFF;
+ import org.apache.joshua.decoder.ff.lm.LanguageModelFF;
+ import org.apache.joshua.decoder.ff.tm.Grammar;
++import org.apache.joshua.decoder.ff.tm.OwnerId;
++import org.apache.joshua.decoder.ff.tm.OwnerMap;
+ import org.apache.joshua.decoder.ff.tm.Rule;
+ import org.apache.joshua.decoder.ff.tm.format.HieroFormatReader;
+ import org.apache.joshua.decoder.ff.tm.hash_based.MemoryBasedBatchGrammar;
+ import org.apache.joshua.decoder.ff.tm.packed.PackedGrammar;
+ import org.apache.joshua.decoder.io.TranslationRequestStream;
+ import org.apache.joshua.decoder.phrase.PhraseTable;
+ import org.apache.joshua.decoder.segment_file.Sentence;
+ import org.apache.joshua.util.FileUtility;
+ import org.apache.joshua.util.FormatUtils;
+ import org.apache.joshua.util.Regex;
+ import org.apache.joshua.util.io.LineReader;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
+ /**
+ * This class handles decoder initialization and the complication introduced by multithreading.
+ *
+ * After initialization, the main entry point to the Decoder object is
+ * decodeAll(TranslationRequest), which returns a set of Translation objects wrapped in an iterable
+ * Translations object. It is important that we support multithreading both (a) across the sentences
+ * within a request and (b) across requests, in a round-robin fashion. This is done by maintaining a
+ * fixed sized concurrent thread pool. When a new request comes in, a RequestParallelizer thread is
+ * launched. This object iterates over the request's sentences, obtaining a thread from the
+ * thread pool, and using that thread to decode the sentence. If a decoding thread is not available,
+ * it will block until one is in a fair (FIFO) manner. RequestParallelizer thereby permits intra-request
+ * parallelization by separating out reading the input stream from processing the translated sentences,
+ * but also ensures that round-robin parallelization occurs, since RequestParallelizer uses the
+ * thread pool before translating each request.
+ *
+ * A decoding thread is handled by DecoderThread and launched from DecoderThreadRunner. The purpose
+ * of the runner is to record where to place the translated sentence when it is done (i.e., which
+ * Translations object). Translations itself is an iterator whose next() call blocks until the next
+ * translation is available.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ * @author Zhifei Li, zhifei.work@gmail.com
+ * @author wren ng thornton wren@users.sourceforge.net
+ * @author Lane Schwartz dowobeha@users.sourceforge.net
+ */
+ public class Decoder {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Decoder.class);
+
+ private final JoshuaConfiguration joshuaConfiguration;
+
+ public JoshuaConfiguration getJoshuaConfiguration() {
+ return joshuaConfiguration;
+ }
+
+ /*
+ * Many of these objects themselves are global objects. We pass them in when constructing other
+ * objects, so that they all share pointers to the same object. This is good because it reduces
+ * overhead, but it can be problematic because of unseen dependencies (for example, in the
+ * Vocabulary shared by language model, translation grammar, etc).
+ */
+ private List<Grammar> grammars;
+ private ArrayList<FeatureFunction> featureFunctions;
+ private Grammar customPhraseTable;
+
+ /* The feature weights. */
+ public static FeatureVector weights;
+
+ public static int VERBOSE = 1;
+
+ private BlockingQueue<DecoderThread> threadPool = null;
+
+ // ===============================================================
+ // Constructors
+ // ===============================================================
+
+ /**
+ * Constructor method that creates a new decoder using the specified configuration file.
+ *
+ * @param joshuaConfiguration a populated {@link org.apache.joshua.decoder.JoshuaConfiguration}
+ * @param configFile name of configuration file.
+ */
+ public Decoder(JoshuaConfiguration joshuaConfiguration, String configFile) {
+ this(joshuaConfiguration);
+ this.initialize(configFile);
+ }
+
+ /**
+ * Factory method that creates a new decoder using the specified configuration file.
+ *
+ * @param configFile Name of configuration file.
+ * @return a configured {@link org.apache.joshua.decoder.Decoder}
+ */
+ public static Decoder createDecoder(String configFile) {
+ JoshuaConfiguration joshuaConfiguration = new JoshuaConfiguration();
+ return new Decoder(joshuaConfiguration, configFile);
+ }
+
+ /**
+ * Constructs an uninitialized decoder for use in testing.
+ * <p>
+ * This method is private because it should only ever be called by the
+ * {@link #getUninitalizedDecoder()} method to provide an uninitialized decoder for use in
+ * testing.
+ */
+ private Decoder(JoshuaConfiguration joshuaConfiguration) {
+ this.joshuaConfiguration = joshuaConfiguration;
+ this.grammars = new ArrayList<Grammar>();
+ this.threadPool = new ArrayBlockingQueue<DecoderThread>(
+ this.joshuaConfiguration.num_parallel_decoders, true);
+ this.customPhraseTable = null;
+ }
+
+ /**
+ * Gets an uninitialized decoder for use in testing.
+ * <p>
+ * This method is called by unit tests or any outside packages (e.g., MERT) relying on the
+ * decoder.
+ * @param joshuaConfiguration a {@link org.apache.joshua.decoder.JoshuaConfiguration} object
+ * @return an uninitialized decoder for use in testing
+ */
+ static public Decoder getUninitalizedDecoder(JoshuaConfiguration joshuaConfiguration) {
+ return new Decoder(joshuaConfiguration);
+ }
+
+ // ===============================================================
+ // Public Methods
+ // ===============================================================
+
+ /**
+ * This class is responsible for getting sentences from the TranslationRequest and procuring a
+ * DecoderThreadRunner to translate it. Each call to decodeAll(TranslationRequest) launches a
+ * thread that will read the request's sentences, obtain a DecoderThread to translate them, and
+ * then place the Translation in the appropriate place.
+ *
+ * @author Matt Post <po...@cs.jhu.edu>
+ *
+ */
+ private class RequestParallelizer extends Thread {
+ /* Source of sentences to translate. */
+ private final TranslationRequestStream request;
+
+ /* Where to put translated sentences. */
+ private final Translations response;
+
+ RequestParallelizer(TranslationRequestStream request, Translations response) {
+ this.request = request;
+ this.response = response;
+ }
+
+ @Override
+ public void run() {
+ /*
+ * Repeatedly get an input sentence, wait for a DecoderThread, and then start a new thread to
+ * translate the sentence. We start a new thread (via DecoderRunnerThread) as opposed to
+ * blocking, so that the RequestHandler can go on to the next sentence in this request, which
+ * allows parallelization across the sentences of the request.
+ */
+ for (;;) {
+ Sentence sentence = request.next();
+
+ if (sentence == null) {
+ response.finish();
+ break;
+ }
+
+ // This will block until a DecoderThread becomes available.
+ DecoderThread thread = Decoder.this.getThread();
+ new DecoderThreadRunner(thread, sentence, response).start();
+ }
+ }
+
+ /**
+ * Strips the nonterminals from the lefthand side of the rule.
+ *
+ * @param rule
+ * @return
+ */
+ private String formatRule(Rule rule) {
+ String ruleString = "";
+ boolean first = true;
+ for (int word: rule.getFrench()) {
+ if (!first)
+ ruleString += " " + Vocabulary.word(word);
+ first = false;
+ }
+
+ ruleString += " |||"; // space will get added with first English word
+ first = true;
+ for (int word: rule.getEnglish()) {
+ if (!first)
+ ruleString += " " + Vocabulary.word(word);
+ first = false;
+ }
+
+ // strip of the leading space
+ return ruleString.substring(1);
+ }
+ }
+
+ /**
+ * Retrieve a thread from the thread pool, blocking until one is available. The blocking occurs in
+ * a fair fashion (i.e,. FIFO across requests).
+ *
+ * @return a thread that can be used for decoding.
+ */
+ public DecoderThread getThread() {
+ try {
+ return threadPool.take();
+ } catch (InterruptedException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ return null;
+ }
+
+ /**
+ * This class handles running a DecoderThread (which takes care of the actual translation of an
+ * input Sentence, returning a Translation object when its done). This is done in a thread so as
+ * not to tie up the RequestHandler that launched it, freeing it to go on to the next sentence in
+ * the TranslationRequest, in turn permitting parallelization across the sentences of a request.
+ *
+ * When the decoder thread is finshed, the Translation object is placed in the correct place in
+ * the corresponding Translations object that was returned to the caller of
+ * Decoder.decodeAll(TranslationRequest).
+ *
+ * @author Matt Post <po...@cs.jhu.edu>
+ */
+ private class DecoderThreadRunner extends Thread {
+
+ private final DecoderThread decoderThread;
+ private final Sentence sentence;
+ private final Translations translations;
+
+ DecoderThreadRunner(DecoderThread thread, Sentence sentence, Translations translations) {
+ this.decoderThread = thread;
+ this.sentence = sentence;
+ this.translations = translations;
+ }
+
+ @Override
+ public void run() {
+ /*
+ * Process any found metadata.
+ */
+
+ /*
+ * Use the thread to translate the sentence. Then record the translation with the
+ * corresponding Translations object, and return the thread to the pool.
+ */
+ try {
+ Translation translation = decoderThread.translate(this.sentence);
+ translations.record(translation);
+
+ /*
+ * This is crucial! It's what makes the thread available for the next sentence to be
+ * translated.
+ */
+ threadPool.put(decoderThread);
+ } catch (Exception e) {
+ throw new RuntimeException(String.format(
+ "Input %d: FATAL UNCAUGHT EXCEPTION: %s", sentence.id(), e.getMessage()), e);
+ // translations.record(new Translation(sentence, null, featureFunctions, joshuaConfiguration));
+ }
+ }
+ }
+
+ /**
+ * This function is the main entry point into the decoder. It translates all the sentences in a
+ * (possibly boundless) set of input sentences. Each request launches its own thread to read the
+ * sentences of the request.
+ *
+ * @param request the populated {@link org.apache.joshua.decoder.io.TranslationRequestStream}
+ * @throws IOException if there is an error with the input stream or writing the output
+ * @return an iterable, asynchronously-filled list of Translations
+ */
+ public Translations decodeAll(TranslationRequestStream request) throws IOException {
+ Translations translations = new Translations(request);
+
+ /* Start a thread to handle requests on the input stream */
+ new RequestParallelizer(request, translations).start();
+
+ return translations;
+ }
+
+
+ /**
+ * We can also just decode a single sentence.
+ *
+ * @param sentence {@link org.apache.joshua.lattice.Lattice} input
+ * @return the sentence {@link org.apache.joshua.decoder.Translation}
+ */
+ public Translation decode(Sentence sentence) {
+ // Get a thread.
+
+ try {
+ DecoderThread thread = threadPool.take();
+ Translation translation = thread.translate(sentence);
+ threadPool.put(thread);
+
+ return translation;
+
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+
+ return null;
+ }
+
+ /**
+ * Clean shutdown of Decoder, resetting all
+ * static variables, such that any other instance of Decoder
+ * afterwards gets a fresh start.
+ */
+ public void cleanUp() {
+ // shut down DecoderThreads
+ for (DecoderThread thread : threadPool) {
+ try {
+ thread.join();
+ } catch (InterruptedException e) {
+ e.printStackTrace();
+ }
+ }
+ resetGlobalState();
+ }
+
+ public static void resetGlobalState() {
+ // clear/reset static variables
++ OwnerMap.clear();
+ DENSE_FEATURE_NAMES.clear();
+ Vocabulary.clear();
+ Vocabulary.unregisterLanguageModels();
+ LanguageModelFF.resetLmIndex();
+ StatefulFF.resetGlobalStateIndex();
+ }
+
+ public static void writeConfigFile(double[] newWeights, String template, String outputFile,
+ String newDiscriminativeModel) {
+ try {
+ int columnID = 0;
+
+ BufferedWriter writer = FileUtility.getWriteFileStream(outputFile);
+ LineReader reader = new LineReader(template);
+ try {
+ for (String line : reader) {
+ line = line.trim();
+ if (Regex.commentOrEmptyLine.matches(line) || line.indexOf("=") != -1) {
+ // comment, empty line, or parameter lines: just copy
+ writer.write(line);
+ writer.newLine();
+
+ } else { // models: replace the weight
+ String[] fds = Regex.spaces.split(line);
+ StringBuffer newSent = new StringBuffer();
+ if (!Regex.floatingNumber.matches(fds[fds.length - 1])) {
+ throw new IllegalArgumentException("last field is not a number; the field is: "
+ + fds[fds.length - 1]);
+ }
+
+ if (newDiscriminativeModel != null && "discriminative".equals(fds[0])) {
+ newSent.append(fds[0]).append(' ');
+ newSent.append(newDiscriminativeModel).append(' ');// change the
+ // file name
+ for (int i = 2; i < fds.length - 1; i++) {
+ newSent.append(fds[i]).append(' ');
+ }
+ } else {// regular
+ for (int i = 0; i < fds.length - 1; i++) {
+ newSent.append(fds[i]).append(' ');
+ }
+ }
+ if (newWeights != null)
+ newSent.append(newWeights[columnID++]);// change the weight
+ else
+ newSent.append(fds[fds.length - 1]);// do not change
+
+ writer.write(newSent.toString());
+ writer.newLine();
+ }
+ }
+ } finally {
+ reader.close();
+ writer.close();
+ }
+
+ if (newWeights != null && columnID != newWeights.length) {
+ throw new IllegalArgumentException("number of models does not match number of weights");
+ }
+
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ // ===============================================================
+ // Initialization Methods
+ // ===============================================================
+
+ /**
+ * Moses requires the pattern .*_.* for sparse features, and prohibits underscores in dense features.
+ * This conforms to that pattern. We assume non-conforming dense features start with tm_ or lm_,
+ * and the only sparse feature that needs converting is OOVPenalty.
+ *
+ * @param feature
+ * @return the feature in Moses format
+ */
+ private String mosesize(String feature) {
+ if (joshuaConfiguration.moses) {
+ if (feature.startsWith("tm_") || feature.startsWith("lm_"))
+ return feature.replace("_", "-");
+ }
+
+ return feature;
+ }
+
+ /**
+ * Initialize all parts of the JoshuaDecoder.
+ *
+ * @param configFile File containing configuration options
+ * @return An initialized decoder
+ */
+ public Decoder initialize(String configFile) {
+ try {
+
+ long pre_load_time = System.currentTimeMillis();
+
+ /* Weights can be listed in a separate file (denoted by parameter "weights-file") or directly
+ * in the Joshua config file. Config file values take precedent.
+ */
+ this.readWeights(joshuaConfiguration.weights_file);
+
+
+ /* Add command-line-passed weights to the weights array for processing below */
+ if (!Strings.isNullOrEmpty(joshuaConfiguration.weight_overwrite)) {
+ String[] tokens = joshuaConfiguration.weight_overwrite.split("\\s+");
+ for (int i = 0; i < tokens.length; i += 2) {
+ String feature = tokens[i];
+ float value = Float.parseFloat(tokens[i+1]);
+
+ if (joshuaConfiguration.moses)
+ feature = demoses(feature);
+
+ joshuaConfiguration.weights.add(String.format("%s %s", feature, tokens[i+1]));
+ LOG.info("COMMAND LINE WEIGHT: {} -> {}", feature, value);
+ }
+ }
+
+ /* Read the weights found in the config file */
+ for (String pairStr: joshuaConfiguration.weights) {
+ String pair[] = pairStr.split("\\s+");
+
+ /* Sanity check for old-style unsupported feature invocations. */
+ if (pair.length != 2) {
+ StringBuilder errMsg = new StringBuilder();
+ errMsg.append("FATAL: Invalid feature weight line found in config file.\n");
+ errMsg.append(String.format("The line was '%s'\n", pairStr));
+ errMsg.append("You might be using an old version of the config file that is no longer supported\n");
+ errMsg.append("Check joshua-decoder.org or email joshua_support@googlegroups.com for help\n");
+ errMsg.append("Code = " + 17);
+ throw new RuntimeException(errMsg.toString());
+ }
+
+ weights.set(pair[0], Float.parseFloat(pair[1]));
+ }
+
+ LOG.info("Read {} weights ({} of them dense)", weights.size(), DENSE_FEATURE_NAMES.size());
+
+ // Do this before loading the grammars and the LM.
+ this.featureFunctions = new ArrayList<FeatureFunction>();
+
+ // Initialize and load grammars. This must happen first, since the vocab gets defined by
+ // the packed grammar (if any)
+ this.initializeTranslationGrammars();
+ LOG.info("Grammar loading took: {} seconds.",
+ (System.currentTimeMillis() - pre_load_time) / 1000);
+
+ // Initialize the features: requires that LM model has been initialized.
+ this.initializeFeatureFunctions();
+
+ // This is mostly for compatibility with the Moses tuning script
+ if (joshuaConfiguration.show_weights_and_quit) {
+ for (int i = 0; i < DENSE_FEATURE_NAMES.size(); i++) {
+ String name = DENSE_FEATURE_NAMES.get(i);
+ if (joshuaConfiguration.moses)
+ System.out.println(String.format("%s= %.5f", mosesize(name), weights.getDense(i)));
+ else
+ System.out.println(String.format("%s %.5f", name, weights.getDense(i)));
+ }
+ System.exit(0);
+ }
+
+ // Sort the TM grammars (needed to do cube pruning)
+ if (joshuaConfiguration.amortized_sorting) {
+ LOG.info("Grammar sorting happening lazily on-demand.");
+ } else {
+ long pre_sort_time = System.currentTimeMillis();
+ for (Grammar grammar : this.grammars) {
+ grammar.sortGrammar(this.featureFunctions);
+ }
+ LOG.info("Grammar sorting took {} seconds.",
+ (System.currentTimeMillis() - pre_sort_time) / 1000);
+ }
+
+ // Create the threads
+ for (int i = 0; i < joshuaConfiguration.num_parallel_decoders; i++) {
+ this.threadPool.put(new DecoderThread(this.grammars, Decoder.weights,
+ this.featureFunctions, joshuaConfiguration));
+ }
+ } catch (IOException | InterruptedException e) {
+ LOG.warn(e.getMessage(), e);
+ }
+
+ return this;
+ }
+
+ /**
+ * Initializes translation grammars Retained for backward compatibility
+ *
+ * @param ownersSeen Records which PhraseModelFF's have been instantiated (one is needed for each
+ * owner)
+ * @throws IOException
+ */
+ private void initializeTranslationGrammars() throws IOException {
+
+ if (joshuaConfiguration.tms.size() > 0) {
+
+ // collect packedGrammars to check if they use a shared vocabulary
+ final List<PackedGrammar> packed_grammars = new ArrayList<>();
+
+ // tm = {thrax/hiero,packed,samt,moses} OWNER LIMIT FILE
+ for (String tmLine : joshuaConfiguration.tms) {
+
+ String type = tmLine.substring(0, tmLine.indexOf(' '));
+ String[] args = tmLine.substring(tmLine.indexOf(' ')).trim().split("\\s+");
+ HashMap<String, String> parsedArgs = FeatureFunction.parseArgs(args);
+
+ String owner = parsedArgs.get("owner");
+ int span_limit = Integer.parseInt(parsedArgs.get("maxspan"));
+ String path = parsedArgs.get("path");
+
+ Grammar grammar = null;
+ if (! type.equals("moses") && ! type.equals("phrase")) {
+ if (new File(path).isDirectory()) {
+ try {
+ PackedGrammar packed_grammar = new PackedGrammar(path, span_limit, owner, type, joshuaConfiguration);
+ packed_grammars.add(packed_grammar);
+ grammar = packed_grammar;
+ } catch (FileNotFoundException e) {
+ String msg = String.format("Couldn't load packed grammar from '%s'", path)
+ + "Perhaps it doesn't exist, or it may be an old packed file format.";
+ throw new RuntimeException(msg);
+ }
+ } else {
+ // thrax, hiero, samt
+ grammar = new MemoryBasedBatchGrammar(type, path, owner,
+ joshuaConfiguration.default_non_terminal, span_limit, joshuaConfiguration);
+ }
+
+ } else {
+
- int maxSourceLen = parsedArgs.containsKey("max-source-len")
- ? Integer.parseInt(parsedArgs.get("max-source-len"))
- : -1;
-
+ joshuaConfiguration.search_algorithm = "stack";
+ grammar = new PhraseTable(path, owner, type, joshuaConfiguration);
+ }
+
+ this.grammars.add(grammar);
+ }
+
+ checkSharedVocabularyChecksumsForPackedGrammars(packed_grammars);
+
+ } else {
+ LOG.warn("no grammars supplied! Supplying dummy glue grammar.");
- MemoryBasedBatchGrammar glueGrammar = new MemoryBasedBatchGrammar("glue", joshuaConfiguration);
- glueGrammar.setSpanLimit(-1);
++ MemoryBasedBatchGrammar glueGrammar = new MemoryBasedBatchGrammar("glue", joshuaConfiguration, -1);
+ glueGrammar.addGlueRules(featureFunctions);
+ this.grammars.add(glueGrammar);
+ }
+
+ /* Add the grammar for custom entries */
+ if (joshuaConfiguration.search_algorithm.equals("stack"))
+ this.customPhraseTable = new PhraseTable(null, "custom", "phrase", joshuaConfiguration);
+ else
- this.customPhraseTable = new MemoryBasedBatchGrammar("custom", joshuaConfiguration);
++ this.customPhraseTable = new MemoryBasedBatchGrammar("custom", joshuaConfiguration, 20);
+ this.grammars.add(this.customPhraseTable);
+
+ /* Create an epsilon-deleting grammar */
+ if (joshuaConfiguration.lattice_decoding) {
+ LOG.info("Creating an epsilon-deleting grammar");
- MemoryBasedBatchGrammar latticeGrammar = new MemoryBasedBatchGrammar("lattice", joshuaConfiguration);
- latticeGrammar.setSpanLimit(-1);
++ MemoryBasedBatchGrammar latticeGrammar = new MemoryBasedBatchGrammar("lattice", joshuaConfiguration, -1);
+ HieroFormatReader reader = new HieroFormatReader();
+
+ String goalNT = FormatUtils.cleanNonTerminal(joshuaConfiguration.goal_symbol);
+ String defaultNT = FormatUtils.cleanNonTerminal(joshuaConfiguration.default_non_terminal);
+
+ //FIXME: too many arguments
+ String ruleString = String.format("[%s] ||| [%s,1] <eps> ||| [%s,1] ||| ", goalNT, goalNT, defaultNT,
+ goalNT, defaultNT);
+
+ Rule rule = reader.parseLine(ruleString);
+ latticeGrammar.addRule(rule);
+ rule.estimateRuleCost(featureFunctions);
+
+ this.grammars.add(latticeGrammar);
+ }
+
+ /* Now create a feature function for each owner */
- HashSet<String> ownersSeen = new HashSet<String>();
++ final Set<OwnerId> ownersSeen = new HashSet<OwnerId>();
+
+ for (Grammar grammar: this.grammars) {
- String owner = Vocabulary.word(grammar.getOwner());
++ OwnerId owner = grammar.getOwner();
+ if (! ownersSeen.contains(owner)) {
- this.featureFunctions.add(new PhraseModel(weights, new String[] { "tm", "-owner", owner },
- joshuaConfiguration, grammar));
++ this.featureFunctions.add(
++ new PhraseModel(
++ weights, new String[] { "tm", "-owner", getOwner(owner) }, joshuaConfiguration, grammar));
+ ownersSeen.add(owner);
+ }
+ }
+
+ LOG.info("Memory used {} MB",
+ ((Runtime.getRuntime().totalMemory() - Runtime.getRuntime().freeMemory()) / 1000000.0));
+ }
+
+ /**
+ * Checks if multiple packedGrammars have the same vocabulary by comparing their vocabulary file checksums.
+ */
+ private static void checkSharedVocabularyChecksumsForPackedGrammars(final List<PackedGrammar> packed_grammars) {
+ String previous_checksum = "";
+ for (PackedGrammar grammar : packed_grammars) {
+ final String checksum = grammar.computeVocabularyChecksum();
+ if (previous_checksum.isEmpty()) {
+ previous_checksum = checksum;
+ } else {
+ if (!checksum.equals(previous_checksum)) {
+ throw new RuntimeException(
+ "Trying to load multiple packed grammars with different vocabularies!" +
+ "Have you packed them jointly?");
+ }
+ previous_checksum = checksum;
+ }
+ }
+ }
+
+ /*
+ * This function reads the weights for the model. Feature names and their weights are listed one
+ * per line in the following format:
+ *
+ * FEATURE_NAME WEIGHT
+ */
+ private void readWeights(String fileName) {
+ Decoder.weights = new FeatureVector();
+
+ if (fileName.equals(""))
+ return;
+
+ try {
+ LineReader lineReader = new LineReader(fileName);
+
+ for (String line : lineReader) {
+ line = line.replaceAll("\\s+", " ");
+
+ if (line.equals("") || line.startsWith("#") || line.startsWith("//")
+ || line.indexOf(' ') == -1)
+ continue;
+
+ String tokens[] = line.split("\\s+");
+ String feature = tokens[0];
+ Float value = Float.parseFloat(tokens[1]);
+
+ // Kludge for compatibility with Moses tuners
+ if (joshuaConfiguration.moses) {
+ feature = demoses(feature);
+ }
+
+ weights.increment(feature, value);
+ }
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+ LOG.info("Read {} weights from file '{}'", weights.size(), fileName);
+ }
+
+ private String demoses(String feature) {
+ if (feature.endsWith("="))
+ feature = feature.replace("=", "");
+ if (feature.equals("OOV_Penalty"))
+ feature = "OOVPenalty";
+ else if (feature.startsWith("tm-") || feature.startsWith("lm-"))
+ feature = feature.replace("-", "_");
+ return feature;
+ }
+
+ /**
+ * Feature functions are instantiated with a line of the form
+ *
+ * <pre>
+ * FEATURE OPTIONS
+ * </pre>
+ *
+ * Weights for features are listed separately.
+ *
+ * @throws IOException
+ *
+ */
+ private void initializeFeatureFunctions() throws IOException {
+
+ for (String featureLine : joshuaConfiguration.features) {
+ // line starts with NAME, followed by args
+ // 1. create new class named NAME, pass it config, weights, and the args
+
+ String fields[] = featureLine.split("\\s+");
+ String featureName = fields[0];
+
+ try {
+
+ Class<?> clas = getFeatureFunctionClass(featureName);
+ Constructor<?> constructor = clas.getConstructor(FeatureVector.class,
+ String[].class, JoshuaConfiguration.class);
+ FeatureFunction feature = (FeatureFunction) constructor.newInstance(weights, fields, joshuaConfiguration);
+ this.featureFunctions.add(feature);
+
+ } catch (Exception e) {
+ throw new RuntimeException(String.format("Unable to instantiate feature function '%s'!", featureLine), e);
+ }
+ }
+
+ for (FeatureFunction feature : featureFunctions) {
+ LOG.info("FEATURE: {}", feature.logString());
+ }
+
+ weights.registerDenseFeatures(featureFunctions);
+ }
+
+ /**
+ * Searches a list of predefined paths for classes, and returns the first one found. Meant for
+ * instantiating feature functions.
+ *
+ * @param name
+ * @return the class, found in one of the search paths
+ * @throws ClassNotFoundException
+ */
+ private Class<?> getFeatureFunctionClass(String featureName) {
+ Class<?> clas = null;
+
+ String[] packages = { "org.apache.joshua.decoder.ff", "org.apache.joshua.decoder.ff.lm", "org.apache.joshua.decoder.ff.phrase" };
+ for (String path : packages) {
+ try {
+ clas = Class.forName(String.format("%s.%s", path, featureName));
+ break;
+ } catch (ClassNotFoundException e) {
+ try {
+ clas = Class.forName(String.format("%s.%sFF", path, featureName));
+ break;
+ } catch (ClassNotFoundException e2) {
+ // do nothing
+ }
+ }
+ }
+ return clas;
+ }
+
+ /**
+ * Adds a rule to the custom grammar.
+ *
+ * @param rule the rule to add
+ */
+ public void addCustomRule(Rule rule) {
+ customPhraseTable.addRule(rule);
+ rule.estimateRuleCost(featureFunctions);
+ }
+
+ public Grammar getCustomPhraseTable() {
+ return customPhraseTable;
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslation.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslation.java
index 0000000,2faacf2..b44a7f0
mode 000000,100644..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslation.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslation.java
@@@ -1,0 -1,164 +1,157 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.decoder;
+
+ import java.util.List;
+ import java.util.Map;
+
+ import org.apache.joshua.decoder.ff.FeatureVector;
+ import org.apache.joshua.decoder.hypergraph.KBestExtractor.DerivationState;
+ import org.apache.joshua.decoder.io.DeNormalize;
+ import org.apache.joshua.decoder.segment_file.Sentence;
+ import org.apache.joshua.decoder.segment_file.Token;
+ import org.apache.joshua.util.FormatUtils;
+
+ /**
+ * A StructuredTranslation instance provides a more structured access to
+ * translation results than the string-based Translation class.
+ * This is useful if the decoder is encapsulated in a larger project, instead
+ * of simply writing to a file or stdout.
+ * StructuredTranslation encodes all relevant information about a derivation,
+ * namely output string, tokens, score, features, and word alignment.
+ *
+ * @author fhieber
+ */
+ public class StructuredTranslation {
+
+ private final Sentence sourceSentence;
+ private final String translationString;
+ private final List<String> translationTokens;
+ private final float translationScore;
+ private final List<List<Integer>> translationWordAlignments;
- private final FeatureVector translationFeatures;
++ private final Map<String,Float> translationFeatures;
+ private final float extractionTime;
+
+ public StructuredTranslation(
+ final Sentence sourceSentence,
+ final String translationString,
+ final List<String> translationTokens,
+ final float translationScore,
+ final List<List<Integer>> translationWordAlignments,
- final FeatureVector translationFeatures,
++ final Map<String,Float> translationFeatures,
+ final float extractionTime) {
+ this.sourceSentence = sourceSentence;
+ this.translationString = translationString;
+ this.translationTokens = translationTokens;
+ this.translationScore = translationScore;
+ this.translationWordAlignments = translationWordAlignments;
+ this.translationFeatures = translationFeatures;
+ this.extractionTime = extractionTime;
+ }
+
+ public Sentence getSourceSentence() {
+ return sourceSentence;
+ }
+
+ public int getSentenceId() {
+ return sourceSentence.id();
+ }
+
+ /**
+ * Produces the raw translation hypothesis (still tokenized).
+ *
+ * @return the raw translation hypothesis
+ */
+ public String getTranslationString() {
+ return translationString;
+ }
+
+ /**
- * Produces the translation formatted according to the value of {@value JoshuaConfiguration.output_format}.
- * Also includes formatting options such as {@value JoshuaConfiguration.project_case}.
++ * Returns the output string formatted according to JoshuaConfiguration.output_format.
+ *
- * @return
++ * @return the formatted string
+ */
+ public String getFormattedTranslationString() {
- JoshuaConfiguration config = sourceSentence.config;
- String outputString = config.outputFormat
- .replace("%s", getTranslationString())
- .replace("%S", DeNormalize.processSingleLine(maybeProjectCase(getTranslationString())))
- .replace("%i", Integer.toString(getSentenceId()))
- .replace("%f", config.moses ? translationFeatures.mosesString() : translationFeatures.toString())
- .replace("%c", String.format("%.3f", getTranslationScore()));
- return outputString;
++ throw new RuntimeException("Not yet implemented");
+ }
+
+ public List<String> getTranslationTokens() {
+ return translationTokens;
+ }
+
+ public float getTranslationScore() {
+ return translationScore;
+ }
+
+ /**
+ * Returns a list of target to source alignments.
++ *
+ * @return a list of target to source alignments
+ */
+ public List<List<Integer>> getTranslationWordAlignments() {
+ return translationWordAlignments;
+ }
+
+ public Map<String,Float> getTranslationFeatures() {
- return translationFeatures.getMap();
++ return translationFeatures;
+ }
+
+ /**
+ * Time taken to build output information from the hypergraph.
+ * @return the time taken to build output information from the hypergraph
+ */
+ public Float getExtractionTime() {
+ return extractionTime;
+ }
+
+ /**
+ * If requested, projects source-side lettercase to target, and appends the alignment from
+ * to the source-side sentence in ||s.
+ *
+ * @param hypothesis todo
+ * @param state todo
+ * @return source-side lettercase to target, and appends the alignment from to the source-side sentence in ||s
+ */
+ private String maybeProjectCase(String hypothesis) {
+ String output = hypothesis;
+
+ JoshuaConfiguration config = sourceSentence.config;
+ if (config.project_case) {
+ String[] tokens = hypothesis.split("\\s+");
+ List<List<Integer>> points = getTranslationWordAlignments();
+ for (int i = 0; i < points.size(); i++) {
+ List<Integer> target = points.get(i);
+ for (int source: target) {
+ Token token = sourceSentence.getTokens().get(source + 1); // skip <s>
+ String annotation = "";
+ if (token != null && token.getAnnotation("lettercase") != null)
+ annotation = token.getAnnotation("lettercase");
+ if (source != 0 && annotation.equals("upper"))
+ tokens[i] = FormatUtils.capitalize(tokens[i]);
+ else if (annotation.equals("all-upper"))
+ tokens[i] = tokens[i].toUpperCase();
+ }
+ }
+
+ output = String.join(" ", tokens);
+ }
+
+ return output;
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslationFactory.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslationFactory.java
index 0000000,4389135..921771e
mode 000000,100644..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslationFactory.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslationFactory.java
@@@ -1,0 -1,117 +1,120 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.decoder;
+
+ import static java.util.Arrays.asList;
+ import static java.util.Collections.emptyList;
+ import static java.util.Collections.emptyMap;
+ import static org.apache.joshua.decoder.hypergraph.ViterbiExtractor.getViterbiFeatures;
+ import static org.apache.joshua.decoder.hypergraph.ViterbiExtractor.getViterbiString;
+ import static org.apache.joshua.decoder.hypergraph.ViterbiExtractor.getViterbiWordAlignmentList;
+ import static org.apache.joshua.util.FormatUtils.removeSentenceMarkers;
+
+ import java.util.List;
+
+ import org.apache.joshua.decoder.ff.FeatureFunction;
+ import org.apache.joshua.decoder.ff.FeatureVector;
+ import org.apache.joshua.decoder.hypergraph.HyperGraph;
+ import org.apache.joshua.decoder.hypergraph.KBestExtractor.DerivationState;
+ import org.apache.joshua.decoder.segment_file.Sentence;
+
+ /**
+ * This factory provides methods to create StructuredTranslation objects
+ * from either Viterbi derivations or KBest derivations.
+ *
+ * @author fhieber
+ */
+ public class StructuredTranslationFactory {
+
+ /**
- * Returns a StructuredTranslation instance from the Viterbi derivation.
++ * Returns a StructuredTranslation instance from the Viterbi derivation. This is used to
++ * obtain the 1-best hypothesis, since traversing the Viterbi derivation directly on the
++ * hypergraph is considerably faster than instantiating the k-best apparatus.
+ *
+ * @param sourceSentence the source sentence
+ * @param hypergraph the hypergraph object
+ * @param featureFunctions the list of active feature functions
+ * @return A StructuredTranslation object representing the Viterbi derivation.
+ */
+ public static StructuredTranslation fromViterbiDerivation(
+ final Sentence sourceSentence,
+ final HyperGraph hypergraph,
+ final List<FeatureFunction> featureFunctions) {
+ final long startTime = System.currentTimeMillis();
+ final String translationString = removeSentenceMarkers(getViterbiString(hypergraph));
+ return new StructuredTranslation(
+ sourceSentence,
+ translationString,
+ extractTranslationTokens(translationString),
+ extractTranslationScore(hypergraph),
+ getViterbiWordAlignmentList(hypergraph),
- getViterbiFeatures(hypergraph, featureFunctions, sourceSentence),
++ getViterbiFeatures(hypergraph, featureFunctions, sourceSentence).getMap(),
+ (System.currentTimeMillis() - startTime) / 1000.0f);
+ }
+
+ /**
+ * Returns a StructuredTranslation from an empty decoder output
+ * @param sourceSentence the source sentence
+ * @return a StructuredTranslation object
+ */
+ public static StructuredTranslation fromEmptyOutput(final Sentence sourceSentence) {
+ return new StructuredTranslation(
- sourceSentence, "", emptyList(), 0, emptyList(), new FeatureVector(), 0f);
++ sourceSentence, "", emptyList(), 0, emptyList(), emptyMap(), 0f);
+ }
+
+ /**
+ * Returns a StructuredTranslation instance from a KBest DerivationState.
++ *
+ * @param sourceSentence Sentence object representing the source.
+ * @param derivationState the KBest DerivationState.
+ * @return A StructuredTranslation object representing the derivation encoded by derivationState.
+ */
+ public static StructuredTranslation fromKBestDerivation(
+ final Sentence sourceSentence,
+ final DerivationState derivationState) {
+ final long startTime = System.currentTimeMillis();
+ final String translationString = removeSentenceMarkers(derivationState.getHypothesis());
+ return new StructuredTranslation(
+ sourceSentence,
+ translationString,
+ extractTranslationTokens(translationString),
+ derivationState.getModelCost(),
+ derivationState.getWordAlignmentList(),
- derivationState.getFeatures(),
++ derivationState.getFeatures().getMap(),
+ (System.currentTimeMillis() - startTime) / 1000.0f);
+ }
+
+ private static float extractTranslationScore(final HyperGraph hypergraph) {
+ if (hypergraph == null) {
+ return 0;
+ } else {
+ return hypergraph.goalNode.getScore();
+ }
+ }
+
+ private static List<String> extractTranslationTokens(final String translationString) {
+ if (translationString.isEmpty()) {
+ return emptyList();
+ } else {
+ return asList(translationString.split("\\s+"));
+ }
+ }
+
+
+ }
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/Chart.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/Chart.java
index 0000000,d0cd96b..355a6f1
mode 000000,100644..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/Chart.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/Chart.java
@@@ -1,0 -1,746 +1,746 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.decoder.chart_parser;
+
+ import java.util.ArrayList;
+ import java.util.Arrays;
+ import java.util.HashSet;
+ import java.util.List;
+ import java.util.PriorityQueue;
+
+ import org.apache.joshua.corpus.Vocabulary;
+ import org.apache.joshua.decoder.JoshuaConfiguration;
+ import org.apache.joshua.decoder.chart_parser.DotChart.DotNode;
+ import org.apache.joshua.decoder.ff.FeatureFunction;
+ import org.apache.joshua.decoder.ff.SourceDependentFF;
+ import org.apache.joshua.decoder.ff.tm.AbstractGrammar;
+ import org.apache.joshua.decoder.ff.tm.Grammar;
+ import org.apache.joshua.decoder.ff.tm.Rule;
+ import org.apache.joshua.decoder.ff.tm.RuleCollection;
+ import org.apache.joshua.decoder.ff.tm.Trie;
+ import org.apache.joshua.decoder.ff.tm.hash_based.MemoryBasedBatchGrammar;
+ import org.apache.joshua.decoder.hypergraph.HGNode;
+ import org.apache.joshua.decoder.hypergraph.HyperGraph;
+ import org.apache.joshua.decoder.segment_file.Sentence;
+ import org.apache.joshua.decoder.segment_file.Token;
+ import org.apache.joshua.lattice.Arc;
+ import org.apache.joshua.lattice.Lattice;
+ import org.apache.joshua.lattice.Node;
+ import org.apache.joshua.util.ChartSpan;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
+ /**
+ * Chart class this class implements chart-parsing: (1) seeding the chart (2)
+ * cky main loop over bins, (3) identify applicable rules in each bin
+ *
+ * Note: the combination operation will be done in Cell
+ *
+ * Signatures of class: Cell: i, j SuperNode (used for CKY check): i,j, lhs
+ * HGNode ("or" node): i,j, lhs, edge ngrams HyperEdge ("and" node)
+ *
+ * index of sentences: start from zero index of cell: cell (i,j) represent span
+ * of words indexed [i,j-1] where i is in [0,n-1] and j is in [1,n]
+ *
+ * @author Zhifei Li, zhifei.work@gmail.com
+ * @author Matt Post post@cs.jhu.edu
+ */
+
+ public class Chart {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Chart.class);
+ private final JoshuaConfiguration config;
+ // ===========================================================
+ // Statistics
+ // ===========================================================
+
+ /**
+ * how many items have been pruned away because its cost is greater than the
+ * cutoff in calling chart.add_deduction_in_chart()
+ */
+ int nMerged = 0;
+ int nAdded = 0;
+ int nDotitemAdded = 0; // note: there is no pruning in dot-item
+
+ public Sentence getSentence() {
+ return this.sentence;
+ }
+
+ // ===============================================================
+ // Private instance fields (maybe could be protected instead)
+ // ===============================================================
+ private ChartSpan<Cell> cells; // note that in some cell, it might be null
+ private int sourceLength;
+ private List<FeatureFunction> featureFunctions;
+ private Grammar[] grammars;
+ private DotChart[] dotcharts; // each grammar should have a dotchart associated with it
+ private Cell goalBin;
+ private int goalSymbolID = -1;
+ private Lattice<Token> inputLattice;
+
+ private Sentence sentence = null;
+ // private SyntaxTree parseTree;
+ private StateConstraint stateConstraint;
+
+
+ // ===============================================================
+ // Constructors
+ // ===============================================================
+
+ /*
+ * TODO: Once the Segment interface is adjusted to provide a Lattice<String>
+ * for the sentence() method, we should just accept a Segment instead of the
+ * sentence, segmentID, and constraintSpans parameters. We have the symbol
+ * table already, so we can do the integerization here instead of in
+ * DecoderThread. GrammarFactory.getGrammarForSentence will want the
+ * integerized sentence as well, but then we'll need to adjust that interface
+ * to deal with (non-trivial) lattices too. Of course, we get passed the
+ * grammars too so we could move all of that into here.
+ */
+
+ public Chart(Sentence sentence, List<FeatureFunction> featureFunctions, Grammar[] grammars,
+ String goalSymbol, JoshuaConfiguration config) {
+ this.config = config;
+ this.inputLattice = sentence.getLattice();
+ this.sourceLength = inputLattice.size() - 1;
+ this.featureFunctions = featureFunctions;
+
+ this.sentence = sentence;
+
+ // TODO: OOV handling no longer handles parse tree input (removed after
+ // commit 748eb69714b26dd67cba8e7c25a294347603bede)
+ // this.parseTree = null;
+ // if (sentence instanceof ParsedSentence)
+ // this.parseTree = ((ParsedSentence) sentence).syntaxTree();
+ //
+ this.cells = new ChartSpan<Cell>(sourceLength, null);
+
+ this.goalSymbolID = Vocabulary.id(goalSymbol);
+ this.goalBin = new Cell(this, this.goalSymbolID);
+
+ /* Create the grammars, leaving space for the OOV grammar. */
+ this.grammars = new Grammar[grammars.length + 1];
+ for (int i = 0; i < grammars.length; i++)
+ this.grammars[i + 1] = grammars[i];
+
- MemoryBasedBatchGrammar oovGrammar = new MemoryBasedBatchGrammar("oov", this.config);
++ MemoryBasedBatchGrammar oovGrammar = new MemoryBasedBatchGrammar("oov", this.config, 20);
+ AbstractGrammar.addOOVRules(oovGrammar, sentence.getLattice(), featureFunctions,
+ this.config.true_oovs_only);
+ this.grammars[0] = oovGrammar;
+
+ // each grammar will have a dot chart
+ this.dotcharts = new DotChart[this.grammars.length];
+ for (int i = 0; i < this.grammars.length; i++)
+ this.dotcharts[i] = new DotChart(this.inputLattice, this.grammars[i], this);
+
+ // Begin to do initialization work
+
+ stateConstraint = null;
+ if (sentence.target() != null)
+ // stateConstraint = new StateConstraint(sentence.target());
+ stateConstraint = new StateConstraint(Vocabulary.START_SYM + " " + sentence.target() + " "
+ + Vocabulary.STOP_SYM);
+
+ /* Find the SourceDependent feature and give it access to the sentence. */
+ for (FeatureFunction ff : this.featureFunctions)
+ if (ff instanceof SourceDependentFF)
+ ((SourceDependentFF) ff).setSource(sentence);
+
+ LOG.debug("Finished seeding chart.");
+ }
+
+ /**
+ * Manually set the goal symbol ID. The constructor expects a String
+ * representing the goal symbol, but there may be time (say, for example, in
+ * the second pass of a synchronous parse) where we want to set the goal
+ * symbol to a particular ID (regardless of String representation).
+ * <p>
+ * This method should be called before expanding the chart, as chart expansion
+ * depends on the goal symbol ID.
+ *
+ * @param i the id of the goal symbol to use
+ */
+ public void setGoalSymbolID(int i) {
+ this.goalSymbolID = i;
+ this.goalBin = new Cell(this, i);
+ return;
+ }
+
+ // ===============================================================
+ // The primary method for filling in the chart
+ // ===============================================================
+
+ /**
+ * Construct the hypergraph with the help from DotChart using cube pruning.
+ * Cube pruning occurs at the span level, with all completed rules from the
+ * dot chart competing against each other; that is, rules with different
+ * source sides *and* rules sharing a source side but with different target
+ * sides are all in competition with each other.
+ *
+ * Terminal rules are added to the chart directly.
+ *
+ * Rules with nonterminals are added to the list of candidates. The candidates
+ * list is seeded with the list of all rules and, for each nonterminal in the
+ * rule, the 1-best tail node for that nonterminal and subspan. If the maximum
+ * arity of a rule is R, then the dimension of the hypercube is R + 1, since
+ * the first dimension is used to record the rule.
+ */
+ private void completeSpan(int i, int j) {
+
+ /* STEP 1: create the heap, and seed it with all of the candidate states */
+ PriorityQueue<CubePruneState> candidates = new PriorityQueue<CubePruneState>();
+
+ /*
+ * Look at all the grammars, seeding the chart with completed rules from the
+ * DotChart
+ */
+ for (int g = 0; g < grammars.length; g++) {
+ if (!grammars[g].hasRuleForSpan(i, j, inputLattice.distance(i, j))
+ || null == dotcharts[g].getDotCell(i, j))
+ continue;
+
+ // for each rule with applicable rules
+ for (DotNode dotNode : dotcharts[g].getDotCell(i, j).getDotNodes()) {
+ RuleCollection ruleCollection = dotNode.getRuleCollection();
+ if (ruleCollection == null)
+ continue;
+
+ List<Rule> rules = ruleCollection.getSortedRules(this.featureFunctions);
+ SourcePath sourcePath = dotNode.getSourcePath();
+
+ if (null == rules || rules.size() == 0)
+ continue;
+
+ if (ruleCollection.getArity() == 0) {
+ /*
+ * The total number of arity-0 items (pre-terminal rules) that we add
+ * is controlled by num_translation_options in the configuration.
+ *
+ * We limit the translation options per DotNode; that is, per LHS.
+ */
+ int numTranslationsAdded = 0;
+
+ /* Terminal productions are added directly to the chart */
+ for (Rule rule : rules) {
+
+ if (config.num_translation_options > 0
+ && numTranslationsAdded >= config.num_translation_options) {
+ break;
+ }
+
+ ComputeNodeResult result = new ComputeNodeResult(this.featureFunctions, rule, null, i,
+ j, sourcePath, this.sentence);
+
+ if (stateConstraint == null || stateConstraint.isLegal(result.getDPStates())) {
+ getCell(i, j).addHyperEdgeInCell(result, rule, i, j, null, sourcePath, true);
+ numTranslationsAdded++;
+ }
+ }
+ } else {
+ /* Productions with rank > 0 are subject to cube pruning */
+
+ Rule bestRule = rules.get(0);
+
+ List<HGNode> currentTailNodes = new ArrayList<HGNode>();
+ List<SuperNode> superNodes = dotNode.getAntSuperNodes();
+ for (SuperNode si : superNodes) {
+ currentTailNodes.add(si.nodes.get(0));
+ }
+
+ /*
+ * `ranks` records the current position in the cube. the 0th index is
+ * the rule, and the remaining indices 1..N correspond to the tail
+ * nodes (= nonterminals in the rule). These tail nodes are
+ * represented by SuperNodes, which group together items with the same
+ * nonterminal but different DP state (e.g., language model state)
+ */
+ int[] ranks = new int[1 + superNodes.size()];
+ Arrays.fill(ranks, 1);
+
+ ComputeNodeResult result = new ComputeNodeResult(featureFunctions, bestRule,
+ currentTailNodes, i, j, sourcePath, sentence);
+ CubePruneState bestState = new CubePruneState(result, ranks, rules, currentTailNodes,
+ dotNode);
+ candidates.add(bestState);
+ }
+ }
+ }
+
+ applyCubePruning(i, j, candidates);
+ }
+
+ /**
+ * Applies cube pruning over a span.
+ *
+ * @param i
+ * @param j
+ * @param stateConstraint
+ * @param candidates
+ */
+ private void applyCubePruning(int i, int j, PriorityQueue<CubePruneState> candidates) {
+
+ // System.err.println(String.format("CUBEPRUNE: %d-%d with %d candidates",
+ // i, j, candidates.size()));
+ // for (CubePruneState cand: candidates) {
+ // System.err.println(String.format(" CAND " + cand));
+ // }
+
+ /*
+ * There are multiple ways to reach each point in the cube, so short-circuit
+ * that.
+ */
+ HashSet<CubePruneState> visitedStates = new HashSet<CubePruneState>();
+
+ int popLimit = config.pop_limit;
+ int popCount = 0;
+ while (candidates.size() > 0 && ((++popCount <= popLimit) || popLimit == 0)) {
+ CubePruneState state = candidates.poll();
+
+ DotNode dotNode = state.getDotNode();
+ List<Rule> rules = state.rules;
+ SourcePath sourcePath = dotNode.getSourcePath();
+ List<SuperNode> superNodes = dotNode.getAntSuperNodes();
+
+ /*
+ * Add the hypothesis to the chart. This can only happen if (a) we're not
+ * doing constrained decoding or (b) we are and the state is legal.
+ */
+ if (stateConstraint == null || stateConstraint.isLegal(state.getDPStates())) {
+ getCell(i, j).addHyperEdgeInCell(state.computeNodeResult, state.getRule(), i, j,
+ state.antNodes, sourcePath, true);
+ }
+
+ /*
+ * Expand the hypothesis by walking down a step along each dimension of
+ * the cube, in turn. k = 0 means we extend the rule being used; k > 0
+ * expands the corresponding tail node.
+ */
+
+ for (int k = 0; k < state.ranks.length; k++) {
+
+ /* Copy the current ranks, then extend the one we're looking at. */
+ int[] nextRanks = new int[state.ranks.length];
+ System.arraycopy(state.ranks, 0, nextRanks, 0, state.ranks.length);
+ nextRanks[k]++;
+
+ /*
+ * We might have reached the end of something (list of rules or tail
+ * nodes)
+ */
+ if (k == 0
+ && (nextRanks[k] > rules.size() || (config.num_translation_options > 0 && nextRanks[k] > config.num_translation_options)))
+ continue;
+ else if ((k != 0 && nextRanks[k] > superNodes.get(k - 1).nodes.size()))
+ continue;
+
+ /* Use the updated ranks to assign the next rule and tail node. */
+ Rule nextRule = rules.get(nextRanks[0] - 1);
+ // HGNode[] nextAntNodes = new HGNode[state.antNodes.size()];
+ List<HGNode> nextAntNodes = new ArrayList<HGNode>(state.antNodes.size());
+ for (int x = 0; x < state.ranks.length - 1; x++)
+ nextAntNodes.add(superNodes.get(x).nodes.get(nextRanks[x + 1] - 1));
+
+ /* Create the next state. */
+ CubePruneState nextState = new CubePruneState(new ComputeNodeResult(featureFunctions,
+ nextRule, nextAntNodes, i, j, sourcePath, this.sentence), nextRanks, rules,
+ nextAntNodes, dotNode);
+
+ /* Skip states that have been explored before. */
+ if (visitedStates.contains(nextState))
+ continue;
+
+ visitedStates.add(nextState);
+ candidates.add(nextState);
+ }
+ }
+ }
+
+ /* Create a priority queue of candidates for each span under consideration */
+ private PriorityQueue<CubePruneState>[] allCandidates;
+
+ private ArrayList<SuperNode> nodeStack;
+
+ /**
+ * Translates the sentence using the CKY+ variation proposed in
+ * "A CYK+ Variant for SCFG Decoding Without A Dot Chart" (Sennrich, SSST
+ * 2014).
+ */
+ private int i = -1;
+
+ public HyperGraph expandSansDotChart() {
+ for (i = sourceLength - 1; i >= 0; i--) {
+ allCandidates = new PriorityQueue[sourceLength - i + 2];
+ for (int id = 0; id < allCandidates.length; id++)
+ allCandidates[id] = new PriorityQueue<CubePruneState>();
+
+ nodeStack = new ArrayList<SuperNode>();
+
+ for (int j = i + 1; j <= sourceLength; j++) {
+ if (!sentence.hasPath(i, j))
+ continue;
+
+ for (int g = 0; g < this.grammars.length; g++) {
+ // System.err.println(String.format("\n*** I=%d J=%d GRAMMAR=%d", i, j, g));
+
+ if (j == i + 1) {
+ /* Handle terminals */
+ Node<Token> node = sentence.getNode(i);
+ for (Arc<Token> arc : node.getOutgoingArcs()) {
+ int word = arc.getLabel().getWord();
+ // disallow lattice decoding for now
+ assert arc.getHead().id() == j;
+ Trie trie = this.grammars[g].getTrieRoot().match(word);
+ if (trie != null && trie.hasRules())
+ addToChart(trie, j, false);
+ }
+ } else {
+ /* Recurse for non-terminal case */
+ consume(this.grammars[g].getTrieRoot(), i, j - 1);
+ }
+ }
+
+ // Now that we've accumulated all the candidates, apply cube pruning
+ applyCubePruning(i, j, allCandidates[j - i]);
+
+ // Add unary nodes
+ addUnaryNodes(this.grammars, i, j);
+ }
+ }
+
+ // transition_final: setup a goal item, which may have many deductions
+ if (null == this.cells.get(0, sourceLength)
+ || !this.goalBin.transitToGoal(this.cells.get(0, sourceLength), this.featureFunctions,
+ this.sourceLength)) {
+ LOG.warn("Input {}: Parse failure (either no derivations exist or pruning is too aggressive",
+ sentence.id());
+ return null;
+ }
+
+ return new HyperGraph(this.goalBin.getSortedNodes().get(0), -1, -1, this.sentence);
+ }
+
+ /**
+ * Recursively consumes the trie, following input nodes, finding applicable
+ * rules and adding them to bins for each span for later cube pruning.
+ *
+ * @param dotNode data structure containing information about what's been
+ * already matched
+ * @param l extension point we're looking at
+ *
+ */
+ private void consume(Trie trie, int j, int l) {
+ /*
+ * 1. If the trie node has any rules, we can add them to the collection
+ *
+ * 2. Next, look at all the outgoing nonterminal arcs of the trie node. If
+ * any of them match an existing chart item, then we know we can extend
+ * (i,j) to (i,l). We then recurse for all m from l+1 to n (the end of the
+ * sentence)
+ *
+ * 3. We also try to match terminals if (j + 1 == l)
+ */
+
+ // System.err.println(String.format("CONSUME %s / %d %d %d", dotNode,
+ // dotNode.begin(), dotNode.end(), l));
+
+ // Try to match terminals
+ if (inputLattice.distance(j, l) == 1) {
+ // Get the current sentence node, and explore all outgoing arcs, since we
+ // might be decoding
+ // a lattice. For sentence decoding, this is trivial: there is only one
+ // outgoing arc.
+ Node<Token> inputNode = sentence.getNode(j);
+ for (Arc<Token> arc : inputNode.getOutgoingArcs()) {
+ int word = arc.getLabel().getWord();
+ Trie nextTrie;
+ if ((nextTrie = trie.match(word)) != null) {
+ // add to chart item over (i, l)
+ addToChart(nextTrie, arc.getHead().id(), i == j);
+ }
+ }
+ }
+
+ // Now try to match nonterminals
+ Cell cell = cells.get(j, l);
+ if (cell != null) {
+ for (int id : cell.getKeySet()) { // for each supernode (lhs), see if you
+ // can match a trie
+ Trie nextTrie = trie.match(id);
+ if (nextTrie != null) {
+ SuperNode superNode = cell.getSuperNode(id);
+ nodeStack.add(superNode);
+ addToChart(nextTrie, superNode.end(), i == j);
+ nodeStack.remove(nodeStack.size() - 1);
+ }
+ }
+ }
+ }
+
+ /**
+ * Adds all rules at a trie node to the chart, unless its a unary rule. A
+ * unary rule is the first outgoing arc of a grammar's root trie. For
+ * terminals, these are added during the seeding stage; for nonterminals,
+ * these confuse cube pruning and can result in infinite loops, and are
+ * handled separately (see addUnaryNodes());
+ *
+ * @param trie the grammar node
+ * @param isUnary whether the rules at this dotnode are unary
+ */
+ private void addToChart(Trie trie, int j, boolean isUnary) {
+
+ // System.err.println(String.format("ADD TO CHART %s unary=%s", dotNode,
+ // isUnary));
+
+ if (!isUnary && trie.hasRules()) {
+ DotNode dotNode = new DotNode(i, j, trie, new ArrayList<SuperNode>(nodeStack), null);
+
+ addToCandidates(dotNode);
+ }
+
+ for (int l = j + 1; l <= sentence.length(); l++)
+ consume(trie, j, l);
+ }
+
+ /**
+ * Record the completed rule with backpointers for later cube-pruning.
+ *
+ * @param width
+ * @param rules
+ * @param tailNodes
+ */
+ private void addToCandidates(DotNode dotNode) {
+ // System.err.println(String.format("ADD TO CANDIDATES %s AT INDEX %d",
+ // dotNode, dotNode.end() - dotNode.begin()));
+
+ // TODO: one entry per rule, or per rule instantiation (rule together with
+ // unique matching of input)?
+ List<Rule> rules = dotNode.getRuleCollection().getSortedRules(featureFunctions);
+ Rule bestRule = rules.get(0);
+ List<SuperNode> superNodes = dotNode.getAntSuperNodes();
+
+ List<HGNode> tailNodes = new ArrayList<HGNode>();
+ for (SuperNode superNode : superNodes)
+ tailNodes.add(superNode.nodes.get(0));
+
+ int[] ranks = new int[1 + superNodes.size()];
+ Arrays.fill(ranks, 1);
+
+ ComputeNodeResult result = new ComputeNodeResult(featureFunctions, bestRule, tailNodes,
+ dotNode.begin(), dotNode.end(), dotNode.getSourcePath(), sentence);
+ CubePruneState seedState = new CubePruneState(result, ranks, rules, tailNodes, dotNode);
+
+ allCandidates[dotNode.end() - dotNode.begin()].add(seedState);
+ }
+
+ /**
+ * This function performs the main work of decoding.
+ *
+ * @return the hypergraph containing the translated sentence.
+ */
+ public HyperGraph expand() {
+
+ for (int width = 1; width <= sourceLength; width++) {
+ for (int i = 0; i <= sourceLength - width; i++) {
+ int j = i + width;
+ if (LOG.isDebugEnabled())
+ LOG.debug("Processing span ({}, {})", i, j);
+
+ /* Skips spans for which no path exists (possible in lattices). */
+ if (inputLattice.distance(i, j) == Float.POSITIVE_INFINITY) {
+ continue;
+ }
+
+ /*
+ * 1. Expand the dot through all rules. This is a matter of (a) look for
+ * rules over (i,j-1) that need the terminal at (j-1,j) and looking at
+ * all split points k to expand nonterminals.
+ */
+ if (LOG.isDebugEnabled())
+ LOG.debug("Expanding cell");
+ for (int k = 0; k < this.grammars.length; k++) {
+ /**
+ * Each dotChart can act individually (without consulting other
+ * dotCharts) because it either consumes the source input or the
+ * complete nonTerminals, which are both grammar-independent.
+ **/
+ this.dotcharts[k].expandDotCell(i, j);
+ }
+
+ /*
+ * 2. The regular CKY part: add completed items onto the chart via cube
+ * pruning.
+ */
+ if (LOG.isDebugEnabled())
+ LOG.debug("Adding complete items into chart");
+ completeSpan(i, j);
+
+ /* 3. Process unary rules. */
+ if (LOG.isDebugEnabled())
+ LOG.debug("Adding unary items into chart");
+ addUnaryNodes(this.grammars, i, j);
+
+ // (4)=== in dot_cell(i,j), add dot-nodes that start from the /complete/
+ // superIterms in
+ // chart_cell(i,j)
+ if (LOG.isDebugEnabled())
+ LOG.debug("Initializing new dot-items that start from complete items in this cell");
+ for (int k = 0; k < this.grammars.length; k++) {
+ if (this.grammars[k].hasRuleForSpan(i, j, inputLattice.distance(i, j))) {
+ this.dotcharts[k].startDotItems(i, j);
+ }
+ }
+
+ /*
+ * 5. Sort the nodes in the cell.
+ *
+ * Sort the nodes in this span, to make them usable for future
+ * applications of cube pruning.
+ */
+ if (null != this.cells.get(i, j)) {
+ this.cells.get(i, j).getSortedNodes();
+ }
+ }
+ }
+
+ logStatistics();
+
+ // transition_final: setup a goal item, which may have many deductions
+ if (null == this.cells.get(0, sourceLength)
+ || !this.goalBin.transitToGoal(this.cells.get(0, sourceLength), this.featureFunctions,
+ this.sourceLength)) {
+ LOG.warn("Input {}: Parse failure (either no derivations exist or pruning is too aggressive",
+ sentence.id());
+ return null;
+ }
+
+ if (LOG.isDebugEnabled())
+ LOG.debug("Finished expand");
+ return new HyperGraph(this.goalBin.getSortedNodes().get(0), -1, -1, this.sentence);
+ }
+
+ /**
+ * Get the requested cell, creating the entry if it doesn't already exist.
+ *
+ * @param i span start
+ * @param j span end
+ * @return the cell item
+ */
+ public Cell getCell(int i, int j) {
+ assert i >= 0;
+ assert i <= sentence.length();
+ assert i <= j;
+ if (cells.get(i, j) == null)
+ cells.set(i, j, new Cell(this, goalSymbolID));
+
+ return cells.get(i, j);
+ }
+
+ // ===============================================================
+ // Private methods
+ // ===============================================================
+
+ private void logStatistics() {
+ if (LOG.isDebugEnabled())
+ LOG.debug("Input {}: Chart: added {} merged {} dot-items added: {}",
+ this.sentence.id(), this.nAdded, this.nMerged, this.nDotitemAdded);
+ }
+
+ /**
+ * Handles expansion of unary rules. Rules are expanded in an agenda-based
+ * manner to avoid constructing infinite unary chains. Assumes a triangle
+ * inequality of unary rule expansion (e.g., A -> B will always be cheaper
+ * than A -> C -> B), which is not a true assumption.
+ *
+ * @param grammars A list of the grammars for the sentence
+ * @param i
+ * @param j
+ * @return the number of nodes added
+ */
+ private int addUnaryNodes(Grammar[] grammars, int i, int j) {
+
+ Cell chartBin = this.cells.get(i, j);
+ if (null == chartBin) {
+ return 0;
+ }
+ int qtyAdditionsToQueue = 0;
+ ArrayList<HGNode> queue = new ArrayList<HGNode>(chartBin.getSortedNodes());
+ HashSet<Integer> seen_lhs = new HashSet<Integer>();
+
+ if (LOG.isDebugEnabled())
+ LOG.debug("Adding unary to [{}, {}]", i, j);
+
+ while (queue.size() > 0) {
+ HGNode node = queue.remove(0);
+ seen_lhs.add(node.lhs);
+
+ for (Grammar gr : grammars) {
+ if (!gr.hasRuleForSpan(i, j, inputLattice.distance(i, j)))
+ continue;
+
+ /*
+ * Match against the node's LHS, and then make sure the rule collection
+ * has unary rules
+ */
+ Trie childNode = gr.getTrieRoot().match(node.lhs);
+ if (childNode != null && childNode.getRuleCollection() != null
+ && childNode.getRuleCollection().getArity() == 1) {
+
+ ArrayList<HGNode> antecedents = new ArrayList<HGNode>();
+ antecedents.add(node);
+
+ List<Rule> rules = childNode.getRuleCollection().getSortedRules(this.featureFunctions);
+ for (Rule rule : rules) { // for each unary rules
+
+ ComputeNodeResult states = new ComputeNodeResult(this.featureFunctions, rule,
+ antecedents, i, j, new SourcePath(), this.sentence);
+ HGNode resNode = chartBin.addHyperEdgeInCell(states, rule, i, j, antecedents,
+ new SourcePath(), true);
+
+ if (LOG.isDebugEnabled())
+ LOG.debug("{}", rule);
+ if (null != resNode && !seen_lhs.contains(resNode.lhs)) {
+ queue.add(resNode);
+ qtyAdditionsToQueue++;
+ }
+ }
+ }
+ }
+ }
+ return qtyAdditionsToQueue;
+ }
+
+ /***
+ * Add a terminal production (X -> english phrase) to the hypergraph.
+ *
+ * @param i the start index
+ * @param j stop index
+ * @param rule the terminal rule applied
+ * @param srcPath the source path cost
+ */
+ public void addAxiom(int i, int j, Rule rule, SourcePath srcPath) {
+ if (null == this.cells.get(i, j)) {
+ this.cells.set(i, j, new Cell(this, this.goalSymbolID));
+ }
+
+ this.cells.get(i, j).addHyperEdgeInCell(
+ new ComputeNodeResult(this.featureFunctions, rule, null, i, j, srcPath, sentence), rule, i,
+ j, null, srcPath, false);
+
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/main/java/org/apache/joshua/decoder/ff/ArityPhrasePenalty.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/ff/ArityPhrasePenalty.java
index 0000000,ae273b7..f544f50
mode 000000,100644..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/ArityPhrasePenalty.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/ArityPhrasePenalty.java
@@@ -1,0 -1,72 +1,73 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.decoder.ff;
+
+ import java.util.List;
+
+ import org.apache.joshua.decoder.JoshuaConfiguration;
++import org.apache.joshua.decoder.chart_parser.SourcePath;
+ import org.apache.joshua.decoder.ff.state_maintenance.DPState;
++import org.apache.joshua.decoder.ff.tm.OwnerId;
++import org.apache.joshua.decoder.ff.tm.OwnerMap;
+ import org.apache.joshua.decoder.ff.tm.Rule;
+ import org.apache.joshua.decoder.hypergraph.HGNode;
+ import org.apache.joshua.decoder.segment_file.Sentence;
-import org.apache.joshua.decoder.chart_parser.SourcePath;
-import org.apache.joshua.corpus.Vocabulary;
+
+ /**
+ * This feature function counts rules from a particular grammar (identified by the owner) having an
+ * arity within a specific range. It expects three parameters upon initialization: the owner, the
+ * minimum arity, and the maximum arity.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ * @author Zhifei Li zhifei.work@gmail.com
+ */
+ public class ArityPhrasePenalty extends StatelessFF {
+
+ // when the rule.arity is in the range, then this feature is activated
- private final int owner;
++ private final OwnerId owner;
+ private final int minArity;
+ private final int maxArity;
+
+ public ArityPhrasePenalty(final FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, "ArityPenalty", args, config);
+
- this.owner = Vocabulary.id(parsedArgs.get("owner"));
++ this.owner = OwnerMap.register(parsedArgs.get("owner"));
+ this.minArity = Integer.parseInt(parsedArgs.get("min-arity"));
+ this.maxArity = Integer.parseInt(parsedArgs.get("max-arity"));
+ }
+
+ /**
+ * Returns 1 if the arity penalty feature applies to the current rule.
+ */
+ private int isEligible(final Rule rule) {
- if (this.owner == rule.getOwner() && rule.getArity() >= this.minArity
++ if (this.owner.equals(rule.getOwner()) && rule.getArity() >= this.minArity
+ && rule.getArity() <= this.maxArity)
+ return 1;
+
+ return 0;
+ }
+
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+ acc.add(name, isEligible(rule));
+
+ return null;
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/main/java/org/apache/joshua/decoder/ff/LexicalFeatures.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/ff/LexicalFeatures.java
index 0000000,58de5f4..75158d0
mode 000000,100644..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/LexicalFeatures.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/LexicalFeatures.java
@@@ -1,0 -1,150 +1,152 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.decoder.ff;
+
+ import static com.google.common.cache.CacheBuilder.newBuilder;
+
+ import java.util.ArrayList;
+ import java.util.List;
+
+ import org.apache.joshua.corpus.Vocabulary;
+ import org.apache.joshua.decoder.JoshuaConfiguration;
+ import org.apache.joshua.decoder.chart_parser.SourcePath;
+ import org.apache.joshua.decoder.ff.state_maintenance.DPState;
++import org.apache.joshua.decoder.ff.tm.OwnerId;
++import org.apache.joshua.decoder.ff.tm.OwnerMap;
+ import org.apache.joshua.decoder.ff.tm.Rule;
+ import org.apache.joshua.decoder.hypergraph.HGNode;
+ import org.apache.joshua.decoder.segment_file.Sentence;
+ import org.apache.joshua.util.FormatUtils;
+
+ import com.google.common.cache.Cache;
+
+ /**
+ * Lexical alignment features denoting alignments, deletions, and insertions.
+ */
+ public class LexicalFeatures extends StatelessFF {
+
+ private final boolean useAlignments;
+ private final boolean useDeletions;
+ private final boolean useInsertions;
+
+ private static final String NAME = "LexicalFeatures";
+ // value to fire for features
+ private static final int VALUE = 1;
+ //whether this feature is restricted to a certain grammar/owner
+ private final boolean ownerRestriction;
+ // the grammar/owner this feature is restricted to fire
- private final int owner;
++ private final OwnerId owner;
+ // Strings separating words
+ private static final String SEPARATOR = "~";
+
+ private final Cache<Rule, List<String>> featureCache;
+
+ public LexicalFeatures(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, NAME, args, config);
+
+ ownerRestriction = (parsedArgs.containsKey("owner")) ? true : false;
- owner = ownerRestriction ? Vocabulary.id(parsedArgs.get("owner")) : 0;
++ owner = ownerRestriction ? OwnerMap.register(parsedArgs.get("owner")) : OwnerMap.UNKNOWN_OWNER_ID;
+
+ useAlignments = parsedArgs.containsKey("alignments");
+ useDeletions = parsedArgs.containsKey("deletions");
+ useInsertions = parsedArgs.containsKey("insertions");
+
+ // initialize cache
+ if (parsedArgs.containsKey("cacheSize")) {
+ featureCache = newBuilder().maximumSize(Integer.parseInt(parsedArgs.get("cacheSize"))).build();
+ } else {
+ featureCache = newBuilder().maximumSize(config.cachedRuleSize).build();
+ }
+ }
+
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+
- if (ownerRestriction && rule.getOwner() != owner) {
++ if (ownerRestriction && rule.getOwner().equals(owner)) {
+ return null;
+ }
+
+ List<String> featureNames = featureCache.getIfPresent(rule);
+ if (featureNames == null) {
+ featureNames = getFeatures(rule);
+ featureCache.put(rule, featureNames);
+ }
+ for (String feature : featureNames) {
+ acc.add(feature, VALUE);
+ }
+
+ return null;
+ }
+
+ /**
+ * Obtains the feature ids for the given rule.
+ * @param rule
+ * @return String representing the feature name.s
+ */
+ private List<String> getFeatures(final Rule rule) {
+ final List<String> result = new ArrayList<>();
+
+ byte[] alignments = rule.getAlignment();
+ if (alignments == null) {
+ return result;
+ }
+ int[] sourceWords = rule.getFrench();
+ int[] targetWords = rule.getEnglish();
+
+ // sourceAligned & targetAligned indicate whether an index is covered by alignments
+ boolean[] sourceAligned = new boolean[sourceWords.length];
+ boolean[] targetAligned = new boolean[targetWords.length];
+
+ // translations: aligned words
+ for (int i = 0; i < alignments.length; i+=2) {
+ byte sourceIndex = alignments[i];
+ byte targetIndex = alignments[i + 1];
+ sourceAligned[sourceIndex] = true;
+ targetAligned[targetIndex] = true;
+ if (useAlignments) {
+ result.add(
+ "T:" +
+ Vocabulary.word(sourceWords[sourceIndex]) +
+ SEPARATOR +
+ Vocabulary.word(targetWords[targetIndex]));
+ }
+ }
+
+ // deletions: unaligned source words
+ if (useDeletions) {
+ for (int i = 0; i < sourceAligned.length; i++) {
+ if (!sourceAligned[i] && ! FormatUtils.isNonterminal(sourceWords[i])) {
+ result.add("D:" + Vocabulary.word(sourceWords[i]));
+ }
+ }
+ }
+
+ // insertions: unaligned target words
+ if (useInsertions) {
+ for (int i = 0; i < targetAligned.length; i++) {
+ if (useInsertions && !targetAligned[i] && ! FormatUtils.isNonterminal(targetWords[i])) {
+ result.add("I:" + Vocabulary.word(targetWords[i]));
+ }
+ }
+ }
+
+ return result;
+ }
+ }
[47/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/resources/phrase_decoder/config
----------------------------------------------------------------------
diff --git a/joshua-core/resources/phrase_decoder/config b/joshua-core/resources/phrase_decoder/config
new file mode 100644
index 0000000..de781e3
--- /dev/null
+++ b/joshua-core/resources/phrase_decoder/config
@@ -0,0 +1,29 @@
+tm = moses -owner pt -maxspan 0 -path resources/phrase_decoder/rules.1.gz -max-source-len 5
+feature-function = StateMinimizingLanguageModel -lm_order 5 -lm_file resources/phrase_decoder/lm.1.gz
+
+search = stack
+
+mark-oovs = false
+pop-limit = 10
+top-n = 1
+
+output-format = %i ||| %s ||| %f ||| %c
+
+include-align-index = true
+reordering-limit = 6
+
+# And these are the feature functions to activate.
+feature-function = OOVPenalty
+feature-function = WordPenalty
+feature-function = Distortion
+feature-function = PhrasePenalty -owner pt
+
+OOVPenalty 1.0
+Distortion 0.114849
+WordPenalty -0.201544
+PhrasePenalty -0.236965
+tm_pt_0 0.0370068
+tm_pt_1 0.0495759
+tm_pt_2 0.196742
+tm_pt_3 0.0745423
+lm_0 0.204412452147565
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/resources/phrase_decoder/constrained.config
----------------------------------------------------------------------
diff --git a/joshua-core/resources/phrase_decoder/constrained.config b/joshua-core/resources/phrase_decoder/constrained.config
new file mode 100644
index 0000000..4642650
--- /dev/null
+++ b/joshua-core/resources/phrase_decoder/constrained.config
@@ -0,0 +1,28 @@
+tm = moses pt 0 resources/phrase_decoder/rules.1.gz
+
+lm = kenlm 5 true false 100 resources/phrase_decoder/lm.1.gz
+
+mark-oovs = false
+pop-limit = 10
+top-n = 5
+
+output-format = %i ||| %s ||| %f ||| %c
+
+include-align-index = true
+reordering-limit = 10
+
+# And these are the feature functions to activate.
+feature-function = OOVPenalty
+feature-function = WordPenalty
+feature-function = Distortion
+feature-function = PhrasePenalty -owner pt
+
+OOVPenalty 1.0
+Distortion 0.114849
+WordPenalty -0.201544
+PhrasePenalty -0.236965
+tm_pt_0 0.0370068
+tm_pt_1 0.0495759
+tm_pt_2 0.196742
+tm_pt_3 0.0745423
+lm_0 0.204412452147565
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/resources/phrase_decoder/constrained.output.gold
----------------------------------------------------------------------
diff --git a/joshua-core/resources/phrase_decoder/constrained.output.gold b/joshua-core/resources/phrase_decoder/constrained.output.gold
new file mode 100644
index 0000000..238387c
--- /dev/null
+++ b/joshua-core/resources/phrase_decoder/constrained.output.gold
@@ -0,0 +1,5 @@
+0 ||| President Obama |8-8| to |7-7| hinder |4-4| a strategy |0-1| for |3-3| Republican |2-2| re @-@ election |5-6| ||| tm_pt_0=-15.792 tm_pt_1=-17.550 tm_pt_2=-14.599 tm_pt_3=-18.298 lm_0=-29.452 OOVPenalty=0.000 WordPenalty=-4.777 Distortion=-24.000 PhrasePenalty=7.000 ||| -15.163
+0 ||| President Obama |8-8| to |7-7| hinder |4-4| a |0-0| strategy |1-1| for |3-3| Republican |2-2| re @-@ election |5-6| ||| tm_pt_0=-16.919 tm_pt_1=-17.550 tm_pt_2=-14.917 tm_pt_3=-18.298 lm_0=-29.452 OOVPenalty=0.000 WordPenalty=-4.777 Distortion=-24.000 PhrasePenalty=8.000 ||| -15.505
+0 ||| President Obama |8-8| to hinder |3-4| a strategy |0-1| for |7-7| Republican |2-2| re @-@ election |5-6| ||| tm_pt_0=-14.986 tm_pt_1=-17.951 tm_pt_2=-14.075 tm_pt_3=-18.699 lm_0=-29.452 OOVPenalty=0.000 WordPenalty=-4.777 Distortion=-32.000 PhrasePenalty=6.000 ||| -15.762
+0 ||| President Obama |8-8| to hinder |3-4| a |0-0| strategy |1-1| for |7-7| Republican |2-2| re @-@ election |5-6| ||| tm_pt_0=-16.112 tm_pt_1=-17.951 tm_pt_2=-14.393 tm_pt_3=-18.699 lm_0=-29.452 OOVPenalty=0.000 WordPenalty=-4.777 Distortion=-32.000 PhrasePenalty=7.000 ||| -16.103
+0 ||| President Obama |8-8| to |3-3| hinder |4-4| a strategy |0-1| for |7-7| Republican |2-2| re @-@ election |5-6| ||| tm_pt_0=-16.329 tm_pt_1=-17.951 tm_pt_2=-15.136 tm_pt_3=-18.699 lm_0=-29.452 OOVPenalty=0.000 WordPenalty=-4.777 Distortion=-32.000 PhrasePenalty=7.000 ||| -16.257
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/resources/phrase_decoder/lm.1.gz
----------------------------------------------------------------------
diff --git a/joshua-core/resources/phrase_decoder/lm.1.gz b/joshua-core/resources/phrase_decoder/lm.1.gz
new file mode 100644
index 0000000..3f4c453
Binary files /dev/null and b/joshua-core/resources/phrase_decoder/lm.1.gz differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/resources/phrase_decoder/output.gold
----------------------------------------------------------------------
diff --git a/joshua-core/resources/phrase_decoder/output.gold b/joshua-core/resources/phrase_decoder/output.gold
new file mode 100644
index 0000000..509a3de
--- /dev/null
+++ b/joshua-core/resources/phrase_decoder/output.gold
@@ -0,0 +1 @@
+0 ||| a strategy |0-1| republican |2-2| to hinder |3-4| reelection |5-6| Obama |7-8| ||| tm_pt_0=-9.702 tm_pt_1=-10.800 tm_pt_2=-7.543 tm_pt_3=-8.555 lm_0=-19.117 OOVPenalty=0.000 WordPenalty=-3.040 Distortion=0.000 PhrasePenalty=5.000 ||| -7.496
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/resources/phrase_decoder/rules.1.gz
----------------------------------------------------------------------
diff --git a/joshua-core/resources/phrase_decoder/rules.1.gz b/joshua-core/resources/phrase_decoder/rules.1.gz
new file mode 100644
index 0000000..14466e9
Binary files /dev/null and b/joshua-core/resources/phrase_decoder/rules.1.gz differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/resources/wa_grammar
----------------------------------------------------------------------
diff --git a/joshua-core/resources/wa_grammar b/joshua-core/resources/wa_grammar
new file mode 100644
index 0000000..82d0052
--- /dev/null
+++ b/joshua-core/resources/wa_grammar
@@ -0,0 +1,3 @@
+[X] ||| A [X,1] B1 [X,2] B2 C ||| a b [X,2] c1 [X,1] c2 ||| 1 1 1 1 1 1 OOV=1 ||| 0-0 2-1 4-1 5-3 5-5
+[X] ||| U Z1 Z2 ||| n1 u z ||| 1 1 1 1 1 1 OOV=2 ||| 0-1 1-2 2-2
+[X] ||| K ||| k1 k2 k3 n1 n2 n3 ||| 1 1 1 1 1 1 OOV=4 ||| 0-0 0-1 0-2
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/resources/wa_grammar.packed/config
----------------------------------------------------------------------
diff --git a/joshua-core/resources/wa_grammar.packed/config b/joshua-core/resources/wa_grammar.packed/config
new file mode 100644
index 0000000..ebd1bf1
--- /dev/null
+++ b/joshua-core/resources/wa_grammar.packed/config
@@ -0,0 +1 @@
+max-source-len = 6
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/resources/wa_grammar.packed/encoding
----------------------------------------------------------------------
diff --git a/joshua-core/resources/wa_grammar.packed/encoding b/joshua-core/resources/wa_grammar.packed/encoding
new file mode 100644
index 0000000..630f69f
Binary files /dev/null and b/joshua-core/resources/wa_grammar.packed/encoding differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/resources/wa_grammar.packed/slice_00000.alignments
----------------------------------------------------------------------
diff --git a/joshua-core/resources/wa_grammar.packed/slice_00000.alignments b/joshua-core/resources/wa_grammar.packed/slice_00000.alignments
new file mode 100644
index 0000000..f1425eb
Binary files /dev/null and b/joshua-core/resources/wa_grammar.packed/slice_00000.alignments differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/resources/wa_grammar.packed/slice_00000.features
----------------------------------------------------------------------
diff --git a/joshua-core/resources/wa_grammar.packed/slice_00000.features b/joshua-core/resources/wa_grammar.packed/slice_00000.features
new file mode 100644
index 0000000..5a4c774
Binary files /dev/null and b/joshua-core/resources/wa_grammar.packed/slice_00000.features differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/resources/wa_grammar.packed/slice_00000.source
----------------------------------------------------------------------
diff --git a/joshua-core/resources/wa_grammar.packed/slice_00000.source b/joshua-core/resources/wa_grammar.packed/slice_00000.source
new file mode 100644
index 0000000..4607b89
Binary files /dev/null and b/joshua-core/resources/wa_grammar.packed/slice_00000.source differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/resources/wa_grammar.packed/slice_00000.target
----------------------------------------------------------------------
diff --git a/joshua-core/resources/wa_grammar.packed/slice_00000.target b/joshua-core/resources/wa_grammar.packed/slice_00000.target
new file mode 100644
index 0000000..fe11a38
Binary files /dev/null and b/joshua-core/resources/wa_grammar.packed/slice_00000.target differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/resources/wa_grammar.packed/slice_00000.target.lookup
----------------------------------------------------------------------
diff --git a/joshua-core/resources/wa_grammar.packed/slice_00000.target.lookup b/joshua-core/resources/wa_grammar.packed/slice_00000.target.lookup
new file mode 100644
index 0000000..7d82179
Binary files /dev/null and b/joshua-core/resources/wa_grammar.packed/slice_00000.target.lookup differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/resources/wa_grammar.packed/vocabulary
----------------------------------------------------------------------
diff --git a/joshua-core/resources/wa_grammar.packed/vocabulary b/joshua-core/resources/wa_grammar.packed/vocabulary
new file mode 100644
index 0000000..637651e
Binary files /dev/null and b/joshua-core/resources/wa_grammar.packed/vocabulary differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/adagrad/AdaGrad.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/adagrad/AdaGrad.java b/joshua-core/src/main/java/org/apache/joshua/adagrad/AdaGrad.java
new file mode 100755
index 0000000..0784318
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/adagrad/AdaGrad.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.adagrad;
+
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.util.FileUtility;
+import org.apache.joshua.util.StreamGobbler;
+
+public class AdaGrad {
+ public static void main(String[] args) throws Exception {
+ JoshuaConfiguration joshuaConfiguration = new JoshuaConfiguration();
+ boolean external = false; // should each AdaGrad iteration be launched externally?
+
+ if (args.length == 1) {
+ if (args[0].equals("-h")) {
+ printAdaGradUsage(args.length, true);
+ System.exit(2);
+ } else {
+ external = false;
+ }
+ } else if (args.length == 3) {
+ external = true;
+ } else {
+ printAdaGradUsage(args.length, false);
+ System.exit(1);
+ }
+
+ if (!external) {
+ AdaGradCore myAdaGrad = new AdaGradCore(args[0], joshuaConfiguration);
+ myAdaGrad.run_AdaGrad(); // optimize lambda[]
+ myAdaGrad.finish();
+ } else {
+
+ int maxMem = Integer.parseInt(args[1]);
+ String configFileName = args[2];
+ String stateFileName = FileUtility.dirname(configFileName) + "/AdaGrad.temp.state";
+ String cp = System.getProperty("java.class.path");
+ boolean done = false;
+ int iteration = 0;
+
+ while (!done) {
+ ++iteration;
+ Runtime rt = Runtime.getRuntime();
+ Process p =
+ rt.exec("java -Xmx" + maxMem + "m -cp " + cp + " org.apache.joshua.adagrad.AdaGradCore " + configFileName
+ + " " + stateFileName + " " + iteration);
+ /*
+ * BufferedReader br_i = new BufferedReader(new InputStreamReader(p.getInputStream()));
+ * BufferedReader br_e = new BufferedReader(new InputStreamReader(p.getErrorStream()));
+ * String dummy_line = null; while ((dummy_line = br_i.readLine()) != null) {
+ * System.out.println(dummy_line); } while ((dummy_line = br_e.readLine()) != null) {
+ * System.out.println(dummy_line); }
+ */
+ StreamGobbler errorGobbler = new StreamGobbler(p.getErrorStream(), 1);
+ StreamGobbler outputGobbler = new StreamGobbler(p.getInputStream(), 1);
+
+ errorGobbler.start();
+ outputGobbler.start();
+
+ int status = p.waitFor();
+
+ if (status == 90) {
+ done = true;
+ } else if (status == 91) {
+ done = false;
+ } else {
+ System.out.println("AdaGrad exiting prematurely (AdaGradCore returned " + status + ")...");
+ break;
+ }
+ }
+ }
+
+ System.exit(0);
+
+ } // main(String[] args)
+
+ public static void printAdaGradUsage(int argsLen, boolean detailed) {
+ if (!detailed) {
+ println("Oops, you provided " + argsLen + " args!");
+ println("");
+ println("Usage:");
+ println(" AdaGrad -maxMem maxMemoryInMB AdaGrad_configFile");
+ println("");
+ println("Where -maxMem specifies the maximum amount of memory (in MB) AdaGrad is");
+ println("allowed to use when performing its calculations (no memroy is needed while");
+ println("the decoder is running),");
+ println("and the config file contains any subset of AdaGrad's 20-some parameters,");
+ println("one per line. Run AdaGrad -h for more details on those parameters.");
+ } else {
+ println("Usage:");
+ println(" AdaGrad -maxMem maxMemoryInMB AdaGrad_configFile");
+ println("");
+ println("Where -maxMem specifies the maximum amount of memory (in MB) AdaGrad is");
+ println("allowed to use when performing its calculations (no memroy is needed while");
+ println("the decoder is running),");
+ println("and the config file contains any subset of AdaGrad's 20-some parameters,");
+ println("one per line. Those parameters, and their default values, are:");
+ println("");
+ println("Relevant files:");
+ println(" -dir dirPrefix: working directory\n [[default: null string (i.e. they are in the current directory)]]");
+ println(" -s sourceFile: source sentences (foreign sentences) of the AdaGrad dataset\n [[default: null string (i.e. file name is not needed by AdaGrad)]]");
+ println(" -r refFile: target sentences (reference translations) of the AdaGrad dataset\n [[default: reference.txt]]");
+ println(" -rps refsPerSen: number of reference translations per sentence\n [[default: 1]]");
+ //println(" -txtNrm textNormMethod: how should text be normalized?\n (0) don't normalize text,\n or (1) \"NIST-style\", and also rejoin 're, *'s, n't, etc,\n or (2) apply 1 and also rejoin dashes between letters,\n or (3) apply 1 and also drop non-ASCII characters,\n or (4) apply 1+2+3\n [[default: 1]]");
+ println(" -p paramsFile: file containing parameter names, initial values, and ranges\n [[default: params.txt]]");
+ //println(" -docInfo documentInfoFile: file informing AdaGrad which document each\n sentence belongs to\n [[default: null string (i.e. all sentences are in one 'document')]]");
+ println(" -fin finalLambda: file name for final lambda[] values\n [[default: null string (i.e. no such file will be created)]]");
+ println("");
+ println("AdaGrad specs:");
+ println(" -m metricName metric options: name of evaluation metric and its options\n [[default: BLEU 4 closest]]");
+ println(" -maxIt maxAdaGradIts: maximum number of AdaGrad iterations\n [[default: 20]]");
+ println(" -prevIt prevAdaGradIts: maximum number of previous AdaGrad iterations to\n construct candidate sets from\n [[default: 20]]");
+ println(" -minIt minAdaGradIts: number of iterations before considering an early exit\n [[default: 5]]");
+ println(" -stopIt stopMinIts: some early stopping criterion must be satisfied in\n stopMinIts *consecutive* iterations before an early exit\n [[default: 3]]");
+ println(" -stopSig sigValue: early AdaGrad exit if no weight changes by more than sigValue\n [[default: -1 (i.e. this criterion is never investigated)]]");
+ //println(" -thrCnt threadCount: number of threads to run in parallel when optimizing\n [[default: 1]]");
+ println(" -save saveInter: save intermediate cfg files (1) or decoder outputs (2)\n or both (3) or neither (0)\n [[default: 3]]");
+ println(" -compress compressFiles: should AdaGrad compress the files it produces (1)\n or not (0)\n [[default: 0]]");
+ //println(" -ipi initsPerIt: number of intermediate initial points per iteration\n [[default: 20]]");
+ //println(" -opi oncePerIt: modify a parameter only once per iteration (1) or not (0)\n [[default: 0]]");
+ //println(" -rand randInit: choose initial point randomly (1) or from paramsFile (0)\n [[default: 0]]");
+ //println(" -seed seed: seed used to initialize random number generator\n [[default: time (i.e. value returned by System.currentTimeMillis()]]");
+ // println(" -ud useDisk: reliance on disk (0-2; higher value => more reliance)\n [[default: 2]]");
+ println("");
+ println("Decoder specs:");
+ println(" -cmd commandFile: name of file containing commands to run the decoder\n [[default: null string (i.e. decoder is a JoshuaDecoder object)]]");
+ println(" -passIt passIterationToDecoder: should iteration number be passed\n to command file (1) or not (0)\n [[default: 0]]");
+ println(" -decOut decoderOutFile: name of the output file produced by the decoder\n [[default: output.nbest]]");
+ println(" -decExit validExit: value returned by decoder to indicate success\n [[default: 0]]");
+ println(" -dcfg decConfigFile: name of decoder config file\n [[default: dec_cfg.txt]]");
+ println(" -N N: size of N-best list (per sentence) generated in each AdaGrad iteration\n [[default: 100]]");
+ println("");
+ println("Output specs:");
+ println(" -v verbosity: AdaGrad verbosity level (0-2; higher value => more verbose)\n [[default: 1]]");
+ println(" -decV decVerbosity: should decoder output be printed (1) or ignored (0)\n [[default: 0]]");
+ println("");
+ }
+ }
+
+ private static void println(Object obj) {
+ System.out.println(obj);
+ }
+
+}
[54/60] incubator-joshua git commit: Merge branch
'maven-multi-module' of https://github.com/logogin/incubator-joshua into
maven-multi-module
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/test/resources/bn-en/hiero/output-classlm.gold
----------------------------------------------------------------------
diff --cc joshua-core/src/test/resources/bn-en/hiero/output-classlm.gold
index 0000000,066773e..001206c
mode 000000,100644..100644
--- a/joshua-core/src/test/resources/bn-en/hiero/output-classlm.gold
+++ b/joshua-core/src/test/resources/bn-en/hiero/output-classlm.gold
@@@ -1,0 -1,678 +1,887 @@@
-0 ||| rabindranath was born in a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-30.409 tm_pt_6=-15.712 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.135 tm_pt_10=-14.979 tm_pt_11=0.000 tm_pt_12=-7.729 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 lm_0=-23.712 lm_1=-22.418 WordPenalty=-5.212 OOVPenalty=-200.000 ||| -254.042
-0 ||| rabindranath was born in one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-29.639 tm_pt_6=-16.710 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.135 tm_pt_10=-13.438 tm_pt_11=0.000 tm_pt_12=-8.350 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 lm_0=-25.676 lm_1=-23.515 WordPenalty=-5.212 OOVPenalty=-200.000 ||| -257.842
-0 ||| rabindranath born in the a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-5.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-32.737 tm_pt_6=-16.092 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.000 tm_pt_10=-12.188 tm_pt_11=0.000 tm_pt_12=-3.876 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 lm_0=-26.980 lm_1=-24.179 WordPenalty=-5.212 OOVPenalty=-200.000 ||| -258.306
-0 ||| rabindranath born in kolkata a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-5.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-29.029 tm_pt_6=-16.002 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.000 tm_pt_10=-10.090 tm_pt_11=0.000 tm_pt_12=-4.282 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 lm_0=-27.803 lm_1=-24.299 WordPenalty=-5.212 OOVPenalty=-200.000 ||| -258.333
-0 ||| rabindranath born in the one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-5.000 tm_pt_3=0.000 tm_pt_4=-4.000 tm_pt_5=-31.967 tm_pt_6=-17.090 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.000 tm_pt_10=-10.415 tm_pt_11=0.000 tm_pt_12=-4.100 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=6.000 lm_0=-27.320 lm_1=-24.209 WordPenalty=-5.212 OOVPenalty=-200.000 ||| -259.542
-0 ||| rabindranath born in kolkata one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-5.000 tm_pt_3=0.000 tm_pt_4=-4.000 tm_pt_5=-28.259 tm_pt_6=-16.999 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.000 tm_pt_10=-8.317 tm_pt_11=0.000 tm_pt_12=-4.505 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=6.000 lm_0=-28.555 lm_1=-24.206 WordPenalty=-5.212 OOVPenalty=-200.000 ||| -259.925
-0 ||| rabindranath was born in a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 \u09aa\u09b0\u09bf\u09ac\u09be\u09b0\u09c7 . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-12.867 tm_pt_6=-7.153 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-0.135 tm_pt_10=-14.988 tm_pt_11=0.000 tm_pt_12=-7.732 tm_pt_13=0.000 tm_pt_14=-6.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-23.574 lm_1=-22.046 WordPenalty=-4.777 OOVPenalty=-300.000 ||| -347.574
-0 ||| rabindranath was born in one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 \u09aa\u09b0\u09bf\u09ac\u09be\u09b0\u09c7 . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-12.097 tm_pt_6=-8.150 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-0.135 tm_pt_10=-13.447 tm_pt_11=0.000 tm_pt_12=-8.353 tm_pt_13=0.000 tm_pt_14=-6.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-25.537 lm_1=-23.143 WordPenalty=-4.777 OOVPenalty=-300.000 ||| -351.374
-0 ||| rabindranath born in the a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 \u09aa\u09b0\u09bf\u09ac\u09be\u09b0\u09c7 . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-5.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-15.195 tm_pt_6=-7.533 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-1.000 tm_pt_10=-12.198 tm_pt_11=0.000 tm_pt_12=-3.880 tm_pt_13=0.000 tm_pt_14=-6.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-26.841 lm_1=-23.807 WordPenalty=-4.777 OOVPenalty=-300.000 ||| -351.838
-1 ||| recently with the united states relation improved . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-18.913 tm_pt_6=-15.946 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.743 tm_pt_10=-14.886 tm_pt_11=0.000 tm_pt_12=-6.633 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-18.873 lm_1=-15.933 WordPenalty=-4.343 OOVPenalty=0.000 ||| -43.205
-1 ||| recently with the united states matters improved . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-20.258 tm_pt_6=-17.332 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.743 tm_pt_10=-14.678 tm_pt_11=0.000 tm_pt_12=-6.633 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-18.768 lm_1=-15.680 WordPenalty=-4.343 OOVPenalty=0.000 ||| -43.440
-1 ||| recently india with united states relation improved . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-6.000 tm_pt_5=-15.661 tm_pt_6=-15.849 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.742 tm_pt_10=-10.885 tm_pt_11=0.000 tm_pt_12=-4.412 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=6.000 lm_0=-21.491 lm_1=-16.859 WordPenalty=-4.343 OOVPenalty=0.000 ||| -44.146
-1 ||| recently with the united states relation between improved . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-23.251 tm_pt_6=-15.828 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.743 tm_pt_10=-13.722 tm_pt_11=0.000 tm_pt_12=-6.633 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-19.500 lm_1=-17.313 WordPenalty=-4.777 OOVPenalty=0.000 ||| -44.244
-1 ||| recently india with united states matters improved . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-6.000 tm_pt_5=-17.007 tm_pt_6=-17.235 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.742 tm_pt_10=-10.677 tm_pt_11=0.000 tm_pt_12=-4.412 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=6.000 lm_0=-21.386 lm_1=-16.606 WordPenalty=-4.343 OOVPenalty=0.000 ||| -44.381
-1 ||| recently with the united states relationship between improved . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-22.357 tm_pt_6=-15.386 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.743 tm_pt_10=-12.806 tm_pt_11=0.000 tm_pt_12=-6.633 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-19.559 lm_1=-17.901 WordPenalty=-4.777 OOVPenalty=0.000 ||| -44.484
-1 ||| recently with united states with the relationship between improved . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-5.000 tm_pt_3=0.000 tm_pt_4=-3.000 tm_pt_5=-25.015 tm_pt_6=-15.386 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.742 tm_pt_10=-3.975 tm_pt_11=0.000 tm_pt_12=-2.234 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=3.000 lm_0=-21.908 lm_1=-18.571 WordPenalty=-5.212 OOVPenalty=0.000 ||| -44.540
-2 ||| mathematics so science language . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-4.000 tm_pt_3=0.000 tm_pt_4=-4.000 tm_pt_5=-6.483 tm_pt_6=-3.387 tm_pt_7=-4.000 tm_pt_8=-10.872 tm_pt_9=-0.002 tm_pt_10=-3.378 tm_pt_11=0.000 tm_pt_12=-1.626 tm_pt_13=0.000 tm_pt_14=-5.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=4.000 lm_0=-15.141 lm_1=-10.511 WordPenalty=-3.040 OOVPenalty=0.000 ||| -23.476
-2 ||| mathematics is science language . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-4.000 tm_pt_3=0.000 tm_pt_4=-4.000 tm_pt_5=-10.375 tm_pt_6=-3.926 tm_pt_7=-4.000 tm_pt_8=-10.872 tm_pt_9=-0.052 tm_pt_10=-8.326 tm_pt_11=0.000 tm_pt_12=-3.330 tm_pt_13=0.000 tm_pt_14=-5.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=4.000 lm_0=-12.890 lm_1=-10.773 WordPenalty=-3.040 OOVPenalty=0.000 ||| -23.873
-2 ||| mathematics that science language . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-4.000 tm_pt_3=0.000 tm_pt_4=-4.000 tm_pt_5=-9.625 tm_pt_6=-3.926 tm_pt_7=-4.000 tm_pt_8=-10.872 tm_pt_9=-0.052 tm_pt_10=-7.607 tm_pt_11=0.000 tm_pt_12=-3.330 tm_pt_13=0.000 tm_pt_14=-5.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=4.000 lm_0=-14.001 lm_1=-10.511 WordPenalty=-3.040 OOVPenalty=0.000 ||| -24.593
-2 ||| science mathematics that language . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-4.000 tm_pt_3=0.000 tm_pt_4=-3.000 tm_pt_5=-9.625 tm_pt_6=-3.926 tm_pt_7=-4.000 tm_pt_8=-10.872 tm_pt_9=-0.138 tm_pt_10=-2.832 tm_pt_11=0.000 tm_pt_12=-1.486 tm_pt_13=0.000 tm_pt_14=-5.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 lm_0=-14.271 lm_1=-10.297 WordPenalty=-3.040 OOVPenalty=0.000 ||| -24.806
-3 ||| from this it will be understood that the \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-3.000 tm_pt_5=-20.366 tm_pt_6=-14.416 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.389 tm_pt_10=-6.943 tm_pt_11=0.000 tm_pt_12=-4.457 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-28.474 lm_1=-26.724 WordPenalty=-6.080 OOVPenalty=-300.000 ||| -358.253
-3 ||| from this it will be understood that this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-7.000 tm_pt_3=0.000 tm_pt_4=-3.000 tm_pt_5=-17.030 tm_pt_6=-13.124 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-1.386 tm_pt_10=-8.291 tm_pt_11=0.000 tm_pt_12=-4.208 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-28.974 lm_1=-26.750 WordPenalty=-6.080 OOVPenalty=-300.000 ||| -359.239
-3 ||| from this it will be it can be understood that the \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-3.000 tm_pt_5=-28.998 tm_pt_6=-11.009 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.389 tm_pt_10=-5.844 tm_pt_11=0.000 tm_pt_12=-4.457 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-31.634 lm_1=-28.993 WordPenalty=-7.383 OOVPenalty=-300.000 ||| -359.373
-3 ||| from this it can be it will be that the \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-7.000 tm_pt_3=0.000 tm_pt_4=-3.000 tm_pt_5=-27.558 tm_pt_6=-10.958 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-1.407 tm_pt_10=-14.576 tm_pt_11=0.000 tm_pt_12=-5.598 tm_pt_13=0.000 tm_pt_14=-11.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=6.000 lm_0=-31.727 lm_1=-26.439 WordPenalty=-6.949 OOVPenalty=-300.000 ||| -360.806
-3 ||| from this it can be it will be that this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-8.000 tm_pt_3=0.000 tm_pt_4=-3.000 tm_pt_5=-24.223 tm_pt_6=-9.666 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-1.389 tm_pt_10=-14.553 tm_pt_11=0.000 tm_pt_12=-5.139 tm_pt_13=0.000 tm_pt_14=-11.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=6.000 lm_0=-32.227 lm_1=-26.088 WordPenalty=-6.949 OOVPenalty=-300.000 ||| -360.882
-3 ||| from this it will it can be understood that the \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-3.000 tm_pt_5=-26.563 tm_pt_6=-11.056 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.524 tm_pt_10=-5.333 tm_pt_11=0.000 tm_pt_12=-5.479 tm_pt_13=0.000 tm_pt_14=-11.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=4.000 lm_0=-31.565 lm_1=-28.432 WordPenalty=-6.949 OOVPenalty=-300.000 ||| -361.417
-3 ||| from this it can be it that this will \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-8.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-21.787 tm_pt_6=-9.713 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-1.524 tm_pt_10=-14.418 tm_pt_11=0.000 tm_pt_12=-6.372 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 lm_0=-32.769 lm_1=-25.204 WordPenalty=-6.514 OOVPenalty=-300.000 ||| -361.526
-4 ||| same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match from this novel . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-9.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-16.212 tm_pt_6=-10.084 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-3.005 tm_pt_10=-12.338 tm_pt_11=0.000 tm_pt_12=-5.018 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 lm_0=-39.641 lm_1=-31.966 WordPenalty=-6.080 OOVPenalty=-200.000 ||| -277.519
-4 ||| same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 presage match from this novel . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-9.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-15.113 tm_pt_6=-10.084 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-3.005 tm_pt_10=-12.338 tm_pt_11=0.000 tm_pt_12=-5.018 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 lm_0=-39.916 lm_1=-31.966 WordPenalty=-6.080 OOVPenalty=-200.000 ||| -277.649
-4 ||| same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match from this novels . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-9.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-15.164 tm_pt_6=-9.637 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-3.005 tm_pt_10=-11.182 tm_pt_11=0.000 tm_pt_12=-4.651 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 lm_0=-41.798 lm_1=-31.966 WordPenalty=-6.080 OOVPenalty=-200.000 ||| -279.404
-4 ||| same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 presage match from this novels . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-9.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-14.066 tm_pt_6=-9.637 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-3.005 tm_pt_10=-11.182 tm_pt_11=0.000 tm_pt_12=-4.651 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 lm_0=-42.073 lm_1=-31.966 WordPenalty=-6.080 OOVPenalty=-200.000 ||| -279.534
-4 ||| with the same earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match from this novel . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-9.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-16.212 tm_pt_6=-10.084 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-3.373 tm_pt_10=-14.188 tm_pt_11=0.000 tm_pt_12=-7.809 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 lm_0=-37.796 lm_1=-32.110 WordPenalty=-6.080 OOVPenalty=-200.000 ||| -280.076
-4 ||| with the same earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 presage match from this novel . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-9.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-15.113 tm_pt_6=-10.084 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-3.373 tm_pt_10=-14.188 tm_pt_11=0.000 tm_pt_12=-7.809 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 lm_0=-38.071 lm_1=-32.110 WordPenalty=-6.080 OOVPenalty=-200.000 ||| -280.206
-4 ||| the with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match from this novel . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-9.000 tm_pt_3=0.000 tm_pt_4=-6.000 tm_pt_5=-20.640 tm_pt_6=-9.983 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-3.024 tm_pt_10=-18.514 tm_pt_11=0.000 tm_pt_12=-6.668 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-37.546 lm_1=-32.463 WordPenalty=-6.080 OOVPenalty=-200.000 ||| -280.483
-4 ||| the same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match from this novel . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-9.000 tm_pt_3=0.000 tm_pt_4=-6.000 tm_pt_5=-21.437 tm_pt_6=-10.032 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-3.005 tm_pt_10=-12.086 tm_pt_11=0.000 tm_pt_12=-4.843 tm_pt_13=0.000 tm_pt_14=-11.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-39.669 lm_1=-33.675 WordPenalty=-6.514 OOVPenalty=-200.000 ||| -280.518
-5 ||| \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 majority in the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-7.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-16.310 tm_pt_6=-6.695 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-0.289 tm_pt_10=-10.344 tm_pt_11=0.000 tm_pt_12=-2.428 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=9.000 lm_0=-25.263 lm_1=-22.356 WordPenalty=-5.212 OOVPenalty=-200.000 ||| -245.557
-5 ||| \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 majority that the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-6.000 tm_pt_5=-16.018 tm_pt_6=-7.571 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.271 tm_pt_10=-5.619 tm_pt_11=0.000 tm_pt_12=-1.161 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-27.702 lm_1=-22.018 WordPenalty=-5.212 OOVPenalty=-200.000 ||| -247.342
-5 ||| \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 majority in . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-13.102 tm_pt_6=-8.482 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-0.289 tm_pt_10=-14.216 tm_pt_11=0.000 tm_pt_12=-2.256 tm_pt_13=0.000 tm_pt_14=-7.000 tm_pt_15=-1.000 tm_pt_16=0.000 tm_glue_0=7.000 lm_0=-25.124 lm_1=-21.005 WordPenalty=-4.777 OOVPenalty=-200.000 ||| -247.913
-5 ||| mujib and his party \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 majority in the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-7.000 tm_pt_3=0.000 tm_pt_4=-4.000 tm_pt_5=-16.310 tm_pt_6=-6.695 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-0.657 tm_pt_10=-10.233 tm_pt_11=0.000 tm_pt_12=-5.719 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-24.288 lm_1=-22.355 WordPenalty=-5.212 OOVPenalty=-200.000 ||| -249.681
-5 ||| \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 . majority in the ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-7.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-16.310 tm_pt_6=-6.695 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-0.370 tm_pt_10=-9.011 tm_pt_11=0.000 tm_pt_12=-4.591 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=6.000 lm_0=-28.212 lm_1=-20.384 WordPenalty=-5.212 OOVPenalty=-200.000 ||| -250.181
-5 ||| \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 the . majority in ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-16.310 tm_pt_6=-6.695 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.271 tm_pt_10=-2.153 tm_pt_11=0.000 tm_pt_12=-0.468 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=6.000 lm_0=-30.418 lm_1=-22.020 WordPenalty=-5.212 OOVPenalty=-200.000 ||| -251.282
-6 ||| \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his work with to that . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-6.000 tm_pt_5=-22.509 tm_pt_6=-11.163 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.370 tm_pt_10=-18.845 tm_pt_11=0.000 tm_pt_12=-2.681 tm_pt_13=0.000 tm_pt_14=-6.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 lm_0=-33.425 lm_1=-25.130 WordPenalty=-5.212 OOVPenalty=-400.000 ||| -463.452
-6 ||| \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his work with a that . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-24.229 tm_pt_6=-13.109 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.418 tm_pt_10=-18.986 tm_pt_11=0.000 tm_pt_12=-2.612 tm_pt_13=0.000 tm_pt_14=-6.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=9.000 lm_0=-33.108 lm_1=-25.595 WordPenalty=-5.212 OOVPenalty=-400.000 ||| -465.645
-6 ||| \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his work with that can . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-19.242 tm_pt_6=-9.832 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-2.368 tm_pt_10=-17.376 tm_pt_11=0.000 tm_pt_12=-3.305 tm_pt_13=0.000 tm_pt_14=-6.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=9.000 lm_0=-33.973 lm_1=-25.956 WordPenalty=-5.212 OOVPenalty=-400.000 ||| -465.694
-6 ||| \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his work with can that . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-6.000 tm_pt_5=-19.242 tm_pt_6=-9.832 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.503 tm_pt_10=-17.011 tm_pt_11=0.000 tm_pt_12=-3.528 tm_pt_13=0.000 tm_pt_14=-6.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 lm_0=-35.292 lm_1=-26.410 WordPenalty=-5.212 OOVPenalty=-400.000 ||| -466.199
-6 ||| \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his work could that with . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-19.925 tm_pt_6=-12.193 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.736 tm_pt_10=-13.793 tm_pt_11=0.000 tm_pt_12=-1.919 tm_pt_13=0.000 tm_pt_14=-6.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-35.742 lm_1=-24.982 WordPenalty=-5.212 OOVPenalty=-400.000 ||| -466.677
-6 ||| \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 its work with that can . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-20.130 tm_pt_6=-11.794 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-2.368 tm_pt_10=-17.845 tm_pt_11=0.000 tm_pt_12=-5.310 tm_pt_13=0.000 tm_pt_14=-6.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=9.000 lm_0=-34.521 lm_1=-25.533 WordPenalty=-5.212 OOVPenalty=-400.000 ||| -467.521
-7 ||| task , \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character is but these a is not . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-40.521 tm_pt_6=-16.440 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-1.889 tm_pt_10=-28.483 tm_pt_11=0.000 tm_pt_12=-9.906 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=15.000 lm_0=-46.205 lm_1=-34.518 WordPenalty=-7.817 OOVPenalty=-200.000 ||| -292.687
-7 ||| task , \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character is but these very is not . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-36.805 tm_pt_6=-15.372 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-1.871 tm_pt_10=-24.380 tm_pt_11=0.000 tm_pt_12=-9.030 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=15.000 lm_0=-47.390 lm_1=-35.336 WordPenalty=-7.817 OOVPenalty=-200.000 ||| -292.701
-7 ||| task , \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character are but these very is not . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-35.727 tm_pt_6=-15.118 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-1.871 tm_pt_10=-23.123 tm_pt_11=0.000 tm_pt_12=-8.647 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=15.000 lm_0=-48.326 lm_1=-35.181 WordPenalty=-7.817 OOVPenalty=-200.000 ||| -292.912
-7 ||| task , \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and some more linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character is but these a is not . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-12.000 tm_pt_5=-38.538 tm_pt_6=-15.147 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-1.892 tm_pt_10=-28.563 tm_pt_11=0.000 tm_pt_12=-10.079 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=14.000 lm_0=-46.230 lm_1=-34.518 WordPenalty=-7.817 OOVPenalty=-200.000 ||| -292.945
-7 ||| task , \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and some more linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character is but these very is not . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-12.000 tm_pt_5=-34.821 tm_pt_6=-14.079 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-1.874 tm_pt_10=-24.460 tm_pt_11=0.000 tm_pt_12=-9.204 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=14.000 lm_0=-47.416 lm_1=-35.336 WordPenalty=-7.817 OOVPenalty=-200.000 ||| -292.960
-7 ||| task , \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and some more linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character are but these very is not . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-12.000 tm_pt_5=-33.743 tm_pt_6=-13.825 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-1.874 tm_pt_10=-23.203 tm_pt_11=0.000 tm_pt_12=-8.821 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=14.000 lm_0=-48.352 lm_1=-35.181 WordPenalty=-7.817 OOVPenalty=-200.000 ||| -293.171
-7 ||| task , \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character are but these is not common . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-12.000 tm_pt_3=0.000 tm_pt_4=-11.000 tm_pt_5=-36.984 tm_pt_6=-17.526 tm_pt_7=-12.000 tm_pt_8=-32.616 tm_pt_9=-2.871 tm_pt_10=-24.262 tm_pt_11=0.000 tm_pt_12=-8.564 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=13.000 lm_0=-47.411 lm_1=-33.956 WordPenalty=-7.817 OOVPenalty=-200.000 ||| -293.188
-7 ||| task , \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character there but these very is not . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-34.439 tm_pt_6=-14.939 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-1.871 tm_pt_10=-22.476 tm_pt_11=0.000 tm_pt_12=-9.541 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=15.000 lm_0=-48.680 lm_1=-35.181 WordPenalty=-7.817 OOVPenalty=-200.000 ||| -293.191
-7 ||| task , \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and some more linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character are but these is not common . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-12.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-35.001 tm_pt_6=-16.233 tm_pt_7=-12.000 tm_pt_8=-32.616 tm_pt_9=-2.874 tm_pt_10=-24.342 tm_pt_11=0.000 tm_pt_12=-8.737 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=12.000 lm_0=-47.437 lm_1=-33.956 WordPenalty=-7.817 OOVPenalty=-200.000 ||| -293.447
-7 ||| task , \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and some more linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character there but these very is not . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-12.000 tm_pt_5=-32.456 tm_pt_6=-13.646 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-1.874 tm_pt_10=-22.557 tm_pt_11=0.000 tm_pt_12=-9.715 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=14.000 lm_0=-48.706 lm_1=-35.181 WordPenalty=-7.817 OOVPenalty=-200.000 ||| -293.450
-8 ||| it social situation in his oppositions \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making with the rules of the the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-14.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-51.949 tm_pt_6=-23.028 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-4.346 tm_pt_10=-40.862 tm_pt_11=0.000 tm_pt_12=-16.582 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 lm_0=-44.904 lm_1=-32.036 WordPenalty=-8.252 OOVPenalty=-200.000 ||| -303.147
-8 ||| it social situation in his oppositions \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making with the rule of the the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-14.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-52.547 tm_pt_6=-22.440 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-4.218 tm_pt_10=-41.230 tm_pt_11=0.000 tm_pt_12=-15.889 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 lm_0=-44.342 lm_1=-33.010 WordPenalty=-8.252 OOVPenalty=-200.000 ||| -303.315
-8 ||| it social situation in his oppositions \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making with the rule of the in the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-14.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-58.491 tm_pt_6=-22.780 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-4.303 tm_pt_10=-38.507 tm_pt_11=0.000 tm_pt_12=-16.177 tm_pt_13=0.000 tm_pt_14=-16.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 lm_0=-45.015 lm_1=-33.550 WordPenalty=-8.686 OOVPenalty=-200.000 ||| -303.603
-8 ||| it social situation in his oppositions \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making with the rule of the fingers . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-48.497 tm_pt_6=-24.838 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-4.536 tm_pt_10=-34.420 tm_pt_11=0.000 tm_pt_12=-15.157 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 lm_0=-45.197 lm_1=-34.200 WordPenalty=-8.252 OOVPenalty=-200.000 ||| -303.936
-8 ||| it social situation in his oppositions \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making with the rules of the is . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-14.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-50.981 tm_pt_6=-23.411 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-4.298 tm_pt_10=-38.862 tm_pt_11=0.000 tm_pt_12=-15.889 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 lm_0=-45.232 lm_1=-33.325 WordPenalty=-8.252 OOVPenalty=-200.000 ||| -304.291
-8 ||| this social situation in his oppositions \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making with the rule of the fingers . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-50.091 tm_pt_6=-25.785 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-4.536 tm_pt_10=-36.353 tm_pt_11=0.000 tm_pt_12=-15.790 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 lm_0=-44.436 lm_1=-34.295 WordPenalty=-8.252 OOVPenalty=-200.000 ||| -304.466
-8 ||| it social situation in his oppositions \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making with the rules of the fingers . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-47.899 tm_pt_6=-25.426 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-4.665 tm_pt_10=-34.052 tm_pt_11=0.000 tm_pt_12=-15.850 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 lm_0=-45.758 lm_1=-33.931 WordPenalty=-8.252 OOVPenalty=-200.000 ||| -304.640
-8 ||| it social situation in his oppositions \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making with the rule of the is . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-14.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-51.579 tm_pt_6=-22.823 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-4.169 tm_pt_10=-39.230 tm_pt_11=0.000 tm_pt_12=-15.196 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 lm_0=-44.670 lm_1=-34.452 WordPenalty=-8.252 OOVPenalty=-200.000 ||| -304.647
-8 ||| this social situation in his oppositions \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making with the rules of the is . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-14.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-52.574 tm_pt_6=-24.358 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-4.298 tm_pt_10=-40.795 tm_pt_11=0.000 tm_pt_12=-16.521 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 lm_0=-44.472 lm_1=-33.420 WordPenalty=-8.252 OOVPenalty=-200.000 ||| -304.820
-8 ||| this social situation in his oppositions \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making with the rules of the fingers . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-49.492 tm_pt_6=-26.373 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-4.665 tm_pt_10=-35.985 tm_pt_11=0.000 tm_pt_12=-16.483 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 lm_0=-44.998 lm_1=-34.026 WordPenalty=-8.252 OOVPenalty=-200.000 ||| -305.170
-9 ||| \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 annual \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf meters ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-3.372 tm_pt_6=-3.054 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.368 tm_pt_10=-2.140 tm_pt_11=0.000 tm_pt_12=-1.263 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-26.340 lm_1=-17.443 WordPenalty=-3.040 OOVPenalty=-300.000 ||| -342.660
-9 ||| \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 annual \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf meter ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-3.885 tm_pt_6=-2.821 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.368 tm_pt_10=-2.140 tm_pt_11=0.000 tm_pt_12=-1.337 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-26.316 lm_1=-17.443 WordPenalty=-3.040 OOVPenalty=-300.000 ||| -342.676
-9 ||| \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 arrange \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf meters ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-2.916 tm_pt_6=-3.748 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-1.000 tm_pt_10=0.000 tm_pt_11=0.000 tm_pt_12=-1.956 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-26.532 lm_1=-17.735 WordPenalty=-3.040 OOVPenalty=-300.000 ||| -343.618
-9 ||| \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 arrange \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf meter ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-3.430 tm_pt_6=-3.514 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-1.000 tm_pt_10=0.000 tm_pt_11=0.000 tm_pt_12=-2.030 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-26.509 lm_1=-17.735 WordPenalty=-3.040 OOVPenalty=-300.000 ||| -343.633
-9 ||| \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 annual \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf is ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-10.780 tm_pt_6=-4.900 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-1.368 tm_pt_10=-9.020 tm_pt_11=0.000 tm_pt_12=-3.902 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-24.572 lm_1=-16.003 WordPenalty=-3.040 OOVPenalty=-300.000 ||| -344.278
-9 ||| \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 annual \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf above ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-6.156 tm_pt_6=-5.306 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-1.368 tm_pt_10=-2.833 tm_pt_11=0.000 tm_pt_12=-3.902 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-26.244 lm_1=-16.235 WordPenalty=-3.040 OOVPenalty=-300.000 ||| -344.284
-9 ||| \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 annual \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf metres ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-3.608 tm_pt_6=-4.389 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.503 tm_pt_10=-2.140 tm_pt_11=0.000 tm_pt_12=-2.803 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-26.771 lm_1=-17.443 WordPenalty=-3.040 OOVPenalty=-300.000 ||| -344.352
-10 ||| in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f in the national conference was he the speech -lrb- keynote speech -rrb- , the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-15.000 tm_pt_3=0.000 tm_pt_4=-12.000 tm_pt_5=-52.291 tm_pt_6=-17.578 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-1.738 tm_pt_10=-37.639 tm_pt_11=0.000 tm_pt_12=-11.291 tm_pt_13=0.000 tm_pt_14=-16.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=17.000 lm_0=-63.388 lm_1=-50.340 WordPenalty=-10.423 OOVPenalty=-600.000 ||| -728.897
-10 ||| in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 he in national \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f party was the main speech -lrb- keynote speech -rrb- , the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-15.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-41.284 tm_pt_6=-16.735 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-1.525 tm_pt_10=-31.599 tm_pt_11=0.000 tm_pt_12=-10.353 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=13.000 lm_0=-63.061 lm_1=-49.582 WordPenalty=-9.989 OOVPenalty=-600.000 ||| -728.950
-10 ||| in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f in the national conference was he the speech -lrb- keynote speech -rrb- on the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-15.000 tm_pt_3=0.000 tm_pt_4=-12.000 tm_pt_5=-50.081 tm_pt_6=-17.884 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-1.420 tm_pt_10=-33.895 tm_pt_11=0.000 tm_pt_12=-10.598 tm_pt_13=0.000 tm_pt_14=-16.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=17.000 lm_0=-64.518 lm_1=-50.766 WordPenalty=-10.423 OOVPenalty=-600.000 ||| -729.027
-10 ||| in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 he in national \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f party was the main speech -lrb- keynote speech -rrb- on the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-15.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-39.075 tm_pt_6=-17.041 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-1.207 tm_pt_10=-27.855 tm_pt_11=0.000 tm_pt_12=-9.660 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=13.000 lm_0=-64.192 lm_1=-50.008 WordPenalty=-9.989 OOVPenalty=-600.000 ||| -729.081
-10 ||| in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f in the national was he the speech -lrb- keynote speech -rrb- , the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-15.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-49.789 tm_pt_6=-17.578 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-2.056 tm_pt_10=-43.723 tm_pt_11=0.000 tm_pt_12=-10.805 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=18.000 lm_0=-61.947 lm_1=-50.201 WordPenalty=-9.989 OOVPenalty=-600.000 ||| -729.099
-10 ||| in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in national \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f party was the he the speech -lrb- keynote speech -rrb- , the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-15.000 tm_pt_3=0.000 tm_pt_4=-11.000 tm_pt_5=-45.731 tm_pt_6=-17.483 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-1.524 tm_pt_10=-36.236 tm_pt_11=0.000 tm_pt_12=-9.345 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=15.000 lm_0=-64.100 lm_1=-48.774 WordPenalty=-9.989 OOVPenalty=-600.000 ||| -729.223
-10 ||| in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f in the national was he the speech -lrb- keynote speech -rrb- on the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-15.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-47.579 tm_pt_6=-17.884 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-1.738 tm_pt_10=-39.979 tm_pt_11=0.000 tm_pt_12=-10.112 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=18.000 lm_0=-63.078 lm_1=-50.627 WordPenalty=-9.989 OOVPenalty=-600.000 ||| -729.230
-10 ||| in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f in the national conference was he main speech -lrb- keynote speech -rrb- , the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-15.000 tm_pt_3=0.000 tm_pt_4=-12.000 tm_pt_5=-47.844 tm_pt_6=-16.831 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-1.738 tm_pt_10=-33.032 tm_pt_11=0.000 tm_pt_12=-9.306 tm_pt_13=0.000 tm_pt_14=-16.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=17.000 lm_0=-65.045 lm_1=-51.497 WordPenalty=-10.423 OOVPenalty=-600.000 ||| -729.376
-10 ||| in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in national \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f party was the he main speech -lrb- keynote speech -rrb- , the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-15.000 tm_pt_3=0.000 tm_pt_4=-11.000 tm_pt_5=-41.284 tm_pt_6=-16.735 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-1.524 tm_pt_10=-31.629 tm_pt_11=0.000 tm_pt_12=-7.360 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=15.000 lm_0=-65.757 lm_1=-49.826 WordPenalty=-9.989 OOVPenalty=-600.000 ||| -729.571
-10 ||| in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f in the national was he main speech -lrb- keynote speech -rrb- , the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-15.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-45.343 tm_pt_6=-16.831 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-2.056 tm_pt_10=-39.115 tm_pt_11=0.000 tm_pt_12=-8.820 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=18.000 lm_0=-63.604 lm_1=-51.358 WordPenalty=-9.989 OOVPenalty=-600.000 ||| -729.578
-11 ||| population , power of east pakistan where to west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac they started the where the west pakistan was considered as a province . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-24.000 tm_pt_3=0.000 tm_pt_4=-21.000 tm_pt_5=-83.737 tm_pt_6=-46.104 tm_pt_7=-24.000 tm_pt_8=-65.232 tm_pt_9=-5.578 tm_pt_10=-65.560 tm_pt_11=0.000 tm_pt_12=-18.490 tm_pt_13=0.000 tm_pt_14=-30.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=21.000 lm_0=-67.857 lm_1=-46.089 WordPenalty=-14.332 OOVPenalty=-100.000 ||| -245.539
-11 ||| population , power of east pakistan where to west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac started with the where the west pakistan was considered as a province . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-24.000 tm_pt_3=0.000 tm_pt_4=-20.000 tm_pt_5=-85.129 tm_pt_6=-46.104 tm_pt_7=-24.000 tm_pt_8=-65.232 tm_pt_9=-5.896 tm_pt_10=-66.642 tm_pt_11=0.000 tm_pt_12=-18.372 tm_pt_13=0.000 tm_pt_14=-30.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=20.000 lm_0=-66.308 lm_1=-46.684 WordPenalty=-14.332 OOVPenalty=-100.000 ||| -246.090
-11 ||| population , power of east pakistan where to west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac given started the where the west pakistan was considered as a province . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-24.000 tm_pt_3=0.000 tm_pt_4=-20.000 tm_pt_5=-83.065 tm_pt_6=-46.104 tm_pt_7=-24.000 tm_pt_8=-65.232 tm_pt_9=-4.713 tm_pt_10=-61.832 tm_pt_11=0.000 tm_pt_12=-17.391 tm_pt_13=0.000 tm_pt_14=-30.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=20.000 lm_0=-69.330 lm_1=-46.228 WordPenalty=-14.332 OOVPenalty=-100.000 ||| -246.304
-11 ||| population , power of east pakistan where to west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac they started the where the regarded as a province west pakistan . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-25.000 tm_pt_3=0.000 tm_pt_4=-21.000 tm_pt_5=-81.357 tm_pt_6=-47.996 tm_pt_7=-25.000 tm_pt_8=-67.950 tm_pt_9=-4.630 tm_pt_10=-63.514 tm_pt_11=0.000 tm_pt_12=-16.959 tm_pt_13=0.000 tm_pt_14=-29.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=21.000 lm_0=-66.590 lm_1=-47.021 WordPenalty=-13.897 OOVPenalty=-100.000 ||| -246.554
-11 ||| population , power of east pakistan where to west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac with started the where the west pakistan was considered as a province . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-24.000 tm_pt_3=0.000 tm_pt_4=-21.000 tm_pt_5=-85.129 tm_pt_6=-46.104 tm_pt_7=-24.000 tm_pt_8=-65.232 tm_pt_9=-5.578 tm_pt_10=-66.463 tm_pt_11=0.000 tm_pt_12=-18.490 tm_pt_13=0.000 tm_pt_14=-30.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=21.000 lm_0=-68.152 lm_1=-46.328 WordPenalty=-14.332 OOVPenalty=-100.000 ||| -246.701
-11 ||| population on power of east pakistan where to west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac started with the where the west pakistan was considered as a province . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-25.000 tm_pt_3=0.000 tm_pt_4=-21.000 tm_pt_5=-81.148 tm_pt_6=-44.637 tm_pt_7=-25.000 tm_pt_8=-67.950 tm_pt_9=-6.528 tm_pt_10=-64.590 tm_pt_11=0.000 tm_pt_12=-19.542 tm_pt_13=0.000 tm_pt_14=-30.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=21.000 lm_0=-67.384 lm_1=-46.965 WordPenalty=-14.332 OOVPenalty=-100.000 ||| -247.096
-11 ||| population , power of east pakistan where to west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac they started the where the considered as a province west pakistan . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-25.000 tm_pt_3=0.000 tm_pt_4=-21.000 tm_pt_5=-81.467 tm_pt_6=-46.438 tm_pt_7=-25.000 tm_pt_8=-67.950 tm_pt_9=-5.262 tm_pt_10=-65.817 tm_pt_11=0.000 tm_pt_12=-17.653 tm_pt_13=0.000 tm_pt_14=-29.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=21.000 lm_0=-66.450 lm_1=-47.021 WordPenalty=-13.897 OOVPenalty=-100.000 ||| -247.231
-11 ||| population , power of east pakistan where to west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac given started the where the regarded as a province west pakistan . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-25.000 tm_pt_3=0.000 tm_pt_4=-20.000 tm_pt_5=-80.686 tm_pt_6=-47.996 tm_pt_7=-25.000 tm_pt_8=-67.950 tm_pt_9=-3.765 tm_pt_10=-59.786 tm_pt_11=0.000 tm_pt_12=-15.861 tm_pt_13=0.000 tm_pt_14=-29.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=20.000 lm_0=-68.063 lm_1=-47.160 WordPenalty=-13.897 OOVPenalty=-100.000 ||| -247.319
-11 ||| population , power of east pakistan where to west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac with started the where the regarded as a province west pakistan . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-25.000 tm_pt_3=0.000 tm_pt_4=-21.000 tm_pt_5=-82.750 tm_pt_6=-47.996 tm_pt_7=-25.000 tm_pt_8=-67.950 tm_pt_9=-4.630 tm_pt_10=-64.417 tm_pt_11=0.000 tm_pt_12=-16.959 tm_pt_13=0.000 tm_pt_14=-29.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=21.000 lm_0=-66.885 lm_1=-47.260 WordPenalty=-13.897 OOVPenalty=-100.000 ||| -247.715
-11 ||| population , power of east pakistan where to west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac given started the where the considered as a province west pakistan . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-25.000 tm_pt_3=0.000 tm_pt_4=-20.000 tm_pt_5=-80.795 tm_pt_6=-46.438 tm_pt_7=-25.000 tm_pt_8=-67.950 tm_pt_9=-4.397 tm_pt_10=-62.089 tm_pt_11=0.000 tm_pt_12=-16.554 tm_pt_13=0.000 tm_pt_14=-29.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=20.000 lm_0=-67.923 lm_1=-47.160 WordPenalty=-13.897 OOVPenalty=-100.000 ||| -247.996
-12 ||| mathematical \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 theory ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-2.869 tm_pt_6=-2.890 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-1.000 tm_pt_10=-4.888 tm_pt_11=0.000 tm_pt_12=-2.010 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=3.000 lm_0=-12.665 lm_1=-9.989 WordPenalty=-2.171 OOVPenalty=-100.000 ||| -123.067
-12 ||| the \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 theory ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-9.309 tm_pt_6=-3.988 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.368 tm_pt_10=-7.916 tm_pt_11=0.000 tm_pt_12=-1.316 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=3.000 lm_0=-11.112 lm_1=-10.414 WordPenalty=-2.171 OOVPenalty=-100.000 ||| -123.301
-12 ||| . \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 theory ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-1.000 tm_pt_5=-7.900 tm_pt_6=-2.990 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.018 tm_pt_10=-3.641 tm_pt_11=0.000 tm_pt_12=-1.712 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 lm_0=-12.758 lm_1=-10.015 WordPenalty=-2.171 OOVPenalty=-100.000 ||| -123.942
-12 ||| \u2022 \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 theory ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-1.046 tm_pt_6=-5.241 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.368 tm_pt_10=-0.422 tm_pt_11=0.000 tm_pt_12=-1.316 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=3.000 lm_0=-14.217 lm_1=-10.419 WordPenalty=-2.171 OOVPenalty=-100.000 ||| -124.052
-13 ||| external links of ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-7.390 tm_pt_6=-2.729 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.018 tm_pt_10=-5.285 tm_pt_11=0.000 tm_pt_12=-1.611 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 lm_0=-6.986 lm_1=-5.946 WordPenalty=-2.171 OOVPenalty=0.000 ||| -11.675
-13 ||| external link of ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-8.091 tm_pt_6=-2.871 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.018 tm_pt_10=-5.285 tm_pt_11=0.000 tm_pt_12=-2.767 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 lm_0=-7.533 lm_1=-5.946 WordPenalty=-2.171 OOVPenalty=0.000 ||| -12.944
-13 ||| outer link of ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-8.247 tm_pt_6=-3.617 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.018 tm_pt_10=-5.285 tm_pt_11=0.000 tm_pt_12=-3.202 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 lm_0=-7.249 lm_1=-5.946 WordPenalty=-2.171 OOVPenalty=0.000 ||| -13.031
-13 ||| external communication of ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-8.265 tm_pt_6=-2.886 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.018 tm_pt_10=-5.285 tm_pt_11=0.000 tm_pt_12=-2.555 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 lm_0=-7.692 lm_1=-5.946 WordPenalty=-2.171 OOVPenalty=0.000 ||| -13.104
-13 ||| outer communication of ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-8.420 tm_pt_6=-3.648 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.018 tm_pt_10=-5.285 tm_pt_11=0.000 tm_pt_12=-3.297 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 lm_0=-7.553 lm_1=-5.946 WordPenalty=-2.171 OOVPenalty=0.000 ||| -13.484
-13 ||| external connection of ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-8.551 tm_pt_6=-3.029 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.037 tm_pt_10=-5.285 tm_pt_11=0.000 tm_pt_12=-3.990 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 lm_0=-8.109 lm_1=-5.946 WordPenalty=-2.171 OOVPenalty=0.000 ||| -14.247
-13 ||| out-links of ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-6.938 tm_pt_6=-4.795 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.018 tm_pt_10=-5.285 tm_pt_11=0.000 tm_pt_12=-3.297 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 lm_0=-8.078 lm_1=-4.396 WordPenalty=-1.737 OOVPenalty=0.000 ||| -14.259
-14 ||| tata communication " foreign sanchar nigam limited building is the telecommunication system of a main providers ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-11.000 tm_pt_5=-40.840 tm_pt_6=-15.009 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-3.507 tm_pt_10=-27.395 tm_pt_11=0.000 tm_pt_12=-13.114 tm_pt_13=0.000 tm_pt_14=-16.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 lm_0=-45.756 lm_1=-35.626 WordPenalty=-7.817 OOVPenalty=0.000 ||| -98.525
-14 ||| tata communication " foreign sanchar nigam limited building is the telecommunication system of a the providers ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-11.000 tm_pt_5=-44.626 tm_pt_6=-15.096 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-3.507 tm_pt_10=-30.934 tm_pt_11=0.000 tm_pt_12=-14.030 tm_pt_13=0.000 tm_pt_14=-16.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 lm_0=-45.036 lm_1=-35.036 WordPenalty=-7.817 OOVPenalty=0.000 ||| -98.899
-14 ||| tata communication " foreign sanchar nigam limited building it the telecommunication system of a main providers ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-38.870 tm_pt_6=-14.378 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-2.524 tm_pt_10=-23.054 tm_pt_11=0.000 tm_pt_12=-10.369 tm_pt_13=0.000 tm_pt_14=-16.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 lm_0=-47.786 lm_1=-36.147 WordPenalty=-7.817 OOVPenalty=0.000 ||| -99.150
-14 ||| tata communication " foreign sanchar nigam limited building it the telecommunication system of a the providers ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-11.000 tm_pt_5=-42.656 tm_pt_6=-14.466 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-3.506 tm_pt_10=-27.222 tm_pt_11=0.000 tm_pt_12=-11.190 tm_pt_13=0.000 tm_pt_14=-16.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 lm_0=-47.066 lm_1=-35.556 WordPenalty=-7.817 OOVPenalty=0.000 ||| -99.494
-14 ||| tata communication " foreign sanchar nigam limited building is the telecommunication system is the main providers ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-44.366 tm_pt_6=-18.068 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-4.390 tm_pt_10=-25.931 tm_pt_11=0.000 tm_pt_12=-12.761 tm_pt_13=0.000 tm_pt_14=-16.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 lm_0=-43.128 lm_1=-36.462 WordPenalty=-7.817 OOVPenalty=0.000 ||| -99.795
-14 ||| tata communication " foreign sanchar nigam limited building is the telecommunication system is one of the providers ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-12.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-49.485 tm_pt_6=-17.961 tm_pt_7=-12.000 tm_pt_8=-32.616 tm_pt_9=-4.371 tm_pt_10=-27.543 tm_pt_11=0.000 tm_pt_12=-15.099 tm_pt_13=0.000 tm_pt_14=-17.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 lm_0=-43.606 lm_1=-37.456 WordPenalty=-8.252 OOVPenalty=0.000 ||| -100.118
-14 ||| tata communication " foreign sanchar nigam limited building is the main providers of telecommunication system is ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-12.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-43.603 tm_pt_6=-17.632 tm_pt_7=-12.000 tm_pt_8=-32.616 tm_pt_9=-3.439 tm_pt_10=-24.629 tm_pt_11=0.000 tm_pt_12=-15.084 tm_pt_13=0.000 tm_pt_14=-16.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 lm_0=-42.899 lm_1=-37.040 WordPenalty=-7.817 OOVPenalty=0.000 ||| -100.827
-15 ||| he that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national assembly in the won all and was elected as the 44th president of the united states . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-10.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-67.250 tm_pt_6=-18.027 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-2.383 tm_pt_10=-13.421 tm_pt_11=0.000 tm_pt_12=-6.471 tm_pt_13=0.000 tm_pt_14=-21.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-46.628 lm_1=-35.442 WordPenalty=-10.857 OOVPenalty=-200.000 ||| -285.117
-15 ||| in that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national assembly in the won all and was elected as the 44th president of the united states . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-10.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-72.353 tm_pt_6=-21.308 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-2.383 tm_pt_10=-17.619 tm_pt_11=0.000 tm_pt_12=-10.104 tm_pt_13=0.000 tm_pt_14=-21.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-43.704 lm_1=-35.202 WordPenalty=-10.857 OOVPenalty=-200.000 ||| -285.675
-15 ||| in the year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national assembly in the won all and was elected as the 44th president of the united states . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-74.382 tm_pt_6=-21.237 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.015 tm_pt_10=-23.652 tm_pt_11=0.000 tm_pt_12=-10.678 tm_pt_13=0.000 tm_pt_14=-21.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=9.000 lm_0=-42.047 lm_1=-35.283 WordPenalty=-10.857 OOVPenalty=-200.000 ||| -285.744
-15 ||| he the year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national assembly in the won all and was elected as the 44th president of the united states . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-69.279 tm_pt_6=-17.956 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.015 tm_pt_10=-19.454 tm_pt_11=0.000 tm_pt_12=-7.046 tm_pt_13=0.000 tm_pt_14=-21.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=9.000 lm_0=-46.478 lm_1=-34.757 WordPenalty=-10.857 OOVPenalty=-200.000 ||| -286.105
-15 ||| he that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national assembly in the won all and elected as the 44th president of the united states . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-10.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-62.544 tm_pt_6=-17.987 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-2.382 tm_pt_10=-14.160 tm_pt_11=0.000 tm_pt_12=-5.506 tm_pt_13=0.000 tm_pt_14=-20.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-46.630 lm_1=-35.981 WordPenalty=-10.423 OOVPenalty=-200.000 ||| -286.655
-16 ||| many indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be from upper stage ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-5.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-14.556 tm_pt_6=-12.125 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.000 tm_pt_10=-2.670 tm_pt_11=0.000 tm_pt_12=-0.912 tm_pt_13=0.000 tm_pt_14=-6.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-35.728 lm_1=-25.931 WordPenalty=-4.777 OOVPenalty=-300.000 ||| -364.209
-16 ||| many indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be to upper stage ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-5.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-16.518 tm_pt_6=-13.004 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.000 tm_pt_10=-4.474 tm_pt_11=0.000 tm_pt_12=-2.241 tm_pt_13=0.000 tm_pt_14=-6.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-35.464 lm_1=-25.936 WordPenalty=-4.777 OOVPenalty=-300.000 ||| -365.503
-16 ||| many indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be upper stage from ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-5.000 tm_pt_3=0.000 tm_pt_4=-4.000 tm_pt_5=-14.556 tm_pt_6=-12.125 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.000 tm_pt_10=-2.830 tm_pt_11=0.000 tm_pt_12=-1.650 tm_pt_13=0.000 tm_pt_14=-6.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 lm_0=-36.278 lm_1=-25.431 WordPenalty=-4.777 OOVPenalty=-300.000 ||| -365.538
-16 ||| many indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 from \u099f\u09c7\u0995\u09cd\u09b8\u09be upper stage ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-5.000 tm_pt_3=0.000 tm_pt_4=-4.000 tm_pt_5=-14.556 tm_pt_6=-12.125 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.000 tm_pt_10=-3.002 tm_pt_11=0.000 tm_pt_12=-1.537 tm_pt_13=0.000 tm_pt_14=-6.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 lm_0=-35.728 lm_1=-26.794 WordPenalty=-4.777 OOVPenalty=-300.000 ||| -366.548
-16 ||| of indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be from upper stage ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-5.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-19.990 tm_pt_6=-14.047 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.050 tm_pt_10=-6.670 tm_pt_11=0.000 tm_pt_12=-3.477 tm_pt_13=0.000 tm_pt_14=-6.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-34.630 lm_1=-26.215 WordPenalty=-4.777 OOVPenalty=-300.000 ||| -366.885
-16 ||| many the \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be from upper stage ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-5.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-18.962 tm_pt_6=-12.755 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.000 tm_pt_10=-7.024 tm_pt_11=0.000 tm_pt_12=-2.862 tm_pt_13=0.000 tm_pt_14=-6.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-35.098 lm_1=-26.535 WordPenalty=-4.777 OOVPenalty=-300.000 ||| -367.054
-16 ||| many indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be from the upper stage ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-5.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-20.408 tm_pt_6=-12.744 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.000 tm_pt_10=-3.443 tm_pt_11=0.000 tm_pt_12=-4.470 tm_pt_13=0.000 tm_pt_14=-7.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-35.506 lm_1=-27.896 WordPenalty=-5.212 OOVPenalty=-300.000 ||| -367.223
-17 ||| britain writers written drama novels and stories recently scripts in \u0986\u09a6\u09c3\u09a4 . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-20.642 tm_pt_6=-10.927 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-0.874 tm_pt_10=-21.259 tm_pt_11=0.000 tm_pt_12=-8.774 tm_pt_13=0.000 tm_pt_14=-11.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=9.000 lm_0=-36.970 lm_1=-24.414 WordPenalty=-6.080 OOVPenalty=-100.000 ||| -170.879
-17 ||| britain writers the drama novels and stories recently scripts in \u0986\u09a6\u09c3\u09a4 . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-25.509 tm_pt_6=-11.095 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-0.959 tm_pt_10=-25.559 tm_pt_11=0.000 tm_pt_12=-9.061 tm_pt_13=0.000 tm_pt_14=-11.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=9.000 lm_0=-36.194 lm_1=-23.463 WordPenalty=-6.080 OOVPenalty=-100.000 ||| -171.018
-17 ||| britain writers written drama novels stories and recently scripts in \u0986\u09a6\u09c3\u09a4 . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-20.642 tm_pt_6=-10.927 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-0.874 tm_pt_10=-16.732 tm_pt_11=0.000 tm_pt_12=-5.024 tm_pt_13=0.000 tm_pt_14=-11.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 lm_0=-40.717 lm_1=-24.768 WordPenalty=-6.080 OOVPenalty=-100.000 ||| -171.523
-17 ||| britain writers written drama novels and stories recently script in \u0986\u09a6\u09c3\u09a4 . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-21.418 tm_pt_6=-10.442 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-1.824 tm_pt_10=-21.547 tm_pt_11=0.000 tm_pt_12=-10.160 tm_pt_13=0.000 tm_pt_14=-11.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=9.000 lm_0=-36.674 lm_1=-24.414 WordPenalty=-6.080 OOVPenalty=-100.000 ||| -171.840
-17 ||| britain writers the drama novels and stories recently script in \u0986\u09a6\u09c3\u09a4 . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-26.285 tm_pt_6=-10.609 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-1.909 tm_pt_10=-25.847 tm_pt_11=0.000 tm_pt_12=-10.448 tm_pt_13=0.000 tm_pt_14=-11.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=9.000 lm_0=-35.899 lm_1=-23.463 WordPenalty=-6.080 OOVPenalty=-100.000 ||| -171.980
-17 ||| britain writers written drama novel stories and recently scripts in \u0986\u09a6\u09c3\u09a4 . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-21.690 tm_pt_6=-11.374 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-0.874 tm_pt_10=-17.888 tm_pt_11=0.000 tm_pt_12=-5.391 tm_pt_13=0.000 tm_pt_14=-11.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 lm_0=-40.536 lm_1=-24.768 WordPenalty=-6.080 OOVPenalty=-100.000 ||| -172.083
-18 ||| 1919 on may month it -lrb- magazine was published . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-8.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-20.792 tm_pt_6=-13.466 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-1.167 tm_pt_10=-12.018 tm_pt_11=0.000 tm_pt_12=-7.168 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-26.824 lm_1=-17.292 WordPenalty=-5.212 OOVPenalty=0.000 ||| -48.758
-18 ||| 1919 on may month it -lrb- paper published . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-8.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-18.147 tm_pt_6=-15.056 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-1.645 tm_pt_10=-12.241 tm_pt_11=0.000 tm_pt_12=-7.456 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 lm_0=-25.565 lm_1=-16.091 WordPenalty=-4.777 OOVPenalty=0.000 ||| -49.202
-18 ||| 1919 on may month it -lrb- in published . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-8.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-21.062 tm_pt_6=-12.823 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-1.327 tm_pt_10=-17.056 tm_pt_11=0.000 tm_pt_12=-8.085 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-25.508 lm_1=-16.231 WordPenalty=-4.777 OOVPenalty=0.000 ||| -49.344
-18 ||| 1919 on may month it -lrb- magazine published . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-8.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-16.574 tm_pt_6=-13.110 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-1.284 tm_pt_10=-12.018 tm_pt_11=0.000 tm_pt_12=-7.679 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-27.629 lm_1=-16.091 WordPenalty=-4.777 OOVPenalty=0.000 ||| -49.550
-18 ||| 1919 on may , it -lrb- paper published . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-7.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-24.019 tm_pt_6=-15.257 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-2.510 tm_pt_10=-12.137 tm_pt_11=0.000 tm_pt_12=-5.525 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-23.603 lm_1=-17.056 WordPenalty=-4.777 OOVPenalty=0.000 ||| -49.933
-18 ||| 1919 on may , it -lrb- in published . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-7.000 tm_pt_3=0.000 tm_pt_4=-6.000 tm_pt_5=-26.935 tm_pt_6=-13.023 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-2.192 tm_pt_10=-16.952 tm_pt_11=0.000 tm_pt_12=-6.153 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=6.000 lm_0=-23.546 lm_1=-17.195 WordPenalty=-4.777 OOVPenalty=0.000 ||| -50.076
-18 ||| 1919 in may , it -lrb- paper published . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-7.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-26.096 tm_pt_6=-15.877 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-1.878 tm_pt_10=-11.444 tm_pt_11=0.000 tm_pt_12=-4.831 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-24.006 lm_1=-17.081 WordPenalty=-4.777 OOVPenalty=0.000 ||| -50.130
-18 ||| 1919 on may , it is -lrb- in published . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-7.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-30.789 tm_pt_6=-13.290 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-2.192 tm_pt_10=-16.360 tm_pt_11=0.000 tm_pt_12=-6.691 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-24.112 lm_1=-16.790 WordPenalty=-5.212 OOVPenalty=0.000 ||| -50.170
-18 ||| 1919 in may , it -lrb- in published . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-7.000 tm_pt_3=0.000 tm_pt_4=-6.000 tm_pt_5=-29.012 tm_pt_6=-13.644 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-1.560 tm_pt_10=-16.259 tm_pt_11=0.000 tm_pt_12=-5.460 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=6.000 lm_0=-23.949 lm_1=-17.221 WordPenalty=-4.777 OOVPenalty=0.000 ||| -50.272
-19 ||| 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium was arranged . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-21.244 tm_pt_6=-8.707 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.249 tm_pt_10=-15.730 tm_pt_11=0.000 tm_pt_12=-5.148 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=14.000 lm_0=-57.604 lm_1=-39.798 WordPenalty=-7.817 OOVPenalty=-400.000 ||| -501.538
-19 ||| 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium was organized . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-22.039 tm_pt_6=-8.841 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.249 tm_pt_10=-16.240 tm_pt_11=0.000 tm_pt_12=-5.148 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=14.000 lm_0=-57.677 lm_1=-39.749 WordPenalty=-7.817 OOVPenalty=-400.000 ||| -501.898
-19 ||| 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium organized in . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-11.000 tm_pt_5=-24.172 tm_pt_6=-9.959 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.249 tm_pt_10=-16.993 tm_pt_11=0.000 tm_pt_12=-6.193 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=15.000 lm_0=-58.336 lm_1=-39.009 WordPenalty=-7.817 OOVPenalty=-400.000 ||| -502.189
-19 ||| 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium organized was . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-11.000 tm_pt_5=-22.039 tm_pt_6=-8.841 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.249 tm_pt_10=-15.520 tm_pt_11=0.000 tm_pt_12=-5.062 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=15.000 lm_0=-59.983 lm_1=-39.504 WordPenalty=-7.817 OOVPenalty=-400.000 ||| -503.267
-20 ||| to prevent this several measures are taken . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-1.000 tm_pt_3=0.000 tm_pt_4=-1.000 tm_pt_5=-22.680 tm_pt_6=-30.812 tm_pt_7=-1.000 tm_pt_8=-2.718 tm_pt_9=-1.000 tm_pt_10=0.000 tm_pt_11=0.000 tm_pt_12=-1.386 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=1.000 lm_0=-11.632 lm_1=-10.983 WordPenalty=-4.343 OOVPenalty=0.000 ||| -25.678
-20 ||| to avoid this possibility several measures are taken . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-1.000 tm_pt_3=0.000 tm_pt_4=-1.000 tm_pt_5=-24.597 tm_pt_6=-31.733 tm_pt_7=-1.000 tm_pt_8=-2.718 tm_pt_9=-1.000 tm_pt_10=0.000 tm_pt_11=0.000 tm_pt_12=-1.386 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=1.000 lm_0=-13.461 lm_1=-12.428 WordPenalty=-4.777 OOVPenalty=0.000 ||| -28.478
-20 ||| to prevent this several measures are the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-3.000 tm_pt_3=0.000 tm_pt_4=-3.000 tm_pt_5=-24.227 tm_pt_6=-27.251 tm_pt_7=-3.000 tm_pt_8=-8.154 tm_pt_9=-1.000 tm_pt_10=-3.426 tm_pt_11=0.000 tm_pt_12=-2.285 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=3.000 lm_0=-14.066 lm_1=-12.982 WordPenalty=-4.343 OOVPenalty=0.000 ||| -31.830
-20 ||| to prevent this several measures are . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-1.000 tm_pt_5=-20.219 tm_pt_6=-29.189 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-1.000 tm_pt_10=-6.851 tm_pt_11=0.000 tm_pt_12=-1.946 tm_pt_13=0.000 tm_pt_14=-7.000 tm_pt_15=-1.000 tm_pt_16=0.000 tm_glue_0=1.000 lm_0=-12.686 lm_1=-11.625 WordPenalty=-3.909 OOVPenalty=0.000 ||| -32.367
-20 ||| to prevent this several measures are in . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-3.000 tm_pt_3=0.000 tm_pt_4=-3.000 tm_pt_5=-23.388 tm_pt_6=-27.344 tm_pt_7=-3.000 tm_pt_8=-8.154 tm_pt_9=-1.000 tm_pt_10=-2.771 tm_pt_11=0.000 tm_pt_12=-2.699 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=3.000 lm_0=-14.649 lm_1=-12.982 WordPenalty=-4.343 OOVPenalty=0.000 ||| -32.400
-20 ||| to prevent this several measures are taken . . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-22.746 tm_pt_6=-30.812 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-1.368 tm_pt_10=0.000 tm_pt_11=0.000 tm_pt_12=-10.427 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 lm_0=-15.278 lm_1=-12.549 WordPenalty=-4.777 OOVPenalty=0.000 ||| -33.898
-21 ||| \u09e7\u09ef\u09ec\u09ec on 5 february \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 against a national was held in . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-8.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-19.901 tm_pt_6=-11.529 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-1.135 tm_pt_10=-5.272 tm_pt_11=0.000 tm_pt_12=-3.806 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 lm_0=-42.873 lm_1=-29.512 WordPenalty=-6.514 OOVPenalty=-300.000 ||| -373.091
-21 ||| \u09e7\u09ef\u09ec\u09ec on 5th february , \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 against a national was held in . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-7.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-25.571 tm_pt_6=-13.159 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-1.503 tm_pt_10=-3.423 tm_pt_11=0.000 tm_pt_12=-3.325 tm_pt_13=0.000 tm_pt_14=-11.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-42.544 lm_1=-30.355 WordPenalty=-6.949 OOVPenalty=-300.000 ||| -373.863
-21 ||| \u09e7\u09ef\u09ec\u09ec on 5 february \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 against \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 a national was held in . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-8.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-19.901 tm_pt_6=-11.529 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-1.135 tm_pt_10=-6.095 tm_pt_11=0.000 tm_pt_12=-4.029 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 lm_0=-43.433 lm_1=-30.334 WordPenalty=-6.514 OOVPenalty=-300.000 ||| -374.129
-21 ||| \u09e7\u09ef\u09ec\u09ec on 5 february \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 of \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 a national was held in . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-8.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-24.651 tm_pt_6=-11.840 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-1.018 tm_pt_10=-8.289 tm_pt_11=0.000 tm_pt_12=-3.518 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 lm_0=-42.147 lm_1=-30.745 WordPenalty=-6.514 OOVPenalty=-300.000 ||| -374.354
-21 ||| \u09e7\u09ef\u09ec\u09ec on 5 february \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 a national was held in against . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-8.000 tm_pt_3=0.000 tm_pt_4=-3.000 tm_pt_5=-19.901 tm_pt_6=-11.529 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-0.154 tm_pt_10=-5.625 tm_pt_11=0.000 tm_pt_12=-3.764 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=6.000 lm_0=-43.384 lm_1=-28.686 WordPenalty=-6.514 OOVPenalty=-300.000 ||| -375.831
-22 ||| bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in in the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-11.000 tm_pt_5=-50.512 tm_pt_6=-9.888 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.050 tm_pt_10=-23.145 tm_pt_11=0.000 tm_pt_12=-2.086 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=14.000 lm_0=-45.104 lm_1=-39.650 WordPenalty=-8.252 OOVPenalty=-300.000 ||| -390.343
-22 ||| bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank took secured its place in the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-46.603 tm_pt_6=-9.693 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.368 tm_pt_10=-19.856 tm_pt_11=0.000 tm_pt_12=-3.210 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=13.000 lm_0=-45.737 lm_1=-39.265 WordPenalty=-8.252 OOVPenalty=-300.000 ||| -390.606
-22 ||| bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in took the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-11.000 tm_pt_5=-46.603 tm_pt_6=-9.693 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.368 tm_pt_10=-19.978 tm_pt_11=0.000 tm_pt_12=-2.779 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=14.000 lm_0=-46.563 lm_1=-39.704 WordPenalty=-8.252 OOVPenalty=-300.000 ||| -391.085
-22 ||| bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in in . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-10.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-47.305 tm_pt_6=-11.674 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-2.050 tm_pt_10=-27.018 tm_pt_11=0.000 tm_pt_12=-1.913 tm_pt_13=0.000 tm_pt_14=-13.000 tm_pt_15=-1.000 tm_pt_16=0.000 tm_glue_0=12.000 lm_0=-44.173 lm_1=-37.800 WordPenalty=-7.817 OOVPenalty=-300.000 ||| -391.101
-22 ||| bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in adopted . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-10.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-42.382 tm_pt_6=-12.116 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-3.000 tm_pt_10=-21.979 tm_pt_11=0.000 tm_pt_12=-3.300 tm_pt_13=0.000 tm_pt_14=-13.000 tm_pt_15=-1.000 tm_pt_16=0.000 tm_glue_0=12.000 lm_0=-45.321 lm_1=-37.892 WordPenalty=-7.817 OOVPenalty=-300.000 ||| -391.808
-22 ||| bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in took . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-10.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-43.395 tm_pt_6=-11.480 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-2.368 tm_pt_10=-23.851 tm_pt_11=0.000 tm_pt_12=-2.607 tm_pt_13=0.000 tm_pt_14=-13.000 tm_pt_15=-1.000 tm_pt_16=0.000 tm_glue_0=12.000 lm_0=-45.354 lm_1=-38.351 WordPenalty=-7.817 OOVPenalty=-300.000 ||| -392.115
-22 ||| \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 bangladesh of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in in the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-50.512 tm_pt_6=-9.888 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.050 tm_pt_10=-25.870 tm_pt_11=0.000 tm_pt_12=-5.409 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=13.000 lm_0=-46.143 lm_1=-37.905 WordPenalty=-8.252 OOVPenalty=-300.000 ||| -392.315
-23 ||| subject : encyclopedia ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-1.657 tm_pt_6=-1.542 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.000 tm_pt_10=-0.420 tm_pt_11=0.000 tm_pt_12=-1.500 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 lm_0=-5.528 lm_1=-7.730 WordPenalty=-2.171 OOVPenalty=0.000 ||| -9.260
-23 ||| subject category : encyclopedia ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-1.727 tm_pt_6=-1.749 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.135 tm_pt_10=0.000 tm_pt_11=0.000 tm_pt_12=-1.881 tm_pt_13=0.000 tm_pt_14=-4.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 lm_0=-5.763 lm_1=-9.076 WordPenalty=-2.606 OOVPenalty=0.000 ||| -9.504
-23 ||| category : encyclopedia ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-1.425 tm_pt_6=-2.012 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.135 tm_pt_10=-0.020 tm_pt_11=0.000 tm_pt_12=-1.817 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 lm_0=-5.707 lm_1=-7.730 WordPenalty=-2.171 OOVPenalty=0.000 ||| -9.718
-23 ||| subject matter : encyclopedia ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-3.165 tm_pt_6=-2.139 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.135 tm_pt_10=-0.032 tm_pt_11=0.000 tm_pt_12=-2.958 tm_pt_13=0.000 tm_pt_14=-4.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 lm_0=-6.126 lm_1=-9.076 WordPenalty=-2.606 OOVPenalty=0.000 ||| -10.751
-23 ||| subject-class : encyclopedia ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-1.379 tm_pt_6=-3.561 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.135 tm_pt_10=0.000 tm_pt_11=0.000 tm_pt_12=-2.703 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 lm_0=-5.989 lm_1=-7.730 WordPenalty=-2.171 OOVPenalty=0.000 ||| -10.896
-24 ||| russia france and israel the main \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 state and defense sub country . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-9.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-18.467 tm_pt_6=-9.206 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-1.474 tm_pt_10=-10.535 tm_pt_11=0.000 tm_pt_12=-5.933 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 lm_0=-38.802 lm_1=-26.775 WordPenalty=-6.514 OOVPenalty=-100.000 ||| -166.562
-24 ||| russia france and israel the main \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 country and defence sub country . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-10.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-18.942 tm_pt_6=-9.742 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-1.738 tm_pt_10=-10.065 tm_pt_11=0.000 tm_pt_12=-6.534 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 lm_0=-38.535 lm_1=-26.552 WordPenalty=-6.514 OOVPenalty=-100.000 ||| -166.728
-24 ||| russia france and israel the main \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 country and defense sub country . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-10.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-18.530 tm_pt_6=-8.356 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-1.106 tm_pt_10=-10.471 tm_pt_11=0.000 tm_pt_12=-5.841 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 lm_0=-38.844 lm_1=-27.238 WordPenalty=-6.514 OOVPenalty=-100.000 ||| -166.746
-24 ||| russia france and israel the main \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 countries and defense sub country . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-9.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-19.426 tm_pt_6=-10.343 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-1.474 tm_pt_10=-10.130 tm_pt_11=0.000 tm_pt_12=-5.933 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 lm_0=-38.815 lm_1=-27.108 WordPenalty=-6.514 OOVPenalty=-100.000 ||| -167.457
-25 ||| this is the known imaginary formed with the help of which mathematics are real number set from \u09b8\u09c7\u099f\u09c7 par with the complex number . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-60.559 tm_pt_6=-22.004 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-6.553 tm_pt_10=-17.782 tm_pt_11=0.000 tm_pt_12=-5.911 tm_pt_13=0.000 tm_pt_14=-23.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=9.000 lm_0=-50.351 lm_1=-44.022 WordPenalty=-11.292 OOVPenalty=-100.000 ||| -205.244
-25 ||| this is the known as imaginary formed with the help of which mathematics are real number set from \u09b8\u09c7\u099f\u09c7 par with the complex number . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-14.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-63.543 tm_pt_6=-22.038 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-5.553 tm_pt_10=-25.182 tm_pt_11=0.000 tm_pt_12=-8.041 tm_pt_13=0.000 tm_pt_14=-24.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=10.000 lm_0=-50.186 lm_1=-43.889 WordPenalty=-11.726 OOVPenalty=-100.000 ||| -205.568
-25 ||| this is the known imaginary formed with the help of which mathematics are real number set to \u09b8\u09c7\u099f\u09c7 par with the complex number . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-62.521 tm_pt_6=-22.883 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-6.553 tm_pt_10=-19.587 tm_pt_11=0.000 tm_pt_12=-7.240 tm_pt_13=0.000 tm_pt_14=-23.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=9.000 lm_0=-50.087 lm_1=-44.043 WordPenalty=-11.292 OOVPenalty=-100.000 ||| -206.559
-25 ||| this is the known imaginary formed with the help of which mathematics are set of real numbers from \u09b8\u09c7\u099f\u09c7 par with the complex number . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-14.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-60.947 tm_pt_6=-22.704 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-5.921 tm_pt_10=-18.256 tm_pt_11=0.000 tm_pt_12=-6.322 tm_pt_13=0.000 tm_pt_14=-24.000 tm_pt_15=0.000 tm_pt_16=-2.000 tm_glue_0=9.000 lm_0=-49.282 lm_1=-46.640 WordPenalty=-11.726 OOVPenalty=-100.000 ||| -206.680
-25 ||| this is the known imaginary formed with the help of which mathematics are set of real numbers to \u09b8\u09c7\u099f\u09c7 par with the complex number . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-14.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-62.909 tm_pt_6=-23.583 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-5.921 tm_pt_10=-20.060 tm_pt_11=0.000 tm_pt_12=-7.652 tm_pt_13=0.000 tm_pt_14=-24.000 tm_pt_15=0.000 tm_pt_16=-2.000 tm_glue_0=9.000 lm_0=-48.838 lm_1=-46.153 WordPenalty=-11.726 OOVPenalty=-100.000 ||| -207.143
-26 ||| <address> ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-1.000 tm_pt_3=0.000 tm_pt_4=-1.000 tm_pt_5=-1.494 tm_pt_6=-38.184 tm_pt_7=-1.000 tm_pt_8=-2.718 tm_pt_9=-0.050 tm_pt_10=0.000 tm_pt_11=0.000 tm_pt_12=0.000 tm_pt_13=0.000 tm_pt_14=-1.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=1.000 lm_0=-4.240 lm_1=-3.850 WordPenalty=-1.303 OOVPenalty=0.000 ||| -18.437
-26 ||| < \u09a0\u09bf\u0995\u09be\u09a8\u09be > ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-2.518 tm_pt_6=-29.231 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.000 tm_pt_10=-0.118 tm_pt_11=0.000 tm_pt_12=0.000 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=3.000 lm_0=-15.853 lm_1=-9.647 WordPenalty=-2.171 OOVPenalty=-100.000 ||| -132.757
-26 ||| the lt \u09a0\u09bf\u0995\u09be\u09a8\u09be > ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-1.000 tm_pt_5=-6.362 tm_pt_6=-20.589 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.135 tm_pt_10=-2.453 tm_pt_11=0.000 tm_pt_12=0.000 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=-1.000 tm_pt_16=0.000 tm_glue_0=2.000 lm_0=-17.709 lm_1=-13.346 WordPenalty=-2.606 OOVPenalty=-200.000 ||| -238.041
-26 ||| < \u09a0\u09bf\u0995\u09be\u09a8\u09be , gt , ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-3.000 tm_pt_3=0.000 tm_pt_4=-3.000 tm_pt_5=-4.258 tm_pt_6=-15.720 tm_pt_7=-3.000 tm_pt_8=-8.154 tm_pt_9=-0.000 tm_pt_10=-5.328 tm_pt_11=0.000 tm_pt_12=-1.262 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-20.678 lm_1=-15.916 WordPenalty=-3.040 OOVPenalty=-200.000 ||| -239.649
-26 ||| < \u09a0\u09bf\u0995\u09be\u09a8\u09be , gt ; ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-3.000 tm_pt_3=0.000 tm_pt_4=-3.000 tm_pt_5=-3.201 tm_pt_6=-18.449 tm_pt_7=-3.000 tm_pt_8=-8.154 tm_pt_9=-0.000 tm_pt_10=-1.596 tm_pt_11=0.000 tm_pt_12=-1.248 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-21.561 lm_1=-15.792 WordPenalty=-3.040 OOVPenalty=-200.000 ||| -240.345
-26 ||| , lt , \u09a0\u09bf\u0995\u09be\u09a8\u09be > ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-3.000 tm_pt_3=0.000 tm_pt_4=-3.000 tm_pt_5=-4.245 tm_pt_6=-15.998 tm_pt_7=-3.000 tm_pt_8=-8.154 tm_pt_9=-0.000 tm_pt_10=-5.446 tm_pt_11=0.000 tm_pt_12=-1.262 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-20.961 lm_1=-16.373 WordPenalty=-3.040 OOVPenalty=-200.000 ||| -240.688
-27 ||| september ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-1.000 tm_pt_3=0.000 tm_pt_4=-1.000 tm_pt_5=-0.176 tm_pt_6=-0.047 tm_pt_7=-1.000 tm_pt_8=-2.718 tm_pt_9=-0.000 tm_pt_10=-0.013 tm_pt_11=0.000 tm_pt_12=-0.025 tm_pt_13=0.000 tm_pt_14=-1.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=1.000 lm_0=-3.024 lm_1=-3.825 WordPenalty=-1.303 OOVPenalty=0.000 ||| -3.585
-27 ||| september . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-1.000 tm_pt_3=0.000 tm_pt_4=-1.000 tm_pt_5=-9.282 tm_pt_6=-0.716 tm_pt_7=-1.000 tm_pt_8=-2.718 tm_pt_9=-0.368 tm_pt_10=-1.099 tm_pt_11=0.000 tm_pt_12=-3.689 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=1.000 lm_0=-4.832 lm_1=-5.754 WordPenalty=-1.737 OOVPenalty=0.000 ||| -10.126
-27 ||| \u09b8\u09c7\u09aa\u09cd\u099f\u09c7\u09ae\u09cd\u09ac\u09b0 ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=0.000 tm_pt_5=0.000 tm_pt_6=0.000 tm_pt_7=0.000 tm_pt_8=0.000 tm_pt_9=0.000 tm_pt_10=0.000 tm_pt_11=0.000 tm_pt_12=0.000 tm_pt_13=0.000 tm_pt_14=0.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=1.000 lm_0=-7.355 lm_1=-4.974 WordPenalty=-1.303 OOVPenalty=-100.000 ||| -109.443
-28 ||| from this theory though big \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 to can not be rather it can be support . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-10.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-29.487 tm_pt_6=-24.381 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-2.525 tm_pt_10=-16.787 tm_pt_11=0.000 tm_pt_12=-5.740 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-36.059 lm_1=-28.266 WordPenalty=-7.817 OOVPenalty=-100.000 ||| -172.006
-28 ||| from this theory though big \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 against can not be rather it can be support . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-10.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-25.087 tm_pt_6=-23.283 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-1.575 tm_pt_10=-12.592 tm_pt_11=0.000 tm_pt_12=-4.353 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-37.068 lm_1=-30.729 WordPenalty=-7.817 OOVPenalty=-100.000 ||| -172.727
-28 ||| from this theory though big \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 opposed can not be rather it can be support . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-10.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-23.358 tm_pt_6=-23.283 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-2.525 tm_pt_10=-10.887 tm_pt_11=0.000 tm_pt_12=-5.740 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-37.843 lm_1=-30.003 WordPenalty=-7.817 OOVPenalty=-100.000 ||| -173.285
-28 ||| from this theory though big \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 to can not be rather it can be supported . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-10.000 tm_pt_3=0.000 tm_pt_4=-6.000 tm_pt_5=-29.070 tm_pt_6=-23.889 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-3.524 tm_pt_10=-17.914 tm_pt_11=0.000 tm_pt_12=-7.821 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 lm_0=-35.106 lm_1=-28.266 WordPenalty=-7.817 OOVPenalty=-100.000 ||| -173.391
-28 ||| from this theory though big \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 against can not be but it can be support . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-10.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-27.960 tm_pt_6=-23.108 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-1.575 tm_pt_10=-14.990 tm_pt_11=0.000 tm_pt_12=-4.353 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-35.950 lm_1=-31.509 WordPenalty=-7.817 OOVPenalty=-100.000 ||| -173.420
-28 ||| from this theory though big \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 against can not be support rather it can be . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-10.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-25.087 tm_pt_6=-23.283 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-1.942 tm_pt_10=-13.423 tm_pt_11=0.000 tm_pt_12=-5.046 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 lm_0=-37.028 lm_1=-30.243 WordPenalty=-7.817 OOVPenalty=-100.000 ||| -173.838
-28 ||| from this theory though big \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 against can not be support but it can be . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-10.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-27.960 tm_pt_6=-23.108 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-1.942 tm_pt_10=-15.821 tm_pt_11=0.000 tm_pt_12=-5.046 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 lm_0=-35.911 lm_1=-30.724 WordPenalty=-7.817 OOVPenalty=-100.000 ||| -174.163
-28 ||| from this theory though big \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 protested can not be rather it can be support . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-23.293 tm_pt_6=-24.158 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-1.525 tm_pt_10=-12.568 tm_pt_11=0.000 tm_pt_12=-6.573 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=9.000 lm_0=-38.421 lm_1=-30.003 WordPenalty=-7.817 OOVPenalty=-100.000 ||| -174.388
-29 ||| agricultural in production france is the most important country ; it is \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af wine cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and export in the whole world . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-15.000 tm_pt_3=0.000 tm_pt_4=-12.000 tm_pt_5=-61.979 tm_pt_6=-21.379 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-3.566 tm_pt_10=-19.501 tm_pt_11=0.000 tm_pt_12=-6.075 tm_pt_13=0.000 tm_pt_14=-24.000 tm_pt_15=-1.000 tm_pt_16=0.000 tm_glue_0=14.000 lm_0=-60.198 lm_1=-49.635 WordPenalty=-12.160 OOVPenalty=-200.000 ||| -316.684
-29 ||| agricultural in production france is the most important country ; it basically \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af wine cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and export in the whole world . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-15.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-56.780 tm_pt_6=-21.542 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-3.477 tm_pt_10=-13.337 tm_pt_11=0.000 tm_pt_12=-5.633 tm_pt_13=0.000 tm_pt_14=-24.000 tm_pt_15=-1.000 tm_pt_16=0.000 tm_glue_0=15.000 lm_0=-63.240 lm_1=-49.990 WordPenalty=-12.160 OOVPenalty=-200.000 ||| -317.154
-29 ||| agricultural in production france country ; it is the most important mainly \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af wine cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and export in the whole world . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-14.000 tm_pt_3=0.000 tm_pt_4=-11.000 tm_pt_5=-54.055 tm_pt_6=-20.128 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-3.477 tm_pt_10=-11.319 tm_pt_11=0.000 tm_pt_12=-3.338 tm_pt_13=0.000 tm_pt_14=-24.000 tm_pt_15=-1.000 tm_pt_16=0.000 tm_glue_0=12.000 lm_0=-62.879 lm_1=-50.811 WordPenalty=-12.160 OOVPenalty=-200.000 ||| -317.157
-29 ||| agricultural in production france is the most important country ; it mainly \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af wine cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and export in the whole world . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-15.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-57.050 tm_pt_6=-20.546 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-3.474 tm_pt_10=-13.783 tm_pt_11=0.000 tm_pt_12=-4.113 tm_pt_13=0.000 tm_pt_14=-24.000 tm_pt_15=-1.000 tm_pt_16=0.000 tm_glue_0=15.000 lm_0=-63.884 lm_1=-49.990 WordPenalty=-12.160 OOVPenalty=-200.000 ||| -317.238
-29 ||| agriculture in production france is the most important country ; it is \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af wine cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and export in the whole world . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-14.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-62.243 tm_pt_6=-21.939 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-2.838 tm_pt_10=-19.370 tm_pt_11=0.000 tm_pt_12=-7.203 tm_pt_13=0.000 tm_pt_14=-24.000 tm_pt_15=-1.000 tm_pt_16=0.000 tm_glue_0=12.000 lm_0=-60.690 lm_1=-49.635 WordPenalty=-12.160 OOVPenalty=-200.000 ||| -318.076
-29 ||| agricultural in production france is the most important country ; it is \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af wine cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and export the in the whole world . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-16.000 tm_pt_3=0.000 tm_pt_4=-12.000 tm_pt_5=-65.187 tm_pt_6=-19.593 tm_pt_7=-16.000 tm_pt_8=-43.488 tm_pt_9=-3.560 tm_pt_10=-16.076 tm_pt_11=0.000 tm_pt_12=-5.939 tm_pt_13=0.000 tm_pt_14=-25.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=14.000 lm_0=-62.535 lm_1=-50.774 WordPenalty=-12.595 OOVPenalty=-200.000 ||| -318.715
-29 ||| agricultural in production france is the most important country ; it basically \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af wine cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and export the in the whole world . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-16.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-59.988 tm_pt_6=-19.755 tm_pt_7=-16.000 tm_pt_8=-43.488 tm_pt_9=-3.470 tm_pt_10=-9.911 tm_pt_11=0.000 tm_pt_12=-5.497 tm_pt_13=0.000 tm_pt_14=-25.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=15.000 lm_0=-65.577 lm_1=-51.128 WordPenalty=-12.595 OOVPenalty=-200.000 ||| -319.185
-29 ||| agricultural in production france is the most important country ; it mainly \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af wine cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and export the in the whole world . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-16.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-60.258 tm_pt_6=-18.760 tm_pt_7=-16.000 tm_pt_8=-43.488 tm_pt_9=
<TRUNCATED>
[24/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/browser/Browser.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/browser/Browser.java b/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/browser/Browser.java
new file mode 100644
index 0000000..ee22b94
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/browser/Browser.java
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.ui.tree_visualizer.browser;
+
+import java.awt.BorderLayout;
+import java.awt.Color;
+import java.awt.event.ActionEvent;
+import java.awt.event.ActionListener;
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Scanner;
+
+import javax.swing.DefaultListModel;
+import javax.swing.JFrame;
+import javax.swing.JList;
+import javax.swing.JScrollPane;
+import javax.swing.JTextField;
+import javax.swing.ListSelectionModel;
+import javax.swing.event.DocumentEvent;
+import javax.swing.event.DocumentListener;
+import javax.swing.event.ListSelectionEvent;
+import javax.swing.event.ListSelectionListener;
+
+import org.apache.joshua.ui.tree_visualizer.tree.Tree;
+import org.apache.joshua.util.io.LineReader;
+
+public class Browser {
+
+ /**
+ * A list that contains the one best translation of each source sentence.
+ */
+ private static JList oneBestList;
+
+ private static JTextField searchBox;
+
+ /**
+ * The current frame that displays a derivation tree.
+ */
+ private static List<DerivationTreeFrame> activeFrame;
+
+ private static List<TranslationInfo> translations;
+ /**
+ * Default width of the chooser frame.
+ */
+ private static final int DEFAULT_WIDTH = 640;
+
+ /**
+ * Default height of the chooser frame.
+ */
+ private static final int DEFAULT_HEIGHT = 480;
+
+ /**
+ * List of colors to be used in derivation trees
+ */
+ static final Color[] dataSetColors = { Color.red, Color.orange, Color.blue, Color.green };
+
+ /**
+ * @param argv the paths to the source, reference, and n-best files
+ * @throws IOException if there is an error reading from standard input
+ */
+ public static void main(String[] argv) throws IOException {
+ String sourcePath = argv.length > 0 ? argv[0] : null;
+ String referencePath = argv.length > 1 ? argv[1] : null;
+ String[] translationPaths = new String[0];
+ if (argv.length > 2) {
+ translationPaths = Arrays.copyOfRange(argv, 2, argv.length);
+ }
+ translations = new ArrayList<TranslationInfo>();
+ readSourcesFromPath(sourcePath);
+ readReferencesFromPath(referencePath);
+ for (String tp : translationPaths) {
+ readTranslationsFromPath(tp);
+ }
+ initializeChooserFrame();
+ return;
+ }
+
+ private static void readSourcesFromPath(String path) throws IOException {
+ for (String line: new LineReader(path)) {
+ TranslationInfo ti = new TranslationInfo();
+ ti.setSourceSentence("<s> " + line + " </s>");
+ translations.add(ti);
+ }
+ }
+
+ private static void readReferencesFromPath(String path) throws IOException {
+ Scanner scanner = new Scanner(new File(path), "UTF-8");
+ for (TranslationInfo ti : translations) {
+ if (scanner.hasNextLine()) {
+ ti.setReference(scanner.nextLine());
+ }
+ }
+ scanner.close();
+ }
+
+ private static void readTranslationsFromPath(String path) throws IOException {
+ Scanner scanner = new Scanner(new File(path), "UTF-8");
+ String sentenceIndex = null;
+ for (TranslationInfo ti : translations) {
+ while (scanner.hasNextLine()) {
+ final String[] fields = scanner.nextLine().split("\\|\\|\\|");
+ final String index = fields[0];
+ final String tree = fields[1].trim();
+ if (!index.equals(sentenceIndex)) {
+ sentenceIndex = index;
+ ti.translations().add(new Tree(tree));
+ break;
+ }
+ }
+ }
+ scanner.close();
+ }
+
+ /**
+ * Initializes the various JComponents in the chooser frame.
+ */
+ private static void initializeChooserFrame() {
+ JFrame chooserFrame = new JFrame("Joshua Derivation Tree Browser");
+ chooserFrame.setLayout(new BorderLayout());
+
+ /*
+ * JMenuBar mb = new JMenuBar(); JMenu openMenu = new JMenu("Control"); JMenuItem src = new
+ * JMenuItem("Open source file ..."); JMenuItem ref = new JMenuItem("Open reference file ...");
+ * JMenuItem tgt = new JMenuItem("Open n-best derivations file ..."); JMenuItem quit = new
+ * JMenuItem("Quit");
+ *
+ * new FileChoiceListener(chooserFrame, src, ref, tgt);
+ *
+ * quit.addActionListener(new ActionListener() { public void actionPerformed(ActionEvent e) {
+ * System.exit(0); } }); openMenu.add(src); openMenu.add(ref); openMenu.add(tgt);
+ * openMenu.add(quit); mb.add(openMenu); chooserFrame.setJMenuBar(mb);
+ */
+
+ searchBox = new JTextField("search");
+ searchBox.getDocument().addDocumentListener(new SearchListener());
+ searchBox.addActionListener(new ActionListener() {
+ public void actionPerformed(ActionEvent e) {
+ final int selectedIndex = oneBestList.getSelectedIndex();
+ Browser.search(selectedIndex < 0 ? 0 : selectedIndex + 1);
+ }
+ });
+ oneBestList = new JList(new DefaultListModel());
+ oneBestList.setFixedCellWidth(200);
+ oneBestList.setSelectionMode(ListSelectionModel.SINGLE_SELECTION);
+ // oneBestList.setCellRenderer(new DerivationBrowserListCellRenderer());
+
+ oneBestList.addListSelectionListener(new ListSelectionListener() {
+ public void valueChanged(ListSelectionEvent e) {
+ for (DerivationTreeFrame frame : activeFrame) {
+ frame.drawGraph(translations.get(oneBestList.getSelectedIndex()));
+ }
+ return;
+ }
+ });
+ chooserFrame.getContentPane().add(searchBox, BorderLayout.NORTH);
+ chooserFrame.getContentPane().add(new JScrollPane(oneBestList), BorderLayout.CENTER);
+
+ refreshLists();
+ chooserFrame.setSize(DEFAULT_WIDTH, DEFAULT_HEIGHT);
+ chooserFrame.setDefaultCloseOperation(JFrame.EXIT_ON_CLOSE);
+
+ activeFrame = new ArrayList<DerivationTreeFrame>();
+ int numNBestFiles = translations.get(0).translations().size();
+ for (int i = 0; i < numNBestFiles; i++)
+ activeFrame.add(new DerivationTreeFrame(i, oneBestList));
+ chooserFrame.setVisible(true);
+ return;
+ }
+
+ /**
+ * Removes and re-adds the appropriate values to the reference and one-best lists.
+ */
+ private static void refreshLists() {
+ oneBestList.removeAll();
+ DefaultListModel oneBestListModel = (DefaultListModel) oneBestList.getModel();
+ for (TranslationInfo ti : translations) {
+ oneBestListModel.addElement(ti.reference());
+ }
+ return;
+ }
+
+ private static void search(int fromIndex) {
+ final String query = searchBox.getText();
+ DefaultListModel oneBestListModel = (DefaultListModel) oneBestList.getModel();
+ for (int i = fromIndex; i < oneBestListModel.getSize(); i++) {
+ String reference = (String) oneBestListModel.getElementAt(i);
+ if (reference.indexOf(query) != -1) {
+ // found the query
+ oneBestList.setSelectedIndex(i);
+ oneBestList.ensureIndexIsVisible(i);
+ searchBox.setBackground(Color.white);
+ return;
+ }
+ }
+ searchBox.setBackground(Color.red);
+ }
+
+ private static class SearchListener implements DocumentListener {
+
+ public void insertUpdate(DocumentEvent e) {
+ final int selectedIndex = oneBestList.getSelectedIndex();
+ Browser.search(selectedIndex < 0 ? 0 : selectedIndex);
+ }
+
+ public void removeUpdate(DocumentEvent e) {
+ final String query = searchBox.getText();
+ if (query.equals("")) {
+ return;
+ } else {
+ insertUpdate(e);
+ }
+ }
+
+ public void changedUpdate(DocumentEvent e) {
+
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/browser/DerivationTreeFrame.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/browser/DerivationTreeFrame.java b/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/browser/DerivationTreeFrame.java
new file mode 100644
index 0000000..56366a0
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/browser/DerivationTreeFrame.java
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.ui.tree_visualizer.browser;
+
+import java.awt.BorderLayout;
+import java.awt.Color;
+import java.awt.GridLayout;
+import java.awt.event.ActionEvent;
+import java.awt.event.ActionListener;
+
+import javax.swing.JButton;
+import javax.swing.JFrame;
+import javax.swing.JLabel;
+import javax.swing.JList;
+import javax.swing.JPanel;
+
+import org.apache.joshua.ui.tree_visualizer.DerivationTree;
+import org.apache.joshua.ui.tree_visualizer.DerivationViewer;
+import org.apache.joshua.ui.tree_visualizer.tree.Tree;
+
+/**
+ * A frame that displays a derivation tree.
+ *
+ * @author jonny
+ *
+ */
+class DerivationTreeFrame extends JFrame {
+ /**
+ * Eclipse seems to think serialVersionUID is important. I don't know why.
+ */
+ private static final long serialVersionUID = -3173826443907629130L;
+
+ /**
+ * A button to move to the next source-side sentence in the file.
+ */
+ JButton nextSource;
+ /**
+ * A button to move to the previous source-side sentence in the file.
+ */
+ JButton previousSource;
+
+ /**
+ * A button to show or hide extra information about the derivation.
+ */
+ private JButton informationButton;
+
+ /**
+ * A panel holding the extra information about the derivation.
+ */
+ private JPanel informationPanel;
+
+ /**
+ * A label holding the current source sentence.
+ */
+ private JLabel sourceLabel;
+
+ /**
+ * A label holding the reference translation of the current source sentence.
+ */
+ private JLabel referenceLabel;
+
+ /**
+ * A label holding the one-best translation of the current source sentence.
+ */
+ private JLabel oneBestLabel;
+
+ /**
+ * A panel that holds the buttons, as well as labels to show which derivation
+ * is currently being displayed.
+ */
+ private JPanel controlPanel;
+ /**
+ * A panel used to display the derivation tree itself.
+ */
+ private JPanel viewPanel;
+
+ /**
+ * This component displays the derivation tree's JUNG graph.
+ */
+ private DerivationViewer dv;
+
+ /**
+ * Index to determine which data set (which n-best file) this frame brings its
+ * graphs from.
+ */
+ private final int dataSetIndex;
+
+ private static final int DEFAULT_WIDTH = 640;
+ private static final int DEFAULT_HEIGHT = 480;
+
+ /**
+ * Color to use to render target-side trees.
+ */
+ private Color targetColor;
+
+ private JList mainList;
+
+ /**
+ * The default constructor.
+ */
+ public DerivationTreeFrame(int index, JList mainList) {
+ super("Joshua Derivation Tree");
+ this.mainList = mainList;
+ setLayout(new BorderLayout());
+ setSize(DEFAULT_WIDTH, DEFAULT_HEIGHT);
+ controlPanel = new JPanel(new BorderLayout());
+ informationPanel = new JPanel(new GridLayout(3, 1));
+
+ sourceLabel = new JLabel("source sentence");
+ referenceLabel = new JLabel("reference translation");
+ oneBestLabel = new JLabel("one best translation");
+
+ informationPanel.add(sourceLabel);
+ informationPanel.add(referenceLabel);
+ informationPanel.add(oneBestLabel);
+ informationPanel.setVisible(false);
+
+ controlPanel.add(informationPanel, BorderLayout.SOUTH);
+
+ initializeButtons();
+ layoutControl();
+
+ viewPanel = new JPanel(new BorderLayout());
+ dv = null;
+
+ dataSetIndex = index;
+ targetColor = Browser.dataSetColors[dataSetIndex % Browser.dataSetColors.length];
+
+ getContentPane().add(viewPanel, BorderLayout.CENTER);
+ getContentPane().add(controlPanel, BorderLayout.SOUTH);
+ // drawGraph();
+ setVisible(true);
+ }
+
+ /**
+ * Lays out the control buttons of this frame.
+ */
+ private void layoutControl() {
+ /*
+ * JPanel ctlLeft = new JPanel(new GridLayout(2, 1)); JPanel ctlCenter = new
+ * JPanel(new GridLayout(2, 1)); JPanel ctlRight = new JPanel(new
+ * GridLayout(2, 1));
+ *
+ * controlPanel.add(ctlLeft, BorderLayout.WEST); controlPanel.add(ctlCenter,
+ * BorderLayout.CENTER); controlPanel.add(ctlRight, BorderLayout.EAST);
+ *
+ * ctlLeft.add(previousSource); ctlRight.add(nextSource);
+ */
+
+ controlPanel.add(previousSource, BorderLayout.WEST);
+ controlPanel.add(nextSource, BorderLayout.EAST);
+ controlPanel.add(informationButton, BorderLayout.CENTER);
+ return;
+ }
+
+ /**
+ * Initializes the control buttons of this frame.
+ */
+ private void initializeButtons() {
+ nextSource = new JButton(">");
+ previousSource = new JButton("<");
+ informationButton = new JButton("More Information");
+
+ nextSource.addActionListener(new ActionListener() {
+ public void actionPerformed(ActionEvent e) {
+ int index = mainList.getSelectedIndex();
+ mainList.setSelectedIndex(index + 1);
+ return;
+ }
+ });
+ previousSource.addActionListener(new ActionListener() {
+ public void actionPerformed(ActionEvent e) {
+ int index = mainList.getSelectedIndex();
+ if (index > 0) {
+ mainList.setSelectedIndex(index - 1);
+ }
+ return;
+ }
+ });
+ informationButton.addActionListener(new ActionListener() {
+ public void actionPerformed(ActionEvent e) {
+ JButton source = (JButton) e.getSource();
+ if (informationPanel.isVisible()) {
+ source.setText("More Information");
+ informationPanel.setVisible(false);
+ } else {
+ source.setText("Less Information");
+ informationPanel.setVisible(true);
+ }
+ return;
+ }
+ });
+ return;
+ }
+
+ /**
+ * Displays the derivation tree for the current candidate translation. The
+ * current candidate translation is whichever translation is currently
+ * highlighted in the Derivation Browser's chooser frame.
+ */
+ public void drawGraph(TranslationInfo ti) {
+ viewPanel.removeAll();
+ String src = ti.sourceSentence();
+ Tree tgt = ti.translations().get(dataSetIndex);
+ String ref = ti.reference();
+
+ sourceLabel.setText(src);
+ referenceLabel.setText(ref);
+ oneBestLabel.setText(tgt.yield());
+
+ DerivationTree tree = new DerivationTree(tgt, src);
+ if (dv == null) {
+ dv = new DerivationViewer(tree, viewPanel.getSize(), targetColor,
+ DerivationViewer.AnchorType.ANCHOR_LEFTMOST_LEAF);
+ } else {
+ dv.setGraph(tree);
+ }
+ viewPanel.add(dv, BorderLayout.CENTER);
+ dv.revalidate();
+ repaint();
+ getContentPane().repaint();
+ return;
+ }
+
+ /**
+ * Makes this frame unmodifiable, so that the tree it displays cannot be
+ * changed. In fact, all that happens is the title is update and the
+ * navigation buttons are disabled. This method is intended to prevent the
+ * user from modifying the frame, not to prevent other code from modifying it.
+ */
+ public void disableNavigationButtons() {
+ setTitle(getTitle() + " (fixed)");
+ nextSource.setEnabled(false);
+ previousSource.setEnabled(false);
+ return;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/browser/TranslationInfo.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/browser/TranslationInfo.java b/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/browser/TranslationInfo.java
new file mode 100644
index 0000000..e23a89d
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/browser/TranslationInfo.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.ui.tree_visualizer.browser;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.joshua.ui.tree_visualizer.tree.Tree;
+
+class TranslationInfo {
+ private String sourceSentence;
+ private String reference;
+ private ArrayList<Tree> translations;
+
+ public TranslationInfo() {
+ translations = new ArrayList<Tree>();
+ }
+
+ public String sourceSentence() {
+ return sourceSentence;
+ }
+
+ public void setSourceSentence(String src) {
+ sourceSentence = src;
+ return;
+ }
+
+ public String reference() {
+ return reference;
+ }
+
+ public void setReference(String ref) {
+ reference = ref;
+ return;
+ }
+
+ public List<Tree> translations() {
+ return translations;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/tree/Tree.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/tree/Tree.java b/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/tree/Tree.java
new file mode 100644
index 0000000..662544b
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/ui/tree_visualizer/tree/Tree.java
@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.ui.tree_visualizer.tree;
+
+import java.util.Stack;
+import java.util.regex.Pattern;
+import java.util.regex.Matcher;
+import java.util.List;
+import java.util.ArrayList;
+import java.util.Comparator;
+
+/**
+ * A class to represent the target-side tree produced by decoding using Joshua
+ * with an SCFG.
+ * <p>
+ * When decoding with use_tree_nbest=true, instead of a flat text output like
+ * "i asked her a question", we get a Penn treebank format tree like
+ * "(ROOT (S (NP i) (VP (V asked) (NP her) (NP (DT a) (N question)))))".
+ * If we also set include_align_index=true, we include source-side alignments
+ * for each internal node of the tree.
+ * <p>
+ * So, if the source input sentence is "je lui ai pose un question", if we
+ * turn on both configuration options, we end up with a decorated tree like
+ * this:
+ * "(ROOT{0-6} (S{0-6} (NP{0-1} i) (VP{1-6} (V{2-4} asked) (NP{1-2} her)
+ * (NP{4-6} (DT{4-5} a) (N{5-6} question)))))".
+ * <p>
+ * This class contains all the information of that flat string representation:
+ * the tree structure, the output (English) words, and the alignments to a
+ * source sentence.
+ * <p>
+ * Using a Tree the source sentence it was aligned to, we can create
+ * a DerivationTree object suitable for display.
+ *
+ * @author Jonny Weese jonny@cs.jhu.edu
+ */
+public class Tree {
+
+ /**
+ * An array holding the label of each node of the tree, in depth-first order.
+ * The label of a node means the NT label assigned to an internal node, or
+ * the terminal symbol (English word) at a leaf.
+ */
+ private final String [] labels;
+
+ /**
+ * The number of children of each node of the tree, in depth-first order.
+ */
+ private final int [] numChildren;
+
+ /**
+ * The smallest source-side index that each node covers, in depth-first order.
+ * Note that we only have this information for internal nodes. For leaves,
+ * this value will always be -1.
+ */
+ private final int [] sourceStartIndices;
+
+ /**
+ * 1 + the largest source-side index that each node covers, in depth-first
+ * order. Note that we only have this informaion for internal nodes. For
+ * leaves, this value will always be -1.
+ */
+ private final int [] sourceEndIndices;
+
+ /**
+ * A pattern to match an aligned internal node and pull out its information.
+ * This pattern matches:
+ *
+ * 1) start-of-string
+ * 2) (
+ * 3) an arbitrary sequence of non-whitespace characters (at least 1)
+ * 4) {
+ * 5) a decimal number
+ * 6) -
+ * 7) a decimal number
+ * 8) }
+ * 9) end-of-string
+ *
+ * That is, it matches something like "(FOO{32-55}". The string and two
+ * decimal numbers (parts 3, 5, and 7) are captured in groups.
+ */
+ private static final Pattern NONTERMINAL_PATTERN =
+ Pattern.compile("^\\((\\S+)\\{(\\d+)-(\\d+)\\}$");
+
+ /**
+ * Creates a Tree object from an input string in Penn treebank format with
+ * source alignment annotations.
+ * @param s an input string in Penn treebank format with source alignment annotations
+ */
+ public Tree(String s) {
+ final String [] tokens = s.replaceAll("\\)", " )").split("\\s+");
+ int numNodes = 0;
+ for (String t : tokens) {
+ if (!t.equals(")")) {
+ numNodes++;
+ }
+ }
+ labels = new String[numNodes];
+ numChildren = new int[numNodes];
+ sourceStartIndices = new int[numNodes];
+ sourceEndIndices = new int[numNodes];
+ try {
+ initialize(tokens);
+ } catch (Exception e) {
+ // This will catch most formatting errors.
+ throw new IllegalArgumentException(
+ String.format("couldn't create tree from string: \"%s\"", s),
+ e);
+ }
+ }
+
+ private void initialize(String [] tokens) {
+ final Stack<Integer> stack = new Stack<Integer>();
+ int nodeIndex = 0;
+ for (String token : tokens) {
+ final Matcher matcher = NONTERMINAL_PATTERN.matcher(token);
+ if (matcher.matches()) {
+ // new non-terminal node
+ labels[nodeIndex] = matcher.group(1);
+ sourceStartIndices[nodeIndex] = Integer.parseInt(matcher.group(2));
+ sourceEndIndices[nodeIndex] = Integer.parseInt(matcher.group(3));
+ stack.push(nodeIndex);
+ nodeIndex++;
+ } else if (token.equals(")")) {
+ // finished a subtree
+ stack.pop();
+ if (stack.empty()) {
+ break;
+ } else {
+ numChildren[stack.peek()]++;
+ }
+ } else {
+ // otherwise, it's a new leaf node
+ labels[nodeIndex] = token;
+ sourceStartIndices[nodeIndex] = -1;
+ sourceEndIndices[nodeIndex] = -1;
+ numChildren[stack.peek()]++;
+ nodeIndex++;
+ }
+ }
+ if (!stack.empty()) {
+ // Not enough close-parentheses at the end of the tree.
+ throw new IllegalArgumentException();
+ }
+ }
+
+ /**
+ * Return the number of nodes in this Tree.
+ * @return the number of nodes in this Tree
+ */
+ public int size() {
+ return labels.length;
+ }
+
+ /**
+ * Get the root Node of this Tree.
+ * @return the Node present at the toom the this Tree
+ */
+ public Node root() {
+ return new Node(0);
+ }
+
+ private List<Integer> childIndices(int index) {
+ List<Integer> result = new ArrayList<Integer>();
+ int remainingChildren = numChildren[index];
+ int childIndex = index + 1;
+ while (remainingChildren > 0) {
+ result.add(childIndex);
+ childIndex = nextSiblingIndex(childIndex);
+ remainingChildren--;
+ }
+ return result;
+ }
+
+ private int nextSiblingIndex(int index) {
+ int result = index + 1;
+ int remainingChildren = numChildren[index];
+ for (int i = 0; i < remainingChildren; i++) {
+ result = nextSiblingIndex(result);
+ }
+ return result;
+ }
+
+ public String yield() {
+ String result = "";
+ for (int i = 0; i < labels.length; i++) {
+ if (numChildren[i] == 0) {
+ if (!result.equals("")) {
+ result += " ";
+ }
+ result += labels[i];
+ }
+ }
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ return root().toString();
+ }
+
+ /**
+ * A class representing the Nodes of a tree.
+ */
+ public class Node {
+
+ /**
+ * The index into the Tree class's internal arrays.
+ */
+ private final int index;
+
+ private Node(int i) {
+ index = i;
+ }
+
+ /**
+ * Get the label for this node. If the node is internal to the tree, its
+ * label is the non-terminal label assigned to it. If it is a leaf node,
+ * the label is the English word at the leaf.
+ * @return a string representing the label for this node
+ */
+ public String label() {
+ return labels[index];
+ }
+
+ public boolean isLeaf() {
+ return numChildren[index] == 0;
+ }
+
+ public int sourceStartIndex() {
+ return sourceStartIndices[index];
+ }
+
+ public int sourceEndIndex() {
+ return sourceEndIndices[index];
+ }
+
+ public List<Node> children() {
+ List<Node> result = new ArrayList<Node>();
+ for (int j : childIndices(index)) {
+ result.add(new Node(j));
+ }
+ return result;
+ }
+
+ @Override
+ public String toString() {
+ if (isLeaf()) {
+ return label();
+ }
+ String result = String.format("(%s{%d-%d}",
+ label(),
+ sourceStartIndex(),
+ sourceEndIndex());
+ for (Node c : children()) {
+ result += String.format(" %s", c);
+ }
+ return result + ")";
+ }
+ }
+
+ public static class NodeSourceStartComparator implements Comparator<Node> {
+ public int compare(Node a, Node b) {
+ return a.sourceStartIndex() - b.sourceStartIndex();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/Algorithms.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/Algorithms.java b/joshua-core/src/main/java/org/apache/joshua/util/Algorithms.java
new file mode 100644
index 0000000..327c882
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/Algorithms.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util;
+
+public final class Algorithms {
+
+ /**
+ * Calculates the Levenshtein Distance for a candidate paraphrase given the source.
+ *
+ * The code is based on the example by Michael Gilleland found at
+ * http://www.merriampark.com/ld.htm.
+ * @param candidate todo
+ * @param source todo
+ * @return the minimum edit distance.
+ */
+ public static final int levenshtein(String[] candidate, String[] source) {
+ // First check to see whether either of the arrays
+ // is empty, in which case the least cost is simply
+ // the length of the other array (which would correspond
+ // to inserting that many elements.
+ if (source.length == 0) return candidate.length;
+ if (candidate.length == 0) return source.length;
+
+ // Initialize a table to the minimum edit distances between
+ // any two points in the arrays. The size of the table is set
+ // to be one beyond the lengths of the two arrays, and the first
+ // row and first column are set to be zero to avoid complicated
+ // checks for out of bounds exceptions.
+ int distances[][] = new int[source.length + 1][candidate.length + 1];
+
+ for (int i = 0; i <= source.length; i++)
+ distances[i][0] = i;
+ for (int j = 0; j <= candidate.length; j++)
+ distances[0][j] = j;
+
+ // Walk through each item in the source and target arrays
+ // and find the minimum cost to move from the previous points
+ // to here.
+ for (int i = 1; i <= source.length; i++) {
+ Object sourceItem = source[i - 1];
+ for (int j = 1; j <= candidate.length; j++) {
+ Object targetItem = candidate[j - 1];
+ int cost;
+ if (sourceItem.equals(targetItem))
+ cost = 0;
+ else
+ cost = 1;
+ int deletionCost = distances[i - 1][j] + 1;
+ int insertionCost = distances[i][j - 1] + 1;
+ int substitutionCost = distances[i - 1][j - 1] + cost;
+ distances[i][j] = minimum(insertionCost, deletionCost, substitutionCost);
+ }
+ }
+ // The point at the end will be the minimum edit distance.
+ return distances[source.length][candidate.length];
+ }
+
+ /**
+ * Returns the minimum of the three values.
+ */
+ private static final int minimum(int a, int b, int c) {
+ int minimum;
+ minimum = a;
+ if (b < minimum) minimum = b;
+ if (c < minimum) minimum = c;
+ return minimum;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/Bits.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/Bits.java b/joshua-core/src/main/java/org/apache/joshua/util/Bits.java
new file mode 100644
index 0000000..b5294f6
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/Bits.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util;
+
+/**
+ * Utility class for bit twiddling.
+ *
+ * @author Lane Schwartz
+ */
+public class Bits {
+
+ /**
+ * Encodes two shorts in an int.
+ *
+ * @param high input high short to encode
+ * @param low input low short to encode
+ * @return encoded int
+ */
+ public static int encodeAsInt(short high, short low) {
+
+ // Store the first short value in the highest 16 bits of the int
+ int key = high | 0x00000000;
+ key <<= 16;
+
+ // Store the second short value in the lowest 16 bits of the int
+ int lowInt = low & 0x0000FFFF;
+ key |= lowInt;
+
+ return key;
+
+ }
+
+ /**
+ * Decodes the high 16 bits of an integer as a short.
+ *
+ * @param i Integer value to decode
+ * @return Short representation of the high 16 bits of the integer
+ */
+ public static short decodeHighBits(int i) {
+
+ long key = i & 0xFFFF0000l;
+
+ key >>= 16;
+
+ return (short) key;
+
+ }
+
+
+ /**
+ * Decodes the low 16 bits of an integer as a short.
+ *
+ * @param i Integer value to decode
+ * @return Short representation of the high 16 bits of the integer
+ */
+ public static short decodeLowBits(int i) {
+
+ return (short) i;
+
+ }
+
+
+ /**
+ * Encodes two integers in a long.
+ *
+ * @param high input high int to encode
+ * @param low input low int to encode
+ * @return encoded long
+ */
+ public static long encodeAsLong(int high, int low) {
+
+ // Store the first int value in the highest 32 bits of the long
+ long key = high | 0x0000000000000000l;
+ key <<= 32;
+
+ // Store the second int value in the lowest 32 bits of the long
+ long lowLong = low & 0x00000000FFFFFFFFl;;
+ key |= lowLong;
+
+ return key;
+
+ }
+
+ /**
+ * Decodes the high 32 bits of a long as an integer.
+ *
+ * @param l Long value to decode
+ * @return Integer representation of the high 32 bits of the long
+ */
+ public static int decodeHighBits(long l) {
+
+ long key = l & 0xFFFFFFFF00000000l;
+
+ key >>= 32;
+
+ return (int) key;
+
+ }
+
+
+ /**
+ * Decodes the low 32 bits of a long as an integer.
+ *
+ * @param l Long value to decode
+ * @return Integer representation of the high 32 bits of the long
+ */
+ public static int decodeLowBits(long l) {
+
+ return (int) l;
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/BotMap.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/BotMap.java b/joshua-core/src/main/java/org/apache/joshua/util/BotMap.java
new file mode 100644
index 0000000..1cc82b5
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/BotMap.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Gets a special map that maps any key to the a particular value.
+ *
+ * @author Lane Schwartz
+ * @see "Lopez (2008), footnote 9 on p73"
+ */
+public class BotMap<K, V> implements Map<K, V> {
+
+ /** Special value, which this map will return for every key. */
+ private final V value;
+
+ /**
+ * Constructs a special map that maps any key to the a particular value.
+ *
+ * @param value Special value, which this map will return for every key.
+ */
+ public BotMap(V value) {
+ this.value = value;
+ }
+
+ public void clear() {
+ throw new UnsupportedOperationException();
+ }
+
+ public boolean containsKey(Object key) {
+ return true;
+ }
+
+ public boolean containsValue(Object value) {
+ return this.value == value;
+ }
+
+ public Set<Map.Entry<K, V>> entrySet() {
+ throw new UnsupportedOperationException();
+ }
+
+ public V get(Object key) {
+ return value;
+ }
+
+ public boolean isEmpty() {
+ return false;
+ }
+
+ public Set<K> keySet() {
+ throw new UnsupportedOperationException();
+ }
+
+ public V put(K key, V value) {
+ throw new UnsupportedOperationException();
+ }
+
+ public void putAll(Map<? extends K, ? extends V> t) {
+ throw new UnsupportedOperationException();
+ }
+
+ public V remove(Object key) {
+ throw new UnsupportedOperationException();
+ }
+
+ public int size() {
+ throw new UnsupportedOperationException();
+ }
+
+ public Collection<V> values() {
+ return Collections.singleton(value);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/Cache.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/Cache.java b/joshua-core/src/main/java/org/apache/joshua/util/Cache.java
new file mode 100644
index 0000000..0d72f8a
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/Cache.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util;
+
+// Imports
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+
+/**
+ * Cache is a class that implements a least recently used cache. It is a straightforward extension
+ * of java.util.LinkedHashMap with its removeEldestEntry method overridden, so that stale entries
+ * are deleted once we reach the specified capacity of the Cache.
+ * <p>
+ * This class is quite useful for storing the results of computations that we would do many times
+ * over in the FeatureFunctions.
+ *
+ * @author Chris Callison-Burch
+ * @since 14 April 2005
+ *
+ */
+public class Cache<K, V> extends LinkedHashMap<K, V> {
+
+ private static final long serialVersionUID = 6073387072740892061L;
+
+ /** Logger for this class. */
+ private static final Logger LOG = LoggerFactory.getLogger(Cache.class);
+ // ===============================================================
+ // Constants
+ // ===============================================================
+
+ /**
+ * A constant is used as the default the cache size if none is specified.
+ */
+ public static final int DEFAULT_CAPACITY = 100000000;
+
+ /** Default initial capacity of the cache. */
+ public static final int INITIAL_CAPACITY = 1000000;
+
+ /** Default load factor of the cache. */
+ public static final float LOAD_FACTOR = 0.75f;
+
+ /**
+ * By default, ordering mode of the cache is access order (true).
+ */
+ public static final boolean ACCESS_ORDER = true;
+
+
+ // ===============================================================
+ // Member variables
+ // ===============================================================
+
+ /** Maximum number of items that the cache can contain. */
+ int maxCapacity;
+
+ // ===============================================================
+ // Constructor(s)
+ // ===============================================================
+
+ /**
+ * Creates a Cache with a set capacity.
+ *
+ * @param maxCapacity the maximum capacity of the cache.
+ */
+ public Cache(int maxCapacity) {
+ super((maxCapacity < INITIAL_CAPACITY) ? maxCapacity : INITIAL_CAPACITY, LOAD_FACTOR,
+ ACCESS_ORDER);
+ this.maxCapacity = maxCapacity;
+ }
+
+
+ /**
+ * Creates a Cache with the DEFAULT_CAPACITY.
+ */
+ public Cache() {
+ this(DEFAULT_CAPACITY);
+ }
+
+ // ===============================================================
+ // Public
+ // ===============================================================
+
+ // ===========================================================
+ // Accessor methods (set/get)
+ // ===========================================================
+
+ @Override
+ public V get(Object key) {
+ LOG.debug("Cache get key: {}", key);
+ return super.get(key);
+ }
+
+
+ @Override
+ public V put(K key, V value) {
+ LOG.debug("Cache put key: {}", key);
+ return super.put(key, value);
+ }
+
+ // ===========================================================
+ // Methods
+ // ===========================================================
+
+ @Override
+ public boolean containsKey(Object key) {
+ boolean contains = super.containsKey(key);
+ if (contains){
+ LOG.debug("Cache has key: {}", key);
+ } else {
+ LOG.debug("Cache lacks key: {}", key);
+ }
+ return contains;
+ }
+
+
+ // ===============================================================
+ // Protected
+ // ===============================================================
+
+ // ===============================================================
+ // Methods
+ // ===============================================================
+
+ /**
+ * This method is invoked by put and putAll after inserting a new entry into the map. Once we
+ * reach the capacity of the cache, we remove the oldest entry each time a new entry is added.
+ * This reduces memory consumption by deleting stale entries.
+ *
+ * @param eldest the eldest entry
+ * @return true if the capacity is greater than the maximum capacity
+ */
+ protected boolean removeEldestEntry(Map.Entry<K, V> eldest) {
+ boolean removing = size() > maxCapacity;
+ if (removing ) {
+ LOG.debug("Cache loses key: {}", eldest.getKey());
+ }
+ return removing;
+ }
+
+ // ===============================================================
+ // Private
+ // ===============================================================
+
+ // ===============================================================
+ // Methods
+ // ===============================================================
+
+
+ // ===============================================================
+ // Static
+ // ===============================================================
+
+
+ // ===============================================================
+ // Main
+ // ===============================================================
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/ChartSpan.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/ChartSpan.java b/joshua-core/src/main/java/org/apache/joshua/util/ChartSpan.java
new file mode 100644
index 0000000..b22d2aa
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/ChartSpan.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util;
+
+/**
+ * CKY-based decoding makes extensive use of charts, which maintain information about spans (i, j)
+ * over the length-n input sentence, 0 <= i <= j <= n. These charts are used for many things; for
+ * example, lattices use a chart to denote whether there is a path between nodes i and j, and what
+ * their costs is, and the decoder uses charts to record the partial application of rules (DotChart})
+ * and the existence of proved items ({@link org.apache.joshua.decoder.phrase.PhraseChart}).
+ *
+ * The dummy way to implement a chart is to initialize a two-dimensional array; however, this wastes
+ * a lot of space, because the constraint (i <= j) means that only half of this space can ever be
+ * used. This is especially a problem for lattices, where the sentence length (n) is the number of
+ * nodes in the lattice!
+ *
+ * Fortunately, there is a smarter way, since there is a simple deterministic mapping between chart
+ * spans under a given maximum length. This class implements that in a generic way, introducing
+ * large savings in both space and time.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ */
+public class ChartSpan<Type> {
+ Object[] chart;
+ int max;
+
+ public ChartSpan(int w, Type defaultValue) {
+ //System.err.println(String.format("ChartSpan::ChartSpan(%d)", w));
+ this.max = w;
+
+ /* offset(max,max) is the last position in the array */
+ chart = new Object[offset(max,max) + 1];
+
+ /* Initialize all arcs to infinity, except self-loops, which have distance 0 */
+ for (int i = 0; i < chart.length; i++)
+ chart[i] = defaultValue;
+ }
+
+ @SuppressWarnings("unchecked")
+ public Type get(int i, int j) {
+ return (Type) chart[offset(i, j)];
+ }
+
+ public void set(int i, int j, Type value) {
+ chart[offset(i, j)] = value;
+ }
+
+ /**
+ * This computes the offset into the one-dimensional array for a given span.
+ *
+ * @param i source node in span
+ * @param j target node in span
+ * @return the offset
+ */
+ private int offset(int i, int j) {
+ if (i < 0 || j > max || i > j) {
+ throw new RuntimeException(String.format("Invalid span (%d,%d | %d)", i, j, max));
+ }
+
+ return i * (max + 1) - i * (i + 1) / 2 + j;
+ }
+
+ /**
+ * Convenience function for setting the values along the diagonal.
+ *
+ * @param value input Type for which to set values
+ */
+ public void setDiagonal(Type value) {
+ for (int i = 0; i <= max; i++)
+ set(i, i, value);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/CommandLineParser.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/CommandLineParser.java b/joshua-core/src/main/java/org/apache/joshua/util/CommandLineParser.java
new file mode 100644
index 0000000..974b973
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/CommandLineParser.java
@@ -0,0 +1,738 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Java Command Line Parser
+ * <p>
+ * The current version supports string and integer options.
+ * <p>
+ * Support is not included for options which take a list of values.
+ *
+ * @author Lane O.B. Schwartz
+ */
+@SuppressWarnings("rawtypes")
+public class CommandLineParser {
+
+ private Map<Character, Option<Integer>> intShortForms;
+ private Map<String, Option<Integer>> intLongForms;
+
+ private Map<Character, Option<String>> stringShortForms;
+ private Map<String, Option<String>> stringLongForms;
+
+ private Map<Character, Option<Boolean>> booleanShortForms;
+ private Map<String, Option<Boolean>> booleanLongForms;
+
+ private List<Option> allOptions;
+
+ private final Set<String> localizedTrueStrings = new HashSet<String>();
+ private final Set<String> localizedFalseStrings = new HashSet<String>();
+
+ public CommandLineParser() {
+ intShortForms = new HashMap<Character, Option<Integer>>();
+ intLongForms = new HashMap<String, Option<Integer>>();
+
+ stringShortForms = new HashMap<Character, Option<String>>();
+ stringLongForms = new HashMap<String, Option<String>>();
+
+ booleanShortForms = new HashMap<Character, Option<Boolean>>();
+ booleanLongForms = new HashMap<String, Option<Boolean>>();
+
+ allOptions = new LinkedList<Option>();
+
+ localizedTrueStrings.add("true");
+ localizedTrueStrings.add("yes");
+ localizedFalseStrings.add("false");
+ localizedFalseStrings.add("no");
+ }
+
+ public CommandLineParser(Set<String> localizedTrueStrings, Set<String> localizedFalseStrings) {
+ this();
+
+ this.localizedTrueStrings.clear();
+ this.localizedFalseStrings.clear();
+
+ this.localizedTrueStrings.addAll(localizedTrueStrings);
+ this.localizedFalseStrings.addAll(localizedFalseStrings);
+ }
+
+ public Option<Integer> addIntegerOption(char shortForm, String longForm, String valueVariable,
+ Integer defaultValue, Set<Integer> legalValues, String comment) {
+ if (shortForm != Option.MISSING_SHORT_FORM && (intShortForms.containsKey(shortForm))
+ || (!longForm.equals(Option.MISSING_LONG_FORM) && intLongForms.containsKey(longForm)))
+ throw new DuplicateOptionException("Duplicate options are not allowed");
+
+ Option<Integer> o =
+ new Option<Integer>(shortForm, longForm, valueVariable, defaultValue, legalValues, comment);
+ intShortForms.put(shortForm, o);
+ intLongForms.put(longForm, o);
+ allOptions.add(o);
+ return o;
+ }
+
+ public Option<Integer> addIntegerOption(char shortForm, String longForm, String valueVariable,
+ Set<Integer> legalValues, String comment) {
+ return addIntegerOption(shortForm, longForm, valueVariable, null, legalValues, comment);
+ }
+
+ public Option<Integer> addIntegerOption(char shortForm, String longForm, String valueVariable,
+ String comment) {
+ return addIntegerOption(shortForm, longForm, valueVariable, null, new UniversalSet<Integer>(),
+ comment);
+ }
+
+ public Option<Integer> addIntegerOption(char shortForm, String longForm, String comment) {
+ return addIntegerOption(shortForm, longForm, null, null, new UniversalSet<Integer>(), comment);
+ }
+
+ public Option<Integer> addIntegerOption(char shortForm, String longForm, String valueVariable,
+ Integer defaultValue, String comment) {
+ return addIntegerOption(shortForm, longForm, valueVariable, defaultValue,
+ new UniversalSet<Integer>(), comment);
+ }
+
+ public Option<Integer> addIntegerOption(String longForm, String valueVariable,
+ Integer defaultValue, String comment) {
+ return addIntegerOption(Option.MISSING_SHORT_FORM, longForm, valueVariable, defaultValue,
+ new UniversalSet<Integer>(), comment);
+ }
+
+ public Option<Integer> addIntegerOption(char shortForm, String longForm) {
+ return addIntegerOption(shortForm, longForm, null, null, new UniversalSet<Integer>(), "");
+ }
+
+ public Option<Integer> addIntegerOption(char shortForm) {
+ return addIntegerOption(shortForm, Option.MISSING_LONG_FORM);
+ }
+
+ public Option<Integer> addIntegerOption(String longForm) {
+ return addIntegerOption(Option.MISSING_SHORT_FORM, longForm);
+ }
+
+ public Option<Integer> addIntegerOption(String longForm, String comment) {
+ return addIntegerOption(Option.MISSING_SHORT_FORM, longForm, comment);
+ }
+
+
+ // String options
+
+
+ public Option<String> addStringOption(char shortForm, String longForm, String valueVariable,
+ String defaultValue, Set<String> legalValues, String comment) {
+ if (shortForm != Option.MISSING_SHORT_FORM && (intShortForms.containsKey(shortForm))
+ || (!longForm.equals(Option.MISSING_LONG_FORM) && intLongForms.containsKey(longForm)))
+ throw new DuplicateOptionException("Duplicate options are not allowed");
+
+ Option<String> o =
+ new Option<String>(shortForm, longForm, valueVariable, defaultValue, legalValues, comment);
+ stringShortForms.put(shortForm, o);
+ stringLongForms.put(longForm, o);
+ allOptions.add(o);
+ return o;
+ }
+
+ public Option<String> addStringOption(char shortForm, String longForm, String valueVariable,
+ Set<String> legalValues, String comment) {
+ return addStringOption(shortForm, longForm, valueVariable, null, legalValues, comment);
+ }
+
+ public Option<String> addStringOption(char shortForm, String longForm, String valueVariable,
+ String comment) {
+ return addStringOption(shortForm, longForm, valueVariable, null, new UniversalSet<String>(),
+ comment);
+ }
+
+ public Option<String> addStringOption(String longForm, String valueVariable, String comment) {
+ return addStringOption(Option.MISSING_SHORT_FORM, longForm, valueVariable, null,
+ new UniversalSet<String>(), comment);
+ }
+
+ public Option<String> addStringOption(char shortForm, String longForm, String comment) {
+ return addStringOption(shortForm, longForm, null, null, new UniversalSet<String>(), comment);
+ }
+
+ public Option<String> addStringOption(char shortForm, String longForm, String valueVariable,
+ String defaultValue, String comment) {
+ return addStringOption(shortForm, longForm, valueVariable, defaultValue,
+ new UniversalSet<String>(), comment);
+ }
+
+ public Option<String> addStringOption(String longForm, String valueVariable, String defaultValue,
+ String comment) {
+ return addStringOption(Option.MISSING_SHORT_FORM, longForm, valueVariable, defaultValue,
+ new UniversalSet<String>(), comment);
+ }
+
+ public Option<String> addStringOption(char shortForm, String longForm) {
+ return addStringOption(shortForm, longForm, null, null, new UniversalSet<String>(), "");
+ }
+
+ public Option<String> addStringOption(char shortForm) {
+ return addStringOption(shortForm, Option.MISSING_LONG_FORM);
+ }
+
+ public Option<String> addStringOption(String longForm) {
+ return addStringOption(Option.MISSING_SHORT_FORM, longForm);
+ }
+
+ public Option<String> addStringOption(String longForm, String comment) {
+ return addStringOption(Option.MISSING_SHORT_FORM, longForm, comment);
+ }
+
+
+ // boolean options
+
+ public Option<Boolean> addBooleanOption(char shortForm, String longForm, String valueVariable,
+ Boolean defaultValue, String comment) {
+ if (shortForm != Option.MISSING_SHORT_FORM && (booleanShortForms.containsKey(shortForm))
+ || (!longForm.equals(Option.MISSING_LONG_FORM) && booleanLongForms.containsKey(longForm)))
+ throw new DuplicateOptionException("Duplicate options are not allowed");
+ Set<Boolean> legalBooleanValues = new HashSet<Boolean>();
+ legalBooleanValues.add(true);
+ legalBooleanValues.add(false);
+
+ Option<Boolean> o =
+ new Option<Boolean>(shortForm, longForm, valueVariable, defaultValue, legalBooleanValues,
+ comment);
+ booleanShortForms.put(shortForm, o);
+ booleanLongForms.put(longForm, o);
+ allOptions.add(o);
+ return o;
+ }
+
+ public Option<Boolean> addBooleanOption(char shortForm, String longForm, String valueVariable,
+ String comment) {
+ return addBooleanOption(shortForm, longForm, valueVariable, null, comment);
+ }
+
+ public Option<Boolean> addBooleanOption(char shortForm, String longForm, String comment) {
+ return addBooleanOption(shortForm, longForm, null, null, comment);
+ }
+
+ public Option<Boolean> addBooleanOption(String longForm, Boolean defaultValue, String comment) {
+ return addBooleanOption(Option.MISSING_SHORT_FORM, longForm, null, defaultValue, comment);
+ }
+
+ public Option<Boolean> addBooleanOption(String longForm, String valueVariable,
+ Boolean defaultValue, String comment) {
+ return addBooleanOption(Option.MISSING_SHORT_FORM, longForm, valueVariable, defaultValue,
+ comment);
+ }
+
+ public Option<Boolean> addBooleanOption(char shortForm, String longForm) {
+ return addBooleanOption(shortForm, longForm, null, null, "");
+ }
+
+ public Option<Boolean> addBooleanOption(char shortForm) {
+ return addBooleanOption(shortForm, Option.MISSING_LONG_FORM);
+ }
+
+ public Option<Boolean> addBooleanOption(String longForm) {
+ return addBooleanOption(Option.MISSING_SHORT_FORM, longForm);
+ }
+
+ public Option<Boolean> addBooleanOption(String longForm, String comment) {
+ return addBooleanOption(Option.MISSING_SHORT_FORM, longForm, comment);
+ }
+
+
+
+ // float options
+
+
+
+ // /
+ /*
+ * public Option<Integer> addIntegerOption(char shortForm, String longForm) { if
+ * (intShortForms.containsKey(shortForm) || intLongForms.containsKey(longForm)) throw new
+ * DuplicateOptionException("Duplicate options are not allowed");
+ *
+ * Option<Integer> o = new Option<Integer>(shortForm, longForm); intShortForms.put(shortForm, o);
+ * intLongForms.put(longForm, o); allOptions.add(o);
+ *
+ * return o; }
+ *
+ * public Option<Integer> addIntegerOption(char shortForm, String longForm, String valueVariable,
+ * int defaultValue, Set<Integer> legalValues, String comment) { if
+ * (intShortForms.containsKey(shortForm) || intLongForms.containsKey(longForm)) throw new
+ * DuplicateOptionException("Duplicate options are not allowed");
+ *
+ * Option<Integer> o = new Option<Integer>(shortForm, longForm, valueVariable, defaultValue,
+ * comment); intShortForms.put(shortForm, o); intLongForms.put(longForm, o); allOptions.add(o);
+ * return o; }
+ *
+ * public Option<Integer> addIntegerOption(char shortForm, String longForm, String valueVariable,
+ * int defaultValue, String comment) { if (intShortForms.containsKey(shortForm) ||
+ * intLongForms.containsKey(longForm)) throw new
+ * DuplicateOptionException("Duplicate options are not allowed");
+ *
+ * Option<Integer> o = new Option<Integer>(shortForm, longForm, valueVariable, defaultValue,
+ * comment); intShortForms.put(shortForm, o); intLongForms.put(longForm, o); allOptions.add(o);
+ * return o; }
+ *
+ * public Option<Integer> addIntegerOption(char shortForm, String longForm, String valueVariable,
+ * String comment) { if (intShortForms.containsKey(shortForm) ||
+ * intLongForms.containsKey(longForm)) throw new
+ * DuplicateOptionException("Duplicate options are not allowed");
+ *
+ * Option<Integer> o = new Option<Integer>(shortForm, longForm, valueVariable, comment);
+ * intShortForms.put(shortForm, o); intLongForms.put(longForm, o); allOptions.add(o); return o; }
+ */
+
+ /*
+ * public Option<String> addStringOption(char shortForm, String longForm) { if
+ * (stringShortForms.containsKey(shortForm) || stringLongForms.containsKey(longForm)) throw new
+ * DuplicateOptionException("Duplicate options are not allowed");
+ *
+ * Option<String> o = new Option<String>(shortForm, longForm); stringShortForms.put(shortForm, o);
+ * stringLongForms.put(longForm, o); allOptions.add(o); return o; }
+ */
+
+ public void parse(String[] argv) {
+
+ Collection<Option> parsedOptions = new HashSet<Option>();
+
+ int index = 0;
+
+ while (index < argv.length) {
+ if (argv[index].startsWith("--")) {
+ int splitPoint = argv[index].indexOf('=');
+ if (splitPoint == 2) {
+ throw new CommandLineParserException("Invalid option: --");
+ } else if (splitPoint >= 0) {
+ String option = argv[index].substring(2, splitPoint);
+ String value = argv[index].substring(splitPoint + 1);
+ parsedOptions.add(parseLongForm(option, value));
+ } else if (index + 1 < argv.length) {
+ String option = argv[index].substring(2);
+ String value = argv[index + 1];
+ if (value.startsWith("-") && !value.equals("-") && !value.equals("--")) {
+ parsedOptions.add(parseLongForm(option));
+ } else {
+ parsedOptions.add(parseLongForm(option, value));
+ index++;
+ }
+ } else {
+ // Must be a boolean option
+ String option = argv[index].substring(2);
+ parsedOptions.add(parseLongForm(option));
+ // throw new CommandLineParserException("No value provided for option " +
+ // argv[index].substring(2));
+ }
+ } else if (argv[index].startsWith("-")) {
+ String option = argv[index].substring(1);
+ if (option.length() == 1) {
+ if (index + 1 < argv.length) {
+ String value = argv[index + 1];
+ if (value.startsWith("-") && !value.equals("-") && !value.equals("--")) {
+ // Must be a boolean option
+ parsedOptions.add(parseShortForm(option.charAt(0)));
+ } else {
+ parsedOptions.add(parseShortForm(option.charAt(0), value));
+ index++;
+ }
+ } else {
+ // Must be a boolean option
+ parsedOptions.add(parseShortForm(option.charAt(0)));
+ }
+ } else {
+ throw new CommandLineParserException(argv[index] + " is not a valid option");
+ }
+ }
+ index++;
+ }
+
+ for (Option o : allOptions) {
+ if (o.isRequired() && !parsedOptions.contains(o)) {
+ die("A required option was not provided:\n " + o + "\n");
+ }
+ }
+
+ }
+
+ public void printUsage() {
+ System.err.println("Usage:");
+ for (Option o : allOptions) {
+ System.err.println(o);
+ }
+ }
+
+ private void die(String error) {
+ System.err.println(error);
+ printUsage();
+ System.exit(1);
+ }
+
+ public Option parseLongForm(String key, String value) {
+
+ if (intLongForms.containsKey(key)) {
+ try {
+ Option<Integer> o = intLongForms.get(key);
+ o.setValue(Integer.valueOf(value));
+ return o;
+ } catch (NumberFormatException e) {
+ die("Option " + key + " requires an integer value.");
+ return null;
+ }
+ } else if (stringLongForms.containsKey(key)) {
+ Option<String> o = stringLongForms.get(key);
+ o.setValue(value);
+ return o;
+ } else if (booleanLongForms.containsKey(key)) {
+ Option<Boolean> o = booleanLongForms.get(key);
+
+ if (localizedTrueStrings.contains(value.toLowerCase())) {
+ o.setValue(true);
+ } else if (localizedFalseStrings.contains(value.toLowerCase())) {
+ o.setValue(false);
+ } else {
+ throw new CommandLineParserException("Invalid value \"" + value + "\" for boolean option "
+ + key);
+ }
+
+ return o;
+ } else {
+
+ throw new Error("Bug in command line parser - unexpected option type encountered for option "
+ + key);
+ }
+ }
+
+ public Option parseLongForm(String key) {
+
+ if (booleanLongForms.containsKey(key)) {
+ Option<Boolean> o = booleanLongForms.get(key);
+ o.setValue(true);
+ return o;
+
+ } else {
+ throw new CommandLineParserException("No such boolean option exists: --" + key);
+ }
+ }
+
+ public Option parseShortForm(Character key) {
+
+ if (booleanShortForms.containsKey(key)) {
+ Option<Boolean> o = booleanShortForms.get(key);
+ o.setValue(true);
+ return o;
+
+ } else {
+ throw new CommandLineParserException("No such boolean option exists: -" + key);
+ }
+ }
+
+ public Option parseShortForm(Character key, String value) {
+ if (intShortForms.containsKey(key)) {
+ try {
+ Option<Integer> o = intShortForms.get(key);
+ o.setValue(Integer.valueOf(value));
+ return o;
+ } catch (NumberFormatException e) {
+ die("Option " + key + " requires an integer value.");
+ return null;
+ }
+ } else if (stringShortForms.containsKey(key)) {
+ Option<String> o = stringShortForms.get(key);
+ o.setValue(value);
+ return o;
+ } else if (booleanShortForms.containsKey(key)) {
+ Option<Boolean> o = booleanShortForms.get(key);
+
+ if (localizedTrueStrings.contains(value.toLowerCase())) {
+ o.setValue(true);
+ } else if (localizedFalseStrings.contains(value.toLowerCase())) {
+ o.setValue(false);
+ } else {
+ throw new CommandLineParserException("Invalid value \"" + value + "\" for boolean option "
+ + key);
+ }
+
+ return o;
+ } else {
+ throw new Error("Bug in command line parser - unexpected option type encountered");
+ }
+ }
+
+ /*
+ * public int intValue(Option o) { if (intOptions.containsKey(o)) return intOptions.get(o); else
+ * throw new RuntimeException("No such integer option"); }
+ *
+ * public String stringValue(Option o) { if (stringOptions.containsKey(o)) return
+ * stringOptions.get(o); else throw new RuntimeException("No such string option"); }
+ */
+
+ public <OptionType> OptionType getValue(Option<OptionType> option) {
+ return option.getValue();
+ }
+
+ public boolean hasValue(Option<?> option) {
+ return option.hasValue();
+ }
+
+ public static void main(String[] args) {
+ CommandLineParser parser = new CommandLineParser();
+ Option<Integer> n = parser.addIntegerOption('n', "number", "NUMBER", "a number to be supplied");
+
+ parser.parse(args);
+
+ // parser.printUsage();
+ System.out.println(parser.getValue(n));
+ }
+
+ @SuppressWarnings("serial")
+ public static class CommandLineParserException extends RuntimeException {
+ public CommandLineParserException(String message) {
+ super(message);
+ }
+ }
+
+ @SuppressWarnings("serial")
+ public static class DuplicateOptionException extends RuntimeException {
+ public DuplicateOptionException(String message) {
+ super(message);
+ }
+ }
+
+ public class Option<OptionType> {
+ private final char shortForm;
+ private final String longForm;
+ private final String comment;
+ private final OptionType defaultValue;
+ private final String valueVariable;
+ private final Set<OptionType> legalValues;
+
+ public static final char MISSING_SHORT_FORM = '\u0000';
+ public static final String MISSING_LONG_FORM = "\u0000";
+
+ private OptionType optionValue;
+
+ public Option(char shortForm, String longForm, String valueVariable, OptionType defaultValue,
+ Set<OptionType> legalValues, String comment) {
+
+ if (longForm == null) throw new NullPointerException("longForm must not be null");
+
+ if (comment == null) throw new NullPointerException("comment must not be null");
+
+ this.shortForm = shortForm;
+ this.longForm = longForm;
+ this.comment = comment;
+ this.valueVariable = valueVariable;
+ this.defaultValue = defaultValue;
+ this.legalValues = legalValues;
+ this.optionValue = null;
+ }
+
+ public Option(char shortForm, String longForm, String valueVariable,
+ Set<OptionType> legalValues, String comment) {
+ this(shortForm, longForm, valueVariable, null, legalValues, comment);
+ }
+
+
+ public Option(char shortForm, String longForm, String valueVariable, String comment) {
+ this(shortForm, longForm, valueVariable, null, new UniversalSet<OptionType>(), comment);
+ }
+
+ public Option(char shortForm, String longForm, String comment) {
+ this(shortForm, longForm, null, null, new UniversalSet<OptionType>(), comment);
+ }
+
+ public Option(char shortForm, String longForm, String valueVariable, OptionType defaultValue,
+ String comment) {
+ this(shortForm, longForm, valueVariable, defaultValue, new UniversalSet<OptionType>(),
+ comment);
+ }
+
+ public Option(String longForm, String valueVariable, OptionType defaultValue, String comment) {
+ this(MISSING_SHORT_FORM, longForm, valueVariable, defaultValue,
+ new UniversalSet<OptionType>(), comment);
+ }
+
+ public Option(char shortForm, String longForm) {
+ this(shortForm, longForm, null, null, new UniversalSet<OptionType>(), "");
+ }
+
+ public Option(char shortForm) {
+ this(shortForm, MISSING_LONG_FORM);
+ }
+
+ public Option(String longForm) {
+ this(MISSING_SHORT_FORM, longForm);
+ }
+
+ public Option(String longForm, String comment) {
+ this(MISSING_SHORT_FORM, longForm, comment);
+ }
+
+ public boolean isOptional() {
+ return (null != defaultValue);
+ }
+
+ public boolean isRequired() {
+ return (null == defaultValue);
+ }
+
+ public char getShortForm() {
+ return shortForm;
+ }
+
+ public String getLongForm() {
+ return longForm;
+ }
+
+ public String getComment() {
+ return comment;
+ }
+
+ void setValue(OptionType value) {
+ this.optionValue = value;
+ }
+
+ OptionType getValue() {
+ if (optionValue != null) {
+ return optionValue;
+ } else if (defaultValue != null) {
+ return defaultValue;
+ } else {
+ throw new CommandLineParserException(
+ "Unable to get value because option has not been initialized and does not have a default value: "
+ + this.toString());
+ }
+ }
+
+ boolean hasValue() {
+ return !(null == optionValue && null == defaultValue);
+ }
+
+ public String toString() {
+
+ String formattedShortForm;
+ if (shortForm == Option.MISSING_SHORT_FORM) {
+ formattedShortForm = "";
+ } else {
+ formattedShortForm = "-" + shortForm;
+ }
+
+ String formattedLongForm;
+ if (longForm.equals(Option.MISSING_LONG_FORM)) {
+ formattedLongForm = "";
+ } else {
+ formattedLongForm = "--" + longForm;
+ }
+
+ if (shortForm != Option.MISSING_SHORT_FORM && !longForm.equals(Option.MISSING_LONG_FORM)) {
+ formattedShortForm += ",";
+ }
+
+ if (valueVariable != null && valueVariable.length() >= 1) {
+ formattedLongForm += "=" + valueVariable;
+ }
+
+ String string = String.format(" %1$3s %2$-21s", formattedShortForm, formattedLongForm);
+
+ if (null != comment) {
+ string += " " + comment;
+ }
+
+ if (!(legalValues instanceof UniversalSet)) {
+ string += " " + legalValues;
+ }
+
+ return string;
+ }
+
+ public boolean equals(Object o) {
+ if (o instanceof Option) {
+ return (shortForm == ((Option) o).shortForm && longForm == ((Option) o).longForm);
+ } else {
+ return false;
+ }
+ }
+
+ public int hashCode() {
+ return (shortForm + longForm).hashCode();
+ }
+ }
+
+ static class UniversalSet<E> implements Set<E> {
+
+ public boolean add(Object o) {
+ throw new UnsupportedOperationException();
+ }
+
+ public boolean addAll(Collection c) {
+ throw new UnsupportedOperationException();
+ }
+
+ public void clear() {
+ throw new UnsupportedOperationException();
+ }
+
+ public boolean contains(Object o) {
+ return true;
+ }
+
+ public boolean containsAll(Collection c) {
+ return true;
+ }
+
+ public boolean isEmpty() {
+ return false;
+ }
+
+ public Iterator<E> iterator() {
+ return null;
+ }
+
+ public boolean remove(Object o) {
+ throw new UnsupportedOperationException();
+ }
+
+ public boolean removeAll(Collection c) {
+ throw new UnsupportedOperationException();
+ }
+
+ public boolean retainAll(Collection c) {
+ throw new UnsupportedOperationException();
+ }
+
+ public int size() {
+ return Integer.MAX_VALUE;
+ }
+
+ public Object[] toArray() {
+ return null;
+ }
+
+ public <T> T[] toArray(T[] a) {
+ return null;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/Constants.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/Constants.java b/joshua-core/src/main/java/org/apache/joshua/util/Constants.java
new file mode 100644
index 0000000..3d4139d
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/Constants.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util;
+
+/***
+ * One day, all constants should be moved here (many are in Vocabulary).
+ *
+ * @author Matt Post post@cs.jhu.edu
+ */
+
+public final class Constants {
+ public static String defaultNT = "[X]";
+
+ public static final String START_SYM = "<s>";
+ public static final String STOP_SYM = "</s>";
+ public static final String UNKNOWN_WORD = "<unk>";
+
+ public static final String fieldDelimiter = "\\s\\|{3}\\s";
+ public static final String spaceSeparator = "\\s+";
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/Counted.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/Counted.java b/joshua-core/src/main/java/org/apache/joshua/util/Counted.java
new file mode 100644
index 0000000..9f719b3
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/Counted.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util;
+
+import java.util.Comparator;
+
+/**
+ * Represents an object being counted, with the associated count.
+ *
+ * @author Lane Schwartz
+ */
+public class Counted<E> implements Comparable<Counted<E>> {
+
+ /** The element being counted. */
+ private final E element;
+
+ /** The count associated with the element. */
+ private final Integer count;
+
+ /**
+ * Constructs an object wrapping an element and its associated count.
+ *
+ * @param element An element being counted
+ * @param count The count associated with the element
+ */
+ public Counted(E element, int count) {
+ this.element = element;
+ this.count = count;
+ }
+
+ /**
+ * Gets the count associated with this object's element.
+ *
+ * @return The count associated with this object's element
+ */
+ public int getCount() {
+ return count;
+ }
+
+ /**
+ * Gets the element associated with this object.
+ *
+ * @return The element associated with this object
+ */
+ public E getElement() {
+ return element;
+ }
+
+ /**
+ * Compares this object to another counted object, according to the natural order of the counts
+ * associated with each object.
+ *
+ * @param o Another counted object
+ * @return -1 if the count of this object is less than the count of the other object, 0 if the
+ * counts are equal, or 1 if the count of this object is greater than the count of the
+ * other object
+ */
+ public int compareTo(Counted<E> o) {
+ return count.compareTo(o.count);
+ }
+
+ /**
+ * Gets a comparator that compares two counted objects based on the reverse of the natural order
+ * of the counts associated with each object.
+ * @param <E> todo
+ * @return A comparator that compares two counted objects based on the reverse of the natural
+ * order of the counts associated with each object
+ */
+ public static <E> Comparator<Counted<E>> getDescendingComparator() {
+ return new Comparator<Counted<E>>() {
+ public int compare(Counted<E> o1, Counted<E> o2) {
+ return (o2.count.compareTo(o1.count));
+ }
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/Counts.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/Counts.java b/joshua-core/src/main/java/org/apache/joshua/util/Counts.java
new file mode 100644
index 0000000..89a9f38
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/Counts.java
@@ -0,0 +1,306 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util;
+
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Map.Entry;
+
+/**
+ * Maintains element co-occurrence data.
+ *
+ * @author Lane Schwartz
+ * @author Chris Callison-Burch
+ */
+public class Counts<A, B> implements Iterable<Pair<A, B>> {
+
+ /**
+ * Stores the number of times instances of A and B co-occur.
+ */
+ private Map<A, Map<B, Integer>> counts;
+
+ /** Stores the number of times instances of B occur. */
+ private Map<B, Integer> bTotals;
+
+ /** Stores relative frequency estimates for p(A | B). */
+ private Map<A, Map<B, Float>> probabilities;
+
+ /** Stores relative frequency estimates for p(B | A). */
+ private Map<B, Map<A, Float>> reverseProbabilities;
+
+ /** Stores the value to return when an unseen pair is queried. */
+ private float floorProbability;
+
+ /**
+ * Constructs an initially empty co-occurrence counter, with floor probability set to
+ * <code>Float.MIN_VALUE</code>.
+ */
+ public Counts() {
+ this(Float.MIN_VALUE);
+ }
+
+ /**
+ * Constructs an initially empty co-occurrence counter.
+ *
+ * @param floorProbability Floor probability to use when an unseen pair is queried.
+ */
+ public Counts(float floorProbability) {
+ this.floorProbability = floorProbability;
+ this.counts = new HashMap<A, Map<B, Integer>>();
+ this.bTotals = new HashMap<B, Integer>();
+ this.probabilities = new HashMap<A, Map<B, Float>>();
+ this.reverseProbabilities = new HashMap<B, Map<A, Float>>();
+ }
+
+
+ /**
+ * Increments the co-occurrence count of the provided objects.
+ *
+ * @param a input object A
+ * @param b input object B
+ */
+ public void incrementCount(A a, B b) {
+ // increment the count and handle the adding of objects to the map if they aren't already there
+ {
+ Map<B, Integer> bMap;
+ if (counts.containsKey(a)) {
+ bMap = counts.get(a);
+ } else {
+ bMap = new HashMap<B, Integer>();
+ counts.put(a, bMap);
+ }
+
+ Integer previousCount;
+ if (bMap.containsKey(b)) {
+ previousCount = bMap.get(b);
+ } else {
+ previousCount = 0;
+ }
+ bMap.put(b, previousCount + 1);
+ }
+
+ // increments total for o2.
+ {
+ Integer previousTotal;
+ if (bTotals.containsKey(b)) {
+ previousTotal = bTotals.get(b);
+ } else {
+ previousTotal = 0;
+ }
+ bTotals.put(b, previousTotal + 1);
+ }
+
+ // Invalidate previously calculated probabilities
+ {
+ if (probabilities.containsKey(a)) {
+ probabilities.get(a).clear();
+ }
+
+ if (reverseProbabilities.containsKey(b)) {
+ reverseProbabilities.get(b).clear();
+ }
+ }
+ }
+
+ /**
+ * Gets the co-occurrence count for the two elements.
+ *
+ * @param a input object A
+ * @param b input object B
+ * @return the co-occurrence count for the two elements
+ */
+ public int getCount(A a, B b) {
+
+ int count = 0;
+ if (counts.containsKey(a)) {
+ Map<B, Integer> bMap = counts.get(a);
+ if (bMap.containsKey(b)) {
+ count = bMap.get(b);
+ }
+ }
+
+ return count;
+ }
+
+ /**
+ * Gets the total number of times the specified element has been seen.
+ *
+ * @param b
+ * @return the total number of times the specified element has been seen
+ */
+ int getCount(B b) {
+
+ return (bTotals.containsKey(b) ? bTotals.get(b) : 0);
+
+ }
+
+ /**
+ * Gets the probability of a given b.
+ * <p>
+ * This value is the relative frequency estimate.
+ *
+ * @param a object A
+ * @param b object B
+ * @return the probability of a given b.
+ */
+ public float getProbability(A a, B b) {
+
+ int count = getCount(a, b);
+ int bCount = getCount(b);
+
+ Float value;
+ if (count == 0 || bCount == 0) {
+
+ value = floorProbability;
+
+ } else {
+
+ Map<B, Float> bMap;
+ if (probabilities.containsKey(a)) {
+ bMap = probabilities.get(a);
+ } else {
+ bMap = new HashMap<B, Float>();
+ }
+
+
+ if (bMap.containsKey(b)) {
+ value = bMap.get(b);
+ } else {
+ value = (float) count / (float) getCount(b);
+ bMap.put(b, value);
+ }
+
+ }
+
+ return value;
+ }
+
+ /**
+ * Gets the probability of b given a.
+ * <p>
+ * This value is the relative frequency estimate in the reverse direction.
+ *
+ * @param b object B
+ * @param a object A
+ * @return the probability of b given a.
+ */
+ public float getReverseProbability(B b, A a) {
+
+ int count = getCount(a, b);
+
+ Float value = floorProbability;
+
+ if (count > 0) {
+
+ int aCount = 0;
+ for (Integer aValue : counts.get(a).values()) {
+ aCount += aValue;
+ }
+
+ if (aCount > 0) {
+
+ Map<A, Float> aMap;
+ if (reverseProbabilities.containsKey(b)) {
+ aMap = reverseProbabilities.get(b);
+ } else {
+ aMap = new HashMap<A, Float>();
+ }
+
+ if (aMap.containsKey(a)) {
+ value = aMap.get(a);
+ } else {
+ value = (float) count / (float) aCount;
+ }
+
+ }
+
+ }
+
+ return value;
+
+ }
+
+ /**
+ * Gets the floor probability that is returned whenever an unseen pair is queried.
+ *
+ * @return The floor probability that is returned whenever an unseen pair is queried
+ */
+ public float getFloorProbability() {
+ return this.floorProbability;
+ }
+
+ public void writeExternal(ObjectOutput out) throws IOException {
+ out.writeObject(counts);
+ out.writeObject(bTotals);
+ out.writeObject(probabilities);
+ out.writeObject(reverseProbabilities);
+ out.writeFloat(floorProbability);
+ // out.close();
+ }
+
+ @SuppressWarnings("unchecked")
+ public void readExternal(ObjectInput in) throws ClassNotFoundException, IOException {
+ this.counts = (HashMap<A, Map<B, Integer>>) in.readObject();
+ this.bTotals = (HashMap<B, Integer>) in.readObject();
+ this.probabilities = (HashMap<A, Map<B, Float>>) in.readObject();
+ this.reverseProbabilities = (HashMap<B, Map<A, Float>>) in.readObject();
+ this.floorProbability = in.readFloat();
+ }
+
+ /**
+ * Gets an iterator over all counted pairs.
+ * <p>
+ * The pairs are not guaranteed to be iterated over in any particular order.
+ *
+ * @return an iterator over all counted pairs
+ */
+ public Iterator<Pair<A, B>> iterator() {
+
+ final Iterator<Entry<A, Map<B, Integer>>> aIterator = counts.entrySet().iterator();
+
+ return new Iterator<Pair<A, B>>() {
+
+ Entry<A, Map<B, Integer>> entry = null;
+ Iterator<B> bIterator = null;
+
+ public boolean hasNext() {
+ return (bIterator != null && bIterator.hasNext()) || aIterator.hasNext();
+ }
+
+ public Pair<A, B> next() {
+ if (bIterator == null || !bIterator.hasNext()) {
+ entry = aIterator.next();
+ bIterator = entry.getValue().keySet().iterator();
+ }
+
+ return new Pair<A, B>(entry.getKey(), bIterator.next());
+ }
+
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ };
+ }
+
+}
[46/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/adagrad/AdaGradCore.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/adagrad/AdaGradCore.java b/joshua-core/src/main/java/org/apache/joshua/adagrad/AdaGradCore.java
new file mode 100755
index 0000000..789757f
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/adagrad/AdaGradCore.java
@@ -0,0 +1,3127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.adagrad;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.text.DecimalFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Random;
+import java.util.Scanner;
+import java.util.TreeSet;
+import java.util.Vector;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.Decoder;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.metrics.EvaluationMetric;
+import org.apache.joshua.util.StreamGobbler;
+
+import EDU.oswego.cs.dl.util.concurrent.ConcurrentHashMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This code was originally written by Yuan Cao, who copied the MERT code to produce this file.
+ */
+
+public class AdaGradCore {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AdaGradCore.class);
+ private final static double NegInf = (-1.0 / 0.0);
+ private final static double PosInf = (+1.0 / 0.0);
+ private final static double epsilon = 1.0 / 1000000;
+ private final static DecimalFormat f4 = new DecimalFormat("###0.0000");
+
+ private final JoshuaConfiguration joshuaConfiguration;
+ private final Runtime myRuntime = Runtime.getRuntime();
+
+ private TreeSet<Integer>[] indicesOfInterest_all;
+
+ private int progress;
+
+ private int verbosity; // anything of priority <= verbosity will be printed
+ // (lower value for priority means more important)
+
+ private Random randGen;
+ private int generatedRands;
+
+ private int numSentences;
+ // number of sentences in the dev set
+ // (aka the "MERT training" set)
+
+ private int numDocuments;
+ // number of documents in the dev set
+ // this should be 1, unless doing doc-level optimization
+
+ private int[] docOfSentence;
+ // docOfSentence[i] stores which document contains the i'th sentence.
+ // docOfSentence is 0-indexed, as are the documents (i.e. first doc is indexed 0)
+
+ private int[] docSubsetInfo;
+ // stores information regarding which subset of the documents are evaluated
+ // [0]: method (0-6)
+ // [1]: first (1-indexed)
+ // [2]: last (1-indexed)
+ // [3]: size
+ // [4]: center
+ // [5]: arg1
+ // [6]: arg2
+ // [1-6] are 0 for method 0, [6] is 0 for methods 1-4 as well
+ // only [1] and [2] are needed for optimization. The rest are only needed for an output message.
+
+ private int refsPerSen;
+ // number of reference translations per sentence
+
+ private int textNormMethod;
+ // 0: no normalization, 1: "NIST-style" tokenization, and also rejoin 'm, 're, *'s, 've, 'll, 'd,
+ // and n't,
+ // 2: apply 1 and also rejoin dashes between letters, 3: apply 1 and also drop non-ASCII
+ // characters
+ // 4: apply 1+2+3
+
+ private int numParams;
+ // total number of firing features
+ // this number may increase overtime as new n-best lists are decoded
+ // initially it is equal to the # of params in the parameter config file
+ private int numParamsOld;
+ // number of features before observing the new features fired in the current iteration
+
+ private double[] normalizationOptions;
+ // How should a lambda[] vector be normalized (before decoding)?
+ // nO[0] = 0: no normalization
+ // nO[0] = 1: scale so that parameter nO[2] has absolute value nO[1]
+ // nO[0] = 2: scale so that the maximum absolute value is nO[1]
+ // nO[0] = 3: scale so that the minimum absolute value is nO[1]
+ // nO[0] = 4: scale so that the L-nO[1] norm equals nO[2]
+
+ /* *********************************************************** */
+ /* NOTE: indexing starts at 1 in the following few arrays: */
+ /* *********************************************************** */
+
+ // private double[] lambda;
+ private ArrayList<Double> lambda = new ArrayList<Double>();
+ // the current weight vector. NOTE: indexing starts at 1.
+ private ArrayList<Double> bestLambda = new ArrayList<Double>();
+ // the best weight vector across all iterations
+
+ private boolean[] isOptimizable;
+ // isOptimizable[c] = true iff lambda[c] should be optimized
+
+ private double[] minRandValue;
+ private double[] maxRandValue;
+ // when choosing a random value for the lambda[c] parameter, it will be
+ // chosen from the [minRandValue[c],maxRandValue[c]] range.
+ // (*) minRandValue and maxRandValue must be real values, but not -Inf or +Inf
+
+ private double[] defaultLambda;
+ // "default" parameter values; simply the values read in the parameter file
+ // USED FOR NON-OPTIMIZABLE (FIXED) FEATURES
+
+ /* *********************************************************** */
+ /* *********************************************************** */
+
+ private Decoder myDecoder;
+ // COMMENT OUT if decoder is not Joshua
+
+ private String decoderCommand;
+ // the command that runs the decoder; read from decoderCommandFileName
+
+ private int decVerbosity;
+ // verbosity level for decoder output. If 0, decoder output is ignored.
+ // If 1, decoder output is printed.
+
+ private int validDecoderExitValue;
+ // return value from running the decoder command that indicates success
+
+ private int numOptThreads;
+ // number of threads to run things in parallel
+
+ private int saveInterFiles;
+ // 0: nothing, 1: only configs, 2: only n-bests, 3: both configs and n-bests
+
+ private int compressFiles;
+ // should AdaGrad gzip the large files? If 0, no compression takes place.
+ // If 1, compression is performed on: decoder output files, temp sents files,
+ // and temp feats files.
+
+ private int sizeOfNBest;
+ // size of N-best list generated by decoder at each iteration
+ // (aka simply N, but N is a bad variable name)
+
+ private long seed;
+ // seed used to create random number generators
+
+ private boolean randInit;
+ // if true, parameters are initialized randomly. If false, parameters
+ // are initialized using values from parameter file.
+
+ private int maxMERTIterations, minMERTIterations, prevMERTIterations;
+ // max: maximum number of MERT iterations
+ // min: minimum number of MERT iterations before an early MERT exit
+ // prev: number of previous MERT iterations from which to consider candidates (in addition to
+ // the candidates from the current iteration)
+
+ private double stopSigValue;
+ // early MERT exit if no weight changes by more than stopSigValue
+ // (but see minMERTIterations above and stopMinIts below)
+
+ private int stopMinIts;
+ // some early stopping criterion must be satisfied in stopMinIts *consecutive* iterations
+ // before an early exit (but see minMERTIterations above)
+
+ private boolean oneModificationPerIteration;
+ // if true, each MERT iteration performs at most one parameter modification.
+ // If false, a new MERT iteration starts (i.e. a new N-best list is
+ // generated) only after the previous iteration reaches a local maximum.
+
+ private String metricName;
+ // name of evaluation metric optimized by MERT
+
+ private String metricName_display;
+ // name of evaluation metric optimized by MERT, possibly with "doc-level " prefixed
+
+ private String[] metricOptions;
+ // options for the evaluation metric (e.g. for BLEU, maxGramLength and effLengthMethod)
+
+ private EvaluationMetric evalMetric;
+ // the evaluation metric used by MERT
+
+ private int suffStatsCount;
+ // number of sufficient statistics for the evaluation metric
+
+ private String tmpDirPrefix;
+ // prefix for the AdaGrad.temp.* files
+
+ private boolean passIterationToDecoder;
+ // should the iteration number be passed as an argument to decoderCommandFileName?
+
+ // used by adagrad
+ private boolean needShuffle = true; // shuffle the training sentences or not
+ private boolean needAvg = true; // average the weihgts or not?
+ private boolean usePseudoBleu = true; // need to use pseudo corpus to compute bleu?
+ private boolean returnBest = true; // return the best weight during tuning
+ private boolean needScale = true; // need scaling?
+ private String trainingMode;
+ private int oraSelectMode = 1;
+ private int predSelectMode = 1;
+ private int adagradIter = 1;
+ private int regularization = 2;
+ private int batchSize = 1;
+ private double eta;
+ private double lam;
+ private double R = 0.99; // corpus decay when pseudo corpus is used for bleu computation
+ // private double sentForScale = 0.15; //percentage of sentences for scale factor estimation
+ private double scoreRatio = 5.0; // sclale so that model_score/metric_score = scoreratio
+ private double prevMetricScore = 0; // final metric score of the previous iteration, used only
+ // when returnBest = true
+
+ private String dirPrefix; // where are all these files located?
+ private String paramsFileName, docInfoFileName, finalLambdaFileName;
+ private String sourceFileName, refFileName, decoderOutFileName;
+ private String decoderConfigFileName, decoderCommandFileName;
+ private String fakeFileNameTemplate, fakeFileNamePrefix, fakeFileNameSuffix;
+
+ // e.g. output.it[1-x].someOldRun would be specified as:
+ // output.it?.someOldRun
+ // and we'd have prefix = "output.it" and suffix = ".sameOldRun"
+
+ // private int useDisk;
+
+ public AdaGradCore(JoshuaConfiguration joshuaConfiguration) {
+ this.joshuaConfiguration = joshuaConfiguration;
+ }
+
+ public AdaGradCore(String[] args, JoshuaConfiguration joshuaConfiguration) {
+ this.joshuaConfiguration = joshuaConfiguration;
+ EvaluationMetric.set_knownMetrics();
+ processArgsArray(args);
+ initialize(0);
+ }
+
+ public AdaGradCore(String configFileName, JoshuaConfiguration joshuaConfiguration) {
+ this.joshuaConfiguration = joshuaConfiguration;
+ EvaluationMetric.set_knownMetrics();
+ processArgsArray(cfgFileToArgsArray(configFileName));
+ initialize(0);
+ }
+
+ private void initialize(int randsToSkip) {
+ println("NegInf: " + NegInf + ", PosInf: " + PosInf + ", epsilon: " + epsilon, 4);
+
+ randGen = new Random(seed);
+ for (int r = 1; r <= randsToSkip; ++r) {
+ randGen.nextDouble();
+ }
+ generatedRands = randsToSkip;
+
+ if (randsToSkip == 0) {
+ println("----------------------------------------------------", 1);
+ println("Initializing...", 1);
+ println("----------------------------------------------------", 1);
+ println("", 1);
+
+ println("Random number generator initialized using seed: " + seed, 1);
+ println("", 1);
+ }
+
+ // count the total num of sentences to be decoded, reffilename is the combined reference file
+ // name(auto generated)
+ numSentences = countLines(refFileName) / refsPerSen;
+
+ // ??
+ processDocInfo();
+ // sets numDocuments and docOfSentence[]
+
+ if (numDocuments > 1)
+ metricName_display = "doc-level " + metricName;
+
+ // ??
+ set_docSubsetInfo(docSubsetInfo);
+
+ // count the number of initial features
+ numParams = countNonEmptyLines(paramsFileName) - 1;
+ numParamsOld = numParams;
+
+ // read parameter config file
+ try {
+ // read dense parameter names
+ BufferedReader inFile_names = new BufferedReader(new FileReader(paramsFileName));
+
+ for (int c = 1; c <= numParams; ++c) {
+ String line = "";
+ while (line != null && line.length() == 0) { // skip empty lines
+ line = inFile_names.readLine();
+ }
+
+ // save feature names
+ String paramName = (line.substring(0, line.indexOf("|||"))).trim();
+ Vocabulary.id(paramName);
+ // System.err.println(String.format("VOCAB(%s) = %d", paramName, id));
+ }
+
+ inFile_names.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ // the parameter file contains one line per parameter
+ // and one line for the normalization method
+ // indexing starts at 1 in these arrays
+ for (int p = 0; p <= numParams; ++p)
+ lambda.add(new Double(0));
+ bestLambda.add(new Double(0));
+ // why only lambda is a list? because the size of lambda
+ // may increase over time, but other arrays are specified in
+ // the param config file, only used for initialization
+ isOptimizable = new boolean[1 + numParams];
+ minRandValue = new double[1 + numParams];
+ maxRandValue = new double[1 + numParams];
+ defaultLambda = new double[1 + numParams];
+ normalizationOptions = new double[3];
+
+ // read initial param values
+ processParamFile();
+ // sets the arrays declared just above
+
+ // SentenceInfo.createV(); // uncomment ONLY IF using vocabulary implementation of SentenceInfo
+
+ String[][] refSentences = new String[numSentences][refsPerSen];
+
+ try {
+
+ // read in reference sentences
+ InputStream inStream_refs = new FileInputStream(new File(refFileName));
+ BufferedReader inFile_refs = new BufferedReader(new InputStreamReader(inStream_refs, "utf8"));
+
+ for (int i = 0; i < numSentences; ++i) {
+ for (int r = 0; r < refsPerSen; ++r) {
+ // read the rth reference translation for the ith sentence
+ refSentences[i][r] = inFile_refs.readLine();
+ }
+ }
+
+ inFile_refs.close();
+
+ // normalize reference sentences
+ for (int i = 0; i < numSentences; ++i) {
+ for (int r = 0; r < refsPerSen; ++r) {
+ // normalize the rth reference translation for the ith sentence
+ refSentences[i][r] = normalize(refSentences[i][r], textNormMethod);
+ }
+ }
+
+ // read in decoder command, if any
+ decoderCommand = null;
+ if (decoderCommandFileName != null) {
+ if (fileExists(decoderCommandFileName)) {
+ BufferedReader inFile_comm = new BufferedReader(new FileReader(decoderCommandFileName));
+ decoderCommand = inFile_comm.readLine(); // READ IN DECODE COMMAND
+ inFile_comm.close();
+ }
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ // set static data members for the EvaluationMetric class
+ EvaluationMetric.set_numSentences(numSentences);
+ EvaluationMetric.set_numDocuments(numDocuments);
+ EvaluationMetric.set_refsPerSen(refsPerSen);
+ EvaluationMetric.set_refSentences(refSentences);
+ EvaluationMetric.set_tmpDirPrefix(tmpDirPrefix);
+
+ evalMetric = EvaluationMetric.getMetric(metricName, metricOptions);
+ // used only if returnBest = true
+ prevMetricScore = evalMetric.getToBeMinimized() ? PosInf : NegInf;
+
+ // length of sufficient statistics
+ // for bleu: suffstatscount=8 (2*ngram+2)
+ suffStatsCount = evalMetric.get_suffStatsCount();
+
+ // set static data members for the IntermediateOptimizer class
+ /*
+ * IntermediateOptimizer.set_MERTparams(numSentences, numDocuments, docOfSentence,
+ * docSubsetInfo, numParams, normalizationOptions, isOptimizable oneModificationPerIteration,
+ * evalMetric, tmpDirPrefix, verbosity);
+ */
+
+ // print info
+ if (randsToSkip == 0) { // i.e. first iteration
+ println("Number of sentences: " + numSentences, 1);
+ println("Number of documents: " + numDocuments, 1);
+ println("Optimizing " + metricName_display, 1);
+
+ /*
+ * print("docSubsetInfo: {", 1); for (int f = 0; f < 6; ++f) print(docSubsetInfo[f] + ", ",
+ * 1); println(docSubsetInfo[6] + "}", 1);
+ */
+
+ println("Number of initial features: " + numParams, 1);
+ print("Initial feature names: {", 1);
+
+ for (int c = 1; c <= numParams; ++c)
+ print("\"" + Vocabulary.word(c) + "\"", 1);
+ println("}", 1);
+ println("", 1);
+
+ // TODO just print the correct info
+ println("c Default value\tOptimizable?\tRand. val. range", 1);
+
+ for (int c = 1; c <= numParams; ++c) {
+ print(c + " " + f4.format(lambda.get(c).doubleValue()) + "\t\t", 1);
+
+ if (!isOptimizable[c]) {
+ println(" No", 1);
+ } else {
+ print(" Yes\t\t", 1);
+ print(" [" + minRandValue[c] + "," + maxRandValue[c] + "]", 1);
+ println("", 1);
+ }
+ }
+
+ println("", 1);
+ print("Weight vector normalization method: ", 1);
+ if (normalizationOptions[0] == 0) {
+ println("none.", 1);
+ } else if (normalizationOptions[0] == 1) {
+ println(
+ "weights will be scaled so that the \""
+ + Vocabulary.word((int) normalizationOptions[2])
+ + "\" weight has an absolute value of " + normalizationOptions[1] + ".", 1);
+ } else if (normalizationOptions[0] == 2) {
+ println("weights will be scaled so that the maximum absolute value is "
+ + normalizationOptions[1] + ".", 1);
+ } else if (normalizationOptions[0] == 3) {
+ println("weights will be scaled so that the minimum absolute value is "
+ + normalizationOptions[1] + ".", 1);
+ } else if (normalizationOptions[0] == 4) {
+ println("weights will be scaled so that the L-" + normalizationOptions[1] + " norm is "
+ + normalizationOptions[2] + ".", 1);
+ }
+
+ println("", 1);
+
+ println("----------------------------------------------------", 1);
+ println("", 1);
+
+ // rename original config file so it doesn't get overwritten
+ // (original name will be restored in finish())
+ renameFile(decoderConfigFileName, decoderConfigFileName + ".AdaGrad.orig");
+ } // if (randsToSkip == 0)
+
+ // by default, load joshua decoder
+ if (decoderCommand == null && fakeFileNameTemplate == null) {
+ println("Loading Joshua decoder...", 1);
+ myDecoder = new Decoder(joshuaConfiguration, decoderConfigFileName + ".AdaGrad.orig");
+ println("...finished loading @ " + (new Date()), 1);
+ println("");
+ } else {
+ myDecoder = null;
+ }
+
+ @SuppressWarnings("unchecked")
+ TreeSet<Integer>[] temp_TSA = new TreeSet[numSentences];
+ indicesOfInterest_all = temp_TSA;
+
+ for (int i = 0; i < numSentences; ++i) {
+ indicesOfInterest_all[i] = new TreeSet<Integer>();
+ }
+ } // void initialize(...)
+
+ // -------------------------
+
+ public void run_AdaGrad() {
+ run_AdaGrad(minMERTIterations, maxMERTIterations, prevMERTIterations);
+ }
+
+ public void run_AdaGrad(int minIts, int maxIts, int prevIts) {
+ // FIRST, CLEAN ALL PREVIOUS TEMP FILES
+ String dir;
+ int k = tmpDirPrefix.lastIndexOf("/");
+ if (k >= 0) {
+ dir = tmpDirPrefix.substring(0, k + 1);
+ } else {
+ dir = "./";
+ }
+ String files;
+ File folder = new File(dir);
+
+ if (folder.exists()) {
+ File[] listOfFiles = folder.listFiles();
+
+ for (int i = 0; i < listOfFiles.length; i++) {
+ if (listOfFiles[i].isFile()) {
+ files = listOfFiles[i].getName();
+ if (files.startsWith("AdaGrad.temp")) {
+ deleteFile(files);
+ }
+ }
+ }
+ }
+
+ println("----------------------------------------------------", 1);
+ println("AdaGrad run started @ " + (new Date()), 1);
+ // printMemoryUsage();
+ println("----------------------------------------------------", 1);
+ println("", 1);
+
+ // if no default lambda is provided
+ if (randInit) {
+ println("Initializing lambda[] randomly.", 1);
+ // initialize optimizable parameters randomly (sampling uniformly from
+ // that parameter's random value range)
+ lambda = randomLambda();
+ }
+
+ println("Initial lambda[]: " + lambdaToString(lambda), 1);
+ println("", 1);
+
+ int[] maxIndex = new int[numSentences];
+
+ // HashMap<Integer,int[]>[] suffStats_array = new HashMap[numSentences];
+ // suffStats_array[i] maps candidates of interest for sentence i to an array
+ // storing the sufficient statistics for that candidate
+
+ int earlyStop = 0;
+ // number of consecutive iteration an early stopping criterion was satisfied
+
+ for (int iteration = 1;; ++iteration) {
+
+ // what does "A" contain?
+ // retA[0]: FINAL_score
+ // retA[1]: earlyStop
+ // retA[2]: should this be the last iteration?
+ double[] A = run_single_iteration(iteration, minIts, maxIts, prevIts, earlyStop, maxIndex);
+ if (A != null) {
+ earlyStop = (int) A[1];
+ if (A[2] == 1)
+ break;
+ } else {
+ break;
+ }
+
+ } // for (iteration)
+
+ println("", 1);
+
+ println("----------------------------------------------------", 1);
+ println("AdaGrad run ended @ " + (new Date()), 1);
+ // printMemoryUsage();
+ println("----------------------------------------------------", 1);
+ println("", 1);
+ if (!returnBest)
+ println("FINAL lambda: " + lambdaToString(lambda), 1);
+ // + " (" + metricName_display + ": " + FINAL_score + ")",1);
+ else
+ println("BEST lambda: " + lambdaToString(lambda), 1);
+
+ // delete intermediate .temp.*.it* decoder output files
+ for (int iteration = 1; iteration <= maxIts; ++iteration) {
+ if (compressFiles == 1) {
+ deleteFile(tmpDirPrefix + "temp.sents.it" + iteration + ".gz");
+ deleteFile(tmpDirPrefix + "temp.feats.it" + iteration + ".gz");
+ if (fileExists(tmpDirPrefix + "temp.stats.it" + iteration + ".copy.gz")) {
+ deleteFile(tmpDirPrefix + "temp.stats.it" + iteration + ".copy.gz");
+ } else {
+ deleteFile(tmpDirPrefix + "temp.stats.it" + iteration + ".gz");
+ }
+ } else {
+ deleteFile(tmpDirPrefix + "temp.sents.it" + iteration);
+ deleteFile(tmpDirPrefix + "temp.feats.it" + iteration);
+ if (fileExists(tmpDirPrefix + "temp.stats.it" + iteration + ".copy")) {
+ deleteFile(tmpDirPrefix + "temp.stats.it" + iteration + ".copy");
+ } else {
+ deleteFile(tmpDirPrefix + "temp.stats.it" + iteration);
+ }
+ }
+ }
+ } // void run_AdaGrad(int maxIts)
+
+ // this is the key function!
+ @SuppressWarnings("unchecked")
+ public double[] run_single_iteration(int iteration, int minIts, int maxIts, int prevIts,
+ int earlyStop, int[] maxIndex) {
+ double FINAL_score = 0;
+
+ double[] retA = new double[3];
+ // retA[0]: FINAL_score
+ // retA[1]: earlyStop
+ // retA[2]: should this be the last iteration?
+
+ boolean done = false;
+ retA[2] = 1; // will only be made 0 if we don't break from the following loop
+
+ // save feats and stats for all candidates(old & new)
+ HashMap<String, String>[] feat_hash = new HashMap[numSentences];
+ for (int i = 0; i < numSentences; i++)
+ feat_hash[i] = new HashMap<String, String>();
+
+ HashMap<String, String>[] stats_hash = new HashMap[numSentences];
+ for (int i = 0; i < numSentences; i++)
+ stats_hash[i] = new HashMap<String, String>();
+
+ while (!done) { // NOTE: this "loop" will only be carried out once
+ println("--- Starting AdaGrad iteration #" + iteration + " @ " + (new Date()) + " ---", 1);
+
+ // printMemoryUsage();
+
+ /******************************/
+ // CREATE DECODER CONFIG FILE //
+ /******************************/
+
+ createConfigFile(lambda, decoderConfigFileName, decoderConfigFileName + ".AdaGrad.orig");
+ // i.e. use the original config file as a template
+
+ /***************/
+ // RUN DECODER //
+ /***************/
+
+ if (iteration == 1) {
+ println("Decoding using initial weight vector " + lambdaToString(lambda), 1);
+ } else {
+ println("Redecoding using weight vector " + lambdaToString(lambda), 1);
+ }
+
+ // generate the n-best file after decoding
+ String[] decRunResult = run_decoder(iteration); // iteration passed in case fake decoder will
+ // be used
+ // [0] name of file to be processed
+ // [1] indicates how the output file was obtained:
+ // 1: external decoder
+ // 2: fake decoder
+ // 3: internal decoder
+
+ if (!decRunResult[1].equals("2")) {
+ println("...finished decoding @ " + (new Date()), 1);
+ }
+
+ checkFile(decRunResult[0]);
+
+ /************* END OF DECODING **************/
+
+ println("Producing temp files for iteration " + iteration, 3);
+
+ produceTempFiles(decRunResult[0], iteration);
+
+ // save intermedidate output files
+ // save joshua.config.adagrad.it*
+ if (saveInterFiles == 1 || saveInterFiles == 3) { // make copy of intermediate config file
+ if (!copyFile(decoderConfigFileName, decoderConfigFileName + ".AdaGrad.it" + iteration)) {
+ println("Warning: attempt to make copy of decoder config file (to create"
+ + decoderConfigFileName + ".AdaGrad.it" + iteration + ") was unsuccessful!", 1);
+ }
+ }
+
+ // save output.nest.AdaGrad.it*
+ if (saveInterFiles == 2 || saveInterFiles == 3) { // make copy of intermediate decoder output
+ // file...
+
+ if (!decRunResult[1].equals("2")) { // ...but only if no fake decoder
+ if (!decRunResult[0].endsWith(".gz")) {
+ if (!copyFile(decRunResult[0], decRunResult[0] + ".AdaGrad.it" + iteration)) {
+ println("Warning: attempt to make copy of decoder output file (to create"
+ + decRunResult[0] + ".AdaGrad.it" + iteration + ") was unsuccessful!", 1);
+ }
+ } else {
+ String prefix = decRunResult[0].substring(0, decRunResult[0].length() - 3);
+ if (!copyFile(prefix + ".gz", prefix + ".AdaGrad.it" + iteration + ".gz")) {
+ println("Warning: attempt to make copy of decoder output file (to create" + prefix
+ + ".AdaGrad.it" + iteration + ".gz" + ") was unsuccessful!", 1);
+ }
+ }
+
+ if (compressFiles == 1 && !decRunResult[0].endsWith(".gz")) {
+ gzipFile(decRunResult[0] + ".AdaGrad.it" + iteration);
+ }
+ } // if (!fake)
+ }
+
+ // ------------- end of saving .adagrad.it* files ---------------
+
+ int[] candCount = new int[numSentences];
+ int[] lastUsedIndex = new int[numSentences];
+
+ ConcurrentHashMap[] suffStats_array = new ConcurrentHashMap[numSentences];
+ for (int i = 0; i < numSentences; ++i) {
+ candCount[i] = 0;
+ lastUsedIndex[i] = -1;
+ // suffStats_array[i].clear();
+ suffStats_array[i] = new ConcurrentHashMap();
+ }
+
+ // initLambda[0] is not used!
+ double[] initialLambda = new double[1 + numParams];
+ for (int i = 1; i <= numParams; ++i)
+ initialLambda[i] = lambda.get(i);
+
+ // the "score" in initialScore refers to that
+ // assigned by the evaluation metric)
+
+ // you may consider all candidates from iter 1, or from iter (iteration-prevIts) to current
+ // iteration
+ int firstIt = Math.max(1, iteration - prevIts);
+ // i.e. only process candidates from the current iteration and candidates
+ // from up to prevIts previous iterations.
+ println("Reading candidate translations from iterations " + firstIt + "-" + iteration, 1);
+ println("(and computing " + metricName
+ + " sufficient statistics for previously unseen candidates)", 1);
+ print(" Progress: ");
+
+ int[] newCandidatesAdded = new int[1 + iteration];
+ for (int it = 1; it <= iteration; ++it)
+ newCandidatesAdded[it] = 0;
+
+ try {
+ // read temp files from all past iterations
+ // 3 types of temp files:
+ // 1. output hypo at iter i
+ // 2. feature value of each hypo at iter i
+ // 3. suff stats of each hypo at iter i
+
+ // each inFile corresponds to the output of an iteration
+ // (index 0 is not used; no corresponding index for the current iteration)
+ BufferedReader[] inFile_sents = new BufferedReader[iteration];
+ BufferedReader[] inFile_feats = new BufferedReader[iteration];
+ BufferedReader[] inFile_stats = new BufferedReader[iteration];
+
+ // temp file(array) from previous iterations
+ for (int it = firstIt; it < iteration; ++it) {
+ InputStream inStream_sents, inStream_feats, inStream_stats;
+ if (compressFiles == 0) {
+ inStream_sents = new FileInputStream(tmpDirPrefix + "temp.sents.it" + it);
+ inStream_feats = new FileInputStream(tmpDirPrefix + "temp.feats.it" + it);
+ inStream_stats = new FileInputStream(tmpDirPrefix + "temp.stats.it" + it);
+ } else {
+ inStream_sents = new GZIPInputStream(new FileInputStream(tmpDirPrefix + "temp.sents.it"
+ + it + ".gz"));
+ inStream_feats = new GZIPInputStream(new FileInputStream(tmpDirPrefix + "temp.feats.it"
+ + it + ".gz"));
+ inStream_stats = new GZIPInputStream(new FileInputStream(tmpDirPrefix + "temp.stats.it"
+ + it + ".gz"));
+ }
+
+ inFile_sents[it] = new BufferedReader(new InputStreamReader(inStream_sents, "utf8"));
+ inFile_feats[it] = new BufferedReader(new InputStreamReader(inStream_feats, "utf8"));
+ inFile_stats[it] = new BufferedReader(new InputStreamReader(inStream_stats, "utf8"));
+ }
+
+ InputStream inStream_sentsCurrIt, inStream_featsCurrIt, inStream_statsCurrIt;
+ // temp file for current iteration!
+ if (compressFiles == 0) {
+ inStream_sentsCurrIt = new FileInputStream(tmpDirPrefix + "temp.sents.it" + iteration);
+ inStream_featsCurrIt = new FileInputStream(tmpDirPrefix + "temp.feats.it" + iteration);
+ } else {
+ inStream_sentsCurrIt = new GZIPInputStream(new FileInputStream(tmpDirPrefix
+ + "temp.sents.it" + iteration + ".gz"));
+ inStream_featsCurrIt = new GZIPInputStream(new FileInputStream(tmpDirPrefix
+ + "temp.feats.it" + iteration + ".gz"));
+ }
+
+ BufferedReader inFile_sentsCurrIt = new BufferedReader(new InputStreamReader(
+ inStream_sentsCurrIt, "utf8"));
+ BufferedReader inFile_featsCurrIt = new BufferedReader(new InputStreamReader(
+ inStream_featsCurrIt, "utf8"));
+
+ BufferedReader inFile_statsCurrIt = null; // will only be used if statsCurrIt_exists below
+ // is set to true
+ PrintWriter outFile_statsCurrIt = null; // will only be used if statsCurrIt_exists below is
+ // set to false
+
+ // just to check if temp.stat.it.iteration exists
+ boolean statsCurrIt_exists = false;
+
+ if (fileExists(tmpDirPrefix + "temp.stats.it" + iteration)) {
+ inStream_statsCurrIt = new FileInputStream(tmpDirPrefix + "temp.stats.it" + iteration);
+ inFile_statsCurrIt = new BufferedReader(new InputStreamReader(inStream_statsCurrIt,
+ "utf8"));
+ statsCurrIt_exists = true;
+ copyFile(tmpDirPrefix + "temp.stats.it" + iteration, tmpDirPrefix + "temp.stats.it"
+ + iteration + ".copy");
+ } else if (fileExists(tmpDirPrefix + "temp.stats.it" + iteration + ".gz")) {
+ inStream_statsCurrIt = new GZIPInputStream(new FileInputStream(tmpDirPrefix
+ + "temp.stats.it" + iteration + ".gz"));
+ inFile_statsCurrIt = new BufferedReader(new InputStreamReader(inStream_statsCurrIt,
+ "utf8"));
+ statsCurrIt_exists = true;
+ copyFile(tmpDirPrefix + "temp.stats.it" + iteration + ".gz", tmpDirPrefix
+ + "temp.stats.it" + iteration + ".copy.gz");
+ } else {
+ outFile_statsCurrIt = new PrintWriter(tmpDirPrefix + "temp.stats.it" + iteration);
+ }
+
+ // output the 4^th temp file: *.temp.stats.merged
+ PrintWriter outFile_statsMerged = new PrintWriter(tmpDirPrefix + "temp.stats.merged");
+ // write sufficient statistics from all the sentences
+ // from the output files into a single file
+ PrintWriter outFile_statsMergedKnown = new PrintWriter(tmpDirPrefix
+ + "temp.stats.mergedKnown");
+ // write sufficient statistics from all the sentences
+ // from the output files into a single file
+
+ // output the 5^th 6^th temp file, but will be deleted at the end of the function
+ FileOutputStream outStream_unknownCands = new FileOutputStream(tmpDirPrefix
+ + "temp.currIt.unknownCands", false);
+ OutputStreamWriter outStreamWriter_unknownCands = new OutputStreamWriter(
+ outStream_unknownCands, "utf8");
+ BufferedWriter outFile_unknownCands = new BufferedWriter(outStreamWriter_unknownCands);
+
+ PrintWriter outFile_unknownIndices = new PrintWriter(tmpDirPrefix
+ + "temp.currIt.unknownIndices");
+
+ String sents_str, feats_str, stats_str;
+
+ // BUG: this assumes a candidate string cannot be produced for two
+ // different source sentences, which is not necessarily true
+ // (It's not actually a bug, but only because existingCandStats gets
+ // cleared before moving to the next source sentence.)
+ // FIX: should be made an array, indexed by i
+ HashMap<String, String> existingCandStats = new HashMap<String, String>();
+ // VERY IMPORTANT:
+ // A CANDIDATE X MAY APPEARED IN ITER 1, ITER 3
+ // BUT IF THE USER SPECIFIED TO CONSIDER ITERATIONS FROM ONLY ITER 2, THEN
+ // X IS NOT A "REPEATED" CANDIDATE IN ITER 3. THEREFORE WE WANT TO KEEP THE
+ // SUFF STATS FOR EACH CANDIDATE(TO SAVE COMPUTATION IN THE FUTURE)
+
+ // Stores precalculated sufficient statistics for candidates, in case
+ // the same candidate is seen again. (SS stored as a String.)
+ // Q: Why do we care? If we see the same candidate again, aren't we going
+ // to ignore it? So, why do we care about the SS of this repeat candidate?
+ // A: A "repeat" candidate may not be a repeat candidate in later
+ // iterations if the user specifies a value for prevMERTIterations
+ // that causes MERT to skip candidates from early iterations.
+
+ double[] currFeatVal = new double[1 + numParams];
+ String[] featVal_str;
+
+ int totalCandidateCount = 0;
+
+ // new candidate size for each sentence
+ int[] sizeUnknown_currIt = new int[numSentences];
+
+ for (int i = 0; i < numSentences; ++i) {
+ // process candidates from previous iterations
+ // low efficiency? for each iteration, it reads in all previous iteration outputs
+ // therefore a lot of overlapping jobs
+ // this is an easy implementation to deal with the situation in which user only specified
+ // "previt" and hopes to consider only the previous previt
+ // iterations, then for each iteration the existing candadites will be different
+ for (int it = firstIt; it < iteration; ++it) {
+ // Why up to but *excluding* iteration?
+ // Because the last iteration is handled a little differently, since
+ // the SS must be calculated (and the corresponding file created),
+ // which is not true for previous iterations.
+
+ for (int n = 0; n <= sizeOfNBest; ++n) {
+ // note that in all temp files, "||||||" is a separator between 2 n-best lists
+
+ // Why up to and *including* sizeOfNBest?
+ // So that it would read the "||||||" separator even if there is
+ // a complete list of sizeOfNBest candidates.
+
+ // for the nth candidate for the ith sentence, read the sentence, feature values,
+ // and sufficient statistics from the various temp files
+
+ // read one line of temp.sent, temp.feat, temp.stats from iteration it
+ sents_str = inFile_sents[it].readLine();
+ feats_str = inFile_feats[it].readLine();
+ stats_str = inFile_stats[it].readLine();
+
+ if (sents_str.equals("||||||")) {
+ n = sizeOfNBest + 1; // move on to the next n-best list
+ } else if (!existingCandStats.containsKey(sents_str)) // if this candidate does not
+ // exist
+ {
+ outFile_statsMergedKnown.println(stats_str);
+
+ // save feats & stats
+ feat_hash[i].put(sents_str, feats_str);
+ stats_hash[i].put(sents_str, stats_str);
+
+ // extract feature value
+ featVal_str = feats_str.split("\\s+");
+
+ if (feats_str.indexOf('=') != -1) {
+ for (String featurePair : featVal_str) {
+ String[] pair = featurePair.split("=");
+ String name = pair[0];
+ Double value = Double.parseDouble(pair[1]);
+ }
+ }
+ existingCandStats.put(sents_str, stats_str);
+ candCount[i] += 1;
+ newCandidatesAdded[it] += 1;
+
+ } // if unseen candidate
+ } // for (n)
+ } // for (it)
+
+ outFile_statsMergedKnown.println("||||||");
+
+ // ---------- end of processing previous iterations ----------
+ // ---------- now start processing new candidates ----------
+
+ // now process the candidates of the current iteration
+ // now determine the new candidates of the current iteration
+
+ /*
+ * remember: BufferedReader inFile_sentsCurrIt BufferedReader inFile_featsCurrIt
+ * PrintWriter outFile_statsCurrIt
+ */
+
+ String[] sentsCurrIt_currSrcSent = new String[sizeOfNBest + 1];
+
+ Vector<String> unknownCands_V = new Vector<String>();
+ // which candidates (of the i'th source sentence) have not been seen before
+ // this iteration?
+
+ for (int n = 0; n <= sizeOfNBest; ++n) {
+ // Why up to and *including* sizeOfNBest?
+ // So that it would read the "||||||" separator even if there is
+ // a complete list of sizeOfNBest candidates.
+
+ // for the nth candidate for the ith sentence, read the sentence,
+ // and store it in the sentsCurrIt_currSrcSent array
+
+ sents_str = inFile_sentsCurrIt.readLine(); // read one candidate from the current
+ // iteration
+ sentsCurrIt_currSrcSent[n] = sents_str; // Note: possibly "||||||"
+
+ if (sents_str.equals("||||||")) {
+ n = sizeOfNBest + 1;
+ } else if (!existingCandStats.containsKey(sents_str)) {
+ unknownCands_V.add(sents_str); // NEW CANDIDATE FROM THIS ITERATION
+ writeLine(sents_str, outFile_unknownCands);
+ outFile_unknownIndices.println(i); // INDEX OF THE NEW CANDIDATES
+ newCandidatesAdded[iteration] += 1;
+ existingCandStats.put(sents_str, "U"); // i.e. unknown
+ // we add sents_str to avoid duplicate entries in unknownCands_V
+ }
+ } // for (n)
+
+ // only compute suff stats for new candidates
+ // now unknownCands_V has the candidates for which we need to calculate
+ // sufficient statistics (for the i'th source sentence)
+ int sizeUnknown = unknownCands_V.size();
+ sizeUnknown_currIt[i] = sizeUnknown;
+
+ existingCandStats.clear();
+
+ } // for (i) each sentence
+
+ // ---------- end of merging candidates stats from previous iterations
+ // and finding new candidates ------------
+
+ /*
+ * int[][] newSuffStats = null; if (!statsCurrIt_exists && sizeUnknown > 0) { newSuffStats =
+ * evalMetric.suffStats(unknownCands, indices); }
+ */
+
+ outFile_statsMergedKnown.close();
+ outFile_unknownCands.close();
+ outFile_unknownIndices.close();
+
+ // want to re-open all temp files and start from scratch again?
+ for (int it = firstIt; it < iteration; ++it) // previous iterations temp files
+ {
+ inFile_sents[it].close();
+ inFile_stats[it].close();
+
+ InputStream inStream_sents, inStream_stats;
+ if (compressFiles == 0) {
+ inStream_sents = new FileInputStream(tmpDirPrefix + "temp.sents.it" + it);
+ inStream_stats = new FileInputStream(tmpDirPrefix + "temp.stats.it" + it);
+ } else {
+ inStream_sents = new GZIPInputStream(new FileInputStream(tmpDirPrefix + "temp.sents.it"
+ + it + ".gz"));
+ inStream_stats = new GZIPInputStream(new FileInputStream(tmpDirPrefix + "temp.stats.it"
+ + it + ".gz"));
+ }
+
+ inFile_sents[it] = new BufferedReader(new InputStreamReader(inStream_sents, "utf8"));
+ inFile_stats[it] = new BufferedReader(new InputStreamReader(inStream_stats, "utf8"));
+ }
+
+ inFile_sentsCurrIt.close();
+ // current iteration temp files
+ if (compressFiles == 0) {
+ inStream_sentsCurrIt = new FileInputStream(tmpDirPrefix + "temp.sents.it" + iteration);
+ } else {
+ inStream_sentsCurrIt = new GZIPInputStream(new FileInputStream(tmpDirPrefix
+ + "temp.sents.it" + iteration + ".gz"));
+ }
+ inFile_sentsCurrIt = new BufferedReader(new InputStreamReader(inStream_sentsCurrIt, "utf8"));
+
+ // calculate SS for unseen candidates and write them to file
+ FileInputStream inStream_statsCurrIt_unknown = null;
+ BufferedReader inFile_statsCurrIt_unknown = null;
+
+ if (!statsCurrIt_exists && newCandidatesAdded[iteration] > 0) {
+ // create the file...
+ evalMetric.createSuffStatsFile(tmpDirPrefix + "temp.currIt.unknownCands", tmpDirPrefix
+ + "temp.currIt.unknownIndices", tmpDirPrefix + "temp.stats.unknown", sizeOfNBest);
+
+ // ...and open it
+ inStream_statsCurrIt_unknown = new FileInputStream(tmpDirPrefix + "temp.stats.unknown");
+ inFile_statsCurrIt_unknown = new BufferedReader(new InputStreamReader(
+ inStream_statsCurrIt_unknown, "utf8"));
+ }
+
+ // open mergedKnown file
+ // newly created by the big loop above
+ FileInputStream instream_statsMergedKnown = new FileInputStream(tmpDirPrefix
+ + "temp.stats.mergedKnown");
+ BufferedReader inFile_statsMergedKnown = new BufferedReader(new InputStreamReader(
+ instream_statsMergedKnown, "utf8"));
+
+ // num of features before observing new firing features from this iteration
+ numParamsOld = numParams;
+
+ for (int i = 0; i < numSentences; ++i) {
+ // reprocess candidates from previous iterations
+ for (int it = firstIt; it < iteration; ++it) {
+ for (int n = 0; n <= sizeOfNBest; ++n) {
+ sents_str = inFile_sents[it].readLine();
+ stats_str = inFile_stats[it].readLine();
+
+ if (sents_str.equals("||||||")) {
+ n = sizeOfNBest + 1;
+ } else if (!existingCandStats.containsKey(sents_str)) {
+ existingCandStats.put(sents_str, stats_str);
+ } // if unseen candidate
+ } // for (n)
+ } // for (it)
+
+ // copy relevant portion from mergedKnown to the merged file
+ String line_mergedKnown = inFile_statsMergedKnown.readLine();
+ while (!line_mergedKnown.equals("||||||")) {
+ outFile_statsMerged.println(line_mergedKnown);
+ line_mergedKnown = inFile_statsMergedKnown.readLine();
+ }
+
+ int[] stats = new int[suffStatsCount];
+
+ for (int n = 0; n <= sizeOfNBest; ++n) {
+ sents_str = inFile_sentsCurrIt.readLine();
+ feats_str = inFile_featsCurrIt.readLine();
+
+ if (sents_str.equals("||||||")) {
+ n = sizeOfNBest + 1;
+ } else if (!existingCandStats.containsKey(sents_str)) {
+
+ if (!statsCurrIt_exists) {
+ stats_str = inFile_statsCurrIt_unknown.readLine();
+
+ String[] temp_stats = stats_str.split("\\s+");
+ for (int s = 0; s < suffStatsCount; ++s) {
+ stats[s] = Integer.parseInt(temp_stats[s]);
+ }
+
+ outFile_statsCurrIt.println(stats_str);
+ } else {
+ stats_str = inFile_statsCurrIt.readLine();
+
+ String[] temp_stats = stats_str.split("\\s+");
+ for (int s = 0; s < suffStatsCount; ++s) {
+ stats[s] = Integer.parseInt(temp_stats[s]);
+ }
+ }
+
+ outFile_statsMerged.println(stats_str);
+
+ // save feats & stats
+ // System.out.println(sents_str+" "+feats_str);
+
+ feat_hash[i].put(sents_str, feats_str);
+ stats_hash[i].put(sents_str, stats_str);
+
+ featVal_str = feats_str.split("\\s+");
+
+ if (feats_str.indexOf('=') != -1) {
+ for (String featurePair : featVal_str) {
+ String[] pair = featurePair.split("=");
+ String name = pair[0];
+ Double value = Double.parseDouble(pair[1]);
+ int featId = Vocabulary.id(name);
+
+ // need to identify newly fired feats here
+ // in this case currFeatVal is not given the value
+ // of the new feat, since the corresponding weight is
+ // initialized as zero anyway
+ if (featId > numParams) {
+ ++numParams;
+ lambda.add(new Double(0));
+ }
+ }
+ }
+ existingCandStats.put(sents_str, stats_str);
+ candCount[i] += 1;
+
+ // newCandidatesAdded[iteration] += 1;
+ // moved to code above detecting new candidates
+ } else {
+ if (statsCurrIt_exists)
+ inFile_statsCurrIt.readLine();
+ else {
+ // write SS to outFile_statsCurrIt
+ stats_str = existingCandStats.get(sents_str);
+ outFile_statsCurrIt.println(stats_str);
+ }
+ }
+
+ } // for (n)
+
+ // now d = sizeUnknown_currIt[i] - 1
+
+ if (statsCurrIt_exists)
+ inFile_statsCurrIt.readLine();
+ else
+ outFile_statsCurrIt.println("||||||");
+
+ existingCandStats.clear();
+ totalCandidateCount += candCount[i];
+
+ // output sentence progress
+ if ((i + 1) % 500 == 0) {
+ print((i + 1) + "\n" + " ", 1);
+ } else if ((i + 1) % 100 == 0) {
+ print("+", 1);
+ } else if ((i + 1) % 25 == 0) {
+ print(".", 1);
+ }
+
+ } // for (i)
+
+ inFile_statsMergedKnown.close();
+ outFile_statsMerged.close();
+
+ // for testing
+ /*
+ * int total_sent = 0; for( int i=0; i<numSentences; i++ ) {
+ * System.out.println(feat_hash[i].size()+" "+candCount[i]); total_sent +=
+ * feat_hash[i].size(); feat_hash[i].clear(); }
+ * System.out.println("----------------total sent: "+total_sent); total_sent = 0; for( int
+ * i=0; i<numSentences; i++ ) { System.out.println(stats_hash[i].size()+" "+candCount[i]);
+ * total_sent += stats_hash[i].size(); stats_hash[i].clear(); }
+ * System.out.println("*****************total sent: "+total_sent);
+ */
+
+ println("", 1); // finish progress line
+
+ for (int it = firstIt; it < iteration; ++it) {
+ inFile_sents[it].close();
+ inFile_feats[it].close();
+ inFile_stats[it].close();
+ }
+
+ inFile_sentsCurrIt.close();
+ inFile_featsCurrIt.close();
+ if (statsCurrIt_exists)
+ inFile_statsCurrIt.close();
+ else
+ outFile_statsCurrIt.close();
+
+ if (compressFiles == 1 && !statsCurrIt_exists) {
+ gzipFile(tmpDirPrefix + "temp.stats.it" + iteration);
+ }
+
+ // clear temp files
+ deleteFile(tmpDirPrefix + "temp.currIt.unknownCands");
+ deleteFile(tmpDirPrefix + "temp.currIt.unknownIndices");
+ deleteFile(tmpDirPrefix + "temp.stats.unknown");
+ deleteFile(tmpDirPrefix + "temp.stats.mergedKnown");
+
+ // cleanupMemory();
+
+ println("Processed " + totalCandidateCount + " distinct candidates " + "(about "
+ + totalCandidateCount / numSentences + " per sentence):", 1);
+ for (int it = firstIt; it <= iteration; ++it) {
+ println("newCandidatesAdded[it=" + it + "] = " + newCandidatesAdded[it] + " (about "
+ + newCandidatesAdded[it] / numSentences + " per sentence)", 1);
+ }
+
+ println("", 1);
+
+ println("Number of features observed so far: " + numParams);
+ println("", 1);
+
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ // n-best list converges
+ if (newCandidatesAdded[iteration] == 0) {
+ if (!oneModificationPerIteration) {
+ println("No new candidates added in this iteration; exiting AdaGrad.", 1);
+ println("", 1);
+ println("--- AdaGrad iteration #" + iteration + " ending @ " + (new Date()) + " ---", 1);
+ println("", 1);
+ deleteFile(tmpDirPrefix + "temp.stats.merged");
+
+ if (returnBest) {
+ // note that bestLambda.size() <= lambda.size()
+ for (int p = 1; p < bestLambda.size(); ++p)
+ lambda.set(p, bestLambda.get(p));
+ // and set the rest of lambda to be 0
+ for (int p = 0; p < lambda.size() - bestLambda.size(); ++p)
+ lambda.set(p + bestLambda.size(), new Double(0));
+ }
+
+ return null; // this means that the old values should be kept by the caller
+ } else {
+ println("Note: No new candidates added in this iteration.", 1);
+ }
+ }
+
+ /************* start optimization **************/
+
+ /*
+ * for( int v=1; v<initialLambda[1].length; v++ ) System.out.print(initialLambda[1][v]+" ");
+ * System.exit(0);
+ */
+
+ Optimizer.sentNum = numSentences; // total number of training sentences
+ Optimizer.needShuffle = needShuffle;
+ Optimizer.adagradIter = adagradIter;
+ Optimizer.oraSelectMode = oraSelectMode;
+ Optimizer.predSelectMode = predSelectMode;
+ Optimizer.needAvg = needAvg;
+ // Optimizer.sentForScale = sentForScale;
+ Optimizer.scoreRatio = scoreRatio;
+ Optimizer.evalMetric = evalMetric;
+ Optimizer.normalizationOptions = normalizationOptions;
+ Optimizer.needScale = needScale;
+ Optimizer.regularization = regularization;
+ Optimizer.batchSize = batchSize;
+ Optimizer.eta = eta;
+ Optimizer.lam = lam;
+
+ // if need to use bleu stats history
+ if (iteration == 1) {
+ if (evalMetric.get_metricName().equals("BLEU") && usePseudoBleu) {
+ Optimizer.initBleuHistory(numSentences, evalMetric.get_suffStatsCount());
+ Optimizer.usePseudoBleu = usePseudoBleu;
+ Optimizer.R = R;
+ }
+ if (evalMetric.get_metricName().equals("TER-BLEU") && usePseudoBleu) {
+ Optimizer.initBleuHistory(numSentences, evalMetric.get_suffStatsCount() - 2); // Stats
+ // count of
+ // TER=2
+ Optimizer.usePseudoBleu = usePseudoBleu;
+ Optimizer.R = R;
+ }
+ }
+
+ Vector<String> output = new Vector<String>();
+
+ // note: initialLambda[] has length = numParamsOld
+ // augmented with new feature weights, initial values are 0
+ double[] initialLambdaNew = new double[1 + numParams];
+ System.arraycopy(initialLambda, 1, initialLambdaNew, 1, numParamsOld);
+
+ // finalLambda[] has length = numParams (considering new features)
+ double[] finalLambda = new double[1 + numParams];
+
+ Optimizer opt = new Optimizer(output, isOptimizable, initialLambdaNew, feat_hash, stats_hash);
+ finalLambda = opt.runOptimizer();
+
+ if (returnBest) {
+ double metricScore = opt.getMetricScore();
+ if (!evalMetric.getToBeMinimized()) {
+ if (metricScore > prevMetricScore) {
+ prevMetricScore = metricScore;
+ for (int p = 1; p < bestLambda.size(); ++p)
+ bestLambda.set(p, finalLambda[p]);
+ if (1 + numParams > bestLambda.size()) {
+ for (int p = bestLambda.size(); p <= numParams; ++p)
+ bestLambda.add(p, finalLambda[p]);
+ }
+ }
+ } else {
+ if (metricScore < prevMetricScore) {
+ prevMetricScore = metricScore;
+ for (int p = 1; p < bestLambda.size(); ++p)
+ bestLambda.set(p, finalLambda[p]);
+ if (1 + numParams > bestLambda.size()) {
+ for (int p = bestLambda.size(); p <= numParams; ++p)
+ bestLambda.add(p, finalLambda[p]);
+ }
+ }
+ }
+ }
+
+ // System.out.println(finalLambda.length);
+ // for( int i=0; i<finalLambda.length-1; i++ )
+ // System.out.println(finalLambda[i+1]);
+
+ /************* end optimization **************/
+
+ for (int i = 0; i < output.size(); i++)
+ println(output.get(i));
+
+ // check if any parameter has been updated
+ boolean anyParamChanged = false;
+ boolean anyParamChangedSignificantly = false;
+
+ for (int c = 1; c <= numParams; ++c) {
+ if (finalLambda[c] != lambda.get(c)) {
+ anyParamChanged = true;
+ }
+ if (Math.abs(finalLambda[c] - lambda.get(c)) > stopSigValue) {
+ anyParamChangedSignificantly = true;
+ }
+ }
+
+ // System.arraycopy(finalLambda,1,lambda,1,numParams);
+
+ println("--- AdaGrad iteration #" + iteration + " ending @ " + (new Date()) + " ---", 1);
+ println("", 1);
+
+ if (!anyParamChanged) {
+ println("No parameter value changed in this iteration; exiting AdaGrad.", 1);
+ println("", 1);
+ break; // exit for (iteration) loop preemptively
+ }
+
+ // was an early stopping criterion satisfied?
+ boolean critSatisfied = false;
+ if (!anyParamChangedSignificantly && stopSigValue >= 0) {
+ println("Note: No parameter value changed significantly " + "(i.e. by more than "
+ + stopSigValue + ") in this iteration.", 1);
+ critSatisfied = true;
+ }
+
+ if (critSatisfied) {
+ ++earlyStop;
+ println("", 1);
+ } else {
+ earlyStop = 0;
+ }
+
+ // if min number of iterations executed, investigate if early exit should happen
+ if (iteration >= minIts && earlyStop >= stopMinIts) {
+ println("Some early stopping criteria has been observed " + "in " + stopMinIts
+ + " consecutive iterations; exiting AdaGrad.", 1);
+ println("", 1);
+
+ if (returnBest) {
+ for (int f = 1; f <= bestLambda.size() - 1; ++f)
+ lambda.set(f, bestLambda.get(f));
+ } else {
+ for (int f = 1; f <= numParams; ++f)
+ lambda.set(f, finalLambda[f]);
+ }
+
+ break; // exit for (iteration) loop preemptively
+ }
+
+ // if max number of iterations executed, exit
+ if (iteration >= maxIts) {
+ println("Maximum number of AdaGrad iterations reached; exiting AdaGrad.", 1);
+ println("", 1);
+
+ if (returnBest) {
+ for (int f = 1; f <= bestLambda.size() - 1; ++f)
+ lambda.set(f, bestLambda.get(f));
+ } else {
+ for (int f = 1; f <= numParams; ++f)
+ lambda.set(f, finalLambda[f]);
+ }
+
+ break; // exit for (iteration) loop
+ }
+
+ // use the new wt vector to decode the next iteration
+ // (interpolation with previous wt vector)
+ double interCoef = 1.0; // no interpolation for now
+ for (int i = 1; i <= numParams; i++)
+ lambda.set(i, interCoef * finalLambda[i] + (1 - interCoef) * lambda.get(i).doubleValue());
+
+ println("Next iteration will decode with lambda: " + lambdaToString(lambda), 1);
+ println("", 1);
+
+ // printMemoryUsage();
+ for (int i = 0; i < numSentences; ++i) {
+ suffStats_array[i].clear();
+ }
+ // cleanupMemory();
+ // println("",2);
+
+ retA[2] = 0; // i.e. this should NOT be the last iteration
+ done = true;
+
+ } // while (!done) // NOTE: this "loop" will only be carried out once
+
+ // delete .temp.stats.merged file, since it is not needed in the next
+ // iteration (it will be recreated from scratch)
+ deleteFile(tmpDirPrefix + "temp.stats.merged");
+
+ retA[0] = FINAL_score;
+ retA[1] = earlyStop;
+ return retA;
+
+ } // run_single_iteration
+
+ private String lambdaToString(ArrayList<Double> lambdaA) {
+ String retStr = "{";
+ int featToPrint = numParams > 15 ? 15 : numParams;
+ // print at most the first 15 features
+
+ retStr += "(listing the first " + featToPrint + " lambdas)";
+ for (int c = 1; c <= featToPrint - 1; ++c) {
+ retStr += "" + String.format("%.4f", lambdaA.get(c).doubleValue()) + ", ";
+ }
+ retStr += "" + String.format("%.4f", lambdaA.get(numParams).doubleValue()) + "}";
+
+ return retStr;
+ }
+
+ private String[] run_decoder(int iteration) {
+ String[] retSA = new String[2];
+
+ // retsa saves the output file name(nbest-file)
+ // and the decoder type
+
+ // [0] name of file to be processed
+ // [1] indicates how the output file was obtained:
+ // 1: external decoder
+ // 2: fake decoder
+ // 3: internal decoder
+
+ // use fake decoder
+ if (fakeFileNameTemplate != null
+ && fileExists(fakeFileNamePrefix + iteration + fakeFileNameSuffix)) {
+ String fakeFileName = fakeFileNamePrefix + iteration + fakeFileNameSuffix;
+ println("Not running decoder; using " + fakeFileName + " instead.", 1);
+ /*
+ * if (fakeFileName.endsWith(".gz")) { copyFile(fakeFileName,decoderOutFileName+".gz");
+ * gunzipFile(decoderOutFileName+".gz"); } else { copyFile(fakeFileName,decoderOutFileName); }
+ */
+ retSA[0] = fakeFileName;
+ retSA[1] = "2";
+
+ } else {
+ println("Running external decoder...", 1);
+
+ try {
+ ArrayList<String> cmd = new ArrayList<String>();
+ cmd.add(decoderCommandFileName);
+
+ if (passIterationToDecoder)
+ cmd.add(Integer.toString(iteration));
+
+ ProcessBuilder pb = new ProcessBuilder(cmd);
+ // this merges the error and output streams of the subprocess
+ pb.redirectErrorStream(true);
+ Process p = pb.start();
+
+ // capture the sub-command's output
+ new StreamGobbler(p.getInputStream(), decVerbosity).start();
+
+ int decStatus = p.waitFor();
+ if (decStatus != validDecoderExitValue) {
+ throw new RuntimeException("Call to decoder returned " + decStatus + "; was expecting "
+ + validDecoderExitValue + ".");
+ }
+ } catch (IOException | InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+
+ retSA[0] = decoderOutFileName;
+ retSA[1] = "1";
+
+ }
+
+ return retSA;
+ }
+
+ private void produceTempFiles(String nbestFileName, int iteration) {
+ try {
+ String sentsFileName = tmpDirPrefix + "temp.sents.it" + iteration;
+ String featsFileName = tmpDirPrefix + "temp.feats.it" + iteration;
+
+ FileOutputStream outStream_sents = new FileOutputStream(sentsFileName, false);
+ OutputStreamWriter outStreamWriter_sents = new OutputStreamWriter(outStream_sents, "utf8");
+ BufferedWriter outFile_sents = new BufferedWriter(outStreamWriter_sents);
+
+ PrintWriter outFile_feats = new PrintWriter(featsFileName);
+
+ InputStream inStream_nbest = null;
+ if (nbestFileName.endsWith(".gz")) {
+ inStream_nbest = new GZIPInputStream(new FileInputStream(nbestFileName));
+ } else {
+ inStream_nbest = new FileInputStream(nbestFileName);
+ }
+ BufferedReader inFile_nbest = new BufferedReader(
+ new InputStreamReader(inStream_nbest, "utf8"));
+
+ String line; // , prevLine;
+ String candidate_str = "";
+ String feats_str = "";
+
+ int i = 0;
+ int n = 0;
+ line = inFile_nbest.readLine();
+
+ while (line != null) {
+
+ /*
+ * line format:
+ *
+ * i ||| words of candidate translation . ||| feat-1_val feat-2_val ... feat-numParams_val
+ * .*
+ */
+
+ // in a well formed file, we'd find the nth candidate for the ith sentence
+
+ int read_i = Integer.parseInt((line.substring(0, line.indexOf("|||"))).trim());
+
+ if (read_i != i) {
+ writeLine("||||||", outFile_sents);
+ outFile_feats.println("||||||");
+ n = 0;
+ ++i;
+ }
+
+ line = (line.substring(line.indexOf("|||") + 3)).trim(); // get rid of initial text
+
+ candidate_str = (line.substring(0, line.indexOf("|||"))).trim();
+ feats_str = (line.substring(line.indexOf("|||") + 3)).trim();
+ // get rid of candidate string
+
+ int junk_i = feats_str.indexOf("|||");
+ if (junk_i >= 0) {
+ feats_str = (feats_str.substring(0, junk_i)).trim();
+ }
+
+ writeLine(normalize(candidate_str, textNormMethod), outFile_sents);
+ outFile_feats.println(feats_str);
+
+ ++n;
+ if (n == sizeOfNBest) {
+ writeLine("||||||", outFile_sents);
+ outFile_feats.println("||||||");
+ n = 0;
+ ++i;
+ }
+
+ line = inFile_nbest.readLine();
+ }
+
+ if (i != numSentences) { // last sentence had too few candidates
+ writeLine("||||||", outFile_sents);
+ outFile_feats.println("||||||");
+ }
+
+ inFile_nbest.close();
+ outFile_sents.close();
+ outFile_feats.close();
+
+ if (compressFiles == 1) {
+ gzipFile(sentsFileName);
+ gzipFile(featsFileName);
+ }
+
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ }
+
+ private void createConfigFile(ArrayList<Double> params, String cfgFileName,
+ String templateFileName) {
+ try {
+ // i.e. create cfgFileName, which is similar to templateFileName, but with
+ // params[] as parameter values
+
+ BufferedReader inFile = new BufferedReader(new FileReader(templateFileName));
+ PrintWriter outFile = new PrintWriter(cfgFileName);
+
+ BufferedReader inFeatDefFile = null;
+ PrintWriter outFeatDefFile = null;
+ int origFeatNum = 0; // feat num in the template file
+
+ String line = inFile.readLine();
+ while (line != null) {
+ int c_match = -1;
+ for (int c = 1; c <= numParams; ++c) {
+ if (line.startsWith(Vocabulary.word(c) + " ")) {
+ c_match = c;
+ ++origFeatNum;
+ break;
+ }
+ }
+
+ if (c_match == -1) {
+ outFile.println(line);
+ } else {
+ if (Math.abs(params.get(c_match).doubleValue()) > 1e-20)
+ outFile.println(Vocabulary.word(c_match) + " " + params.get(c_match));
+ }
+
+ line = inFile.readLine();
+ }
+
+ // now append weights of new features
+ for (int c = origFeatNum + 1; c <= numParams; ++c) {
+ if (Math.abs(params.get(c).doubleValue()) > 1e-20)
+ outFile.println(Vocabulary.word(c) + " " + params.get(c));
+ }
+
+ inFile.close();
+ outFile.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void processParamFile() {
+ // process parameter file
+ Scanner inFile_init = null;
+ try {
+ inFile_init = new Scanner(new FileReader(paramsFileName));
+ } catch (FileNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+
+ String dummy = "";
+
+ // initialize lambda[] and other related arrays
+ for (int c = 1; c <= numParams; ++c) {
+ // skip parameter name
+ while (!dummy.equals("|||")) {
+ dummy = inFile_init.next();
+ }
+
+ // read default value
+ lambda.set(c, inFile_init.nextDouble());
+ defaultLambda[c] = lambda.get(c).doubleValue();
+
+ // read isOptimizable
+ dummy = inFile_init.next();
+ if (dummy.equals("Opt")) {
+ isOptimizable[c] = true;
+ } else if (dummy.equals("Fix")) {
+ isOptimizable[c] = false;
+ } else {
+ throw new RuntimeException("Unknown isOptimizable string " + dummy + " (must be either Opt or Fix)");
+ }
+
+ if (!isOptimizable[c]) { // skip next two values
+ dummy = inFile_init.next();
+ dummy = inFile_init.next();
+ dummy = inFile_init.next();
+ dummy = inFile_init.next();
+ } else {
+ // the next two values are not used, only to be consistent with ZMERT's params file format
+ dummy = inFile_init.next();
+ dummy = inFile_init.next();
+ // set minRandValue[c] and maxRandValue[c] (range for random values)
+ dummy = inFile_init.next();
+ if (dummy.equals("-Inf") || dummy.equals("+Inf")) {
+ throw new RuntimeException("minRandValue[" + c + "] cannot be -Inf or +Inf!");
+ } else {
+ minRandValue[c] = Double.parseDouble(dummy);
+ }
+
+ dummy = inFile_init.next();
+ if (dummy.equals("-Inf") || dummy.equals("+Inf")) {
+ throw new RuntimeException("maxRandValue[" + c + "] cannot be -Inf or +Inf!");
+ } else {
+ maxRandValue[c] = Double.parseDouble(dummy);
+ }
+
+ // check for illogical values
+ if (minRandValue[c] > maxRandValue[c]) {
+ throw new RuntimeException("minRandValue[" + c + "]=" + minRandValue[c] + " > " + maxRandValue[c]
+ + "=maxRandValue[" + c + "]!");
+ }
+
+ // check for odd values
+ if (minRandValue[c] == maxRandValue[c]) {
+ println("Warning: lambda[" + c + "] has " + "minRandValue = maxRandValue = "
+ + minRandValue[c] + ".", 1);
+ }
+ } // if (!isOptimizable[c])
+
+ /*
+ * precision[c] = inFile_init.nextDouble(); if (precision[c] < 0) { println("precision[" + c +
+ * "]=" + precision[c] + " < 0! Must be non-negative."); System.exit(21); }
+ */
+
+ }
+
+ // set normalizationOptions[]
+ String origLine = "";
+ while (origLine != null && origLine.length() == 0) {
+ origLine = inFile_init.nextLine();
+ }
+
+ // How should a lambda[] vector be normalized (before decoding)?
+ // nO[0] = 0: no normalization
+ // nO[0] = 1: scale so that parameter nO[2] has absolute value nO[1]
+ // nO[0] = 2: scale so that the maximum absolute value is nO[1]
+ // nO[0] = 3: scale so that the minimum absolute value is nO[1]
+ // nO[0] = 4: scale so that the L-nO[1] norm equals nO[2]
+
+ // normalization = none
+ // normalization = absval 1 lm
+ // normalization = maxabsval 1
+ // normalization = minabsval 1
+ // normalization = LNorm 2 1
+
+ dummy = (origLine.substring(origLine.indexOf("=") + 1)).trim();
+ String[] dummyA = dummy.split("\\s+");
+
+ if (dummyA[0].equals("none")) {
+ normalizationOptions[0] = 0;
+ } else if (dummyA[0].equals("absval")) {
+ normalizationOptions[0] = 1;
+ normalizationOptions[1] = Double.parseDouble(dummyA[1]);
+ String pName = dummyA[2];
+ for (int i = 3; i < dummyA.length; ++i) { // in case parameter name has multiple words
+ pName = pName + " " + dummyA[i];
+ }
+ normalizationOptions[2] = Vocabulary.id(pName);
+
+ if (normalizationOptions[1] <= 0) {
+ throw new RuntimeException("Value for the absval normalization method must be positive.");
+ }
+ if (normalizationOptions[2] == 0) {
+ throw new RuntimeException("Unrecognized feature name " + normalizationOptions[2]
+ + " for absval normalization method.");
+ }
+ } else if (dummyA[0].equals("maxabsval")) {
+ normalizationOptions[0] = 2;
+ normalizationOptions[1] = Double.parseDouble(dummyA[1]);
+ if (normalizationOptions[1] <= 0) {
+ throw new RuntimeException("Value for the maxabsval normalization method must be positive.");
+ }
+ } else if (dummyA[0].equals("minabsval")) {
+ normalizationOptions[0] = 3;
+ normalizationOptions[1] = Double.parseDouble(dummyA[1]);
+ if (normalizationOptions[1] <= 0) {
+ throw new RuntimeException("Value for the minabsval normalization method must be positive.");
+ }
+ } else if (dummyA[0].equals("LNorm")) {
+ normalizationOptions[0] = 4;
+ normalizationOptions[1] = Double.parseDouble(dummyA[1]);
+ normalizationOptions[2] = Double.parseDouble(dummyA[2]);
+ if (normalizationOptions[1] <= 0 || normalizationOptions[2] <= 0) {
+ throw new RuntimeException("Both values for the LNorm normalization method must be positive.");
+ }
+ } else {
+ throw new RuntimeException("Unrecognized normalization method " + dummyA[0] + "; "
+ + "must be one of none, absval, maxabsval, and LNorm.");
+ } // if (dummyA[0])
+
+ inFile_init.close();
+ } // processParamFile()
+
+ private void processDocInfo() {
+ // sets numDocuments and docOfSentence[]
+ docOfSentence = new int[numSentences];
+
+ if (docInfoFileName == null) {
+ for (int i = 0; i < numSentences; ++i)
+ docOfSentence[i] = 0;
+ numDocuments = 1;
+ } else {
+
+ try {
+
+ // 4 possible formats:
+ // 1) List of numbers, one per document, indicating # sentences in each document.
+ // 2) List of "docName size" pairs, one per document, indicating name of document and #
+ // sentences.
+ // 3) List of docName's, one per sentence, indicating which doument each sentence belongs
+ // to.
+ // 4) List of docName_number's, one per sentence, indicating which doument each sentence
+ // belongs to,
+ // and its order in that document. (can also use '-' instead of '_')
+
+ int docInfoSize = countNonEmptyLines(docInfoFileName);
+
+ if (docInfoSize < numSentences) { // format #1 or #2
+ numDocuments = docInfoSize;
+ int i = 0;
+
+ BufferedReader inFile = new BufferedReader(new FileReader(docInfoFileName));
+ String line = inFile.readLine();
+ boolean format1 = (!(line.contains(" ")));
+
+ for (int doc = 0; doc < numDocuments; ++doc) {
+
+ if (doc != 0)
+ line = inFile.readLine();
+
+ int docSize = 0;
+ if (format1) {
+ docSize = Integer.parseInt(line);
+ } else {
+ docSize = Integer.parseInt(line.split("\\s+")[1]);
+ }
+
+ for (int i2 = 1; i2 <= docSize; ++i2) {
+ docOfSentence[i] = doc;
+ ++i;
+ }
+
+ }
+
+ // now i == numSentences
+
+ inFile.close();
+
+ } else if (docInfoSize == numSentences) { // format #3 or #4
+
+ boolean format3 = false;
+
+ HashSet<String> seenStrings = new HashSet<String>();
+ BufferedReader inFile = new BufferedReader(new FileReader(docInfoFileName));
+ for (int i = 0; i < numSentences; ++i) {
+ // set format3 = true if a duplicate is found
+ String line = inFile.readLine();
+ if (seenStrings.contains(line))
+ format3 = true;
+ seenStrings.add(line);
+ }
+
+ inFile.close();
+
+ HashSet<String> seenDocNames = new HashSet<String>();
+ HashMap<String, Integer> docOrder = new HashMap<String, Integer>();
+ // maps a document name to the order (0-indexed) in which it was seen
+
+ inFile = new BufferedReader(new FileReader(docInfoFileName));
+ for (int i = 0; i < numSentences; ++i) {
+ String line = inFile.readLine();
+
+ String docName = "";
+ if (format3) {
+ docName = line;
+ } else {
+ int sep_i = Math.max(line.lastIndexOf('_'), line.lastIndexOf('-'));
+ docName = line.substring(0, sep_i);
+ }
+
+ if (!seenDocNames.contains(docName)) {
+ seenDocNames.add(docName);
+ docOrder.put(docName, seenDocNames.size() - 1);
+ }
+
+ int docOrder_i = docOrder.get(docName);
+
+ docOfSentence[i] = docOrder_i;
+
+ }
+
+ inFile.close();
+
+ numDocuments = seenDocNames.size();
+
+ } else { // badly formatted
+
+ }
+
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ }
+
+ private boolean copyFile(String origFileName, String newFileName) {
+ try {
+ File inputFile = new File(origFileName);
+ File outputFile = new File(newFileName);
+
+ InputStream in = new FileInputStream(inputFile);
+ OutputStream out = new FileOutputStream(outputFile);
+
+ byte[] buffer = new byte[1024];
+ int len;
+ while ((len = in.read(buffer)) > 0) {
+ out.write(buffer, 0, len);
+ }
+ in.close();
+ out.close();
+
+ /*
+ * InputStream inStream = new FileInputStream(new File(origFileName)); BufferedReader inFile =
+ * new BufferedReader(new InputStreamReader(inStream, "utf8"));
+ *
+ * FileOutputStream outStream = new FileOutputStream(newFileName, false); OutputStreamWriter
+ * outStreamWriter = new OutputStreamWriter(outStream, "utf8"); BufferedWriter outFile = new
+ * BufferedWriter(outStreamWriter);
+ *
+ * String line; while(inFile.ready()) { line = inFile.readLine(); writeLine(line, outFile); }
+ *
+ * inFile.close(); outFile.close();
+ */
+ return true;
+ } catch (IOException e) {
+ LOG.error(e.getMessage(), e);
+ return false;
+ }
+ }
+
+ private void renameFile(String origFileName, String newFileName) {
+ if (fileExists(origFileName)) {
+ deleteFile(newFileName);
+ File oldFile = new File(origFileName);
+ File newFile = new File(newFileName);
+ if (!oldFile.renameTo(newFile)) {
+ println("Warning: attempt to rename " + origFileName + " to " + newFileName
+ + " was unsuccessful!", 1);
+ }
+ } else {
+ println("Warning: file " + origFileName + " does not exist! (in AdaGradCore.renameFile)", 1);
+ }
+ }
+
+ private void deleteFile(String fileName) {
+ if (fileExists(fileName)) {
+ File fd = new File(fileName);
+ if (!fd.delete()) {
+ println("Warning: attempt to delete " + fileName + " was unsuccessful!", 1);
+ }
+ }
+ }
+
+ private void writeLine(String line, BufferedWriter writer) throws IOException {
+ writer.write(line, 0, line.length());
+ writer.newLine();
+ writer.flush();
+ }
+
+ // need to re-write to handle different forms of lambda
+ public void finish() {
+ if (myDecoder != null) {
+ myDecoder.cleanUp();
+ }
+
+ // create config file with final values
+ createConfigFile(lambda, decoderConfigFileName + ".AdaGrad.final", decoderConfigFileName
+ + ".AdaGrad.orig");
+
+ // delete current decoder config file and decoder output
+ deleteFile(decoderConfigFileName);
+ deleteFile(decoderOutFileName);
+
+ // restore original name for config file (name was changed
+ // in initialize() so it doesn't get overwritten)
+ renameFile(decoderConfigFileName + ".AdaGrad.orig", decoderConfigFileName);
+
+ if (finalLambdaFileName != null) {
+ try {
+ PrintWriter outFile_lambdas = new PrintWriter(finalLambdaFileName);
+ for (int c = 1; c <= numParams; ++c) {
+ outFile_lambdas.println(Vocabulary.word(c) + " ||| " + lambda.get(c).doubleValue());
+ }
+ outFile_lambdas.close();
+
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ }
+
+ private String[] cfgFileToArgsArray(String fileName) {
+ checkFile(fileName);
+
+ Vector<String> argsVector = new Vector<String>();
+
+ BufferedReader inFile = null;
+ try {
+ inFile = new BufferedReader(new FileReader(fileName));
+ String line, origLine;
+ do {
+ line = inFile.readLine();
+ origLine = line; // for error reporting purposes
+
+ if (line != null && line.length() > 0 && line.charAt(0) != '#') {
+
+ if (line.indexOf("#") != -1) { // discard comment
+ line = line.substring(0, line.indexOf("#"));
+ }
+
+ line = line.trim();
+
+ // now line should look like "-xxx XXX"
+
+ /*
+ * OBSOLETE MODIFICATION //SPECIAL HANDLING FOR AdaGrad CLASSIFIER PARAMETERS String[]
+ * paramA = line.split("\\s+");
+ *
+ * if( paramA[0].equals("-classifierParams") ) { String classifierParam = ""; for(int p=1;
+ * p<=paramA.length-1; p++) classifierParam += paramA[p]+" ";
+ *
+ * if(paramA.length>=2) { String[] tmpParamA = new String[2]; tmpParamA[0] = paramA[0];
+ * tmpParamA[1] = classifierParam; paramA = tmpParamA; } else {
+ * println("Malformed line in config file:"); println(origLine); System.exit(70); } }//END
+ * MODIFICATION
+ */
+
+ // cmu modification(from meteor for zmert)
+ // Parse args
+ ArrayList<String> argList = new ArrayList<String>();
+ StringBuilder arg = new StringBuilder();
+ boolean quoted = false;
+ for (int i = 0; i < line.length(); i++) {
+ if (Character.isWhitespace(line.charAt(i))) {
+ if (quoted)
+ arg.append(line.charAt(i));
+ else if (arg.length() > 0) {
+ argList.add(arg.toString());
+ arg = new StringBuilder();
+ }
+ } else if (line.charAt(i) == '\'') {
+ if (quoted) {
+ argList.add(arg.toString());
+ arg = new StringBuilder();
+ }
+ quoted = !quoted;
+ } else
+ arg.append(line.charAt(i));
+ }
+ if (arg.length() > 0)
+ argList.add(arg.toString());
+ // Create paramA
+ String[] paramA = new String[argList.size()];
+ for (int i = 0; i < paramA.length; paramA[i] = argList.get(i++))
+ ;
+ // END CMU MODIFICATION
+
+ if (paramA.length == 2 && paramA[0].charAt(0) == '-') {
+ argsVector.add(paramA[0]);
+ argsVector.add(paramA[1]);
+ } else if (paramA.length > 2 && (paramA[0].equals("-m") || paramA[0].equals("-docSet"))) {
+ // -m (metricName), -docSet are allowed to have extra optinos
+ for (int opt = 0; opt < paramA.length; ++opt) {
+ argsVector.add(paramA[opt]);
+ }
+ } else {
+ String msg = "Malformed line in config file:" + origLine;
+ throw new RuntimeException(msg);
+ }
+
+ }
+ } while (line != null);
+
+ inFile.close();
+ } catch (FileNotFoundException e) {
+ println("AdaGrad configuration file " + fileName + " was not found!");
+ throw new RuntimeException(e);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ String[] argsArray = new String[argsVector.size()];
+
+ for (int i = 0; i < argsVector.size(); ++i) {
+ argsArray[i] = argsVector.elementAt(i);
+ }
+
+ return argsArray;
+ }
+
+ private void processArgsArray(String[] args) {
+ processArgsArray(args, true);
+ }
+
+ private void processArgsArray(String[] args, boolean firstTime) {
+ /* set default values */
+ // Relevant files
+ dirPrefix = null;
+ sourceFileName = null;
+ refFileName = "reference.txt";
+ refsPerSen = 1;
+ textNormMethod = 1;
+ paramsFileName = "params.txt";
+ docInfoFileName = null;
+ finalLambdaFileName = null;
+ // MERT specs
+ metricName = "BLEU";
+ metricName_display = metricName;
+ metricOptions = new String[2];
+ metricOptions[0] = "4";
+ metricOptions[1] = "closest";
+ docSubsetInfo = new int[7];
+ docSubsetInfo[0] = 0;
+ maxMERTIterations = 20;
+ prevMERTIterations = 20;
+ minMERTIterations = 5;
+ stopMinIts = 3;
+ stopSigValue = -1;
+ //
+ // /* possibly other early stopping criteria here */
+ //
+ numOptThreads = 1;
+ saveInterFiles = 3;
+ compressFiles = 0;
+ oneModificationPerIteration = false;
+ randInit = false;
+ seed = System.currentTimeMillis();
+ // useDisk = 2;
+ // Decoder specs
+ decoderCommandFileName = null;
+ passIterationToDecoder = false;
+ decoderOutFileName = "output.nbest";
+ validDecoderExitValue = 0;
+ decoderConfigFileName = "dec_cfg.txt";
+ sizeOfNBest = 100;
+ fakeFileNameTemplate = null;
+ fakeFileNamePrefix = null;
+ fakeFileNameSuffix = null;
+ // Output specs
+ verbosity = 1;
+ decVerbosity = 0;
+
+ int i = 0;
+
+ while (i < args.length) {
+ String option = args[i];
+ // Relevant files
+ if (option.equals("-dir")) {
+ dirPrefix = args[i + 1];
+ } else if (option.equals("-s")) {
+ sourceFileName = args[i + 1];
+ } else if (option.equals("-r")) {
+ refFileName = args[i + 1];
+ } else if (option.equals("-rps")) {
+ refsPerSen = Integer.parseInt(args[i + 1]);
+ if (refsPerSen < 1) {
+ throw new RuntimeException("refsPerSen must be positive.");
+ }
+ } else if (option.equals("-txtNrm")) {
+ textNormMethod = Integer.parseInt(args[i + 1]);
+ if (textNormMethod < 0 || textNormMethod > 4) {
+ throw new RuntimeException("textNormMethod should be between 0 and 4");
+ }
+ } else if (option.equals("-p")) {
+ paramsFileName = args[i + 1];
+ } else if (option.equals("-docInfo")) {
+ docInfoFileName = args[i + 1];
+ } else if (option.equals("-fin")) {
+ finalLambdaFileName = args[i + 1];
+ // MERT specs
+ } else if (option.equals("-m")) {
+ metricName = args[i + 1];
+ metricName_display = metricName;
+ if (EvaluationMetric.knownMetricName(metricName)) {
+ int optionCount = EvaluationMetric.metricOptionCount(metricName);
+ metricOptions = new String[optionCount];
+ for (int opt = 0; opt < optionCount; ++opt) {
+ metricOptions[opt] = args[i + opt + 2];
+ }
+ i += optionCount;
+ } else {
+ throw new RuntimeException("Unknown metric name " + metricName + ".");
+ }
+ } else if (option.equals("-docSet")) {
+ String method = args[i + 1];
+
+ if (method.equals("all")) {
+ docSubsetInfo[0] = 0;
+ i += 0;
+ } else if (method.equals("bottom")) {
+ String a = args[i + 2];
+ if (a.endsWith("d")) {
+ docSubsetInfo[0] = 1;
+ a = a.substring(0, a.indexOf("d"));
+ } else {
+ docSubsetInfo[0] = 2;
+ a = a.substring(0, a.indexOf("%"));
+ }
+ docSubsetInfo[5] = Integer.parseInt(a);
+ i += 1;
+ } else if (method.equals("top")) {
+ String a = args[i + 2];
+ if (a.endsWith("d")) {
+ docSubsetInfo[0] = 3;
+ a = a.substring(0, a.indexOf("d"));
+ } else {
+ docSubsetInfo[0] = 4;
+ a = a.substring(0, a.indexOf("%"));
+ }
+ docSubsetInfo[5] = Integer.parseInt(a);
+ i += 1;
+ } else if (method.equals("window")) {
+ String a1 = args[i + 2];
+ a1 = a1.substring(0, a1.indexOf("d")); // size of window
+ String a2 = args[i + 4];
+ if (a2.indexOf("p") > 0) {
+ docSubsetInfo[0] = 5;
+ a2 = a2.substring(0, a2.indexOf("p"));
+ } else {
+ docSubsetInfo[0] = 6;
+ a2 = a2.substring(0, a2.indexOf("r"));
+ }
+ docSubsetInfo[5] = Integer.parseInt(a1);
+ docSubsetInfo[6] = Integer.parseInt(a2);
+ i += 3;
+ } else {
+ throw new RuntimeException("Unknown docSet method " + method + ".");
+ }
+ } else if (option.equals("-maxIt")) {
+ maxMERTIterations = Integer.parseInt(args[i + 1]);
+ if (maxMERTIterations < 1) {
+ throw new RuntimeException("maxIt must be positive.");
+ }
+ } else if (option.equals("-minIt")) {
+ minMERTIterations = Integer.parseInt(args[i + 1]);
+ if (minMERTIterations < 1) {
+ throw new RuntimeException("minIt must be positive.");
+ }
+ } else if (option.equals("-prevIt")) {
+ prevMERTIterations = Integer.parseInt(args[i + 1]);
+ if (prevMERTIterations < 0) {
+ throw new RuntimeException("prevIt must be non-negative.");
+ }
+ } else if (option.equals("-stopIt")) {
+ stopMinIts = Integer.parseInt(args[i + 1]);
+ if (stopMinIts < 1) {
+ throw new RuntimeException("stopIts must be positive.");
+ }
+ } else if (option.equals("-stopSig")) {
+ stopSigValue = Double.parseDouble(args[i + 1]);
+ }
+ //
+ // /* possibly other early stopping criteria here */
+ //
+ else if (option.equals("-thrCnt")) {
+ numOptThreads = Integer.parseInt(args[i + 1]);
+ if (numOptThreads < 1) {
+ throw new RuntimeException("threadCount must be positive.");
+ }
+ } else if (option.equals("-save")) {
+ saveInterFiles = Integer.parseInt(args[i + 1]);
+ if (saveInterFiles < 0 || saveInterFiles > 3) {
+ throw new RuntimeException("save should be between 0 and 3");
+ }
+ } else if (option.equals("-compress")) {
+ compressFiles = Integer.parseInt(args[i + 1]);
+ if (compressFiles < 0 || compressFiles > 1) {
+ throw new RuntimeException("compressFiles should be either 0 or 1");
+ }
+ } else if (option.equals("-opi")) {
+ int opi = Integer.parseInt(args[i + 1]);
+ if (opi == 1) {
+ oneModificationPerIteration = true;
+ } else if (opi == 0) {
+ oneModificationPerIteration = false;
+ } else {
+ throw new RuntimeException("oncePerIt must be either 0 or 1.");
+ }
+ } else if (option.equals("-rand")) {
+ int rand = Integer.parseInt(args[i + 1]);
+ if (ran
<TRUNCATED>
[58/60] incubator-joshua git commit: Merge branch
'maven-multi-module' of https://github.com/logogin/incubator-joshua into
maven-multi-module
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/main/java/org/apache/joshua/decoder/ff/OOVPenalty.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/ff/OOVPenalty.java
index 0000000,5278172..92ee740
mode 000000,100644..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/OOVPenalty.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/OOVPenalty.java
@@@ -1,0 -1,106 +1,108 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.decoder.ff;
+
+ import java.util.ArrayList;
+ import java.util.HashMap;
+ import java.util.List;
+
+ import org.apache.joshua.decoder.JoshuaConfiguration;
+ import org.apache.joshua.decoder.JoshuaConfiguration.OOVItem;
+ import org.apache.joshua.decoder.ff.state_maintenance.DPState;
++import org.apache.joshua.decoder.ff.tm.OwnerId;
++import org.apache.joshua.decoder.ff.tm.OwnerMap;
+ import org.apache.joshua.decoder.ff.tm.Rule;
+ import org.apache.joshua.decoder.hypergraph.HGNode;
+ import org.apache.joshua.decoder.segment_file.Sentence;
+ import org.apache.joshua.corpus.Vocabulary;
+ import org.apache.joshua.decoder.chart_parser.SourcePath;
+
+ /**
+ * This feature is fired when an out-of-vocabulary word (with respect to the translation model) is
+ * entered into the chart. OOVs work in the following manner: for each word in the input that is OOV
+ * with respect to the translation model, we create a rule that pushes that word through
+ * untranslated (the suffix "_OOV" can optionally be appended according to the runtime parameter
+ * "mark-oovs") . These rules are all stored in a grammar whose owner is "oov". The OOV feature
+ * function template then fires the "OOVPenalty" feature whenever it is asked to score an OOV rule.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ */
+ public class OOVPenalty extends StatelessFF {
- private final int ownerID;
++ private final OwnerId ownerID;
+
+ /* The default value returned for OOVs. Can be overridden with -oov-list */
+ private final float defaultValue = -100f;
+ private final HashMap<Integer,Float> oovWeights;
+
+ public OOVPenalty(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, "OOVPenalty", args, config);
+
- ownerID = Vocabulary.id("oov");
++ ownerID = OwnerMap.register("oov");
+ oovWeights = new HashMap<Integer,Float>();
+
+ if (config.oovList != null) {
+ for (OOVItem item: config.oovList) {
+ oovWeights.put(Vocabulary.id(item.label), item.weight);
+ }
+ }
+ }
+
+ @Override
+ public ArrayList<String> reportDenseFeatures(int index) {
+ denseFeatureIndex = index;
+
+ ArrayList<String> names = new ArrayList<>(1);
+ names.add(name);
+ return names;
+ }
+
+ /**
+ * OOV rules cover exactly one word, and such rules belong to a grammar whose owner is "oov". Each
+ * OOV fires the OOVPenalty feature with a value of 1, so the cost is simply the weight, which was
+ * cached when the feature was created.
+ */
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+
- if (rule != null && this.ownerID == rule.getOwner()) {
++ if (rule != null && this.ownerID.equals(rule.getOwner())) {
+ acc.add(denseFeatureIndex, getValue(rule.getLHS()));
+ }
+
+ return null;
+ }
+
+ /**
+ * It's important for the OOV feature to contribute to the rule's estimated cost, so that OOV
+ * rules (which are added for all words, not just ones without translation options) get sorted
+ * to the bottom during cube pruning.
+ *
+ * Important! estimateCost returns the *weighted* feature value.
+ */
+ @Override
+ public float estimateCost(Rule rule, Sentence sentence) {
- if (rule != null && this.ownerID == rule.getOwner())
++ if (rule != null && this.ownerID.equals(rule.getOwner()))
+ return weights.getDense(denseFeatureIndex) * getValue(rule.getLHS());
+ return 0.0f;
+ }
+
+ private float getValue(int lhs) {
+ return oovWeights.containsKey(lhs) ? oovWeights.get(lhs) : defaultValue;
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/main/java/org/apache/joshua/decoder/ff/PhraseModel.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/ff/PhraseModel.java
index 0000000,2324292..7ae3dbc
mode 000000,100644..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/PhraseModel.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/PhraseModel.java
@@@ -1,0 -1,135 +1,134 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.decoder.ff;
+
+ import java.util.ArrayList;
+ import java.util.List;
+
-import org.apache.joshua.corpus.Vocabulary;
+ import org.apache.joshua.decoder.JoshuaConfiguration;
+ import org.apache.joshua.decoder.chart_parser.SourcePath;
+ import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+ import org.apache.joshua.decoder.ff.tm.Grammar;
++import org.apache.joshua.decoder.ff.tm.OwnerId;
++import org.apache.joshua.decoder.ff.tm.OwnerMap;
+ import org.apache.joshua.decoder.ff.tm.Rule;
+ import org.apache.joshua.decoder.hypergraph.HGNode;
+ import org.apache.joshua.decoder.segment_file.Sentence;
+
+ /**
+ * This feature handles the list of features that are found with grammar rules in the grammar file.
+ * dense features that may be associated with the rules in a grammar file. The feature names of
+ * these dense rules are a function of the phrase model owner. When the feature is loaded, it
+ * queries the weights for the set of features that are active for this grammar, storing them in an
+ * array.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ * @author Zhifei Li zhifei.work@gmail.com
+ */
+
+ public class PhraseModel extends StatelessFF {
+
+ /* The owner of the grammar. */
- private int ownerID;
- private String owner;
++ private final OwnerId ownerID;
++ private final String owner;
+
+ private float[] phrase_weights = null;
+
+ public PhraseModel(FeatureVector weights, String[] args, JoshuaConfiguration config, Grammar g) {
+ super(weights, "tm_", args, config);
+
- String owner = parsedArgs.get("owner");
- this.name = String.format("tm_%s", owner);
++ // Store the owner and name
++ this.owner = parsedArgs.get("owner");
++ this.ownerID = OwnerMap.register(owner);
++ this.name = String.format("tm_%s", this.owner);
+
+ /*
+ * Determine the number of features by querying the example grammar that was passed in.
+ */
+ phrase_weights = new float[g.getNumDenseFeatures()];
-// System.err.println(String.format("GOT %d FEATURES FOR %s", g.getNumDenseFeatures(), owner));
+ for (int i = 0; i < phrase_weights.length; i++)
+ phrase_weights[i] = weights.getSparse(String.format("tm_%s_%d", owner, i));
-
- // Store the owner.
- this.owner = owner;
- this.ownerID = Vocabulary.id(owner);
++
+ }
+
+ /**
+ * Just register a single weight, tm_OWNER, and use that to set its precomputed cost
+ */
+ @Override
+ public ArrayList<String> reportDenseFeatures(int index) {
+ denseFeatureIndex = index;
+
+ ArrayList<String> names = new ArrayList<String>();
+ for (int i = 0; i < phrase_weights.length; i++)
+ names.add(String.format("tm_%s_%d", owner, i));
+ return names;
+ }
+
+ /**
+ * Estimates the cost of applying this rule, which is just the score of the precomputable feature
+ * functions.
+ */
+ @Override
+ public float estimateCost(final Rule rule, Sentence sentence) {
+
- if (rule != null && rule.getOwner() == ownerID) {
++ if (rule != null && rule.getOwner().equals(ownerID)) {
+ if (rule.getPrecomputableCost() <= Float.NEGATIVE_INFINITY)
+ rule.setPrecomputableCost(phrase_weights, weights);
+
+ return rule.getPrecomputableCost();
+ }
+
+ return 0.0f;
+ }
+
+ /**
+ * Just chain to computeFeatures(rule), since this feature doesn't use the sourcePath or sentID. *
+ */
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+
- if (rule != null && rule.getOwner() == ownerID) {
++ if (rule != null && rule.getOwner().equals(ownerID)) {
+ /*
+ * Here, we peak at the Accumulator object. If it's asking for scores, then we don't bother to
+ * add each feature, but rather compute the inner product and add *that*. This is totally
+ * cheating; the Accumulator is supposed to be a generic object. But without this cheat
+ */
+ if (rule.getPrecomputableCost() <= Float.NEGATIVE_INFINITY) {
+ // float score = rule.getFeatureVector().innerProduct(weights);
+ rule.setPrecomputableCost(phrase_weights, weights);
+ }
+
+ // System.err.println(String.format("RULE = %s / %f", rule.getEnglishWords(), rule.getPrecomputableCost()));
+ for (int k = 0; k < phrase_weights.length; k++) {
+ // System.err.println(String.format("k = %d, denseFeatureIndex = %d, owner = %s, ownerID = %d", k, denseFeatureIndex, owner, ownerID));
+ acc.add(k + denseFeatureIndex, rule.getDenseFeature(k));
+ }
+
+ for (String key: rule.getFeatureVector().keySet())
+ acc.add(key, rule.getFeatureVector().getSparse(key));
+ }
+
+ return null;
+ }
+
+ public String toString() {
- return name + " " + Vocabulary.word(ownerID);
++ return name;
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/main/java/org/apache/joshua/decoder/ff/PhrasePenalty.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/ff/PhrasePenalty.java
index 0000000,3c38e60..9eecd0c
mode 000000,100644..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/PhrasePenalty.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/PhrasePenalty.java
@@@ -1,0 -1,86 +1,87 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.decoder.ff;
+
+ import java.util.ArrayList;
-import java.util.List;
++import java.util.List;
+
-import org.apache.joshua.corpus.Vocabulary;
+ import org.apache.joshua.decoder.JoshuaConfiguration;
+ import org.apache.joshua.decoder.chart_parser.SourcePath;
+ import org.apache.joshua.decoder.ff.state_maintenance.DPState;
++import org.apache.joshua.decoder.ff.tm.OwnerId;
++import org.apache.joshua.decoder.ff.tm.OwnerMap;
+ import org.apache.joshua.decoder.ff.tm.Rule;
+ import org.apache.joshua.decoder.hypergraph.HGNode;
+ import org.apache.joshua.decoder.phrase.Hypothesis;
+ import org.apache.joshua.decoder.segment_file.Sentence;
+
+ /**
+ * This feature just counts rules that are used. You can restrict it with a number of flags:
+ *
+ * -owner OWNER
+ * Only count rules owned by OWNER
+ * -target|-source
+ * Only count the target or source side (plus the LHS)
+ *
+ * TODO: add an option to separately provide a list of rule counts, restrict to counts above a threshold.
+ */
+ public class PhrasePenalty extends StatelessFF {
+
- private int owner = 0;
++ private final OwnerId owner;
+ private float value = 1.0f;
+
+ public PhrasePenalty(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, "PhrasePenalty", args, config);
+ if (parsedArgs.containsKey("owner"))
- this.owner = Vocabulary.id(parsedArgs.get("owner"));
++ this.owner = OwnerMap.register(parsedArgs.get("owner"));
+ else // default
- this.owner = Vocabulary.id("pt");
++ this.owner = OwnerMap.register("pt");
+ }
+
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+
+ if (rule != null && rule != Hypothesis.BEGIN_RULE && rule != Hypothesis.END_RULE
- && (owner == 0 || rule.getOwner() == owner))
++ && (rule.getOwner().equals(owner)))
+ acc.add(denseFeatureIndex, value);
+
+ return null;
+ }
+
+ @Override
+ public ArrayList<String> reportDenseFeatures(int index) {
+ denseFeatureIndex = index;
+ ArrayList<String> names = new ArrayList<String>();
+ names.add(name);
+ return names;
+ }
+
+ /**
+ * Returns the *weighted* estimate.
+ *
+ */
+ @Override
+ public float estimateCost(Rule rule, Sentence sentence) {
+ if (rule != null && rule != Hypothesis.BEGIN_RULE && rule != Hypothesis.END_RULE
- && (owner == 0 || rule.getOwner() == owner))
++ && (rule.getOwner().equals(owner)))
+ return weights.getDense(denseFeatureIndex) * value;
+ return 0.0f;
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/main/java/org/apache/joshua/decoder/ff/RuleCountBin.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/ff/RuleCountBin.java
index 0000000,5ba0c66..3ffbf65
mode 000000,100644..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/RuleCountBin.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/RuleCountBin.java
@@@ -1,0 -1,74 +1,77 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.decoder.ff;
+
+ import java.util.List;
+
-import org.apache.joshua.corpus.Vocabulary;
+ import org.apache.joshua.decoder.JoshuaConfiguration;
+ import org.apache.joshua.decoder.chart_parser.SourcePath;
+ import org.apache.joshua.decoder.ff.state_maintenance.DPState;
++import org.apache.joshua.decoder.ff.tm.OwnerId;
++import org.apache.joshua.decoder.ff.tm.OwnerMap;
+ import org.apache.joshua.decoder.ff.tm.Rule;
+ import org.apache.joshua.decoder.hypergraph.HGNode;
+ import org.apache.joshua.decoder.segment_file.Sentence;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
+ /*
+ * This feature computes a bin for the rule and activates a feature for it. It requires access to
+ * the index of the RarityPenalty field, from which the rule count can be computed.
+ */
+ public class RuleCountBin extends StatelessFF {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RuleCountBin.class);
+ private int field = -1;
++ private final OwnerId owner;
+
+ public RuleCountBin(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, "RuleCountBin", args, config);
++ owner = OwnerMap.register("pt");
+
+ field = Integer.parseInt(parsedArgs.get("field"));
+ }
+
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+
- if (rule.getOwner() != Vocabulary.id("pt"))
++ if (rule.getOwner().equals(owner))
+ return null;
+
+ float rarityPenalty = -rule.getFeatureVector().getSparse(String.format("tm_pt_%d", field));
+ int count = (int) (1.0 - Math.log(rarityPenalty));
+
+ String feature = "RuleCountBin_inf";
+
+ int[] bins = { 1, 2, 4, 8, 16, 32, 64, 128, 1000, 10000 };
+ for (int k : bins) {
+ if (count <= k) {
+ feature = String.format("RuleCountBin_%d", k);
+ break;
+ }
+ }
+
+ LOG.debug("RuleCountBin({}) = {} ==> {}", rarityPenalty, count, feature);
+
+ acc.add(feature, 1.0f);
+
+ return null;
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/main/java/org/apache/joshua/decoder/ff/RuleFF.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/ff/RuleFF.java
index 0000000,909e481..308d38a
mode 000000,100644..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/RuleFF.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/RuleFF.java
@@@ -1,0 -1,123 +1,126 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.decoder.ff;
+
+ import static com.google.common.cache.CacheBuilder.newBuilder;
++import static org.apache.joshua.decoder.ff.tm.OwnerMap.UNKNOWN_OWNER_ID;
+
+ import java.util.List;
+
+ import org.apache.joshua.corpus.Vocabulary;
+ import org.apache.joshua.decoder.JoshuaConfiguration;
+ import org.apache.joshua.decoder.chart_parser.SourcePath;
+ import org.apache.joshua.decoder.ff.state_maintenance.DPState;
++import org.apache.joshua.decoder.ff.tm.OwnerId;
++import org.apache.joshua.decoder.ff.tm.OwnerMap;
+ import org.apache.joshua.decoder.ff.tm.Rule;
+ import org.apache.joshua.decoder.hypergraph.HGNode;
+ import org.apache.joshua.decoder.segment_file.Sentence;
+
+ import com.google.common.cache.Cache;
+
+ /**
+ * This feature fires for rule ids.
+ * Firing can be restricted to rules from a certain owner, and rule ids
+ * can be generated from source side and/or target side.
+ */
+ public class RuleFF extends StatelessFF {
+
+ private enum Sides { SOURCE, TARGET, BOTH };
+
+ private static final String NAME = "RuleFF";
+ // value to fire for features
+ private static final int VALUE = 1;
+ // whether this feature is restricted to a certain grammar/owner
+ private final boolean ownerRestriction;
+ // the grammar/owner this feature is restricted to fire
- private final int owner;
++ private final OwnerId owner;
+ // what part of the rule should be extracted;
+ private final Sides sides;
+ // Strings separating words and rule sides
+ private static final String SEPARATOR = "~";
+ private static final String SIDES_SEPARATOR = "->";
+
+ private final Cache<Rule, String> featureCache;
+
+ public RuleFF(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, NAME, args, config);
+
+ ownerRestriction = (parsedArgs.containsKey("owner")) ? true : false;
- owner = ownerRestriction ? Vocabulary.id(parsedArgs.get("owner")) : 0;
++ owner = ownerRestriction ? OwnerMap.register(parsedArgs.get("owner")) : UNKNOWN_OWNER_ID;
+
+ if (parsedArgs.containsKey("sides")) {
+ final String sideValue = parsedArgs.get("sides");
+ if (sideValue.equalsIgnoreCase("source")) {
+ sides = Sides.SOURCE;
+ } else if (sideValue.equalsIgnoreCase("target")) {
+ sides = Sides.TARGET;
+ } else if (sideValue.equalsIgnoreCase("both")){
+ sides = Sides.BOTH;
+ } else {
+ throw new RuntimeException("Unknown side value.");
+ }
+ } else {
+ sides = Sides.BOTH;
+ }
+
+ // initialize cache
+ if (parsedArgs.containsKey("cacheSize")) {
+ featureCache = newBuilder().maximumSize(Integer.parseInt(parsedArgs.get("cacheSize"))).build();
+ } else {
+ featureCache = newBuilder().maximumSize(config.cachedRuleSize).build();
+ }
+ }
+
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+
- if (ownerRestriction && rule.getOwner() != owner) {
++ if (ownerRestriction && !rule.getOwner().equals(owner)) {
+ return null;
+ }
+
+ String featureName = featureCache.getIfPresent(rule);
+ if (featureName == null) {
+ featureName = getRuleString(rule);
+ featureCache.put(rule, featureName);
+ }
+ acc.add(featureName, VALUE);
+
+ return null;
+ }
+
+ /**
+ * Obtains the feature id for the given rule.
+ * @param rule
+ * @return String representing the feature name.s
+ */
+ private String getRuleString(final Rule rule) {
+ final StringBuilder sb = new StringBuilder(Vocabulary.word(rule.getLHS()))
+ .append(SIDES_SEPARATOR);
+ if (sides == Sides.SOURCE || sides == Sides.BOTH) {
+ sb.append(Vocabulary.getWords(rule.getFrench(), SEPARATOR));
+ }
+ sb.append(SIDES_SEPARATOR);
+ if (sides == Sides.TARGET || sides == Sides.BOTH) {
+ sb.append(Vocabulary.getWords(rule.getEnglish(), SEPARATOR));
+ }
+ return sb.toString();
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/main/java/org/apache/joshua/decoder/ff/fragmentlm/FragmentLMFF.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/ff/fragmentlm/FragmentLMFF.java
index 0000000,fa4c4af..861cf35
mode 000000,100644..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/fragmentlm/FragmentLMFF.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/fragmentlm/FragmentLMFF.java
@@@ -1,0 -1,365 +1,368 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.decoder.ff.fragmentlm;
+
+ import java.io.IOException;
+ import java.util.ArrayList;
+ import java.util.Collection;
+ import java.util.Collections;
+ import java.util.HashMap;
+ import java.util.List;
+ import java.util.Stack;
+
+ import org.apache.joshua.decoder.JoshuaConfiguration;
+ import org.apache.joshua.decoder.chart_parser.SourcePath;
+ import org.apache.joshua.decoder.ff.FeatureVector;
+ import org.apache.joshua.decoder.ff.StatefulFF;
+ import org.apache.joshua.decoder.ff.state_maintenance.DPState;
++import org.apache.joshua.decoder.ff.tm.OwnerId;
++import org.apache.joshua.decoder.ff.tm.OwnerMap;
+ import org.apache.joshua.decoder.ff.tm.Rule;
+ import org.apache.joshua.decoder.ff.tm.format.HieroFormatReader;
+ import org.apache.joshua.decoder.hypergraph.HGNode;
+ import org.apache.joshua.decoder.hypergraph.HyperEdge;
+ import org.apache.joshua.decoder.segment_file.Sentence;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
+ /**
+ * <p>Feature function that reads in a list of language model fragments and matches them against the
+ * hypergraph. This allows for language model fragment "glue" features, which fire when LM fragments
+ * (supplied as input) are assembled. These LM fragments are presumably useful in ensuring
+ * grammaticality and can be independent of the translation model fragments.</p>
+ *
+ * <p>Usage: in the Joshua Configuration file, put</p>
+ *
+ * <code>feature-function = FragmentLM -lm LM_FRAGMENTS_FILE -map RULE_FRAGMENTS_MAP_FILE</code>
+ *
+ * <p>LM_FRAGMENTS_FILE is a pointer to a file containing a list of fragments that it should look for.
+ * The format of the file is one fragment per line in PTB format, e.g.:</p>
+ *
+ * <code>(S NP (VP (VBD said) SBAR) (. .))</code>
+ *
+ * <p>RULE_FRAGMENTS_MAP_FILE points to a file that maps fragments to the flattened SCFG rule format
+ * that Joshua uses. This mapping is necessary because Joshua's rules have been flattened, meaning
+ * that their internal structure has been removed, yet this structure is needed for matching LM
+ * fragments. The format of the file is</p>
+ *
+ * <code>FRAGMENT ||| RULE-TARGET-SIDE</code>
+ *
+ * <p>for example,</p>
+ *
+ * <code>(S (NP (DT the) (NN man)) VP .) ||| the man [VP,1] [.,2] (SBAR (IN that) (S (NP (PRP he)) (VP
+ * (VBD was) (VB done)))) ||| that he was done (VP (VBD said) SBAR) ||| said SBAR</code>
+ *
+ * @author Matt Post post@cs.jhu.edu
+ */
+ public class FragmentLMFF extends StatefulFF {
+
+ private static final Logger LOG = LoggerFactory.getLogger(FragmentLMFF.class);
+
+ /*
+ * When building a fragment from a rule rooted in the hypergraph, this parameter determines how
+ * deep we'll go. Smaller values mean less hypergraph traversal but may also limit the LM
+ * fragments that can be fired.
+ */
+ private int BUILD_DEPTH = 1;
+
+ /*
+ * The maximum depth of a fragment, defined as the longest path from the fragment root to any of
+ * its leaves.
+ */
+ private int MAX_DEPTH = 0;
+
+ /*
+ * This is the minimum depth for lexicalized LM fragments. This allows you to easily exclude small
+ * depth-one fragments that may be overfit to the training data. A depth of 1 (the default) does
+ * not exclude any fragments.
+ */
+ private int MIN_LEX_DEPTH = 1;
+
+ /*
+ * Set to true to activate meta-features.
+ */
+ private boolean OPTS_DEPTH = false;
+
+ /*
+ * This contains a list of the language model fragments, indexed by LHS.
+ */
+ private HashMap<String, ArrayList<Tree>> lmFragments = null;
+
+ private int numFragments = 0;
+
+ /* The location of the file containing the language model fragments */
+ private String fragmentLMFile = "";
+
+ /**
+ * @param weights a {@link org.apache.joshua.decoder.ff.FeatureVector} with weights
+ * @param args arguments passed to the feature function
+ * @param config the {@link org.apache.joshua.decoder.JoshuaConfiguration}
+ */
+ public FragmentLMFF(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, "FragmentLMFF", args, config);
+
+ lmFragments = new HashMap<String, ArrayList<Tree>>();
+
+ fragmentLMFile = parsedArgs.get("lm");
+ BUILD_DEPTH = Integer.parseInt(parsedArgs.get("build-depth"));
+ MAX_DEPTH = Integer.parseInt(parsedArgs.get("max-depth"));
+ MIN_LEX_DEPTH = Integer.parseInt(parsedArgs.get("min-lex-depth"));
+
+ /* Read in the language model fragments */
+ try {
+ Collection<Tree> trees = PennTreebankReader.readTrees(fragmentLMFile);
+ for (Tree fragment : trees) {
+ addLMFragment(fragment);
+
+ // System.err.println(String.format("Read fragment: %s",
+ // lmFragments.get(lmFragments.size()-1)));
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(String.format("* WARNING: couldn't read fragment LM file '%s'",
+ fragmentLMFile), e);
+ }
+ LOG.info("FragmentLMFF: Read {} LM fragments from '{}'", numFragments, fragmentLMFile);
+ }
+
+ /**
+ * Add the provided fragment to the language model, subject to some filtering.
+ *
+ * @param fragment a {@link org.apache.joshua.decoder.ff.fragmentlm.Tree} fragment
+ */
+ public void addLMFragment(Tree fragment) {
+ if (lmFragments == null)
+ return;
+
+ int fragmentDepth = fragment.getDepth();
+
+ if (MAX_DEPTH != 0 && fragmentDepth > MAX_DEPTH) {
+ LOG.warn("Skipping fragment {} (depth {} > {})", fragment, fragmentDepth, MAX_DEPTH);
+ return;
+ }
+
+ if (MIN_LEX_DEPTH > 1 && fragment.isLexicalized() && fragmentDepth < MIN_LEX_DEPTH) {
+ LOG.warn("Skipping fragment {} (lex depth {} < {})", fragment, fragmentDepth, MIN_LEX_DEPTH);
+ return;
+ }
+
+ if (lmFragments.get(fragment.getRule()) == null) {
+ lmFragments.put(fragment.getRule(), new ArrayList<Tree>());
+ }
+ lmFragments.get(fragment.getRule()).add(fragment);
+ numFragments++;
+ }
+
+ /**
+ * This function computes the features that fire when the current rule is applied. The features
+ * that fire are any LM fragments that match the fragment associated with the current rule. LM
+ * fragments may recurse over the tail nodes, following 1-best backpointers until the fragment
+ * either matches or fails.
+ *
+ * @param rule {@link org.apache.joshua.decoder.ff.tm.Rule} to be utilized within computation
+ * @param tailNodes {@link java.util.List} of {@link org.apache.joshua.decoder.hypergraph.HGNode} tail nodes
+ * @param i todo
+ * @param j todo
+ * @param sourcePath information about a path taken through the source {@link org.apache.joshua.lattice.Lattice}
+ * @param sentence {@link org.apache.joshua.lattice.Lattice} input
+ * @param acc {@link org.apache.joshua.decoder.ff.FeatureFunction.Accumulator} object permitting generalization of feature computation
+ * @return the new dynamic programming state (null for stateless features)
+ */
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+
+ /*
+ * Get the fragment associated with the target side of this rule.
+ *
+ * This could be done more efficiently. For example, just build the tree fragment once and then
+ * pattern match against it. This would circumvent having to build the tree possibly once every
+ * time you try to apply a rule.
+ */
+ Tree baseTree = Tree.buildTree(rule, tailNodes, BUILD_DEPTH);
+
+ Stack<Tree> nodeStack = new Stack<Tree>();
+ nodeStack.add(baseTree);
+ while (!nodeStack.empty()) {
+ Tree tree = nodeStack.pop();
+ if (tree == null)
+ continue;
+
+ if (lmFragments.get(tree.getRule()) != null) {
+ for (Tree fragment : lmFragments.get(tree.getRule())) {
+ // System.err.println(String.format("Does\n %s match\n %s??\n -> %s", fragment, tree,
+ // match(fragment, tree)));
+
+ if (fragment.getLabel() == tree.getLabel() && match(fragment, tree)) {
+ // System.err.println(String.format(" FIRING: matched %s against %s", fragment, tree));
+ acc.add(fragment.escapedString(), 1);
+ if (OPTS_DEPTH)
+ if (fragment.isLexicalized())
+ acc.add(String.format("FragmentFF_lexdepth%d", fragment.getDepth()), 1);
+ else
+ acc.add(String.format("FragmentFF_depth%d", fragment.getDepth()), 1);
+ }
+ }
+ }
+
+ // We also need to try matching rules against internal nodes of the fragment corresponding to
+ // this
+ // rule
+ if (tree.getChildren() != null)
+ for (Tree childNode : tree.getChildren()) {
+ if (!childNode.isBoundary())
+ nodeStack.add(childNode);
+ }
+ }
+
+ return new FragmentState(baseTree);
+ }
+
+ /**
+ * Matches the fragment against the (possibly partially-built) tree. Assumption
+ *
+ * @param fragment the language model fragment
+ * @param tree the tree to match against (expanded from the hypergraph)
+ * @return
+ */
+ private boolean match(Tree fragment, Tree tree) {
+ // System.err.println(String.format("MATCH(%s,%s)", fragment, tree));
+
+ /* Make sure the root labels match. */
+ if (fragment.getLabel() != tree.getLabel()) {
+ return false;
+ }
+
+ /* Same number of kids? */
+ List<Tree> fkids = fragment.getChildren();
+ if (fkids.size() > 0) {
+ List<Tree> tkids = tree.getChildren();
+ if (fkids.size() != tkids.size()) {
+ return false;
+ }
+
+ /* Do the kids match on all labels? */
+ for (int i = 0; i < fkids.size(); i++)
+ if (fkids.get(i).getLabel() != tkids.get(i).getLabel())
+ return false;
+
+ /* Recursive match. */
+ for (int i = 0; i < fkids.size(); i++) {
+ if (!match(fkids.get(i), tkids.get(i)))
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ @Override
+ public DPState computeFinal(HGNode tailNodes, int i, int j, SourcePath sourcePath, Sentence sentence,
+ Accumulator acc) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public float estimateFutureCost(Rule rule, DPState state, Sentence sentence) {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+
+ @Override
+ public float estimateCost(Rule rule, Sentence sentence) {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+
+ public static void main(String[] args) {
+ /* Add an LM fragment, then create a dummy multi-level hypergraph to match the fragment against. */
+ // FragmentLMFF fragmentLMFF = new FragmentLMFF(new FeatureVector(), (StateComputer) null, "");
+ FragmentLMFF fragmentLMFF = new FragmentLMFF(new FeatureVector(),
+ new String[] {"-lm", "test/fragments.txt", "-map", "test/mapping.txt"}, null);
+
+ Tree fragment = Tree.fromString("(S NP (VP (VBD \"said\") SBAR) (. \".\"))");
+
+ Rule ruleS = new HieroFormatReader()
+ .parseLine("[S] ||| the man [VP,1] [.,2] ||| the man [VP,1] [.,2] ||| 0");
+ Rule ruleVP = new HieroFormatReader()
+ .parseLine("[VP] ||| said [SBAR,1] ||| said [SBAR,1] ||| 0");
+ Rule ruleSBAR = new HieroFormatReader()
+ .parseLine("[SBAR] ||| that he was done ||| that he was done ||| 0");
+ Rule rulePERIOD = new HieroFormatReader().parseLine("[.] ||| . ||| . ||| 0");
+
- ruleS.setOwner(0);
- ruleVP.setOwner(0);
- ruleSBAR.setOwner(0);
- rulePERIOD.setOwner(0);
++ final OwnerId owner = OwnerMap.register("0");
++ ruleS.setOwner(owner);
++ ruleVP.setOwner(owner);
++ ruleSBAR.setOwner(owner);
++ rulePERIOD.setOwner(owner);
+
+ HyperEdge edgeSBAR = new HyperEdge(ruleSBAR, 0.0f, 0.0f, null, (SourcePath) null);
+
+ HGNode nodeSBAR = new HGNode(3, 7, ruleSBAR.getLHS(), null, edgeSBAR, 0.0f);
+ ArrayList<HGNode> tailNodesVP = new ArrayList<HGNode>();
+ Collections.addAll(tailNodesVP, nodeSBAR);
+ HyperEdge edgeVP = new HyperEdge(ruleVP, 0.0f, 0.0f, tailNodesVP, (SourcePath) null);
+ HGNode nodeVP = new HGNode(2, 7, ruleVP.getLHS(), null, edgeVP, 0.0f);
+
+ HyperEdge edgePERIOD = new HyperEdge(rulePERIOD, 0.0f, 0.0f, null, (SourcePath) null);
+ HGNode nodePERIOD = new HGNode(7, 8, rulePERIOD.getLHS(), null, edgePERIOD, 0.0f);
+
+ ArrayList<HGNode> tailNodes = new ArrayList<HGNode>();
+ Collections.addAll(tailNodes, nodeVP, nodePERIOD);
+
+ Tree tree = Tree.buildTree(ruleS, tailNodes, 1);
+ boolean matched = fragmentLMFF.match(fragment, tree);
+ LOG.info("Does\n {} match\n {}??\n -> {}", fragment, tree, matched);
+ }
+
+ /**
+ * Maintains a state pointer used by KenLM to implement left-state minimization.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ * @author Juri Ganitkevitch juri@cs.jhu.edu
+ */
+ public class FragmentState extends DPState {
+
+ private Tree tree = null;
+
+ public FragmentState(Tree tree) {
+ this.tree = tree;
+ }
+
+ /**
+ * Every tree is unique.
+ *
+ * Some savings could be had here if we grouped together items with the same string.
+ */
+ @Override
+ public int hashCode() {
+ return tree.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ return (other instanceof FragmentState && this == other);
+ }
+
+ @Override
+ public String toString() {
+ return String.format("[FragmentState %s]", tree);
+ }
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/LanguageModelFF.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/LanguageModelFF.java
index 0000000,3fea410..9388ed7
mode 000000,100644..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/LanguageModelFF.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/LanguageModelFF.java
@@@ -1,0 -1,527 +1,495 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.decoder.ff.lm;
+
+ import java.io.IOException;
+ import java.util.ArrayList;
+ import java.util.Arrays;
-import java.util.HashMap;
+ import java.util.LinkedList;
+ import java.util.List;
+
-import com.google.common.primitives.Ints;
-
+ import org.apache.joshua.corpus.Vocabulary;
+ import org.apache.joshua.decoder.JoshuaConfiguration;
+ import org.apache.joshua.decoder.Support;
+ import org.apache.joshua.decoder.chart_parser.SourcePath;
+ import org.apache.joshua.decoder.ff.FeatureVector;
+ import org.apache.joshua.decoder.ff.StatefulFF;
+ import org.apache.joshua.decoder.ff.lm.berkeley_lm.LMGrammarBerkeley;
-import org.apache.joshua.decoder.ff.lm.KenLM;
+ import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+ import org.apache.joshua.decoder.ff.state_maintenance.NgramDPState;
+ import org.apache.joshua.decoder.ff.tm.Rule;
+ import org.apache.joshua.decoder.hypergraph.HGNode;
+ import org.apache.joshua.decoder.segment_file.Sentence;
+ import org.apache.joshua.util.FormatUtils;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
++import com.google.common.annotations.VisibleForTesting;
++import com.google.common.primitives.Ints;
++
+ /**
+ * This class performs the following:
+ * <ol>
+ * <li>Gets the additional LM score due to combinations of small items into larger ones by using
+ * rules</li>
+ * <li>Gets the LM state</li>
+ * <li>Gets the left-side LM state estimation score</li>
+ * </ol>
- *
++ *
+ * @author Matt Post post@cs.jhu.edu
+ * @author Juri Ganitkevitch juri@cs.jhu.edu
+ * @author Zhifei Li, zhifei.work@gmail.com
+ */
+ public class LanguageModelFF extends StatefulFF {
+
- private static final Logger LOG = LoggerFactory.getLogger(LanguageModelFF.class);
++ static final Logger LOG = LoggerFactory.getLogger(LanguageModelFF.class);
+
+ public static int LM_INDEX = 0;
+ private int startSymbolId;
+
+ /**
+ * N-gram language model. We assume the language model is in ARPA format for equivalent state:
- *
++ *
+ * <ol>
+ * <li>We assume it is a backoff lm, and high-order ngram implies low-order ngram; absense of
+ * low-order ngram implies high-order ngram</li>
+ * <li>For a ngram, existence of backoffweight => existence a probability Two ways of dealing with
+ * low counts:
+ * <ul>
+ * <li>SRILM: don't multiply zeros in for unknown words</li>
+ * <li>Pharaoh: cap at a minimum score exp(-10), including unknown words</li>
+ * </ul>
+ * </li>
+ * </ol>
+ */
+ protected NGramLanguageModel languageModel;
+
+ /**
+ * We always use this order of ngram, though the LMGrammar may provide higher order probability.
+ */
+ protected final int ngramOrder;
+
+ /*
+ * We cache the weight of the feature since there is only one.
+ */
+ protected float weight;
+ protected String type;
+ protected String path;
+
+ /* Whether this is a class-based LM */
- private boolean isClassLM;
++ protected boolean isClassLM;
+ private ClassMap classMap;
+
- protected class ClassMap {
-
- private final int OOV_id = Vocabulary.getUnknownId();
- private HashMap<Integer, Integer> classMap;
-
- public ClassMap(String file_name) throws IOException {
- this.classMap = new HashMap<Integer, Integer>();
- read(file_name);
- }
-
- public int getClassID(int wordID) {
- return this.classMap.getOrDefault(wordID, OOV_id);
- }
-
- /**
- * Reads a class map from file.
- *
- * @param file_name
- * @throws IOException
- */
- private void read(String file_name) throws IOException {
-
- int lineno = 0;
- for (String line: new org.apache.joshua.util.io.LineReader(file_name, false)) {
- lineno++;
- String[] lineComp = line.trim().split("\\s+");
- try {
- this.classMap.put(Vocabulary.id(lineComp[0]), Vocabulary.id(lineComp[1]));
- } catch (java.lang.ArrayIndexOutOfBoundsException e) {
- LOG.warn("bad vocab line #{} '{}'", lineno, line);
- LOG.warn(e.getMessage(), e);
- }
- }
- }
-
- }
-
+ public LanguageModelFF(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, String.format("lm_%d", LanguageModelFF.LM_INDEX++), args, config);
+
+ this.type = parsedArgs.get("lm_type");
- this.ngramOrder = Integer.parseInt(parsedArgs.get("lm_order"));
++ this.ngramOrder = Integer.parseInt(parsedArgs.get("lm_order"));
+ this.path = parsedArgs.get("lm_file");
+
- if (parsedArgs.containsKey("class_map"))
- try {
- this.isClassLM = true;
- this.classMap = new ClassMap(parsedArgs.get("class_map"));
- } catch (IOException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
++ if (parsedArgs.containsKey("class_map")) {
++ this.isClassLM = true;
++ this.classMap = new ClassMap(parsedArgs.get("class_map"));
++ }
+
+ // The dense feature initialization hasn't happened yet, so we have to retrieve this as sparse
+ this.weight = weights.getSparse(name);
+
+ initializeLM();
+ }
+
+ @Override
+ public ArrayList<String> reportDenseFeatures(int index) {
+ denseFeatureIndex = index;
+
- ArrayList<String> names = new ArrayList<String>();
++ final ArrayList<String> names = new ArrayList<String>(1);
+ names.add(name);
+ return names;
+ }
+
+ /**
+ * Initializes the underlying language model.
+ */
+ protected void initializeLM() {
+ if (type.equals("kenlm")) {
+ this.languageModel = new KenLM(ngramOrder, path);
+
+ } else if (type.equals("berkeleylm")) {
+ this.languageModel = new LMGrammarBerkeley(ngramOrder, path);
+
+ } else {
+ String msg = String.format("* FATAL: Invalid backend lm_type '%s' for LanguageModel", type)
+ + "* Permissible values for 'lm_type' are 'kenlm' and 'berkeleylm'";
+ throw new RuntimeException(msg);
+ }
+
+ Vocabulary.registerLanguageModel(this.languageModel);
+ Vocabulary.id(config.default_non_terminal);
+
+ startSymbolId = Vocabulary.id(Vocabulary.START_SYM);
+ }
+
+ public NGramLanguageModel getLM() {
+ return this.languageModel;
+ }
+
++ public boolean isClassLM() {
++ return this.isClassLM;
++ }
++
+ public String logString() {
- if (languageModel != null)
- return String.format("%s, order %d (weight %.3f)", name, languageModel.getOrder(), weight);
- else
- return "WHOA";
++ return String.format("%s, order %d (weight %.3f), classLm=%s", name, languageModel.getOrder(), weight, isClassLM);
+ }
+
+ /**
- * Computes the features incurred along this edge. Note that these features are unweighted costs
- * of the feature; they are the feature cost, not the model cost, or the inner product of them.
++ * Computes the features incurred along this edge. Note that these features
++ * are unweighted costs of the feature; they are the feature cost, not the
++ * model cost, or the inner product of them.
+ */
+ @Override
- public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
- Sentence sentence, Accumulator acc) {
-
- NgramDPState newState = null;
- if (rule != null) {
- if (config.source_annotations) {
- // Get source side annotations and project them to the target side
- newState = computeTransition(getTags(rule, i, j, sentence), tailNodes, acc);
- }
- else {
- if (this.isClassLM) {
- // Use a class language model
- // Return target side classes
- newState = computeTransition(getClasses(rule), tailNodes, acc);
- }
- else {
- // Default LM
- newState = computeTransition(rule.getEnglish(), tailNodes, acc);
- }
- }
++ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j,
++ SourcePath sourcePath, Sentence sentence, Accumulator acc) {
+
++ if (rule == null) {
++ return null;
+ }
+
- return newState;
++ int[] words;
++ if (config.source_annotations) {
++ // get source side annotations and project them to the target side
++ words = getTags(rule, i, j, sentence);
++ } else {
++ words = getRuleIds(rule);
++ }
++
++ return computeTransition(words, tailNodes, acc);
++
++ }
++
++ /**
++ * Retrieve ids from rule. These are either simply the rule ids on the target
++ * side, their corresponding class map ids, or the configured source-side
++ * annotation tags.
++ */
++ @VisibleForTesting
++ public int[] getRuleIds(final Rule rule) {
++ if (this.isClassLM) {
++ // map words to class ids
++ return getClasses(rule);
++ }
++ // Regular LM: use rule word ids
++ return rule.getEnglish();
+ }
+
+ /**
+ * Input sentences can be tagged with information specific to the language model. This looks for
+ * such annotations by following a word's alignments back to the source words, checking for
+ * annotations, and replacing the surface word if such annotations are found.
+ * @param rule the {@link org.apache.joshua.decoder.ff.tm.Rule} to use
+ * @param begin todo
+ * @param end todo
+ * @param sentence {@link org.apache.joshua.lattice.Lattice} input
+ * @return todo
+ */
+ protected int[] getTags(Rule rule, int begin, int end, Sentence sentence) {
+ /* Very important to make a copy here, so the original rule is not modified */
+ int[] tokens = Arrays.copyOf(rule.getEnglish(), rule.getEnglish().length);
+ byte[] alignments = rule.getAlignment();
+
+ // System.err.println(String.format("getTags() %s", rule.getRuleString()));
+
+ /* For each target-side token, project it to each of its source-language alignments. If any of those
+ * are annotated, take the first annotation and quit.
+ */
+ if (alignments != null) {
+ for (int i = 0; i < tokens.length; i++) {
+ if (tokens[i] > 0) { // skip nonterminals
+ for (int j = 0; j < alignments.length; j += 2) {
+ if (alignments[j] == i) {
+ String annotation = sentence.getAnnotation((int)alignments[i] + begin, "class");
+ if (annotation != null) {
- // System.err.println(String.format(" word %d source %d abs %d annotation %d/%s",
++ // System.err.println(String.format(" word %d source %d abs %d annotation %d/%s",
+ // i, alignments[i], alignments[i] + begin, annotation, Vocabulary.word(annotation)));
+ tokens[i] = Vocabulary.id(annotation);
+ break;
+ }
+ }
+ }
+ }
+ }
+ }
+
+ return tokens;
+ }
+
- /**
- * Sets the class map if this is a class LM
++ /**
++ * Sets the class map if this is a class LM
+ * @param fileName a string path to a file
+ * @throws IOException if there is an error reading the input file
+ */
+ public void setClassMap(String fileName) throws IOException {
+ this.classMap = new ClassMap(fileName);
+ }
+
+ /**
+ * Replace each word in a rule with the target side classes.
+ * @param rule {@link org.apache.joshua.decoder.ff.tm.Rule} to use when obtaining tokens
+ * @return int[] of tokens
+ */
+ protected int[] getClasses(Rule rule) {
+ if (this.classMap == null) {
+ throw new RuntimeException("The class map is not set. Cannot use the class LM ");
+ }
+ /* Very important to make a copy here, so the original rule is not modified */
+ int[] tokens = Arrays.copyOf(rule.getEnglish(), rule.getEnglish().length);
+ for (int i = 0; i < tokens.length; i++) {
+ if (tokens[i] > 0 ) {
+ tokens[i] = this.classMap.getClassID(tokens[i]);
+ }
+ }
+ return tokens;
+ }
+
+ @Override
+ public DPState computeFinal(HGNode tailNode, int i, int j, SourcePath sourcePath, Sentence sentence,
+ Accumulator acc) {
+ return computeFinalTransition((NgramDPState) tailNode.getDPState(stateIndex), acc);
+ }
+
+ /**
+ * This function computes all the complete n-grams found in the rule, as well as the incomplete
+ * n-grams on the left-hand side.
+ */
+ @Override
+ public float estimateCost(Rule rule, Sentence sentence) {
+
+ float estimate = 0.0f;
+ boolean considerIncompleteNgrams = true;
+
- int[] enWords = rule.getEnglish();
++ int[] enWords = getRuleIds(rule);
+
+ List<Integer> words = new ArrayList<Integer>();
+ boolean skipStart = (enWords[0] == startSymbolId);
+
+ /*
+ * Move through the words, accumulating language model costs each time we have an n-gram (n >=
+ * 2), and resetting the series of words when we hit a nonterminal.
+ */
+ for (int c = 0; c < enWords.length; c++) {
+ int currentWord = enWords[c];
+ if (FormatUtils.isNonterminal(currentWord)) {
+ estimate += scoreChunkLogP(words, considerIncompleteNgrams, skipStart);
+ words.clear();
+ skipStart = false;
+ } else {
+ words.add(currentWord);
+ }
+ }
+ estimate += scoreChunkLogP(words, considerIncompleteNgrams, skipStart);
+
+ return weight * estimate;
+ }
+
+ /**
+ * Estimates the future cost of a rule. For the language model feature, this is the sum of the
+ * costs of the leftmost k-grams, k = [1..n-1].
+ */
+ @Override
+ public float estimateFutureCost(Rule rule, DPState currentState, Sentence sentence) {
+ NgramDPState state = (NgramDPState) currentState;
+
+ float estimate = 0.0f;
+ int[] leftContext = state.getLeftLMStateWords();
+
+ if (null != leftContext) {
+ boolean skipStart = true;
+ if (leftContext[0] != startSymbolId) {
+ skipStart = false;
+ }
+ estimate += scoreChunkLogP(leftContext, true, skipStart);
+ }
+ return weight * estimate;
+ }
+
+ /**
+ * Compute the cost of a rule application. The cost of applying a rule is computed by determining
+ * the n-gram costs for all n-grams created by this rule application, and summing them. N-grams
+ * are created when (a) terminal words in the rule string are followed by a nonterminal (b)
+ * terminal words in the rule string are preceded by a nonterminal (c) we encounter adjacent
+ * nonterminals. In all of these situations, the corresponding boundary words of the node in the
+ * hypergraph represented by the nonterminal must be retrieved.
- *
++ *
+ * IMPORTANT: only complete n-grams are scored. This means that hypotheses with fewer words
+ * than the complete n-gram state remain *unscored*. This fact adds a lot of complication to the
+ * code, including the use of the computeFinal* family of functions, which correct this fact for
+ * sentences that are too short on the final transition.
+ */
+ private NgramDPState computeTransition(int[] enWords, List<HGNode> tailNodes, Accumulator acc) {
+
+ int[] current = new int[this.ngramOrder];
+ int[] shadow = new int[this.ngramOrder];
+ int ccount = 0;
+ float transitionLogP = 0.0f;
+ int[] left_context = null;
+
+ for (int c = 0; c < enWords.length; c++) {
+ int curID = enWords[c];
+
+ if (FormatUtils.isNonterminal(curID)) {
+ int index = -(curID + 1);
+
+ NgramDPState state = (NgramDPState) tailNodes.get(index).getDPState(stateIndex);
+ int[] left = state.getLeftLMStateWords();
+ int[] right = state.getRightLMStateWords();
+
+ // Left context.
+ for (int i = 0; i < left.length; i++) {
+ current[ccount++] = left[i];
+
+ if (left_context == null && ccount == this.ngramOrder - 1)
+ left_context = Arrays.copyOf(current, ccount);
+
+ if (ccount == this.ngramOrder) {
+ // Compute the current word probability, and remove it.
+ float prob = this.languageModel.ngramLogProbability(current, this.ngramOrder);
+ // System.err.println(String.format("-> prob(%s) = %f", Vocabulary.getWords(current), prob));
+ transitionLogP += prob;
+ System.arraycopy(current, 1, shadow, 0, this.ngramOrder - 1);
+ int[] tmp = current;
+ current = shadow;
+ shadow = tmp;
+ --ccount;
+ }
+ }
+ System.arraycopy(right, 0, current, ccount - right.length, right.length);
+ } else { // terminal words
+ current[ccount++] = curID;
+
+ if (left_context == null && ccount == this.ngramOrder - 1)
+ left_context = Arrays.copyOf(current, ccount);
+
+ if (ccount == this.ngramOrder) {
+ // Compute the current word probability, and remove it.s
+ float prob = this.languageModel.ngramLogProbability(current, this.ngramOrder);
+ // System.err.println(String.format("-> prob(%s) = %f", Vocabulary.getWords(current), prob));
+ transitionLogP += prob;
+ System.arraycopy(current, 1, shadow, 0, this.ngramOrder - 1);
+ int[] tmp = current;
+ current = shadow;
+ shadow = tmp;
+ --ccount;
+ }
+ }
+ }
+ // acc.add(name, transitionLogP);
+ acc.add(denseFeatureIndex, transitionLogP);
+
+ if (left_context != null) {
+ return new NgramDPState(left_context, Arrays.copyOfRange(current, ccount - this.ngramOrder
+ + 1, ccount));
+ } else {
+ int[] context = Arrays.copyOf(current, ccount);
+ return new NgramDPState(context, context);
+ }
+ }
+
+ /**
+ * This function differs from regular transitions because we incorporate the cost of incomplete
+ * left-hand ngrams, as well as including the start- and end-of-sentence markers (if they were
+ * requested when the object was created).
- *
++ *
+ * @param state the dynamic programming state
+ * @return the final transition probability (including incomplete n-grams)
+ */
+ private NgramDPState computeFinalTransition(NgramDPState state, Accumulator acc) {
+
+ // System.err.println(String.format("LanguageModel::computeFinalTransition()"));
+
+ float res = 0.0f;
+ LinkedList<Integer> currentNgram = new LinkedList<Integer>();
+ int[] leftContext = state.getLeftLMStateWords();
+ int[] rightContext = state.getRightLMStateWords();
+
+ for (int i = 0; i < leftContext.length; i++) {
+ int t = leftContext[i];
+ currentNgram.add(t);
+
+ if (currentNgram.size() >= 2) { // start from bigram
+ float prob = this.languageModel.ngramLogProbability(Support.toArray(currentNgram),
+ currentNgram.size());
+ res += prob;
+ }
+ if (currentNgram.size() == this.ngramOrder)
+ currentNgram.removeFirst();
+ }
+
+ // Tell the accumulator
+ // acc.add(name, res);
+ acc.add(denseFeatureIndex, res);
+
+ // State is the same
+ return new NgramDPState(leftContext, rightContext);
+ }
+
+
+ /**
+ * Compatibility method for {@link #scoreChunkLogP(int[], boolean, boolean)}
+ */
+ private float scoreChunkLogP(List<Integer> words, boolean considerIncompleteNgrams,
+ boolean skipStart) {
+ return scoreChunkLogP(Ints.toArray(words), considerIncompleteNgrams, skipStart);
+ }
+
+ /**
+ * This function is basically a wrapper for NGramLanguageModel::sentenceLogProbability(). It
+ * computes the probability of a phrase ("chunk"), using lower-order n-grams for the first n-1
+ * words.
- *
++ *
+ * @param words
+ * @param considerIncompleteNgrams
+ * @param skipStart
+ * @return the phrase log probability
+ */
+ private float scoreChunkLogP(int[] words, boolean considerIncompleteNgrams,
+ boolean skipStart) {
+
+ float score = 0.0f;
+ if (words.length > 0) {
+ int startIndex;
+ if (!considerIncompleteNgrams) {
+ startIndex = this.ngramOrder;
+ } else if (skipStart) {
+ startIndex = 2;
+ } else {
+ startIndex = 1;
+ }
+ score = this.languageModel.sentenceLogProbability(words, this.ngramOrder, startIndex);
+ }
+
+ return score;
+ }
+
+ /**
+ * Public method to set LM_INDEX back to 0.
+ * Required if multiple instances of the JoshuaDecoder live in the same JVM.
+ */
+ public static void resetLmIndex() {
+ LM_INDEX = 0;
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/StateMinimizingLanguageModel.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/StateMinimizingLanguageModel.java
index 0000000,533365c..88dc647
mode 000000,100644..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/StateMinimizingLanguageModel.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/StateMinimizingLanguageModel.java
@@@ -1,0 -1,202 +1,193 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.decoder.ff.lm;
+
-import java.util.ArrayList;
++import static org.apache.joshua.util.FormatUtils.isNonterminal;
++
+ import java.util.List;
+ import java.util.concurrent.ConcurrentHashMap;
+
+ import org.apache.joshua.corpus.Vocabulary;
+ import org.apache.joshua.decoder.JoshuaConfiguration;
+ import org.apache.joshua.decoder.chart_parser.SourcePath;
+ import org.apache.joshua.decoder.ff.FeatureVector;
-import org.apache.joshua.decoder.ff.lm.KenLM;
+ import org.apache.joshua.decoder.ff.lm.KenLM.StateProbPair;
+ import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+ import org.apache.joshua.decoder.ff.state_maintenance.KenLMState;
+ import org.apache.joshua.decoder.ff.tm.Rule;
+ import org.apache.joshua.decoder.hypergraph.HGNode;
+ import org.apache.joshua.decoder.segment_file.Sentence;
-import org.apache.joshua.util.FormatUtils;
+
+ /**
+ * Wrapper for KenLM LMs with left-state minimization. We inherit from the regular
- *
++ *
+ * @author Matt Post post@cs.jhu.edu
+ * @author Juri Ganitkevitch juri@cs.jhu.edu
+ */
+ public class StateMinimizingLanguageModel extends LanguageModelFF {
+
+ // maps from sentence numbers to KenLM-side pools used to allocate state
+ private static final ConcurrentHashMap<Integer, Long> poolMap = new ConcurrentHashMap<Integer, Long>();
+
+ public StateMinimizingLanguageModel(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, args, config);
+ this.type = "kenlm";
+ if (parsedArgs.containsKey("lm_type") && ! parsedArgs.get("lm_type").equals("kenlm")) {
+ String msg = "* FATAL: StateMinimizingLanguageModel only supports 'kenlm' lm_type backend"
+ + "* Remove lm_type from line or set to 'kenlm'";
+ throw new RuntimeException(msg);
+ }
+ }
-
- @Override
- public ArrayList<String> reportDenseFeatures(int index) {
- denseFeatureIndex = index;
-
- ArrayList<String> names = new ArrayList<String>();
- names.add(name);
- return names;
- }
+
+ /**
+ * Initializes the underlying language model.
+ */
+ @Override
+ public void initializeLM() {
-
++
+ // Override type (only KenLM supports left-state minimization)
+ this.languageModel = new KenLM(ngramOrder, path);
+
+ Vocabulary.registerLanguageModel(this.languageModel);
+ Vocabulary.id(config.default_non_terminal);
-
++
+ }
-
++
+ /**
- * Estimates the cost of a rule. We override here since KenLM can do it more efficiently
- * than the default {@link LanguageModelFF} class.
- *
- * Most of this function implementation is redundant with compute().
++ * Estimates the cost of a rule. We override here since KenLM can do it more
++ * efficiently than the default {@link LanguageModelFF} class.
+ */
+ @Override
+ public float estimateCost(Rule rule, Sentence sentence) {
-
- int[] ruleWords = rule.getEnglish();
-
- // The IDs we'll pass to KenLM
- long[] words = new long[ruleWords.length];
+
- for (int x = 0; x < ruleWords.length; x++) {
- int id = ruleWords[x];
++ int[] ruleWords = getRuleIds(rule);
+
- if (FormatUtils.isNonterminal(id)) {
- // For the estimate, we can just mark negative values
- words[x] = -1;
++ // map to ken lm ids
++ final long[] words = mapToKenLmIds(ruleWords, null, true);
+
- } else {
- // Terminal: just add it
- words[x] = id;
- }
- }
-
+ // Get the probability of applying the rule and the new state
+ return weight * ((KenLM) languageModel).estimateRule(words);
+ }
-
++
+ /**
+ * Computes the features incurred along this edge. Note that these features are unweighted costs
+ * of the feature; they are the feature cost, not the model cost, or the inner product of them.
+ */
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+
- int[] ruleWords = config.source_annotations
- ? getTags(rule, i, j, sentence)
- : rule.getEnglish();
-
- // The IDs we'll pass to KenLM
- long[] words = new long[ruleWords.length];
++ if (rule == null ) {
++ return null;
++ }
+
- for (int x = 0; x < ruleWords.length; x++) {
- int id = ruleWords[x];
++ int[] ruleWords;
++ if (config.source_annotations) {
++ // get source side annotations and project them to the target side
++ ruleWords = getTags(rule, i, j, sentence);
++ } else {
++ ruleWords = getRuleIds(rule);
++ }
+
- if (FormatUtils.isNonterminal(id)) {
- // Nonterminal: retrieve the KenLM long that records the state
- int index = -(id + 1);
- KenLMState state = (KenLMState) tailNodes.get(index).getDPState(stateIndex);
- words[x] = -state.getState();
++ // map to ken lm ids
++ final long[] words = mapToKenLmIds(ruleWords, tailNodes, false);
+
- } else {
- // Terminal: just add it
- words[x] = id;
- }
- }
-
- int sentID = sentence.id();
++ final int sentID = sentence.id();
+ // Since sentId is unique across threads, next operations are safe, but not atomic!
+ if (!poolMap.containsKey(sentID)) {
+ poolMap.put(sentID, KenLM.createPool());
+ }
+
+ // Get the probability of applying the rule and the new state
- StateProbPair pair = ((KenLM) languageModel).probRule(words, poolMap.get(sentID));
++ final StateProbPair pair = ((KenLM) languageModel).probRule(words, poolMap.get(sentID));
+
+ // Record the prob
-// acc.add(name, pair.prob);
+ acc.add(denseFeatureIndex, pair.prob);
+
+ // Return the state
+ return pair.state;
+ }
+
+ /**
++ * Maps given array of word/class ids to KenLM ids. For estimating cost and computing,
++ * state retrieval differs slightly.
++ */
++ private long[] mapToKenLmIds(int[] ids, List<HGNode> tailNodes, boolean isOnlyEstimate) {
++ // The IDs we will to KenLM
++ long[] kenIds = new long[ids.length];
++ for (int x = 0; x < ids.length; x++) {
++ int id = ids[x];
++
++ if (isNonterminal(id)) {
++
++ if (isOnlyEstimate) {
++ // For the estimate, we can just mark negative values
++ kenIds[x] = -1;
++ } else {
++ // Nonterminal: retrieve the KenLM long that records the state
++ int index = -(id + 1);
++ final KenLMState state = (KenLMState) tailNodes.get(index).getDPState(stateIndex);
++ kenIds[x] = -state.getState();
++ }
++
++ } else {
++ // Terminal: just add it
++ kenIds[x] = id;
++ }
++ }
++ return kenIds;
++ }
++
++ /**
+ * Destroys the pool created to allocate state for this sentence. Called from the
+ * {@link org.apache.joshua.decoder.Translation} class after outputting the sentence or k-best list. Hosting
+ * this map here in KenLMFF statically allows pools to be shared across KenLM instances.
- *
++ *
+ * @param sentId a key in the poolmap table to destroy
+ */
+ public void destroyPool(int sentId) {
+ if (poolMap.containsKey(sentId))
+ KenLM.destroyPool(poolMap.get(sentId));
+ poolMap.remove(sentId);
+ }
+
+ /**
+ * This function differs from regular transitions because we incorporate the cost of incomplete
+ * left-hand ngrams, as well as including the start- and end-of-sentence markers (if they were
+ * requested when the object was created).
- *
++ *
+ * KenLM already includes the prefix probabilities (of shorter n-grams on the left-hand side), so
+ * there's nothing that needs to be done.
+ */
+ @Override
+ public DPState computeFinal(HGNode tailNode, int i, int j, SourcePath sourcePath, Sentence sentence,
+ Accumulator acc) {
-
- // KenLMState state = (KenLMState) tailNode.getDPState(getStateIndex());
-
- // This is unnecessary
- // acc.add(name, 0.0f);
-
+ // The state is the same since no rule was applied
+ return new KenLMState();
+ }
+
+ /**
+ * KenLM probs already include the prefix probabilities (they are substracted out when merging
+ * states), so this doesn't need to do anything.
+ */
+ @Override
+ public float estimateFutureCost(Rule rule, DPState currentState, Sentence sentence) {
+ return 0.0f;
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/AbstractGrammar.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/AbstractGrammar.java
index 0000000,6c512bd..bc66331
mode 000000,100644..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/AbstractGrammar.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/AbstractGrammar.java
@@@ -1,0 -1,228 +1,229 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.decoder.ff.tm;
+
+ import java.util.Arrays;
+ import java.util.HashSet;
+ import java.util.List;
+
+ import org.apache.joshua.corpus.Vocabulary;
+ import org.apache.joshua.decoder.JoshuaConfiguration;
+ import org.apache.joshua.decoder.ff.FeatureFunction;
+ import org.apache.joshua.decoder.phrase.PhraseTable;
+ import org.apache.joshua.decoder.segment_file.Token;
+ import org.apache.joshua.lattice.Arc;
+ import org.apache.joshua.lattice.Lattice;
+ import org.apache.joshua.lattice.Node;
-
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
++import cern.colt.Arrays;
++
+ /**
+ * Partial implementation of the <code>Grammar</code> interface that provides logic for sorting a
+ * grammar.
+ * <p>
+ * <em>Note</em>: New classes implementing the <code>Grammar</code> interface should probably
+ * inherit from this class, unless a specific sorting technique different from that implemented by
+ * this class is required.
+ *
+ * @author Zhifei Li
+ * @author Lane Schwartz
+ * @author Matt Post post@cs.jhu.edu
+ */
+ public abstract class AbstractGrammar implements Grammar {
+
+ /** Logger for this class. */
+ private static final Logger LOG = LoggerFactory.getLogger(AbstractGrammar.class);
+ /**
+ * Indicates whether the rules in this grammar have been sorted based on the latest feature
+ * function values.
+ */
+ protected boolean sorted = false;
+
+ /*
+ * The grammar's owner, used to determine which weights are applicable to the dense features found
+ * within.
+ */
- protected int owner = -1;
-
++ protected final OwnerId owner;
++
+ /*
+ * The maximum length of a source-side phrase. Mostly used by the phrase-based decoder.
+ */
+ protected int maxSourcePhraseLength = -1;
+
+ /**
+ * Returns the longest source phrase read.
+ *
+ * @return the longest source phrase read (nonterminal + terminal symbols).
+ */
+ @Override
+ public int getMaxSourcePhraseLength() {
+ return maxSourcePhraseLength;
+ }
+
+ @Override
- public int getOwner() {
++ public OwnerId getOwner() {
+ return owner;
+ }
++
++ public int getSpanLimit() {
++ return spanLimit;
++ }
+
- /* The maximum span of the input this rule can be applied to. */
- protected int spanLimit = 1;
++ /* The maximum span of the input this grammar rules can be applied to. */
++ protected final int spanLimit;
+
- protected JoshuaConfiguration joshuaConfiguration;
++ protected final JoshuaConfiguration joshuaConfiguration;
+
+ /**
- * Constructs an empty, unsorted grammar.
- *
++ * Creates an empty, unsorted grammar with given owner and spanlimit
++ *
+ * @see Grammar#isSorted()
+ * @param config a {@link org.apache.joshua.decoder.JoshuaConfiguration} object
+ */
- public AbstractGrammar(JoshuaConfiguration config) {
- this.joshuaConfiguration = config;
++ public AbstractGrammar(final String owner, final JoshuaConfiguration config, final int spanLimit) {
+ this.sorted = false;
- }
-
- public AbstractGrammar(int owner, int spanLimit) {
- this.sorted = false;
- this.owner = owner;
++ this.owner = OwnerMap.register(owner);
++ this.joshuaConfiguration = config;
+ this.spanLimit = spanLimit;
+ }
+
+ public static final int OOV_RULE_ID = 0;
+
+ /**
+ * Cube-pruning requires that the grammar be sorted based on the latest feature functions. To
+ * avoid synchronization, this method should be called before multiple threads are initialized for
+ * parallel decoding
+ * @param models {@link java.util.List} of {@link org.apache.joshua.decoder.ff.FeatureFunction}'s
+ */
+ public void sortGrammar(List<FeatureFunction> models) {
+ Trie root = getTrieRoot();
+ if (root != null) {
+ sort(root, models);
+ setSorted(true);
+ }
+ }
+
+ /* See Javadoc comments for Grammar interface. */
+ public boolean isSorted() {
+ return sorted;
+ }
+
+ /**
+ * Sets the flag indicating whether this grammar is sorted.
+ * <p>
+ * This method is called by {@link org.apache.joshua.decoder.ff.tm.AbstractGrammar#sortGrammar(List)}
+ * to indicate that the grammar has been sorted.</p>
+ *
+ * <p>Its scope is protected so that child classes that override <code>sortGrammar</code> will also
+ * be able to call this method to indicate that the grammar has been sorted.</p>
+ *
+ * @param sorted set to true if the grammar is sorted
+ */
+ protected void setSorted(boolean sorted) {
+ this.sorted = sorted;
+ LOG.debug("This grammar is now sorted: {}", this);
+ }
+
+ /**
+ * Recursively sorts the grammar using the provided feature functions.
+ * <p>
+ * This method first sorts the rules stored at the provided node, then recursively calls itself on
+ * the child nodes of the provided node.
+ *
+ * @param node Grammar node in the <code>Trie</code> whose rules should be sorted.
+ * @param models Feature function models to use during sorting.
+ */
+ private void sort(Trie node, List<FeatureFunction> models) {
+
+ if (node != null) {
+ if (node.hasRules()) {
+ RuleCollection rules = node.getRuleCollection();
+ LOG.debug("Sorting node {}", Arrays.toString(rules.getSourceSide()));
+
+ /* This causes the rules at this trie node to be sorted */
+ rules.getSortedRules(models);
+
+ if (LOG.isDebugEnabled()) {
+ StringBuilder s = new StringBuilder();
+ for (Rule r : rules.getSortedRules(models)) {
+ s.append("\n\t" + r.getLHS() + " ||| " + Arrays.toString(r.getFrench()) + " ||| "
+ + Arrays.toString(r.getEnglish()) + " ||| " + r.getFeatureVector() + " ||| "
+ + r.getEstimatedCost() + " " + r.getClass().getName() + "@"
+ + Integer.toHexString(System.identityHashCode(r)));
+ }
+ LOG.debug("{}", s);
+ }
+ }
+
+ if (node.hasExtensions()) {
+ for (Trie child : node.getExtensions()) {
+ sort(child, models);
+ }
+ } else {
+ LOG.debug("Node has 0 children to extend: {}", node);
+ }
+ }
+ }
+
+ // write grammar to disk
+ public void writeGrammarOnDisk(String file) {
+ }
+
+ /**
+ * Adds OOV rules for all words in the input lattice to the current grammar. Uses addOOVRule() so that
+ * sub-grammars can define different types of OOV rules if needed (as is used in {@link PhraseTable}).
+ *
+ * @param grammar Grammar in the Trie
+ * @param inputLattice the lattice representing the input sentence
+ * @param featureFunctions a list of feature functions used for scoring
+ * @param onlyTrue determine if word is actual OOV.
+ */
+ public static void addOOVRules(Grammar grammar, Lattice<Token> inputLattice,
+ List<FeatureFunction> featureFunctions, boolean onlyTrue) {
+ /*
+ * Add OOV rules; This should be called after the manual constraints have
+ * been set up.
+ */
+ HashSet<Integer> words = new HashSet<Integer>();
+ for (Node<Token> node : inputLattice) {
+ for (Arc<Token> arc : node.getOutgoingArcs()) {
+ // create a rule, but do not add into the grammar trie
+ // TODO: which grammar should we use to create an OOV rule?
+ int sourceWord = arc.getLabel().getWord();
+ if (sourceWord == Vocabulary.id(Vocabulary.START_SYM)
+ || sourceWord == Vocabulary.id(Vocabulary.STOP_SYM))
+ continue;
+
+ // Determine if word is actual OOV.
+ if (onlyTrue && ! Vocabulary.hasId(sourceWord))
+ continue;
+
+ words.add(sourceWord);
+ }
+ }
+
+ for (int sourceWord: words)
+ grammar.addOOVRules(sourceWord, featureFunctions);
+
+ // Sort all the rules (not much to actually do, this just marks it as sorted)
+ grammar.sortGrammar(featureFunctions);
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/Grammar.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/Grammar.java
index 0000000,06252a1..8f90d1b
mode 000000,100644..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/Grammar.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/Grammar.java
@@@ -1,0 -1,120 +1,120 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.decoder.ff.tm;
+
+ import java.util.List;
+
+ import org.apache.joshua.decoder.ff.FeatureFunction;
+
+ /**
+ * Grammar is a class for wrapping a trie of TrieGrammar in order to store holistic metadata.
+ *
+ * @author wren ng thornton wren@users.sourceforge.net
+ * @author Zhifei Li, zhifei.work@gmail.com
+ */
+ public interface Grammar {
+
+ /**
+ * Gets the root of the <code>Trie</code> backing this grammar.
+ * <p>
+ * <em>Note</em>: This method should run as a small constant-time function.
+ *
+ * @return the root of the <code>Trie</code> backing this grammar
+ */
+ Trie getTrieRoot();
+
+ /**
+ * After calling this method, the rules in this grammar are guaranteed to be sorted based on the
+ * latest feature function values.
+ * <p>
+ * Cube-pruning requires that the grammar be sorted based on the latest feature functions.
+ *
+ * @param models list of {@link org.apache.joshua.decoder.ff.FeatureFunction}'s
+ */
+ void sortGrammar(List<FeatureFunction> models);
+
+ /**
+ * Determines whether the rules in this grammar have been sorted based on the latest feature
+ * function values.
+ * <p>
+ * This method is needed for the cube-pruning algorithm.
+ *
+ * @return <code>true</code> if the rules in this grammar have been sorted based on the latest
+ * feature function values, <code>false</code> otherwise
+ */
+ boolean isSorted();
+
+ /**
+ * Returns whether this grammar has any valid rules for covering a particular span of a sentence.
+ * Hiero's "glue" grammar will only say True if the span is longer than our span limit, and is
+ * anchored at startIndex==0. Hiero's "regular" grammar will only say True if the span is less
+ * than the span limit. Other grammars, e.g. for rule-based systems, may have different behaviors.
+ *
+ * @param startIndex Indicates the starting index of a phrase in a source input phrase, or a
+ * starting node identifier in a source input lattice
+ * @param endIndex Indicates the ending index of a phrase in a source input phrase, or an ending
+ * node identifier in a source input lattice
+ * @param pathLength Length of the input path in a source input lattice. If a source input phrase
+ * is used instead of a lattice, this value will likely be ignored by the underlying
+ * implementation, but would normally be defined as <code>endIndex-startIndex</code>
+ * @return true if there is a rule for this span
+ */
+ boolean hasRuleForSpan(int startIndex, int endIndex, int pathLength);
+
+ /**
+ * Gets the number of rules stored in the grammar.
+ *
+ * @return the number of rules stored in the grammar
+ */
+ int getNumRules();
+
+ /**
+ * Returns the number of dense features.
+ *
+ * @return the number of dense features
+ */
+ int getNumDenseFeatures();
+
+ /**
+ * Return the grammar's owner.
+ * @return grammar owner
+ */
- int getOwner();
++ OwnerId getOwner();
+
+ /**
+ * Return the maximum source phrase length (terminals + nonterminals)
+ * @return the maximum source phrase length
+ */
+ int getMaxSourcePhraseLength();
+
+ /**
+ * Add an OOV rule for the requested word for the grammar.
+ *
+ * @param word input word to add rules to
+ * @param featureFunctions a {@link java.util.List} of {@link org.apache.joshua.decoder.ff.FeatureFunction}'s
+ */
+ void addOOVRules(int word, List<FeatureFunction> featureFunctions);
+
+ /**
+ * Add a rule to the grammar.
+ *
+ * @param rule the {@link org.apache.joshua.decoder.ff.tm.Rule}
+ */
+ void addRule(Rule rule);
+ }
[50/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/resources/kbest_extraction/grammar
----------------------------------------------------------------------
diff --git a/joshua-core/resources/kbest_extraction/grammar b/joshua-core/resources/kbest_extraction/grammar
new file mode 100644
index 0000000..a03b2d9
--- /dev/null
+++ b/joshua-core/resources/kbest_extraction/grammar
@@ -0,0 +1,25 @@
+[X] ||| a ||| A ||| 2
+[X] ||| a ||| B ||| 3
+[X] ||| a ||| C ||| 5
+[X] ||| a ||| D ||| 7
+[X] ||| a ||| E ||| 11
+[X] ||| b ||| A ||| 13
+[X] ||| b ||| B ||| 17
+[X] ||| b ||| C ||| 19
+[X] ||| b ||| D ||| 23
+[X] ||| b ||| E ||| 29
+[X] ||| c ||| A ||| 31
+[X] ||| c ||| B ||| 37
+[X] ||| c ||| C ||| 41
+[X] ||| c ||| D ||| 43
+[X] ||| c ||| E ||| 47
+[X] ||| d ||| A ||| 53
+[X] ||| d ||| B ||| 59
+[X] ||| d ||| C ||| 61
+[X] ||| d ||| D ||| 67
+[X] ||| d ||| E ||| 71
+[X] ||| e ||| A ||| 73
+[X] ||| e ||| B ||| 79
+[X] ||| e ||| C ||| 83
+[X] ||| e ||| D ||| 89
+[X] ||| e ||| E ||| 97
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/resources/kbest_extraction/joshua.config
----------------------------------------------------------------------
diff --git a/joshua-core/resources/kbest_extraction/joshua.config b/joshua-core/resources/kbest_extraction/joshua.config
new file mode 100644
index 0000000..cdab98e
--- /dev/null
+++ b/joshua-core/resources/kbest_extraction/joshua.config
@@ -0,0 +1,27 @@
+feature-function = StateMinimizingLanguageModel -lm_type kenlm -lm_order 5 -lm_file resources/kbest_extraction/lm.gz
+
+tm = thrax -owner pt -maxspan 12 -path resources/kbest_extraction/grammar
+tm = thrax -owner glue -maxspan -1 -path resources/kbest_extraction/glue-grammar
+
+mark_oovs=false
+
+#tm config
+default_non_terminal=X
+goalSymbol=GOAL
+
+#pruning config
+pop-limit=100
+
+#nbest config
+use_unique_nbest=true
+top-n = 3126
+
+#feature_function = WordPenalty
+feature_function = OOVPenalty
+
+# Model Weights ####
+
+lm_0 1
+tm_pt_0 1
+tm_glue_0 1
+OOVPenalty 10000
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/resources/kbest_extraction/lm.gz
----------------------------------------------------------------------
diff --git a/joshua-core/resources/kbest_extraction/lm.gz b/joshua-core/resources/kbest_extraction/lm.gz
new file mode 100644
index 0000000..a26335e
Binary files /dev/null and b/joshua-core/resources/kbest_extraction/lm.gz differ
[07/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/packed/grammar.glue
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/packed/grammar.glue b/joshua-core/src/test/resources/bn-en/packed/grammar.glue
new file mode 100644
index 0000000..d6667cd
--- /dev/null
+++ b/joshua-core/src/test/resources/bn-en/packed/grammar.glue
@@ -0,0 +1,5673 @@
+[GOAL] ||| <s> ||| <s> ||| 0
+[GOAL] ||| [GOAL,1] [SYM+RB+DT,2] ||| [GOAL,1] [SYM+RB+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+NN+IN,2] ||| [GOAL,1] [:+NN+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+.,2] ||| [GOAL,1] [CC+NP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [RP+IN+DT,2] ||| [GOAL,1] [RP+IN+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+:,2] ||| [GOAL,1] [CC+NP+:,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+PP+DT,2] ||| [GOAL,1] [PP+PP+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+VBP+DT,2] ||| [GOAL,1] [PP+VBP+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VBZ+WRB,2] ||| [GOAL,1] [NN+VBZ+WRB,2] ||| -1
+[GOAL] ||| [GOAL,1] [POS+CC,2] ||| [GOAL,1] [POS+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [NX+COMMA+ADJP,2] ||| [GOAL,1] [NX+COMMA+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+FW,2] ||| [GOAL,1] [:+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+RB+.,2] ||| [GOAL,1] [PP+RB+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+IN+OOV,2] ||| [GOAL,1] [COMMA+IN+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+:+NP,2] ||| [GOAL,1] [CD+:+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+PP+.,2] ||| [GOAL,1] [VBN+PP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+MD,2] ||| [GOAL,1] [CC+NP+MD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+EX,2] ||| [GOAL,1] [NN+EX,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+NN+NP,2] ||| [GOAL,1] [SYM+NN+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJS+WDT,2] ||| [GOAL,1] [JJS+WDT,2] ||| -1
+[GOAL] ||| [GOAL,1] [OOV+COMMA+OOV,2] ||| [GOAL,1] [OOV+COMMA+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+CD+.,2] ||| [GOAL,1] [CD+CD+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+VP+.,2] ||| [GOAL,1] [CC+VP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+RB,2] ||| [GOAL,1] [CD+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+.,2] ||| [GOAL,1] [NP+VBD+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+RB+ADVP,2] ||| [GOAL,1] [COMMA+RB+ADVP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+RB+VP,2] ||| [GOAL,1] [NN+RB+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+:,2] ||| [GOAL,1] [NP+VBD+:,2] ||| -1
+[GOAL] ||| [GOAL,1] [.,2] ||| [GOAL,1] [.,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+SYM,2] ||| [GOAL,1] [RB+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+FW+SYM,2] ||| [GOAL,1] [PP+FW+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJS+NN+VBZ,2] ||| [GOAL,1] [JJS+NN+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+VBG+COMMA,2] ||| [GOAL,1] [CC+VBG+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR+COMMA+VB,2] ||| [GOAL,1] [SBAR+COMMA+VB,2] ||| -1
+[GOAL] ||| [GOAL,1] [QP+PRP+$,2] ||| [GOAL,1] [QP+PRP+$,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+NN,2] ||| [GOAL,1] [CC+NP+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VB+RB,2] ||| [GOAL,1] [NN+VB+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP/'',2] ||| [GOAL,1] [ADJP/'',2] ||| -1
+[GOAL] ||| [GOAL,1] [JJS+NN+VBP,2] ||| [GOAL,1] [JJS+NN+VBP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+PP+CC,2] ||| [GOAL,1] [PP+PP+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [:,2] ||| [GOAL,1] [:,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+RB+NP,2] ||| [GOAL,1] [NNS+RB+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+VBP+DT,2] ||| [GOAL,1] [PRP+VBP+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+FRAG,2] ||| [GOAL,1] [NN+FRAG,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+FW,2] ||| [GOAL,1] [NN+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+JJ+CD,2] ||| [GOAL,1] [NP+JJ+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+JJ+CC,2] ||| [GOAL,1] [NP+JJ+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+DT+ADJP,2] ||| [GOAL,1] [VBD+DT+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [.+OOV+JJ,2] ||| [GOAL,1] [.+OOV+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+CD,2] ||| [GOAL,1] [NN+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+:+NN,2] ||| [GOAL,1] [VBD+:+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+COMMA+IN,2] ||| [GOAL,1] [PP+COMMA+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+VBZ+CD,2] ||| [GOAL,1] [PRP+VBZ+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+:+NP,2] ||| [GOAL,1] [VBD+:+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [OOV\S,2] ||| [GOAL,1] [OOV\S,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+NP,2] ||| [GOAL,1] [CC+NP+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+CC,2] ||| [GOAL,1] [NN+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [OOV\X,2] ||| [GOAL,1] [OOV\X,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+IN+CC,2] ||| [GOAL,1] [NN+IN+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [OOV+SYM+RB,2] ||| [GOAL,1] [OOV+SYM+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [.+DT,2] ||| [GOAL,1] [.+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NN+WP$,2] ||| [GOAL,1] [JJ+NN+WP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [S,2] ||| [GOAL,1] [S,2] ||| -1
+[GOAL] ||| [GOAL,1] [FRAG+COMMA+DT,2] ||| [GOAL,1] [FRAG+COMMA+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+IN+CD,2] ||| [GOAL,1] [NN+IN+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+PRP+VBD,2] ||| [GOAL,1] [ADVP+PRP+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+RB+DT,2] ||| [GOAL,1] [RB+RB+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+RB+CD,2] ||| [GOAL,1] [SYM+RB+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+OOV+COMMA,2] ||| [GOAL,1] [VB+OOV+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+CD+NNP,2] ||| [GOAL,1] [DT+CD+NNP,2] ||| -1
+[GOAL] ||| [GOAL,1] [X,2] ||| [GOAL,1] [X,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP/S,2] ||| [GOAL,1] [PP/S,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+TO,2] ||| [GOAL,1] [CD+TO,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+:+PP,2] ||| [GOAL,1] [CD+:+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+DT,2] ||| [GOAL,1] [:+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNP+CC+IN,2] ||| [GOAL,1] [NNP+CC+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+:+VBP,2] ||| [GOAL,1] [PP+:+VBP,2] ||| -1
+[GOAL] ||| [GOAL,1] [S+CC+RB,2] ||| [GOAL,1] [S+CC+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+VBD+SYM,2] ||| [GOAL,1] [NNS+VBD+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+:+VP,2] ||| [GOAL,1] [PP+:+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+SYM,2] ||| [GOAL,1] [PP+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+PP,2] ||| [GOAL,1] [CC+NP+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+VBN+DT,2] ||| [GOAL,1] [VBD+VBN+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [OOV+POS,2] ||| [GOAL,1] [OOV+POS,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+DT,2] ||| [GOAL,1] [NN+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+COMMA+CD,2] ||| [GOAL,1] [PP+COMMA+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+CD,2] ||| [GOAL,1] [:+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [POS+NX+CC,2] ||| [GOAL,1] [POS+NX+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+COMMA+CC,2] ||| [GOAL,1] [PP+COMMA+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+NX,2] ||| [GOAL,1] [CD+NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+CC,2] ||| [GOAL,1] [:+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBG\SINV,2] ||| [GOAL,1] [VBG\SINV,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+NP+NP,2] ||| [GOAL,1] [ADVP+NP+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+MD+VB,2] ||| [GOAL,1] [CC+MD+VB,2] ||| -1
+[GOAL] ||| [GOAL,1] [WRB+VBZ,2] ||| [GOAL,1] [WRB+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [FW+VP+'',2] ||| [GOAL,1] [FW+VP+'',2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VBP+NNS,2] ||| [GOAL,1] [NN+VBP+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+VBZ+DT,2] ||| [GOAL,1] [PRP+VBZ+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+CD,2] ||| [GOAL,1] [VBD+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+COMMA+ADVP,2] ||| [GOAL,1] [RB+COMMA+ADVP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+IN,2] ||| [GOAL,1] [NN+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+NP,2] ||| [GOAL,1] [CD+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+NN,2] ||| [GOAL,1] [CD+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [TO+VB+DT,2] ||| [GOAL,1] [TO+VB+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+SYM+FW,2] ||| [GOAL,1] [:+SYM+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+CC,2] ||| [GOAL,1] [VBD+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+JJ,2] ||| [GOAL,1] [NN+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR+COMMA+RB,2] ||| [GOAL,1] [SBAR+COMMA+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+COMMA+PRP$,2] ||| [GOAL,1] [PP+COMMA+PRP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [POS+WDT+VP,2] ||| [GOAL,1] [POS+WDT+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+SYM,2] ||| [GOAL,1] [NP+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+PP+WRB,2] ||| [GOAL,1] [ADVP+PP+WRB,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+:+DT,2] ||| [GOAL,1] [VBP+:+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+NN+JJ,2] ||| [GOAL,1] [SYM+NN+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NNS+.,2] ||| [GOAL,1] [CC+NNS+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [TO+VB+CC,2] ||| [GOAL,1] [TO+VB+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+COMMA+SYM,2] ||| [GOAL,1] [PP+COMMA+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VP+CC,2] ||| [GOAL,1] [COMMA+VP+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+RB,2] ||| [GOAL,1] [CC+NP+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [NX+OOV,2] ||| [GOAL,1] [NX+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+VBP+JJ,2] ||| [GOAL,1] [PP+VBP+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+COMMA+SYM,2] ||| [GOAL,1] [NP+COMMA+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+MD,2] ||| [GOAL,1] [CD+MD,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NNS+:,2] ||| [GOAL,1] [CC+NNS+:,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+VBP+.,2] ||| [GOAL,1] [NNS+VBP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [TO+VBG,2] ||| [GOAL,1] [TO+VBG,2] ||| -1
+[GOAL] ||| [GOAL,1] [OOV+NNS+COMMA,2] ||| [GOAL,1] [OOV+NNS+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+X,2] ||| [GOAL,1] [NP+VBD+X,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+$+PRN,2] ||| [GOAL,1] [PRP+$+PRN,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+PP+VP,2] ||| [GOAL,1] [ADVP+PP+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+NN+CC,2] ||| [GOAL,1] [VBD+NN+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ\NP,2] ||| [GOAL,1] [JJ\NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+DT,2] ||| [GOAL,1] [VBD+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [''+COMMA,2] ||| [GOAL,1] [''+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+RB+VP,2] ||| [GOAL,1] [NNS+RB+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+WDT+VBZ,2] ||| [GOAL,1] [PP+WDT+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+DT+ADJP,2] ||| [GOAL,1] [NP+DT+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [''+NN+COMMA,2] ||| [GOAL,1] [''+NN+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBZ+OOV,2] ||| [GOAL,1] [VBZ+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+VBP+:,2] ||| [GOAL,1] [NNS+VBP+:,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB\INTJ,2] ||| [GOAL,1] [VB\INTJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+PP,2] ||| [GOAL,1] [CD+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [TO+VBN,2] ||| [GOAL,1] [TO+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+VBP+SYM,2] ||| [GOAL,1] [SYM+VBP+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+:+FW,2] ||| [GOAL,1] [VBP+:+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBP+VBG,2] ||| [GOAL,1] [NP+VBP+VBG,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+WDT,2] ||| [GOAL,1] [JJ+WDT,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NX+CD,2] ||| [GOAL,1] [CC+NX+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NX+CC,2] ||| [GOAL,1] [CC+NX+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+ADJP+JJ,2] ||| [GOAL,1] [NN+ADJP+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+VBD+VBN,2] ||| [GOAL,1] [PP+VBD+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [WP+VBG+IN,2] ||| [GOAL,1] [WP+VBG+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+NN+VBZ,2] ||| [GOAL,1] [SYM+NN+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+VBN+TO,2] ||| [GOAL,1] [RB+VBN+TO,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+JJR+JJ,2] ||| [GOAL,1] [VBP+JJR+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [.+JJ,2] ||| [GOAL,1] [.+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+NN+NN,2] ||| [GOAL,1] [:+NN+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+ADJP+COMMA,2] ||| [GOAL,1] [COMMA+ADJP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+NP+ADVP,2] ||| [GOAL,1] [COMMA+NP+ADVP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ\NX,2] ||| [GOAL,1] [JJ\NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+COMMA+DT,2] ||| [GOAL,1] [PP+COMMA+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+VBZ+NN,2] ||| [GOAL,1] [JJ+VBZ+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+VBZ+NP,2] ||| [GOAL,1] [JJ+VBZ+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [OOV+CC+VBN,2] ||| [GOAL,1] [OOV+CC+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBP+VBN,2] ||| [GOAL,1] [NP+VBP+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [POS+JJ,2] ||| [GOAL,1] [POS+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+SBAR+.,2] ||| [GOAL,1] [COMMA+SBAR+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+NP+COMMA,2] ||| [GOAL,1] [VB+NP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+NP+IN,2] ||| [GOAL,1] [ADVP+NP+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBN+IN,2] ||| [GOAL,1] [NP+VBN+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+NP+:,2] ||| [GOAL,1] [NP+NP+:,2] ||| -1
+[GOAL] ||| [GOAL,1] [UCP+NNS,2] ||| [GOAL,1] [UCP+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+COMMA+PP,2] ||| [GOAL,1] [JJ+COMMA+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+JJ,2] ||| [GOAL,1] [CD+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+NN+VBZ,2] ||| [GOAL,1] [PP+NN+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+DT+PRP$,2] ||| [GOAL,1] [CC+DT+PRP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+VBN+PP,2] ||| [GOAL,1] [RB+VBN+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [FRAG+COMMA+NP,2] ||| [GOAL,1] [FRAG+COMMA+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+MD,2] ||| [GOAL,1] [NN+MD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+NP+.,2] ||| [GOAL,1] [NP+NP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+DT+RB,2] ||| [GOAL,1] [CC+DT+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+RBR+NP,2] ||| [GOAL,1] [NN+RBR+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [LS+SYM,2] ||| [GOAL,1] [LS+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+DT,2] ||| [GOAL,1] [CC+NP+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+JJ+JJR,2] ||| [GOAL,1] [JJ+JJ+JJR,2] ||| -1
+[GOAL] ||| [GOAL,1] [CONJP+NP+IN,2] ||| [GOAL,1] [CONJP+NP+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+VP+COMMA,2] ||| [GOAL,1] [JJ+VP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+RB+NP,2] ||| [GOAL,1] [RB+RB+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+IN,2] ||| [GOAL,1] [CD+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBN+JJ,2] ||| [GOAL,1] [NP+VBN+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBZ+CD+IN,2] ||| [GOAL,1] [VBZ+CD+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+NX,2] ||| [GOAL,1] [NN+NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [LST+OOV+JJ,2] ||| [GOAL,1] [LST+OOV+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+VBP+NN,2] ||| [GOAL,1] [PP+VBP+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+VBP+NP,2] ||| [GOAL,1] [PP+VBP+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [TO+VB+OOV,2] ||| [GOAL,1] [TO+VB+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP+COMMA+VBZ,2] ||| [GOAL,1] [ADJP+COMMA+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+''+JJ,2] ||| [GOAL,1] [:+''+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [''+NN+WDT,2] ||| [GOAL,1] [''+NN+WDT,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+JJ+NNS,2] ||| [GOAL,1] [COMMA+JJ+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJR+COMMA,2] ||| [GOAL,1] [JJR+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [POS+VP+.,2] ||| [GOAL,1] [POS+VP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [.+CD+JJ,2] ||| [GOAL,1] [.+CD+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+JJ+NNS,2] ||| [GOAL,1] [IN+JJ+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+COMMA+NP,2] ||| [GOAL,1] [PP+COMMA+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [FW+WRB,2] ||| [GOAL,1] [FW+WRB,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+COMMA,2] ||| [GOAL,1] [IN+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [X/CD,2] ||| [GOAL,1] [X/CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+COMMA+NN,2] ||| [GOAL,1] [PP+COMMA+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+NNS+VBD,2] ||| [GOAL,1] [NP+NNS+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+NN,2] ||| [GOAL,1] [NN+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+NP,2] ||| [GOAL,1] [NN+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+VBD+OOV,2] ||| [GOAL,1] [PRP+VBD+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP+NN+JJR,2] ||| [GOAL,1] [ADJP+NN+JJR,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP,2] ||| [GOAL,1] [ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+CC+NN,2] ||| [GOAL,1] [VB+CC+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+:+FW,2] ||| [GOAL,1] [JJ+:+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+COMMA,2] ||| [GOAL,1] [NN+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+POS,2] ||| [GOAL,1] [NN+POS,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR+COMMA+NP,2] ||| [GOAL,1] [SBAR+COMMA+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [WRB+FW+CC,2] ||| [GOAL,1] [WRB+FW+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+COMMA+RB,2] ||| [GOAL,1] [PP+COMMA+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+FW+FW,2] ||| [GOAL,1] [VBP+FW+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR+COMMA+NN,2] ||| [GOAL,1] [SBAR+COMMA+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [WRB+NNS,2] ||| [GOAL,1] [WRB+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+NN+IN,2] ||| [GOAL,1] [SYM+NN+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+FW,2] ||| [GOAL,1] [CC+NP+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [TO+VB+JJ,2] ||| [GOAL,1] [TO+VB+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+:+JJ,2] ||| [GOAL,1] [ADVP+:+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+WRB,2] ||| [GOAL,1] [JJ+WRB,2] ||| -1
+[GOAL] ||| [GOAL,1] [''+ADJP+NNS,2] ||| [GOAL,1] [''+ADJP+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+COMMA+QP,2] ||| [GOAL,1] [PP+COMMA+QP,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+OOV,2] ||| [GOAL,1] [DT+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+IN+JJ,2] ||| [GOAL,1] [NN+IN+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NX+FW,2] ||| [GOAL,1] [CC+NX+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VB+COMMA,2] ||| [GOAL,1] [COMMA+VB+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP+JJ+ADJP,2] ||| [GOAL,1] [ADJP+JJ+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [TO+VB+IN,2] ||| [GOAL,1] [TO+VB+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+SBAR+VP,2] ||| [GOAL,1] [NN+SBAR+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+ADJP+NN,2] ||| [GOAL,1] [NN+ADJP+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP+VBZ+CD,2] ||| [GOAL,1] [ADJP+VBZ+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [.+NP+COMMA,2] ||| [GOAL,1] [.+NP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+IN+VBG,2] ||| [GOAL,1] [NN+IN+VBG,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+TO+CC,2] ||| [GOAL,1] [JJ+TO+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+''+NP,2] ||| [GOAL,1] [COMMA+''+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+WRB+NP,2] ||| [GOAL,1] [NP+WRB+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN\VP,2] ||| [GOAL,1] [IN\VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+NN+VBD,2] ||| [GOAL,1] [PP+NN+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+DT+NNS,2] ||| [GOAL,1] [NN+DT+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+IN+IN,2] ||| [GOAL,1] [NN+IN+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+.+``,2] ||| [GOAL,1] [NP+.+``,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+NN+NNS,2] ||| [GOAL,1] [SYM+NN+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP/SYM,2] ||| [GOAL,1] [NP/SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+COMMA+PP,2] ||| [GOAL,1] [PP+COMMA+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [WRB+NNS+IN,2] ||| [GOAL,1] [WRB+NNS+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+VBZ+CD,2] ||| [GOAL,1] [JJ+VBZ+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+RB,2] ||| [GOAL,1] [NN+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+ADJP+.,2] ||| [GOAL,1] [RB+ADJP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+JJ+JJ,2] ||| [GOAL,1] [VBN+JJ+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+NNS+COMMA,2] ||| [GOAL,1] [NN+NNS+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBG+TO+DT,2] ||| [GOAL,1] [VBG+TO+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+PP+ADVP,2] ||| [GOAL,1] [VBD+PP+ADVP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+COMMA,2] ||| [GOAL,1] [VB+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+TO+DT,2] ||| [GOAL,1] [JJ+TO+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+JJ+RBS,2] ||| [GOAL,1] [JJ+JJ+RBS,2] ||| -1
+[GOAL] ||| [GOAL,1] [MD\S,2] ||| [GOAL,1] [MD\S,2] ||| -1
+[GOAL] ||| [GOAL,1] [QP+NNS,2] ||| [GOAL,1] [QP+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+QP,2] ||| [GOAL,1] [NN+QP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VBN+NNS,2] ||| [GOAL,1] [COMMA+VBN+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+OOV+CC,2] ||| [GOAL,1] [NN+OOV+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+NP+VP,2] ||| [GOAL,1] [RB+NP+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+QP+JJ,2] ||| [GOAL,1] [PP+QP+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+RB+JJ,2] ||| [GOAL,1] [RB+RB+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+SBAR+RB,2] ||| [GOAL,1] [COMMA+SBAR+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+NN+CC,2] ||| [GOAL,1] [SYM+NN+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+NN+CD,2] ||| [GOAL,1] [SYM+NN+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+IN+CD,2] ||| [GOAL,1] [NNS+IN+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [WRB+COMMA+SYM,2] ||| [GOAL,1] [WRB+COMMA+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [.+CD,2] ||| [GOAL,1] [.+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+''+.,2] ||| [GOAL,1] [JJ+''+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NX+JJ,2] ||| [GOAL,1] [CC+NX+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+FW,2] ||| [GOAL,1] [CD+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+COMMA+VBZ,2] ||| [GOAL,1] [JJ+COMMA+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+IN+DT,2] ||| [GOAL,1] [NN+IN+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN\QP,2] ||| [GOAL,1] [IN\QP,2] ||| -1
+[GOAL] ||| [GOAL,1] [X/FW,2] ||| [GOAL,1] [X/FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+''+NP,2] ||| [GOAL,1] [:+''+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+WDT,2] ||| [GOAL,1] [IN+WDT,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+:+CC,2] ||| [GOAL,1] [CD+:+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NN+WRB,2] ||| [GOAL,1] [JJ+NN+WRB,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR+COMMA+IN,2] ||| [GOAL,1] [SBAR+COMMA+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBN+NN,2] ||| [GOAL,1] [NP+VBN+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+IN+DT,2] ||| [GOAL,1] [NNS+IN+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBN+NP,2] ||| [GOAL,1] [NP+VBN+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+JJ,2] ||| [GOAL,1] [CC+NP+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJR\NP,2] ||| [GOAL,1] [JJR\NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+IN,2] ||| [GOAL,1] [CC+NP+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+VBP+RB,2] ||| [GOAL,1] [PP+VBP+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NX+IN,2] ||| [GOAL,1] [CC+NX+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [VP/NNS,2] ||| [GOAL,1] [VP/NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+RB+IN,2] ||| [GOAL,1] [RB+RB+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+COMMA+JJ,2] ||| [GOAL,1] [PP+COMMA+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+VBZ+DT,2] ||| [GOAL,1] [JJ+VBZ+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+QP+JJ,2] ||| [GOAL,1] [RB+QP+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+RB+SYM,2] ||| [GOAL,1] [JJ+RB+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+ADVP+COMMA,2] ||| [GOAL,1] [COMMA+ADVP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [CONJP+NP,2] ||| [GOAL,1] [CONJP+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN\PP,2] ||| [GOAL,1] [IN\PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+SYM+NP,2] ||| [GOAL,1] [:+SYM+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [.+CD+NN,2] ||| [GOAL,1] [.+CD+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+NNS+COMMA,2] ||| [GOAL,1] [NNS+NNS+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+ADJP,2] ||| [GOAL,1] [SYM+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+COMMA+NX,2] ||| [GOAL,1] [JJ+COMMA+NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [VP+PRP+NN,2] ||| [GOAL,1] [VP+PRP+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+SBAR+TO,2] ||| [GOAL,1] [COMMA+SBAR+TO,2] ||| -1
+[GOAL] ||| [GOAL,1] [TO+VB+NP,2] ||| [GOAL,1] [TO+VB+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NNS,2] ||| [GOAL,1] [JJ+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR+COMMA+JJ,2] ||| [GOAL,1] [SBAR+COMMA+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+OOV+NNS,2] ||| [GOAL,1] [COMMA+OOV+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [$+PRN+PRP,2] ||| [GOAL,1] [$+PRN+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+DT+NNS,2] ||| [GOAL,1] [CC+DT+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+COMMA+NP,2] ||| [GOAL,1] [JJ+COMMA+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN\VP,2] ||| [GOAL,1] [NN\VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+COMMA+NN,2] ||| [GOAL,1] [JJ+COMMA+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NNP,2] ||| [GOAL,1] [JJ+NNP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VP+VBZ,2] ||| [GOAL,1] [COMMA+VP+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+NP+ADJP,2] ||| [GOAL,1] [IN+NP+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+VP+.,2] ||| [GOAL,1] [ADVP+VP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+WRB+NN,2] ||| [GOAL,1] [JJ+WRB+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+PP+SYM,2] ||| [GOAL,1] [NN+PP+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VP+VBP,2] ||| [GOAL,1] [COMMA+VP+VBP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+PRP+NNS,2] ||| [GOAL,1] [COMMA+PRP+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+SYM+QP,2] ||| [GOAL,1] [:+SYM+QP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNP+SYM+JJ,2] ||| [GOAL,1] [NNP+SYM+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [TO+VB+NN,2] ||| [GOAL,1] [TO+VB+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NX+COMMA+NNS,2] ||| [GOAL,1] [NX+COMMA+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VB+CC,2] ||| [GOAL,1] [NN+VB+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+JJ+.,2] ||| [GOAL,1] [RB+JJ+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+IN+FW,2] ||| [GOAL,1] [NN+IN+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+COMMA+OOV,2] ||| [GOAL,1] [NN+COMMA+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VP+VBD,2] ||| [GOAL,1] [COMMA+VP+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+:+IN,2] ||| [GOAL,1] [JJ+:+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+PP+IN,2] ||| [GOAL,1] [PP+PP+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+VBZ+WRB,2] ||| [GOAL,1] [DT+VBZ+WRB,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBN+PP,2] ||| [GOAL,1] [NP+VBN+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+X,2] ||| [GOAL,1] [CC+NP+X,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+VP+COMMA,2] ||| [GOAL,1] [VBP+VP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+:+DT,2] ||| [GOAL,1] [CD+:+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+CD+COMMA,2] ||| [GOAL,1] [NN+CD+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+PP,2] ||| [GOAL,1] [NN+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+S+COMMA,2] ||| [GOAL,1] [IN+S+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [LS+SYM+JJ,2] ||| [GOAL,1] [LS+SYM+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+WRB+JJ,2] ||| [GOAL,1] [NP+WRB+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+NNS+COMMA,2] ||| [GOAL,1] [IN+NNS+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+''+JJ,2] ||| [GOAL,1] [COMMA+''+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+:+JJ,2] ||| [GOAL,1] [JJ+:+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [``+SYM+SYM,2] ||| [GOAL,1] [``+SYM+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+RB,2] ||| [GOAL,1] [NP+VBD+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP\NX,2] ||| [GOAL,1] [ADJP\NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+PRP,2] ||| [GOAL,1] [NNS+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [WHPP+NP+VBZ,2] ||| [GOAL,1] [WHPP+NP+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+X+.,2] ||| [GOAL,1] [SYM+X+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+NNS+NP,2] ||| [GOAL,1] [CD+NNS+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VP,2] ||| [GOAL,1] [NN+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+PRN,2] ||| [GOAL,1] [NNS+PRN,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+NNS+NN,2] ||| [GOAL,1] [CD+NNS+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+OOV+SYM,2] ||| [GOAL,1] [CC+OOV+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP\NP,2] ||| [GOAL,1] [ADJP\NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NNS+COMMA,2] ||| [GOAL,1] [CC+NNS+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+''+RB,2] ||| [GOAL,1] [:+''+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+VBG+NNS,2] ||| [GOAL,1] [IN+VBG+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+SYM+RB,2] ||| [GOAL,1] [:+SYM+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [MD+NP+VP,2] ||| [GOAL,1] [MD+NP+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP+JJR,2] ||| [GOAL,1] [ADJP+JJR,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+VBN+SYM,2] ||| [GOAL,1] [RB+VBN+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+CC+VBG,2] ||| [GOAL,1] [NP+CC+VBG,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+DT+NP,2] ||| [GOAL,1] [IN+DT+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+WHNP+PP,2] ||| [GOAL,1] [NN+WHNP+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+DT+NN,2] ||| [GOAL,1] [IN+DT+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+NP+VBP,2] ||| [GOAL,1] [CD+NP+VBP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+NNS,2] ||| [GOAL,1] [CD+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+COMMA,2] ||| [GOAL,1] [CC+NP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [S+WDT+PRP,2] ||| [GOAL,1] [S+WDT+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+MD+CD,2] ||| [GOAL,1] [NP+MD+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [WHPP+NP,2] ||| [GOAL,1] [WHPP+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+CC+VBD,2] ||| [GOAL,1] [NP+CC+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [WHPP+NN,2] ||| [GOAL,1] [WHPP+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+VBN+COMMA,2] ||| [GOAL,1] [CC+VBN+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+JJ+VBN,2] ||| [GOAL,1] [NN+JJ+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+PDT+DT,2] ||| [GOAL,1] [RB+PDT+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [X/JJ,2] ||| [GOAL,1] [X/JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+VP+.,2] ||| [GOAL,1] [VB+VP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+CC+VBZ,2] ||| [GOAL,1] [NP+CC+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+WP$,2] ||| [GOAL,1] [COMMA+WP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+COMMA+PRP,2] ||| [GOAL,1] [ADVP+COMMA+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [MD+VBZ,2] ||| [GOAL,1] [MD+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [X\S,2] ||| [GOAL,1] [X\S,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+CC+VBP,2] ||| [GOAL,1] [NP+CC+VBP,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR+COMMA+DT,2] ||| [GOAL,1] [SBAR+COMMA+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [WDT+S+.,2] ||| [GOAL,1] [WDT+S+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+MD+IN,2] ||| [GOAL,1] [CC+MD+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+VBN+RB,2] ||| [GOAL,1] [VBD+VBN+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+:+DT,2] ||| [GOAL,1] [PP+:+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBZ+NP+RBS,2] ||| [GOAL,1] [VBZ+NP+RBS,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VB,2] ||| [GOAL,1] [NN+VB,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+VBP+TO,2] ||| [GOAL,1] [PRP+VBP+TO,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+TO+CD,2] ||| [GOAL,1] [CD+TO+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+QP,2] ||| [GOAL,1] [NP+VBD+QP,2] ||| -1
+[GOAL] ||| [GOAL,1] [X\X,2] ||| [GOAL,1] [X\X,2] ||| -1
+[GOAL] ||| [GOAL,1] [S/SBAR,2] ||| [GOAL,1] [S/SBAR,2] ||| -1
+[GOAL] ||| [GOAL,1] [.+ADJP,2] ||| [GOAL,1] [.+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+FW+CC,2] ||| [GOAL,1] [SYM+FW+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+''+QP,2] ||| [GOAL,1] [:+''+QP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+JJ+NN,2] ||| [GOAL,1] [VBN+JJ+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+NN+.,2] ||| [GOAL,1] [DT+NN+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+IN+JJ,2] ||| [GOAL,1] [NNS+IN+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+:+NN,2] ||| [GOAL,1] [JJ+:+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR+PRP,2] ||| [GOAL,1] [SBAR+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+PP+VP,2] ||| [GOAL,1] [PP+PP+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+ADJP+VP,2] ||| [GOAL,1] [NN+ADJP+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+WHADVP,2] ||| [GOAL,1] [COMMA+WHADVP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+:+NP,2] ||| [GOAL,1] [JJ+:+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP$+VP,2] ||| [GOAL,1] [PRP$+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+TO,2] ||| [GOAL,1] [NN+TO,2] ||| -1
+[GOAL] ||| [GOAL,1] [TO+VB+QP,2] ||| [GOAL,1] [TO+VB+QP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+WRB,2] ||| [GOAL,1] [COMMA+WRB,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NN+COMMA,2] ||| [GOAL,1] [CC+NN+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+VBP,2] ||| [GOAL,1] [CC+NP+VBP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+VBN,2] ||| [GOAL,1] [CC+NP+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP/VP,2] ||| [GOAL,1] [PP/VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NX+NX+COMMA,2] ||| [GOAL,1] [NX+NX+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+NP+VBD,2] ||| [GOAL,1] [NNS+NP+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP/SBAR,2] ||| [GOAL,1] [ADJP/SBAR,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP+S+.,2] ||| [GOAL,1] [ADJP+S+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+NP+ADJP,2] ||| [GOAL,1] [COMMA+NP+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP+IN+NP,2] ||| [GOAL,1] [ADJP+IN+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+PP,2] ||| [GOAL,1] [NP+VBD+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+CD,2] ||| [GOAL,1] [CD+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+CC,2] ||| [GOAL,1] [CD+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB\VP,2] ||| [GOAL,1] [VB\VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+VBZ,2] ||| [GOAL,1] [CC+NP+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+VBZ+RB,2] ||| [GOAL,1] [PRP+VBZ+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [POS+JJ+CC,2] ||| [GOAL,1] [POS+JJ+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+CC+JJR,2] ||| [GOAL,1] [JJ+CC+JJR,2] ||| -1
+[GOAL] ||| [GOAL,1] [OOV+FW+.,2] ||| [GOAL,1] [OOV+FW+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+JJ+COMMA,2] ||| [GOAL,1] [VBN+JJ+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+DT,2] ||| [GOAL,1] [CD+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBZ+CD+NN,2] ||| [GOAL,1] [VBZ+CD+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+VBN+TO,2] ||| [GOAL,1] [VBD+VBN+TO,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+ADJP+COMMA,2] ||| [GOAL,1] [PP+ADJP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+VBD,2] ||| [GOAL,1] [CC+NP+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+:+FW,2] ||| [GOAL,1] [PP+:+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+VBG,2] ||| [GOAL,1] [CC+NP+VBG,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+OOV,2] ||| [GOAL,1] [COMMA+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+ADVP+.,2] ||| [GOAL,1] [JJ+ADVP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VBP+JJ,2] ||| [GOAL,1] [NN+VBP+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+NP+PRP,2] ||| [GOAL,1] [COMMA+NP+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NN+PRP$,2] ||| [GOAL,1] [JJ+NN+PRP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+NX+COMMA,2] ||| [GOAL,1] [NP+NX+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+VBP+:,2] ||| [GOAL,1] [DT+VBP+:,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+DT+RBS,2] ||| [GOAL,1] [COMMA+DT+RBS,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+NNS+.,2] ||| [GOAL,1] [CD+NNS+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+S+COMMA,2] ||| [GOAL,1] [:+S+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [WDT+PRP+VBD,2] ||| [GOAL,1] [WDT+PRP+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+VB,2] ||| [GOAL,1] [NP+VBD+VB,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+VP,2] ||| [GOAL,1] [NP+VBD+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [WHPP+JJ,2] ||| [GOAL,1] [WHPP+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+DT+JJ,2] ||| [GOAL,1] [IN+DT+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+POS,2] ||| [GOAL,1] [NNS+POS,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+FW+FW,2] ||| [GOAL,1] [SYM+FW+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [X/NP,2] ||| [GOAL,1] [X/NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+CC+PRP,2] ||| [GOAL,1] [COMMA+CC+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [X/NN,2] ||| [GOAL,1] [X/NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+:+VB,2] ||| [GOAL,1] [VBP+:+VB,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+VP+.,2] ||| [GOAL,1] [DT+VP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+SYM+WP,2] ||| [GOAL,1] [:+SYM+WP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VBZ+NP,2] ||| [GOAL,1] [COMMA+VBZ+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+RB+CC,2] ||| [GOAL,1] [NNS+RB+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [NX+NX+OOV,2] ||| [GOAL,1] [NX+NX+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+WHPP+CD,2] ||| [GOAL,1] [RB+WHPP+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+NNS+VBZ,2] ||| [GOAL,1] [IN+NNS+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+OOV,2] ||| [GOAL,1] [NN+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+NNS+RB,2] ||| [GOAL,1] [CD+NNS+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+PP+RB,2] ||| [GOAL,1] [PP+PP+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+VBZ+.,2] ||| [GOAL,1] [SYM+VBZ+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+OOV+JJ,2] ||| [GOAL,1] [NN+OOV+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS\S,2] ||| [GOAL,1] [NNS\S,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+CC+COMMA,2] ||| [GOAL,1] [NP+CC+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+JJ+VBN,2] ||| [GOAL,1] [DT+JJ+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+RB+PRP,2] ||| [GOAL,1] [NN+RB+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+IN+NN,2] ||| [GOAL,1] [NNS+IN+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+WP$+JJ,2] ||| [GOAL,1] [NNS+WP$+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN\NX,2] ||| [GOAL,1] [NN\NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+OOV,2] ||| [GOAL,1] [CC+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [WRB+QP+IN,2] ||| [GOAL,1] [WRB+QP+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBZ+:+NN,2] ||| [GOAL,1] [VBZ+:+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR+COMMA+CC,2] ||| [GOAL,1] [SBAR+COMMA+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+CONJP+DT,2] ||| [GOAL,1] [NN+CONJP+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+RB+NP,2] ||| [GOAL,1] [SYM+RB+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+COMMA+RB,2] ||| [GOAL,1] [VBN+COMMA+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+IN+NP,2] ||| [GOAL,1] [NNS+IN+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+TO,2] ||| [GOAL,1] [NP+VBD+TO,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+OOV,2] ||| [GOAL,1] [VBD+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+JJ+OOV,2] ||| [GOAL,1] [:+JJ+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJS+NNS,2] ||| [GOAL,1] [JJS+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+COMMA+'',2] ||| [GOAL,1] [NN+COMMA+'',2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+JJ+VP,2] ||| [GOAL,1] [NNS+JJ+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+NN+COMMA,2] ||| [GOAL,1] [NNS+NN+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+JJ+VBD,2] ||| [GOAL,1] [NN+JJ+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+TO+NP,2] ||| [GOAL,1] [JJ+TO+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VB+ADVP,2] ||| [GOAL,1] [COMMA+VB+ADVP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+WP,2] ||| [GOAL,1] [NN+WP,2] ||| -1
+[GOAL] ||| [GOAL,1] [FW+ADJP,2] ||| [GOAL,1] [FW+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VBP+NN,2] ||| [GOAL,1] [NN+VBP+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [VP+PRP+VP,2] ||| [GOAL,1] [VP+PRP+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN\NP,2] ||| [GOAL,1] [NN\NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [LST+JJ,2] ||| [GOAL,1] [LST+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VBP+NP,2] ||| [GOAL,1] [NN+VBP+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+WRB,2] ||| [GOAL,1] [ADVP+WRB,2] ||| -1
+[GOAL] ||| [GOAL,1] [.+PRP+RB,2] ||| [GOAL,1] [.+PRP+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [WHADVP,2] ||| [GOAL,1] [WHADVP,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR+RB+DT,2] ||| [GOAL,1] [SBAR+RB+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [NX+.,2] ||| [GOAL,1] [NX+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+JJ,2] ||| [GOAL,1] [NP+VBD+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+VP,2] ||| [GOAL,1] [CC+NP+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VP+RB+DT,2] ||| [GOAL,1] [VP+RB+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+NNS,2] ||| [GOAL,1] [SYM+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+VBP+RBR,2] ||| [GOAL,1] [NNS+VBP+RBR,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+QP+NNS,2] ||| [GOAL,1] [IN+QP+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [MD+VB+NNP,2] ||| [GOAL,1] [MD+VB+NNP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VBP+CC,2] ||| [GOAL,1] [NN+VBP+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VBP+CD,2] ||| [GOAL,1] [NN+VBP+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+VP+COMMA,2] ||| [GOAL,1] [VBD+VP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [S\FRAG,2] ||| [GOAL,1] [S\FRAG,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+OOV+.,2] ||| [GOAL,1] [CC+OOV+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [NX+:,2] ||| [GOAL,1] [NX+:,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+X+SYM,2] ||| [GOAL,1] [SYM+X+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+IN+JJ,2] ||| [GOAL,1] [VBN+IN+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+OOV+CC,2] ||| [GOAL,1] [VBP+OOV+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP/PP,2] ||| [GOAL,1] [PP/PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+IN,2] ||| [GOAL,1] [NP+VBD+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+COMMA+SBAR,2] ||| [GOAL,1] [NNS+COMMA+SBAR,2] ||| -1
+[GOAL] ||| [GOAL,1] [X/RB,2] ||| [GOAL,1] [X/RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+IN+S,2] ||| [GOAL,1] [VBN+IN+S,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+CC+PRP,2] ||| [GOAL,1] [NN+CC+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+NN+NNS,2] ||| [GOAL,1] [IN+NN+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [OOV,2] ||| [GOAL,1] [OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRN+PRP+NNP,2] ||| [GOAL,1] [PRN+PRP+NNP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NX+''+NX,2] ||| [GOAL,1] [NX+''+NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+JJ+COMMA,2] ||| [GOAL,1] [COMMA+JJ+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+NNS+VP,2] ||| [GOAL,1] [CD+NNS+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+:+JJ,2] ||| [GOAL,1] [VBP+:+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP+RB+ADJP,2] ||| [GOAL,1] [ADJP+RB+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+PRP$+RBS,2] ||| [GOAL,1] [IN+PRP$+RBS,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+WRB,2] ||| [GOAL,1] [IN+WRB,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+VBN+IN,2] ||| [GOAL,1] [VBD+VBN+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [FW+VP+CC,2] ||| [GOAL,1] [FW+VP+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+JJ+COMMA,2] ||| [GOAL,1] [VB+JJ+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+VBN+JJ,2] ||| [GOAL,1] [VBD+VBN+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [PDT+PRP$,2] ||| [GOAL,1] [PDT+PRP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+WRB+DT,2] ||| [GOAL,1] [COMMA+WRB+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NNS+WDT,2] ||| [GOAL,1] [JJ+NNS+WDT,2] ||| -1
+[GOAL] ||| [GOAL,1] [NX+''+NN,2] ||| [GOAL,1] [NX+''+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+JJ+COMMA,2] ||| [GOAL,1] [CC+JJ+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [WHNP\S,2] ||| [GOAL,1] [WHNP\S,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+VBZ+IN,2] ||| [GOAL,1] [PRP+VBZ+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP/NP,2] ||| [GOAL,1] [PP/NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [''+COMMA+CC,2] ||| [GOAL,1] [''+COMMA+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [RP+IN+NP,2] ||| [GOAL,1] [RP+IN+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+VBZ+COMMA,2] ||| [GOAL,1] [DT+VBZ+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [.+NN+VP,2] ||| [GOAL,1] [.+NN+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP/NX,2] ||| [GOAL,1] [PP/NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+JJ+IN,2] ||| [GOAL,1] [NNS+JJ+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP$+NN,2] ||| [GOAL,1] [PRP$+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP$+NP,2] ||| [GOAL,1] [PRP$+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+IN+PRP$,2] ||| [GOAL,1] [CD+IN+PRP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+NN+WRB,2] ||| [GOAL,1] [IN+NN+WRB,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB\NP,2] ||| [GOAL,1] [VB\NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VB+JJS,2] ||| [GOAL,1] [COMMA+VB+JJS,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+:+NN,2] ||| [GOAL,1] [PP+:+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+PP+VBZ,2] ||| [GOAL,1] [CC+PP+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+:+NP,2] ||| [GOAL,1] [PP+:+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+OOV+NN,2] ||| [GOAL,1] [NN+OOV+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNP+COMMA,2] ||| [GOAL,1] [NNP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP/NN,2] ||| [GOAL,1] [PP/NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+CC+'',2] ||| [GOAL,1] [:+CC+'',2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+OOV+NP,2] ||| [GOAL,1] [NN+OOV+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+PP+VBP,2] ||| [GOAL,1] [CC+PP+VBP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+VBZ+JJ,2] ||| [GOAL,1] [PRP+VBZ+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+ADJP+.,2] ||| [GOAL,1] [COMMA+ADJP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+COMMA+NP,2] ||| [GOAL,1] [VBN+COMMA+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [RBS+NN,2] ||| [GOAL,1] [RBS+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+MD+CD,2] ||| [GOAL,1] [JJ+MD+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+JJ+JJ,2] ||| [GOAL,1] [NNS+JJ+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+RB+SYM,2] ||| [GOAL,1] [CD+RB+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+RB+IN,2] ||| [GOAL,1] [NNS+RB+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+RB+WDT,2] ||| [GOAL,1] [COMMA+RB+WDT,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+VP,2] ||| [GOAL,1] [ADVP+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+WRB+NP,2] ||| [GOAL,1] [NNS+WRB+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [POS+JJ+JJ,2] ||| [GOAL,1] [POS+JJ+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP\S,2] ||| [GOAL,1] [ADVP\S,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+CC+S,2] ||| [GOAL,1] [COMMA+CC+S,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+PRP$,2] ||| [GOAL,1] [RB+PRP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+VBP+IN,2] ||| [GOAL,1] [PRP+VBP+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [X/PP,2] ||| [GOAL,1] [X/PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+FW,2] ||| [GOAL,1] [NP+VBD+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NNS+POS,2] ||| [GOAL,1] [CC+NNS+POS,2] ||| -1
+[GOAL] ||| [GOAL,1] [OOV+COMMA+JJ,2] ||| [GOAL,1] [OOV+COMMA+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+WRB,2] ||| [GOAL,1] [CC+WRB,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+RB+JJ,2] ||| [GOAL,1] [NNS+RB+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+RB+VBD,2] ||| [GOAL,1] [CC+RB+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP/CD,2] ||| [GOAL,1] [ADJP/CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [S/SYM,2] ||| [GOAL,1] [S/SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+NP,2] ||| [GOAL,1] [NP+VBD+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+NN,2] ||| [GOAL,1] [NP+VBD+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+SYM+ADJP,2] ||| [GOAL,1] [NP+SYM+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+CC+ADJP,2] ||| [GOAL,1] [NNS+CC+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+PRP+VBD,2] ||| [GOAL,1] [PP+PRP+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+RB+VBZ,2] ||| [GOAL,1] [CC+RB+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VBP+VBN,2] ||| [GOAL,1] [NN+VBP+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [INTJ\FRAG,2] ||| [GOAL,1] [INTJ\FRAG,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+NNS+COMMA,2] ||| [GOAL,1] [VBP+NNS+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+RB+VBP,2] ||| [GOAL,1] [CC+RB+VBP,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+DT+RB,2] ||| [GOAL,1] [IN+DT+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+IN+NNS,2] ||| [GOAL,1] [NP+IN+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+IN+WRB,2] ||| [GOAL,1] [VB+IN+WRB,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+RB+VBN,2] ||| [GOAL,1] [CC+RB+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+:+NP,2] ||| [GOAL,1] [VBP+:+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+:+NN,2] ||| [GOAL,1] [VBP+:+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+:+PRP$,2] ||| [GOAL,1] [VBP+:+PRP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [''+RBS,2] ||| [GOAL,1] [''+RBS,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP,2] ||| [GOAL,1] [ADVP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+IN+DT,2] ||| [GOAL,1] [VBN+IN+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+PP,2] ||| [GOAL,1] [ADVP+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+PRP+VBZ,2] ||| [GOAL,1] [PP+PRP+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+JJ+VBN,2] ||| [GOAL,1] [IN+JJ+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+ADJP+ADVP,2] ||| [GOAL,1] [DT+ADJP+ADVP,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR+VBZ+VBG,2] ||| [GOAL,1] [SBAR+VBZ+VBG,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+VBN+IN,2] ||| [GOAL,1] [CC+VBN+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [WRB+SYM+CC,2] ||| [GOAL,1] [WRB+SYM+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBZ+CD+TO,2] ||| [GOAL,1] [VBZ+CD+TO,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP/POS,2] ||| [GOAL,1] [PP/POS,2] ||| -1
+[GOAL] ||| [GOAL,1] [RP+NP+.,2] ||| [GOAL,1] [RP+NP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP/DT,2] ||| [GOAL,1] [ADJP/DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+VBN+NP,2] ||| [GOAL,1] [VBD+VBN+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [''+COMMA+FW,2] ||| [GOAL,1] [''+COMMA+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [X/VP,2] ||| [GOAL,1] [X/VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+ADVP+COMMA,2] ||| [GOAL,1] [NN+ADVP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBG+NP+COMMA,2] ||| [GOAL,1] [VBG+NP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+COMMA+IN,2] ||| [GOAL,1] [VB+COMMA+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VBD+SYM,2] ||| [GOAL,1] [NN+VBD+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+NP+DT,2] ||| [GOAL,1] [NP+NP+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [WHPP+PRP,2] ||| [GOAL,1] [WHPP+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NN+WHPP,2] ||| [GOAL,1] [JJ+NN+WHPP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJR+SYM,2] ||| [GOAL,1] [JJR+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+VBZ+NP,2] ||| [GOAL,1] [PRP+VBZ+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VBP+DT,2] ||| [GOAL,1] [NN+VBP+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+RB,2] ||| [GOAL,1] [ADVP+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+RB+COMMA,2] ||| [GOAL,1] [SYM+RB+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+VBZ+NN,2] ||| [GOAL,1] [PRP+VBZ+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [VP+:+SYM,2] ||| [GOAL,1] [VP+:+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+ADVP,2] ||| [GOAL,1] [CC+ADVP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+VBN+QP,2] ||| [GOAL,1] [VBD+VBN+QP,2] ||| -1
+[GOAL] ||| [GOAL,1] [POS+WDT,2] ||| [GOAL,1] [POS+WDT,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP+VBD+NP,2] ||| [GOAL,1] [ADJP+VBD+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+JJ+SYM,2] ||| [GOAL,1] [SYM+JJ+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+VP+CC,2] ||| [GOAL,1] [PRP+VP+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB\X,2] ||| [GOAL,1] [RB\X,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP/RB,2] ||| [GOAL,1] [PP/RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP+ADJP,2] ||| [GOAL,1] [ADJP+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+JJ+NX,2] ||| [GOAL,1] [NNS+JJ+NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+VP+:,2] ||| [GOAL,1] [:+VP+:,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+WRB+NN,2] ||| [GOAL,1] [VBN+WRB+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [POS+JJ+NN,2] ||| [GOAL,1] [POS+JJ+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+NN+COMMA,2] ||| [GOAL,1] [RB+NN+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB\S,2] ||| [GOAL,1] [RB\S,2] ||| -1
+[GOAL] ||| [GOAL,1] [OOV+COMMA+NX,2] ||| [GOAL,1] [OOV+COMMA+NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+JJ+NP,2] ||| [GOAL,1] [NNS+JJ+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+JJ+NN,2] ||| [GOAL,1] [NNS+JJ+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP/PRP,2] ||| [GOAL,1] [PP/PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+JJR+.,2] ||| [GOAL,1] [NN+JJR+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP/QP,2] ||| [GOAL,1] [PP/QP,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+VP+.,2] ||| [GOAL,1] [:+VP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+OOV,2] ||| [GOAL,1] [IN+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+NP+CC,2] ||| [GOAL,1] [NP+NP+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+:+PRP$,2] ||| [GOAL,1] [PP+:+PRP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBZ+:+VB,2] ||| [GOAL,1] [VBZ+:+VB,2] ||| -1
+[GOAL] ||| [GOAL,1] [WRB+PRP+VP,2] ||| [GOAL,1] [WRB+PRP+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+VBN+PP,2] ||| [GOAL,1] [VBD+VBN+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+WRB+FW,2] ||| [GOAL,1] [COMMA+WRB+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+WDT+VBN,2] ||| [GOAL,1] [COMMA+WDT+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+COMMA,2] ||| [GOAL,1] [NP+VBD+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+VBZ+RB,2] ||| [GOAL,1] [CC+VBZ+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+WDT+VBP,2] ||| [GOAL,1] [COMMA+WDT+VBP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+CD+IN,2] ||| [GOAL,1] [NN+CD+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+CD+NNS,2] ||| [GOAL,1] [RB+CD+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+NNS+CC,2] ||| [GOAL,1] [IN+NNS+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+OOV+JJ,2] ||| [GOAL,1] [VBP+OOV+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+CC+PP,2] ||| [GOAL,1] [:+CC+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+NP,2] ||| [GOAL,1] [ADVP+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+SYM+SYM,2] ||| [GOAL,1] [NN+SYM+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [FW+NN+COMMA,2] ||| [GOAL,1] [FW+NN+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+JJ+VBN,2] ||| [GOAL,1] [CC+JJ+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+JJ+VBP,2] ||| [GOAL,1] [CC+JJ+VBP,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+NN,2] ||| [GOAL,1] [ADVP+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+NN+COMMA,2] ||| [GOAL,1] [NN+NN+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+WHPP+NP,2] ||| [GOAL,1] [NP+WHPP+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+JJ+VBD,2] ||| [GOAL,1] [NP+JJ+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+ADVP,2] ||| [GOAL,1] [NP+ADVP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+WDT+VBZ,2] ||| [GOAL,1] [COMMA+WDT+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+CD+JJ,2] ||| [GOAL,1] [NN+CD+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NX+.,2] ||| [GOAL,1] [CC+NX+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+TO,2] ||| [GOAL,1] [VB+TO,2] ||| -1
+[GOAL] ||| [GOAL,1] [VP+RB+NP,2] ||| [GOAL,1] [VP+RB+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+FW+JJ,2] ||| [GOAL,1] [RB+FW+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+JJ+VBG,2] ||| [GOAL,1] [CC+JJ+VBG,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+JJ+VBD,2] ||| [GOAL,1] [CC+JJ+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [MD+VBZ+DT,2] ||| [GOAL,1] [MD+VBZ+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR+TO,2] ||| [GOAL,1] [SBAR+TO,2] ||| -1
+[GOAL] ||| [GOAL,1] [FW+X+NP,2] ||| [GOAL,1] [FW+X+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+WDT+VBD,2] ||| [GOAL,1] [COMMA+WDT+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP+NNS+CC,2] ||| [GOAL,1] [ADJP+NNS+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+RB+NP,2] ||| [GOAL,1] [CC+RB+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+WRB+JJ,2] ||| [GOAL,1] [COMMA+WRB+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+JJ+JJ,2] ||| [GOAL,1] [:+JJ+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+VBD+ADJP,2] ||| [GOAL,1] [PRP+VBD+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+RB+NN,2] ||| [GOAL,1] [CC+RB+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+$+.,2] ||| [GOAL,1] [PRP+$+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+WDT,2] ||| [GOAL,1] [CD+WDT,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+'',2] ||| [GOAL,1] [:+'',2] ||| -1
+[GOAL] ||| [GOAL,1] [:+CC+NP,2] ||| [GOAL,1] [:+CC+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+NP+PRP,2] ||| [GOAL,1] [IN+NP+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+SYM+JJ,2] ||| [GOAL,1] [RB+SYM+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN\S,2] ||| [GOAL,1] [NN\S,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+IN+PRP,2] ||| [GOAL,1] [COMMA+IN+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+PRP$+JJ,2] ||| [GOAL,1] [VBD+PRP$+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+''+JJ,2] ||| [GOAL,1] [CC+''+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+S+COMMA,2] ||| [GOAL,1] [COMMA+S+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [FRAG/NN,2] ||| [GOAL,1] [FRAG/NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN\X,2] ||| [GOAL,1] [NN\X,2] ||| -1
+[GOAL] ||| [GOAL,1] [FRAG/NP,2] ||| [GOAL,1] [FRAG/NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NNS+NP,2] ||| [GOAL,1] [CC+NNS+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+''+RBS,2] ||| [GOAL,1] [IN+''+RBS,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNP+VBZ,2] ||| [GOAL,1] [NNP+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+JJ+NNS,2] ||| [GOAL,1] [DT+JJ+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NNS+NN,2] ||| [GOAL,1] [CC+NNS+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+WHNP,2] ||| [GOAL,1] [NNS+WHNP,2] ||| -1
+[GOAL] ||| [GOAL,1] [EX+VP,2] ||| [GOAL,1] [EX+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VP\SINV,2] ||| [GOAL,1] [VP\SINV,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+VBZ+OOV,2] ||| [GOAL,1] [NNS+VBZ+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+COMMA+NP,2] ||| [GOAL,1] [VB+COMMA+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+RP+NP,2] ||| [GOAL,1] [VBD+RP+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+RB+COMMA,2] ||| [GOAL,1] [:+RB+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+NN+VBD,2] ||| [GOAL,1] [COMMA+NN+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+VP,2] ||| [GOAL,1] [VB+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+NP+CC,2] ||| [GOAL,1] [RB+NP+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NNS+VP,2] ||| [GOAL,1] [JJ+NNS+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+CC+PRP$,2] ||| [GOAL,1] [VBN+CC+PRP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+NN+VBG,2] ||| [GOAL,1] [COMMA+NN+VBG,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBZ+NNS,2] ||| [GOAL,1] [NP+VBZ+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR+VP,2] ||| [GOAL,1] [SBAR+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [''+JJ+NNS,2] ||| [GOAL,1] [''+JJ+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+NN+VBN,2] ||| [GOAL,1] [COMMA+NN+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJS+NN+PP,2] ||| [GOAL,1] [JJS+NN+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+WP$+JJ,2] ||| [GOAL,1] [COMMA+WP$+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+NN+VBP,2] ||| [GOAL,1] [COMMA+NN+VBP,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+MD+CD,2] ||| [GOAL,1] [IN+MD+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+DT+RBS,2] ||| [GOAL,1] [VBP+DT+RBS,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+CD+COMMA,2] ||| [GOAL,1] [CC+CD+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VB+CC,2] ||| [GOAL,1] [COMMA+VB+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+NN+VBZ,2] ||| [GOAL,1] [COMMA+NN+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+SBAR+CC,2] ||| [GOAL,1] [VBN+SBAR+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [WHADJP+VBZ,2] ||| [GOAL,1] [WHADJP+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NP+JJR,2] ||| [GOAL,1] [JJ+NP+JJR,2] ||| -1
+[GOAL] ||| [GOAL,1] [FW+NP+VBD,2] ||| [GOAL,1] [FW+NP+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP$+NN+COMMA,2] ||| [GOAL,1] [PRP$+NN+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP$+NN+CC,2] ||| [GOAL,1] [PRP$+NN+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+NNS+CC,2] ||| [GOAL,1] [CD+NNS+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+VBG+DT,2] ||| [GOAL,1] [VBP+VBG+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+NP+NP,2] ||| [GOAL,1] [SYM+NP+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+CC+RB,2] ||| [GOAL,1] [DT+CC+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+NN+COMMA,2] ||| [GOAL,1] [COMMA+NN+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+CD,2] ||| [GOAL,1] [JJ+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+PRP,2] ||| [GOAL,1] [DT+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+NP,2] ||| [GOAL,1] [VB+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+CD,2] ||| [GOAL,1] [NP+VBD+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+JJR+JJ,2] ||| [GOAL,1] [COMMA+JJR+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NX+COMMA,2] ||| [GOAL,1] [JJ+NX+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+RB+PRP,2] ||| [GOAL,1] [COMMA+RB+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+OOV+COMMA,2] ||| [GOAL,1] [JJ+OOV+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+''+DT,2] ||| [GOAL,1] [CC+''+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+VBZ,2] ||| [GOAL,1] [JJ+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+CC,2] ||| [GOAL,1] [JJ+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+CD+NN,2] ||| [GOAL,1] [NN+CD+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+EX+VP,2] ||| [GOAL,1] [NNS+EX+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+WHPP,2] ||| [GOAL,1] [NNS+WHPP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NNS+PP,2] ||| [GOAL,1] [CC+NNS+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+JJ,2] ||| [GOAL,1] [ADVP+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+WP$+JJ,2] ||| [GOAL,1] [NP+WP$+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+NP+WP$,2] ||| [GOAL,1] [VBP+NP+WP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+VBN,2] ||| [GOAL,1] [JJ+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+IN,2] ||| [GOAL,1] [ADVP+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+VBG+PRP,2] ||| [GOAL,1] [CC+VBG+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+RB+RB,2] ||| [GOAL,1] [CC+RB+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+DT,2] ||| [GOAL,1] [JJ+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+VBG,2] ||| [GOAL,1] [JJ+VBG,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+PP+COMMA,2] ||| [GOAL,1] [COMMA+PP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] ['',2] ||| [GOAL,1] ['',2] ||| -1
+[GOAL] ||| [GOAL,1] [VP+SYM,2] ||| [GOAL,1] [VP+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [VP/FRAG,2] ||| [GOAL,1] [VP/FRAG,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NNS+RB,2] ||| [GOAL,1] [CC+NNS+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+PP,2] ||| [GOAL,1] [VB+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+VBP,2] ||| [GOAL,1] [JJ+VBP,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+DT+ADJP,2] ||| [GOAL,1] [IN+DT+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+WRB+NN,2] ||| [GOAL,1] [COMMA+WRB+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+WRB+NP,2] ||| [GOAL,1] [COMMA+WRB+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+NNS+DT,2] ||| [GOAL,1] [CD+NNS+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+NX+COMMA,2] ||| [GOAL,1] [:+NX+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+WHNP+VBD,2] ||| [GOAL,1] [NN+WHNP+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+VBD,2] ||| [GOAL,1] [JJ+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+VBD+VBN,2] ||| [GOAL,1] [PRP+VBD+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VBG+NP,2] ||| [GOAL,1] [COMMA+VBG+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+MD+VB,2] ||| [GOAL,1] [NN+MD+VB,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+JJ+DT,2] ||| [GOAL,1] [:+JJ+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [FRAG/JJ,2] ||| [GOAL,1] [FRAG/JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+DT,2] ||| [GOAL,1] [NP+VBD+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBZ+NP+SBAR,2] ||| [GOAL,1] [VBZ+NP+SBAR,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+JJ+ADVP,2] ||| [GOAL,1] [COMMA+JJ+ADVP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+CD+VBZ,2] ||| [GOAL,1] [JJ+CD+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+NN+COMMA,2] ||| [GOAL,1] [SYM+NN+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+NP+IN,2] ||| [GOAL,1] [NP+NP+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+ADJP+CC,2] ||| [GOAL,1] [DT+ADJP+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+NNS,2] ||| [GOAL,1] [DT+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+WRB+QP,2] ||| [GOAL,1] [COMMA+WRB+QP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+RB+VB,2] ||| [GOAL,1] [CC+RB+VB,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+WHPP,2] ||| [GOAL,1] [NP+VBD+WHPP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+VBP+JJ,2] ||| [GOAL,1] [JJ+VBP+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+CD+COMMA,2] ||| [GOAL,1] [IN+CD+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NNS+TO,2] ||| [GOAL,1] [CC+NNS+TO,2] ||| -1
+[GOAL] ||| [GOAL,1] [WHNP+VBZ+NP,2] ||| [GOAL,1] [WHNP+VBZ+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+JJ+PRP$,2] ||| [GOAL,1] [NNS+JJ+PRP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+FW,2] ||| [GOAL,1] [JJ+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VBP+.,2] ||| [GOAL,1] [NN+VBP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBG+OOV+NN,2] ||| [GOAL,1] [VBG+OOV+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+NP+JJ,2] ||| [GOAL,1] [NP+NP+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [POS+NNS,2] ||| [GOAL,1] [POS+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VBG+COMMA,2] ||| [GOAL,1] [COMMA+VBG+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+PP+NN,2] ||| [GOAL,1] [NNS+PP+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+SYM+NP,2] ||| [GOAL,1] [RB+SYM+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [EX+.,2] ||| [GOAL,1] [EX+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [FW+NNS+.,2] ||| [GOAL,1] [FW+NNS+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+PP+NP,2] ||| [GOAL,1] [NNS+PP+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+DT+S,2] ||| [GOAL,1] [COMMA+DT+S,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+RB,2] ||| [GOAL,1] [VB+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+VBZ+VBN,2] ||| [GOAL,1] [NNS+VBZ+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+OOV+OOV,2] ||| [GOAL,1] [SYM+OOV+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP/VB,2] ||| [GOAL,1] [NP/VB,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+VBZ+IN,2] ||| [GOAL,1] [CC+VBZ+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+MD+VB,2] ||| [GOAL,1] [SYM+MD+VB,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+EX,2] ||| [GOAL,1] [ADVP+EX,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VBG+CC,2] ||| [GOAL,1] [COMMA+VBG+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+POS+JJS,2] ||| [GOAL,1] [NN+POS+JJS,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+PRP$,2] ||| [GOAL,1] [NN+PRP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+RP+IN,2] ||| [GOAL,1] [VBN+RP+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NNS+DT,2] ||| [GOAL,1] [CC+NNS+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJS+NN+IN,2] ||| [GOAL,1] [JJS+NN+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+.,2] ||| [GOAL,1] [ADVP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+SYM+.,2] ||| [GOAL,1] [NN+SYM+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+JJ+COMMA,2] ||| [GOAL,1] [RB+JJ+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+:,2] ||| [GOAL,1] [ADVP+:,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+PRP+DT,2] ||| [GOAL,1] [VBD+PRP+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+S+VBG,2] ||| [GOAL,1] [:+S+VBG,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+JJ+TO,2] ||| [GOAL,1] [IN+JJ+TO,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+ADJP,2] ||| [GOAL,1] [CC+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [S\UCP,2] ||| [GOAL,1] [S\UCP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VBG+JJ,2] ||| [GOAL,1] [NN+VBG+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN\FRAG,2] ||| [GOAL,1] [NN\FRAG,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB\SINV,2] ||| [GOAL,1] [RB\SINV,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+ADJP+JJ,2] ||| [GOAL,1] [DT+ADJP+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [FRAG+VP+JJ,2] ||| [GOAL,1] [FRAG+VP+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [S+COMMA+JJ,2] ||| [GOAL,1] [S+COMMA+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+.+COMMA,2] ||| [GOAL,1] [NN+.+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+ADVP+DT,2] ||| [GOAL,1] [COMMA+ADVP+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+:+'',2] ||| [GOAL,1] [VBD+:+'',2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+RB+VP,2] ||| [GOAL,1] [CC+RB+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+NP+PP,2] ||| [GOAL,1] [NP+NP+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBZ+VBG+NN,2] ||| [GOAL,1] [VBZ+VBG+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [FW+VBP+DT,2] ||| [GOAL,1] [FW+VBP+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBZ+VBG+NP,2] ||| [GOAL,1] [VBZ+VBG+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+JJR+CC,2] ||| [GOAL,1] [RB+JJR+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP/VP,2] ||| [GOAL,1] [NP/VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+X+:,2] ||| [GOAL,1] [COMMA+X+:,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJS+NN+JJ,2] ||| [GOAL,1] [JJS+NN+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBZ+NP+JJS,2] ||| [GOAL,1] [VBZ+NP+JJS,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+PRP,2] ||| [GOAL,1] [PRP+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VBG+IN,2] ||| [GOAL,1] [NN+VBG+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBG+WP$+JJ,2] ||| [GOAL,1] [VBG+WP$+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+NP+VP,2] ||| [GOAL,1] [DT+NP+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+MD+IN,2] ||| [GOAL,1] [NN+MD+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VP+.,2] ||| [GOAL,1] [NN+VP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+VBZ+NP,2] ||| [GOAL,1] [CD+VBZ+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+SYM+S,2] ||| [GOAL,1] [NN+SYM+S,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+VBZ+.,2] ||| [GOAL,1] [JJ+VBZ+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NNS+NN,2] ||| [GOAL,1] [JJ+NNS+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJR+IN+CD,2] ||| [GOAL,1] [JJR+IN+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NNS+NNS,2] ||| [GOAL,1] [JJ+NNS+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NNS+NP,2] ||| [GOAL,1] [JJ+NNS+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+DT,2] ||| [GOAL,1] [ADVP+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+IN,2] ||| [GOAL,1] [JJ+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [FRAG/VP,2] ||| [GOAL,1] [FRAG/VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+COMMA+VB,2] ||| [GOAL,1] [VB+COMMA+VB,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+PRP+NN,2] ||| [GOAL,1] [CD+PRP+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+DT+DT,2] ||| [GOAL,1] [IN+DT+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+RB+WRB,2] ||| [GOAL,1] [CC+RB+WRB,2] ||| -1
+[GOAL] ||| [GOAL,1] [''+CD+NN,2] ||| [GOAL,1] [''+CD+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VP+:,2] ||| [GOAL,1] [NN+VP+:,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+FW+.,2] ||| [GOAL,1] [COMMA+FW+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+CC,2] ||| [GOAL,1] [ADVP+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP/SINV,2] ||| [GOAL,1] [NP/SINV,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+VBZ+JJ,2] ||| [GOAL,1] [CC+VBZ+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+SYM+:,2] ||| [GOAL,1] [NN+SYM+:,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNP+:+NX,2] ||| [GOAL,1] [NNP+:+NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [WHNP+VBD,2] ||| [GOAL,1] [WHNP+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+DT+VBN,2] ||| [GOAL,1] [IN+DT+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [WHNP+VBP,2] ||| [GOAL,1] [WHNP+VBP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+JJ,2] ||| [GOAL,1] [JJ+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+NNS+VBZ,2] ||| [GOAL,1] [VBN+NNS+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+DT+VBP,2] ||| [GOAL,1] [IN+DT+VBP,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+JJ+PP,2] ||| [GOAL,1] [:+JJ+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+NP+SYM,2] ||| [GOAL,1] [PP+NP+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+NN,2] ||| [GOAL,1] [VB+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP\S,2] ||| [GOAL,1] [PP\S,2] ||| -1
+[GOAL] ||| [GOAL,1] [WHNP+VBZ,2] ||| [GOAL,1] [WHNP+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR+NP,2] ||| [GOAL,1] [SBAR+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+''+CC,2] ||| [GOAL,1] [VB+''+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [NX+JJ+JJ,2] ||| [GOAL,1] [NX+JJ+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+NP+NP,2] ||| [GOAL,1] [NP+NP+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+WDT,2] ||| [GOAL,1] [CC+WDT,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+NP+NN,2] ||| [GOAL,1] [NP+NP+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+RP+DT,2] ||| [GOAL,1] [VBD+RP+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [.+``,2] ||| [GOAL,1] [.+``,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBG+VBN,2] ||| [GOAL,1] [VBG+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP\X,2] ||| [GOAL,1] [PP\X,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+CD+DT,2] ||| [GOAL,1] [NN+CD+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+DT+VBZ,2] ||| [GOAL,1] [IN+DT+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [WP+NNS,2] ||| [GOAL,1] [WP+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [TO+PRP$,2] ||| [GOAL,1] [TO+PRP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB\FRAG,2] ||| [GOAL,1] [RB\FRAG,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+IN+'',2] ||| [GOAL,1] [NN+IN+'',2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+DT+CD,2] ||| [GOAL,1] [IN+DT+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NNS+PP,2] ||| [GOAL,1] [JJ+NNS+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+NNS+CC,2] ||| [GOAL,1] [RB+NNS+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+CC+NNS,2] ||| [GOAL,1] [NP+CC+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [VP+CC+WHPP,2] ||| [GOAL,1] [VP+CC+WHPP,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+FW+FW,2] ||| [GOAL,1] [RB+FW+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+MD+X,2] ||| [GOAL,1] [NN+MD+X,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR/NNP,2] ||| [GOAL,1] [SBAR/NNP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+WDT,2] ||| [GOAL,1] [COMMA+WDT,2] ||| -1
+[GOAL] ||| [GOAL,1] [FRAG/PP,2] ||| [GOAL,1] [FRAG/PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR/NNS,2] ||| [GOAL,1] [SBAR/NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+WHPP,2] ||| [GOAL,1] [NN+WHPP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NNS+WP$,2] ||| [GOAL,1] [JJ+NNS+WP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [S+PRP,2] ||| [GOAL,1] [S+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJS+CC+RB,2] ||| [GOAL,1] [JJS+CC+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+ADJP+NN,2] ||| [GOAL,1] [DT+ADJP+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+JJ+RB,2] ||| [GOAL,1] [IN+JJ+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+NN+NNS,2] ||| [GOAL,1] [COMMA+NN+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+JJ+NN,2] ||| [GOAL,1] [:+JJ+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NNS+IN,2] ||| [GOAL,1] [CC+NNS+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [LST+NP+:,2] ||| [GOAL,1] [LST+NP+:,2] ||| -1
+[GOAL] ||| [GOAL,1] [S+COMMA+NN,2] ||| [GOAL,1] [S+COMMA+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+DT+OOV,2] ||| [GOAL,1] [CC+DT+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+COMMA+PRP,2] ||| [GOAL,1] [NN+COMMA+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+NX+VBN,2] ||| [GOAL,1] [NN+NX+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NX+COMMA+OOV,2] ||| [GOAL,1] [NX+COMMA+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJS+NN+NN,2] ||| [GOAL,1] [JJS+NN+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+:+PRP,2] ||| [GOAL,1] [NN+:+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR+IN+SYM,2] ||| [GOAL,1] [SBAR+IN+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [S+COMMA+NP,2] ||| [GOAL,1] [S+COMMA+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP/S,2] ||| [GOAL,1] [ADVP/S,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VBZ+DT,2] ||| [GOAL,1] [COMMA+VBZ+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+:+PRP$,2] ||| [GOAL,1] [NP+:+PRP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBZ+JJ+COMMA,2] ||| [GOAL,1] [VBZ+JJ+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NX+:,2] ||| [GOAL,1] [CC+NX+:,2] ||| -1
+[GOAL] ||| [GOAL,1] [''+ADJP,2] ||| [GOAL,1] [''+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+MD,2] ||| [GOAL,1] [JJ+MD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+ADJP,2] ||| [GOAL,1] [NN+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR+RB,2] ||| [GOAL,1] [SBAR+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+COMMA+S,2] ||| [GOAL,1] [PP+COMMA+S,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NNS+JJ,2] ||| [GOAL,1] [CC+NNS+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [S+COMMA+RB,2] ||| [GOAL,1] [S+COMMA+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+''+NP,2] ||| [GOAL,1] [CC+''+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PDT,2] ||| [GOAL,1] [PDT,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+NNS+IN,2] ||| [GOAL,1] [CD+NNS+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+JJ+NP,2] ||| [GOAL,1] [IN+JJ+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+JJ+NN,2] ||| [GOAL,1] [IN+JJ+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VBG+DT,2] ||| [GOAL,1] [NN+VBG+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+JJ+NX,2] ||| [GOAL,1] [IN+JJ+NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+COMMA+WDT,2] ||| [GOAL,1] [JJ+COMMA+WDT,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+WHNP,2] ||| [GOAL,1] [NN+WHNP,2] ||| -1
+[GOAL] ||| [GOAL,1] [S+COMMA+PP,2] ||| [GOAL,1] [S+COMMA+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+IN,2] ||| [GOAL,1] [VB+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+NN+VBD,2] ||| [GOAL,1] [CD+NN+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NN,2] ||| [GOAL,1] [JJ+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NP,2] ||| [GOAL,1] [JJ+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+NP+VP,2] ||| [GOAL,1] [SYM+NP+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+PP+NNS,2] ||| [GOAL,1] [CC+PP+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+PP+'',2] ||| [GOAL,1] [PP+PP+'',2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+OOV,2] ||| [GOAL,1] [JJ+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+IN+PRP,2] ||| [GOAL,1] [:+IN+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+VBP+PRP$,2] ||| [GOAL,1] [NNS+VBP+PRP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+VBZ+NP,2] ||| [GOAL,1] [CC+VBZ+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [WRB+RB+SYM,2] ||| [GOAL,1] [WRB+RB+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP+SYM,2] ||| [GOAL,1] [ADJP+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NNS+RB,2] ||| [GOAL,1] [JJ+NNS+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+NP+COMMA,2] ||| [GOAL,1] [COMMA+NP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+NNS+JJ,2] ||| [GOAL,1] [CD+NNS+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [FW+NN+SYM,2] ||| [GOAL,1] [FW+NN+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VBG+DT,2] ||| [GOAL,1] [COMMA+VBG+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+JJ,2] ||| [GOAL,1] [VB+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+NP+WDT,2] ||| [GOAL,1] [VBP+NP+WDT,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+NN+CC,2] ||| [GOAL,1] [ADVP+NN+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [NX+JJ+NN,2] ||| [GOAL,1] [NX+JJ+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NX,2] ||| [GOAL,1] [JJ+NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [MD+CD+:,2] ||| [GOAL,1] [MD+CD+:,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+PP+VP,2] ||| [GOAL,1] [NNS+PP+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [POS+NNS+CC,2] ||| [GOAL,1] [POS+NNS+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+FW+SBAR,2] ||| [GOAL,1] [SYM+FW+SBAR,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN\NP,2] ||| [GOAL,1] [IN\NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+WDT+VBZ,2] ||| [GOAL,1] [NN+WDT+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP/NN,2] ||| [GOAL,1] [NP/NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP/NP,2] ||| [GOAL,1] [NP/NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+NN+VB,2] ||| [GOAL,1] [NNS+NN+VB,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+SBAR+COMMA,2] ||| [GOAL,1] [PP+SBAR+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+WHPP,2] ||| [GOAL,1] [VBN+WHPP,2] ||| -1
+[GOAL] ||| [GOAL,1] [OOV+SYM+CC,2] ||| [GOAL,1] [OOV+SYM+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+CC+VP,2] ||| [GOAL,1] [COMMA+CC+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [S+VBZ+VBG,2] ||| [GOAL,1] [S+VBZ+VBG,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+CC+PRP,2] ||| [GOAL,1] [PP+CC+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+NNS+COMMA,2] ||| [GOAL,1] [VBN+NNS+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+COMMA,2] ||| [GOAL,1] [VBN+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP/S,2] ||| [GOAL,1] [NP/S,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+COMMA+SBAR,2] ||| [GOAL,1] [NN+COMMA+SBAR,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+WDT+VBD,2] ||| [GOAL,1] [NN+WDT+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+PP,2] ||| [GOAL,1] [JJ+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [QP+MD,2] ||| [GOAL,1] [QP+MD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP/X,2] ||| [GOAL,1] [NP/X,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+EX+VP,2] ||| [GOAL,1] [PP+EX+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NNS+CC,2] ||| [GOAL,1] [JJ+NNS+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+SYM+SYM,2] ||| [GOAL,1] [NNS+SYM+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+OOV+SYM,2] ||| [GOAL,1] [NN+OOV+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP/NX,2] ||| [GOAL,1] [NP/NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+NNS,2] ||| [GOAL,1] [COMMA+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+JJ+JJ,2] ||| [GOAL,1] [CD+JJ+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+NNP,2] ||| [GOAL,1] [COMMA+NNP,2] ||| -1
+[GOAL] ||| [GOAL,1] [WRB+NP+VBD,2] ||| [GOAL,1] [WRB+NP+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+ADJP+.,2] ||| [GOAL,1] [:+ADJP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NNS+DT,2] ||| [GOAL,1] [JJ+NNS+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [WRB+NP+VBP,2] ||| [GOAL,1] [WRB+NP+VBP,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+DT+MD,2] ||| [GOAL,1] [:+DT+MD,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+NP+WP$,2] ||| [GOAL,1] [COMMA+NP+WP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+:+OOV,2] ||| [GOAL,1] [IN+:+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+S+.,2] ||| [GOAL,1] [NP+S+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+JJ+TO,2] ||| [GOAL,1] [NP+JJ+TO,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+FW+.,2] ||| [GOAL,1] [CD+FW+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [QP+NN,2] ||| [GOAL,1] [QP+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+JJS,2] ||| [GOAL,1] [NP+JJS,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+JJ+MD,2] ||| [GOAL,1] [IN+JJ+MD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+''+VBD,2] ||| [GOAL,1] [NN+''+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+MD+CD,2] ||| [GOAL,1] [NN+MD+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+JJR,2] ||| [GOAL,1] [NP+JJR,2] ||| -1
+[GOAL] ||| [GOAL,1] [S+COMMA+CC,2] ||| [GOAL,1] [S+COMMA+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+NN+SBAR,2] ||| [GOAL,1] [NN+NN+SBAR,2] ||| -1
+[GOAL] ||| [GOAL,1] [S+COMMA+CD,2] ||| [GOAL,1] [S+COMMA+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NN+ADJP,2] ||| [GOAL,1] [JJ+NN+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+COMMA+JJ,2] ||| [GOAL,1] [JJ+COMMA+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+PP+SYM,2] ||| [GOAL,1] [NP+PP+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+:+X,2] ||| [GOAL,1] [PP+:+X,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP/PP,2] ||| [GOAL,1] [NP/PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [WP$\SBAR,2] ||| [GOAL,1] [WP$\SBAR,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+DT+JJ,2] ||| [GOAL,1] [:+DT+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+CC+SYM,2] ||| [GOAL,1] [SYM+CC+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [CONJP+JJ,2] ||| [GOAL,1] [CONJP+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+CD+ADJP,2] ||| [GOAL,1] [SYM+CD+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [S+COMMA+DT,2] ||| [GOAL,1] [S+COMMA+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJS+NNS+CC,2] ||| [GOAL,1] [JJS+NNS+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+POS+.,2] ||| [GOAL,1] [NNS+POS+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+IN+PRP$,2] ||| [GOAL,1] [COMMA+IN+PRP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+COMMA+IN,2] ||| [GOAL,1] [ADVP+COMMA+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+DT+IN,2] ||| [GOAL,1] [CC+DT+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [RP+DT+NN,2] ||| [GOAL,1] [RP+DT+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+VBD+NNS,2] ||| [GOAL,1] [PP+VBD+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+VBZ+DT,2] ||| [GOAL,1] [VBN+VBZ+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+COMMA+IN,2] ||| [GOAL,1] [JJ+COMMA+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+WRB+DT,2] ||| [GOAL,1] [NP+WRB+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR/NN,2] ||| [GOAL,1] [SBAR/NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+OOV+COMMA,2] ||| [GOAL,1] [COMMA+OOV+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+NN+.,2] ||| [GOAL,1] [PRP+NN+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+NP+SYM,2] ||| [GOAL,1] [NP+NP+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [FW+.+JJ,2] ||| [GOAL,1] [FW+.+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+JJ+JJ,2] ||| [GOAL,1] [IN+JJ+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NX+ADJP,2] ||| [GOAL,1] [NX+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [S+:+VBD,2] ||| [GOAL,1] [S+:+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+RB,2] ||| [GOAL,1] [JJ+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [QP+PP,2] ||| [GOAL,1] [QP+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+NP+NN,2] ||| [GOAL,1] [DT+NP+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+PP+COMMA,2] ||| [GOAL,1] [JJ+PP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+NP+WHNP,2] ||| [GOAL,1] [RB+NP+WHNP,2] ||| -1
+[GOAL] ||| [GOAL,1] [X+COMMA,2] ||| [GOAL,1] [X+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VB+TO,2] ||| [GOAL,1] [COMMA+VB+TO,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP/.,2] ||| [GOAL,1] [NP/.,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR/NX,2] ||| [GOAL,1] [SBAR/NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+OOV,2] ||| [GOAL,1] [CD+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+NP+VP,2] ||| [GOAL,1] [NP+NP+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP/:,2] ||| [GOAL,1] [NP/:,2] ||| -1
+[GOAL] ||| [GOAL,1] [WRB+LS+SYM,2] ||| [GOAL,1] [WRB+LS+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+NNS+CC,2] ||| [GOAL,1] [PP+NNS+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+DT+JJ,2] ||| [GOAL,1] [CC+DT+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR/NP,2] ||| [GOAL,1] [SBAR/NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+CC+JJ,2] ||| [GOAL,1] [VBP+CC+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+QP,2] ||| [GOAL,1] [JJ+QP,2] ||| -1
+[GOAL] ||| [GOAL,1] [QP+IN,2] ||| [GOAL,1] [QP+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [FW+SYM+CC,2] ||| [GOAL,1] [FW+SYM+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [TO+CD+NX,2] ||| [GOAL,1] [TO+CD+NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+CC+CD,2] ||| [GOAL,1] [:+CC+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [VP+:+S,2] ||| [GOAL,1] [VP+:+S,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+JJ+VB,2] ||| [GOAL,1] [:+JJ+VB,2] ||| -1
+[GOAL] ||| [GOAL,1] [.+SYM,2] ||| [GOAL,1] [.+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VB+VB,2] ||| [GOAL,1] [COMMA+VB+VB,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+NP+WP$,2] ||| [GOAL,1] [RB+NP+WP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+CC+OOV,2] ||| [GOAL,1] [NN+CC+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+NNS+VP,2] ||| [GOAL,1] [IN+NNS+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [OOV+OOV+SYM,2] ||| [GOAL,1] [OOV+OOV+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [TO+RB+VB,2] ||| [GOAL,1] [TO+RB+VB,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+PRP,2] ||| [GOAL,1] [NN+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+NAC,2] ||| [GOAL,1] [DT+NAC,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+NX+CC,2] ||| [GOAL,1] [PP+NX+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+PRN,2] ||| [GOAL,1] [NN+PRN,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+NP+RB,2] ||| [GOAL,1] [RB+NP+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+CD+COMMA,2] ||| [GOAL,1] [JJ+CD+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+TO,2] ||| [GOAL,1] [JJ+TO,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR+PRP$,2] ||| [GOAL,1] [SBAR+PRP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+COMMA+VB,2] ||| [GOAL,1] [PP+COMMA+VB,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+S+.,2] ||| [GOAL,1] [:+S+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+COMMA+FW,2] ||| [GOAL,1] [JJ+COMMA+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBG+.,2] ||| [GOAL,1] [VBG+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [CONJP+IN,2] ||| [GOAL,1] [CONJP+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR/PP,2] ||| [GOAL,1] [SBAR/PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+OOV,2] ||| [GOAL,1] [NNS+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+WRB+NP,2] ||| [GOAL,1] [PP+WRB+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+SBAR,2] ||| [GOAL,1] [PP+SBAR,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+COMMA+VP,2] ||| [GOAL,1] [PP+COMMA+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [RP+PP,2] ||| [GOAL,1] [RP+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN\SINV,2] ||| [GOAL,1] [IN\SINV,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+NNS,2] ||| [GOAL,1] [VBD+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP+RB+.,2] ||| [GOAL,1] [ADJP+RB+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [TO+CD+NN,2] ||| [GOAL,1] [TO+CD+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP+NNS+VP,2] ||| [GOAL,1] [ADJP+NNS+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NNP,2] ||| [GOAL,1] [CC+NNP,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+JJ+NNS,2] ||| [GOAL,1] [:+JJ+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+CC,2] ||| [GOAL,1] [CC+NP+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+RB+CD,2] ||| [GOAL,1] [CC+RB+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+NN+VBZ,2] ||| [GOAL,1] [IN+NN+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJR+NN+VBN,2] ||| [GOAL,1] [JJR+NN+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [TO+VB+ADJP,2] ||| [GOAL,1] [TO+VB+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+COMMA+JJ,2] ||| [GOAL,1] [ADVP+COMMA+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+''+COMMA,2] ||| [GOAL,1] [NN+''+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NNS,2] ||| [GOAL,1] [CC+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+FW+.,2] ||| [GOAL,1] [PP+FW+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [QP+JJ,2] ||| [GOAL,1] [QP+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP/RB,2] ||| [GOAL,1] [NP/RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP$+ADJP+NN,2] ||| [GOAL,1] [PRP$+ADJP+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+VBZ,2] ||| [GOAL,1] [NNS+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBZ+:,2] ||| [GOAL,1] [NP+VBZ+:,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+:,2] ||| [GOAL,1] [VB+:,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+NX+COMMA,2] ||| [GOAL,1] [DT+NX+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+VBD,2] ||| [GOAL,1] [NNS+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [.+PRP,2] ||| [GOAL,1] [.+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+VBG,2] ||| [GOAL,1] [NNS+VBG,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+VBP,2] ||| [GOAL,1] [NNS+VBP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP$+ADJP,2] ||| [GOAL,1] [PRP$+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP\NP,2] ||| [GOAL,1] [NP\NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBZ+.,2] ||| [GOAL,1] [NP+VBZ+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+.,2] ||| [GOAL,1] [VB+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR/RB,2] ||| [GOAL,1] [SBAR/RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NNS+CC,2] ||| [GOAL,1] [CC+NNS+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+VP,2] ||| [GOAL,1] [JJ+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+S+CC,2] ||| [GOAL,1] [CC+S+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NNS+IN,2] ||| [GOAL,1] [JJ+NNS+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+NN+.,2] ||| [GOAL,1] [VBN+NN+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+VBN,2] ||| [GOAL,1] [NNS+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+COMMA+DT,2] ||| [GOAL,1] [JJ+COMMA+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [VP+VBP+DT,2] ||| [GOAL,1] [VP+VBP+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP\NX,2] ||| [GOAL,1] [NP\NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+RB+DT,2] ||| [GOAL,1] [CC+RB+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNP+:+CD,2] ||| [GOAL,1] [NNP+:+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+FW+.,2] ||| [GOAL,1] [NNS+FW+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+VB,2] ||| [GOAL,1] [JJ+VB,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+SYM,2] ||| [GOAL,1] [NP+VBD+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+RB+VBZ,2] ||| [GOAL,1] [PP+RB+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [TO+CD+PP,2] ||| [GOAL,1] [TO+CD+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+DT+NN,2] ||| [GOAL,1] [CC+DT+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NNS+JJ,2] ||| [GOAL,1] [JJ+NNS+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+ADJP+COMMA,2] ||| [GOAL,1] [NNS+ADJP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+CC+OOV,2] ||| [GOAL,1] [COMMA+CC+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+DT+NP,2] ||| [GOAL,1] [CC+DT+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+CD+NNS,2] ||| [GOAL,1] [JJ+CD+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [RP+NP,2] ||| [GOAL,1] [RP+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+RB+VBD,2] ||| [GOAL,1] [PP+RB+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+'',2] ||| [GOAL,1] [RB+'',2] ||| -1
+[GOAL] ||| [GOAL,1] [NP/FW,2] ||| [GOAL,1] [NP/FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+NP+CC,2] ||| [GOAL,1] [DT+NP+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+SYM+SYM,2] ||| [GOAL,1] [CD+SYM+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+OOV+NN,2] ||| [GOAL,1] [RB+OOV+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+CC+NP,2] ||| [GOAL,1] [COMMA+CC+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+CC+NN,2] ||| [GOAL,1] [COMMA+CC+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR/CD,2] ||| [GOAL,1] [SBAR/CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+COMMA+CC,2] ||| [GOAL,1] [JJ+COMMA+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [QP+VBP,2] ||| [GOAL,1] [QP+VBP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+CC+NX,2] ||| [GOAL,1] [COMMA+CC+NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [QP+COMMA+CD,2] ||| [GOAL,1] [QP+COMMA+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+NNS+IN,2] ||| [GOAL,1] [IN+NNS+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+COMMA+CD,2] ||| [GOAL,1] [JJ+COMMA+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+JJ+CD,2] ||| [GOAL,1] [CD+JJ+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+JJ+CC,2] ||| [GOAL,1] [CD+JJ+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [''+JJ+NN,2] ||| [GOAL,1] [''+JJ+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+VBZ+NP,2] ||| [GOAL,1] [VBN+VBZ+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+PP+DT,2] ||| [GOAL,1] [:+PP+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [RP+NP+COMMA,2] ||| [GOAL,1] [RP+NP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [QP\S,2] ||| [GOAL,1] [QP\S,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+:+SYM,2] ||| [GOAL,1] [VBP+:+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+WRB+PRP,2] ||| [GOAL,1] [COMMA+WRB+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [.+NN,2] ||| [GOAL,1] [.+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NX+NNS+NNS,2] ||| [GOAL,1] [NX+NNS+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [.+NP,2] ||| [GOAL,1] [.+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+NX+COMMA,2] ||| [GOAL,1] [NN+NX+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+WHPP+DT,2] ||| [GOAL,1] [NP+WHPP+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [FRAG/CD,2] ||| [GOAL,1] [FRAG/CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+NNS+IN,2] ||| [GOAL,1] [PP+NNS+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+NX+NN,2] ||| [GOAL,1] [PP+NX+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NX+CC+OOV,2] ||| [GOAL,1] [NX+CC+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+ADJP+.,2] ||| [GOAL,1] [NN+ADJP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [POS,2] ||| [GOAL,1] [POS,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VB+JJ,2] ||| [GOAL,1] [COMMA+VB+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [PDT+DT,2] ||| [GOAL,1] [PDT+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [QP+VP,2] ||| [GOAL,1] [QP+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+NP+COMMA,2] ||| [GOAL,1] [:+NP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+JJ+DT,2] ||| [GOAL,1] [IN+JJ+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+COMMA,2] ||| [GOAL,1] [CC+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+VBZ+RB,2] ||| [GOAL,1] [JJ+VBZ+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [.+MD,2] ||| [GOAL,1] [.+MD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+NN+NN,2] ||| [GOAL,1] [NNS+NN+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+NP+JJ,2] ||| [GOAL,1] [SYM+NP+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBZ+PDT,2] ||| [GOAL,1] [VBZ+PDT,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+NN+NX,2] ||| [GOAL,1] [NNS+NN+NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+CC+MD,2] ||| [GOAL,1] [COMMA+CC+MD,2] ||| -1
+[GOAL] ||| [GOAL,1] [FW+NP+.,2] ||| [GOAL,1] [FW+NP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [OOV+SYM+JJ,2] ||| [GOAL,1] [OOV+SYM+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP+NNS+IN,2] ||| [GOAL,1] [ADJP+NNS+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+JJ+ADJP,2] ||| [GOAL,1] [CC+JJ+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+:+OOV,2] ||| [GOAL,1] [NP+:+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+PRP$+JJ,2] ||| [GOAL,1] [CC+PRP$+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+JJ+UCP,2] ||| [GOAL,1] [IN+JJ+UCP,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+SYM+NP,2] ||| [GOAL,1] [DT+SYM+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+RBS,2] ||| [GOAL,1] [NP+RBS,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+JJ+NNP,2] ||| [GOAL,1] [NN+JJ+NNP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+JJ+NNS,2] ||| [GOAL,1] [NN+JJ+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+NNS+POS,2] ||| [GOAL,1] [NN+NNS+POS,2] ||| -1
+[GOAL] ||| [GOAL,1] [VP+VBP+RB,2] ||| [GOAL,1] [VP+VBP+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+JJ+CC,2] ||| [GOAL,1] [IN+JJ+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBG+NN+COMMA,2] ||| [GOAL,1] [VBG+NN+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+JJ+NN,2] ||| [GOAL,1] [NP+JJ+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+JJ+CD,2] ||| [GOAL,1] [IN+JJ+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP$+NN+JJ,2] ||| [GOAL,1] [PRP$+NN+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+NP+IN,2] ||| [GOAL,1] [SYM+NP+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+DT+NN,2] ||| [GOAL,1] [VB+DT+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+COMMA+NNS,2] ||| [GOAL,1] [JJ+COMMA+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [S/RRC,2] ||| [GOAL,1] [S/RRC,2] ||| -1
+[GOAL] ||| [GOAL,1] [VP+SYM+ADJP,2] ||| [GOAL,1] [VP+SYM+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+RB+IN,2] ||| [GOAL,1] [CC+RB+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+NP+JJ,2] ||| [GOAL,1] [RB+NP+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [QP+VBZ,2] ||| [GOAL,1] [QP+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+PRN+CC,2] ||| [GOAL,1] [NN+PRN+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+COMMA,2] ||| [GOAL,1] [PRP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+DT+CD,2] ||| [GOAL,1] [:+DT+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+PRP$,2] ||| [GOAL,1] [COMMA+PRP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBP+NNS,2] ||| [GOAL,1] [NP+VBP+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+JJ+ADJP,2] ||| [GOAL,1] [COMMA+JJ+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+TO+SYM,2] ||| [GOAL,1] [VBD+TO+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [WRB,2] ||| [GOAL,1] [WRB,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+OOV,2] ||| [GOAL,1] [SYM+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR/EX,2] ||| [GOAL,1] [SBAR/EX,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+ADVP,2] ||| [GOAL,1] [NN+ADVP,2] ||| -1
+[GOAL] ||| [GOAL,1] [X/.,2] ||| [GOAL,1] [X/.,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+JJ+VBN,2] ||| [GOAL,1] [COMMA+JJ+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+NNP,2] ||| [GOAL,1] [IN+NNP,2] ||| -1
+[GOAL] ||| [GOAL,1] [POS+VBD,2] ||| [GOAL,1] [POS+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+NP+IN,2] ||| [GOAL,1] [RB+NP+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+S,2] ||| [GOAL,1] [NP+S,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+X,2] ||| [GOAL,1] [NP+X,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+NNS,2] ||| [GOAL,1] [IN+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+JJR+NN,2] ||| [GOAL,1] [PP+JJR+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [POS+VBN,2] ||| [GOAL,1] [POS+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+VP+.,2] ||| [GOAL,1] [VBN+VP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+PRP+VBD,2] ||| [GOAL,1] [DT+PRP+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+EX+VP,2] ||| [GOAL,1] [RB+EX+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [WHNP+VBZ+TO,2] ||| [GOAL,1] [WHNP+VBZ+TO,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+VBN+IN,2] ||| [GOAL,1] [PP+VBN+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+SYM+SYM,2] ||| [GOAL,1] [PP+SYM+SYM,2] ||| -1
+[GOAL] |
<TRUNCATED>
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/packed/grammar.gz
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/packed/grammar.gz b/joshua-core/src/test/resources/bn-en/packed/grammar.gz
new file mode 100644
index 0000000..ae73430
Binary files /dev/null and b/joshua-core/src/test/resources/bn-en/packed/grammar.gz differ
[43/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/JoshuaConfiguration.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/JoshuaConfiguration.java b/joshua-core/src/main/java/org/apache/joshua/decoder/JoshuaConfiguration.java
new file mode 100644
index 0000000..e7ad3b4
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/JoshuaConfiguration.java
@@ -0,0 +1,729 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder;
+
+import static org.apache.joshua.util.FormatUtils.cleanNonTerminal;
+import static org.apache.joshua.util.FormatUtils.ensureNonTerminalBrackets;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.util.ArrayList;
+import java.util.Collections;
+
+import org.apache.joshua.decoder.ff.StatefulFF;
+import org.apache.joshua.decoder.ff.fragmentlm.Tree;
+import org.apache.joshua.util.FormatUtils;
+import org.apache.joshua.util.Regex;
+import org.apache.joshua.util.io.LineReader;
+import org.apache.log4j.Level;
+import org.apache.log4j.LogManager;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Configuration file for Joshua decoder.
+ *
+ * When adding new features to Joshua, any new configurable parameters should be added to this
+ * class.
+ *
+ * @author Zhifei Li, zhifei.work@gmail.com
+ * @author Matt Post post@cs.jhu.edu
+ */
+public class JoshuaConfiguration {
+
+ private static final Logger LOG = LoggerFactory.getLogger(JoshuaConfiguration.class);
+
+ // whether to construct a StructuredTranslation object for each request instead of
+ // printing to stdout. Used when the Decoder is used from Java directly.
+ public Boolean use_structured_output = false;
+
+ // If set to true, Joshua will lowercase the input, creating an annotation that marks the
+ // original case
+ public boolean lowercase = false;
+
+ // If set to true, Joshua will recapitalize the output by projecting the case from aligned
+ // source-side words
+ public boolean project_case = false;
+
+ // List of grammar files to read
+ public ArrayList<String> tms = new ArrayList<String>();
+
+ // A rule cache for commonly used tries to avoid excess object allocations
+ // Testing shows there's up to ~95% hit rate when cache size is 5000 Trie nodes.
+ public Integer cachedRuleSize = new Integer(5000);
+
+ /*
+ * The file to read the weights from (part of the sparse features implementation). Weights can
+ * also just be listed in the main config file.
+ */
+ public String weights_file = "";
+ // Default symbols. The symbol here should be enclosed in square brackets.
+ public String default_non_terminal = FormatUtils.ensureNonTerminalBrackets("X");
+ public String goal_symbol = FormatUtils.ensureNonTerminalBrackets("GOAL");
+
+ /*
+ * A list of OOV symbols in the form
+ *
+ * [X1] weight [X2] weight [X3] weight ...
+ *
+ * where the [X] symbols are nonterminals and the weights are weights. For each OOV word w in the
+ * input sentence, Joshua will create rules of the form
+ *
+ * X1 -> w (weight)
+ *
+ * If this is empty, an unweighted default_non_terminal is used.
+ */
+ public class OOVItem implements Comparable<OOVItem> {
+ public String label;
+
+ public float weight;
+
+ OOVItem(String l, float w) {
+ label = l;
+ weight = w;
+ }
+ @Override
+ public int compareTo(OOVItem other) {
+ if (weight > other.weight)
+ return -1;
+ else if (weight < other.weight)
+ return 1;
+ return 0;
+ }
+ }
+
+ public ArrayList<OOVItem> oovList = null;
+
+ /*
+ * Whether to segment OOVs into a lattice
+ */
+ public boolean segment_oovs = false;
+
+ /*
+ * Enable lattice decoding.
+ */
+ public boolean lattice_decoding = false;
+
+ /*
+ * If false, sorting of the complete grammar is done at load time. If true, grammar tries are not
+ * sorted till they are first accessed. Amortized sorting means you get your first translation
+ * much, much quicker (good for debugging), but that per-sentence decoding is a bit slower.
+ */
+ public boolean amortized_sorting = true;
+ // syntax-constrained decoding
+ public boolean constrain_parse = false;
+
+ public boolean use_pos_labels = false;
+
+ // oov-specific
+ public boolean true_oovs_only = false;
+
+ /* Dynamic sentence-level filtering. */
+ public boolean filter_grammar = false;
+
+ /* The cube pruning pop limit. Set to 0 for exhaustive pruning. */
+ public int pop_limit = 100;
+
+ /* Maximum sentence length. Sentences longer than this are truncated. */
+ public int maxlen = 200;
+
+ /*
+ * N-best configuration.
+ */
+ // Make sure output strings in the n-best list are unique.
+ public boolean use_unique_nbest = true;
+
+ /* Include the phrasal alignments in the output (not word-level alignmetns at the moment). */
+ public boolean include_align_index = false;
+
+ /* The number of hypotheses to output by default. */
+ public int topN = 1;
+
+ /**
+ * This string describes the format of each line of output from the decoder (i.e., the
+ * translations). The string can include arbitrary text and also variables. The following
+ * variables are available:
+ *
+ * <pre>
+ * - %i the 0-indexed sentence number
+ * - %e the source string %s the translated sentence
+ * - %S the translated sentence with some basic capitalization and denormalization
+ * - %t the synchronous derivation
+ * - %f the list of feature values (as name=value pairs)
+ * - %c the model cost
+ * - %w the weight vector
+ * - %a the alignments between source and target words (currently unimplemented)
+ * - %d a verbose, many-line version of the derivation
+ * </pre>
+ */
+ public String outputFormat = "%i ||| %s ||| %f ||| %c";
+
+ /* The number of decoding threads to use (-threads). */
+ public int num_parallel_decoders = 1;
+
+ /*
+ * When true, _OOV is appended to all words that are passed through (useful for something like
+ * transliteration on the target side
+ */
+ public boolean mark_oovs = false;
+
+ /* Enables synchronous parsing. */
+ public boolean parse = false; // perform synchronous parsing
+
+
+ /* A list of the feature functions. */
+ public ArrayList<String> features = new ArrayList<String>();
+
+ /* A list of weights found in the main config file (instead of in a separate weights file) */
+ public ArrayList<String> weights = new ArrayList<String>();
+
+ /* Determines whether to expect JSON input or plain lines */
+ public enum INPUT_TYPE { plain, json };
+ public INPUT_TYPE input_type = INPUT_TYPE.plain;
+
+ /* Type of server. Not sure we need to keep the regular TCP one around. */
+ public enum SERVER_TYPE { none, TCP, HTTP };
+ public SERVER_TYPE server_type = SERVER_TYPE.TCP;
+
+ /* If set, Joshua will start a (multi-threaded, per "threads") TCP/IP server on this port. */
+ public int server_port = 0;
+
+ /*
+ * Whether to do forest rescoring. If set to true, the references are expected on STDIN along with
+ * the input sentences in the following format:
+ *
+ * input sentence ||| ||| reference1 ||| reference2 ...
+ *
+ * (The second field is reserved for the output sentence for alignment and forced decoding).
+ */
+
+ public boolean rescoreForest = false;
+ public float rescoreForestWeight = 10.0f;
+
+ /*
+ * Location of fragment mapping file, which maps flattened SCFG rules to their internal
+ * representation.
+ */
+ public String fragmentMapFile = null;
+
+ /*
+ * Whether to use soft syntactic constraint decoding /fuzzy matching, which allows that any
+ * nonterminal may be substituted for any other nonterminal (except for OOV and GOAL)
+ */
+ public boolean fuzzy_matching = false;
+
+ public static final String SOFT_SYNTACTIC_CONSTRAINT_DECODING_PROPERTY_NAME = "fuzzy_matching";
+
+ /***
+ * Phrase-based decoding parameters.
+ */
+
+ /* The search algorithm: currently either "cky" or "stack" */
+ public String search_algorithm = "cky";
+
+ /* The distortion limit */
+ public int reordering_limit = 8;
+
+ /* The number of target sides considered for each source side (after sorting by model weight) */
+ public int num_translation_options = 20;
+
+ /* If true, decode using a dot chart (standard CKY+); if false, use the much more efficient
+ * version of Sennrich (SSST 2014)
+ */
+ public boolean use_dot_chart = true;
+
+ /* Moses compatibility */
+ public boolean moses = false;
+
+ /* If true, just print out the weights found in the config file, and exit. */
+ public boolean show_weights_and_quit = false;
+
+ /* Read input from a file (Moses compatible flag) */
+ public String input_file = null;
+
+ /* Write n-best output to this file */
+ public String n_best_file = null;
+
+ /* Whether to look at source side for special annotations */
+ public boolean source_annotations = false;
+
+ /* Weights overridden from the command line */
+ public String weight_overwrite = "";
+
+ /**
+ * This method resets the state of JoshuaConfiguration back to the state after initialization.
+ * This is useful when for example making different calls to the decoder within the same java
+ * program, which otherwise leads to potential errors due to inconsistent state as a result of
+ * loading the configuration multiple times without resetting etc.
+ *
+ * This leads to the insight that in fact it may be an even better idea to refactor the code and
+ * make JoshuaConfiguration an object that is is created and passed as an argument, rather than a
+ * shared static object. This is just a suggestion for the next step.
+ *
+ */
+ public void reset() {
+ LOG.info("Resetting the JoshuaConfiguration to its defaults ...");
+ LOG.info("\n\tResetting the StatefullFF global state index ...");
+ LOG.info("\n\t...done");
+ StatefulFF.resetGlobalStateIndex();
+ tms = new ArrayList<String>();
+ weights_file = "";
+ default_non_terminal = "[X]";
+ oovList = new ArrayList<OOVItem>();
+ oovList.add(new OOVItem(default_non_terminal, 1.0f));
+ goal_symbol = "[GOAL]";
+ amortized_sorting = true;
+ constrain_parse = false;
+ use_pos_labels = false;
+ true_oovs_only = false;
+ filter_grammar = false;
+ pop_limit = 100;
+ maxlen = 200;
+ use_unique_nbest = false;
+ include_align_index = false;
+ topN = 1;
+ outputFormat = "%i ||| %s ||| %f ||| %c";
+ num_parallel_decoders = 1;
+ mark_oovs = false;
+ // oracleFile = null;
+ parse = false; // perform synchronous parsing
+ features = new ArrayList<String>();
+ weights = new ArrayList<String>();
+ server_port = 0;
+
+ reordering_limit = 8;
+ num_translation_options = 20;
+ LOG.info("...done");
+ }
+
+ // ===============================================================
+ // Methods
+ // ===============================================================
+
+ /**
+ * To process command-line options, we write them to a file that looks like the config file, and
+ * then call readConfigFile() on it. It would be more general to define a class that sits on a
+ * stream and knows how to chop it up, but this was quicker to implement.
+ *
+ * @param options string array of command line options
+ */
+ public void processCommandLineOptions(String[] options) {
+ try {
+ File tmpFile = File.createTempFile("options", null, null);
+ PrintWriter out = new PrintWriter(new FileWriter(tmpFile));
+
+ for (int i = 0; i < options.length; i++) {
+ String key = options[i].substring(1);
+ if (i + 1 == options.length || options[i + 1].startsWith("-")) {
+ // if this is the last item, or if the next item
+ // is another flag, then this is a boolean flag
+ out.println(key + " = true");
+
+ } else {
+ out.print(key + " =");
+ while (i + 1 < options.length && ! options[i + 1].startsWith("-")) {
+ out.print(String.format(" %s", options[i + 1]));
+ i++;
+ }
+ out.println();
+ }
+ }
+ out.close();
+
+// LOG.info("Parameters overridden from the command line:");
+ this.readConfigFile(tmpFile.getCanonicalPath());
+
+ tmpFile.delete();
+
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public void readConfigFile(String configFile) throws IOException {
+
+ LineReader configReader = new LineReader(configFile, false);
+ try {
+ for (String line : configReader) {
+ line = line.trim(); // .toLowerCase();
+
+ if (Regex.commentOrEmptyLine.matches(line))
+ continue;
+
+ /*
+ * There are two kinds of substantive (non-comment, non-blank) lines: parameters and feature
+ * values. Parameters match the pattern "key = value"; all other substantive lines are
+ * interpreted as features.
+ */
+
+ if (line.indexOf("=") != -1) { // parameters; (not feature function)
+ String[] fds = Regex.equalsWithSpaces.split(line, 2);
+ if (fds.length < 2) {
+ LOG.warn("skipping config file line '{}'", line);
+ continue;
+ }
+
+ String parameter = normalize_key(fds[0]);
+
+ if (parameter.equals(normalize_key("lm"))) {
+ /* This is deprecated. This support old LM lines of the form
+ *
+ * lm = berkeleylm 5 false false 100 lm.gz
+ *
+ * LMs are now loaded as general feature functions, so we transform that to either
+ *
+ * LanguageModel -lm_order 5 -lm_type berkeleylm -lm_file lm.gz
+ *
+ * If the line were state minimizing:
+ *
+ * lm = kenlm 5 true false 100 lm.gz
+ *
+ * StateMinimizingLanguageModel -lm_order 5 -lm_file lm.gz
+ */
+
+ String[] tokens = fds[1].split("\\s+");
+ if (tokens[2].equals("true"))
+ features.add(String.format("StateMinimizingLanguageModel -lm_type kenlm -lm_order %s -lm_file %s",
+ tokens[1], tokens[5]));
+ else
+ features.add(String.format("LanguageModel -lm_type %s -lm_order %s -lm_file %s",
+ tokens[0], tokens[1], tokens[5]));
+
+ } else if (parameter.equals(normalize_key("tm"))) {
+ /* If found, convert old format:
+ * tm = TYPE OWNER MAXSPAN PATH
+ * to new format
+ * tm = TYPE -owner OWNER -maxspan MAXSPAN -path PATH
+ */
+ String tmLine = fds[1];
+
+ String[] tokens = fds[1].split("\\s+");
+ if (! tokens[1].startsWith("-")) { // old format
+ tmLine = String.format("%s -owner %s -maxspan %s -path %s", tokens[0], tokens[1], tokens[2], tokens[3]);
+ LOG.warn("Converting deprecated TM line from '{}' -> '{}'", fds[1], tmLine);
+ }
+ tms.add(tmLine);
+
+ } else if (parameter.equals("v")) {
+
+ // This is already handled in ArgsParser, skip it here, easier than removing it there
+
+ } else if (parameter.equals(normalize_key("parse"))) {
+ parse = Boolean.parseBoolean(fds[1]);
+ LOG.debug("parse: {}", parse);
+
+ } else if (parameter.equals(normalize_key("oov-list"))) {
+ if (new File(fds[1]).exists()) {
+ oovList = new ArrayList<OOVItem>();
+ try {
+ File file = new File(fds[1]);
+ BufferedReader br = new BufferedReader(new FileReader(file));
+ try {
+ String str = br.readLine();
+ while (str != null) {
+ String[] tokens = str.trim().split("\\s+");
+
+ oovList.add(new OOVItem(FormatUtils.ensureNonTerminalBrackets(tokens[0]),
+ (float) Math.log(Float.parseFloat(tokens[1]))));
+
+ str = br.readLine();
+ }
+ br.close();
+ } catch(IOException e){
+ System.out.println(e);
+ }
+ } catch(IOException e){
+ System.out.println(e);
+ }
+ Collections.sort(oovList);
+
+ } else {
+ String[] tokens = fds[1].trim().split("\\s+");
+ if (tokens.length % 2 != 0) {
+ throw new RuntimeException(String.format("* FATAL: invalid format for '%s'", fds[0]));
+ }
+ oovList = new ArrayList<OOVItem>();
+
+ for (int i = 0; i < tokens.length; i += 2)
+ oovList.add(new OOVItem(FormatUtils.ensureNonTerminalBrackets(tokens[i]),
+ (float) Math.log(Float.parseFloat(tokens[i + 1]))));
+
+ Collections.sort(oovList);
+ }
+
+ } else if (parameter.equals(normalize_key("lattice-decoding"))) {
+ lattice_decoding = true;
+
+ } else if (parameter.equals(normalize_key("segment-oovs"))) {
+ segment_oovs = true;
+ lattice_decoding = true;
+
+ } else if (parameter.equals(normalize_key("default-non-terminal"))) {
+ default_non_terminal = ensureNonTerminalBrackets(cleanNonTerminal(fds[1].trim()));
+ LOG.debug("default_non_terminal: {}", default_non_terminal);
+
+ } else if (parameter.equals(normalize_key("goal-symbol"))) {
+ goal_symbol = ensureNonTerminalBrackets(cleanNonTerminal(fds[1].trim()));
+ LOG.debug("goalSymbol: {}", goal_symbol);
+
+ } else if (parameter.equals(normalize_key("weights-file"))) {
+ weights_file = fds[1];
+
+ } else if (parameter.equals(normalize_key("constrain_parse"))) {
+ constrain_parse = Boolean.parseBoolean(fds[1]);
+
+ } else if (parameter.equals(normalize_key("true_oovs_only"))) {
+ true_oovs_only = Boolean.parseBoolean(fds[1]);
+
+ } else if (parameter.equals(normalize_key("filter-grammar"))) {
+ filter_grammar = Boolean.parseBoolean(fds[1]);
+
+ } else if (parameter.equals(normalize_key("amortize"))) {
+ amortized_sorting = Boolean.parseBoolean(fds[1]);
+
+ } else if (parameter.equals(normalize_key("use_pos_labels"))) {
+ use_pos_labels = Boolean.parseBoolean(fds[1]);
+
+ } else if (parameter.equals(normalize_key("use_unique_nbest"))) {
+ use_unique_nbest = Boolean.valueOf(fds[1]);
+ LOG.debug("use_unique_nbest: {}", use_unique_nbest);
+
+ } else if (parameter.equals(normalize_key("output-format"))) {
+ outputFormat = fds[1];
+ LOG.debug("output-format: {}", outputFormat);
+
+ } else if (parameter.equals(normalize_key("include_align_index"))) {
+ include_align_index = Boolean.valueOf(fds[1]);
+ LOG.debug("include_align_index: {}", include_align_index);
+
+ } else if (parameter.equals(normalize_key("top_n"))) {
+ topN = Integer.parseInt(fds[1]);
+ LOG.debug("topN: {}", topN);
+
+ } else if (parameter.equals(normalize_key("num_parallel_decoders"))
+ || parameter.equals(normalize_key("threads"))) {
+ num_parallel_decoders = Integer.parseInt(fds[1]);
+ if (num_parallel_decoders <= 0) {
+ throw new IllegalArgumentException(
+ "Must specify a positive number for num_parallel_decoders");
+ }
+ LOG.debug("num_parallel_decoders: {}", num_parallel_decoders);
+
+ } else if (parameter.equals(normalize_key("mark_oovs"))) {
+ mark_oovs = Boolean.valueOf(fds[1]);
+ LOG.debug("mark_oovs: {}", mark_oovs);
+
+ } else if (parameter.equals(normalize_key("pop-limit"))) {
+ pop_limit = Integer.parseInt(fds[1]);
+ LOG.info("pop-limit: {}", pop_limit);
+
+ } else if (parameter.equals(normalize_key("input-type"))) {
+ if (fds[1].equals("json")) {
+ input_type = INPUT_TYPE.json;
+ } else if (fds[1].equals("plain")) {
+ input_type = INPUT_TYPE.plain;
+ } else {
+ throw new RuntimeException(String.format("* FATAL: invalid server type '%s'", fds[1]));
+ }
+ LOG.info(" input-type: {}", input_type);
+
+ } else if (parameter.equals(normalize_key("server-type"))) {
+ if (fds[1].toLowerCase().equals("tcp"))
+ server_type = SERVER_TYPE.TCP;
+ else if (fds[1].toLowerCase().equals("http"))
+ server_type = SERVER_TYPE.HTTP;
+
+ LOG.info(" server-type: {}", server_type);
+
+ } else if (parameter.equals(normalize_key("server-port"))) {
+ server_port = Integer.parseInt(fds[1]);
+ LOG.info(" server-port: {}", server_port);
+
+ } else if (parameter.equals(normalize_key("rescore-forest"))) {
+ rescoreForest = true;
+ LOG.info(" rescore-forest: {}", rescoreForest);
+
+ } else if (parameter.equals(normalize_key("rescore-forest-weight"))) {
+ rescoreForestWeight = Float.parseFloat(fds[1]);
+ LOG.info(" rescore-forest-weight: {}", rescoreForestWeight);
+
+ } else if (parameter.equals(normalize_key("maxlen"))) {
+ // reset the maximum length
+ maxlen = Integer.parseInt(fds[1]);
+
+ } else if (parameter.equals("c") || parameter.equals("config")) {
+ // this was used to send in the config file, just ignore it
+ ;
+
+ } else if (parameter.equals(normalize_key("feature-function"))) {
+ // add the feature to the list of features for later processing
+ features.add(fds[1]);
+
+ } else if (parameter.equals(normalize_key("maxlen"))) {
+ // add the feature to the list of features for later processing
+ maxlen = Integer.parseInt(fds[1]);
+
+ } else if (parameter
+ .equals(normalize_key(SOFT_SYNTACTIC_CONSTRAINT_DECODING_PROPERTY_NAME))) {
+ fuzzy_matching = Boolean.parseBoolean(fds[1]);
+ LOG.debug("fuzzy_matching: {}", fuzzy_matching);
+
+ } else if (parameter.equals(normalize_key("fragment-map"))) {
+ fragmentMapFile = fds[1];
+ Tree.readMapping(fragmentMapFile);
+
+ /** PHRASE-BASED PARAMETERS **/
+ } else if (parameter.equals(normalize_key("search"))) {
+ search_algorithm = fds[1];
+
+ if (!search_algorithm.equals("cky") && !search_algorithm.equals("stack")) {
+ throw new RuntimeException(
+ "-search must be one of 'stack' (for phrase-based decoding) " +
+ "or 'cky' (for hierarchical / syntactic decoding)");
+ }
+
+ if (search_algorithm.equals("cky") && include_align_index) {
+ throw new RuntimeException(
+ "include_align_index is currently not supported with cky search");
+ }
+
+ } else if (parameter.equals(normalize_key("reordering-limit"))) {
+ reordering_limit = Integer.parseInt(fds[1]);
+
+ } else if (parameter.equals(normalize_key("num-translation-options"))) {
+ num_translation_options = Integer.parseInt(fds[1]);
+
+ } else if (parameter.equals(normalize_key("no-dot-chart"))) {
+ use_dot_chart = false;
+
+ } else if (parameter.equals(normalize_key("moses"))) {
+ moses = true; // triggers some Moses-specific compatibility options
+
+ } else if (parameter.equals(normalize_key("show-weights"))) {
+ show_weights_and_quit = true;
+
+ } else if (parameter.equals(normalize_key("n-best-list"))) {
+ // for Moses compatibility
+ String[] tokens = fds[1].split("\\s+");
+ n_best_file = tokens[0];
+ if (tokens.length > 1)
+ topN = Integer.parseInt(tokens[1]);
+
+ } else if (parameter.equals(normalize_key("input-file"))) {
+ // for Moses compatibility
+ input_file = fds[1];
+
+ } else if (parameter.equals(normalize_key("weight-file"))) {
+ // for Moses, ignore
+
+ } else if (parameter.equals(normalize_key("weight-overwrite"))) {
+ weight_overwrite = fds[1];
+
+ } else if (parameter.equals(normalize_key("source-annotations"))) {
+ // Check source sentence
+ source_annotations = true;
+
+ } else if (parameter.equals(normalize_key("cached-rules-size"))) {
+ // Check source sentence
+ cachedRuleSize = Integer.parseInt(fds[1]);
+ } else if (parameter.equals(normalize_key("lowercase"))) {
+ lowercase = true;
+
+ } else if (parameter.equals(normalize_key("project-case"))) {
+ project_case = true;
+
+ } else {
+
+ if (parameter.equals(normalize_key("use-sent-specific-tm"))
+ || parameter.equals(normalize_key("add-combined-cost"))
+ || parameter.equals(normalize_key("use-tree-nbest"))
+ || parameter.equals(normalize_key("use-kenlm"))
+ || parameter.equals(normalize_key("useCubePrune"))
+ || parameter.equals(normalize_key("useBeamAndThresholdPrune"))
+ || parameter.equals(normalize_key("regexp-grammar"))) {
+ LOG.warn("ignoring deprecated parameter '{}'", fds[0]);
+
+ } else {
+ throw new RuntimeException("FATAL: unknown configuration parameter '" + fds[0] + "'");
+ }
+ }
+
+ LOG.info(" {} = '{}'", normalize_key(fds[0]), fds[1]);
+
+ } else {
+ /*
+ * Lines that don't have an equals sign and are not blank lines, empty lines, or comments,
+ * are feature values, which can be present in this file
+ */
+
+ weights.add(line);
+ }
+ }
+ } finally {
+ configReader.close();
+ }
+ }
+
+ /**
+ * Checks for invalid variable configurations
+ */
+ public void sanityCheck() {
+ }
+
+ /**
+ * Sets the verbosity level to v (0: OFF; 1: INFO; 2: DEBUG).
+ *
+ * @param v the verbosity level (0, 1, or 2)
+ */
+ public void setVerbosity(int v) {
+ Decoder.VERBOSE = v;
+ switch (Decoder.VERBOSE) {
+ case 0:
+ LogManager.getRootLogger().setLevel(Level.OFF);
+ break;
+ case 1:
+ LogManager.getRootLogger().setLevel(Level.INFO);
+ break;
+ case 2:
+ LogManager.getRootLogger().setLevel(Level.DEBUG);
+ break;
+ }
+ }
+
+ /**
+ * Normalizes parameter names by removing underscores and hyphens and lowercasing. This defines
+ * equivalence classes on external use of parameter names, permitting arbitrary_under_scores and
+ * camelCasing in paramter names without forcing the user to memorize them all. Here are some
+ * examples of equivalent ways to refer to parameter names:
+ * <pre>
+ * {pop-limit, poplimit, PopLimit, popLimit, pop_lim_it} {lmfile, lm-file, LM-FILE, lm_file}
+ * </pre>
+ *
+ * @param text the string to be normalized
+ * @return normalized key
+ *
+ */
+ public static String normalize_key(String text) {
+ return text.replaceAll("[-_]", "").toLowerCase();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/JoshuaDecoder.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/JoshuaDecoder.java b/joshua-core/src/main/java/org/apache/joshua/decoder/JoshuaDecoder.java
new file mode 100644
index 0000000..4c31655
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/JoshuaDecoder.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder;
+
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.net.InetSocketAddress;
+
+import com.sun.net.httpserver.HttpServer;
+
+import org.apache.joshua.decoder.JoshuaConfiguration.SERVER_TYPE;
+import org.apache.joshua.decoder.io.TranslationRequestStream;
+import org.apache.joshua.server.TcpServer;
+import org.apache.log4j.Level;
+import org.apache.log4j.LogManager;
+import org.apache.joshua.server.ServerThread;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Implements decoder initialization, including interaction with <code>JoshuaConfiguration</code>
+ * and <code>DecoderThread</code>.
+ *
+ * @author Zhifei Li, zhifei.work@gmail.com
+ * @author wren ng thornton wren@users.sourceforge.net
+ * @author Lane Schwartz dowobeha@users.sourceforge.net
+ */
+public class JoshuaDecoder {
+
+ private static final Logger LOG = LoggerFactory.getLogger(JoshuaDecoder.class);
+
+ // ===============================================================
+ // Main
+ // ===============================================================
+ public static void main(String[] args) throws IOException {
+
+ // default log level
+ LogManager.getRootLogger().setLevel(Level.INFO);
+
+ JoshuaConfiguration joshuaConfiguration = new JoshuaConfiguration();
+ ArgsParser userArgs = new ArgsParser(args,joshuaConfiguration);
+
+ long startTime = System.currentTimeMillis();
+
+ /* Step-0: some sanity checking */
+ joshuaConfiguration.sanityCheck();
+
+ /* Step-1: initialize the decoder, test-set independent */
+ Decoder decoder = new Decoder(joshuaConfiguration, userArgs.getConfigFile());
+
+ LOG.info("Model loading took {} seconds", (System.currentTimeMillis() - startTime) / 1000);
+ LOG.info("Memory used {} MB", ((Runtime.getRuntime().totalMemory()
+ - Runtime.getRuntime().freeMemory()) / 1000000.0));
+
+ /* Step-2: Decoding */
+ // create a server if requested, which will create TranslationRequest objects
+ if (joshuaConfiguration.server_port > 0) {
+ int port = joshuaConfiguration.server_port;
+ if (joshuaConfiguration.server_type == SERVER_TYPE.TCP) {
+ new TcpServer(decoder, port, joshuaConfiguration).start();
+
+ } else if (joshuaConfiguration.server_type == SERVER_TYPE.HTTP) {
+ joshuaConfiguration.use_structured_output = true;
+
+ HttpServer server = HttpServer.create(new InetSocketAddress(port), 0);
+ LOG.info("HTTP Server running and listening on port {}.", port);
+ server.createContext("/", new ServerThread(null, decoder, joshuaConfiguration));
+ server.setExecutor(null); // creates a default executor
+ server.start();
+ } else {
+ LOG.error("Unknown server type");
+ System.exit(1);
+ }
+ return;
+ }
+
+ // Create a TranslationRequest object, reading from a file if requested, or from STDIN
+ InputStream input = (joshuaConfiguration.input_file != null)
+ ? new FileInputStream(joshuaConfiguration.input_file)
+ : System.in;
+
+ BufferedReader reader = new BufferedReader(new InputStreamReader(input));
+ TranslationRequestStream fileRequest = new TranslationRequestStream(reader, joshuaConfiguration);
+ Translations translations = decoder.decodeAll(fileRequest);
+
+ // Create the n-best output stream
+ FileWriter nbest_out = null;
+ if (joshuaConfiguration.n_best_file != null)
+ nbest_out = new FileWriter(joshuaConfiguration.n_best_file);
+
+ for (Translation translation: translations) {
+
+ /**
+ * We need to munge the feature value outputs in order to be compatible with Moses tuners.
+ * Whereas Joshua writes to STDOUT whatever is specified in the `output-format` parameter,
+ * Moses expects the simple translation on STDOUT and the n-best list in a file with a fixed
+ * format.
+ */
+ if (joshuaConfiguration.moses) {
+ String text = translation.toString().replaceAll("=", "= ");
+ // Write the complete formatted string to STDOUT
+ if (joshuaConfiguration.n_best_file != null)
+ nbest_out.write(text);
+
+ // Extract just the translation and output that to STDOUT
+ text = text.substring(0, text.indexOf('\n'));
+ String[] fields = text.split(" \\|\\|\\| ");
+ text = fields[1];
+
+ System.out.println(text);
+
+ } else {
+ System.out.print(translation.toString());
+ }
+ }
+
+ if (joshuaConfiguration.n_best_file != null)
+ nbest_out.close();
+
+ LOG.info("Decoding completed.");
+ LOG.info("Memory used {} MB", ((Runtime.getRuntime().totalMemory()
+ - Runtime.getRuntime().freeMemory()) / 1000000.0));
+
+ /* Step-3: clean up */
+ decoder.cleanUp();
+ LOG.info("Total running time: {} seconds", (System.currentTimeMillis() - startTime) / 1000);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/NbestMinRiskReranker.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/NbestMinRiskReranker.java b/joshua-core/src/main/java/org/apache/joshua/decoder/NbestMinRiskReranker.java
new file mode 100644
index 0000000..9f63cad
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/NbestMinRiskReranker.java
@@ -0,0 +1,446 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Scanner;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.PriorityBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.joshua.util.Ngram;
+import org.apache.joshua.util.Regex;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * this class implements: (1) nbest min risk (MBR) reranking using BLEU as a gain funtion.
+ * <p>
+ * This assume that the string is unique in the nbest list In Hiero, due to spurious ambiguity, a
+ * string may correspond to many possible derivations, and ideally the probability of a string
+ * should be the sum of all the derivataions leading to that string. But, in practice, one normally
+ * uses a Viterbi approximation: the probability of a string is its best derivation probability So,
+ * if one want to deal with spurious ambiguity, he/she should do that before calling this class
+ *
+ * @author Zhifei Li, zhifei.work@gmail.com
+ */
+public class NbestMinRiskReranker {
+
+ private static final Logger LOG = LoggerFactory.getLogger(NbestMinRiskReranker.class);
+
+ // TODO: this functionality is not implemented yet; default is to produce 1best without any
+ // feature scores;
+ boolean produceRerankedNbest = false;
+
+ double scalingFactor = 1.0;
+
+ static int bleuOrder = 4;
+ static boolean doNgramClip = true;
+
+ static boolean useGoogleLinearCorpusGain = false;
+
+ final PriorityBlockingQueue<RankerResult> resultsQueue =
+ new PriorityBlockingQueue<RankerResult>();
+
+ public NbestMinRiskReranker(boolean produceRerankedNbest, double scalingFactor) {
+ this.produceRerankedNbest = produceRerankedNbest;
+ this.scalingFactor = scalingFactor;
+ }
+
+
+ public String processOneSent(List<String> nbest, int sentID) {
+ LOG.info("Now process sentence {}", sentID);
+
+ // step-0: preprocess
+ // assumption: each hyp has a formate:
+ // "sent_id ||| hyp_itself ||| feature scores ||| linear-combination-of-feature-scores(this should be logP)"
+
+ /* Quit if you find an empty hypothesis. */
+ if (nbest.size() == 1) {
+ String[] fields = Regex.threeBarsWithSpace.split(nbest.get(0));
+ if (fields[1].equals("") || Regex.spaces.matches(fields[1])) {
+ LOG.warn("-> sentence is empty");
+ return "";
+ }
+ }
+
+ List<String> hypsItself = new ArrayList<String>();
+ // ArrayList<String> l_feat_scores = new ArrayList<String>();
+ List<Double> baselineScores = new ArrayList<Double>(); // linear combination of all baseline
+ // features
+ List<HashMap<String, Integer>> ngramTbls = new ArrayList<HashMap<String, Integer>>();
+ List<Integer> sentLens = new ArrayList<Integer>();
+
+ for (String hyp : nbest) {
+ String[] fds = Regex.threeBarsWithSpace.split(hyp);
+ int tSentID = Integer.parseInt(fds[0]);
+ if (sentID != tSentID) {
+ throw new RuntimeException("sentence_id does not match");
+ }
+ String hypothesis = (fds.length >= 4) ? fds[1] : "";
+ hypsItself.add(hypothesis);
+
+ String[] words = Regex.spaces.split(hypothesis);
+ sentLens.add(words.length);
+
+ HashMap<String, Integer> ngramTbl = new HashMap<String, Integer>();
+ Ngram.getNgrams(ngramTbl, 1, bleuOrder, words);
+ ngramTbls.add(ngramTbl);
+
+ // l_feat_scores.add(fds[2]);
+
+ // The value of finalIndex is expected to be 3,
+ // unless the hyp_itself is empty,
+ // in which case finalIndex will be 2.
+ int finalIndex = fds.length - 1;
+ baselineScores.add(Double.parseDouble(fds[finalIndex]));
+
+ }
+
+ // step-1: get normalized distribution
+
+ /**
+ * value in baselineScores will be changed to normalized probability
+ * */
+ computeNormalizedProbs(baselineScores, scalingFactor);
+
+ List<Double> normalizedProbs = baselineScores;
+
+ // === required by google linear corpus gain
+ HashMap<String, Double> posteriorCountsTbl = null;
+ if (useGoogleLinearCorpusGain) {
+ posteriorCountsTbl = new HashMap<String, Double>();
+ getGooglePosteriorCounts(ngramTbls, normalizedProbs, posteriorCountsTbl);
+ }
+
+
+ // step-2: rerank the nbest
+ /**
+ * TODO: zhifei: now the re-ranking takes O(n^2) where n is the size of the nbest. But, we can
+ * significantly speed up this (leadding to O(n)) by first estimating a model on nbest, and then
+ * rerank the nbest using the estimated model.
+ * */
+ double bestGain = -1000000000;// set as worst gain
+ String bestHyp = null;
+ List<Double> gains = new ArrayList<Double>();
+ for (int i = 0; i < hypsItself.size(); i++) {
+ String curHyp = hypsItself.get(i);
+ int curHypLen = sentLens.get(i);
+ HashMap<String, Integer> curHypNgramTbl = ngramTbls.get(i);
+ // double cur_gain = computeGain(cur_hyp, l_hyp_itself, l_normalized_probs);
+ double curGain = 0;
+ if (useGoogleLinearCorpusGain) {
+ curGain = computeExpectedLinearCorpusGain(curHypLen, curHypNgramTbl, posteriorCountsTbl);
+ } else {
+ curGain =
+ computeExpectedGain(curHypLen, curHypNgramTbl, ngramTbls, sentLens, normalizedProbs);
+ }
+
+ gains.add(curGain);
+ if (i == 0 || curGain > bestGain) { // maximize
+ bestGain = curGain;
+ bestHyp = curHyp;
+ }
+ }
+
+ // step-3: output the 1best or nbest
+ if (this.produceRerankedNbest) {
+ // TOTO: sort the list and write the reranked nbest; Use Collections.sort(List list,
+ // Comparator c)
+ } else {
+ /*
+ * this.out.write(best_hyp); this.out.write("\n"); out.flush();
+ */
+ }
+
+ LOG.info("best gain: {}", bestGain);
+ if (null == bestHyp) {
+ throw new RuntimeException("mbr reranked one best is null, must be wrong");
+ }
+ return bestHyp;
+ }
+
+
+ /**
+ * based on a list of log-probabilities in nbestLogProbs, obtain a normalized distribution, and
+ * put the normalized probability (real value in [0,1]) into nbestLogProbs
+ *
+ * @param nbestLogProbs a {@link java.util.List} of {@link java.lang.Double} representing nbestLogProbs
+ * @param scalingFactor double value representing scaling factor
+ */
+ // get a normalized distributeion and put it back to nbestLogProbs
+ static public void computeNormalizedProbs(List<Double> nbestLogProbs, double scalingFactor) {
+
+ // === get noralization constant, remember features, remember the combined linear score
+ double normalizationConstant = Double.NEGATIVE_INFINITY;// log-semiring
+
+ for (double logp : nbestLogProbs) {
+ normalizationConstant = addInLogSemiring(normalizationConstant, logp * scalingFactor, 0);
+ }
+ // System.out.println("normalization_constant (logP) is " + normalization_constant);
+
+ // === get normalized prob for each hyp
+ double tSum = 0;
+ for (int i = 0; i < nbestLogProbs.size(); i++) {
+
+ double normalizedProb =
+ Math.exp(nbestLogProbs.get(i) * scalingFactor - normalizationConstant);
+ tSum += normalizedProb;
+ nbestLogProbs.set(i, normalizedProb);
+
+ if (Double.isNaN(normalizedProb)) {
+ throw new RuntimeException("prob is NaN, must be wrong\nnbest_logps.get(i): "
+ + nbestLogProbs.get(i) + "; scaling_factor: " + scalingFactor
+ + "; normalization_constant:" + normalizationConstant);
+ }
+ // logger.info("probability: " + normalized_prob);
+ }
+
+ // sanity check
+ if (Math.abs(tSum - 1.0) > 1e-4) {
+ throw new RuntimeException("probabilities not sum to one, must be wrong");
+ }
+
+ }
+
+
+ // Gain(e) = negative risk = \sum_{e'} G(e, e')P(e')
+ // curHyp: e
+ // trueHyp: e'
+ public double computeExpectedGain(int curHypLen, HashMap<String, Integer> curHypNgramTbl,
+ List<HashMap<String, Integer>> ngramTbls, List<Integer> sentLens, List<Double> nbestProbs) {
+
+ // ### get noralization constant, remember features, remember the combined linear score
+ double gain = 0;
+
+ for (int i = 0; i < nbestProbs.size(); i++) {
+ HashMap<String, Integer> trueHypNgramTbl = ngramTbls.get(i);
+ double trueProb = nbestProbs.get(i);
+ int trueLen = sentLens.get(i);
+ gain +=
+ trueProb
+ * BLEU.computeSentenceBleu(trueLen, trueHypNgramTbl, curHypLen, curHypNgramTbl,
+ doNgramClip, bleuOrder);
+ }
+ // System.out.println("Gain is " + gain);
+ return gain;
+ }
+
+ // Gain(e) = negative risk = \sum_{e'} G(e, e')P(e')
+ // curHyp: e
+ // trueHyp: e'
+ static public double computeExpectedGain(String curHyp, List<String> nbestHyps,
+ List<Double> nbestProbs) {
+ // ### get noralization constant, remember features, remember the combined linear score
+ double gain = 0;
+
+ for (int i = 0; i < nbestHyps.size(); i++) {
+ String trueHyp = nbestHyps.get(i);
+ double trueProb = nbestProbs.get(i);
+ gain += trueProb * BLEU.computeSentenceBleu(trueHyp, curHyp, doNgramClip, bleuOrder);
+ }
+ // System.out.println("Gain is " + gain);
+ return gain;
+ }
+
+ void getGooglePosteriorCounts(List<HashMap<String, Integer>> ngramTbls,
+ List<Double> normalizedProbs, HashMap<String, Double> posteriorCountsTbl) {
+ // TODO
+ }
+
+ double computeExpectedLinearCorpusGain(int curHypLen, HashMap<String, Integer> curHypNgramTbl,
+ HashMap<String, Double> posteriorCountsTbl) {
+ // TODO
+ double[] thetas = {-1, 1, 1, 1, 1};
+
+ double res = 0;
+ res += thetas[0] * curHypLen;
+ for (Entry<String, Integer> entry : curHypNgramTbl.entrySet()) {
+ String key = entry.getKey();
+ String[] tem = Regex.spaces.split(key);
+
+ double post_prob = posteriorCountsTbl.get(key);
+ res += entry.getValue() * post_prob * thetas[tem.length];
+ }
+ return res;
+ }
+
+ // OR: return Math.log(Math.exp(x) + Math.exp(y));
+ static private double addInLogSemiring(double x, double y, int addMode) {// prevent over-flow
+ if (addMode == 0) { // sum
+ if (x == Double.NEGATIVE_INFINITY) {// if y is also n-infinity, then return n-infinity
+ return y;
+ }
+ if (y == Double.NEGATIVE_INFINITY) {
+ return x;
+ }
+
+ if (y <= x) {
+ return x + Math.log(1 + Math.exp(y - x));
+ } else {
+ return y + Math.log(1 + Math.exp(x - y));
+ }
+ } else if (addMode == 1) { // viter-min
+ return (x <= y) ? x : y;
+ } else if (addMode == 2) { // viter-max
+ return (x >= y) ? x : y;
+ } else {
+ throw new RuntimeException("invalid add mode");
+ }
+ }
+
+
+
+ public static void main(String[] args) throws IOException {
+
+ // If you don't know what to use for scaling factor, try using 1
+
+ if (args.length < 2) {
+ String msg = "usage: java NbestMinRiskReranker <produce_reranked_nbest> <scaling_factor> "
+ + "[numThreads]";
+ System.err.println(msg);
+ LOG.error(msg);
+ return;
+ }
+ long startTime = System.currentTimeMillis();
+ boolean produceRerankedNbest = Boolean.valueOf(args[0].trim());
+ double scalingFactor = Double.parseDouble(args[1].trim());
+ int numThreads = (args.length > 2) ? Integer.parseInt(args[2].trim()) : 1;
+
+
+ NbestMinRiskReranker mbrReranker =
+ new NbestMinRiskReranker(produceRerankedNbest, scalingFactor);
+
+ LOG.info("Running mbr reranking");
+
+ int oldSentID = -1;
+ List<String> nbest = new ArrayList<String>();
+
+ Scanner scanner = new Scanner(System.in, "UTF-8");
+
+ if (numThreads == 1) {
+
+ while (scanner.hasNextLine()) {
+ String line = scanner.nextLine();
+ String[] fds = Regex.threeBarsWithSpace.split(line);
+ int newSentID = Integer.parseInt(fds[0]);
+ if (oldSentID != -1 && oldSentID != newSentID) {
+ if (nbest.size() > 0) {
+ String best_hyp = mbrReranker.processOneSent(nbest, oldSentID);// nbest: list of unique
+ // strings
+ System.out.println(best_hyp);
+ } else {
+ System.out.println();
+ }
+ nbest.clear();
+ }
+ oldSentID = newSentID;
+ if (!fds[1].matches("^\\s*$")) nbest.add(line);
+ }
+
+ // last nbest
+ if (oldSentID >= 0) {
+ String bestHyp = mbrReranker.processOneSent(nbest, oldSentID);
+ System.out.println(bestHyp);
+ nbest.clear();
+ }
+
+ } else {
+
+ ExecutorService threadPool = Executors.newFixedThreadPool(numThreads);
+
+ while (scanner.hasNextLine()) {
+ String line = scanner.nextLine();
+ String[] fds = Regex.threeBarsWithSpace.split(line);
+ int newSentID = Integer.parseInt(fds[0]);
+ if (oldSentID != -1 && oldSentID != newSentID) {
+
+ threadPool.execute(mbrReranker.new RankerTask(nbest, oldSentID));
+
+ nbest.clear();
+ }
+ oldSentID = newSentID;
+ nbest.add(line);
+ }
+
+ // last nbest
+ threadPool.execute(mbrReranker.new RankerTask(nbest, oldSentID));
+ nbest.clear();
+
+ threadPool.shutdown();
+
+ try {
+ threadPool.awaitTermination(Integer.MAX_VALUE, TimeUnit.SECONDS);
+
+ while (!mbrReranker.resultsQueue.isEmpty()) {
+ RankerResult result = mbrReranker.resultsQueue.remove();
+ String best_hyp = result.toString();
+ System.out.println(best_hyp);
+ }
+ } catch (InterruptedException e) {
+ LOG.error(e.getMessage(), e);
+ }
+ }
+
+ scanner.close();
+
+ LOG.info("Total running time (seconds) is {} ",
+ (System.currentTimeMillis() - startTime) / 1000.0);
+ }
+
+ private class RankerTask implements Runnable {
+
+ final List<String> nbest;
+ final int sentID;
+
+ RankerTask(final List<String> nbest, final int sentID) {
+ this.nbest = new ArrayList<String>(nbest);
+ this.sentID = sentID;
+ }
+
+ public void run() {
+ String result = processOneSent(nbest, sentID);
+ resultsQueue.add(new RankerResult(result, sentID));
+ }
+
+ }
+
+ private static class RankerResult implements Comparable<RankerResult> {
+ final String result;
+ final Integer sentenceNumber;
+
+ RankerResult(String result, int sentenceNumber) {
+ this.result = result;
+ this.sentenceNumber = sentenceNumber;
+ }
+
+ public int compareTo(RankerResult o) {
+ return sentenceNumber.compareTo(o.sentenceNumber);
+ }
+
+ public String toString() {
+ return result;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslation.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslation.java b/joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslation.java
new file mode 100644
index 0000000..2faacf2
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslation.java
@@ -0,0 +1,164 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder;
+
+import java.util.List;
+import java.util.Map;
+
+import org.apache.joshua.decoder.ff.FeatureVector;
+import org.apache.joshua.decoder.hypergraph.KBestExtractor.DerivationState;
+import org.apache.joshua.decoder.io.DeNormalize;
+import org.apache.joshua.decoder.segment_file.Sentence;
+import org.apache.joshua.decoder.segment_file.Token;
+import org.apache.joshua.util.FormatUtils;
+
+/**
+ * A StructuredTranslation instance provides a more structured access to
+ * translation results than the string-based Translation class.
+ * This is useful if the decoder is encapsulated in a larger project, instead
+ * of simply writing to a file or stdout.
+ * StructuredTranslation encodes all relevant information about a derivation,
+ * namely output string, tokens, score, features, and word alignment.
+ *
+ * @author fhieber
+ */
+public class StructuredTranslation {
+
+ private final Sentence sourceSentence;
+ private final String translationString;
+ private final List<String> translationTokens;
+ private final float translationScore;
+ private final List<List<Integer>> translationWordAlignments;
+ private final FeatureVector translationFeatures;
+ private final float extractionTime;
+
+ public StructuredTranslation(
+ final Sentence sourceSentence,
+ final String translationString,
+ final List<String> translationTokens,
+ final float translationScore,
+ final List<List<Integer>> translationWordAlignments,
+ final FeatureVector translationFeatures,
+ final float extractionTime) {
+ this.sourceSentence = sourceSentence;
+ this.translationString = translationString;
+ this.translationTokens = translationTokens;
+ this.translationScore = translationScore;
+ this.translationWordAlignments = translationWordAlignments;
+ this.translationFeatures = translationFeatures;
+ this.extractionTime = extractionTime;
+ }
+
+ public Sentence getSourceSentence() {
+ return sourceSentence;
+ }
+
+ public int getSentenceId() {
+ return sourceSentence.id();
+ }
+
+ /**
+ * Produces the raw translation hypothesis (still tokenized).
+ *
+ * @return the raw translation hypothesis
+ */
+ public String getTranslationString() {
+ return translationString;
+ }
+
+ /**
+ * Produces the translation formatted according to the value of {@value JoshuaConfiguration.output_format}.
+ * Also includes formatting options such as {@value JoshuaConfiguration.project_case}.
+ *
+ * @return
+ */
+ public String getFormattedTranslationString() {
+ JoshuaConfiguration config = sourceSentence.config;
+ String outputString = config.outputFormat
+ .replace("%s", getTranslationString())
+ .replace("%S", DeNormalize.processSingleLine(maybeProjectCase(getTranslationString())))
+ .replace("%i", Integer.toString(getSentenceId()))
+ .replace("%f", config.moses ? translationFeatures.mosesString() : translationFeatures.toString())
+ .replace("%c", String.format("%.3f", getTranslationScore()));
+ return outputString;
+ }
+
+ public List<String> getTranslationTokens() {
+ return translationTokens;
+ }
+
+ public float getTranslationScore() {
+ return translationScore;
+ }
+
+ /**
+ * Returns a list of target to source alignments.
+ * @return a list of target to source alignments
+ */
+ public List<List<Integer>> getTranslationWordAlignments() {
+ return translationWordAlignments;
+ }
+
+ public Map<String,Float> getTranslationFeatures() {
+ return translationFeatures.getMap();
+ }
+
+ /**
+ * Time taken to build output information from the hypergraph.
+ * @return the time taken to build output information from the hypergraph
+ */
+ public Float getExtractionTime() {
+ return extractionTime;
+ }
+
+ /**
+ * If requested, projects source-side lettercase to target, and appends the alignment from
+ * to the source-side sentence in ||s.
+ *
+ * @param hypothesis todo
+ * @param state todo
+ * @return source-side lettercase to target, and appends the alignment from to the source-side sentence in ||s
+ */
+ private String maybeProjectCase(String hypothesis) {
+ String output = hypothesis;
+
+ JoshuaConfiguration config = sourceSentence.config;
+ if (config.project_case) {
+ String[] tokens = hypothesis.split("\\s+");
+ List<List<Integer>> points = getTranslationWordAlignments();
+ for (int i = 0; i < points.size(); i++) {
+ List<Integer> target = points.get(i);
+ for (int source: target) {
+ Token token = sourceSentence.getTokens().get(source + 1); // skip <s>
+ String annotation = "";
+ if (token != null && token.getAnnotation("lettercase") != null)
+ annotation = token.getAnnotation("lettercase");
+ if (source != 0 && annotation.equals("upper"))
+ tokens[i] = FormatUtils.capitalize(tokens[i]);
+ else if (annotation.equals("all-upper"))
+ tokens[i] = tokens[i].toUpperCase();
+ }
+ }
+
+ output = String.join(" ", tokens);
+ }
+
+ return output;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslationFactory.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslationFactory.java b/joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslationFactory.java
new file mode 100644
index 0000000..4389135
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/StructuredTranslationFactory.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder;
+
+import static java.util.Arrays.asList;
+import static java.util.Collections.emptyList;
+import static java.util.Collections.emptyMap;
+import static org.apache.joshua.decoder.hypergraph.ViterbiExtractor.getViterbiFeatures;
+import static org.apache.joshua.decoder.hypergraph.ViterbiExtractor.getViterbiString;
+import static org.apache.joshua.decoder.hypergraph.ViterbiExtractor.getViterbiWordAlignmentList;
+import static org.apache.joshua.util.FormatUtils.removeSentenceMarkers;
+
+import java.util.List;
+
+import org.apache.joshua.decoder.ff.FeatureFunction;
+import org.apache.joshua.decoder.ff.FeatureVector;
+import org.apache.joshua.decoder.hypergraph.HyperGraph;
+import org.apache.joshua.decoder.hypergraph.KBestExtractor.DerivationState;
+import org.apache.joshua.decoder.segment_file.Sentence;
+
+/**
+ * This factory provides methods to create StructuredTranslation objects
+ * from either Viterbi derivations or KBest derivations.
+ *
+ * @author fhieber
+ */
+public class StructuredTranslationFactory {
+
+ /**
+ * Returns a StructuredTranslation instance from the Viterbi derivation.
+ *
+ * @param sourceSentence the source sentence
+ * @param hypergraph the hypergraph object
+ * @param featureFunctions the list of active feature functions
+ * @return A StructuredTranslation object representing the Viterbi derivation.
+ */
+ public static StructuredTranslation fromViterbiDerivation(
+ final Sentence sourceSentence,
+ final HyperGraph hypergraph,
+ final List<FeatureFunction> featureFunctions) {
+ final long startTime = System.currentTimeMillis();
+ final String translationString = removeSentenceMarkers(getViterbiString(hypergraph));
+ return new StructuredTranslation(
+ sourceSentence,
+ translationString,
+ extractTranslationTokens(translationString),
+ extractTranslationScore(hypergraph),
+ getViterbiWordAlignmentList(hypergraph),
+ getViterbiFeatures(hypergraph, featureFunctions, sourceSentence),
+ (System.currentTimeMillis() - startTime) / 1000.0f);
+ }
+
+ /**
+ * Returns a StructuredTranslation from an empty decoder output
+ * @param sourceSentence the source sentence
+ * @return a StructuredTranslation object
+ */
+ public static StructuredTranslation fromEmptyOutput(final Sentence sourceSentence) {
+ return new StructuredTranslation(
+ sourceSentence, "", emptyList(), 0, emptyList(), new FeatureVector(), 0f);
+ }
+
+ /**
+ * Returns a StructuredTranslation instance from a KBest DerivationState.
+ * @param sourceSentence Sentence object representing the source.
+ * @param derivationState the KBest DerivationState.
+ * @return A StructuredTranslation object representing the derivation encoded by derivationState.
+ */
+ public static StructuredTranslation fromKBestDerivation(
+ final Sentence sourceSentence,
+ final DerivationState derivationState) {
+ final long startTime = System.currentTimeMillis();
+ final String translationString = removeSentenceMarkers(derivationState.getHypothesis());
+ return new StructuredTranslation(
+ sourceSentence,
+ translationString,
+ extractTranslationTokens(translationString),
+ derivationState.getModelCost(),
+ derivationState.getWordAlignmentList(),
+ derivationState.getFeatures(),
+ (System.currentTimeMillis() - startTime) / 1000.0f);
+ }
+
+ private static float extractTranslationScore(final HyperGraph hypergraph) {
+ if (hypergraph == null) {
+ return 0;
+ } else {
+ return hypergraph.goalNode.getScore();
+ }
+ }
+
+ private static List<String> extractTranslationTokens(final String translationString) {
+ if (translationString.isEmpty()) {
+ return emptyList();
+ } else {
+ return asList(translationString.split("\\s+"));
+ }
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/Support.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/Support.java b/joshua-core/src/main/java/org/apache/joshua/decoder/Support.java
new file mode 100644
index 0000000..e513aef
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/Support.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder;
+
+import java.util.List;
+
+/**
+ * @author Zhifei Li, zhifei.work@gmail.com
+ */
+public class Support {
+
+ public static double findMin(double a, double b) {
+ return (a <= b) ? a : b;
+ }
+
+ public static double findMax(double a, double b) {
+ return (a > b) ? a : b;
+ }
+
+ public static int[] toArray(List<Integer> in) {
+ return subIntArray(in, 0, in.size());
+ }
+
+ /**
+ * @param in a {@link java.util.List} of Integer
+ * @param start inclusive
+ * @param end exclusive
+ * @return sub int[] from start to end
+ */
+ public static int[] subIntArray(List<Integer> in, int start, int end) {
+ int[] res = new int[end - start];
+ for (int i = start; i < end; i++) {
+ res[i - start] = in.get(i);
+ }
+ return res;
+ }
+
+ public static long current_time() {
+ return 0;
+ // return System.currentTimeMillis();
+ // return System.nanoTime();
+ }
+
+ // Only used in LMGrammarJAVA
+ public static long getMemoryUse() {
+ putOutTheGarbage();
+ long totalMemory = Runtime.getRuntime().totalMemory();// all the memory I get from the system
+ putOutTheGarbage();
+ long freeMemory = Runtime.getRuntime().freeMemory();
+ return (totalMemory - freeMemory) / 1024;// in terms of kb
+ }
+
+ private static void putOutTheGarbage() {
+ collectGarbage();
+ collectGarbage();
+ }
+
+ private static void collectGarbage() {
+ long fSLEEP_INTERVAL = 100;
+ try {
+ System.gc();
+ Thread.sleep(fSLEEP_INTERVAL);
+ System.runFinalization();
+ Thread.sleep(fSLEEP_INTERVAL);
+
+ } catch (InterruptedException ex) {
+ ex.printStackTrace();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/Translation.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/Translation.java b/joshua-core/src/main/java/org/apache/joshua/decoder/Translation.java
new file mode 100644
index 0000000..46f3061
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/Translation.java
@@ -0,0 +1,239 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder;
+
+import static org.apache.joshua.decoder.hypergraph.ViterbiExtractor.getViterbiFeatures;
+import static org.apache.joshua.decoder.hypergraph.ViterbiExtractor.getViterbiString;
+import static org.apache.joshua.decoder.hypergraph.ViterbiExtractor.getViterbiWordAlignments;
+import static org.apache.joshua.decoder.StructuredTranslationFactory.fromViterbiDerivation;
+import static org.apache.joshua.util.FormatUtils.removeSentenceMarkers;
+import static java.util.Arrays.asList;
+
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.List;
+
+import org.apache.joshua.decoder.ff.FeatureFunction;
+import org.apache.joshua.decoder.ff.FeatureVector;
+import org.apache.joshua.decoder.ff.lm.StateMinimizingLanguageModel;
+import org.apache.joshua.decoder.hypergraph.HyperGraph;
+import org.apache.joshua.decoder.hypergraph.KBestExtractor;
+import org.apache.joshua.decoder.io.DeNormalize;
+import org.apache.joshua.decoder.segment_file.Sentence;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class represents translated input objects (sentences or lattices). It is aware of the source
+ * sentence and id and contains the decoded hypergraph. Translation objects are returned by
+ * DecoderThread instances to the InputHandler, where they are assembled in order for output.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ * @author Felix Hieber fhieber@amazon.com
+ */
+
+public class Translation {
+ private static final Logger LOG = LoggerFactory.getLogger(Translation.class);
+ private Sentence source;
+
+ /**
+ * This stores the output of the translation so we don't have to hold onto the hypergraph while we
+ * wait for the outputs to be assembled.
+ */
+ private String output = null;
+
+ /**
+ * Stores the list of StructuredTranslations.
+ * If joshuaConfig.topN == 0, will only contain the Viterbi translation.
+ * Else it will use KBestExtractor to populate this list.
+ */
+ private List<StructuredTranslation> structuredTranslations = null;
+
+ public Translation(Sentence source, HyperGraph hypergraph,
+ List<FeatureFunction> featureFunctions, JoshuaConfiguration joshuaConfiguration) {
+ this.source = source;
+
+ /**
+ * Structured output from Joshua provides a way to programmatically access translation results
+ * from downstream applications, instead of writing results as strings to an output buffer.
+ */
+ if (joshuaConfiguration.use_structured_output) {
+
+ if (joshuaConfiguration.topN == 0) {
+ /*
+ * Obtain Viterbi StructuredTranslation
+ */
+ StructuredTranslation translation = fromViterbiDerivation(source, hypergraph, featureFunctions);
+ this.output = translation.getTranslationString();
+ structuredTranslations = asList(translation);
+
+ } else {
+ /*
+ * Get K-Best list of StructuredTranslations
+ */
+ final KBestExtractor kBestExtractor = new KBestExtractor(source, featureFunctions, Decoder.weights, false, joshuaConfiguration);
+ structuredTranslations = kBestExtractor.KbestExtractOnHG(hypergraph, joshuaConfiguration.topN);
+ if (structuredTranslations.isEmpty()) {
+ structuredTranslations = asList(StructuredTranslationFactory.fromEmptyOutput(source));
+ this.output = "";
+ } else {
+ this.output = structuredTranslations.get(0).getTranslationString();
+ }
+ // TODO: We omit the BLEU rescoring for now since it is not clear whether it works at all and what the desired output is below.
+ }
+
+ } else {
+
+ StringWriter sw = new StringWriter();
+ BufferedWriter out = new BufferedWriter(sw);
+
+ try {
+
+ if (hypergraph != null) {
+
+ long startTime = System.currentTimeMillis();
+
+ // We must put this weight as zero, otherwise we get an error when we try to retrieve it
+ // without checking
+ Decoder.weights.increment("BLEU", 0);
+
+ if (joshuaConfiguration.topN == 0) {
+
+ /* construct Viterbi output */
+ final String best = getViterbiString(hypergraph);
+
+ LOG.info("Translation {}: {} {}", source.id(), hypergraph.goalNode.getScore(), best);
+
+ /*
+ * Setting topN to 0 turns off k-best extraction, in which case we need to parse through
+ * the output-string, with the understanding that we can only substitute variables for the
+ * output string, sentence number, and model score.
+ */
+ String translation = joshuaConfiguration.outputFormat
+ .replace("%s", removeSentenceMarkers(best))
+ .replace("%S", DeNormalize.processSingleLine(best))
+ .replace("%c", String.format("%.3f", hypergraph.goalNode.getScore()))
+ .replace("%i", String.format("%d", source.id()));
+
+ if (joshuaConfiguration.outputFormat.contains("%a")) {
+ translation = translation.replace("%a", getViterbiWordAlignments(hypergraph));
+ }
+
+ if (joshuaConfiguration.outputFormat.contains("%f")) {
+ final FeatureVector features = getViterbiFeatures(hypergraph, featureFunctions, source);
+ translation = translation.replace("%f", joshuaConfiguration.moses ? features.mosesString() : features.toString());
+ }
+
+ out.write(translation);
+ out.newLine();
+
+ } else {
+
+ final KBestExtractor kBestExtractor = new KBestExtractor(
+ source, featureFunctions, Decoder.weights, false, joshuaConfiguration);
+ kBestExtractor.lazyKBestExtractOnHG(hypergraph, joshuaConfiguration.topN, out);
+
+ if (joshuaConfiguration.rescoreForest) {
+ Decoder.weights.increment("BLEU", joshuaConfiguration.rescoreForestWeight);
+ kBestExtractor.lazyKBestExtractOnHG(hypergraph, joshuaConfiguration.topN, out);
+
+ Decoder.weights.increment("BLEU", -joshuaConfiguration.rescoreForestWeight);
+ kBestExtractor.lazyKBestExtractOnHG(hypergraph, joshuaConfiguration.topN, out);
+ }
+ }
+
+ float seconds = (float) (System.currentTimeMillis() - startTime) / 1000.0f;
+ LOG.info("Input {}: {}-best extraction took {} seconds", id(),
+ joshuaConfiguration.topN, seconds);
+
+ } else {
+
+ // Failed translations and blank lines get empty formatted outputs
+ out.write(getFailedTranslationOutput(source, joshuaConfiguration));
+ out.newLine();
+
+ }
+
+ out.flush();
+
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ this.output = sw.toString();
+
+ }
+
+ // remove state from StateMinimizingLanguageModel instances in features.
+ destroyKenLMStates(featureFunctions);
+
+ }
+
+ public Sentence getSourceSentence() {
+ return this.source;
+ }
+
+ public int id() {
+ return source.id();
+ }
+
+ @Override
+ public String toString() {
+ return output;
+ }
+
+ private String getFailedTranslationOutput(final Sentence source, final JoshuaConfiguration joshuaConfiguration) {
+ return joshuaConfiguration.outputFormat
+ .replace("%s", source.source())
+ .replace("%e", "")
+ .replace("%S", "")
+ .replace("%t", "()")
+ .replace("%i", Integer.toString(source.id()))
+ .replace("%f", "")
+ .replace("%c", "0.000");
+ }
+
+ /**
+ * Returns the StructuredTranslations
+ * if JoshuaConfiguration.use_structured_output == True.
+ * @throws RuntimeException if JoshuaConfiguration.use_structured_output == False.
+ * @return List of StructuredTranslations.
+ */
+ public List<StructuredTranslation> getStructuredTranslations() {
+ if (structuredTranslations == null) {
+ throw new RuntimeException(
+ "No StructuredTranslation objects created. You should set JoshuaConfigration.use_structured_output = true");
+ }
+ return structuredTranslations;
+ }
+
+ /**
+ * KenLM hack. If using KenLMFF, we need to tell KenLM to delete the pool used to create chart
+ * objects for this sentence.
+ */
+ private void destroyKenLMStates(final List<FeatureFunction> featureFunctions) {
+ for (FeatureFunction feature : featureFunctions) {
+ if (feature instanceof StateMinimizingLanguageModel) {
+ ((StateMinimizingLanguageModel) feature).destroyPool(getSourceSentence().id());
+ break;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/Translations.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/Translations.java b/joshua-core/src/main/java/org/apache/joshua/decoder/Translations.java
new file mode 100644
index 0000000..e607225
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/Translations.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder;
+
+import java.util.Iterator;
+import java.util.LinkedList;
+import org.apache.joshua.decoder.io.TranslationRequestStream;
+
+/**
+ * This class represents a streaming sequence of translations. It is returned by the main entry
+ * point to the Decoder object, the call to decodeAll. The translations here are parallel to the
+ * input sentences in the corresponding TranslationRequest object. Because of parallelization, the
+ * translated sentences might be computed out of order. Each Translation is sent to this
+ * Translations object by a DecoderThreadRunner via the record() function, which places the
+ * Translation in the right place. When the next translation in a sequence is available, next() is
+ * notified.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ */
+public class Translations implements Iterator<Translation>, Iterable<Translation> {
+
+ /* The source sentences to be translated. */
+ private TranslationRequestStream request = null;
+
+ /*
+ * This records the index of the sentence at the head of the underlying list. The iterator's
+ * next() blocks when the value at this position in the translations LinkedList is null.
+ */
+ private int currentID = 0;
+
+ /* The set of translated sentences. */
+ private LinkedList<Translation> translations = null;
+
+ private boolean spent = false;
+
+ private Translation nextTranslation;
+
+ public Translations(TranslationRequestStream request) {
+ this.request = request;
+ this.translations = new LinkedList<Translation>();
+ }
+
+ /**
+ * This is called when null is received from the TranslationRequest, indicating that there are no
+ * more input sentences to translated. That in turn means that the request size will no longer
+ * grow. We then notify any waiting thread if the last ID we've processed is the last one, period.
+ */
+ public void finish() {
+ synchronized (this) {
+ spent = true;
+ if (currentID == request.size()) {
+ this.notifyAll();
+ }
+ }
+ }
+
+ /**
+ * This is called whenever a translation is completed by one of the decoder threads. There may be
+ * a current output thread waiting for the current translation, which is determined by checking if
+ * the ID of the translation is the same as the one being waited for (currentID). If so, the
+ * thread waiting for it is notified.
+ *
+ * @param translation a translated input object
+ */
+ public void record(Translation translation) {
+ synchronized (this) {
+
+ /* Pad the set of translations with nulls to accommodate the new translation. */
+ int offset = translation.id() - currentID;
+ while (offset >= translations.size())
+ translations.add(null);
+ translations.set(offset, translation);
+
+ /*
+ * If the id of the current translation is at the head of the list (first element), then we
+ * have the next Translation to be return, and we should notify anyone waiting on next(),
+ * which will then remove the item and increment the currentID.
+ */
+ if (translation.id() == currentID) {
+ this.notify();
+ }
+ }
+ }
+
+ /**
+ * Returns the next Translation, blocking if necessary until it's available, since the next
+ * Translation might not have been produced yet.
+ *
+ * @return first element from the list of {@link org.apache.joshua.decoder.Translation}'s
+ */
+ @Override
+ public Translation next() {
+ synchronized(this) {
+ if (this.hasNext()) {
+ Translation t = this.nextTranslation;
+ this.nextTranslation = null;
+ return t;
+ }
+
+ return null;
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ synchronized (this) {
+
+ if (nextTranslation != null)
+ return true;
+
+ /*
+ * If there are no more input sentences, and we've already distributed what we then know is
+ * the last one, we're done.
+ */
+ if (spent && currentID == request.size())
+ return false;
+
+ /*
+ * Otherwise, there is another sentence. If it's not available already, we need to wait for
+ * it.
+ */
+ if (translations.size() == 0 || translations.peek() == null) {
+ try {
+ this.wait();
+ } catch (InterruptedException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ }
+
+ /* We now have the sentence and can return it. */
+ currentID++;
+ this.nextTranslation = translations.poll();
+ return this.nextTranslation != null;
+ }
+ }
+
+ @Override
+ public Iterator<Translation> iterator() {
+ return this;
+ }
+}
\ No newline at end of file
[30/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/mira/MIRACore.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/mira/MIRACore.java b/joshua-core/src/main/java/org/apache/joshua/mira/MIRACore.java
new file mode 100755
index 0000000..42dd995
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/mira/MIRACore.java
@@ -0,0 +1,3112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.mira;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.text.DecimalFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Random;
+import java.util.Scanner;
+import java.util.TreeSet;
+import java.util.Vector;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+
+import org.apache.joshua.decoder.Decoder;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.metrics.EvaluationMetric;
+import org.apache.joshua.util.StreamGobbler;
+import org.apache.joshua.corpus.Vocabulary;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This code was originally written by Yuan Cao, who copied the MERT code to produce this file.
+ */
+
+public class MIRACore {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MIRACore.class);
+
+ private final JoshuaConfiguration joshuaConfiguration;
+ private TreeSet<Integer>[] indicesOfInterest_all;
+
+ private final static DecimalFormat f4 = new DecimalFormat("###0.0000");
+ private final Runtime myRuntime = Runtime.getRuntime();
+
+ private final static double NegInf = (-1.0 / 0.0);
+ private final static double PosInf = (+1.0 / 0.0);
+ private final static double epsilon = 1.0 / 1000000;
+
+ private int progress;
+
+ private int verbosity; // anything of priority <= verbosity will be printed
+ // (lower value for priority means more important)
+
+ private Random randGen;
+ private int generatedRands;
+
+ private int numSentences;
+ // number of sentences in the dev set
+ // (aka the "MERT training" set)
+
+ private int numDocuments;
+ // number of documents in the dev set
+ // this should be 1, unless doing doc-level optimization
+
+ private int[] docOfSentence;
+ // docOfSentence[i] stores which document contains the i'th sentence.
+ // docOfSentence is 0-indexed, as are the documents (i.e. first doc is indexed 0)
+
+ private int[] docSubsetInfo;
+ // stores information regarding which subset of the documents are evaluated
+ // [0]: method (0-6)
+ // [1]: first (1-indexed)
+ // [2]: last (1-indexed)
+ // [3]: size
+ // [4]: center
+ // [5]: arg1
+ // [6]: arg2
+ // [1-6] are 0 for method 0, [6] is 0 for methods 1-4 as well
+ // only [1] and [2] are needed for optimization. The rest are only needed for an output message.
+
+ private int refsPerSen;
+ // number of reference translations per sentence
+
+ private int textNormMethod;
+ // 0: no normalization, 1: "NIST-style" tokenization, and also rejoin 'm, 're, *'s, 've, 'll, 'd,
+ // and n't,
+ // 2: apply 1 and also rejoin dashes between letters, 3: apply 1 and also drop non-ASCII
+ // characters
+ // 4: apply 1+2+3
+
+ private int numParams;
+ // total number of firing features
+ // this number may increase overtime as new n-best lists are decoded
+ // initially it is equal to the # of params in the parameter config file
+ private int numParamsOld;
+ // number of features before observing the new features fired in the current iteration
+
+ private double[] normalizationOptions;
+ // How should a lambda[] vector be normalized (before decoding)?
+ // nO[0] = 0: no normalization
+ // nO[0] = 1: scale so that parameter nO[2] has absolute value nO[1]
+ // nO[0] = 2: scale so that the maximum absolute value is nO[1]
+ // nO[0] = 3: scale so that the minimum absolute value is nO[1]
+ // nO[0] = 4: scale so that the L-nO[1] norm equals nO[2]
+
+ /* *********************************************************** */
+ /* NOTE: indexing starts at 1 in the following few arrays: */
+ /* *********************************************************** */
+
+ // private double[] lambda;
+ private ArrayList<Double> lambda = new ArrayList<Double>();
+ // the current weight vector. NOTE: indexing starts at 1.
+ private ArrayList<Double> bestLambda = new ArrayList<Double>();
+ // the best weight vector across all iterations
+
+ private boolean[] isOptimizable;
+ // isOptimizable[c] = true iff lambda[c] should be optimized
+
+ private double[] minRandValue;
+ private double[] maxRandValue;
+ // when choosing a random value for the lambda[c] parameter, it will be
+ // chosen from the [minRandValue[c],maxRandValue[c]] range.
+ // (*) minRandValue and maxRandValue must be real values, but not -Inf or +Inf
+
+ private double[] defaultLambda;
+ // "default" parameter values; simply the values read in the parameter file
+ // USED FOR NON-OPTIMIZABLE (FIXED) FEATURES
+
+ /* *********************************************************** */
+ /* *********************************************************** */
+
+ private Decoder myDecoder;
+ // COMMENT OUT if decoder is not Joshua
+
+ private String decoderCommand;
+ // the command that runs the decoder; read from decoderCommandFileName
+
+ private int decVerbosity;
+ // verbosity level for decoder output. If 0, decoder output is ignored.
+ // If 1, decoder output is printed.
+
+ private int validDecoderExitValue;
+ // return value from running the decoder command that indicates success
+
+ private int numOptThreads;
+ // number of threads to run things in parallel
+
+ private int saveInterFiles;
+ // 0: nothing, 1: only configs, 2: only n-bests, 3: both configs and n-bests
+
+ private int compressFiles;
+ // should MIRA gzip the large files? If 0, no compression takes place.
+ // If 1, compression is performed on: decoder output files, temp sents files,
+ // and temp feats files.
+
+ private int sizeOfNBest;
+ // size of N-best list generated by decoder at each iteration
+ // (aka simply N, but N is a bad variable name)
+
+ private long seed;
+ // seed used to create random number generators
+
+ private boolean randInit;
+ // if true, parameters are initialized randomly. If false, parameters
+ // are initialized using values from parameter file.
+
+ private int maxMERTIterations, minMERTIterations, prevMERTIterations;
+ // max: maximum number of MERT iterations
+ // min: minimum number of MERT iterations before an early MERT exit
+ // prev: number of previous MERT iterations from which to consider candidates (in addition to
+ // the candidates from the current iteration)
+
+ private double stopSigValue;
+ // early MERT exit if no weight changes by more than stopSigValue
+ // (but see minMERTIterations above and stopMinIts below)
+
+ private int stopMinIts;
+ // some early stopping criterion must be satisfied in stopMinIts *consecutive* iterations
+ // before an early exit (but see minMERTIterations above)
+
+ private boolean oneModificationPerIteration;
+ // if true, each MERT iteration performs at most one parameter modification.
+ // If false, a new MERT iteration starts (i.e. a new N-best list is
+ // generated) only after the previous iteration reaches a local maximum.
+
+ private String metricName;
+ // name of evaluation metric optimized by MERT
+
+ private String metricName_display;
+ // name of evaluation metric optimized by MERT, possibly with "doc-level " prefixed
+
+ private String[] metricOptions;
+ // options for the evaluation metric (e.g. for BLEU, maxGramLength and effLengthMethod)
+
+ private EvaluationMetric evalMetric;
+ // the evaluation metric used by MERT
+
+ private int suffStatsCount;
+ // number of sufficient statistics for the evaluation metric
+
+ private String tmpDirPrefix;
+ // prefix for the MIRA.temp.* files
+
+ private boolean passIterationToDecoder;
+ // should the iteration number be passed as an argument to decoderCommandFileName?
+
+ // used by mira
+ private boolean needShuffle = true; // shuffle the training sentences or not
+ private boolean needAvg = true; // average the weihgts or not?
+ private boolean runPercep = false; // run perceptron instead of mira
+ private boolean usePseudoBleu = true; // need to use pseudo corpus to compute bleu?
+ private boolean returnBest = false; // return the best weight during tuning
+ private boolean needScale = true; // need scaling?
+ private String trainingMode;
+ private int oraSelectMode = 1;
+ private int predSelectMode = 1;
+ private int miraIter = 1;
+ private int batchSize = 1;
+ private double C = 0.01; // relaxation coefficient
+ private double R = 0.99; // corpus decay when pseudo corpus is used for bleu computation
+ // private double sentForScale = 0.15; //percentage of sentences for scale factor estimation
+ private double scoreRatio = 5.0; // sclale so that model_score/metric_score = scoreratio
+ private double prevMetricScore = 0; // final metric score of the previous iteration, used only
+ // when returnBest = true
+
+ private String dirPrefix; // where are all these files located?
+ private String paramsFileName, docInfoFileName, finalLambdaFileName;
+ private String sourceFileName, refFileName, decoderOutFileName;
+ private String decoderConfigFileName, decoderCommandFileName;
+ private String fakeFileNameTemplate, fakeFileNamePrefix, fakeFileNameSuffix;
+
+ // e.g. output.it[1-x].someOldRun would be specified as:
+ // output.it?.someOldRun
+ // and we'd have prefix = "output.it" and suffix = ".sameOldRun"
+
+ // private int useDisk;
+
+ public MIRACore(JoshuaConfiguration joshuaConfiguration) {
+ this.joshuaConfiguration = joshuaConfiguration;
+ }
+
+ public MIRACore(String[] args, JoshuaConfiguration joshuaConfiguration) {
+ this.joshuaConfiguration = joshuaConfiguration;
+ EvaluationMetric.set_knownMetrics();
+ processArgsArray(args);
+ initialize(0);
+ }
+
+ public MIRACore(String configFileName, JoshuaConfiguration joshuaConfiguration) {
+ this.joshuaConfiguration = joshuaConfiguration;
+ EvaluationMetric.set_knownMetrics();
+ processArgsArray(cfgFileToArgsArray(configFileName));
+ initialize(0);
+ }
+
+ private void initialize(int randsToSkip) {
+ println("NegInf: " + NegInf + ", PosInf: " + PosInf + ", epsilon: " + epsilon, 4);
+
+ randGen = new Random(seed);
+ for (int r = 1; r <= randsToSkip; ++r) {
+ randGen.nextDouble();
+ }
+ generatedRands = randsToSkip;
+
+ if (randsToSkip == 0) {
+ println("----------------------------------------------------", 1);
+ println("Initializing...", 1);
+ println("----------------------------------------------------", 1);
+ println("", 1);
+
+ println("Random number generator initialized using seed: " + seed, 1);
+ println("", 1);
+ }
+
+ // count the total num of sentences to be decoded, reffilename is the combined reference file
+ // name(auto generated)
+ numSentences = countLines(refFileName) / refsPerSen;
+
+ // ??
+ processDocInfo();
+ // sets numDocuments and docOfSentence[]
+
+ if (numDocuments > 1)
+ metricName_display = "doc-level " + metricName;
+
+ // ??
+ set_docSubsetInfo(docSubsetInfo);
+
+ // count the number of initial features
+ numParams = countNonEmptyLines(paramsFileName) - 1;
+ numParamsOld = numParams;
+
+ // read parameter config file
+ try {
+ // read dense parameter names
+ BufferedReader inFile_names = new BufferedReader(new FileReader(paramsFileName));
+
+ for (int c = 1; c <= numParams; ++c) {
+ String line = "";
+ while (line != null && line.length() == 0) { // skip empty lines
+ line = inFile_names.readLine();
+ }
+
+ // save feature names
+ String paramName = (line.substring(0, line.indexOf("|||"))).trim();
+ Vocabulary.id(paramName);
+ // System.err.println(String.format("VOCAB(%s) = %d", paramName, id));
+ }
+
+ inFile_names.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ // the parameter file contains one line per parameter
+ // and one line for the normalization method
+ // indexing starts at 1 in these arrays
+ for (int p = 0; p <= numParams; ++p)
+ lambda.add(new Double(0));
+ bestLambda.add(new Double(0));
+ // why only lambda is a list? because the size of lambda
+ // may increase over time, but other arrays are specified in
+ // the param config file, only used for initialization
+ isOptimizable = new boolean[1 + numParams];
+ minRandValue = new double[1 + numParams];
+ maxRandValue = new double[1 + numParams];
+ defaultLambda = new double[1 + numParams];
+ normalizationOptions = new double[3];
+
+ // read initial param values
+ processParamFile();
+ // sets the arrays declared just above
+
+ // SentenceInfo.createV(); // uncomment ONLY IF using vocabulary implementation of SentenceInfo
+
+ String[][] refSentences = new String[numSentences][refsPerSen];
+
+ try {
+
+ // read in reference sentences
+ InputStream inStream_refs = new FileInputStream(new File(refFileName));
+ BufferedReader inFile_refs = new BufferedReader(new InputStreamReader(inStream_refs, "utf8"));
+
+ for (int i = 0; i < numSentences; ++i) {
+ for (int r = 0; r < refsPerSen; ++r) {
+ // read the rth reference translation for the ith sentence
+ refSentences[i][r] = inFile_refs.readLine();
+ }
+ }
+
+ inFile_refs.close();
+
+ // normalize reference sentences
+ for (int i = 0; i < numSentences; ++i) {
+ for (int r = 0; r < refsPerSen; ++r) {
+ // normalize the rth reference translation for the ith sentence
+ refSentences[i][r] = normalize(refSentences[i][r], textNormMethod);
+ }
+ }
+
+ // read in decoder command, if any
+ decoderCommand = null;
+ if (decoderCommandFileName != null) {
+ if (fileExists(decoderCommandFileName)) {
+ BufferedReader inFile_comm = new BufferedReader(new FileReader(decoderCommandFileName));
+ decoderCommand = inFile_comm.readLine(); // READ IN DECODE COMMAND
+ inFile_comm.close();
+ }
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ // set static data members for the EvaluationMetric class
+ EvaluationMetric.set_numSentences(numSentences);
+ EvaluationMetric.set_numDocuments(numDocuments);
+ EvaluationMetric.set_refsPerSen(refsPerSen);
+ EvaluationMetric.set_refSentences(refSentences);
+ EvaluationMetric.set_tmpDirPrefix(tmpDirPrefix);
+
+ evalMetric = EvaluationMetric.getMetric(metricName, metricOptions);
+ // used only if returnBest = true
+ prevMetricScore = evalMetric.getToBeMinimized() ? PosInf : NegInf;
+
+ // length of sufficient statistics
+ // for bleu: suffstatscount=8 (2*ngram+2)
+ suffStatsCount = evalMetric.get_suffStatsCount();
+
+ // set static data members for the IntermediateOptimizer class
+ /*
+ * IntermediateOptimizer.set_MERTparams(numSentences, numDocuments, docOfSentence,
+ * docSubsetInfo, numParams, normalizationOptions, isOptimizable oneModificationPerIteration,
+ * evalMetric, tmpDirPrefix, verbosity);
+ */
+
+ // print info
+ if (randsToSkip == 0) { // i.e. first iteration
+ println("Number of sentences: " + numSentences, 1);
+ println("Number of documents: " + numDocuments, 1);
+ println("Optimizing " + metricName_display, 1);
+
+ /*
+ * print("docSubsetInfo: {", 1); for (int f = 0; f < 6; ++f) print(docSubsetInfo[f] + ", ",
+ * 1); println(docSubsetInfo[6] + "}", 1);
+ */
+
+ println("Number of initial features: " + numParams, 1);
+ print("Initial feature names: {", 1);
+
+ for (int c = 1; c <= numParams; ++c)
+ print("\"" + Vocabulary.word(c) + "\"", 1);
+ println("}", 1);
+ println("", 1);
+
+ // TODO just print the correct info
+ println("c Default value\tOptimizable?\tRand. val. range", 1);
+
+ for (int c = 1; c <= numParams; ++c) {
+ print(c + " " + f4.format(lambda.get(c).doubleValue()) + "\t\t", 1);
+
+ if (!isOptimizable[c]) {
+ println(" No", 1);
+ } else {
+ print(" Yes\t\t", 1);
+ print(" [" + minRandValue[c] + "," + maxRandValue[c] + "]", 1);
+ println("", 1);
+ }
+ }
+
+ println("", 1);
+ print("Weight vector normalization method: ", 1);
+ if (normalizationOptions[0] == 0) {
+ println("none.", 1);
+ } else if (normalizationOptions[0] == 1) {
+ println(
+ "weights will be scaled so that the \""
+ + Vocabulary.word((int) normalizationOptions[2])
+ + "\" weight has an absolute value of " + normalizationOptions[1] + ".", 1);
+ } else if (normalizationOptions[0] == 2) {
+ println("weights will be scaled so that the maximum absolute value is "
+ + normalizationOptions[1] + ".", 1);
+ } else if (normalizationOptions[0] == 3) {
+ println("weights will be scaled so that the minimum absolute value is "
+ + normalizationOptions[1] + ".", 1);
+ } else if (normalizationOptions[0] == 4) {
+ println("weights will be scaled so that the L-" + normalizationOptions[1] + " norm is "
+ + normalizationOptions[2] + ".", 1);
+ }
+
+ println("", 1);
+
+ println("----------------------------------------------------", 1);
+ println("", 1);
+
+ // rename original config file so it doesn't get overwritten
+ // (original name will be restored in finish())
+ renameFile(decoderConfigFileName, decoderConfigFileName + ".MIRA.orig");
+ } // if (randsToSkip == 0)
+
+ // by default, load joshua decoder
+ if (decoderCommand == null && fakeFileNameTemplate == null) {
+ println("Loading Joshua decoder...", 1);
+ myDecoder = new Decoder(joshuaConfiguration, decoderConfigFileName + ".MIRA.orig");
+ println("...finished loading @ " + (new Date()), 1);
+ println("");
+ } else {
+ myDecoder = null;
+ }
+
+ @SuppressWarnings("unchecked")
+ TreeSet<Integer>[] temp_TSA = new TreeSet[numSentences];
+ indicesOfInterest_all = temp_TSA;
+
+ for (int i = 0; i < numSentences; ++i) {
+ indicesOfInterest_all[i] = new TreeSet<Integer>();
+ }
+ } // void initialize(...)
+
+ // -------------------------
+
+ public void run_MIRA() {
+ run_MIRA(minMERTIterations, maxMERTIterations, prevMERTIterations);
+ }
+
+ public void run_MIRA(int minIts, int maxIts, int prevIts) {
+ // FIRST, CLEAN ALL PREVIOUS TEMP FILES
+ String dir;
+ int k = tmpDirPrefix.lastIndexOf("/");
+ if (k >= 0) {
+ dir = tmpDirPrefix.substring(0, k + 1);
+ } else {
+ dir = "./";
+ }
+ String files;
+ File folder = new File(dir);
+
+ if (folder.exists()) {
+ File[] listOfFiles = folder.listFiles();
+
+ for (int i = 0; i < listOfFiles.length; i++) {
+ if (listOfFiles[i].isFile()) {
+ files = listOfFiles[i].getName();
+ if (files.startsWith("MIRA.temp")) {
+ deleteFile(files);
+ }
+ }
+ }
+ }
+
+ println("----------------------------------------------------", 1);
+ println("MIRA run started @ " + (new Date()), 1);
+ // printMemoryUsage();
+ println("----------------------------------------------------", 1);
+ println("", 1);
+
+ // if no default lambda is provided
+ if (randInit) {
+ println("Initializing lambda[] randomly.", 1);
+ // initialize optimizable parameters randomly (sampling uniformly from
+ // that parameter's random value range)
+ lambda = randomLambda();
+ }
+
+ println("Initial lambda[]: " + lambdaToString(lambda), 1);
+ println("", 1);
+
+ int[] maxIndex = new int[numSentences];
+
+ // HashMap<Integer,int[]>[] suffStats_array = new HashMap[numSentences];
+ // suffStats_array[i] maps candidates of interest for sentence i to an array
+ // storing the sufficient statistics for that candidate
+
+ int earlyStop = 0;
+ // number of consecutive iteration an early stopping criterion was satisfied
+
+ for (int iteration = 1;; ++iteration) {
+
+ // what does "A" contain?
+ // retA[0]: FINAL_score
+ // retA[1]: earlyStop
+ // retA[2]: should this be the last iteration?
+ double[] A = run_single_iteration(iteration, minIts, maxIts, prevIts, earlyStop, maxIndex);
+ if (A != null) {
+ earlyStop = (int) A[1];
+ if (A[2] == 1)
+ break;
+ } else {
+ break;
+ }
+
+ } // for (iteration)
+
+ println("", 1);
+
+ println("----------------------------------------------------", 1);
+ println("MIRA run ended @ " + (new Date()), 1);
+ // printMemoryUsage();
+ println("----------------------------------------------------", 1);
+ println("", 1);
+ if (!returnBest)
+ println("FINAL lambda: " + lambdaToString(lambda), 1);
+ // + " (" + metricName_display + ": " + FINAL_score + ")",1);
+ else
+ println("BEST lambda: " + lambdaToString(lambda), 1);
+
+ // delete intermediate .temp.*.it* decoder output files
+ for (int iteration = 1; iteration <= maxIts; ++iteration) {
+ if (compressFiles == 1) {
+ deleteFile(tmpDirPrefix + "temp.sents.it" + iteration + ".gz");
+ deleteFile(tmpDirPrefix + "temp.feats.it" + iteration + ".gz");
+ if (fileExists(tmpDirPrefix + "temp.stats.it" + iteration + ".copy.gz")) {
+ deleteFile(tmpDirPrefix + "temp.stats.it" + iteration + ".copy.gz");
+ } else {
+ deleteFile(tmpDirPrefix + "temp.stats.it" + iteration + ".gz");
+ }
+ } else {
+ deleteFile(tmpDirPrefix + "temp.sents.it" + iteration);
+ deleteFile(tmpDirPrefix + "temp.feats.it" + iteration);
+ if (fileExists(tmpDirPrefix + "temp.stats.it" + iteration + ".copy")) {
+ deleteFile(tmpDirPrefix + "temp.stats.it" + iteration + ".copy");
+ } else {
+ deleteFile(tmpDirPrefix + "temp.stats.it" + iteration);
+ }
+ }
+ }
+ } // void run_MIRA(int maxIts)
+
+ // this is the key function!
+ @SuppressWarnings("unchecked")
+ public double[] run_single_iteration(int iteration, int minIts, int maxIts, int prevIts,
+ int earlyStop, int[] maxIndex) {
+ double FINAL_score = 0;
+
+ double[] retA = new double[3];
+ // retA[0]: FINAL_score
+ // retA[1]: earlyStop
+ // retA[2]: should this be the last iteration?
+
+ boolean done = false;
+ retA[2] = 1; // will only be made 0 if we don't break from the following loop
+
+ // save feats and stats for all candidates(old & new)
+ HashMap<String, String>[] feat_hash = new HashMap[numSentences];
+ for (int i = 0; i < numSentences; i++)
+ feat_hash[i] = new HashMap<String, String>();
+
+ HashMap<String, String>[] stats_hash = new HashMap[numSentences];
+ for (int i = 0; i < numSentences; i++)
+ stats_hash[i] = new HashMap<String, String>();
+
+ while (!done) { // NOTE: this "loop" will only be carried out once
+ println("--- Starting MIRA iteration #" + iteration + " @ " + (new Date()) + " ---", 1);
+
+ // printMemoryUsage();
+
+ /******************************/
+ // CREATE DECODER CONFIG FILE //
+ /******************************/
+
+ createConfigFile(lambda, decoderConfigFileName, decoderConfigFileName + ".MIRA.orig");
+ // i.e. use the original config file as a template
+
+ /***************/
+ // RUN DECODER //
+ /***************/
+
+ if (iteration == 1) {
+ println("Decoding using initial weight vector " + lambdaToString(lambda), 1);
+ } else {
+ println("Redecoding using weight vector " + lambdaToString(lambda), 1);
+ }
+
+ // generate the n-best file after decoding
+ String[] decRunResult = run_decoder(iteration); // iteration passed in case fake decoder will
+ // be used
+ // [0] name of file to be processed
+ // [1] indicates how the output file was obtained:
+ // 1: external decoder
+ // 2: fake decoder
+ // 3: internal decoder
+
+ if (!decRunResult[1].equals("2")) {
+ println("...finished decoding @ " + (new Date()), 1);
+ }
+
+ checkFile(decRunResult[0]);
+
+ /************* END OF DECODING **************/
+
+ println("Producing temp files for iteration " + iteration, 3);
+
+ produceTempFiles(decRunResult[0], iteration);
+
+ // save intermedidate output files
+ // save joshua.config.mira.it*
+ if (saveInterFiles == 1 || saveInterFiles == 3) { // make copy of intermediate config file
+ if (!copyFile(decoderConfigFileName, decoderConfigFileName + ".MIRA.it" + iteration)) {
+ println("Warning: attempt to make copy of decoder config file (to create"
+ + decoderConfigFileName + ".MIRA.it" + iteration + ") was unsuccessful!", 1);
+ }
+ }
+
+ // save output.nest.MIRA.it*
+ if (saveInterFiles == 2 || saveInterFiles == 3) { // make copy of intermediate decoder output
+ // file...
+
+ if (!decRunResult[1].equals("2")) { // ...but only if no fake decoder
+ if (!decRunResult[0].endsWith(".gz")) {
+ if (!copyFile(decRunResult[0], decRunResult[0] + ".MIRA.it" + iteration)) {
+ println("Warning: attempt to make copy of decoder output file (to create"
+ + decRunResult[0] + ".MIRA.it" + iteration + ") was unsuccessful!", 1);
+ }
+ } else {
+ String prefix = decRunResult[0].substring(0, decRunResult[0].length() - 3);
+ if (!copyFile(prefix + ".gz", prefix + ".MIRA.it" + iteration + ".gz")) {
+ println("Warning: attempt to make copy of decoder output file (to create" + prefix
+ + ".MIRA.it" + iteration + ".gz" + ") was unsuccessful!", 1);
+ }
+ }
+
+ if (compressFiles == 1 && !decRunResult[0].endsWith(".gz")) {
+ gzipFile(decRunResult[0] + ".MIRA.it" + iteration);
+ }
+ } // if (!fake)
+ }
+
+ // ------------- end of saving .mira.it* files ---------------
+
+ int[] candCount = new int[numSentences];
+ int[] lastUsedIndex = new int[numSentences];
+
+ ConcurrentHashMap<Integer, int[]>[] suffStats_array = new ConcurrentHashMap[numSentences];
+ for (int i = 0; i < numSentences; ++i) {
+ candCount[i] = 0;
+ lastUsedIndex[i] = -1;
+ // suffStats_array[i].clear();
+ suffStats_array[i] = new ConcurrentHashMap<Integer, int[]>();
+ }
+
+ // initLambda[0] is not used!
+ double[] initialLambda = new double[1 + numParams];
+ for (int i = 1; i <= numParams; ++i)
+ initialLambda[i] = lambda.get(i);
+
+ // the "score" in initialScore refers to that
+ // assigned by the evaluation metric)
+
+ // you may consider all candidates from iter 1, or from iter (iteration-prevIts) to current
+ // iteration
+ int firstIt = Math.max(1, iteration - prevIts);
+ // i.e. only process candidates from the current iteration and candidates
+ // from up to prevIts previous iterations.
+ println("Reading candidate translations from iterations " + firstIt + "-" + iteration, 1);
+ println("(and computing " + metricName
+ + " sufficient statistics for previously unseen candidates)", 1);
+ print(" Progress: ");
+
+ int[] newCandidatesAdded = new int[1 + iteration];
+ for (int it = 1; it <= iteration; ++it)
+ newCandidatesAdded[it] = 0;
+
+ try {
+ // read temp files from all past iterations
+ // 3 types of temp files:
+ // 1. output hypo at iter i
+ // 2. feature value of each hypo at iter i
+ // 3. suff stats of each hypo at iter i
+
+ // each inFile corresponds to the output of an iteration
+ // (index 0 is not used; no corresponding index for the current iteration)
+ BufferedReader[] inFile_sents = new BufferedReader[iteration];
+ BufferedReader[] inFile_feats = new BufferedReader[iteration];
+ BufferedReader[] inFile_stats = new BufferedReader[iteration];
+
+ // temp file(array) from previous iterations
+ for (int it = firstIt; it < iteration; ++it) {
+ InputStream inStream_sents, inStream_feats, inStream_stats;
+ if (compressFiles == 0) {
+ inStream_sents = new FileInputStream(tmpDirPrefix + "temp.sents.it" + it);
+ inStream_feats = new FileInputStream(tmpDirPrefix + "temp.feats.it" + it);
+ inStream_stats = new FileInputStream(tmpDirPrefix + "temp.stats.it" + it);
+ } else {
+ inStream_sents = new GZIPInputStream(new FileInputStream(tmpDirPrefix + "temp.sents.it"
+ + it + ".gz"));
+ inStream_feats = new GZIPInputStream(new FileInputStream(tmpDirPrefix + "temp.feats.it"
+ + it + ".gz"));
+ inStream_stats = new GZIPInputStream(new FileInputStream(tmpDirPrefix + "temp.stats.it"
+ + it + ".gz"));
+ }
+
+ inFile_sents[it] = new BufferedReader(new InputStreamReader(inStream_sents, "utf8"));
+ inFile_feats[it] = new BufferedReader(new InputStreamReader(inStream_feats, "utf8"));
+ inFile_stats[it] = new BufferedReader(new InputStreamReader(inStream_stats, "utf8"));
+ }
+
+ InputStream inStream_sentsCurrIt, inStream_featsCurrIt, inStream_statsCurrIt;
+ // temp file for current iteration!
+ if (compressFiles == 0) {
+ inStream_sentsCurrIt = new FileInputStream(tmpDirPrefix + "temp.sents.it" + iteration);
+ inStream_featsCurrIt = new FileInputStream(tmpDirPrefix + "temp.feats.it" + iteration);
+ } else {
+ inStream_sentsCurrIt = new GZIPInputStream(new FileInputStream(tmpDirPrefix
+ + "temp.sents.it" + iteration + ".gz"));
+ inStream_featsCurrIt = new GZIPInputStream(new FileInputStream(tmpDirPrefix
+ + "temp.feats.it" + iteration + ".gz"));
+ }
+
+ BufferedReader inFile_sentsCurrIt = new BufferedReader(new InputStreamReader(
+ inStream_sentsCurrIt, "utf8"));
+ BufferedReader inFile_featsCurrIt = new BufferedReader(new InputStreamReader(
+ inStream_featsCurrIt, "utf8"));
+
+ BufferedReader inFile_statsCurrIt = null; // will only be used if statsCurrIt_exists below
+ // is set to true
+ PrintWriter outFile_statsCurrIt = null; // will only be used if statsCurrIt_exists below is
+ // set to false
+
+ // just to check if temp.stat.it.iteration exists
+ boolean statsCurrIt_exists = false;
+
+ if (fileExists(tmpDirPrefix + "temp.stats.it" + iteration)) {
+ inStream_statsCurrIt = new FileInputStream(tmpDirPrefix + "temp.stats.it" + iteration);
+ inFile_statsCurrIt = new BufferedReader(new InputStreamReader(inStream_statsCurrIt,
+ "utf8"));
+ statsCurrIt_exists = true;
+ copyFile(tmpDirPrefix + "temp.stats.it" + iteration, tmpDirPrefix + "temp.stats.it"
+ + iteration + ".copy");
+ } else if (fileExists(tmpDirPrefix + "temp.stats.it" + iteration + ".gz")) {
+ inStream_statsCurrIt = new GZIPInputStream(new FileInputStream(tmpDirPrefix
+ + "temp.stats.it" + iteration + ".gz"));
+ inFile_statsCurrIt = new BufferedReader(new InputStreamReader(inStream_statsCurrIt,
+ "utf8"));
+ statsCurrIt_exists = true;
+ copyFile(tmpDirPrefix + "temp.stats.it" + iteration + ".gz", tmpDirPrefix
+ + "temp.stats.it" + iteration + ".copy.gz");
+ } else {
+ outFile_statsCurrIt = new PrintWriter(tmpDirPrefix + "temp.stats.it" + iteration);
+ }
+
+ // output the 4^th temp file: *.temp.stats.merged
+ PrintWriter outFile_statsMerged = new PrintWriter(tmpDirPrefix + "temp.stats.merged");
+ // write sufficient statistics from all the sentences
+ // from the output files into a single file
+ PrintWriter outFile_statsMergedKnown = new PrintWriter(tmpDirPrefix
+ + "temp.stats.mergedKnown");
+ // write sufficient statistics from all the sentences
+ // from the output files into a single file
+
+ // output the 5^th 6^th temp file, but will be deleted at the end of the function
+ FileOutputStream outStream_unknownCands = new FileOutputStream(tmpDirPrefix
+ + "temp.currIt.unknownCands", false);
+ OutputStreamWriter outStreamWriter_unknownCands = new OutputStreamWriter(
+ outStream_unknownCands, "utf8");
+ BufferedWriter outFile_unknownCands = new BufferedWriter(outStreamWriter_unknownCands);
+
+ PrintWriter outFile_unknownIndices = new PrintWriter(tmpDirPrefix
+ + "temp.currIt.unknownIndices");
+
+ String sents_str, feats_str, stats_str;
+
+ // BUG: this assumes a candidate string cannot be produced for two
+ // different source sentences, which is not necessarily true
+ // (It's not actually a bug, but only because existingCandStats gets
+ // cleared before moving to the next source sentence.)
+ // FIX: should be made an array, indexed by i
+ HashMap<String, String> existingCandStats = new HashMap<String, String>();
+ // VERY IMPORTANT:
+ // A CANDIDATE X MAY APPEARED IN ITER 1, ITER 3
+ // BUT IF THE USER SPECIFIED TO CONSIDER ITERATIONS FROM ONLY ITER 2, THEN
+ // X IS NOT A "REPEATED" CANDIDATE IN ITER 3. THEREFORE WE WANT TO KEEP THE
+ // SUFF STATS FOR EACH CANDIDATE(TO SAVE COMPUTATION IN THE FUTURE)
+
+ // Stores precalculated sufficient statistics for candidates, in case
+ // the same candidate is seen again. (SS stored as a String.)
+ // Q: Why do we care? If we see the same candidate again, aren't we going
+ // to ignore it? So, why do we care about the SS of this repeat candidate?
+ // A: A "repeat" candidate may not be a repeat candidate in later
+ // iterations if the user specifies a value for prevMERTIterations
+ // that causes MERT to skip candidates from early iterations.
+
+ double[] currFeatVal = new double[1 + numParams];
+ String[] featVal_str;
+
+ int totalCandidateCount = 0;
+
+ // new candidate size for each sentence
+ int[] sizeUnknown_currIt = new int[numSentences];
+
+ for (int i = 0; i < numSentences; ++i) {
+ // process candidates from previous iterations
+ // low efficiency? for each iteration, it reads in all previous iteration outputs
+ // therefore a lot of overlapping jobs
+ // this is an easy implementation to deal with the situation in which user only specified
+ // "previt" and hopes to consider only the previous previt
+ // iterations, then for each iteration the existing candadites will be different
+ for (int it = firstIt; it < iteration; ++it) {
+ // Why up to but *excluding* iteration?
+ // Because the last iteration is handled a little differently, since
+ // the SS must be calculated (and the corresponding file created),
+ // which is not true for previous iterations.
+
+ for (int n = 0; n <= sizeOfNBest; ++n) {
+ // note that in all temp files, "||||||" is a separator between 2 n-best lists
+
+ // Why up to and *including* sizeOfNBest?
+ // So that it would read the "||||||" separator even if there is
+ // a complete list of sizeOfNBest candidates.
+
+ // for the nth candidate for the ith sentence, read the sentence, feature values,
+ // and sufficient statistics from the various temp files
+
+ // read one line of temp.sent, temp.feat, temp.stats from iteration it
+ sents_str = inFile_sents[it].readLine();
+ feats_str = inFile_feats[it].readLine();
+ stats_str = inFile_stats[it].readLine();
+
+ if (sents_str.equals("||||||")) {
+ n = sizeOfNBest + 1; // move on to the next n-best list
+ } else if (!existingCandStats.containsKey(sents_str)) // if this candidate does not
+ // exist
+ {
+ outFile_statsMergedKnown.println(stats_str);
+
+ // save feats & stats
+ feat_hash[i].put(sents_str, feats_str);
+ stats_hash[i].put(sents_str, stats_str);
+
+ // extract feature value
+ featVal_str = feats_str.split("\\s+");
+
+ existingCandStats.put(sents_str, stats_str);
+ candCount[i] += 1;
+ newCandidatesAdded[it] += 1;
+
+ } // if unseen candidate
+ } // for (n)
+ } // for (it)
+
+ outFile_statsMergedKnown.println("||||||");
+
+ // ---------- end of processing previous iterations ----------
+ // ---------- now start processing new candidates ----------
+
+ // now process the candidates of the current iteration
+ // now determine the new candidates of the current iteration
+
+ /*
+ * remember: BufferedReader inFile_sentsCurrIt BufferedReader inFile_featsCurrIt
+ * PrintWriter outFile_statsCurrIt
+ */
+
+ String[] sentsCurrIt_currSrcSent = new String[sizeOfNBest + 1];
+
+ Vector<String> unknownCands_V = new Vector<String>();
+ // which candidates (of the i'th source sentence) have not been seen before
+ // this iteration?
+
+ for (int n = 0; n <= sizeOfNBest; ++n) {
+ // Why up to and *including* sizeOfNBest?
+ // So that it would read the "||||||" separator even if there is
+ // a complete list of sizeOfNBest candidates.
+
+ // for the nth candidate for the ith sentence, read the sentence,
+ // and store it in the sentsCurrIt_currSrcSent array
+
+ sents_str = inFile_sentsCurrIt.readLine(); // read one candidate from the current
+ // iteration
+ sentsCurrIt_currSrcSent[n] = sents_str; // Note: possibly "||||||"
+
+ if (sents_str.equals("||||||")) {
+ n = sizeOfNBest + 1;
+ } else if (!existingCandStats.containsKey(sents_str)) {
+ unknownCands_V.add(sents_str); // NEW CANDIDATE FROM THIS ITERATION
+ writeLine(sents_str, outFile_unknownCands);
+ outFile_unknownIndices.println(i); // INDEX OF THE NEW CANDIDATES
+ newCandidatesAdded[iteration] += 1;
+ existingCandStats.put(sents_str, "U"); // i.e. unknown
+ // we add sents_str to avoid duplicate entries in unknownCands_V
+ }
+ } // for (n)
+
+ // only compute suff stats for new candidates
+ // now unknownCands_V has the candidates for which we need to calculate
+ // sufficient statistics (for the i'th source sentence)
+ int sizeUnknown = unknownCands_V.size();
+ sizeUnknown_currIt[i] = sizeUnknown;
+
+ existingCandStats.clear();
+
+ } // for (i) each sentence
+
+ // ---------- end of merging candidates stats from previous iterations
+ // and finding new candidates ------------
+
+ /*
+ * int[][] newSuffStats = null; if (!statsCurrIt_exists && sizeUnknown > 0) { newSuffStats =
+ * evalMetric.suffStats(unknownCands, indices); }
+ */
+
+ outFile_statsMergedKnown.close();
+ outFile_unknownCands.close();
+ outFile_unknownIndices.close();
+
+ // want to re-open all temp files and start from scratch again?
+ for (int it = firstIt; it < iteration; ++it) // previous iterations temp files
+ {
+ inFile_sents[it].close();
+ inFile_stats[it].close();
+
+ InputStream inStream_sents, inStream_stats;
+ if (compressFiles == 0) {
+ inStream_sents = new FileInputStream(tmpDirPrefix + "temp.sents.it" + it);
+ inStream_stats = new FileInputStream(tmpDirPrefix + "temp.stats.it" + it);
+ } else {
+ inStream_sents = new GZIPInputStream(new FileInputStream(tmpDirPrefix + "temp.sents.it"
+ + it + ".gz"));
+ inStream_stats = new GZIPInputStream(new FileInputStream(tmpDirPrefix + "temp.stats.it"
+ + it + ".gz"));
+ }
+
+ inFile_sents[it] = new BufferedReader(new InputStreamReader(inStream_sents, "utf8"));
+ inFile_stats[it] = new BufferedReader(new InputStreamReader(inStream_stats, "utf8"));
+ }
+
+ inFile_sentsCurrIt.close();
+ // current iteration temp files
+ if (compressFiles == 0) {
+ inStream_sentsCurrIt = new FileInputStream(tmpDirPrefix + "temp.sents.it" + iteration);
+ } else {
+ inStream_sentsCurrIt = new GZIPInputStream(new FileInputStream(tmpDirPrefix
+ + "temp.sents.it" + iteration + ".gz"));
+ }
+ inFile_sentsCurrIt = new BufferedReader(new InputStreamReader(inStream_sentsCurrIt, "utf8"));
+
+ // calculate SS for unseen candidates and write them to file
+ FileInputStream inStream_statsCurrIt_unknown = null;
+ BufferedReader inFile_statsCurrIt_unknown = null;
+
+ if (!statsCurrIt_exists && newCandidatesAdded[iteration] > 0) {
+ // create the file...
+ evalMetric.createSuffStatsFile(tmpDirPrefix + "temp.currIt.unknownCands", tmpDirPrefix
+ + "temp.currIt.unknownIndices", tmpDirPrefix + "temp.stats.unknown", sizeOfNBest);
+
+ // ...and open it
+ inStream_statsCurrIt_unknown = new FileInputStream(tmpDirPrefix + "temp.stats.unknown");
+ inFile_statsCurrIt_unknown = new BufferedReader(new InputStreamReader(
+ inStream_statsCurrIt_unknown, "utf8"));
+ }
+
+ // open mergedKnown file
+ // newly created by the big loop above
+ FileInputStream instream_statsMergedKnown = new FileInputStream(tmpDirPrefix
+ + "temp.stats.mergedKnown");
+ BufferedReader inFile_statsMergedKnown = new BufferedReader(new InputStreamReader(
+ instream_statsMergedKnown, "utf8"));
+
+ // num of features before observing new firing features from this iteration
+ numParamsOld = numParams;
+
+ for (int i = 0; i < numSentences; ++i) {
+ // reprocess candidates from previous iterations
+ for (int it = firstIt; it < iteration; ++it) {
+ for (int n = 0; n <= sizeOfNBest; ++n) {
+ sents_str = inFile_sents[it].readLine();
+ stats_str = inFile_stats[it].readLine();
+
+ if (sents_str.equals("||||||")) {
+ n = sizeOfNBest + 1;
+ } else if (!existingCandStats.containsKey(sents_str)) {
+ existingCandStats.put(sents_str, stats_str);
+ } // if unseen candidate
+ } // for (n)
+ } // for (it)
+
+ // copy relevant portion from mergedKnown to the merged file
+ String line_mergedKnown = inFile_statsMergedKnown.readLine();
+ while (!line_mergedKnown.equals("||||||")) {
+ outFile_statsMerged.println(line_mergedKnown);
+ line_mergedKnown = inFile_statsMergedKnown.readLine();
+ }
+
+ int[] stats = new int[suffStatsCount];
+
+ for (int n = 0; n <= sizeOfNBest; ++n) {
+ sents_str = inFile_sentsCurrIt.readLine();
+ feats_str = inFile_featsCurrIt.readLine();
+
+ if (sents_str.equals("||||||")) {
+ n = sizeOfNBest + 1;
+ } else if (!existingCandStats.containsKey(sents_str)) {
+
+ if (!statsCurrIt_exists) {
+ stats_str = inFile_statsCurrIt_unknown.readLine();
+
+ String[] temp_stats = stats_str.split("\\s+");
+ for (int s = 0; s < suffStatsCount; ++s) {
+ stats[s] = Integer.parseInt(temp_stats[s]);
+ }
+
+ outFile_statsCurrIt.println(stats_str);
+ } else {
+ stats_str = inFile_statsCurrIt.readLine();
+
+ String[] temp_stats = stats_str.split("\\s+");
+ for (int s = 0; s < suffStatsCount; ++s) {
+ stats[s] = Integer.parseInt(temp_stats[s]);
+ }
+ }
+
+ outFile_statsMerged.println(stats_str);
+
+ // save feats & stats
+ // System.out.println(sents_str+" "+feats_str);
+
+ feat_hash[i].put(sents_str, feats_str);
+ stats_hash[i].put(sents_str, stats_str);
+
+ featVal_str = feats_str.split("\\s+");
+
+ if (feats_str.indexOf('=') != -1) {
+ for (String featurePair : featVal_str) {
+ String[] pair = featurePair.split("=");
+ String name = pair[0];
+ Double value = Double.parseDouble(pair[1]);
+ int featId = Vocabulary.id(name);
+
+ // need to identify newly fired feats here
+ // in this case currFeatVal is not given the value
+ // of the new feat, since the corresponding weight is
+ // initialized as zero anyway
+ if (featId > numParams) {
+ ++numParams;
+ lambda.add(new Double(0));
+ }
+ }
+ }
+ existingCandStats.put(sents_str, stats_str);
+ candCount[i] += 1;
+
+ // newCandidatesAdded[iteration] += 1;
+ // moved to code above detecting new candidates
+ } else {
+ if (statsCurrIt_exists)
+ inFile_statsCurrIt.readLine();
+ else {
+ // write SS to outFile_statsCurrIt
+ stats_str = existingCandStats.get(sents_str);
+ outFile_statsCurrIt.println(stats_str);
+ }
+ }
+
+ } // for (n)
+
+ // now d = sizeUnknown_currIt[i] - 1
+
+ if (statsCurrIt_exists)
+ inFile_statsCurrIt.readLine();
+ else
+ outFile_statsCurrIt.println("||||||");
+
+ existingCandStats.clear();
+ totalCandidateCount += candCount[i];
+
+ // output sentence progress
+ if ((i + 1) % 500 == 0) {
+ print((i + 1) + "\n" + " ", 1);
+ } else if ((i + 1) % 100 == 0) {
+ print("+", 1);
+ } else if ((i + 1) % 25 == 0) {
+ print(".", 1);
+ }
+
+ } // for (i)
+
+ inFile_statsMergedKnown.close();
+ outFile_statsMerged.close();
+
+ // for testing
+ /*
+ * int total_sent = 0; for( int i=0; i<numSentences; i++ ) {
+ * System.out.println(feat_hash[i].size()+" "+candCount[i]); total_sent +=
+ * feat_hash[i].size(); feat_hash[i].clear(); }
+ * System.out.println("----------------total sent: "+total_sent); total_sent = 0; for( int
+ * i=0; i<numSentences; i++ ) { System.out.println(stats_hash[i].size()+" "+candCount[i]);
+ * total_sent += stats_hash[i].size(); stats_hash[i].clear(); }
+ * System.out.println("*****************total sent: "+total_sent);
+ */
+
+ println("", 1); // finish progress line
+
+ for (int it = firstIt; it < iteration; ++it) {
+ inFile_sents[it].close();
+ inFile_feats[it].close();
+ inFile_stats[it].close();
+ }
+
+ inFile_sentsCurrIt.close();
+ inFile_featsCurrIt.close();
+ if (statsCurrIt_exists)
+ inFile_statsCurrIt.close();
+ else
+ outFile_statsCurrIt.close();
+
+ if (compressFiles == 1 && !statsCurrIt_exists) {
+ gzipFile(tmpDirPrefix + "temp.stats.it" + iteration);
+ }
+
+ // clear temp files
+ deleteFile(tmpDirPrefix + "temp.currIt.unknownCands");
+ deleteFile(tmpDirPrefix + "temp.currIt.unknownIndices");
+ deleteFile(tmpDirPrefix + "temp.stats.unknown");
+ deleteFile(tmpDirPrefix + "temp.stats.mergedKnown");
+
+ // cleanupMemory();
+
+ println("Processed " + totalCandidateCount + " distinct candidates " + "(about "
+ + totalCandidateCount / numSentences + " per sentence):", 1);
+ for (int it = firstIt; it <= iteration; ++it) {
+ println("newCandidatesAdded[it=" + it + "] = " + newCandidatesAdded[it] + " (about "
+ + newCandidatesAdded[it] / numSentences + " per sentence)", 1);
+ }
+
+ println("", 1);
+
+ println("Number of features observed so far: " + numParams);
+ println("", 1);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ // n-best list converges
+ if (newCandidatesAdded[iteration] == 0) {
+ if (!oneModificationPerIteration) {
+ println("No new candidates added in this iteration; exiting MIRA.", 1);
+ println("", 1);
+ println("--- MIRA iteration #" + iteration + " ending @ " + (new Date()) + " ---", 1);
+ println("", 1);
+ deleteFile(tmpDirPrefix + "temp.stats.merged");
+
+ if (returnBest) {
+ // note that bestLambda.size() <= lambda.size()
+ for (int p = 1; p < bestLambda.size(); ++p)
+ lambda.set(p, bestLambda.get(p));
+ // and set the rest of lambda to be 0
+ for (int p = 0; p < lambda.size() - bestLambda.size(); ++p)
+ lambda.set(p + bestLambda.size(), new Double(0));
+ }
+
+ return null; // this means that the old values should be kept by the caller
+ } else {
+ println("Note: No new candidates added in this iteration.", 1);
+ }
+ }
+
+ /************* start optimization **************/
+
+ /*
+ * for( int v=1; v<initialLambda[1].length; v++ ) System.out.print(initialLambda[1][v]+" ");
+ * System.exit(0);
+ */
+
+ Optimizer.sentNum = numSentences; // total number of training sentences
+ Optimizer.needShuffle = needShuffle;
+ Optimizer.miraIter = miraIter;
+ Optimizer.oraSelectMode = oraSelectMode;
+ Optimizer.predSelectMode = predSelectMode;
+ Optimizer.runPercep = runPercep;
+ Optimizer.C = C;
+ Optimizer.needAvg = needAvg;
+ // Optimizer.sentForScale = sentForScale;
+ Optimizer.scoreRatio = scoreRatio;
+ Optimizer.evalMetric = evalMetric;
+ Optimizer.normalizationOptions = normalizationOptions;
+ Optimizer.needScale = needScale;
+ Optimizer.batchSize = batchSize;
+
+ // if need to use bleu stats history
+ if (iteration == 1) {
+ if (evalMetric.get_metricName().equals("BLEU") && usePseudoBleu) {
+ Optimizer.initBleuHistory(numSentences, evalMetric.get_suffStatsCount());
+ Optimizer.usePseudoBleu = usePseudoBleu;
+ Optimizer.R = R;
+ }
+ if (evalMetric.get_metricName().equals("TER-BLEU") && usePseudoBleu) {
+ Optimizer.initBleuHistory(numSentences, evalMetric.get_suffStatsCount() - 2); // Stats
+ // count of
+ // TER=2
+ Optimizer.usePseudoBleu = usePseudoBleu;
+ Optimizer.R = R;
+ }
+ }
+
+ Vector<String> output = new Vector<String>();
+
+ // note: initialLambda[] has length = numParamsOld
+ // augmented with new feature weights, initial values are 0
+ double[] initialLambdaNew = new double[1 + numParams];
+ System.arraycopy(initialLambda, 1, initialLambdaNew, 1, numParamsOld);
+
+ // finalLambda[] has length = numParams (considering new features)
+ double[] finalLambda = new double[1 + numParams];
+
+ Optimizer opt = new Optimizer(output, isOptimizable, initialLambdaNew, feat_hash, stats_hash);
+ finalLambda = opt.runOptimizer();
+
+ if (returnBest) {
+ double metricScore = opt.getMetricScore();
+ if (!evalMetric.getToBeMinimized()) {
+ if (metricScore > prevMetricScore) {
+ prevMetricScore = metricScore;
+ for (int p = 1; p < bestLambda.size(); ++p)
+ bestLambda.set(p, finalLambda[p]);
+ if (1 + numParams > bestLambda.size()) {
+ for (int p = bestLambda.size(); p <= numParams; ++p)
+ bestLambda.add(p, finalLambda[p]);
+ }
+ }
+ } else {
+ if (metricScore < prevMetricScore) {
+ prevMetricScore = metricScore;
+ for (int p = 1; p < bestLambda.size(); ++p)
+ bestLambda.set(p, finalLambda[p]);
+ if (1 + numParams > bestLambda.size()) {
+ for (int p = bestLambda.size(); p <= numParams; ++p)
+ bestLambda.add(p, finalLambda[p]);
+ }
+ }
+ }
+ }
+
+ // System.out.println(finalLambda.length);
+ // for( int i=0; i<finalLambda.length-1; i++ )
+ // System.out.println(finalLambda[i+1]);
+
+ /************* end optimization **************/
+
+ for (int i = 0; i < output.size(); i++)
+ println(output.get(i));
+
+ // check if any parameter has been updated
+ boolean anyParamChanged = false;
+ boolean anyParamChangedSignificantly = false;
+
+ for (int c = 1; c <= numParams; ++c) {
+ if (finalLambda[c] != lambda.get(c)) {
+ anyParamChanged = true;
+ }
+ if (Math.abs(finalLambda[c] - lambda.get(c)) > stopSigValue) {
+ anyParamChangedSignificantly = true;
+ }
+ }
+
+ // System.arraycopy(finalLambda,1,lambda,1,numParams);
+
+ println("--- MIRA iteration #" + iteration + " ending @ " + (new Date()) + " ---", 1);
+ println("", 1);
+
+ if (!anyParamChanged) {
+ println("No parameter value changed in this iteration; exiting MIRA.", 1);
+ println("", 1);
+ break; // exit for (iteration) loop preemptively
+ }
+
+ // was an early stopping criterion satisfied?
+ boolean critSatisfied = false;
+ if (!anyParamChangedSignificantly && stopSigValue >= 0) {
+ println("Note: No parameter value changed significantly " + "(i.e. by more than "
+ + stopSigValue + ") in this iteration.", 1);
+ critSatisfied = true;
+ }
+
+ if (critSatisfied) {
+ ++earlyStop;
+ println("", 1);
+ } else {
+ earlyStop = 0;
+ }
+
+ // if min number of iterations executed, investigate if early exit should happen
+ if (iteration >= minIts && earlyStop >= stopMinIts) {
+ println("Some early stopping criteria has been observed " + "in " + stopMinIts
+ + " consecutive iterations; exiting MIRA.", 1);
+ println("", 1);
+
+ if (returnBest) {
+ for (int f = 1; f <= bestLambda.size() - 1; ++f)
+ lambda.set(f, bestLambda.get(f));
+ } else {
+ for (int f = 1; f <= numParams; ++f)
+ lambda.set(f, finalLambda[f]);
+ }
+
+ break; // exit for (iteration) loop preemptively
+ }
+
+ // if max number of iterations executed, exit
+ if (iteration >= maxIts) {
+ println("Maximum number of MIRA iterations reached; exiting MIRA.", 1);
+ println("", 1);
+
+ if (returnBest) {
+ for (int f = 1; f <= bestLambda.size() - 1; ++f)
+ lambda.set(f, bestLambda.get(f));
+ } else {
+ for (int f = 1; f <= numParams; ++f)
+ lambda.set(f, finalLambda[f]);
+ }
+
+ break; // exit for (iteration) loop
+ }
+
+ // use the new wt vector to decode the next iteration
+ // (interpolation with previous wt vector)
+ double interCoef = 1.0; // no interpolation for now
+ for (int i = 1; i <= numParams; i++)
+ lambda.set(i, interCoef * finalLambda[i] + (1 - interCoef) * lambda.get(i).doubleValue());
+
+ println("Next iteration will decode with lambda: " + lambdaToString(lambda), 1);
+ println("", 1);
+
+ // printMemoryUsage();
+ for (int i = 0; i < numSentences; ++i) {
+ suffStats_array[i].clear();
+ }
+ // cleanupMemory();
+ // println("",2);
+
+ retA[2] = 0; // i.e. this should NOT be the last iteration
+ done = true;
+
+ } // while (!done) // NOTE: this "loop" will only be carried out once
+
+ // delete .temp.stats.merged file, since it is not needed in the next
+ // iteration (it will be recreated from scratch)
+ deleteFile(tmpDirPrefix + "temp.stats.merged");
+
+ retA[0] = FINAL_score;
+ retA[1] = earlyStop;
+ return retA;
+
+ } // run_single_iteration
+
+ private String lambdaToString(ArrayList<Double> lambdaA) {
+ String retStr = "{";
+ int featToPrint = numParams > 15 ? 15 : numParams;
+ // print at most the first 15 features
+
+ retStr += "(listing the first " + featToPrint + " lambdas)";
+ for (int c = 1; c <= featToPrint - 1; ++c) {
+ retStr += "" + String.format("%.4f", lambdaA.get(c).doubleValue()) + ", ";
+ }
+ retStr += "" + String.format("%.4f", lambdaA.get(numParams).doubleValue()) + "}";
+
+ return retStr;
+ }
+
+ private String[] run_decoder(int iteration) {
+ String[] retSA = new String[2];
+
+ // retsa saves the output file name(nbest-file)
+ // and the decoder type
+
+ // [0] name of file to be processed
+ // [1] indicates how the output file was obtained:
+ // 1: external decoder
+ // 2: fake decoder
+ // 3: internal decoder
+
+ // use fake decoder
+ if (fakeFileNameTemplate != null
+ && fileExists(fakeFileNamePrefix + iteration + fakeFileNameSuffix)) {
+ String fakeFileName = fakeFileNamePrefix + iteration + fakeFileNameSuffix;
+ println("Not running decoder; using " + fakeFileName + " instead.", 1);
+ /*
+ * if (fakeFileName.endsWith(".gz")) { copyFile(fakeFileName,decoderOutFileName+".gz");
+ * gunzipFile(decoderOutFileName+".gz"); } else { copyFile(fakeFileName,decoderOutFileName); }
+ */
+ retSA[0] = fakeFileName;
+ retSA[1] = "2";
+
+ } else {
+ println("Running external decoder...", 1);
+
+ try {
+ ArrayList<String> cmd = new ArrayList<String>();
+ cmd.add(decoderCommandFileName);
+
+ if (passIterationToDecoder)
+ cmd.add(Integer.toString(iteration));
+
+ ProcessBuilder pb = new ProcessBuilder(cmd);
+ // this merges the error and output streams of the subprocess
+ pb.redirectErrorStream(true);
+ Process p = pb.start();
+
+ // capture the sub-command's output
+ new StreamGobbler(p.getInputStream(), decVerbosity).start();
+
+ int decStatus = p.waitFor();
+ if (decStatus != validDecoderExitValue) {
+ throw new RuntimeException("Call to decoder returned " + decStatus + "; was expecting "
+ + validDecoderExitValue + ".");
+ }
+ } catch (IOException| InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+
+ retSA[0] = decoderOutFileName;
+ retSA[1] = "1";
+
+ }
+
+ return retSA;
+ }
+
+ private void produceTempFiles(String nbestFileName, int iteration) {
+ try {
+ String sentsFileName = tmpDirPrefix + "temp.sents.it" + iteration;
+ String featsFileName = tmpDirPrefix + "temp.feats.it" + iteration;
+
+ FileOutputStream outStream_sents = new FileOutputStream(sentsFileName, false);
+ OutputStreamWriter outStreamWriter_sents = new OutputStreamWriter(outStream_sents, "utf8");
+ BufferedWriter outFile_sents = new BufferedWriter(outStreamWriter_sents);
+
+ PrintWriter outFile_feats = new PrintWriter(featsFileName);
+
+ InputStream inStream_nbest = null;
+ if (nbestFileName.endsWith(".gz")) {
+ inStream_nbest = new GZIPInputStream(new FileInputStream(nbestFileName));
+ } else {
+ inStream_nbest = new FileInputStream(nbestFileName);
+ }
+ BufferedReader inFile_nbest = new BufferedReader(
+ new InputStreamReader(inStream_nbest, "utf8"));
+
+ String line; // , prevLine;
+ String candidate_str = "";
+ String feats_str = "";
+
+ int i = 0;
+ int n = 0;
+ line = inFile_nbest.readLine();
+
+ while (line != null) {
+
+ /*
+ * line format:
+ *
+ * i ||| words of candidate translation . ||| feat-1_val feat-2_val ... feat-numParams_val
+ * .*
+ */
+
+ // in a well formed file, we'd find the nth candidate for the ith sentence
+
+ int read_i = Integer.parseInt((line.substring(0, line.indexOf("|||"))).trim());
+
+ if (read_i != i) {
+ writeLine("||||||", outFile_sents);
+ outFile_feats.println("||||||");
+ n = 0;
+ ++i;
+ }
+
+ line = (line.substring(line.indexOf("|||") + 3)).trim(); // get rid of initial text
+
+ candidate_str = (line.substring(0, line.indexOf("|||"))).trim();
+ feats_str = (line.substring(line.indexOf("|||") + 3)).trim();
+ // get rid of candidate string
+
+ int junk_i = feats_str.indexOf("|||");
+ if (junk_i >= 0) {
+ feats_str = (feats_str.substring(0, junk_i)).trim();
+ }
+
+ writeLine(normalize(candidate_str, textNormMethod), outFile_sents);
+ outFile_feats.println(feats_str);
+
+ ++n;
+ if (n == sizeOfNBest) {
+ writeLine("||||||", outFile_sents);
+ outFile_feats.println("||||||");
+ n = 0;
+ ++i;
+ }
+
+ line = inFile_nbest.readLine();
+ }
+
+ if (i != numSentences) { // last sentence had too few candidates
+ writeLine("||||||", outFile_sents);
+ outFile_feats.println("||||||");
+ }
+
+ inFile_nbest.close();
+ outFile_sents.close();
+ outFile_feats.close();
+
+ if (compressFiles == 1) {
+ gzipFile(sentsFileName);
+ gzipFile(featsFileName);
+ }
+
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ }
+
+ private void createConfigFile(ArrayList<Double> params, String cfgFileName,
+ String templateFileName) {
+ try {
+ // i.e. create cfgFileName, which is similar to templateFileName, but with
+ // params[] as parameter values
+
+ BufferedReader inFile = new BufferedReader(new FileReader(templateFileName));
+ PrintWriter outFile = new PrintWriter(cfgFileName);
+
+ BufferedReader inFeatDefFile = null;
+ PrintWriter outFeatDefFile = null;
+ int origFeatNum = 0; // feat num in the template file
+
+ String line = inFile.readLine();
+ while (line != null) {
+ int c_match = -1;
+ for (int c = 1; c <= numParams; ++c) {
+ if (line.startsWith(Vocabulary.word(c) + " ")) {
+ c_match = c;
+ ++origFeatNum;
+ break;
+ }
+ }
+
+ if (c_match == -1) {
+ outFile.println(line);
+ } else {
+ if (Math.abs(params.get(c_match).doubleValue()) > 1e-20)
+ outFile.println(Vocabulary.word(c_match) + " " + params.get(c_match));
+ }
+
+ line = inFile.readLine();
+ }
+
+ // now append weights of new features
+ for (int c = origFeatNum + 1; c <= numParams; ++c) {
+ if (Math.abs(params.get(c).doubleValue()) > 1e-20)
+ outFile.println(Vocabulary.word(c) + " " + params.get(c));
+ }
+
+ inFile.close();
+ outFile.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void processParamFile() {
+ // process parameter file
+ Scanner inFile_init = null;
+ try {
+ inFile_init = new Scanner(new FileReader(paramsFileName));
+ } catch (FileNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+
+ String dummy = "";
+
+ // initialize lambda[] and other related arrays
+ for (int c = 1; c <= numParams; ++c) {
+ // skip parameter name
+ while (!dummy.equals("|||")) {
+ dummy = inFile_init.next();
+ }
+
+ // read default value
+ lambda.set(c, inFile_init.nextDouble());
+ defaultLambda[c] = lambda.get(c).doubleValue();
+
+ // read isOptimizable
+ dummy = inFile_init.next();
+ if (dummy.equals("Opt")) {
+ isOptimizable[c] = true;
+ } else if (dummy.equals("Fix")) {
+ isOptimizable[c] = false;
+ } else {
+ throw new RuntimeException("Unknown isOptimizable string " + dummy + " (must be either Opt or Fix)");
+ }
+
+ if (!isOptimizable[c]) { // skip next two values
+ dummy = inFile_init.next();
+ dummy = inFile_init.next();
+ dummy = inFile_init.next();
+ dummy = inFile_init.next();
+ } else {
+ // the next two values are not used, only to be consistent with ZMERT's params file format
+ dummy = inFile_init.next();
+ dummy = inFile_init.next();
+ // set minRandValue[c] and maxRandValue[c] (range for random values)
+ dummy = inFile_init.next();
+ if (dummy.equals("-Inf") || dummy.equals("+Inf")) {
+ throw new RuntimeException("minRandValue[" + c + "] cannot be -Inf or +Inf!");
+ } else {
+ minRandValue[c] = Double.parseDouble(dummy);
+ }
+
+ dummy = inFile_init.next();
+ if (dummy.equals("-Inf") || dummy.equals("+Inf")) {
+ throw new RuntimeException("maxRandValue[" + c + "] cannot be -Inf or +Inf!");
+ } else {
+ maxRandValue[c] = Double.parseDouble(dummy);
+ }
+
+ // check for illogical values
+ if (minRandValue[c] > maxRandValue[c]) {
+ throw new RuntimeException("minRandValue[" + c + "]=" + minRandValue[c]
+ + " > " + maxRandValue[c] + "=maxRandValue[" + c + "]!");
+ }
+
+ // check for odd values
+ if (minRandValue[c] == maxRandValue[c]) {
+ println("Warning: lambda[" + c + "] has " + "minRandValue = maxRandValue = "
+ + minRandValue[c] + ".", 1);
+ }
+ } // if (!isOptimizable[c])
+
+ /*
+ * precision[c] = inFile_init.nextDouble(); if (precision[c] < 0) { println("precision[" + c +
+ * "]=" + precision[c] + " < 0! Must be non-negative."); System.exit(21); }
+ */
+
+ }
+
+ // set normalizationOptions[]
+ String origLine = "";
+ while (origLine != null && origLine.length() == 0) {
+ origLine = inFile_init.nextLine();
+ }
+
+ // How should a lambda[] vector be normalized (before decoding)?
+ // nO[0] = 0: no normalization
+ // nO[0] = 1: scale so that parameter nO[2] has absolute value nO[1]
+ // nO[0] = 2: scale so that the maximum absolute value is nO[1]
+ // nO[0] = 3: scale so that the minimum absolute value is nO[1]
+ // nO[0] = 4: scale so that the L-nO[1] norm equals nO[2]
+
+ // normalization = none
+ // normalization = absval 1 lm
+ // normalization = maxabsval 1
+ // normalization = minabsval 1
+ // normalization = LNorm 2 1
+
+ dummy = (origLine.substring(origLine.indexOf("=") + 1)).trim();
+ String[] dummyA = dummy.split("\\s+");
+
+ if (dummyA[0].equals("none")) {
+ normalizationOptions[0] = 0;
+ } else if (dummyA[0].equals("absval")) {
+ normalizationOptions[0] = 1;
+ normalizationOptions[1] = Double.parseDouble(dummyA[1]);
+ String pName = dummyA[2];
+ for (int i = 3; i < dummyA.length; ++i) { // in case parameter name has multiple words
+ pName = pName + " " + dummyA[i];
+ }
+ normalizationOptions[2] = Vocabulary.id(pName);
+
+ if (normalizationOptions[1] <= 0) {
+ throw new RuntimeException("Value for the absval normalization method must be positive.");
+ }
+ if (normalizationOptions[2] == 0) {
+ throw new RuntimeException("Unrecognized feature name " + normalizationOptions[2]
+ + " for absval normalization method.");
+ }
+ } else if (dummyA[0].equals("maxabsval")) {
+ normalizationOptions[0] = 2;
+ normalizationOptions[1] = Double.parseDouble(dummyA[1]);
+ if (normalizationOptions[1] <= 0) {
+ throw new RuntimeException("Value for the maxabsval normalization method must be positive.");
+ }
+ } else if (dummyA[0].equals("minabsval")) {
+ normalizationOptions[0] = 3;
+ normalizationOptions[1] = Double.parseDouble(dummyA[1]);
+ if (normalizationOptions[1] <= 0) {
+ throw new RuntimeException("Value for the minabsval normalization method must be positive.");
+ }
+ } else if (dummyA[0].equals("LNorm")) {
+ normalizationOptions[0] = 4;
+ normalizationOptions[1] = Double.parseDouble(dummyA[1]);
+ normalizationOptions[2] = Double.parseDouble(dummyA[2]);
+ if (normalizationOptions[1] <= 0 || normalizationOptions[2] <= 0) {
+ throw new RuntimeException("Both values for the LNorm normalization method must be"
+ + " positive.");
+ }
+ } else {
+ throw new RuntimeException("Unrecognized normalization method " + dummyA[0] + "; "
+ + "must be one of none, absval, maxabsval, and LNorm.");
+ } // if (dummyA[0])
+
+ inFile_init.close();
+ } // processParamFile()
+
+ private void processDocInfo() {
+ // sets numDocuments and docOfSentence[]
+ docOfSentence = new int[numSentences];
+
+ if (docInfoFileName == null) {
+ for (int i = 0; i < numSentences; ++i)
+ docOfSentence[i] = 0;
+ numDocuments = 1;
+ } else {
+
+ try {
+
+ // 4 possible formats:
+ // 1) List of numbers, one per document, indicating # sentences in each document.
+ // 2) List of "docName size" pairs, one per document, indicating name of document and #
+ // sentences.
+ // 3) List of docName's, one per sentence, indicating which doument each sentence belongs
+ // to.
+ // 4) List of docName_number's, one per sentence, indicating which doument each sentence
+ // belongs to,
+ // and its order in that document. (can also use '-' instead of '_')
+
+ int docInfoSize = countNonEmptyLines(docInfoFileName);
+
+ if (docInfoSize < numSentences) { // format #1 or #2
+ numDocuments = docInfoSize;
+ int i = 0;
+
+ BufferedReader inFile = new BufferedReader(new FileReader(docInfoFileName));
+ String line = inFile.readLine();
+ boolean format1 = (!(line.contains(" ")));
+
+ for (int doc = 0; doc < numDocuments; ++doc) {
+
+ if (doc != 0)
+ line = inFile.readLine();
+
+ int docSize = 0;
+ if (format1) {
+ docSize = Integer.parseInt(line);
+ } else {
+ docSize = Integer.parseInt(line.split("\\s+")[1]);
+ }
+
+ for (int i2 = 1; i2 <= docSize; ++i2) {
+ docOfSentence[i] = doc;
+ ++i;
+ }
+
+ }
+
+ // now i == numSentences
+
+ inFile.close();
+
+ } else if (docInfoSize == numSentences) { // format #3 or #4
+
+ boolean format3 = false;
+
+ HashSet<String> seenStrings = new HashSet<String>();
+ BufferedReader inFile = new BufferedReader(new FileReader(docInfoFileName));
+ for (int i = 0; i < numSentences; ++i) {
+ // set format3 = true if a duplicate is found
+ String line = inFile.readLine();
+ if (seenStrings.contains(line))
+ format3 = true;
+ seenStrings.add(line);
+ }
+
+ inFile.close();
+
+ HashSet<String> seenDocNames = new HashSet<String>();
+ HashMap<String, Integer> docOrder = new HashMap<String, Integer>();
+ // maps a document name to the order (0-indexed) in which it was seen
+
+ inFile = new BufferedReader(new FileReader(docInfoFileName));
+ for (int i = 0; i < numSentences; ++i) {
+ String line = inFile.readLine();
+
+ String docName = "";
+ if (format3) {
+ docName = line;
+ } else {
+ int sep_i = Math.max(line.lastIndexOf('_'), line.lastIndexOf('-'));
+ docName = line.substring(0, sep_i);
+ }
+
+ if (!seenDocNames.contains(docName)) {
+ seenDocNames.add(docName);
+ docOrder.put(docName, seenDocNames.size() - 1);
+ }
+
+ int docOrder_i = docOrder.get(docName);
+
+ docOfSentence[i] = docOrder_i;
+
+ }
+
+ inFile.close();
+
+ numDocuments = seenDocNames.size();
+
+ } else { // badly formatted
+
+ }
+
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ }
+
+ private boolean copyFile(String origFileName, String newFileName) {
+ try {
+ File inputFile = new File(origFileName);
+ File outputFile = new File(newFileName);
+
+ InputStream in = new FileInputStream(inputFile);
+ OutputStream out = new FileOutputStream(outputFile);
+
+ byte[] buffer = new byte[1024];
+ int len;
+ while ((len = in.read(buffer)) > 0) {
+ out.write(buffer, 0, len);
+ }
+ in.close();
+ out.close();
+
+ /*
+ * InputStream inStream = new FileInputStream(new File(origFileName)); BufferedReader inFile =
+ * new BufferedReader(new InputStreamReader(inStream, "utf8"));
+ *
+ * FileOutputStream outStream = new FileOutputStream(newFileName, false); OutputStreamWriter
+ * outStreamWriter = new OutputStreamWriter(outStream, "utf8"); BufferedWriter outFile = new
+ * BufferedWriter(outStreamWriter);
+ *
+ * String line; while(inFile.ready()) { line = inFile.readLine(); writeLine(line, outFile); }
+ *
+ * inFile.close(); outFile.close();
+ */
+ return true;
+ } catch (IOException e) {
+ LOG.error(e.getMessage(), e);
+ return false;
+ }
+ }
+
+ private void renameFile(String origFileName, String newFileName) {
+ if (fileExists(origFileName)) {
+ deleteFile(newFileName);
+ File oldFile = new File(origFileName);
+ File newFile = new File(newFileName);
+ if (!oldFile.renameTo(newFile)) {
+ println("Warning: attempt to rename " + origFileName + " to " + newFileName
+ + " was unsuccessful!", 1);
+ }
+ } else {
+ println("Warning: file " + origFileName + " does not exist! (in MIRACore.renameFile)", 1);
+ }
+ }
+
+ private void deleteFile(String fileName) {
+ if (fileExists(fileName)) {
+ File fd = new File(fileName);
+ if (!fd.delete()) {
+ println("Warning: attempt to delete " + fileName + " was unsuccessful!", 1);
+ }
+ }
+ }
+
+ private void writeLine(String line, BufferedWriter writer) throws IOException {
+ writer.write(line, 0, line.length());
+ writer.newLine();
+ writer.flush();
+ }
+
+ // need to re-write to handle different forms of lambda
+ public void finish() {
+ if (myDecoder != null) {
+ myDecoder.cleanUp();
+ }
+
+ // create config file with final values
+ createConfigFile(lambda, decoderConfigFileName + ".MIRA.final", decoderConfigFileName
+ + ".MIRA.orig");
+
+ // delete current decoder config file and decoder output
+ deleteFile(decoderConfigFileName);
+ deleteFile(decoderOutFileName);
+
+ // restore original name for config file (name was changed
+ // in initialize() so it doesn't get overwritten)
+ renameFile(decoderConfigFileName + ".MIRA.orig", decoderConfigFileName);
+
+ if (finalLambdaFileName != null) {
+ try {
+ PrintWriter outFile_lambdas = new PrintWriter(finalLambdaFileName);
+ for (int c = 1; c <= numParams; ++c) {
+ outFile_lambdas.println(Vocabulary.word(c) + " ||| " + lambda.get(c).doubleValue());
+ }
+ outFile_lambdas.close();
+
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ }
+
+ private String[] cfgFileToArgsArray(String fileName) {
+ checkFile(fileName);
+
+ Vector<String> argsVector = new Vector<String>();
+
+ BufferedReader inFile = null;
+ try {
+ inFile = new BufferedReader(new FileReader(fileName));
+ String line, origLine;
+ do {
+ line = inFile.readLine();
+ origLine = line; // for error reporting purposes
+
+ if (line != null && line.length() > 0 && line.charAt(0) != '#') {
+
+ if (line.indexOf("#") != -1) { // discard comment
+ line = line.substring(0, line.indexOf("#"));
+ }
+
+ line = line.trim();
+
+ // now line should look like "-xxx XXX"
+
+ /*
+ * OBSOLETE MODIFICATION //SPECIAL HANDLING FOR MIRA CLASSIFIER PARAMETERS String[] paramA
+ * = line.split("\\s+");
+ *
+ * if( paramA[0].equals("-classifierParams") ) { String classifierParam = ""; for(int p=1;
+ * p<=paramA.length-1; p++) classifierParam += paramA[p]+" ";
+ *
+ * if(paramA.length>=2) { String[] tmpParamA = new String[2]; tmpParamA[0] = paramA[0];
+ * tmpParamA[1] = classifierParam; paramA = tmpParamA; } else {
+ * println("Malformed line in config file:"); println(origLine); System.exit(70); } }//END
+ * MODIFICATION
+ */
+
+ // cmu modification(from meteor for zmert)
+ // Parse args
+ ArrayList<String> argList = new ArrayList<String>();
+ StringBuilder arg = new StringBuilder();
+ boolean quoted = false;
+ for (int i = 0; i < line.length(); i++) {
+ if (Character.isWhitespace(line.charAt(i))) {
+ if (quoted)
+ arg.append(line.charAt(i));
+ else if (arg.length() > 0) {
+ argList.add(arg.toString());
+ arg = new StringBuilder();
+ }
+ } else if (line.charAt(i) == '\'') {
+ if (quoted) {
+ argList.add(arg.toString());
+ arg = new StringBuilder();
+ }
+ quoted = !quoted;
+ } else
+ arg.append(line.charAt(i));
+ }
+ if (arg.length() > 0)
+ argList.add(arg.toString());
+ // Create paramA
+ String[] paramA = new String[argList.size()];
+ for (int i = 0; i < paramA.length; paramA[i] = argList.get(i++))
+ ;
+ // END CMU MODIFICATION
+
+ if (paramA.length == 2 && paramA[0].charAt(0) == '-') {
+ argsVector.add(paramA[0]);
+ argsVector.add(paramA[1]);
+ } else if (paramA.length > 2 && (paramA[0].equals("-m") || paramA[0].equals("-docSet"))) {
+ // -m (metricName), -docSet are allowed to have extra optinos
+ for (int opt = 0; opt < paramA.length; ++opt) {
+ argsVector.add(paramA[opt]);
+ }
+ } else {
+ throw new RuntimeException("Malformed line in config file:" + origLine);
+ }
+
+ }
+ } while (line != null);
+
+ inFile.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ String[] argsArray = new String[argsVector.size()];
+
+ for (int i = 0; i < argsVector.size(); ++i) {
+ argsArray[i] = argsVector.elementAt(i);
+ }
+
+ return argsArray;
+ }
+
+ private void processArgsArray(String[] args) {
+ processArgsArray(args, true);
+ }
+
+ private void processArgsArray(String[] args, boolean firstTime) {
+ /* set default values */
+ // Relevant files
+ dirPrefix = null;
+ sourceFileName = null;
+ refFileName = "reference.txt";
+ refsPerSen = 1;
+ textNormMethod = 1;
+ paramsFileName = "params.txt";
+ docInfoFileName = null;
+ finalLambdaFileName = null;
+ // MERT specs
+ metricName = "BLEU";
+ metricName_display = metricName;
+ metricOptions = new String[2];
+ metricOptions[0] = "4";
+ metricOptions[1] = "closest";
+ docSubsetInfo = new int[7];
+ docSubsetInfo[0] = 0;
+ maxMERTIterations = 20;
+ prevMERTIterations = 20;
+ minMERTIterations = 5;
+ stopMinIts = 3;
+ stopSigValue = -1;
+ //
+ // /* possibly other early stopping criteria here */
+ //
+ numOptThreads = 1;
+ saveInterFiles = 3;
+ compressFiles = 0;
+ oneModificationPerIteration = false;
+ randInit = false;
+ seed = System.currentTimeMillis();
+ // useDisk = 2;
+ // Decoder specs
+ decoderCommandFileName = null;
+ passIterationToDecoder = false;
+ decoderOutFileName = "output.nbest";
+ validDecoderExitValue = 0;
+ decoderConfigFileName = "dec_cfg.txt";
+ sizeOfNBest = 100;
+ fakeFileNameTemplate = null;
+ fakeFileNamePrefix = null;
+ fakeFileNameSuffix = null;
+ // Output specs
+ verbosity = 1;
+ decVerbosity = 0;
+
+ int i = 0;
+
+ while (i < args.length) {
+ String option = args[i];
+ // Relevant files
+ if (option.equals("-dir")) {
+ dirPrefix = args[i + 1];
+ } else if (option.equals("-s")) {
+ sourceFileName = args[i + 1];
+ } else if (option.equals("-r")) {
+ refFileName = args[i + 1];
+ } else if (option.equals("-rps")) {
+ refsPerSen = Integer.parseInt(args[i + 1]);
+ if (refsPerSen < 1) {
+ throw new RuntimeException("refsPerSen must be positive.");
+ }
+ } else if (option.equals("-txtNrm")) {
+ textNormMethod = Integer.parseInt(args[i + 1]);
+ if (textNormMethod < 0 || textNormMethod > 4) {
+ throw new RuntimeException("textNormMethod should be between 0 and 4");
+ }
+ } else if (option.equals("-p")) {
+ paramsFileName = args[i + 1];
+ } else if (option.equals("-docInfo")) {
+ docInfoFileName = args[i + 1];
+ } else if (option.equals("-fin")) {
+ finalLambdaFileName = args[i + 1];
+ // MERT specs
+ } else if (option.equals("-m")) {
+ metricName = args[i + 1];
+ metricName_display = metricName;
+ if (EvaluationMetric.knownMetricName(metricName)) {
+ int optionCount = EvaluationMetric.metricOptionCount(metricName);
+ metricOptions = new String[optionCount];
+ for (int opt = 0; opt < optionCount; ++opt) {
+ metricOptions[opt] = args[i + opt + 2];
+ }
+ i += optionCount;
+ } else {
+ throw new RuntimeException("Unknown metric name " + metricName + ".");
+ }
+ } else if (option.equals("-docSet")) {
+ String method = args[i + 1];
+
+ if (method.equals("all")) {
+ docSubsetInfo[0] = 0;
+ i += 0;
+ } else if (method.equals("bottom")) {
+ String a = args[i + 2];
+ if (a.endsWith("d")) {
+ docSubsetInfo[0] = 1;
+ a = a.substring(0, a.indexOf("d"));
+ } else {
+ docSubsetInfo[0] = 2;
+ a = a.substring(0, a.indexOf("%"));
+ }
+ docSubsetInfo[5] = Integer.parseInt(a);
+ i += 1;
+ } else if (method.equals("top")) {
+ String a = args[i + 2];
+ if (a.endsWith("d")) {
+ docSubsetInfo[0] = 3;
+ a = a.substring(0, a.indexOf("d"));
+ } else {
+ docSubsetInfo[0] = 4;
+ a = a.substring(0, a.indexOf("%"));
+ }
+ docSubsetInfo[5] = Integer.parseInt(a);
+ i += 1;
+ } else if (method.equals("window")) {
+ String a1 = args[i + 2];
+ a1 = a1.substring(0, a1.indexOf("d")); // size of window
+ String a2 = args[i + 4];
+ if (a2.indexOf("p") > 0) {
+ docSubsetInfo[0] = 5;
+ a2 = a2.substring(0, a2.indexOf("p"));
+ } else {
+ docSubsetInfo[0] = 6;
+ a2 = a2.substring(0, a2.indexOf("r"));
+ }
+ docSubsetInfo[5] = Integer.parseInt(a1);
+ docSubsetInfo[6] = Integer.parseInt(a2);
+ i += 3;
+ } else {
+ throw new RuntimeException("Unknown docSet method " + method + ".");
+ }
+ } else if (option.equals("-maxIt")) {
+ maxMERTIterations = Integer.parseInt(args[i + 1]);
+ if (maxMERTIterations < 1) {
+ throw new RuntimeException("maxIt must be positive.");
+ }
+ } else if (option.equals("-minIt")) {
+ minMERTIterations = Integer.parseInt(args[i + 1]);
+ if (minMERTIterations < 1) {
+ throw new RuntimeException("minIt must be positive.");
+ }
+ } else if (option.equals("-prevIt")) {
+ prevMERTIterations = Integer.parseInt(args[i + 1]);
+ if (prevMERTIterations < 0) {
+ throw new RuntimeException("prevIt must be non-negative.");
+ }
+ } else if (option.equals("-stopIt")) {
+ stopMinIts = Integer.parseInt(args[i + 1]);
+ if (stopMinIts < 1) {
+ throw new RuntimeException("stopIts must be positive.");
+ }
+ } else if (option.equals("-stopSig")) {
+ stopSigValue = Double.parseDouble(args[i + 1]);
+ }
+ //
+ // /* possibly other early stopping criteria here */
+ //
+ else if (option.equals("-thrCnt")) {
+ numOptThreads = Integer.parseInt(args[i + 1]);
+ if (numOptThreads < 1) {
+ throw new RuntimeException("threadCount must be positive.");
+ }
+ } else if (option.equals("-save")) {
+ saveInterFiles = Integer.parseInt(args[i + 1]);
+ if (saveInterFiles < 0 || saveInterFiles > 3) {
+ throw new RuntimeException("save should be between 0 and 3");
+ }
+ } else if (option.equals("-compress")) {
+ compressFiles = Integer.parseInt(args[i + 1]);
+ if (compressFiles < 0 || compressFiles > 1) {
+ throw new RuntimeException("compressFiles should be either 0 or 1");
+ }
+ } else if (option.equals("-opi")) {
+ int opi = Integer.parseInt(args[i + 1]);
+ if (opi == 1) {
+ oneModificationPerIteration = true;
+ } else if (opi == 0) {
+ oneModificationPerIteration = false;
+ } else {
+ throw new RuntimeException("oncePerIt must be either 0 or 1.");
+ }
+ } else if (option.equals("-rand")) {
+ int rand = Integer.parseInt(args[i + 1]);
+ if (rand == 1) {
+ randInit = true;
+ } else if (rand == 0) {
+ randInit = false;
+ } else {
+ throw new RuntimeException("randInit must be either 0 or 1.");
+ }
+ } else if (option.equals("-seed")) {
+ if (args[i + 1].equals("time")) {
+ seed = System.currentTimeMillis();
+ } else {
+ seed = Long.parseLong(args[i + 1]);
+ }
+ }
+ /*
+ * else if (option.equals("-ud")) { useDisk = Integer.parseInt(args[i+1]); if (useDisk < 0 ||
+ * useDisk > 2) { println("useDisk should be between 0 and 2"); System.exit(10); } }
+
<TRUNCATED>
[41/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/FeatureFunction.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/FeatureFunction.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/FeatureFunction.java
new file mode 100644
index 0000000..6f231ae
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/FeatureFunction.java
@@ -0,0 +1,364 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.chart_parser.SourcePath;
+import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.hypergraph.HGNode;
+import org.apache.joshua.decoder.segment_file.Sentence;
+
+/**
+ * <p>This class defines Joshua's feature function interface, for both sparse and
+ * dense features. It is immediately inherited by StatelessFF and StatefulFF,
+ * which provide functionality common to stateless and stateful features,
+ * respectively. Any feature implementation should extend those classes, and not
+ * this one. The distinction between stateless and stateful features is somewhat
+ * narrow: all features have the opportunity to return an instance of a
+ * {@link DPState} object, and stateless ones just return null.</p>
+ *
+ * <p>Features in Joshua work like templates. Each feature function defines any
+ * number of actual features, which are associated with weights. The task of the
+ * feature function is to compute the features that are fired in different
+ * circumstances and then return the inner product of those features with the
+ * weight vector. Feature functions can also produce estimates of their future
+ * cost (via {@link org.apache.joshua.decoder.ff.FeatureFunction#estimateCost(Rule, Sentence)});
+ * these values are not used in computing the
+ * score, but are only used for sorting rules during cube pruning. The
+ * individual features produced by each template should have globally unique
+ * names; a good convention is to prefix each feature with the name of the
+ * template that produced it.</p>
+ *
+ * <p>Joshua does not retain individual feature values while decoding, since this
+ * requires keeping a sparse feature vector along every hyperedge, which can be
+ * expensive. Instead, it computes only the weighted cost of each edge. If the
+ * individual feature values are requested, the feature functions are replayed
+ * in post-processing, say during k-best list extraction. This is implemented in
+ * a generic way by passing an {@link Accumulator} object to the compute()
+ * function. During decoding, the accumulator simply sums weighted features in a
+ * scalar. During k-best extraction, when individual feature values are needed,
+ * a {@link FeatureAccumulator} is used to retain the individual values.</p>
+ *
+ * @author Matt Post post@cs.jhu.edu
+ * @author Juri Ganitkevich juri@cs.jhu.edu
+ */
+public abstract class FeatureFunction {
+
+ /*
+ * The name of the feature function; this generally matches the weight name on
+ * the config file. This can also be used as a prefix for feature / weight
+ * names, for templates that define multiple features.
+ */
+ protected String name = null;
+
+ /*
+ * The list of features each function can contribute, along with the dense feature IDs.
+ */
+ protected String[] denseFeatureNames = null;
+ protected int[] denseFeatureIDs = null;
+
+ /*
+ * The first dense feature index
+ */
+ protected int denseFeatureIndex = -1;
+
+ // The list of arguments passed to the feature, and the hash for the parsed args
+ protected String[] args;
+ protected HashMap<String, String> parsedArgs = null;
+
+ /*
+ * The global weight vector used by the decoder, passed it when the feature is
+ * instantiated
+ */
+ protected FeatureVector weights;
+
+ /* The config */
+ protected JoshuaConfiguration config;
+
+ public String getName() {
+ return name;
+ }
+
+ // Whether the feature has state.
+ public abstract boolean isStateful();
+
+ public FeatureFunction(FeatureVector weights, String name, String[] args, JoshuaConfiguration config) {
+ this.weights = weights;
+ this.name = name;
+ this.args = args;
+ this.config = config;
+
+ this.parsedArgs = FeatureFunction.parseArgs(args);
+ }
+
+ /**
+ * Any feature function can use this to report dense features names to the master code. The
+ * parameter tells the feature function the index of the first available dense feature ID; the feature
+ * function will then use IDs (id..id+names.size()-1).
+ *
+ * @param id the id of the first dense feature id to use
+ * @return a list of dense feature names
+ */
+ public ArrayList<String> reportDenseFeatures(int id) {
+ return new ArrayList<String>();
+ }
+
+ public String logString() {
+ try {
+ return String.format("%s (weight %.3f)", name, weights.getSparse(name));
+ } catch (RuntimeException e) {
+ return name;
+ }
+ }
+
+ /**
+ * This is the main function for defining feature values. The implementor
+ * should compute all the features along the hyperedge, calling
+ * {@link org.apache.joshua.decoder.ff.FeatureFunction.Accumulator#add(String, float)}
+ * for each feature. It then returns the newly-computed dynamic
+ * programming state for this feature (for example, for the
+ * {@link org.apache.joshua.decoder.ff.lm.LanguageModelFF} feature, this returns the new language model
+ * context). For stateless features, this value is null.
+ *
+ * Note that the accumulator accumulates *unweighted* feature values. The
+ * feature vector is multiplied times the weight vector later on.
+ *
+ * @param rule {@link org.apache.joshua.decoder.ff.tm.Rule} to be utilized within computation
+ * @param tailNodes {@link java.util.List} of {@link org.apache.joshua.decoder.hypergraph.HGNode} tail nodes
+ * @param i todo
+ * @param j todo
+ * @param sourcePath information about a path taken through the source {@link org.apache.joshua.lattice.Lattice}
+ * @param sentence {@link org.apache.joshua.lattice.Lattice} input
+ * @param acc {@link org.apache.joshua.decoder.ff.FeatureFunction.Accumulator} object permitting generalization of feature computation
+ * @return the new dynamic programming state (null for stateless features)
+ */
+ public abstract DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j,
+ SourcePath sourcePath, Sentence sentence, Accumulator acc);
+
+ /**
+ * Feature functions must overrided this. StatefulFF and StatelessFF provide
+ * reasonable defaults since most features do not fire on the goal node.
+ *
+ * @param tailNode single {@link org.apache.joshua.decoder.hypergraph.HGNode} representing tail node
+ * @param i todo
+ * @param j todo
+ * @param sourcePath information about a path taken through the source {@link org.apache.joshua.lattice.Lattice}
+ * @param sentence {@link org.apache.joshua.lattice.Lattice} input
+ * @param acc {@link org.apache.joshua.decoder.ff.FeatureFunction.Accumulator} object permitting generalization of feature computation
+ * @return the DPState (null if none)
+ */
+ public abstract DPState computeFinal(HGNode tailNode, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc);
+
+ /**
+ * This is a convenience function for retrieving the features fired when
+ * applying a rule, provided for backward compatibility.
+ *
+ * Returns the *unweighted* cost of the features delta computed at this
+ * position. Note that this is a feature delta, so existing feature costs of
+ * the tail nodes should not be incorporated, and it is very important not to
+ * incorporate the feature weights. This function is used in the kbest
+ * extraction code but could also be used in computing the cost.
+ *
+ * @param rule {@link org.apache.joshua.decoder.ff.tm.Rule} to be utilized within computation
+ * @param tailNodes {@link java.util.List} of {@link org.apache.joshua.decoder.hypergraph.HGNode} tail nodes
+ * @param i todo
+ * @param j todo
+ * @param sourcePath information about a path taken through the source {@link org.apache.joshua.lattice.Lattice}
+ * @param sentence {@link org.apache.joshua.lattice.Lattice} input
+ * @return an *unweighted* feature delta
+ */
+ public final FeatureVector computeFeatures(Rule rule, List<HGNode> tailNodes, int i, int j,
+ SourcePath sourcePath, Sentence sentence) {
+
+ FeatureAccumulator features = new FeatureAccumulator();
+ compute(rule, tailNodes, i, j, sourcePath, sentence, features);
+ return features.getFeatures();
+ }
+
+ /**
+ * This function is called for the final transition. For example, the
+ * LanguageModel feature function treats the last rule specially. It needs to
+ * return the *weighted* cost of applying the feature. Provided for backward
+ * compatibility.
+ *
+ * @param tailNode single {@link org.apache.joshua.decoder.hypergraph.HGNode} representing tail node
+ * @param i todo
+ * @param j todo
+ * @param sourcePath information about a path taken through the source {@link org.apache.joshua.lattice.Lattice}
+ * @param sentence {@link org.apache.joshua.lattice.Lattice} input
+ * @return a *weighted* feature cost
+ */
+ public final float computeFinalCost(HGNode tailNode, int i, int j, SourcePath sourcePath,
+ Sentence sentence) {
+
+ ScoreAccumulator score = new ScoreAccumulator();
+ computeFinal(tailNode, i, j, sourcePath, sentence, score);
+ return score.getScore();
+ }
+
+ /**
+ * Returns the *unweighted* feature delta for the final transition (e.g., for
+ * the language model feature function). Provided for backward compatibility.
+ *
+ * @param tailNode single {@link org.apache.joshua.decoder.hypergraph.HGNode} representing tail node
+ * @param i todo
+ * @param j todo
+ * @param sourcePath information about a path taken through the source {@link org.apache.joshua.lattice.Lattice}
+ * @param sentence {@link org.apache.joshua.lattice.Lattice} input
+ * @return an *weighted* feature vector
+ */
+ public final FeatureVector computeFinalFeatures(HGNode tailNode, int i, int j,
+ SourcePath sourcePath, Sentence sentence) {
+
+ FeatureAccumulator features = new FeatureAccumulator();
+ computeFinal(tailNode, i, j, sourcePath, sentence, features);
+ return features.getFeatures();
+ }
+
+ /**
+ * This function is called when sorting rules for cube pruning. It must return
+ * the *weighted* estimated cost of applying a feature. This need not be the
+ * actual cost of applying the rule in context. Basically, it's the inner
+ * product of the weight vector and all features found in the grammar rule,
+ * though some features (like LanguageModelFF) can also compute some of their
+ * values. This is just an estimate of the cost, which helps do better
+ * sorting. Later, the real cost of this feature function is called via
+ * compute();
+ *
+ * @param rule {@link org.apache.joshua.decoder.ff.tm.Rule} to be utilized within computation
+ * @param sentence {@link org.apache.joshua.lattice.Lattice} input
+ * @return the *weighted* cost of applying the feature.
+ */
+ public abstract float estimateCost(Rule rule, Sentence sentence);
+
+ /**
+ * This feature is called to produce a *weighted estimate* of the future cost
+ * of applying this feature. This value is not incorporated into the model
+ * score but is used in pruning decisions. Stateless features return 0.0f by
+ * default, but Stateful features might want to override this.
+ *
+ * @param rule {@link org.apache.joshua.decoder.ff.tm.Rule} to be utilized within computation
+ * @param state todo
+ * @param sentence {@link org.apache.joshua.lattice.Lattice} input
+ * @return the *weighted* future cost estimate of applying this rule in
+ * context.
+ */
+ public abstract float estimateFutureCost(Rule rule, DPState state, Sentence sentence);
+
+ /**
+ * Parses the arguments passed to a feature function in the Joshua config file TODO: Replace this
+ * with a proper CLI library at some point Expects key value pairs in the form : -argname value
+ * Any key without a value is added with an empty string as value Multiple values for the same key
+ * are not parsed. The first one is used.
+ *
+ * @param args A string with the raw arguments and their names
+ * @return A hash with the keys and the values of the string
+ */
+ public static HashMap<String, String> parseArgs(String[] args) {
+ HashMap<String, String> parsedArgs = new HashMap<String, String>();
+ boolean lookingForValue = false;
+ String currentKey = "";
+ for (int i = 0; i < args.length; i++) {
+
+ Pattern argKeyPattern = Pattern.compile("^-[a-zA-Z]\\S+");
+ Matcher argKey = argKeyPattern.matcher(args[i]);
+ if (argKey.find()) {
+ // This is a key
+ // First check to see if there is a key that is waiting to be written
+ if (lookingForValue) {
+ // This is a key with no specified value
+ parsedArgs.put(currentKey, "");
+ }
+ // Now store the new key and look for its value
+ currentKey = args[i].substring(1);
+ lookingForValue = true;
+ } else {
+ // This is a value
+ if (lookingForValue) {
+ parsedArgs.put(currentKey, args[i]);
+ lookingForValue = false;
+ }
+ }
+ }
+ return parsedArgs;
+ }
+
+ /**
+ * Accumulator objects allow us to generalize feature computation.
+ * ScoreAccumulator takes (feature,value) pairs and simple stores the weighted
+ * sum (for decoding). FeatureAccumulator records the named feature values
+ * (for k-best extraction).
+ */
+ public interface Accumulator {
+ public void add(String name, float value);
+ public void add(int id, float value);
+ }
+
+ public class ScoreAccumulator implements Accumulator {
+ private float score;
+
+ public ScoreAccumulator() {
+ this.score = 0.0f;
+ }
+
+ @Override
+ public void add(String name, float value) {
+ score += value * weights.getSparse(name);
+ }
+
+ @Override
+ public void add(int id, float value) {
+ score += value * weights.getDense(id);
+ }
+
+ public float getScore() {
+ return score;
+ }
+ }
+
+ public class FeatureAccumulator implements Accumulator {
+ private FeatureVector features;
+
+ public FeatureAccumulator() {
+ this.features = new FeatureVector();
+ }
+
+ @Override
+ public void add(String name, float value) {
+ features.increment(name, value);
+ }
+
+ @Override
+ public void add(int id, float value) {
+ features.increment(id, value);
+ }
+
+ public FeatureVector getFeatures() {
+ return features;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/FeatureVector.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/FeatureVector.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/FeatureVector.java
new file mode 100644
index 0000000..1b39c78
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/FeatureVector.java
@@ -0,0 +1,385 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * An implementation of a sparse feature vector, using for representing both weights and feature
+ * values.
+ *
+ * This class is used to hold both the decoder weights and the feature values accumulated across
+ * each edge. When features are read in upon decoder startup, they all start out as sparse features
+ * and are stored in the hash table. After the feature functions have been loaded, the decoder
+ * queries each of them for their sparse features via {@link registerDenseFeatures}. Those features
+ * returned by each decoder are then *removed* from the sparse feature hash and placed in the dense
+ * feature array. Therefore, when a feature registers a dense feature, it should take care to
+ * query either {@link org.apache.joshua.decoder.ff.FeatureVector#getDense(int)} or
+ * {@link org.apache.joshua.decoder.ff.FeatureVector#getSparse(String)} when asking for the feature
+ * values later on.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ */
+
+public class FeatureVector {
+ /*
+ * A list of the dense feature names. Increased via calls to registerDenseFeatures()
+ */
+ public static ArrayList<String> DENSE_FEATURE_NAMES = new ArrayList<String>();
+
+ /*
+ * The values of each of the dense features, defaulting to 0.
+ */
+ private ArrayList<Float> denseFeatures = null;
+
+ /*
+ * Value of sparse features.
+ */
+ private HashMap<String, Float> sparseFeatures;
+
+ public FeatureVector() {
+ sparseFeatures = new HashMap<String, Float>();
+ denseFeatures = new ArrayList<Float>(DENSE_FEATURE_NAMES.size());
+ for (int i = 0; i < denseFeatures.size(); i++)
+ denseFeatures.set(i, 0.0f);
+ }
+
+ /**
+ * This version of the constructor takes an uninitialized feature with potentially intermingled
+ * labeled and unlabeled feature values, of the format:
+ *
+ * [feature1=]value [feature2=]value
+ *
+ * It produces a Feature Vector where all unlabeled features have been labeled by appending the
+ * unlabeled feature index (starting at 0) to the defaultPrefix value.
+ *
+ * **IMPORTANT** The feature values are inverted, for historical reasons, which leads to a lot
+ * of confusion. They have to be inverted here and when the score is actually computed. They
+ * are inverted here (which is used to build the feature vector representation of a rule's dense
+ * features) and in {@link org.apache.joshua.decoder.ff.tm.Rule#estimateRuleCost(java.util.List)}
+ * , where the rule's precomputable (weighted) score is cached.
+ *
+ * @param featureString, the string of labeled and unlabeled features (probably straight from the
+ * grammar text file)
+ * @param prefix, the prefix to use for unlabeled features (probably "tm_OWNER_")
+ */
+ public FeatureVector(String featureString, String prefix) {
+
+// System.err.println(String.format("FEATURES_OF(%s, %s)", featureString, prefix));
+
+ /*
+ * Read through the features on this rule, adding them to the feature vector. Unlabeled features
+ * are converted to a canonical form.
+ *
+ * Note that it's bad form to mix unlabeled features and the named feature index they are mapped
+ * to, but we are being liberal in what we accept.
+ *
+ * IMPORTANT: Note that, for historical reasons, the sign is reversed on all *dense* scores.
+ * This is the source of *no end* of confusion and should be done away with.
+ */
+ this();
+
+ int denseFeatureIndex = 0;
+
+ if (!featureString.trim().equals("")) {
+ for (String token : featureString.split("\\s+")) {
+ if (token.indexOf('=') == -1) {
+ /*
+ * If we encounter an unlabeled feature, it is the next dense feature
+ */
+ while (denseFeatures.size() <= denseFeatureIndex)
+ denseFeatures.add(0.0f);
+ denseFeatures.set(denseFeatureIndex, -Float.parseFloat(token));
+ denseFeatureIndex++;
+ } else {
+ /*
+ * Labeled features are of two types: if they start with the prefix, they are actually
+ * dense feature in disguise; otherwise, they are proper sparse features.
+ */
+ int splitPoint = token.indexOf('=');
+ if (token.startsWith(prefix)) {
+// System.err.println(String.format(" PREFIX=%s '%s'.substring(%d,%d) = %s", prefix, token, prefix.length(), splitPoint,
+// token.substring(prefix.length(), splitPoint)));
+ int index = Integer.parseInt(token.substring(prefix.length(), splitPoint));
+ while (denseFeatures.size() <= index)
+ denseFeatures.add(0.0f);
+ denseFeatures.set(index, 1.0f * Float.parseFloat(token.substring(splitPoint + 1)));
+ } else {
+ sparseFeatures.put(token.substring(0, splitPoint),
+ Float.parseFloat(token.substring(splitPoint + 1)));
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Register one or more dense features with the global weight vector. This assumes them global
+ * IDs, and then returns the index of the first feature (from which the calling feature function
+ * can infer them all). This *must* be called by every feature function wishing to register
+ * dense features!
+ *
+ * @param featureFunctions {@link java.util.ArrayList} of {@link org.apache.joshua.decoder.ff.FeatureFunction}'s
+ */
+ public void registerDenseFeatures(ArrayList<FeatureFunction> featureFunctions) {
+ for (FeatureFunction feature: featureFunctions) {
+ ArrayList<String> names = feature.reportDenseFeatures(denseFeatures.size());
+ for (String name: names) {
+ DENSE_FEATURE_NAMES.add(name);
+ denseFeatures.add(getSparse(name));
+ sparseFeatures.remove(name);
+ }
+ }
+ }
+
+ public ArrayList<Float> getDenseFeatures() {
+ return denseFeatures;
+ }
+
+ public HashMap<String,Float> getSparseFeatures() {
+ return sparseFeatures;
+ }
+
+ public Set<String> keySet() {
+ return sparseFeatures.keySet();
+ }
+
+ public int size() {
+ return sparseFeatures.size() + denseFeatures.size();
+ }
+
+ public FeatureVector clone() {
+ FeatureVector newOne = new FeatureVector();
+ for (String key : this.sparseFeatures.keySet())
+ newOne.set(key, this.sparseFeatures.get(key));
+ for (int i = 0; i < denseFeatures.size(); i++)
+ newOne.set(i, getDense(i));
+ return newOne;
+ }
+
+ /**
+ * Subtracts the weights in the other feature vector from this one. Note that this is not set
+ * subtraction; keys found in the other FeatureVector but not in this one will be initialized with
+ * a value of 0.0f before subtraction.
+ *
+ * @param other another {@link org.apache.joshua.decoder.ff.FeatureVector} from which to subtract its score
+ */
+ public void subtract(FeatureVector other) {
+ for (int i = 0; i < denseFeatures.size(); i++)
+ denseFeatures.set(i, getDense(i) - other.getDense(i));
+
+ for (String key : other.keySet()) {
+ float oldValue = (sparseFeatures.containsKey(key)) ? sparseFeatures.get(key) : 0.0f;
+ sparseFeatures.put(key, oldValue - other.getSparse(key));
+ }
+ }
+
+ /**
+ * Adds the weights in the other feature vector to this one. This is set union, with values shared
+ * between the two being summed.
+ *
+ * @param other another {@link org.apache.joshua.decoder.ff.FeatureVector} from which to add its score
+ */
+ public void add(FeatureVector other) {
+ while (denseFeatures.size() < other.denseFeatures.size())
+ denseFeatures.add(0.0f);
+
+ for (int i = 0; i < other.denseFeatures.size(); i++)
+ increment(i, other.getDense(i));
+
+ for (String key : other.keySet()) {
+ if (!sparseFeatures.containsKey(key))
+ sparseFeatures.put(key, other.getSparse(key));
+ else
+ sparseFeatures.put(key, sparseFeatures.get(key) + other.getSparse(key));
+ }
+ }
+
+ /**
+ * Return the weight of a feature by name, after checking to determine if it is sparse or dense.
+ *
+ * @param feature String name of some feature
+ * @return the feature's weight
+ */
+ public float getWeight(String feature) {
+ for (int i = 0; i < DENSE_FEATURE_NAMES.size(); i++) {
+ if (DENSE_FEATURE_NAMES.get(i).equals(feature)) {
+ return getDense(i);
+ }
+ }
+ return getSparse(feature);
+ }
+
+ /**
+ * Return the weight of a sparse feature, indexed by its name.
+ *
+ * @param feature String name of some feature
+ * @return the sparse feature's weight, or 0 if not found.
+ */
+ public float getSparse(String feature) {
+ if (sparseFeatures.containsKey(feature))
+ return sparseFeatures.get(feature);
+ return 0.0f;
+ }
+
+ public boolean hasValue(String name) {
+ return sparseFeatures.containsKey(name);
+ }
+
+ /**
+ * Return the weight of a dense feature, indexed by its feature index, or 0.0f, if the feature
+ * is not found. In other words, this is a safe way to query the dense feature vector.
+ *
+ * @param id int representing of some dense feature
+ * @return the dense feature's value, or 0 if not found.
+ */
+ public float getDense(int id) {
+ if (id < denseFeatures.size())
+ return denseFeatures.get(id);
+ return 0.0f;
+ }
+
+ public void increment(String feature, float value) {
+ sparseFeatures.put(feature, getSparse(feature) + value);
+ }
+
+ public void increment(int id, float value) {
+ while (id >= denseFeatures.size())
+ denseFeatures.add(0.0f);
+ denseFeatures.set(id, getDense(id) + value);
+ }
+
+ /**
+ * Set the value of a feature. We need to first determine whether the feature is a dense or
+ * sparse one, then set accordingly.
+ *
+ * @param feature String name of some feature
+ * @param value float value to set to the featue with the associated name
+ */
+ public void set(String feature, float value) {
+ for (int i = 0; i < DENSE_FEATURE_NAMES.size(); i++) {
+ if (DENSE_FEATURE_NAMES.get(i).equals(feature)) {
+ denseFeatures.set(i, value);
+ return;
+ }
+ }
+ // No dense feature was found; assume it's sparse
+ sparseFeatures.put(feature, value);
+ }
+
+ public void set(int id, float value) {
+ while (id >= denseFeatures.size())
+ denseFeatures.add(0.0f);
+ denseFeatures.set(id, value);
+ }
+
+ public Map<String, Float> getMap() {
+ Map<String, Float> allFeatures = new HashMap<>(sparseFeatures.size() + denseFeatures.size());
+ allFeatures.putAll(sparseFeatures);
+ for (int i = 0; i < DENSE_FEATURE_NAMES.size(); i++) {
+ allFeatures.put(DENSE_FEATURE_NAMES.get(i), getDense(i));
+ }
+ return allFeatures;
+ }
+
+ /**
+ * Computes the inner product between this feature vector and another one.
+ *
+ * @param other a {@link org.apache.joshua.decoder.ff.FeatureVector} with which to compute the inner product
+ * @return float value representing the computation
+ */
+ public float innerProduct(FeatureVector other) {
+ float cost = 0.0f;
+ for (int i = 0; i < DENSE_FEATURE_NAMES.size(); i++)
+ cost += getDense(i) * other.getDense(i);
+
+ for (String key : sparseFeatures.keySet())
+ cost += sparseFeatures.get(key) * other.getSparse(key);
+
+ return cost;
+ }
+
+ public void times(float value) {
+ for (String key : sparseFeatures.keySet())
+ sparseFeatures.put(key, sparseFeatures.get(key) * value);
+ }
+
+ /***
+ * Moses distinguishes sparse features as those containing an underscore, so we have to fake it
+ * to be compatible with their tuners.
+ *
+ * @return trimmed Moses output string
+ */
+ public String mosesString() {
+ StringBuilder outputString = new StringBuilder();
+
+ HashSet<String> printed_keys = new HashSet<String>();
+
+ // First print all the dense feature names in order
+ for (int i = 0; i < DENSE_FEATURE_NAMES.size(); i++) {
+ outputString.append(String.format("%s=%.3f ", DENSE_FEATURE_NAMES.get(i).replaceAll("_", "-"), getDense(i)));
+ printed_keys.add(DENSE_FEATURE_NAMES.get(i));
+ }
+
+ // Now print the sparse features
+ ArrayList<String> keys = new ArrayList<String>(sparseFeatures.keySet());
+ Collections.sort(keys);
+ for (String key: keys) {
+ if (! printed_keys.contains(key)) {
+ float value = sparseFeatures.get(key);
+ if (key.equals("OOVPenalty"))
+ // force moses to see it as sparse
+ key = "OOV_Penalty";
+ outputString.append(String.format("%s=%.3f ", key, value));
+ }
+ }
+ return outputString.toString().trim();
+ }
+
+ /***
+ * Outputs a list of feature names. All dense features are printed. Feature names are printed
+ * in the order they were read in.
+ */
+ @Override
+ public String toString() {
+ StringBuilder outputString = new StringBuilder();
+
+ HashSet<String> printed_keys = new HashSet<String>();
+
+ // First print all the dense feature names in order
+ for (int i = 0; i < DENSE_FEATURE_NAMES.size(); i++) {
+ outputString.append(String.format("%s=%.3f ", DENSE_FEATURE_NAMES.get(i), getDense(i)));
+ printed_keys.add(DENSE_FEATURE_NAMES.get(i));
+ }
+
+ // Now print the rest of the features
+ ArrayList<String> keys = new ArrayList<String>(sparseFeatures.keySet());
+ Collections.sort(keys);
+ for (String key: keys)
+ if (! printed_keys.contains(key))
+ outputString.append(String.format("%s=%.3f ", key, sparseFeatures.get(key)));
+
+ return outputString.toString().trim();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/LabelCombinationFF.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/LabelCombinationFF.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/LabelCombinationFF.java
new file mode 100644
index 0000000..bfebaa5
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/LabelCombinationFF.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff;
+
+/***
+ * @author Gideon Wenniger
+ */
+
+import java.util.List;
+
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.chart_parser.SourcePath;
+import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.hypergraph.HGNode;
+import org.apache.joshua.decoder.segment_file.Sentence;
+
+public class LabelCombinationFF extends StatelessFF {
+
+ public LabelCombinationFF(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, "LabelCombination", args, config);
+ }
+
+ public String getLowerCasedFeatureName() {
+ return name.toLowerCase();
+ }
+
+ private final String computeRuleLabelCombinationDescriptor(Rule rule) {
+ StringBuilder result = new StringBuilder(getLowerCasedFeatureName() + "_");
+ result.append(RulePropertiesQuerying.getLHSAsString(rule));
+ // System.out.println("Rule: " + rule);
+ for (String foreignNonterminalString : RulePropertiesQuerying.getRuleSourceNonterminalStrings(rule)) {
+ result.append("_").append(foreignNonterminalString);
+ }
+ return result.toString();
+ }
+
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+ if (rule != null)
+ acc.add(computeRuleLabelCombinationDescriptor(rule), 1);
+
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/LabelSubstitutionFF.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/LabelSubstitutionFF.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/LabelSubstitutionFF.java
new file mode 100644
index 0000000..8735be6
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/LabelSubstitutionFF.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff;
+
+/***
+ * @author Gideon Wenniger
+ */
+
+import java.util.List;
+
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.chart_parser.SourcePath;
+import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.hypergraph.HGNode;
+import org.apache.joshua.decoder.segment_file.Sentence;
+import org.apache.joshua.util.ListUtil;
+
+public class LabelSubstitutionFF extends StatelessFF {
+ private static final String MATCH_SUFFIX = "MATCH";
+ private static final String NO_MATCH_SUFFIX = "NOMATCH";
+
+ public LabelSubstitutionFF(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, "LabelSubstitution", args, config);
+ }
+
+ public String getLowerCasedFeatureName() {
+ return name.toLowerCase();
+ }
+
+ public String getMatchFeatureSuffix(String ruleNonterminal, String substitutionNonterminal) {
+ if (ruleNonterminal.equals(substitutionNonterminal)) {
+ return MATCH_SUFFIX;
+ } else {
+ return NO_MATCH_SUFFIX;
+ }
+ }
+
+ public static String getSubstitutionSuffix(String ruleNonterminal, String substitutionNonterminal) {
+ return substitutionNonterminal + "_substitutes_" + ruleNonterminal;
+ }
+
+ private final String computeLabelMatchingFeature(String ruleNonterminal,
+ String substitutionNonterminal) {
+ String result = getLowerCasedFeatureName() + "_";
+ result += getMatchFeatureSuffix(ruleNonterminal, substitutionNonterminal);
+ return result;
+ }
+
+ private final String computeLabelSubstitutionFeature(String ruleNonterminal,
+ String substitutionNonterminal) {
+ String result = getLowerCasedFeatureName() + "_";
+ result += getSubstitutionSuffix(ruleNonterminal, substitutionNonterminal);
+ return result;
+ }
+
+ private static final String getRuleLabelsDescriptorString(Rule rule) {
+ String result = "";
+ String leftHandSide = RulePropertiesQuerying.getLHSAsString(rule);
+ List<String> ruleSourceNonterminals = RulePropertiesQuerying
+ .getRuleSourceNonterminalStrings(rule);
+ boolean isInverting = rule.isInverting();
+ result += "<LHS>" + leftHandSide + "</LHS>";
+ result += "_<Nont>";
+ result += ListUtil.stringListStringWithoutBracketsCommaSeparated(ruleSourceNonterminals);
+ result += "</Nont>";
+ if(isInverting)
+ {
+ result += "_INV";
+ }
+ else
+ {
+ result += "_MONO";
+ }
+
+ return result;
+ }
+
+ private static final String getSubstitutionsDescriptorString(List<HGNode> tailNodes) {
+ String result = "_<Subst>";
+ List<String> substitutionNonterminals = RulePropertiesQuerying
+ .getSourceNonterminalStrings(tailNodes);
+ result += ListUtil.stringListStringWithoutBracketsCommaSeparated(substitutionNonterminals);
+ result += "</Subst>";
+ return result;
+ }
+
+ public final String getGapLabelsForRuleSubstitutionSuffix(Rule rule, List<HGNode> tailNodes) {
+ String result = getLowerCasedFeatureName() + "_";
+ result += getRuleLabelsDescriptorString(rule);
+ result += getSubstitutionsDescriptorString(tailNodes);
+ return result;
+ }
+
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+ if (rule != null && (tailNodes != null)) {
+
+ List<String> ruleSourceNonterminals = RulePropertiesQuerying
+ .getRuleSourceNonterminalStrings(rule);
+ List<String> substitutionNonterminals = RulePropertiesQuerying
+ .getSourceNonterminalStrings(tailNodes);
+ // Assert.assertEquals(ruleSourceNonterminals.size(), substitutionNonterminals.size());
+ for (int nonterinalIndex = 0; nonterinalIndex < ruleSourceNonterminals.size(); nonterinalIndex++) {
+ String ruleNonterminal = ruleSourceNonterminals.get(nonterinalIndex);
+ String substitutionNonterminal = substitutionNonterminals.get(nonterinalIndex);
+ acc.add(computeLabelMatchingFeature(ruleNonterminal, substitutionNonterminal), 1);
+ acc.add(computeLabelSubstitutionFeature(ruleNonterminal, substitutionNonterminal), 1);
+ }
+ acc.add(getGapLabelsForRuleSubstitutionSuffix(rule, tailNodes), 1);
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/LexicalFeatures.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/LexicalFeatures.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/LexicalFeatures.java
new file mode 100644
index 0000000..58de5f4
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/LexicalFeatures.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff;
+
+import static com.google.common.cache.CacheBuilder.newBuilder;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.chart_parser.SourcePath;
+import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.hypergraph.HGNode;
+import org.apache.joshua.decoder.segment_file.Sentence;
+import org.apache.joshua.util.FormatUtils;
+
+import com.google.common.cache.Cache;
+
+/**
+ * Lexical alignment features denoting alignments, deletions, and insertions.
+ */
+public class LexicalFeatures extends StatelessFF {
+
+ private final boolean useAlignments;
+ private final boolean useDeletions;
+ private final boolean useInsertions;
+
+ private static final String NAME = "LexicalFeatures";
+ // value to fire for features
+ private static final int VALUE = 1;
+ //whether this feature is restricted to a certain grammar/owner
+ private final boolean ownerRestriction;
+ // the grammar/owner this feature is restricted to fire
+ private final int owner;
+ // Strings separating words
+ private static final String SEPARATOR = "~";
+
+ private final Cache<Rule, List<String>> featureCache;
+
+ public LexicalFeatures(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, NAME, args, config);
+
+ ownerRestriction = (parsedArgs.containsKey("owner")) ? true : false;
+ owner = ownerRestriction ? Vocabulary.id(parsedArgs.get("owner")) : 0;
+
+ useAlignments = parsedArgs.containsKey("alignments");
+ useDeletions = parsedArgs.containsKey("deletions");
+ useInsertions = parsedArgs.containsKey("insertions");
+
+ // initialize cache
+ if (parsedArgs.containsKey("cacheSize")) {
+ featureCache = newBuilder().maximumSize(Integer.parseInt(parsedArgs.get("cacheSize"))).build();
+ } else {
+ featureCache = newBuilder().maximumSize(config.cachedRuleSize).build();
+ }
+ }
+
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+
+ if (ownerRestriction && rule.getOwner() != owner) {
+ return null;
+ }
+
+ List<String> featureNames = featureCache.getIfPresent(rule);
+ if (featureNames == null) {
+ featureNames = getFeatures(rule);
+ featureCache.put(rule, featureNames);
+ }
+ for (String feature : featureNames) {
+ acc.add(feature, VALUE);
+ }
+
+ return null;
+ }
+
+ /**
+ * Obtains the feature ids for the given rule.
+ * @param rule
+ * @return String representing the feature name.s
+ */
+ private List<String> getFeatures(final Rule rule) {
+ final List<String> result = new ArrayList<>();
+
+ byte[] alignments = rule.getAlignment();
+ if (alignments == null) {
+ return result;
+ }
+ int[] sourceWords = rule.getFrench();
+ int[] targetWords = rule.getEnglish();
+
+ // sourceAligned & targetAligned indicate whether an index is covered by alignments
+ boolean[] sourceAligned = new boolean[sourceWords.length];
+ boolean[] targetAligned = new boolean[targetWords.length];
+
+ // translations: aligned words
+ for (int i = 0; i < alignments.length; i+=2) {
+ byte sourceIndex = alignments[i];
+ byte targetIndex = alignments[i + 1];
+ sourceAligned[sourceIndex] = true;
+ targetAligned[targetIndex] = true;
+ if (useAlignments) {
+ result.add(
+ "T:" +
+ Vocabulary.word(sourceWords[sourceIndex]) +
+ SEPARATOR +
+ Vocabulary.word(targetWords[targetIndex]));
+ }
+ }
+
+ // deletions: unaligned source words
+ if (useDeletions) {
+ for (int i = 0; i < sourceAligned.length; i++) {
+ if (!sourceAligned[i] && ! FormatUtils.isNonterminal(sourceWords[i])) {
+ result.add("D:" + Vocabulary.word(sourceWords[i]));
+ }
+ }
+ }
+
+ // insertions: unaligned target words
+ if (useInsertions) {
+ for (int i = 0; i < targetAligned.length; i++) {
+ if (useInsertions && !targetAligned[i] && ! FormatUtils.isNonterminal(targetWords[i])) {
+ result.add("I:" + Vocabulary.word(targetWords[i]));
+ }
+ }
+ }
+
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/OOVPenalty.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/OOVPenalty.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/OOVPenalty.java
new file mode 100644
index 0000000..5278172
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/OOVPenalty.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.JoshuaConfiguration.OOVItem;
+import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.hypergraph.HGNode;
+import org.apache.joshua.decoder.segment_file.Sentence;
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.chart_parser.SourcePath;
+
+/**
+ * This feature is fired when an out-of-vocabulary word (with respect to the translation model) is
+ * entered into the chart. OOVs work in the following manner: for each word in the input that is OOV
+ * with respect to the translation model, we create a rule that pushes that word through
+ * untranslated (the suffix "_OOV" can optionally be appended according to the runtime parameter
+ * "mark-oovs") . These rules are all stored in a grammar whose owner is "oov". The OOV feature
+ * function template then fires the "OOVPenalty" feature whenever it is asked to score an OOV rule.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ */
+public class OOVPenalty extends StatelessFF {
+ private final int ownerID;
+
+ /* The default value returned for OOVs. Can be overridden with -oov-list */
+ private final float defaultValue = -100f;
+ private final HashMap<Integer,Float> oovWeights;
+
+ public OOVPenalty(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, "OOVPenalty", args, config);
+
+ ownerID = Vocabulary.id("oov");
+ oovWeights = new HashMap<Integer,Float>();
+
+ if (config.oovList != null) {
+ for (OOVItem item: config.oovList) {
+ oovWeights.put(Vocabulary.id(item.label), item.weight);
+ }
+ }
+ }
+
+ @Override
+ public ArrayList<String> reportDenseFeatures(int index) {
+ denseFeatureIndex = index;
+
+ ArrayList<String> names = new ArrayList<>(1);
+ names.add(name);
+ return names;
+ }
+
+ /**
+ * OOV rules cover exactly one word, and such rules belong to a grammar whose owner is "oov". Each
+ * OOV fires the OOVPenalty feature with a value of 1, so the cost is simply the weight, which was
+ * cached when the feature was created.
+ */
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+
+ if (rule != null && this.ownerID == rule.getOwner()) {
+ acc.add(denseFeatureIndex, getValue(rule.getLHS()));
+ }
+
+ return null;
+ }
+
+ /**
+ * It's important for the OOV feature to contribute to the rule's estimated cost, so that OOV
+ * rules (which are added for all words, not just ones without translation options) get sorted
+ * to the bottom during cube pruning.
+ *
+ * Important! estimateCost returns the *weighted* feature value.
+ */
+ @Override
+ public float estimateCost(Rule rule, Sentence sentence) {
+ if (rule != null && this.ownerID == rule.getOwner())
+ return weights.getDense(denseFeatureIndex) * getValue(rule.getLHS());
+ return 0.0f;
+ }
+
+ private float getValue(int lhs) {
+ return oovWeights.containsKey(lhs) ? oovWeights.get(lhs) : defaultValue;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/PhraseModel.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/PhraseModel.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/PhraseModel.java
new file mode 100644
index 0000000..2324292
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/PhraseModel.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.chart_parser.SourcePath;
+import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+import org.apache.joshua.decoder.ff.tm.Grammar;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.hypergraph.HGNode;
+import org.apache.joshua.decoder.segment_file.Sentence;
+
+/**
+ * This feature handles the list of features that are found with grammar rules in the grammar file.
+ * dense features that may be associated with the rules in a grammar file. The feature names of
+ * these dense rules are a function of the phrase model owner. When the feature is loaded, it
+ * queries the weights for the set of features that are active for this grammar, storing them in an
+ * array.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ * @author Zhifei Li zhifei.work@gmail.com
+ */
+
+public class PhraseModel extends StatelessFF {
+
+ /* The owner of the grammar. */
+ private int ownerID;
+ private String owner;
+
+ private float[] phrase_weights = null;
+
+ public PhraseModel(FeatureVector weights, String[] args, JoshuaConfiguration config, Grammar g) {
+ super(weights, "tm_", args, config);
+
+ String owner = parsedArgs.get("owner");
+ this.name = String.format("tm_%s", owner);
+
+ /*
+ * Determine the number of features by querying the example grammar that was passed in.
+ */
+ phrase_weights = new float[g.getNumDenseFeatures()];
+// System.err.println(String.format("GOT %d FEATURES FOR %s", g.getNumDenseFeatures(), owner));
+ for (int i = 0; i < phrase_weights.length; i++)
+ phrase_weights[i] = weights.getSparse(String.format("tm_%s_%d", owner, i));
+
+ // Store the owner.
+ this.owner = owner;
+ this.ownerID = Vocabulary.id(owner);
+ }
+
+ /**
+ * Just register a single weight, tm_OWNER, and use that to set its precomputed cost
+ */
+ @Override
+ public ArrayList<String> reportDenseFeatures(int index) {
+ denseFeatureIndex = index;
+
+ ArrayList<String> names = new ArrayList<String>();
+ for (int i = 0; i < phrase_weights.length; i++)
+ names.add(String.format("tm_%s_%d", owner, i));
+ return names;
+ }
+
+ /**
+ * Estimates the cost of applying this rule, which is just the score of the precomputable feature
+ * functions.
+ */
+ @Override
+ public float estimateCost(final Rule rule, Sentence sentence) {
+
+ if (rule != null && rule.getOwner() == ownerID) {
+ if (rule.getPrecomputableCost() <= Float.NEGATIVE_INFINITY)
+ rule.setPrecomputableCost(phrase_weights, weights);
+
+ return rule.getPrecomputableCost();
+ }
+
+ return 0.0f;
+ }
+
+ /**
+ * Just chain to computeFeatures(rule), since this feature doesn't use the sourcePath or sentID. *
+ */
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+
+ if (rule != null && rule.getOwner() == ownerID) {
+ /*
+ * Here, we peak at the Accumulator object. If it's asking for scores, then we don't bother to
+ * add each feature, but rather compute the inner product and add *that*. This is totally
+ * cheating; the Accumulator is supposed to be a generic object. But without this cheat
+ */
+ if (rule.getPrecomputableCost() <= Float.NEGATIVE_INFINITY) {
+ // float score = rule.getFeatureVector().innerProduct(weights);
+ rule.setPrecomputableCost(phrase_weights, weights);
+ }
+
+// System.err.println(String.format("RULE = %s / %f", rule.getEnglishWords(), rule.getPrecomputableCost()));
+ for (int k = 0; k < phrase_weights.length; k++) {
+// System.err.println(String.format("k = %d, denseFeatureIndex = %d, owner = %s, ownerID = %d", k, denseFeatureIndex, owner, ownerID));
+ acc.add(k + denseFeatureIndex, rule.getDenseFeature(k));
+ }
+
+ for (String key: rule.getFeatureVector().keySet())
+ acc.add(key, rule.getFeatureVector().getSparse(key));
+ }
+
+ return null;
+ }
+
+ public String toString() {
+ return name + " " + Vocabulary.word(ownerID);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/PhrasePenalty.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/PhrasePenalty.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/PhrasePenalty.java
new file mode 100644
index 0000000..3c38e60
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/PhrasePenalty.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.chart_parser.SourcePath;
+import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.hypergraph.HGNode;
+import org.apache.joshua.decoder.phrase.Hypothesis;
+import org.apache.joshua.decoder.segment_file.Sentence;
+
+/**
+ * This feature just counts rules that are used. You can restrict it with a number of flags:
+ *
+ * -owner OWNER
+ * Only count rules owned by OWNER
+ * -target|-source
+ * Only count the target or source side (plus the LHS)
+ *
+ * TODO: add an option to separately provide a list of rule counts, restrict to counts above a threshold.
+ */
+public class PhrasePenalty extends StatelessFF {
+
+ private int owner = 0;
+ private float value = 1.0f;
+
+ public PhrasePenalty(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, "PhrasePenalty", args, config);
+ if (parsedArgs.containsKey("owner"))
+ this.owner = Vocabulary.id(parsedArgs.get("owner"));
+ else // default
+ this.owner = Vocabulary.id("pt");
+ }
+
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+
+ if (rule != null && rule != Hypothesis.BEGIN_RULE && rule != Hypothesis.END_RULE
+ && (owner == 0 || rule.getOwner() == owner))
+ acc.add(denseFeatureIndex, value);
+
+ return null;
+ }
+
+ @Override
+ public ArrayList<String> reportDenseFeatures(int index) {
+ denseFeatureIndex = index;
+ ArrayList<String> names = new ArrayList<String>();
+ names.add(name);
+ return names;
+ }
+
+ /**
+ * Returns the *weighted* estimate.
+ *
+ */
+ @Override
+ public float estimateCost(Rule rule, Sentence sentence) {
+ if (rule != null && rule != Hypothesis.BEGIN_RULE && rule != Hypothesis.END_RULE
+ && (owner == 0 || rule.getOwner() == owner))
+ return weights.getDense(denseFeatureIndex) * value;
+ return 0.0f;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/RuleCountBin.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/RuleCountBin.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/RuleCountBin.java
new file mode 100644
index 0000000..5ba0c66
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/RuleCountBin.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff;
+
+import java.util.List;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.chart_parser.SourcePath;
+import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.hypergraph.HGNode;
+import org.apache.joshua.decoder.segment_file.Sentence;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/*
+ * This feature computes a bin for the rule and activates a feature for it. It requires access to
+ * the index of the RarityPenalty field, from which the rule count can be computed.
+ */
+public class RuleCountBin extends StatelessFF {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RuleCountBin.class);
+ private int field = -1;
+
+ public RuleCountBin(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, "RuleCountBin", args, config);
+
+ field = Integer.parseInt(parsedArgs.get("field"));
+ }
+
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+
+ if (rule.getOwner() != Vocabulary.id("pt"))
+ return null;
+
+ float rarityPenalty = -rule.getFeatureVector().getSparse(String.format("tm_pt_%d", field));
+ int count = (int) (1.0 - Math.log(rarityPenalty));
+
+ String feature = "RuleCountBin_inf";
+
+ int[] bins = { 1, 2, 4, 8, 16, 32, 64, 128, 1000, 10000 };
+ for (int k : bins) {
+ if (count <= k) {
+ feature = String.format("RuleCountBin_%d", k);
+ break;
+ }
+ }
+
+ LOG.debug("RuleCountBin({}) = {} ==> {}", rarityPenalty, count, feature);
+
+ acc.add(feature, 1.0f);
+
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/RuleFF.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/RuleFF.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/RuleFF.java
new file mode 100644
index 0000000..909e481
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/RuleFF.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff;
+
+import static com.google.common.cache.CacheBuilder.newBuilder;
+
+import java.util.List;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.chart_parser.SourcePath;
+import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.hypergraph.HGNode;
+import org.apache.joshua.decoder.segment_file.Sentence;
+
+import com.google.common.cache.Cache;
+
+/**
+ * This feature fires for rule ids.
+ * Firing can be restricted to rules from a certain owner, and rule ids
+ * can be generated from source side and/or target side.
+ */
+public class RuleFF extends StatelessFF {
+
+ private enum Sides { SOURCE, TARGET, BOTH };
+
+ private static final String NAME = "RuleFF";
+ // value to fire for features
+ private static final int VALUE = 1;
+ // whether this feature is restricted to a certain grammar/owner
+ private final boolean ownerRestriction;
+ // the grammar/owner this feature is restricted to fire
+ private final int owner;
+ // what part of the rule should be extracted;
+ private final Sides sides;
+ // Strings separating words and rule sides
+ private static final String SEPARATOR = "~";
+ private static final String SIDES_SEPARATOR = "->";
+
+ private final Cache<Rule, String> featureCache;
+
+ public RuleFF(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, NAME, args, config);
+
+ ownerRestriction = (parsedArgs.containsKey("owner")) ? true : false;
+ owner = ownerRestriction ? Vocabulary.id(parsedArgs.get("owner")) : 0;
+
+ if (parsedArgs.containsKey("sides")) {
+ final String sideValue = parsedArgs.get("sides");
+ if (sideValue.equalsIgnoreCase("source")) {
+ sides = Sides.SOURCE;
+ } else if (sideValue.equalsIgnoreCase("target")) {
+ sides = Sides.TARGET;
+ } else if (sideValue.equalsIgnoreCase("both")){
+ sides = Sides.BOTH;
+ } else {
+ throw new RuntimeException("Unknown side value.");
+ }
+ } else {
+ sides = Sides.BOTH;
+ }
+
+ // initialize cache
+ if (parsedArgs.containsKey("cacheSize")) {
+ featureCache = newBuilder().maximumSize(Integer.parseInt(parsedArgs.get("cacheSize"))).build();
+ } else {
+ featureCache = newBuilder().maximumSize(config.cachedRuleSize).build();
+ }
+ }
+
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+
+ if (ownerRestriction && rule.getOwner() != owner) {
+ return null;
+ }
+
+ String featureName = featureCache.getIfPresent(rule);
+ if (featureName == null) {
+ featureName = getRuleString(rule);
+ featureCache.put(rule, featureName);
+ }
+ acc.add(featureName, VALUE);
+
+ return null;
+ }
+
+ /**
+ * Obtains the feature id for the given rule.
+ * @param rule
+ * @return String representing the feature name.s
+ */
+ private String getRuleString(final Rule rule) {
+ final StringBuilder sb = new StringBuilder(Vocabulary.word(rule.getLHS()))
+ .append(SIDES_SEPARATOR);
+ if (sides == Sides.SOURCE || sides == Sides.BOTH) {
+ sb.append(Vocabulary.getWords(rule.getFrench(), SEPARATOR));
+ }
+ sb.append(SIDES_SEPARATOR);
+ if (sides == Sides.TARGET || sides == Sides.BOTH) {
+ sb.append(Vocabulary.getWords(rule.getEnglish(), SEPARATOR));
+ }
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/RuleLength.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/RuleLength.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/RuleLength.java
new file mode 100644
index 0000000..02c520b
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/RuleLength.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff;
+
+import java.util.List;
+
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.chart_parser.SourcePath;
+import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.hypergraph.HGNode;
+import org.apache.joshua.decoder.segment_file.Sentence;
+
+/*
+ * This feature computes three feature templates: a feature indicating the length of the rule's
+ * source side, its target side, and a feature that pairs them.
+ */
+public abstract class RuleLength extends StatelessFF {
+
+ private static final int VALUE = 1;
+
+ public RuleLength(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, "RuleLength", args, config);
+ }
+
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+ int sourceLength = rule.getFrench().length;
+ int targetLength = rule.getEnglish().length;
+ acc.add(name + "_source" + sourceLength, VALUE);
+ acc.add(name + "_target" + sourceLength, VALUE);
+ acc.add(name + "_sourceTarget" + sourceLength + "-" + targetLength, VALUE);
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/RulePropertiesQuerying.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/RulePropertiesQuerying.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/RulePropertiesQuerying.java
new file mode 100644
index 0000000..a1867a3
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/RulePropertiesQuerying.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.hypergraph.HGNode;
+
+public class RulePropertiesQuerying {
+
+ public static final String getLHSAsString(Rule rule) {
+ return Vocabulary.word(rule.getLHS());
+ }
+
+ public static List<String> getRuleSourceNonterminalStrings(Rule rule) {
+ List<String> result = new ArrayList<String>();
+ for (int nonTerminalIndex : rule.getForeignNonTerminals()) {
+ result.add(Vocabulary.word(nonTerminalIndex));
+ }
+ return result;
+ }
+
+ public static List<String> getSourceNonterminalStrings(List<HGNode> tailNodes) {
+ List<String> result = new ArrayList<String>();
+ for (HGNode tailNode : tailNodes) {
+ result.add(Vocabulary.word(tailNode.lhs));
+ }
+ return result;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/RuleShape.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/RuleShape.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/RuleShape.java
new file mode 100644
index 0000000..8483ad6
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/RuleShape.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff;
+
+import java.util.List;
+
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.chart_parser.SourcePath;
+import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.hypergraph.HGNode;
+import org.apache.joshua.decoder.segment_file.Sentence;
+import org.apache.joshua.util.FormatUtils;
+
+/*
+ * Implements the RuleShape feature for source, target, and paired source+target sides.
+ */
+public class RuleShape extends StatelessFF {
+
+ public RuleShape(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, "RuleShape", args, config);
+ }
+
+ private enum WordType {
+ N("N"), T("x"), P("+");
+ private final String string;
+ private boolean repeats;
+
+ private WordType(final String string) {
+ this.string = string;
+ this.repeats = false;
+ }
+
+ private void setRepeats() {
+ repeats = true;
+ }
+
+ @Override
+ public String toString() {
+ if (repeats) {
+ return this.string + "+";
+ }
+ return this.string;
+ }
+ }
+
+ private WordType getWordType(int id) {
+ if (FormatUtils.isNonterminal(id)) {
+ return WordType.N;
+ } else {
+ return WordType.T;
+ }
+ }
+
+ /**
+ * Returns a String describing the rule pattern.
+ */
+ private String getRulePattern(int[] ids) {
+ final StringBuilder pattern = new StringBuilder();
+ WordType currentType = getWordType(ids[0]);
+ for (int i = 1; i < ids.length; i++) {
+ if (getWordType(ids[i]) != currentType) {
+ pattern.append(currentType.toString());
+ currentType = getWordType(ids[i]);
+ } else {
+ currentType.setRepeats();
+ }
+ }
+ pattern.append(currentType.toString());
+ return pattern.toString();
+ }
+
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i_, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+ final String sourceShape = getRulePattern(rule.getFrench());
+ final String targetShape = getRulePattern(rule.getEnglish());
+ acc.add(name + "_source_" + sourceShape, 1);
+ acc.add(name + "_target_" + sourceShape, 1);
+ acc.add(name + "_sourceTarget_" + sourceShape + "_" + targetShape, 1);
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/SourceDependentFF.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/SourceDependentFF.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/SourceDependentFF.java
new file mode 100644
index 0000000..841402a
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/SourceDependentFF.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff;
+
+import org.apache.joshua.decoder.segment_file.Sentence;
+
+public interface SourceDependentFF extends Cloneable {
+
+ public void setSource(Sentence sentence);
+
+ public FeatureFunction clone();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/SourcePathFF.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/SourcePathFF.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/SourcePathFF.java
new file mode 100644
index 0000000..b138426
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/SourcePathFF.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.chart_parser.SourcePath;
+import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.hypergraph.HGNode;
+import org.apache.joshua.decoder.segment_file.Sentence;
+
+/**
+ * This feature returns the scored path through the source lattice, which is recorded in a
+ * SourcePath object.
+ *
+ * @author Chris Dyer redpony@umd.edu
+ * @author Matt Post post@cs.jhu.edu
+ */
+public final class SourcePathFF extends StatelessFF {
+
+ /*
+ * This is a single-value feature template, so we cache the weight here.
+ */
+ public SourcePathFF(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, "SourcePath", args, config);
+ }
+
+ @Override
+ public ArrayList<String> reportDenseFeatures(int index) {
+ denseFeatureIndex = index;
+
+ ArrayList<String> names = new ArrayList<String>();
+ names.add(name);
+ return names;
+ }
+
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+
+ acc.add(denseFeatureIndex, sourcePath.getPathCost());
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/StatefulFF.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/StatefulFF.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/StatefulFF.java
new file mode 100644
index 0000000..1f5d0ed
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/StatefulFF.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff;
+
+import java.util.List;
+
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.chart_parser.SourcePath;
+import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.hypergraph.HGNode;
+import org.apache.joshua.decoder.segment_file.Sentence;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Stateful features contribute dynamic programming state. Unlike earlier versions of Joshua, the
+ * stateful feature itself is responsible for computing and return its updated state. Each
+ * state-computing feature function is assigned a global index, which is used to index the list of
+ * state-contributing objects in each HGNode. State can no longer be shared among different feature
+ * functions.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ * @author Juri Ganitkevich juri@cs.jhu.edu
+ */
+public abstract class StatefulFF extends FeatureFunction {
+
+ private static final Logger LOG = LoggerFactory.getLogger(StatefulFF.class);
+ /* Every stateful FF takes a unique index value and increments this. */
+ static int GLOBAL_STATE_INDEX = 0;
+
+ /* This records the state index for each instantiated stateful feature function. */
+ protected int stateIndex = 0;
+
+ public StatefulFF(FeatureVector weights, String name, String[] args, JoshuaConfiguration config) {
+ super(weights, name, args, config);
+
+ LOG.info("Stateful object with state index {}", GLOBAL_STATE_INDEX);
+ stateIndex = GLOBAL_STATE_INDEX++;
+ }
+
+ public static void resetGlobalStateIndex() {
+ GLOBAL_STATE_INDEX = 0;
+ }
+
+ public final boolean isStateful() {
+ return true;
+ }
+
+ public final int getStateIndex() {
+ return stateIndex;
+ }
+
+ /**
+ * Function computing the features that this function fires when a rule is applied. Must return
+ * its updated DPState. The accumulator is used to record every feature that fires.
+ */
+ @Override
+ public abstract DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j,
+ SourcePath sourcePath, Sentence sentence, Accumulator acc);
+
+ @Override
+ public abstract DPState computeFinal(HGNode tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc);
+
+ /**
+ * Computes an estimated future cost of this rule. Note that this is not compute as part of the
+ * score but is used for pruning.
+ */
+ @Override
+ public abstract float estimateFutureCost(Rule rule, DPState state, Sentence sentence);
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/StatelessFF.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/StatelessFF.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/StatelessFF.java
new file mode 100644
index 0000000..e473c37
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/StatelessFF.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff;
+
+import java.util.List;
+
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.chart_parser.SourcePath;
+import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.hypergraph.HGNode;
+import org.apache.joshua.decoder.segment_file.Sentence;
+
+/**
+ * Stateless feature functions do not contribute any state. You need not implement this class to
+ * create a stateless feature function, but it provides a few convenience functions.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ * @author Juri Ganitkevich juri@cs.jhu.edu
+ */
+
+public abstract class StatelessFF extends FeatureFunction {
+
+ public StatelessFF(FeatureVector weights, String name, String[] args, JoshuaConfiguration config) {
+ super(weights, name, args, config);
+ }
+
+ public final boolean isStateful() {
+ return false;
+ }
+
+ /**
+ * The estimated cost of applying this feature, given only the rule. This is used in sorting the
+ * rules for cube pruning. For most features, this will be 0.0.
+ */
+ public float estimateCost(Rule rule, Sentence sentence) {
+ return 0.0f;
+ }
+
+ /**
+ * Implementations of this should return null, since no state is contributed.
+ */
+ @Override
+ public abstract DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j,
+ SourcePath sourcePath, Sentence sentence, Accumulator acc);
+
+ /**
+ * Implementations of this should return null, since no state is contributed.
+ */
+ @Override
+ public DPState computeFinal(HGNode tailNode, int i, int j, SourcePath sourcePath, Sentence sentence,
+ Accumulator acc) {
+ return null;
+ }
+
+ /**
+ * Stateless functions do not have an estimate of the future cost because they do not have access
+ * to the state.
+ */
+ public final float estimateFutureCost(Rule rule, DPState state, Sentence sentence) {
+ return 0.0f;
+ }
+}
[55/60] incubator-joshua git commit: Merge branch
'maven-multi-module' of https://github.com/logogin/incubator-joshua into
maven-multi-module
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/main/java/org/apache/joshua/metrics/EvaluationMetric.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/metrics/EvaluationMetric.java
index 0000000,9a8786c..9ac77f1
mode 000000,100644..100644
--- a/joshua-core/src/main/java/org/apache/joshua/metrics/EvaluationMetric.java
+++ b/joshua-core/src/main/java/org/apache/joshua/metrics/EvaluationMetric.java
@@@ -1,0 -1,405 +1,409 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.metrics;
+
+ import java.io.BufferedReader;
+ import java.io.FileInputStream;
+ import java.io.IOException;
+ import java.io.InputStreamReader;
+ import java.io.PrintWriter;
+ import java.text.DecimalFormat;
+ import java.util.Arrays;
+ import java.util.TreeMap;
+
+ /***
+ * @author Omar Zaidan
+ */
+ public abstract class EvaluationMetric {
+ /* static data members */
+ private static TreeMap<String, Integer> metricOptionCount; // maps metric names -> number of
+ // options for that metric
+ protected static int numSentences; // number of sentences in the MERT set
+ protected static int numDocuments; // number of documents in the MERT set
+ protected static int refsPerSen;
+ protected static String[][] refSentences;
+ protected final static DecimalFormat f0 = new DecimalFormat("###0");
+ protected final static DecimalFormat f4 = new DecimalFormat("###0.0000");
+ protected static String tmpDirPrefix;
+
+ /* non-static data members */
+ protected int suffStatsCount; // number of sufficient statistics
+ protected String metricName; // number of metric
+ protected boolean toBeMinimized;
+
+ // is this a metric that should be minimized?
+ // e.g. toBeMinimized = true for 01LOSS, WER, TER
+ // toBeMinimized = false for BLEU
+
+ /* static (=> also non-abstract) methods */
+ public static void set_knownMetrics() {
+ metricOptionCount = new TreeMap<String, Integer>();
+
+ metricOptionCount.put("BLEU", 2);
+ // the "BLEU" metric expects an options array of length 2
+ metricOptionCount.put("BLEU_SBP", 2);
+ // the "BLEU_SBP" metric expects an options array of length 2
+ metricOptionCount.put("01LOSS", 0);
+ // the "01LOSS" metric expects an options array of length 0
+ metricOptionCount.put("TER", 6);
+ // the "TER" metric expects an options array of length 5
+ // metricOptionCount.put("METEOR",4);
+ // the "METEOR" metric expects an options array of length 4
+ // metricOptionCount.put("RYPT",5);
+ // the "RYPT" metric expects an options array of length 5
+ metricOptionCount.put("TER-BLEU", 8);
+ // the "TER-BLEU" metric expects an options array of length 7
+ // metricOptionCount.put("WER",0);
+ // the "WER" metric expects an options array of length 0
+ metricOptionCount.put("MC_BLEU", 4);
+ metricOptionCount.put("PRECIS", 6);
+ metricOptionCount.put("SRC_BLEU", 4);
+ metricOptionCount.put("PRECIS-SRC_BLEU", 6);
+ metricOptionCount.put("GL_BLEU", 3);
+ metricOptionCount.put("SARI", 2); // options: max-ngram source-path
++ metricOptionCount.put("CHRF", 2); // options: beta (how much to weight recall vs precision) and max-ngram
+ }
+
+ public static EvaluationMetric getMetric(String metricName, String[] metricOptions) {
+ EvaluationMetric retMetric = null;
+
+ if (metricName.equals("BLEU")) {
+ retMetric = new BLEU(metricOptions); // the "BLEU" metric corresponds to the BLEU class
+ } else if (metricName.equals("BLEU_SBP")) {
+ retMetric = new BLEU_SBP(metricOptions); // the "BLEU_SBP" metric corresponds to the BLEU_SBP
+ // class
+ } else if (metricName.equals("01LOSS")) {
+ retMetric = new ZeroOneLoss(metricOptions); // the "01LOSS" metric corresponds to the
+ // ZeroOneLoss class
+ } else if (metricName.equals("TER")) {
+ retMetric = new TER(metricOptions); // the "TER" metric corresponds to the TER class
+ // } else if (metricName.equals("METEOR")) {
+ // retMetric = new METEOR(metricOptions); // the "METEOR" metric corresponds to the METEOR
+ // class
+ // } else if (metricName.equals("RYPT")) {
+ // retMetric = new RYPT(metricOptions); // the "RYPT" metric corresponds to the RYPT class
+ } else if (metricName.equals("TER-BLEU")) {
+ retMetric = new TERMinusBLEU(metricOptions); // the "TER-BLEU" metric corresponds to the
+ // TERMinusBLEU class
+ // } else if (metricName.equals("WER")) {
+ // retMetric = new WordErrorRate(metricOptions); // the "WER" metric corresponds to the
+ // WordErrorRate class
+ } else if (metricName.equals("MC_BLEU")) {
+ retMetric = new MinimumChangeBLEU(metricOptions); // the "MC_BLEU" metric corresponds to the
+ // ParaphraseBLEU class
+ } else if (metricName.equals("PRECIS")) {
+ retMetric = new Precis(metricOptions);
+ } else if (metricName.equals("SRC_BLEU")) {
+ retMetric = new SourceBLEU(metricOptions);
+ } else if (metricName.equals("PRECIS-SRC_BLEU")) {
+ retMetric = new PrecisMinusSourceBLEU(metricOptions);
+ } else if (metricName.equals("GL_BLEU")) {
+ retMetric = new GradeLevelBLEU(metricOptions); // the "GL_BLEU" metric corresponds to the
+ // GradeLevelBLEU class
+ } else if (metricName.equals("SARI")) {
+ retMetric = new SARI(metricOptions);
- }
++
++ } else if (metricName.equals("CHRF")) {
++ retMetric = new CHRF(metricOptions);
++ }
+
+ return retMetric;
+ }
+
+ public static void set_numSentences(int x) {
+ numSentences = x;
+ }
+
+ public static void set_numDocuments(int x) {
+ numDocuments = x;
+ }
+
+ public static void set_refsPerSen(int x) {
+ refsPerSen = x;
+ }
+
+ public static void set_tmpDirPrefix(String S) {
+ tmpDirPrefix = S;
+ }
+
+ public static void set_refSentences(String[][] refs) {
+ refSentences = new String[numSentences][refsPerSen];
+ for (int i = 0; i < numSentences; ++i) {
+ for (int r = 0; r < refsPerSen; ++r) {
+ refSentences[i][r] = refs[i][r];
+ }
+ }
+ }
+
+ public static boolean knownMetricName(String name) {
+ return metricOptionCount.containsKey(name);
+ }
+
+ public static int metricOptionCount(String name) {
+ return metricOptionCount.get(name);
+ }
+
+ /* non-abstract, non-static methods */
+ public int get_suffStatsCount() {
+ return suffStatsCount;
+ }
+
+ public String get_metricName() {
+ return metricName;
+ }
+
+ public boolean getToBeMinimized() {
+ return toBeMinimized;
+ }
+
+ public boolean isBetter(double x, double y) {
+ // return true if x is better than y
+ if (toBeMinimized) {
+ return (x < y);
+ } else {
+ return (x > y);
+ }
+ }
+
+ public double score(String cand_str, int i) {
+ String[] SA = new String[1];
+ SA[0] = cand_str;
+ int[] IA = new int[1];
+ IA[0] = i;
+
+ int[][] SS = suffStats(SA, IA);
+
+ int[] stats = new int[suffStatsCount];
+ for (int s = 0; s < suffStatsCount; ++s) {
+ stats[s] = SS[0][s];
+ }
+
+ return score(stats);
+ }
+
+ public double score(String[] topCand_str) {
+ int[] stats = suffStats(topCand_str);
+ return score(stats);
+ }
+
+ public int[] suffStats(String[] topCand_str) {
+ int[] IA = new int[numSentences];
+ for (int i = 0; i < numSentences; ++i) {
+ IA[i] = i;
+ }
+
+ int[][] SS = suffStats(topCand_str, IA);
+
+ int[] totStats = new int[suffStatsCount];
+ for (int s = 0; s < suffStatsCount; ++s) {
+ totStats[s] = 0;
+ for (int i = 0; i < numSentences; ++i) {
+ totStats[s] += SS[i][s];
+ }
+ }
+
+ return totStats;
+ }
+
+ /**
+ * Calculates sufficient statistics on each sentence in the corpus, returning them as arrays.
+ *
+ * @param cand_strings todo
+ * @param cand_indices todo
+ * @return todo
+ */
+ public int[][] suffStats(String[] cand_strings, int[] cand_indices) {
+
+ int candCount = cand_strings.length;
+ if (cand_indices.length != candCount) {
+ System.out.println("Array lengths mismatch in suffStats(String[],int[]); returning null.");
+ return null;
+ }
+
+ int[][] stats = new int[candCount][suffStatsCount];
+
+ for (int d = 0; d < candCount; ++d) {
+ int[] currStats = suffStats(cand_strings[d], cand_indices[d]);
+
+ for (int s = 0; s < suffStatsCount; ++s) {
+ stats[d][s] = currStats[s];
+ }
+ } // for (d)
+
+ return stats;
+ }
+
+ public void createSuffStatsFile(String cand_strings_fileName, String cand_indices_fileName,
+ String outputFileName, int maxBatchSize) {
+ // similar to the above suffStats(String[], int[])
+
+ try {
+ FileInputStream inStream_cands = new FileInputStream(cand_strings_fileName);
+ BufferedReader inFile_cands =
+ new BufferedReader(new InputStreamReader(inStream_cands, "utf8"));
+
+ FileInputStream inStream_indices = new FileInputStream(cand_indices_fileName);
+ BufferedReader inFile_indices =
+ new BufferedReader(new InputStreamReader(inStream_indices, "utf8"));
+
+ PrintWriter outFile = new PrintWriter(outputFileName);
+
+ String[] cand_strings = new String[maxBatchSize];
+ int[] cand_indices = new int[maxBatchSize];
+
+ String line_cand = inFile_cands.readLine();
+ String line_index = inFile_indices.readLine();
+
+ while (line_cand != null) {
+ int size = 0;
+ while (line_cand != null) {
+ cand_strings[size] = line_cand;
+ cand_indices[size] = Integer.parseInt(line_index);
+ ++size; // now size is how many were read for this currnet batch
+ if (size == maxBatchSize) break;
+
+ line_cand = inFile_cands.readLine();
+ line_index = inFile_indices.readLine();
+ }
+
+ if (size < maxBatchSize) { // last batch, and smaller than maxBatchSize
+ String[] cand_strings_temp = new String[size];
+ int[] cand_indices_temp = new int[size];
+ for (int d = 0; d < size; ++d) {
+ cand_strings_temp[d] = cand_strings[d];
+ cand_indices_temp[d] = cand_indices[d];
+ }
+ cand_strings = cand_strings_temp;
+ cand_indices = cand_indices_temp;
+ }
+
+ int[][] SS = suffStats(cand_strings, cand_indices);
+ for (int d = 0; d < size; ++d) {
+ StringBuilder stats_str = new StringBuilder();
+
+ for (int s = 0; s < suffStatsCount - 1; ++s) {
+ stats_str.append(SS[d][s]).append(" ");
+ }
+ stats_str.append(SS[d][suffStatsCount - 1]);
+
+ outFile.println(stats_str);
+ }
+
+ line_cand = inFile_cands.readLine();
+ line_index = inFile_indices.readLine();
+ }
+
+ inFile_cands.close();
+ inFile_indices.close();
+ outFile.close();
+
+ } catch (IOException e) {
+ throw new RuntimeException("IOException in EvaluationMetric.createSuffStatsFile(...): "
+ + e.getMessage(), e);
+ }
+
+ }
+
+ public void printDetailedScore(String[] topCand_str, boolean oneLiner) {
+ int[] stats = suffStats(topCand_str);
+ printDetailedScore_fromStats(stats, oneLiner);
+ }
+
+ public double score(int[][] stats) {
+ // returns an average of document scores (aka the document-level score, as opposed to
+ // corpus-level score)
+ // stats[][] is indexed [doc][s]
+
+ double retVal = 0.0;
+ for (int doc = 0; doc < numDocuments; ++doc) {
+ retVal += score(stats[doc]);
+ }
+ return retVal / numDocuments;
+ }
+
+ public double score(int[][] stats, int firstRank, int lastRank) {
+ // returns an average of document scores, restricted to the documents
+ // ranked firstRank-lastRank, inclusive (ranks are 1-indexed, even though the docs are
+ // 0-indexed)
+
+ double[] scores = docScores(stats);
+
+ Arrays.sort(scores);
+ // sorts into ascending order
+
+ double retVal = 0.0;
+
+ if (toBeMinimized) {
+ // scores[0] is rank 1, scores[numDocuments-1] is rank numDocuments
+ // => scores[j] is rank j+1
+ // => rank r is scores[r-1]
+ for (int j = firstRank - 1; j < lastRank; ++j) {
+ retVal += scores[j];
+ }
+ } else {
+ // scores[numDocuments-1] is rank 1, scores[0] is rank numDocuments
+ // => scores[j] is rank numDocuments-j
+ // => rank r is scores[numDocuments-r]
+ for (int j = numDocuments - firstRank; j >= numDocuments - lastRank; --j) {
+ retVal += scores[j];
+ }
+ }
+
+ return retVal / (lastRank - firstRank + 1);
+
+ }
+
+ public double[] docScores(int[][] stats) {
+ // returns an array of document scores
+ // stats[][] is indexed [doc][s]
+
+ double[] scores = new double[numDocuments];
+ for (int doc = 0; doc < numDocuments; ++doc) {
+ scores[doc] = score(stats[doc]);
+ }
+ return scores;
+ }
+
+ public void printDetailedScore_fromStats(int[][] stats, String[] docNames) {
+ // prints individual document scores
+ // stats[][] is indexed [doc][s]
+
+ for (int doc = 0; doc < numDocuments; ++doc) {
+ if (docNames == null) {
+ System.out.print("Document #" + doc + ": ");
+ } else {
+ System.out.print(docNames[doc] + ": ");
+ }
+ printDetailedScore_fromStats(stats[doc], true);
+ }
+ }
+
+ /* abstract (=> also non-static) methods */
+ protected abstract void initialize();
+
+ public abstract double bestPossibleScore();
+
+ public abstract double worstPossibleScore();
+
+ public abstract int[] suffStats(String cand_str, int i);
+
+ public abstract double score(int[] stats);
+
+ public abstract void printDetailedScore_fromStats(int[] stats, boolean oneLiner);
+ }
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/test/java/org/apache/joshua/system/AlignmentMapTest.java
----------------------------------------------------------------------
diff --cc joshua-core/src/test/java/org/apache/joshua/system/AlignmentMapTest.java
index 0000000,eba732a..a28edc7
mode 000000,100644..100644
--- a/joshua-core/src/test/java/org/apache/joshua/system/AlignmentMapTest.java
+++ b/joshua-core/src/test/java/org/apache/joshua/system/AlignmentMapTest.java
@@@ -1,0 -1,72 +1,73 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.system;
+
+ import static org.junit.Assert.*;
+
+ import java.util.Arrays;
+ import java.util.HashMap;
+ import java.util.List;
+ import java.util.Map;
+
+ import org.apache.joshua.corpus.Vocabulary;
++import org.apache.joshua.decoder.ff.FeatureVector;
+ import org.apache.joshua.decoder.ff.tm.Rule;
+
+ import org.junit.Before;
+ import org.junit.Test;
+
+ public class AlignmentMapTest {
+
+ private Rule rule1 = null;
+ private Rule rule2 = null;
+ private static Map<Integer, List<Integer>> expectedAlignmentMap = null;
+ private static final int[] expectedNonTerminalPositions = {2,5};
+
+ @Before
+ public void setUp() throws Exception {
+ Vocabulary.clear();
+ int[] sourceRhs = {Vocabulary.id("A1"),Vocabulary.id("A2"),-1,Vocabulary.id("B"),Vocabulary.id("C"),-2};
+ int[] targetRhs = {Vocabulary.id("c"),Vocabulary.id("b1"),-1,Vocabulary.id("b2"),-4,Vocabulary.id("a")};
+ int arity = 2; // 2 non terminals
+ String alignment = "0-5 1-5 3-1 3-3 4-0";
+ expectedAlignmentMap = new HashMap<Integer, List<Integer>>();
+ expectedAlignmentMap.put(0, Arrays.asList(4));
+ expectedAlignmentMap.put(5, Arrays.asList(0,1));
+ expectedAlignmentMap.put(1, Arrays.asList(3));
+ expectedAlignmentMap.put(3, Arrays.asList(3));
+ rule1 = new Rule(-1, sourceRhs, targetRhs, "", arity, alignment);
- rule2 = new Rule(-1, sourceRhs, targetRhs, "", arity, null); // rule with no alignment
++ rule2 = new Rule(-1, sourceRhs, targetRhs, new FeatureVector(), arity, null); // rule with no alignment
+ }
+
+ @Test
+ public void test() {
+ // test regular rule with arity 2
+ Map<Integer, List<Integer>> alignmentMap1 = rule1.getAlignmentMap();
+ assertEquals(expectedAlignmentMap, alignmentMap1);
+ int[] nonTerminalPositions1 = rule1.getNonTerminalSourcePositions();
+ assertArrayEquals(expectedNonTerminalPositions, nonTerminalPositions1);
+
+ // test rule with no alignment
+ Map<Integer, List<Integer>> alignmentMap2 = rule2.getAlignmentMap();
+ assertTrue(alignmentMap2.isEmpty());
+ int[] nonTerminalPositions2 = rule2.getNonTerminalSourcePositions();
+ assertArrayEquals(expectedNonTerminalPositions, nonTerminalPositions2);
+ }
+
+ }
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/test/resources/bn-en/hiero/joshua-classlm.config
----------------------------------------------------------------------
diff --cc joshua-core/src/test/resources/bn-en/hiero/joshua-classlm.config
index 0000000,970b9b7..3be7392
mode 000000,100644..100644
--- a/joshua-core/src/test/resources/bn-en/hiero/joshua-classlm.config
+++ b/joshua-core/src/test/resources/bn-en/hiero/joshua-classlm.config
@@@ -1,0 -1,51 +1,51 @@@
-feature-function = LanguageModel -lm_type kenlm -lm_order 5 -minimizing false -lm_file lm.gz
++feature-function = StateMinimizingLanguageModel -lm_type kenlm -lm_order 5 -lm_file lm.gz
+
+ # Class LM feature
-feature-function = LanguageModel -lm_type kenlm -lm_order 9 -minimizing false -lm_file class_lm_9gram.gz -lm_class -class_map class.map
++feature-function = StateMinimizingLanguageModel -lm_type kenlm -lm_order 9 -lm_file class_lm_9gram.gz -class_map class.map
+
+ ###### Old format for lms
+ # lm = kenlm 5 false false 100 lm.gz
+
+ # tm = TYPE OWNER MAX_SPAN PATH
+ tm = thrax pt 12 grammar.gz
+ tm = thrax glue -1 glue-grammar
+
+ mark_oovs=false
+
+ #tm config
+ default_non_terminal=X
+ goalSymbol=GOAL
+
+ #pruning config
+ pop-limit = 10
+
+ #nbest config
+ use_unique_nbest=true
+ top_n = 10
+
+ feature-function = WordPenalty
+ feature-function = OOVPenalty
+
+ ###### model weights
+ lm_0 1.2373676802179452
+ lm_1 1.2373676802179452
+ tm_pt_0 -2.4497429277910214
+ tm_pt_1 0.7224581556224123
+ tm_pt_2 -0.31689069155153504
+ tm_pt_3 0.33861043967238036
+ tm_pt_4 0.03553113401320236
+ tm_pt_5 0.19138972284064748
+ tm_pt_6 0.3417994095521415
+ tm_pt_7 -0.9936312455671283
+ tm_pt_8 0.9070737587091975
+ tm_pt_9 0.8202511858619419
+ tm_pt_10 0.2593091306160006
+ tm_pt_11 0.25597137004462134
+ tm_pt_12 0.3538894647790496
+ tm_pt_13 -0.36212061186692646
+ tm_pt_14 -0.32923261148678096
+ tm_pt_15 0.5524863522177359
+ tm_pt_16 0.23451595442127693
+ tm_glue_0 1
+ WordPenalty -3.6942747832593694
+ OOVPenalty 1.0
[17/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/zmert/ZMERT.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/zmert/ZMERT.java b/joshua-core/src/main/java/org/apache/joshua/zmert/ZMERT.java
new file mode 100644
index 0000000..7e4c2cc
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/zmert/ZMERT.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.zmert;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.util.FileUtility;
+
+public class ZMERT {
+ public static void main(String[] args) throws Exception {
+ boolean external = false; // should each MERT iteration be launched externally?
+
+ if (args.length == 1) {
+ if (args[0].equals("-h")) {
+ printZMERTUsage(args.length, true);
+ System.exit(2);
+ } else {
+ external = false;
+ }
+ } else if (args.length == 3) {
+ external = true;
+ } else {
+ printZMERTUsage(args.length, false);
+ System.exit(1);
+ }
+
+ if (!external) {
+ JoshuaConfiguration joshuaConfiguration = new JoshuaConfiguration();
+ MertCore myMert = new MertCore(args[0],joshuaConfiguration);
+ myMert.run_MERT(); // optimize lambda[]!!!
+ myMert.finish();
+ } else {
+ int maxMem = Integer.parseInt(args[1]);
+ String configFileName = args[2];
+ String stateFileName = FileUtility.dirname(configFileName) + "/ZMERT.temp.state";
+ String cp = System.getProperty("java.class.path");
+ boolean done = false;
+ int iteration = 0;
+ while (!done) {
+ ++iteration;
+ Runtime rt = Runtime.getRuntime();
+ Process p =
+ rt.exec("java -Xmx" + maxMem + "m -cp " + cp + " org.apache.joshua.zmert.MertCore "
+ + configFileName + " " + stateFileName + " " + iteration);
+ BufferedReader br_i = new BufferedReader(new InputStreamReader(p.getInputStream()));
+ BufferedReader br_e = new BufferedReader(new InputStreamReader(p.getErrorStream()));
+ String dummy_line = null;
+ while ((dummy_line = br_i.readLine()) != null) {
+ System.out.println(dummy_line);
+ }
+ while ((dummy_line = br_e.readLine()) != null) {
+ System.out.println(dummy_line);
+ }
+ int status = p.waitFor();
+
+ if (status == 90) {
+ done = true;
+ } else if (status == 91) {
+ done = false;
+ } else {
+ System.out.println("Z-MERT exiting prematurely (MertCore returned " + status + ")...");
+ System.exit(status);
+ }
+ }
+ }
+
+ System.exit(0);
+
+ } // main(String[] args)
+
+ public static void printZMERTUsage(int argsLen, boolean detailed) {
+ if (!detailed) {
+ println("Oops, you provided " + argsLen + " args!");
+ println("");
+ println("Usage:");
+ println(" ZMERT -maxMem maxMemoryInMB MERT_configFile");
+ println("");
+ println("Where -maxMem specifies the maximum amount of memory (in MB) Z-MERT is");
+ println("allowed to use when performing its calculations (no memroy is needed while");
+ println("the decoder is running),");
+ println("and the config file contains any subset of Z-MERT's 20-some parameters,");
+ println("one per line. Run ZMERT -h for more details on those parameters.");
+ } else {
+ println("Usage:");
+ println(" ZMERT -maxMem maxMemoryInMB MERT_configFile");
+ println("");
+ println("Where -maxMem specifies the maximum amount of memory (in MB) Z-MERT is");
+ println("allowed to use when performing its calculations (no memroy is needed while");
+ println("the decoder is running),");
+ println("and the config file contains any subset of Z-MERT's 20-some parameters,");
+ println("one per line. Those parameters, and their default values, are:");
+ println("");
+ println("Relevant files:");
+ println(" -dir dirPrefix: working directory\n [[default: null string (i.e. they are in the current directory)]]");
+ println(" -s sourceFile: source sentences (foreign sentences) of the MERT dataset\n [[default: null string (i.e. file name is not needed by MERT)]]");
+ println(" -r refFile: target sentences (reference translations) of the MERT dataset\n [[default: reference.txt]]");
+ println(" -rps refsPerSen: number of reference translations per sentence\n [[default: 1]]");
+ println(" -txtNrm textNormMethod: how should text be normalized?\n (0) don't normalize text,\n or (1) \"NIST-style\", and also rejoin 're, *'s, n't, etc,\n or (2) apply 1 and also rejoin dashes between letters,\n or (3) apply 1 and also drop non-ASCII characters,\n or (4) apply 1+2+3\n [[default: 1]]");
+ println(" -p paramsFile: file containing parameter names, initial values, and ranges\n [[default: params.txt]]");
+ println(" -docInfo documentInfoFile: file informing Z-MERT which document each\n sentence belongs to\n [[default: null string (i.e. all sentences are in one 'document')]]");
+ println(" -fin finalLambda: file name for final lambda[] values\n [[default: null string (i.e. no such file will be created)]]");
+ println("");
+ println("MERT specs:");
+ println(" -m metricName metric options: name of evaluation metric and its options\n [[default: BLEU 4 closest]]");
+ println(" -maxIt maxMERTIts: maximum number of MERT iterations\n [[default: 20]]");
+ println(" -prevIt prevMERTIts: maximum number of previous MERT iterations to\n construct candidate sets from\n [[default: 20]]");
+ println(" -minIt minMERTIts: number of iterations before considering an early exit\n [[default: 5]]");
+ println(" -stopIt stopMinIts: some early stopping criterion must be satisfied in\n stopMinIts *consecutive* iterations before an early exit\n [[default: 3]]");
+ println(" -stopSig sigValue: early MERT exit if no weight changes by more than sigValue\n [[default: -1 (i.e. this criterion is never investigated)]]");
+ println(" -thrCnt threadCount: number of threads to run in parallel when optimizing\n [[default: 1]]");
+ println(" -save saveInter: save intermediate cfg files (1) or decoder outputs (2)\n or both (3) or neither (0)\n [[default: 3]]");
+ println(" -compress compressFiles: should Z-MERT compress the files it produces (1)\n or not (0)\n [[default: 0]]");
+ println(" -ipi initsPerIt: number of intermediate initial points per iteration\n [[default: 20]]");
+ println(" -opi oncePerIt: modify a parameter only once per iteration (1) or not (0)\n [[default: 0]]");
+ println(" -rand randInit: choose initial point randomly (1) or from paramsFile (0)\n [[default: 0]]");
+ println(" -seed seed: seed used to initialize random number generator\n [[default: time (i.e. value returned by System.currentTimeMillis()]]");
+ // println(" -ud useDisk: reliance on disk (0-2; higher value => more reliance)\n [[default: 2]]");
+ println("");
+ println("Decoder specs:");
+ println(" -cmd commandFile: name of file containing commands to run the decoder\n [[default: null string (i.e. decoder is a JoshuaDecoder object)]]");
+ println(" -passIt passIterationToDecoder: should iteration number be passed\n to command file (1) or not (0)\n [[default: 0]]");
+ println(" -decOut decoderOutFile: name of the output file produced by the decoder\n [[default: output.nbest]]");
+ println(" -decExit validExit: value returned by decoder to indicate success\n [[default: 0]]");
+ println(" -dcfg decConfigFile: name of decoder config file\n [[default: dec_cfg.txt]]");
+ println(" -N N: size of N-best list (per sentence) generated in each MERT iteration\n [[default: 100]]");
+ println("");
+ println("Output specs:");
+ println(" -v verbosity: Z-MERT verbosity level (0-2; higher value => more verbose)\n [[default: 1]]");
+ println(" -decV decVerbosity: should decoder output be printed (1) or ignored (0)\n [[default: 0]]");
+ println("");
+ }
+ }
+
+ private static void println(Object obj) {
+ System.out.println(obj);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/zmert/package-info.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/zmert/package-info.java b/joshua-core/src/main/java/org/apache/joshua/zmert/package-info.java
new file mode 100644
index 0000000..571b524
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/zmert/package-info.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * Provides code for performing minimum error rate training.
+ * Much of the code in this package is based on Och (2003).
+ * A deeper description of the algorithm is in Zaidan (2009).
+ */
+package org.apache.joshua.zmert;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/resources/log4j.properties b/joshua-core/src/main/resources/log4j.properties
new file mode 100644
index 0000000..acca5e9
--- /dev/null
+++ b/joshua-core/src/main/resources/log4j.properties
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# log4j settings
+log4j.rootLogger=WARN, stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target=System.err
+log4j.appender.stdout.layout=org.apache.log4j.SimpleLayout
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/overview.html
----------------------------------------------------------------------
diff --git a/joshua-core/src/overview.html b/joshua-core/src/overview.html
new file mode 100644
index 0000000..7efe5b3
--- /dev/null
+++ b/joshua-core/src/overview.html
@@ -0,0 +1,41 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2 Final//EN">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head></head>
+<body bgcolor="white">
+
+<!--
+##### THIS IS THE TEMPLATE FOR THE PACKAGE DOC COMMENTS. #####
+##### TYPE YOUR PACKAGE COMMENTS HERE. BEGIN WITH A #####
+##### ONE-SENTENCE SUMMARY STARTING WITH A VERB LIKE: #####
+-->
+
+Apache Joshua is an extensible, open source statistical
+hierarchical phrase-based machine translation system.
+
+<!--
+<h2>Related Documentation</h2>
+-->
+
+<!-- Put @see and @since tags down here. -->
+
+@see <a href="http://joshua.incubator.apache.org/">Joshua Website</a>
+
+
+</body>
+</html>
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/corpus/CorpusArrayTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/corpus/CorpusArrayTest.java b/joshua-core/src/test/java/org/apache/joshua/corpus/CorpusArrayTest.java
new file mode 100644
index 0000000..19cb20c
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/corpus/CorpusArrayTest.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.corpus;
+
+import java.util.logging.Logger;
+
+public class CorpusArrayTest {
+
+ /** Logger for this class. */
+ private static Logger logger =
+ Logger.getLogger(CorpusArrayTest.class.getName());
+}
+
+// @Test
+// public void writePartsToDisk() {
+//
+// String filename = "data/tiny.en";
+// int numSentences = 5; // Should be 5 sentences in tiny.en
+// int numWords = 89; // Should be 89 words in tiny.en
+//
+//
+// try {
+//
+// // FIX: can't use createVocabulary(String) because we set numWords and numSentences
+// Vocabulary vocab = new Vocabulary();
+// SuffixArrayFactory.createVocabulary(filename, vocab);
+// Corpus corpus = SuffixArrayFactory.createCorpusArray(filename, vocab, numWords, numSentences);
+//
+// corpus.writeWordIDsToFile(filename+".bin");
+// corpus.writeSentenceLengthsToFile(filename+".sbin");
+//
+// MemoryMappedCorpusArray mmCorpus = new MemoryMappedCorpusArray(corpus.getVocabulary(), filename+".bin", numWords*4, filename+".sbin", numSentences*4);
+//
+// // For each word in the corpus,
+// for (int i=0; i<corpus.size(); i++) {
+//
+// // Verify that the memory-mapped corpus and the in-memory corpus have the same value
+// Assert.assertEquals(mmCorpus.getWordID(i), corpus.getWordID(i));
+// }
+//
+//
+// // For each sentence in the corpus
+// for (int i=0; i<corpus.sentences.length; i++) {
+//
+// // Verify that the sentence position in the memory-mapped corpus and the in-memory corpus have the same value
+// Assert.assertEquals(corpus.getSentencePosition(i), mmCorpus.getSentencePosition(i));
+// }
+//
+// } catch (IOException e) {
+// Assert.fail(e.getLocalizedMessage());
+// }
+//
+// }
+//
+// @Test
+// public void iterate() {
+//
+// String[] sentences = {
+// "scientists complete sequencing of the chromosome linked to early dementia",
+// "( afp , paris , january 2 ) an international team of scientists said that they have completed the sequencing of human chromosome 14 that is linked to many diseases , including the early-onset alzheimer's that may strike people in their 30s .",
+// "this is the fourth chromosome whose sequence has been completed to date . it comprises more than 87 million pairs of dna .",
+// "this study published in the weekly british scientific journal nature illustrates that the sequence of chromosome 14 comprises 1,050 genes and gene fragments .",
+// "the goal of geneticists is to provide diagnostic tools to identify defective genes that cause diseases so as to arrive eventually at treatments that can prevent those genes from malfunctioning ."
+// };
+//
+//
+//
+// // Tell System.out and System.err to use UTF8
+// FormatUtil.useUTF8();
+//
+// try {
+//
+// File sourceFile = File.createTempFile("source", new Date().toString());
+// PrintStream sourcePrintStream = new PrintStream(sourceFile, "UTF-8");
+// for (String sentence : sentences) {
+// sourcePrintStream.println(sentence);
+// }
+// sourcePrintStream.close();
+// String corpusFileName = sourceFile.getAbsolutePath();
+//
+// Vocabulary vocabulary;
+//
+// logger.fine("Constructing vocabulary from file " + corpusFileName);
+// vocabulary = new Vocabulary();
+// int[] lengths = Vocabulary.initializeVocabulary(corpusFileName, vocabulary, true);
+//
+// logger.fine("Constructing corpus array from file " + corpusFileName);
+// Corpus corpus = SuffixArrayFactory.createCorpusArray(corpusFileName, vocabulary, lengths[0], lengths[1]);
+//
+// int expectedIndex = 0;
+// for (int actualIndex : corpus.corpusPositions()) {
+// Assert.assertEquals(actualIndex, expectedIndex);
+// expectedIndex += 1;
+// }
+//
+// Assert.assertEquals(corpus.size(), expectedIndex);
+//
+//
+// } catch (IOException e) {
+// Assert.fail("Unable to write temporary file. " + e.toString());
+// }
+//
+//
+//
+// }
+//
+//
+// @Test
+// public void writeAllToDisk() throws ClassNotFoundException {
+//
+// String filename = "data/tiny.en";
+// int numSentences = 5; // Should be 5 sentences in tiny.en
+// int numWords = 89; // Should be 89 words in tiny.en
+//
+//
+// try {
+//
+// // FIX: can't use createVocabulary(String) because we set numWords and numSentences
+// Vocabulary vocab = new Vocabulary();
+// Vocabulary.initializeVocabulary(filename, vocab, true);
+// CorpusArray corpus = SuffixArrayFactory.createCorpusArray(filename, vocab, numWords, numSentences);
+//
+// corpus.write(filename+".corpus", filename+".vocab", "UTF-8");
+//
+// MemoryMappedCorpusArray mmCorpus = new MemoryMappedCorpusArray(filename+".corpus", filename+".vocab");
+//
+// Assert.assertEquals(mmCorpus.size(), corpus.size());
+// Assert.assertEquals(mmCorpus.getNumSentences(), corpus.getNumSentences());
+//
+// // For each word in the corpus,
+// for (int i=0; i<corpus.size(); i++) {
+//
+// // Verify that the memory-mapped corpus and the in-memory corpus have the same value
+// Assert.assertEquals(mmCorpus.getWordID(i), corpus.getWordID(i));
+// }
+//
+//
+// // For each sentence in the corpus
+// for (int i=0; i<corpus.sentences.length; i++) {
+//
+// // Verify that the sentence start position in the memory-mapped corpus and the in-memory corpus have the same value
+// Assert.assertEquals(mmCorpus.getSentencePosition(i), corpus.getSentencePosition(i));
+//
+// // Verify that the sentence end position in the memory-mapped corpus and the in-memory corpus have the same value
+// Assert.assertEquals(mmCorpus.getSentenceEndPosition(i), corpus.getSentenceEndPosition(i));
+//
+// // Verify that the phrase corresponding to this sentence is the same
+// Phrase sentence = corpus.getSentence(i);
+// Phrase mmSentence = mmCorpus.getSentence(i);
+// Assert.assertNotNull(sentence);
+// Assert.assertNotNull(mmSentence);
+// Assert.assertEquals(mmSentence, sentence);
+// }
+//
+// } catch (IOException e) {
+// Assert.fail(e.getLocalizedMessage());
+// }
+//
+// }
+//
+//}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/corpus/SpanTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/corpus/SpanTest.java b/joshua-core/src/test/java/org/apache/joshua/corpus/SpanTest.java
new file mode 100644
index 0000000..3558b79
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/corpus/SpanTest.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.corpus;
+
+import org.apache.joshua.corpus.Span;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+/**
+ *
+ *
+ * @author Lane Schwartz
+ */
+public class SpanTest {
+
+ @Test
+ public void iterator() {
+
+ Span span = new Span(1,10);
+
+ int expected = 1;
+
+ for (int actual : span) {
+ Assert.assertEquals(actual, expected);
+ expected++;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/corpus/VocabularyTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/corpus/VocabularyTest.java b/joshua-core/src/test/java/org/apache/joshua/corpus/VocabularyTest.java
new file mode 100644
index 0000000..834d68b
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/corpus/VocabularyTest.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.corpus;
+
+import static org.apache.joshua.util.FormatUtils.isNonterminal;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+public class VocabularyTest {
+ private static final String WORD1 = "word1";
+ private static final String WORD2 = "word2";
+ private static final String NON_TERMINAL = "[X]";
+ private static final String GOAL = "[GOAL]";
+
+ @Before
+ public void init() {
+ Vocabulary.clear();
+ }
+
+ @After
+ public void deinit() {
+ Vocabulary.clear();
+ }
+
+ @Test
+ public void givenVocabulary_whenEmpty_thenOnlyContainsUnknownWord() {
+ assertTrue(Vocabulary.hasId(Vocabulary.UNKNOWN_ID));
+ assertFalse(Vocabulary.hasId(1));
+ assertFalse(Vocabulary.hasId(-1));
+ assertEquals(Vocabulary.UNKNOWN_WORD, Vocabulary.word(Vocabulary.UNKNOWN_ID));
+ assertEquals(1, Vocabulary.size());
+ }
+
+ @Test
+ public void givenVocabulary_whenNewWord_thenMappingIsAdded() {
+ final int FIRST_WORD_ID = 1;
+ assertFalse(Vocabulary.hasId(FIRST_WORD_ID));
+ assertEquals(FIRST_WORD_ID, Vocabulary.id(WORD1));
+ //should return same id after second call:
+ assertEquals(FIRST_WORD_ID, Vocabulary.id(WORD1));
+ assertTrue(Vocabulary.hasId(FIRST_WORD_ID));
+ assertEquals(WORD1, Vocabulary.word(FIRST_WORD_ID));
+ assertEquals(2, Vocabulary.size());
+ }
+
+ @Test
+ public void givenVocabulary_whenCheckingStringInBracketsOrNegativeNumber_thenIsNonTerminal() {
+ //non-terminals
+ assertTrue(isNonterminal(NON_TERMINAL));
+ //terminals
+ assertFalse(isNonterminal(WORD1));
+ assertFalse(isNonterminal("[]"));
+ assertFalse(isNonterminal("["));
+ assertFalse(isNonterminal("]"));
+ assertFalse(isNonterminal(""));
+
+ //negative numbers indicate non-terminals
+ assertTrue(isNonterminal(-1));
+ assertTrue(isNonterminal(-5));
+
+ //positive numbers indicate terminals:
+ assertFalse(isNonterminal(0));
+ assertFalse(isNonterminal(5));
+ }
+
+ @Test
+ public void givenVocabulary_whenNonTerminal_thenReturnsStrictlyPositiveNonTerminalIndices() {
+ final int FIRST_NON_TERMINAL_INDEX = 1;
+ assertTrue(Vocabulary.id(NON_TERMINAL) < 0);
+ assertTrue(Vocabulary.hasId(FIRST_NON_TERMINAL_INDEX));
+ assertTrue(Vocabulary.hasId(-FIRST_NON_TERMINAL_INDEX));
+
+ assertTrue(Vocabulary.id("") > 0);
+ assertTrue(Vocabulary.id(WORD1) > 0);
+
+ final int SECOND_NON_TERMINAL_INDEX = 4;
+ assertTrue(Vocabulary.id(GOAL) < 0);
+ assertTrue(Vocabulary.hasId(SECOND_NON_TERMINAL_INDEX));
+ assertTrue(Vocabulary.hasId(-SECOND_NON_TERMINAL_INDEX));
+
+ assertTrue(Vocabulary.id(WORD2) > 0);
+ }
+
+ @Rule
+ public TemporaryFolder folder = new TemporaryFolder();
+
+ @Test
+ public void givenVocabulary_whenWritenAndReading_thenVocabularyStaysTheSame() throws IOException {
+ File vocabFile = folder.newFile();
+
+ int id1 = Vocabulary.id(WORD1);
+ int id2 = Vocabulary.id(NON_TERMINAL);
+ int id3 = Vocabulary.id(WORD2);
+
+ Vocabulary.write(vocabFile.getAbsolutePath());
+
+ Vocabulary.clear();
+
+ Vocabulary.read(vocabFile);
+
+ assertEquals(4, Vocabulary.size()); //unknown word + 3 other words
+ assertTrue(Vocabulary.hasId(id1));
+ assertTrue(Vocabulary.hasId(id2));
+ assertTrue(Vocabulary.hasId(id3));
+ assertEquals(id1, Vocabulary.id(WORD1));
+ assertEquals(id2, Vocabulary.id(NON_TERMINAL));
+ assertEquals(id3, Vocabulary.id(WORD2));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/corpus/vocab/VocabularyTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/corpus/vocab/VocabularyTest.java b/joshua-core/src/test/java/org/apache/joshua/corpus/vocab/VocabularyTest.java
new file mode 100644
index 0000000..c1af5ab
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/corpus/vocab/VocabularyTest.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.corpus.vocab;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.Date;
+import org.apache.joshua.corpus.Vocabulary;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+/**
+ *
+ *
+ * @author Lane Schwartz
+ */
+public class VocabularyTest {
+
+ /** [X], [X,1], [X,2], [S], [S,1] <unk>, <s>, </s>, -pau-*/
+ int numBuiltInSymbols = 9;
+
+ /** <unk>, <s>, </s>, -pau- */
+ int numBuiltInTerminals = 4;
+
+ @Test
+ public void basicVocabTest() {
+
+ Vocabulary vocab1 = new Vocabulary();
+ Vocabulary vocab2 = new Vocabulary();
+
+ Assert.assertEquals(vocab1, vocab2);
+
+ Assert.assertFalse(vocab1.size() == 0);
+ //Assert.assertTrue(vocab1.intToString.get(0)==Vocabulary.UNKNOWN_WORD_STRING);
+ //Assert.assertFalse(vocab1.getWords().isEmpty());
+ // Assert.assertTrue(vocab1.getWords(0)==Vocabulary.UNKNOWN_WORD_STRING);
+ // Assert.assertEquals(vocab1.getWords(), vocab1.intToString.values());
+
+ Assert.assertNotEquals(vocab1.size(), numBuiltInSymbols);
+ // Assert.assertEquals(vocab1.getWord(Vocabulary.UNKNOWN_WORD), Vocabulary.UNKNOWN_WORD_STRING);
+
+ //Assert.assertEquals(vocab1.getID("sample"), Vocabulary.UNKNOWN_WORD);
+ //Assert.assertEquals(vocab1.getID(null), Vocabulary.UNKNOWN_WORD);
+
+ // Assert.assertFalse(vocab1.terminalToInt.isEmpty());
+ // Assert.assertEquals(vocab1.terminalToInt.size(), this.numBuiltInTerminals);
+ // Assert.assertFalse(vocab1.isFixed);
+ //
+ // vocab1.fixVocabulary();
+ // Assert.assertTrue(vocab1.isFixed);
+
+ // Assert.assertEquals(vocab1.getID(Vocabulary.X_STRING), -1);
+ // Assert.assertEquals(vocab1.getID(Vocabulary.X1_STRING), -2);
+ // Assert.assertEquals(vocab1.getID(Vocabulary.X2_STRING), -3);
+ //
+ // Assert.assertEquals(vocab1.getWord(-1), Vocabulary.X_STRING);
+ // Assert.assertEquals(vocab1.getWord(-2), Vocabulary.X1_STRING);
+ // Assert.assertEquals(vocab1.getWord(-3), Vocabulary.X2_STRING);
+
+
+
+ // Assert.assertFalse(vocab2.intToString.isEmpty());
+ // Assert.assertTrue(vocab2.intToString.get(0)==Vocabulary.UNKNOWN_WORD_STRING);
+ // Assert.assertFalse(vocab2.getWords().isEmpty());
+ // Assert.assertTrue(vocab2.getWord(0)==Vocabulary.UNKNOWN_WORD_STRING);
+ // Assert.assertEquals(vocab2.getWords(), vocab2.intToString.values());
+
+ Assert.assertNotEquals(vocab2.size(), numBuiltInSymbols);
+ // Assert.assertEquals(vocab2.getWord(Vocabulary.UNKNOWN_WORD), Vocabulary.UNKNOWN_WORD_STRING);
+
+ // Assert.assertEquals(vocab2.getID("sample"), Vocabulary.UNKNOWN_WORD);
+ // Assert.assertEquals(vocab2.getID(null), Vocabulary.UNKNOWN_WORD);
+
+ // Assert.assertFalse(vocab2.terminalToInt.isEmpty());
+ // Assert.assertEquals(vocab2.terminalToInt.size(), this.numBuiltInTerminals);
+ // Assert.assertTrue(vocab2.isFixed);
+ }
+
+ @Test
+ public void verifyWordIDs() throws IOException {
+
+ // Adam Lopez's example...
+ String corpusString = "it makes him and it mars him , it sets him on and it takes him off .";
+ // String queryString = "it persuades him and it disheartens him";
+
+ String sourceFileName;
+ {
+ File sourceFile = File.createTempFile("source", new Date().toString());
+ PrintStream sourcePrintStream = new PrintStream(sourceFile, "UTF-8");
+ sourcePrintStream.println(corpusString);
+ sourcePrintStream.close();
+ sourceFileName = sourceFile.getAbsolutePath();
+ }
+
+ Vocabulary vocab = new Vocabulary();
+ // Vocabulary.initializeVocabulary(sourceFileName, vocab, true);
+
+// Assert.assertEquals(vocab.getWords(Vocabulary.id("it")), "it");
+// Assert.assertEquals(vocab.getWord(vocab.getID("makes")), "makes");
+// Assert.assertEquals(vocab.getWord(vocab.getID("him")), "him");
+// Assert.assertEquals(vocab.getWord(vocab.getID("and")), "and");
+// Assert.assertEquals(vocab.getWord(vocab.getID("mars")), "mars");
+// Assert.assertEquals(vocab.getWord(vocab.getID(",")), ",");
+// Assert.assertEquals(vocab.getWord(vocab.getID("sets")), "sets");
+// Assert.assertEquals(vocab.getWord(vocab.getID("on")), "on");
+// Assert.assertEquals(vocab.getWord(vocab.getID("takes")), "takes");
+// Assert.assertEquals(vocab.getWord(vocab.getID("off")), "off");
+
+ // Assert.assertEquals(vocab.getWord(vocab.getID("persuades")), Vocabulary.UNKNOWN_WORD_STRING);
+ // Assert.assertEquals(vocab.getWord(vocab.getID("disheartens")), Vocabulary.UNKNOWN_WORD_STRING);
+ }
+
+ @SuppressWarnings("static-access")
+ @Test(enabled=false)
+ public void loadVocabFromFile() {
+
+ String filename = "data/tiny.en";
+ int numSentences = 5; // Should be 5 sentences in tiny.en
+ int numWords = 89; // Should be 89 words in tiny.en
+ int numUniqWords = 60; // Should be 60 unique words in tiny.en
+
+ Vocabulary vocab = new Vocabulary();
+ Vocabulary vocab2 = new Vocabulary();
+
+ Assert.assertTrue(vocab.equals(vocab2));
+ Assert.assertTrue(vocab2.equals(vocab));
+ Assert.assertEquals(vocab, vocab2);
+
+ try {
+ vocab.read(new File(getClass().getClassLoader().getResource(filename).getFile()));
+ //int[] result = Vocabulary.initializeVocabulary(filename, vocab, true);
+ Assert.assertNotNull(vocab);
+ Assert.assertEquals(vocab.size(), 2);
+ //Assert.assertEquals(vocab.getWords(numWords), numWords);
+ // Assert.assertEquals(result[1], numSentences);
+
+ //Assert.assertTrue(vocab.isFixed);
+ Assert.assertEquals(Vocabulary.size(), numUniqWords+numBuiltInSymbols);
+
+ } catch (IOException e) {
+ Assert.fail("Error processing " + filename +"; Reason: " + e);
+ }
+
+ Assert.assertFalse(vocab.equals(vocab2));
+
+ try {
+ vocab2.read(new File(filename));
+ //int[] result = Vocabulary.initializeVocabulary(filename, vocab2, true);
+ Assert.assertNotNull(vocab2);
+ Assert.assertEquals(vocab2.size(), 2);
+ // Assert.assertEquals(result[0], numWords);
+ // Assert.assertEquals(result[1], numSentences);
+
+ // Assert.assertTrue(vocab2.isFixed);
+ Assert.assertEquals(Vocabulary.size(), numUniqWords+numBuiltInSymbols);
+
+ } catch (IOException e) {
+ Assert.fail("Could not load file " + filename);
+ }
+
+ Assert.assertEquals(vocab, vocab2);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/decoder/ArtificialGrammarAndCorpusCreater.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/ArtificialGrammarAndCorpusCreater.java b/joshua-core/src/test/java/org/apache/joshua/decoder/ArtificialGrammarAndCorpusCreater.java
new file mode 100644
index 0000000..5cc5996
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/ArtificialGrammarAndCorpusCreater.java
@@ -0,0 +1,130 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder;
+
+import java.io.BufferedWriter;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.joshua.util.FileUtility;
+
+public class ArtificialGrammarAndCorpusCreater {
+
+ private static final String JOSHUA_RULE_SEPARATOR = " ||| ";
+ private static final String ARTIFICAL_TERMINAL_RULE1 = "[T1]" + JOSHUA_RULE_SEPARATOR + "garcon"
+ + JOSHUA_RULE_SEPARATOR + "boy" + JOSHUA_RULE_SEPARATOR + "0.5 0.4";
+ private static final String ARTIFICAL_TERMINAL_RULE2 = "[T2]" + JOSHUA_RULE_SEPARATOR + "fille"
+ + JOSHUA_RULE_SEPARATOR + "girl" + JOSHUA_RULE_SEPARATOR + "0.5 0.4";
+ private static final String ARTIFICAL_TERMINAL_RULE3 = "[T3]" + JOSHUA_RULE_SEPARATOR + "garcon"
+ + JOSHUA_RULE_SEPARATOR + "mister" + JOSHUA_RULE_SEPARATOR + "0.5 0.4";
+ private static final String ARTIFICAL_TERMINAL_RULE4 = "[T4]" + JOSHUA_RULE_SEPARATOR + "fille"
+ + JOSHUA_RULE_SEPARATOR + "woman" + JOSHUA_RULE_SEPARATOR + "0.5 0.4";
+ private static final String ARTIFICAL_TERMINAL_RULE5 = "[T5]" + JOSHUA_RULE_SEPARATOR + "fille"
+ + JOSHUA_RULE_SEPARATOR + "lady" + JOSHUA_RULE_SEPARATOR + "0.5 0.4";
+ private static final String ARTIFICAL_NONTERTERMINAL_RULE1 = "[NT1]" + JOSHUA_RULE_SEPARATOR
+ + "le [T1,1] aime la [T2,2]" + JOSHUA_RULE_SEPARATOR + "the [T1,1] loves the [T2,2]"
+ + JOSHUA_RULE_SEPARATOR + "0.5 0.4";
+ private static final String ARTIFICAL_NONTERTERMINAL_RULE_INVERTED = "[NT1]"
+ + JOSHUA_RULE_SEPARATOR + "le [T1,1] aime la [T2,2]" + JOSHUA_RULE_SEPARATOR
+ + "the [T2,2] loves the [T1,1]" + JOSHUA_RULE_SEPARATOR + "0.5 0.4";
+ private static final String ARTIFICAL_TERMINAL_RULE6 = "[T6]" + JOSHUA_RULE_SEPARATOR + "garcon"
+ + JOSHUA_RULE_SEPARATOR + "sir" + JOSHUA_RULE_SEPARATOR + "0.5 0.4";
+
+ private static final String GLUE_RULE_BEGIN = "[GOAL] ||| <s> ||| <s> ||| 0";
+ private static final String GLUE_RULE_NT = "[GOAL] ||| [GOAL,1] [NT1,2] ||| [GOAL,1] [NT1,2] ||| -1";
+ private static final String GLUE_RULE_END = "[GOAL] ||| [GOAL,1] </s> ||| [GOAL,1] </s> ||| 0";
+
+ private static final String TEST_SENTENCE1 = "le garcon aime la fille";
+
+ private static final List<String> getArtificalGrammarsList1() {
+ List<String> result = Arrays.asList(ARTIFICAL_TERMINAL_RULE1, ARTIFICAL_TERMINAL_RULE2,
+ ARTIFICAL_TERMINAL_RULE3, ARTIFICAL_TERMINAL_RULE4, ARTIFICAL_TERMINAL_RULE5,
+ ARTIFICAL_TERMINAL_RULE6, ARTIFICAL_NONTERTERMINAL_RULE1);
+ return result;
+ }
+
+ private static List<String> getArtificalGrammarsList2() {
+ List<String> result = new ArrayList<String>(getArtificalGrammarsList1());
+ result.add(ARTIFICAL_NONTERTERMINAL_RULE_INVERTED);
+ return result;
+ }
+
+ private static final List<String> ARTIFICIAL_GLUE_GRAMMAR_RULES_LIST = Arrays.asList(
+ GLUE_RULE_BEGIN, GLUE_RULE_NT, GLUE_RULE_END);
+
+ private final String mainGrammarFilePath;
+ private final String glueGrammarFilePath;
+ private final String testSentencesFilePath;
+
+ private ArtificialGrammarAndCorpusCreater(String mainGrammarFilePath, String glueGrammarFilePath,
+ String testSentencesFilePath) {
+ this.mainGrammarFilePath = mainGrammarFilePath;
+ this.glueGrammarFilePath = glueGrammarFilePath;
+ this.testSentencesFilePath = testSentencesFilePath;
+ }
+
+ public static ArtificialGrammarAndCorpusCreater createArtificialGrammarAndCorpusCreater(
+ String mainGrammarFilePath, String glueGrammarFilePath, String testSentencesFilePath) {
+ return new ArtificialGrammarAndCorpusCreater(mainGrammarFilePath, glueGrammarFilePath,
+ testSentencesFilePath);
+ }
+
+ private static final void writeFile(String filePath, List<String> lines) {
+ BufferedWriter outputWriter = null;
+ try {
+ outputWriter = new BufferedWriter(new FileWriter(filePath));
+ for (int i = 0; i < lines.size() - 1; i++) {
+ outputWriter.write(lines.get(i) + "\n");
+ }
+ if (!lines.isEmpty()) {
+ outputWriter.write(lines.get(lines.size() - 1));
+ }
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ } finally {
+ FileUtility.closeCloseableIfNotNull(outputWriter);
+ }
+ }
+
+ protected final void writeMainGrammar(boolean includeInvertingNonterminalRule) {
+ List<String> ruleList;
+ if(includeInvertingNonterminalRule)
+ {
+ ruleList = getArtificalGrammarsList2();
+ }
+ else{
+ ruleList = getArtificalGrammarsList1();
+ }
+
+ writeFile(mainGrammarFilePath,ruleList);
+ }
+
+ protected final void writeGlueGrammar() {
+ writeFile(glueGrammarFilePath, ARTIFICIAL_GLUE_GRAMMAR_RULES_LIST);
+ }
+
+ protected final void writeTestSentencesFile1() {
+ writeFile(testSentencesFilePath, Arrays.asList(TEST_SENTENCE1));
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/decoder/DecoderThreadTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/DecoderThreadTest.java b/joshua-core/src/test/java/org/apache/joshua/decoder/DecoderThreadTest.java
new file mode 100644
index 0000000..326ab23
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/DecoderThreadTest.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.Date;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+/**
+ * Unit tests for decoder thread.
+ *
+ * @author Lane Schwartz
+ * @version $LastChangedDate$
+ */
+public class DecoderThreadTest {
+
+ @Test
+ public void setup() {
+
+ String[] sourceSentences = {
+ "a b c d",
+ "a b c d",
+ "a b c d"
+ };
+
+ String[] targetSentences = {
+ "w x y z",
+ "w t u v",
+ "s x y z"
+ };
+
+ String[] alignmentLines = {
+ "0-0 1-1 2-2 3-3",
+ "0-0 1-1 2-2 3-3",
+ "0-0 1-1 2-2 3-3"
+ };
+
+ String[] testSentences = {
+ "a b c"
+ };
+
+ try {
+
+ // Set up source corpus
+ File sourceFile = File.createTempFile("source", new Date().toString());
+ PrintStream sourcePrintStream = new PrintStream(sourceFile, "UTF-8");
+ for (String sentence : sourceSentences) {
+ sourcePrintStream.println(sentence);
+ }
+ sourcePrintStream.close();
+ String sourceCorpusFileName = sourceFile.getAbsolutePath();
+
+// Vocabulary vocabulary = new Vocabulary();
+// int[] sourceLengths = Vocabulary.initializeVocabulary(sourceCorpusFileName, vocabulary, true);
+// Assert.assertEquals(sourceLengths.length, 2);
+// int numberOfSentences = sourceLengths[1];
+//
+// Corpus sourceCorpus = SuffixArrayFactory.createCorpusArray(sourceCorpusFileName, vocabulary, sourceLengths[0], sourceLengths[1]);
+
+
+ // Set up target corpus
+ File targetFile = File.createTempFile("target", new Date().toString());
+ PrintStream targetPrintStream = new PrintStream(targetFile, "UTF-8");
+ for (String sentence : targetSentences) {
+ targetPrintStream.println(sentence);
+ }
+ targetPrintStream.close();
+ String targetCorpusFileName = targetFile.getAbsolutePath();
+
+// int[] targetLengths = Vocabulary.initializeVocabulary(targetCorpusFileName, vocabulary, true);
+// Assert.assertEquals(targetLengths.length, sourceLengths.length);
+// for (int i=0, n=targetLengths.length; i<n; i++) {
+// Assert.assertEquals(targetLengths[i], sourceLengths[i]);
+// }
+//
+// Corpus targetCorpus = SuffixArrayFactory.createCorpusArray(targetCorpusFileName, vocabulary, targetLengths[0], targetLengths[1]);
+
+
+ // Construct alignments data structure
+ File alignmentsFile = File.createTempFile("alignments", new Date().toString());
+ PrintStream alignmentsPrintStream = new PrintStream(alignmentsFile, "UTF-8");
+ for (String sentence : alignmentLines) {
+ alignmentsPrintStream.println(sentence);
+ }
+ alignmentsPrintStream.close();
+ String alignmentFileName = alignmentsFile.getAbsolutePath();
+
+// AlignmentGrids grids = new AlignmentGrids(
+// new Scanner(alignmentsFile),
+// sourceCorpus,
+// targetCorpus,
+// numberOfSentences);
+
+
+ // Set up test corpus
+ File testFile = File.createTempFile("test", new Date().toString());
+ PrintStream testPrintStream = new PrintStream(testFile, "UTF-8");
+ for (String sentence : testSentences) {
+ testPrintStream.println(sentence);
+ }
+ testPrintStream.close();
+ String testFileName = testFile.getAbsolutePath();
+
+ // Filename of the extracted rules file.
+ String rulesFileName; {
+ File rulesFile = File.createTempFile("rules", new Date().toString());
+ rulesFileName = rulesFile.getAbsolutePath();
+ }
+
+ String joshDirName; {
+ File joshDir = File.createTempFile(new Date().toString(), "josh");
+ joshDirName = joshDir.getAbsolutePath();
+ joshDir.delete();
+ }
+
+
+// Compile compileJoshDir = new Compile();
+// compileJoshDir.setSourceCorpus(sourceCorpusFileName);
+// compileJoshDir.setTargetCorpus(targetCorpusFileName);
+// compileJoshDir.setAlignments(alignmentFileName);
+// compileJoshDir.setOutputDir(joshDirName);
+// compileJoshDir.execute();
+//
+// ExtractRules extractRules = new ExtractRules();
+// extractRules.setJoshDir(joshDirName);
+// extractRules.setTestFile(testFileName);
+// extractRules.setOutputFile(rulesFileName);
+// extractRules.execute();
+
+ } catch (IOException e) {
+ Assert.fail("Unable to write temporary file. " + e.toString());
+ }
+// } catch (ClassNotFoundException e) {
+// Assert.fail("Unable to extract rules. " + e.toString());
+// }
+ }
+
+ @Test
+ public void basicSuffixArrayGrammar() {
+
+ // Write configuration to temp file on disk
+ // String configFile;
+
+
+ // JoshuaDecoder decoder =
+ // JoshuaDecoder.getUninitalizedDecoder(configFile);
+
+
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/decoder/JoshuaDecoderTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/JoshuaDecoderTest.java b/joshua-core/src/test/java/org/apache/joshua/decoder/JoshuaDecoderTest.java
new file mode 100644
index 0000000..2a878f3
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/JoshuaDecoderTest.java
@@ -0,0 +1,83 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Scanner;
+
+import org.testng.Assert;
+import org.testng.annotations.Parameters;
+import org.testng.annotations.Test;
+
+/**
+ * Performs regression tests to verify that the decoder produces expected output
+ * on known data sets.
+ *
+ * @author Lane Schwartz
+ */
+public class JoshuaDecoderTest {
+
+ @Parameters({ "configFile", "sourceInput", "referenceOutput" })
+ @Test
+ public void regressionTest(String configFile, String sourceInput, String referenceOutput)
+ throws IOException {
+
+ File referenceFile = new File(referenceOutput);
+ File output = File.createTempFile("output", null);// ,
+ // referenceFile.getParentFile());
+
+ String[] args = { configFile, sourceInput, output.getAbsoluteFile().toString() };
+ JoshuaDecoder.main(args);
+
+ Scanner resultScanner = new Scanner(output);
+ Scanner refScanner = new Scanner(referenceFile);
+
+ while (resultScanner.hasNextLine() && refScanner.hasNextLine()) {
+
+ String resultLine = resultScanner.nextLine();
+ String refLine = refScanner.nextLine();
+
+ String[] resultParts = resultLine.split(" \\|\\|\\| ");
+ String[] refParts = refLine.split(" \\|\\|\\| ");
+
+ Assert.assertEquals(resultParts.length, 4);
+ Assert.assertEquals(refParts.length, 4);
+
+ Assert.assertEquals(Integer.parseInt(resultParts[0]), Integer.parseInt(refParts[0]));
+ Assert.assertEquals(resultParts[1], refParts[1]);
+
+ String[] resultFeatures = resultParts[2].split(" ");
+ String[] refFeatures = refParts[2].split(" ");
+
+ Assert.assertEquals(resultFeatures.length, 5);
+ Assert.assertEquals(refFeatures.length, 5);
+
+ float acceptableDelta = 0.001f;
+ for (int i = 0; i < refFeatures.length; i++) {
+ Assert.assertEquals(Float.valueOf(resultFeatures[i]), Float.valueOf(refFeatures[i]),
+ acceptableDelta);
+ }
+ }
+
+ resultScanner.close();
+ refScanner.close();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/decoder/TestConfigFileCreater.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/TestConfigFileCreater.java b/joshua-core/src/test/java/org/apache/joshua/decoder/TestConfigFileCreater.java
new file mode 100644
index 0000000..5399bab
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/TestConfigFileCreater.java
@@ -0,0 +1,184 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder;
+
+import java.io.BufferedWriter;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.List;
+import org.apache.joshua.util.FileUtility;
+
+public class TestConfigFileCreater {
+
+
+ protected static String LANGUAGE_MODEL_FILE_NAME = "lm.gz";
+ private static final String NL = "\n";
+ private static final Double NEW_FEATURES_WEIGHT = 0.2;
+
+ private final String testTempFilesFolderName;
+ private final String mainGrammarFileName;
+ private final String glueGrammarFileName;
+ private final List<Double> phraseTableWeights;
+ private final boolean useSoftSyntacticDecoding;
+ private final boolean switchOfPruning;
+
+ private TestConfigFileCreater(String testTemFilesFolderName, String mainGrammarFileName,
+ String glueGrammarFileName, List<Double> phraseTableWeights,
+ boolean useSoftSyntacticDecoding, boolean switchOfPruning) {
+ this.testTempFilesFolderName = testTemFilesFolderName;
+ this.mainGrammarFileName = mainGrammarFileName;
+ this.glueGrammarFileName = glueGrammarFileName;
+ this.phraseTableWeights = phraseTableWeights;
+ this.useSoftSyntacticDecoding = useSoftSyntacticDecoding;
+ this.switchOfPruning = switchOfPruning;
+ }
+
+ public static TestConfigFileCreater createFeaturesTestConfigFileCreater(
+ String testTemFilesFolderName, String mainGrammarFileName, String glueGrammarFileName,
+
+ List<Double> phraseTableWeights, boolean useSoftSyntacticDecoding, boolean switchOfPruning) {
+ return new TestConfigFileCreater(testTemFilesFolderName, mainGrammarFileName,
+ glueGrammarFileName, phraseTableWeights, useSoftSyntacticDecoding, switchOfPruning);
+ }
+
+ private final String createGlueGrammarFileSpecificationLine() {
+ return "tm = thrax glue -1 " + "./" + testTempFilesFolderName + "/" + glueGrammarFileName;
+ }
+
+ private final String createMainGrammarFileSpecificationLine() {
+ return "tm = thrax pt 12 " + "./" + testTempFilesFolderName + "/" + mainGrammarFileName;
+ }
+
+ private static String getFeatureSwitchOnString(String featureFunctionName) {
+ return "feature-function = " + featureFunctionName;
+ }
+
+ public String getPruningSpecification() {
+ if (switchOfPruning) {
+ return "pop-limit = 0" + NL;
+ } else {
+ return "pop-limit = 100" + NL;
+ }
+ }
+
+ // Large String containing the mostly static, partly dynamic generated mose config
+ // file contents used for the test
+ private final String getJoshuaConfigFileFirstPart(boolean useSoftSyntacticDecoding) {
+ String result = "lm = kenlm 5 false false 100 " + createFullPath(LANGUAGE_MODEL_FILE_NAME) + NL
+ + createMainGrammarFileSpecificationLine() + NL + createGlueGrammarFileSpecificationLine()
+ + NL + "mark_oovs=false" + NL + "#tm config" + NL + "default_non_terminal = OOV" + NL
+ + "goalSymbol = GOAL" + NL + "#pruning config" + NL + getPruningSpecification()
+ + JoshuaConfiguration.SOFT_SYNTACTIC_CONSTRAINT_DECODING_PROPERTY_NAME + " = "
+ + useSoftSyntacticDecoding + NL + "#nbest config" + NL + "use_unique_nbest = true" + NL
+
+ + "top_n = 100" // + NL +
+ // "feature-function = OOVPenalty"
+ + NL + "feature-function = WordPenalty";
+ return result;
+ }
+
+ private final String createPhraseTableSpecificationString() {
+ String result = "";
+ for (int i = 0; i < phraseTableWeights.size(); i++) {
+ double phraseTableWeight = phraseTableWeights.get(i);
+ result += "tm_pt_" + i + " " + phraseTableWeight + NL;
+ }
+ return result;
+ }
+
+ private final String getMosesConfigFilePart2() {
+ String retsult = "###### model weights" + NL + "#lm order weight" + NL
+ + "WordPenalty -3.0476045270236662" + NL + createPhraseTableSpecificationString()
+ + "lm_0 1.3200621467242506"
+ // "#phrasemodel owner column(0-indexed)"
+ + NL + "tm_glue_0 1" + NL + "oovpenalty -100.0" + NL;
+ return retsult;
+ }
+
+ // private static final int NO_PHRASE_WEIGTHS = 22;
+
+ /*
+ * private static String createPhraseWeightsSpecification() { String result =
+ * "#phrasemodel owner column(0-indexed) weight" + NL; for (int i = 0; i < NO_PHRASE_WEIGTHS; i++)
+ * { result += "tm_pt_" + i + 0.5; } return result; }
+ */
+
+ private static String createFeatureWeightSpecifications(List<String> featureNames,
+ double featureWeight) {
+ String result = "";
+ for (String featureName : featureNames) {
+ result += featureName + " " + featureWeight + "\n";
+ }
+ return result;
+ }
+
+ protected String createJoshuaConfigFileContentsWithExtraFeatures(String featureFunctionName,
+ List<String> featureNames) {
+ String result = createJoshuaConfigFileContents(featureFunctionName);
+ result += createFeatureWeightSpecifications(featureNames, NEW_FEATURES_WEIGHT);
+ return result;
+ }
+
+ protected String createJoshuaConfigFileContents(String featureFunctionName) {
+ String result = getJoshuaConfigFileFirstPart(useSoftSyntacticDecoding);
+ result += NL + getFeatureSwitchOnString(featureFunctionName) + NL;
+ result += getMosesConfigFilePart2();
+ return result;
+ }
+
+ protected String createJoshuaConfigFileContents() {
+ String result = getJoshuaConfigFileFirstPart(useSoftSyntacticDecoding);
+ result += NL;
+ result += getMosesConfigFilePart2();
+ return result;
+ }
+
+ protected static void writeContents(String filePath, String contents) {
+ BufferedWriter outputWriter = null;
+ try {
+ outputWriter = new BufferedWriter(new FileWriter(filePath));
+ outputWriter.write(contents);
+ } catch (IOException e) {
+ e.printStackTrace();
+ throw new RuntimeException(e);
+ } finally {
+ FileUtility.closeCloseableIfNotNull(outputWriter);
+ }
+ }
+
+ String createFullPath(String fileName) {
+ return testTempFilesFolderName + "/" + fileName;
+ }
+
+ protected void writeBasicJoshuaConfigFile(String configFileName) {
+ writeContents(createFullPath(configFileName), createJoshuaConfigFileContents());
+ }
+
+ protected void writeBasicJoshuaConfigFile(String configFileName, String featureFunctionName) {
+ writeContents(createFullPath(configFileName),
+ createJoshuaConfigFileContents(featureFunctionName));
+ }
+
+ protected void writeJoshuaExtraFeaturesConfigFile(String configFileName,
+ String featureFunctionName, List<String> featureNames) {
+ TestConfigFileCreater.writeContents(createFullPath(configFileName),
+ createJoshuaConfigFileContentsWithExtraFeatures(featureFunctionName, featureNames));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/decoder/TranslationsTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/TranslationsTest.java b/joshua-core/src/test/java/org/apache/joshua/decoder/TranslationsTest.java
new file mode 100644
index 0000000..9d2cb34
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/TranslationsTest.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder;
+
+import static org.testng.Assert.*;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.Charset;
+
+import org.testng.annotations.Test;
+import org.testng.annotations.BeforeTest;
+import org.apache.joshua.decoder.io.TranslationRequestStream;
+import org.testng.annotations.AfterTest;
+
+public class TranslationsTest {
+ private final JoshuaConfiguration joshuaConfiguration = new JoshuaConfiguration();
+ @BeforeTest
+ public void beforeTest() {
+ }
+
+ @AfterTest
+ public void afterTest() {
+ }
+
+
+ @Test(enabled = false)
+ public void Translations() {
+ throw new RuntimeException("Test not implemented");
+ }
+
+ /**
+ * Test method for {@link joshua.decoder.io.TranslationRequest#next()}.
+ */
+ @Test(enabled = false)
+ public void testNext() {
+ fail("Not yet implemented");
+ }
+
+ @Test(enabled = false)
+ public void iterator() {
+ throw new RuntimeException("Test not implemented");
+ }
+
+ // @Test(expectedExceptions = TestException.class)
+ @Test(enabled = false)
+ public void next() {
+ byte[] data = "1\n2\n".getBytes();
+ ByteArrayInputStream input = new ByteArrayInputStream(data);
+ TranslationRequestStream request = new TranslationRequestStream(
+ new BufferedReader(new InputStreamReader(input, Charset.defaultCharset())), joshuaConfiguration);
+ Translations translations = new Translations(request);
+ assertEquals(translations.next().getSourceSentence().source(), "1");
+ // Remove the next two.
+ assertEquals(translations.next().getSourceSentence().source(), "2");
+ // Should throw exception
+ translations.next();
+ translations.next();
+ }
+
+ @Test(enabled = false)
+ public void record() {
+ throw new RuntimeException("Test not implemented");
+ }
+
+ @Test(enabled = false)
+ public void remove() {
+ throw new RuntimeException("Test not implemented");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/decoder/ff/ArityPhrasePenaltyFFTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/ff/ArityPhrasePenaltyFFTest.java b/joshua-core/src/test/java/org/apache/joshua/decoder/ff/ArityPhrasePenaltyFFTest.java
new file mode 100644
index 0000000..9899298
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/ff/ArityPhrasePenaltyFFTest.java
@@ -0,0 +1,64 @@
+///*
+// * Licensed to the Apache Software Foundation (ASF) under one
+// * or more contributor license agreements. See the NOTICE file
+// * distributed with this work for additional information
+// * regarding copyright ownership. The ASF licenses this file
+// * to you under the Apache License, Version 2.0 (the
+// * "License"); you may not use this file except in compliance
+// * with the License. You may obtain a copy of the License at
+// *
+// * http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing,
+// * software distributed under the License is distributed on an
+// * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// * KIND, either express or implied. See the License for the
+// * specific language governing permissions and limitations
+// * under the License.
+// */
+//package org.apache.joshua.decoder.ff;
+//
+//import org.apache.joshua.decoder.ff.tm.BilingualRule;
+//import org.apache.joshua.decoder.ff.tm.MonolingualRule;
+//import org.apache.joshua.decoder.ff.tm.Rule;
+//
+//import org.testng.Assert;
+//import org.testng.annotations.Test;
+//
+///**
+// * Unit tests for ArityPhrasePenaltyFF.
+// *
+// * @author Lane Schwartz
+// * @version $LastChangedDate$
+// */
+//public class ArityPhrasePenaltyFFTest {
+//
+// @Test
+// public void alpha() {
+// Assert.assertEquals(ArityPhrasePenaltyFF.ALPHA, - Math.log10(Math.E));
+// }
+//
+// @Test
+// public void estimate() {
+//
+// int featureID = 0;
+// double weight = 0.0;
+// int owner = MonolingualRule.DUMMY_OWNER;
+// int min = 1;
+// int max = 5;
+//
+// ArityPhrasePenaltyFF featureFunction = new ArityPhrasePenaltyFF(featureID, weight, owner, min, max);
+//
+// int lhs = -1;
+// int[] sourceRHS = {24, -1, 42, 738};
+// int[] targetRHS = {-1, 7, 8};
+// float[] featureScores = {-2.35f, -1.78f, -0.52f};
+// int arity = 1;
+//
+// Rule dummyRule = new BilingualRule(lhs, sourceRHS, targetRHS, featureScores, arity);
+//
+// Assert.assertEquals(featureFunction.estimateLogP(dummyRule, -1), ArityPhrasePenaltyFF.ALPHA);
+//
+// }
+//
+//}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/decoder/ff/lm/ArpaFileTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/ff/lm/ArpaFileTest.java b/joshua-core/src/test/java/org/apache/joshua/decoder/ff/lm/ArpaFileTest.java
new file mode 100644
index 0000000..233a6ed
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/ff/lm/ArpaFileTest.java
@@ -0,0 +1,226 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.lm;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.ff.lm.berkeley_lm.LMGrammarBerkeley;
+import org.apache.joshua.decoder.ff.lm.buildin_lm.TrieLM;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+/**
+ * Unit tests for testing ARPA language model class.
+ *
+ * @author Lane Schwartz
+ */
+public class ArpaFileTest {
+
+ String arpaFileName;
+
+ Vocabulary vocab;
+
+ @Test
+ public void setup() {
+
+ vocab = new Vocabulary();
+ vocab.id("a");
+ vocab.id("because");
+ vocab.id("boycott");
+ vocab.id("of");
+ vocab.id("parliament");
+ vocab.id("potato");
+ vocab.id("resumption");
+ vocab.id("the");
+
+ try {
+ File file = File.createTempFile("testLM", "arpa");
+ PrintStream out = new PrintStream(file, "UTF-8");
+
+ out.println();
+ out.println("\\data\\");
+ out.println("ngram 1=8");
+ out.println("ngram 2=4");
+ out.println("ngram 3=1");
+ out.println();
+
+ out.println("\\1-grams:");
+ out.println("-1.992672 a -0.1195484");
+ out.println("-2.713723 because -0.4665429");
+ out.println("-4.678545 boycott -0.0902521");
+ out.println("-1.609573 of -0.1991907");
+ out.println("-3.875917 parliament -0.1274891");
+ out.println("-9.753210 potato");
+ out.println("-4.678545 resumption -0.07945678");
+ out.println("-1.712444 the -0.1606644");
+
+ out.println();
+ out.println("\\2-grams:");
+ out.println("-0.3552987 because of -0.03083654");
+ out.println("-1.403534 of a");
+ out.println("-0.7507797 of the -0.05237135");
+ out.println("-0.7266324 resumption of");
+ out.println("-3.936147 the resumption");
+
+ out.println();
+ out.println("\\3-grams:");
+ out.println("-0.6309999 because of the");
+ out.println();
+
+ out.println("\\end\\");
+
+ out.close();
+ this.arpaFileName = file.getAbsolutePath();
+
+ } catch (IOException e) {
+ Assert.fail("Unable to create temporary file: " + e.toString());
+ }
+
+ }
+
+ @Test(dependsOnMethods = { "setup" })
+ public void testOrder() {
+ ArpaFile arpaFile = new ArpaFile(arpaFileName, vocab);
+
+ try {
+ Assert.assertEquals(arpaFile.getOrder(), 3);
+ } catch (FileNotFoundException e) {
+ Assert.fail(e.toString());
+ }
+ }
+
+ @Test(dependsOnMethods = { "setup" })
+ public void testIteration() {
+
+ ArpaFile arpaFile = new ArpaFile(arpaFileName, vocab);
+
+ Map<Integer, Integer> counts = new HashMap<Integer, Integer>();
+
+ boolean iterationOccurred = false;
+
+ for (ArpaNgram ngram : arpaFile) {
+
+ iterationOccurred = true;
+
+ int order = ngram.order();
+ // System.err.println("Order = " + order);
+
+ int count;
+ if (counts.containsKey(order)) {
+ count = counts.get(order) + 1;
+ } else {
+ count = 1;
+ }
+
+ counts.put(order, count);
+
+ }
+
+ Assert.assertTrue(iterationOccurred);
+
+ Assert.assertTrue(counts.containsKey(1));
+ Assert.assertTrue(counts.containsKey(2));
+ Assert.assertTrue(counts.containsKey(3));
+
+ Assert.assertEquals((int) counts.get(1), 8);
+ Assert.assertEquals((int) counts.get(2), 5);
+ Assert.assertEquals((int) counts.get(3), 1);
+
+ }
+
+ @Test(dependsOnMethods = { "setup" })
+ public void testSize() {
+ ArpaFile arpaFile = new ArpaFile(arpaFileName, vocab);
+
+ Assert.assertEquals(arpaFile.size(), 14);
+ }
+
+ @Test(dependsOnMethods = { "setup", "testIteration" })
+ public void testChildren() throws FileNotFoundException {
+ ArpaFile arpaFile = new ArpaFile(arpaFileName, vocab);
+
+ TrieLM lm = new TrieLM(arpaFile);
+ // System.err.println(lm.getChildren().size());
+ Assert.assertNotSame(lm.getChildren().size(), 0);
+ }
+
+ @Test(dependsOnMethods = { "setup", "testIteration", "testChildren" })
+ public void testTrie() throws FileNotFoundException {
+ ArpaFile arpaFile = new ArpaFile(arpaFileName, vocab);
+
+ TrieLM lm = new TrieLM(arpaFile);
+
+ testLm(lm);
+
+ }
+
+ @Test(dependsOnMethods = { "setup", "testIteration", "testChildren" })
+ public void testBerkeley() throws FileNotFoundException {
+
+ LMGrammarBerkeley lm = new LMGrammarBerkeley(3, arpaFileName);
+
+ testLm(lm);
+
+ }
+
+ /**
+ * @param lm
+ */
+ private void testLm(NGramLanguageModel lm) {
+ // Test unigrams known to be in the language model
+// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("a")), -1.992672, 0.000001f);
+// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("because")), -2.713723, 0.000001f);
+// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("boycott")), -4.678545, 0.000001f);
+// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("of")), -1.609573, 0.000001f);
+// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("parliament")), -3.875917, 0.000001f);
+// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("potato")), -9.753210, 0.000001f);
+// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("resumption")), -4.678545, 0.000001f);
+// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("the")), -1.712444, 0.000001f);
+
+ // Test unigrams known to NOT be in the language model
+// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("banana")), -JoshuaConfiguration.lm_ceiling_cost, 0.000001f);
+
+ // Test bigrams known to be in the language model
+// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("because of")), -0.3552987, 0.000001f);
+// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("of the")), -0.7507797, 0.000001f);
+// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("resumption of")), -0.7266324, 0.000001f);
+// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("the resumption")), -3.936147, 0.000001f);
+
+ // Test trigrams known to be in the language model
+// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("because of the")), -0.6309999f, 0.000001f);
+
+ // Test bigrams know to NOT be in the language model (but the unigrams are)
+// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("a boycott")), -4.678545f + -0.1195484f, 0.000001f);
+// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("of parliament")), -3.875917f + -0.1991907f, 0.000001f);
+// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("the potato")), -9.753210f + -0.1606644f, 0.000001f);
+// Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("potato parliament")), -3.875917f + -0.0f, 0.000001f);
+
+ // Test trigrams know to NOT be in the language model (but the bigrams are)
+// int[] words = vocab.getIDs("because of a");
+// double f = lm.ngramLogProbability(words);
+// Assert.assertEquals(f, -1.403534f + -0.03083654f, 0.000001f);
+ // //Assert.assertEquals(lm.ngramLogProbability(vocab.getIDs("of the parliament")), -3.875917f + -0.05237135f, 0.000001f);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/decoder/ff/lm/LanguageModelFFTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/ff/lm/LanguageModelFFTest.java b/joshua-core/src/test/java/org/apache/joshua/decoder/ff/lm/LanguageModelFFTest.java
new file mode 100644
index 0000000..d541fdc
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/ff/lm/LanguageModelFFTest.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.lm;
+
+import static org.junit.Assert.*;
+import static org.hamcrest.CoreMatchers.*;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.Decoder;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.ff.FeatureVector;
+import org.apache.joshua.decoder.ff.state_maintenance.NgramDPState;
+
+public class LanguageModelFFTest {
+
+ private static final float WEIGHT = 0.5f;
+
+ private LanguageModelFF ff;
+
+ @Before
+ public void setUp() {
+ Decoder.resetGlobalState();
+
+ FeatureVector weights = new FeatureVector();
+ weights.set("lm_0", WEIGHT);
+ String[] args = {"-lm_type", "berkeleylm", "-lm_order", "2", "-lm_file", "./src/test/resources/lm/berkeley/lm"};
+
+ JoshuaConfiguration config = new JoshuaConfiguration();
+ ff = new LanguageModelFF(weights, args, config);
+ }
+
+ @After
+ public void tearDown() {
+ Decoder.resetGlobalState();
+ }
+
+ @Test
+ public void givenNonStartSymbol_whenEstimateFutureCost_thenMultipleWeightAndLogProbabilty() {
+ int[] left = {3};
+ NgramDPState currentState = new NgramDPState(left, new int[left.length]);
+
+ float score = ff.languageModel.sentenceLogProbability(left, 2, 1);
+ assertEquals(-99.0f, score, 0.0);
+
+ float cost = ff.estimateFutureCost(null, currentState, null);
+ assertEquals(score * WEIGHT, cost, 0.0);
+ }
+
+ @Test
+ public void givenOnlyStartSymbol_whenEstimateFutureCost_thenZeroResult() {
+ int startSymbolId = Vocabulary.id(Vocabulary.START_SYM);
+ int[] left = {startSymbolId};
+ NgramDPState currentState = new NgramDPState(left, new int[left.length]);
+
+ float score = ff.languageModel.sentenceLogProbability(left, 2, 2);
+ assertEquals(0.0f, score, 0.0);
+
+ float cost = ff.estimateFutureCost(null, currentState, null);
+ assertEquals(score * WEIGHT, cost, 0.0);
+ }
+
+ @Test
+ public void givenStartAndOneMoreSymbol_whenEstimateFutureCost_thenMultipleWeightAndLogProbabilty() {
+ int startSymbolId = Vocabulary.id(Vocabulary.START_SYM);
+ assertThat(startSymbolId, not(equalTo(3)));
+ int[] left = {startSymbolId, 3};
+ NgramDPState currentState = new NgramDPState(left, new int[left.length]);
+
+ float score = ff.languageModel.sentenceLogProbability(left, 2, 2);
+ assertEquals(-100.752754f, score, 0.0f);
+
+ float cost = ff.estimateFutureCost(null, currentState, null);
+ assertEquals(score * WEIGHT, cost, 0.0f);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/LMBerkeleySentenceProbablityTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/LMBerkeleySentenceProbablityTest.java b/joshua-core/src/test/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/LMBerkeleySentenceProbablityTest.java
new file mode 100644
index 0000000..bcc1039
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/LMBerkeleySentenceProbablityTest.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.lm.berkeley_lm;
+
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Test;
+
+import edu.berkeley.nlp.lm.ArrayEncodedNgramLanguageModel;
+
+public class LMBerkeleySentenceProbablityTest {
+
+ @Test
+ public void verifySentenceLogProbability() {
+ LMGrammarBerkeley grammar = new LMGrammarBerkeley(2, "resources/berkeley_lm/lm");
+ grammar.registerWord("the", 2);
+ grammar.registerWord("chat-rooms", 3);
+ grammar.registerWord("<unk>", 0);
+
+ ArrayEncodedNgramLanguageModel<String> lm = grammar.getLM();
+ float expected =
+ lm.getLogProb(new int[] {}, 0, 0)
+ + lm.getLogProb(new int[] {0}, 0, 1)
+ + lm.getLogProb(new int[] {0, 2}, 0, 2)
+ + lm.getLogProb(new int[] {2, 3}, 0, 2)
+ + lm.getLogProb(new int[] {3, 0}, 0, 2);
+
+ float result = grammar.sentenceLogProbability(new int[] {0, 2, 3, 0}, 2, 0);
+ assertEquals(expected, result, 0.0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/LMGrammarBerkeleyTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/LMGrammarBerkeleyTest.java b/joshua-core/src/test/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/LMGrammarBerkeleyTest.java
new file mode 100644
index 0000000..e5b2d69
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/LMGrammarBerkeleyTest.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.lm.berkeley_lm;
+
+import static org.junit.Assert.assertEquals;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.junit.After;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameter;
+import org.junit.runners.Parameterized.Parameters;
+
+import org.apache.joshua.decoder.Decoder;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.Translation;
+import org.apache.joshua.decoder.segment_file.Sentence;
+
+/**
+ * Replacement for test/lm/berkeley/test.sh regression test
+ */
+@RunWith(value = Parameterized.class)
+public class LMGrammarBerkeleyTest {
+
+ private static final String INPUT = "the chat-rooms";
+ private static final String[] OPTIONS = "-v 0 -output-format %f".split(" ");
+
+ private JoshuaConfiguration joshuaConfig;
+ private Decoder decoder;
+
+ @Parameters
+ public static List<String> lmFiles() {
+ return Arrays.asList("resources/berkeley_lm/lm",
+ "resources/berkeley_lm/lm.gz",
+ "resources/berkeley_lm/lm.berkeleylm",
+ "resources/berkeley_lm/lm.berkeleylm.gz");
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ decoder.cleanUp();
+ }
+
+ @Parameter
+ public String lmFile;
+
+ @Test
+ public void verifyLM() {
+ joshuaConfig = new JoshuaConfiguration();
+ joshuaConfig.processCommandLineOptions(OPTIONS);
+ joshuaConfig.features.add("LanguageModel -lm_type berkeleylm -lm_order 2 -lm_file " + lmFile);
+ decoder = new Decoder(joshuaConfig, null);
+ String translation = decode(INPUT).toString();
+ assertEquals(lmFile, "tm_glue_0=2.000 lm_0=-7.153\n", translation);
+ }
+
+ private Translation decode(String input) {
+ final Sentence sentence = new Sentence(input, 0, joshuaConfig);
+ return decoder.decode(sentence);
+ }
+}
[08/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/hiero/reference.en.0
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/hiero/reference.en.0 b/joshua-core/src/test/resources/bn-en/hiero/reference.en.0
new file mode 100644
index 0000000..a91dd56
--- /dev/null
+++ b/joshua-core/src/test/resources/bn-en/hiero/reference.en.0
@@ -0,0 +1,100 @@
+rabindranath was born in a pirali brahmin family of kolkata .
+recently the relationship between india and united stated has improved .
+mathematics is , hence , the language of science .
+from this it can easily be understood that this metric will be frw metric .
+at the same time this novel had also indicated to the fall of the land basis feudalism in bengal .
+mujib and his party could attain the absolute majority in the elections .
+hitlar continued his work in the city of munich at bavariar .
+besides tux there are os-tan and some other characters representing linux but are not much popular .
+it explains the conventional rules for decision making followed by the decision maker in the field of sports in relation to the social background
+annual rainfall is 2540 millimeter .
+he provided his main keynote speech in the national conference of his democrat party in the year 2004 in the boston city of machechuest states .
+since the distribution of powers was in favor of the east pakistan if the population was taken into the account , therefore the west pakistan introduced a unique idea under the name of " one unit theory " whereby the entire pakistan was considered as one province .
+measurement theory .
+external connections
+videsh sanchar nigam limited bhavan of tata communications , this is a major telecom service provider in the city .
+that year on the 4th november , he became victorious in the general election and got elected as the 44th president of the united states .
+many indian species originated from gondwana born taxa .
+plays , novels , stories , and recently screenplays written by the british writers are acknowledged worldwide .
+on 1919 it was published in a magazine called swagat .
+in 2005 , the women 's tennis association tour 's tier-iii tournament , sunfeast open was held in netaji indoor stadium .
+several provisions are taken to avoid this possibility
+in lahore , a national conference of the opponent parties was held on 5th february , 1955 .
+bangladesh became the member of the organization of the islamic conference and islamic development bank .
+special class : world dictionary
+russia , france and israel are the main countries supplying arms to india and defense associates .
+this is our familiar imaginary unit which relegates theory of mathematics to concern itself with set of complex numbers from that of real numbers .
+</address>
+september
+according to this philosophy you can not disagree universe , though the logic has been accepted .
+france is the most important country in europe in agriculture ; it export mainly food crop , wine , cheese , and other agricultural product to europe and the world .
+arithmetic was prevalent in their mathematics .
+the countries are : france , hong kong , china , belgium , switzerland , germany , denmark , sweden , austria , czechoslovakia , argentina , italy , norway , hungary , yugoslavia , bulgaria , romania , greece , egypt , singapore , indonesia , thailand , japan , burma , holland , soviet russia , iran , iraq , and sri lanka .
+in the place of this basilica bank of england is located now .
+to the north and south of spain there are bay of biscay and the gibraltar respectively , morocco lies to south of gibraltar and the atlantic ocean is situated at the west and south-west part of this country .
+except that , in this situation , the inability of united nations to take decision quickly in emergency situation was realized .
+this was popularised by karl marx
+its subject is often borrowed from hindu mythology , medieval romances and news of social and political events .
+depending on the three measures , the age of the universe has been found to be 13.7 � 0.2 billion years .
+east russia is close by , which is situated on the opposite side of the sea of okhotsk and sea of japan .
+the indian national library in kolkata is the leading public library of the country .
+mycology
+the secretary-general of the un at present is ban ki-moon .
+the creator of minix was andrew tunnenbom who was a famous teacher of operating system designing .
+the times of india wrote that " it is absurd to compare it with any other indian cinema ... pather panchali is pure cinema " .
+after that in 1953 , during the month of may , nazrul and his wife pramila devi were sent to london for better treatment .
+it has got plain lands in its south and east sides and rough hills and mountains in its west and north sides .
+trademark
+lord wellesley , the governor general between 1797 \u2013 1805 , was largely responsible for the growth of the city
+complex numbers are indispensible for solving many important and real problems .
+an important consequence of the big bang is that the present state of the universe is completely different from its past and future states .
+windows millennium
+although rabindranath had subjugated a number of styles .
+labor economy
+britain was once the most powerful and advance economic power of the world .
+he is highly admired due to the movement against the military ruler of pakistan and the protest against the racial inequities in pakistan and moving the movement of bengalees towards independence .
+though it is thought that the meaning of internet and world wide web but actually they refer two completely different things .
+the two relevant polar coordinates of the z are r = 1
+november
+the first electronics mail used was in 1972 ad in the arpanet .
+the section of biology which discusses fungi and its practical source is called mycology .
+the water was carried up with some pur , this was actually a method of pulling up water by animals with an arrangement of ropes and buckets .
+these are tribal dance , folk dance , classical dance etc .
+the indian literature was spread earlier as orally and later as written
+his direction in his film " gana satru " in 1989 was weak and it was considered as satyajit 's comeback in making movie after returning from a long-lasting illness .
+mechanics .
+linux is different from various angle than the other right-protected operating systems such as windows and mac os .
+according to the language of the asian times ,
+open source means free distribution of computer software source code .
+dhaka in bangladesh online
+in first war world germany was defeated .
+but in order to understand this subject experiments are going on at present also .
+super h
+he was declared unfit for the armed force .
+threatening to boycott the assembly bhutto announced that , if yahya khan call mujib to form the government he will not do accept that government .
+and the word computer stands for the machine that does the work of counting .
+on 4th of july , 1776 these colonies introduced a declaration of independence .
+germany -lrb- in german language : deutschland -rrb- is a country of middle europe .
+christianity is the main religion of russia .
+but the development of the educational systems of the gols into roman styles was a bit slow .
+subject : foundation
+it also studies the financial and revenue strategies .
+among them there were : ' hoyto tomar pabo dekha ' -lrb- perhaps i will meet you -rrb- , ' ore e kon sneho-surdhani ' -lrb- what an affectionate music is this -rrb- .
+he died on 23 april 1992 .
+the medical reports of nazrul was sent to the renowned doctors of vienna at this time .
+apart from acting rani mukherjee is involved with many charitable organisations .
+on december 9 , 1974 , dhaka university honored him with d.lit degree , for his excellent contribution in the field of bengali culture and literature .
+durgapuja in kolkata is a tourist attraction too .
+but when the millions of east germans started to emigrate into the economically developed and democratic country of west germany , the government of east germany had built a wall in berlin and protected the borders of the country in 1962 .
+the first one is the first seven speechless minutes of the film which expresses the monotonous life of charu and second one is " the scene of the cradle in the garden " where charu faces with her love for amal .
+a group of 18th century thinkers and writers , developed the idea of the economy as a circular flow of income and output .
+foreplay is a set of intimate psychological and physical acts and sexual arousal activities before penetrating sex organ .
+this virus could only be found in the pigs which would cause influenza among the pigs .
+it constitutes mycelium together .
+russia is currently a democratic country .
+sex
+this state of intercourse is called orgasm
+several large empires had been built here in different course of history .
+macro economics
+computer interface is the most visible part to a computer user .
+details : the temporary government of bangladesh of 1971
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/hiero/reference.en.1
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/hiero/reference.en.1 b/joshua-core/src/test/resources/bn-en/hiero/reference.en.1
new file mode 100644
index 0000000..7f83452
--- /dev/null
+++ b/joshua-core/src/test/resources/bn-en/hiero/reference.en.1
@@ -0,0 +1,100 @@
+rabindranath was born in a " pirali brahmin " family in kolkata .
+recently , the relation between india and united states developed .
+therefore mathematics is the language of science .
+this is verygood machine which can be used
+simultaneously , a presage of decline of bengal 's landed feudal system is also found in this novel .
+mujibur and his party got an absolute majority in the election .
+hitler also worked at the city of munich in bavaria .
+other than tux there were many other characters like os tyan but these are not so popular .
+it explains the common rules for the decision maker in taking decisions in the playground while dealing with the opponents in the face of social condition .
+rain : total amount of rain in a year is 2540 milli meter .
+in 2008 he gave the keynote speech in national conference of democrat party organised in boston , massachusetts .
+as the division of power based on population favored east pakistn , west pakistan proposed a novel idea of ' one unit theory ' wherein entire west pakistan was considered as a province .
+measurement function
+outer communication
+tata communication bidesh sanchar nigam ltd bhavan , this is one of the unique telecommunication service in the city .
+he won in the national election on that year on 4th november and became as the 44th president of the united states .
+many indian races have been emerged from the texa community of the gondoana lands .
+dramas , novels , stories and recent screenplays of british authors are appreciated worldwide .
+it published in may , year 1919 in shwagat magazine .
+in year 2005 sunfeast open of women tennis association was organized in netaji indoor stadium .
+many steps are taken to eradicate this likely occurance .
+on february 5,1966 a national conference of the opposition parties was held in lahore .
+accepted membership of bangladesh organization of the islamic conference and islamic development bank .
+subject : world cell
+russia , france and italy are the main countries to supply arms and are helpful neighborhood countries .
+this is our known imaginary unit with the help of which mathematical theories get promoted to the set of complex numbers from the set of real numbers .
+<address>
+september
+this theory , however , does not oppose big bang theory rather supports it .
+france has historically been an important producer of agricultural products ; basically they export crops , wine , cheese and other agricultural products to europe and other parts of the world .
+arithmetic was the most important in their mathematics .
+these countries are : france , hong kong , china , belgium , switzerland , germany , denmark , sweden , austria , czechoslovakia , argentina , italy , norway , hungary , yugoslavia , bulgaria , romania , greece , egypt , singapore , indonesia , thailand , japan , myanmar , holland , soviet russia , iran , iraq and sri lanka .
+presently , the bank of london is situated in the place of basilica .
+the bay of biscay is in its north and strait of gibraltarto is in the south and morocco is in the south of the strait and the atlantic ocean in the west and south-west .
+besides , this also demonstrate the inability of the united nations to take quick decisions at the moment of emergency .
+it derives from the work of karl marx .
+its subjects are adopted sometimes from mythologies , sometimes from love affairs of the middle age and even from present day social and political events .
+on the basis of three measurements , the age has been determined as almost 13.7 � 0.2 billion years .
+there is east russia nearby , which is situated on the other side of sea of okhotsk and sea of japan .
+national library of india located in kolkata is one of the leading libraries in the country .
+mycology
+ban ki moon is the secretary general of the united nations .
+the creator of minix was andrew tanenbaum , a famous teacher of operating system design .
+in the times of india it was written that , " it is absurd to compare it with any other indian cinema . pather panchali is pure cinema " .
+after that in 1953 of may , nazrul and pamila devi were sent to london for treatment .
+there are wide flat lands in its southern and eastern side ; while there are wild hills and mountains in the western and northern side .
+trademark
+during the rule of lord wellesley -lrb- governor general 1797-1805 -rrb- there had been a considerable growth of cities .
+complex numbers are must for solving many important and real problems .
+one mention worthy result of big bang theory is that , the recent condition of the universe is completely different from that of the past and future .
+windows millennium
+rabindranath , however , mastered more than one style .
+labor economy .
+britain was once the prime and aggressive economic power of the world .
+he was appreciated for taking forward the protest of the bengali community towards the independence against pakistan military government and clearing the differences within the groups .
+it should be mentioned here that although the terms internet and world wide web are often used in everyday speech without much distinction these are not one and the same .
+the two accompanying polar co-ordinates of z are r = -pipe-
+november
+in 1972 bc at the mean time electronic mail is sent at first in orpanet .
+in that branch of biology where there is discussion about fungus is called as mycology .
+water was drawn from the river by a series of purs , an animal-powered rope and bucket mechanism .
+among them there are : tribal dance , folk dance , traditional or classical dance etc .
+the oldest literature in india became popular initially in oral form and then in writing .
+in 1989 he made ganashatru but his direction was comparably week and was considered as a retry of making film after a long suffering illness .
+mechanics
+linux is different in many ways from other licenses protected operating systems like windows and mac os .
+according to " asia times " : -
+open source means the free distribution of source code of author 's computer software
+dhaka on bangladesh online
+germans lost the 1st world war
+it only takes a few hours to be completed.the social customs are certain
+super h
+he was declared unfit for the military force .
+bhutto declared by threatening to boycott the assembly that he would not recognize that government if yahya khan would call mujib to form the government .
+and the meaning of " computer " is enumerator .
+on 4th july of 1776 , these colonies issued a declaration of independence .
+germany -lrb- deutschland in german -rrb- is a country of the middle europe .
+christianity is the main religion for russian .
+but the romanization of education was slower for the gaul .
+subject : gonu foundation
+it also studies effects of monetary policy and fiscal policy .
+these contain " haito tomar pabo dekha , " o re e kone sneha-surdhuni
+on 23th april 1992 his life came to an end .
+at this time nazrul 's medical report was sent to the famous doctors in vienna .
+other than acting rani mukherjee was found many times to get attached to many of the charitable institutions .
+on 9th december1974 , the bangladesh university honored him with d. lit as recognition of his contribution in bengali literacy and culture .
+kolkata is also the most attractive place for durga puja .
+but lakhs of german started to come in democratic western germany for its rich heritage and wealth from the year 1961 and so the government built up a wall in the southern germany and made the boundary of the country stronger .
+the first one is the silent seven minutes at the beginning which reflects the monotony of charu 's life and the second one is the " swinging in a cradle in the garden " where charu realised her love for amal .
+some thinkers and writers of 18th century by the cycling of income and production made development in economical thoughts .
+the sexual activities before intercourse is called foreplay .
+it was mainly seen in pigs by which the pigs were being infected .
+all of these together formed mycelium .
+russia is at present a democratic country .
+penis
+this is called organism .
+different vast empires were established here in different periods of history .
+micro economics
+to a user , the most visible form of an operating system is the computer interface .
+descriptive : temporary bangladesh government of 1971
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/hiero/reference.en.2
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/hiero/reference.en.2 b/joshua-core/src/test/resources/bn-en/hiero/reference.en.2
new file mode 100644
index 0000000..9588db6
--- /dev/null
+++ b/joshua-core/src/test/resources/bn-en/hiero/reference.en.2
@@ -0,0 +1,100 @@
+rabindranath was born of a pirali brahmin family of calcutta .
+recently the relationship between india and united states has improved .
+so mathematics is the language of science .
+it can be easily understood from it that this metric would be frw metric .
+at the same time indication of the end of bengal 's land centric feudalism is also obtained from this novel .
+mujib and his party got absolute majority in the election .
+hitler also started his work in the state named bavaria miunik
+other than task there are some other western and some other agencies of linux are also present but these are not very much introduced .
+it explains the rules of decision making in a game with one 's rival in the context of social condition .
+rain : yearly 2540 millimeter
+in 2004 he delivered the keynote speech in the national conference of the democrat party in the boston city of massachusetts .
+since a straightforward system of representation based on population would have concentrated political power in east pakistan , the west pakistani establishment came up with the " one unit " scheme , where all of west pakistan was considered one province
+quantum theory
+outer link
+bidesh sanchar nigam limited bhavan of tata communication is one of the main telecom service provider of the city .
+he won the national election on 4th november that year and was elected as the 44th us president .
+today , every one of us is in a problem of ageing in life.quick tips to look fresh and glamorous
+drama , novel , story and recently screenplay written by british writers are adored throughout the world .
+it was published in the sowgat in 1919
+sunfeast open , the tier three tournament of women \u2019 s tennis association tour was held in netaji indoor stadium in 2005 .
+to remove this possibility various steps are taken .
+on 5th february 1966 a religional meeting was arranged by the opposition party .
+he became the members of bangladesh organization of the islamic conference and islamic development bank .
+topics : encyclopedia
+russia , france and israel are the main suppliers of arms and also help in the defence system .
+this is our known imaginary number with which the mathematical theory to deduce complex number set from real number set .
+<address>
+september
+though the theory of big bang can not be opposed through this theory , rather it can be supported .
+france is the most important country of europe ; mainly it exports wine , paneer , food grain to whole europe and other part of the world .
+the dominance of arithmetics can be found in their mathematics .
+the countries are : france , hong kong , china , belgium , switzerland , germany , denmark , sweden , austria , czechoslovakia , argentina , italy , norway , hungary , yugoslavia , bulgaria , romania , greece , egypt , singapore , indonesia , thailand , japan , burma , holland , soviet russia , iran , iraq and sri lanka .
+at the place of that basilica , bank of england is now situated .
+north side of this country has bay of biskay , south side has gibralta and in south west side has atlantic ocean .
+moreover in this urgent situation , it declares the inability of leage of nations for fast decision .
+it is conducted through the works of karl marx .
+it \u2019 s content are taken sometime from mythology , love stories of middle ages or sometime from social and political incidents of recent time .
+the age of the universe that is derived based on three measurements is almost 13.7 � 0.2 billion years
+eastern russia is near , which is situated opposite of the okhotsk sea and japan sea .
+national library of india which is in kolkata is the oldest library .
+mycology
+the president of united nation is " wan ki moon " .
+the writer of minix was andrew tanenbaum , a famous operating system design teacher .
+the times of india wrote that " it is absurd to compare it with any other indian cinema ... pather panchali is pure cinema " .
+after this , in the month of may , 1953b.c nazrul and prameladevi were sent to london for treatment .
+in its south are vast plains , in the west and north are rough hills and mountains .
+trademark
+notable development of the city happens in time of lord wellesley -lrb- governor general 1797-1805 -rrb- .
+complex numbers are indispensable to solve many important and real life problems .
+one important result of the big bang is completely different situations of the past , present and future universe .
+windows millennium
+though rabindranath brought multiple saili within range .
+labour economics
+once upon a time in whole world britain has the most power full and fastest economy .
+he is greatly praised for organizing movement against the military rule of pakistan and protesting against racial discrimination by pakistan and for channelizing the movement of bengalis towards attaining freedom .
+it must be mentioned here that nevertheless most of us consider the internet and the www to be the same thing but these two words , infact , direct to completely different subjects .
+polar coordinate corresponding to z are r = -pipe-
+novewrmber
+electronic mail was sent for the first time with arpanet in 1972 .
+mycology is a science in which mushroom and its uses are described .
+water was collected from the river by the help of rope and domestic animals . rope was tied with a bucket and that was pulled up by domestic animals . this is an old process of collecting water from river
+tribal dance , folk dance , classical dance etc are few among them .
+early works of indian literature was practiced orally at first and later in written form .
+the first , ganashatru is considered the weakest of the three and endeavor to restore his energy to make films
+mechanics
+linux is different in many ways from other proprietary softwares like windows and mac .
+according to the asia times ,
+the meaning of open source is freely publishing the source code of computer software .
+dhaka in bangladesh online .
+germany lost the first world war .
+however , researches are even now going on to have a further understanding of the subject .
+not shown here ...
+he was declared unfit for the army .
+threatening to boycott assembly , he declared that he will not accept the government , if yahia khan calls mujib to form government
+and the meaning of the word ' computer ' is calculating machine .
+these colonies announced declaration of independence on july 4 , 1716 .
+germany -lrb- in german language : deutschland , do-yoch-lant -rrb- is a country of middle europe .
+christianity is the principal religion of russia .
+but the romanization of the education of gauls was slow moving .
+category : gnu foundation .
+monetary and fiscal policies are studied .
+among these were ' hayta tomar pabo dekha ' and ' ore e kon sneha-suradhuni ' .
+he died on 23 april 1992 .
+at this time , nazrul 's medical report was sent to famous surgeons in vienna .
+apart from acting rani mukherjee has kept herself attached with several charitable organizations in different times .
+on 9th december , year 1974 dhaka university gave him d.lit degree to give him respect .
+the durgapuja festival of calcutta is also a tourists ' attraction .
+but when lakhs of east germans started moving to the west german because it was economically developed and a republic , the east german government strengthened the boarders by constructing a wall in berlin in 1961 .
+first one is the speechless first seven minutes , which portrayed the monotony of charu 's life , and the second one is the scene of the " swing in the garden " , where charu first found his love for amol .
+in the 18th century a group of intellectuals improved the economic ideologies by incorporating the circle of income and manufacturing .
+before copulation the things done to get sex is called as sringer .
+it was generally got in between the pigs which attacked the pigs or its kinds .
+all this together form mycelium .
+presently russia is a democratic country .
+penis
+this condition is called as ragmochon .
+in different phases of history , a number of huge empires were established here .
+micro economics
+user can see the operating system as computer interface .
+in details : the temporary government of bangladesh of 1971 .
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/hiero/reference.en.3
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/hiero/reference.en.3 b/joshua-core/src/test/resources/bn-en/hiero/reference.en.3
new file mode 100644
index 0000000..20a8c75
--- /dev/null
+++ b/joshua-core/src/test/resources/bn-en/hiero/reference.en.3
@@ -0,0 +1,100 @@
+robindranath was born in a pirali brahman family .
+recently relation of india with the united states has improved .
+mathematics is thus the language of science .
+it easily understood from it that this metric is frw metric .
+the decline of imperilism is also reflected from this novel
+mujib and his party secured sweeping majority in the election .
+hitler continued to operate at the city of munich in bavaria .
+other than tux , there are o s tan and few more character to represent linux , but those are not very popular .
+this is not an identical task to complete with but to do some thing incredible and spectacular
+rainfall : annually 2580 mm
+in 2004 he give keynote speech in national assembly of democrat party in boston city of massachusetts province .
+division of political power according to the popoulation was favouring eest pakistan , therfore they invented an unique " one unit " theory in which the entire west pakistan was considered as a seperate state .
+measurement theory
+bringing together the relation with outside
+videsh sanchar nigam limited of tata communications is one of the main provider of cities telecommunication service .
+he won the national election on november , 4 in the same year and elected the 44th us president
+many indian tribes came up from the gondwana 's tribe teska .
+drama , story , literature , art of london is still very famous .
+in may 1919 it was published in ' sawgath ' magazine .
+the 2005 sunfeast open , the tier three tournament of the women 's tennis association tours , was organized in the netaji indoor stadium .
+to make this probability zero many kind pf protection is taken during sex .
+on 5th february , 1966 a national conference held of the all opposition parties in lahore .
+bangladesh take the membership of organization of the islamic conference and the islamic development bank .
+subject : encyclopedia
+russia , france and israel are the main ammunition supplier and security assisting countries to india
+this is our known imaginary unit through which the theories of the mathematics are developed to the set of complex numbers from the the set of real numbers .
+<address>
+september
+through this theory the big bang can not be protested but can be supported .
+france is one of the most important countries of europe in agriculture ; it mainly exports different agricultural products including wheat , wines , cottage cheese etc to europe and other parts of the world .
+their mathematics was basically airthmatic .
+the countries are : france , hongkong , china , belgium , switzerland , germany . denmark , sweden , austria , czechoslovakia , argentina , italy , norway , hungary , yugoslavia bulgaria , rumania , greece , singapore , indonesia , thiland , japan , burma , holland , soviet russia , iraq and srilanka
+in the place of this basilica now the bank of england is situated .
+on the north , it borders bay of biscay ; on the south , it borders gibraltar and morocco , and the atlantic ocean on the northwest and southwest .
+moreover , uno 's incapability to initiate quick action in the emergencies was also came into the lights .
+there is a scenario of togetherness spread at every every corner .
+the subject matters are taken sometimes from puran , sometimes from middle age love stories and sometimes from the modern social and political incidents .
+depending upon the three measurements the age that is calculated of the universe is 13.7 + - 0.2 billion years .
+in the near by there is east russia which is at the opposite banks of the okhotsk sea and japan sea .
+the national library of india is situated in kolkata and is the country \u2019 s leading public library .
+mycology
+the secretary general of the united nation is ban ki moon .
+the writer of minx was andrew tannenbaum , a renowned teacher of operating design .
+in the times of india it was written that " it is absurd to compare with any other indian cinema .. pather panchali is pure cinema " .
+thereafter in may 1953 nazrul and promila devi were sent to london for medical treatment .
+in south and east the vast plane and in west and north mountain are found in this state .
+trademark .
+during the ruling period of lord welesly -lrb- governor-general 1917-1805 -rrb- in the city mentionable increment took place .
+to solve several important and practical problems , the complex numbers are inevitable .
+a mentionable fact about great explosive theory is that , the present state of the earth is very much dissimilar from the past and the future state of the earth .
+windows millennium
+though rabindranath mastered more than one style .
+labour economics
+once britain was the prime advanced economic power of the world .
+revolt against the inequality in group and to enforce the bengali movement and also to revolt against the pakisthani military rule , for all this activity mujib is honored by all .
+it is necessary to mention that many people considers internet and world wide web as similar meaning words but in actuality they are different subjects .
+two polar coordinate of .z is r = -pipe- .
+november
+in 1972 the first electronic mail was sent on arpanet .
+mycology is the branch of biology concerned with the study of fungi and their use to humans .
+water used to be drawn from the river with thick rope and bucket pulled by the animals .
+among these are : tribal dance , folk dance , classical dance etc .
+the oldest literature of india were first in vogue through verbally and the in written .
+his direction was comparatively weak in his film ' ganashatru ' -lrb- the enemy of the people -rrb- in 1989 and this film has been considered as his trial of making films after recovering from long illness .
+powerful
+linux is different in many ways from other copy righted operating systems such as windows and mac oc
+according the news of the asia times ,
+however , the company 's army led by robert clive was able to recapture kolkata .
+dhaka is in bangladesh .
+germany was defeated in the first world war .
+but to understand this aspect the research works are progressing .
+superh
+he was declared unfit for the army .
+the day has since been observed as language movement day in bangladesh ,
+and the meaning of the word computer is machine to calculate .
+on the 4th july , 1776 these colonies had collectively issued a declaration of independence .
+germany -lrb- deutschland in german language -rrb- is a country of central europe .
+christ is the main religion of russia .
+but the speed of the romanization of gall 's education system was slow .
+subject : gonu foundation
+it also studies economic and revenue policy .
+among this are : hoyto tomar pap dekha , o hein a kon sneho sur dhoni .
+on the 23rd april , 1992 satyajit died .
+this time medical report of najrul was sent to the famous doctor of vienna .
+in addition to acting , rani mukerji remained involved with various charity organizations .
+for his great contribution to the bengali literature and culture , he was conferred on with the honorary d.lit. by the dhaka university on 9th december , 1974 .
+' durga puja ' is also a tourist attraction of kolkata city .
+but when many people started to migrate to the economically enriched and democratic west germany , east germany built a wall in berlin in 1961 and tightened the country 's border .
+the first one is the silent first seven minutes where the boring life of charu was shown and the second one is the scene of the cradle at the garden where charu realised her love for amal .
+in 18th century a group of intellectuals and writers made advancement in the field of economic thinking using the cyclic movement of earning and production .
+the sexually stimulating activities carried on before actual penetration are called foreplay .
+this virus was primarily found in pigs , which caused the flu in the pigs .
+these are collectively called a mycelium
+now russia is a democratic country .
+at that time , conflict between the french and the company occurred frequently .
+this is termed as orgasm .
+in different phases of history , multiple vast empires were established here .
+microeconomics
+to the user , computer interface is the most visible impersonation of operating systems .
+details : temporary government of bangladesh , 1971
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/hiero/test-berkeleylm.sh
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/hiero/test-berkeleylm.sh b/joshua-core/src/test/resources/bn-en/hiero/test-berkeleylm.sh
new file mode 100755
index 0000000..646d9a7
--- /dev/null
+++ b/joshua-core/src/test/resources/bn-en/hiero/test-berkeleylm.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -u
+
+cat input.bn | $JOSHUA/bin/joshua-decoder -m 1g -threads 2 -c joshua-berkeleylm.config > output 2> log
+
+# Extract the translations and model scores
+cat output | awk -F\| '{print $4 " ||| " $10}' > output.scores
+
+# Compare
+diff -u output.scores output.scores.berkeleylm.gold > diff
+
+if [ $? -eq 0 ]; then
+ rm -f output log output.scores diff
+ exit 0
+else
+ exit 1
+fi
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/hiero/test-classlm.sh
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/hiero/test-classlm.sh b/joshua-core/src/test/resources/bn-en/hiero/test-classlm.sh
new file mode 100755
index 0000000..43e1076
--- /dev/null
+++ b/joshua-core/src/test/resources/bn-en/hiero/test-classlm.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -u
+
+cat input.bn | $JOSHUA/bin/joshua-decoder -c joshua-classlm.config > output 2> log
+
+# Compare
+diff -u output output-classlm.gold > diff
+
+if [ $? -eq 0 ]; then
+ rm -f diff output log output.scores
+ exit 0
+else
+ exit 1
+fi
+
+
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/hiero/test-filter.sh
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/hiero/test-filter.sh b/joshua-core/src/test/resources/bn-en/hiero/test-filter.sh
new file mode 100755
index 0000000..e48a91d
--- /dev/null
+++ b/joshua-core/src/test/resources/bn-en/hiero/test-filter.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Tests dynamic sentence-level filtering.
+
+set -u
+
+cat input.bn | $JOSHUA/bin/joshua-decoder -m 1g -threads 2 -c joshua.config -filter-grammar > output.filter 2> log.filter
+
+# Extract the translations and model scores
+cat output.filter | awk -F\| '{print $4 " ||| " $10}' > output.scores
+
+# Compare
+diff -u output.scores output.scores.gold > diff
+
+if [ $? -eq 0 ]; then
+ rm -rf output.scores diff output.filter log.filter
+ exit 0
+else
+ exit 1
+fi
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/hiero/test.sh
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/hiero/test.sh b/joshua-core/src/test/resources/bn-en/hiero/test.sh
new file mode 100755
index 0000000..474936c
--- /dev/null
+++ b/joshua-core/src/test/resources/bn-en/hiero/test.sh
@@ -0,0 +1,35 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -u
+
+cat input.bn | $JOSHUA/bin/joshua-decoder -m 1g -threads 2 -c joshua.config > output 2> log
+
+# Extract the translations and model scores
+cat output | awk -F\| '{print $4 " ||| " $10}' > output.scores
+
+# Compare
+diff -u output.scores output.scores.gold > diff
+
+if [ $? -eq 0 ]; then
+ rm -f diff output log output.scores
+ exit 0
+else
+ exit 1
+fi
+
+
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/hiero/topN.pl
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/hiero/topN.pl b/joshua-core/src/test/resources/bn-en/hiero/topN.pl
new file mode 100755
index 0000000..d47e7b1
--- /dev/null
+++ b/joshua-core/src/test/resources/bn-en/hiero/topN.pl
@@ -0,0 +1,18 @@
+#!/usr/bin/perl
+
+my $N = shift || 10;
+my $field = -1;
+my $count = 0;
+while (<>) {
+ my @tokens = split;
+ if ($tokens[0] != $field) {
+ $count = 0;
+ print;
+ $field = $tokens[0];
+ } elsif ($count >= $N) {
+ next;
+ } else {
+ print;
+ }
+ $count++;
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/packed/.gitignore
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/packed/.gitignore b/joshua-core/src/test/resources/bn-en/packed/.gitignore
new file mode 100644
index 0000000..cb8243b
--- /dev/null
+++ b/joshua-core/src/test/resources/bn-en/packed/.gitignore
@@ -0,0 +1,3 @@
+output
+diff
+output.bleu
[34/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/WalkerFunction.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/WalkerFunction.java b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/WalkerFunction.java
new file mode 100644
index 0000000..811521c
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/WalkerFunction.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.hypergraph;
+
+/**
+ * Classes implementing this interface define a single function that is applied to each node. This
+ * interface is used for various walkers (ViterbiExtractor).
+ */
+public interface WalkerFunction {
+
+ /**
+ * Function that is applied to node at tail node index nodeIndex.
+ * nodeIndex indicates the index of node in the list of tailnodes for the
+ * outgoing edge.
+ * @param node the {{@link org.apache.joshua.decoder.hypergraph.HGNode} we
+ * wish to apply some Walker Function to.
+ * @param nodeIndex node in the list of tailnodes for the outgoing edge
+ */
+ void apply(HGNode node, int nodeIndex);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/WordAlignmentExtractor.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/WordAlignmentExtractor.java b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/WordAlignmentExtractor.java
new file mode 100644
index 0000000..04d0897
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/WordAlignmentExtractor.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.hypergraph;
+
+import static java.util.Collections.emptyList;
+
+import java.util.List;
+import java.util.Stack;
+
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.hypergraph.KBestExtractor.DerivationState;
+import org.apache.joshua.decoder.hypergraph.KBestExtractor.DerivationVisitor;
+
+/**
+ * This class enables extraction of word-level alignments from hypotheses.
+ * It implements two interfaces, WalkerFunction and DerivationVisitor.
+ * The former is for using the Viterbi walk function, the latter is for
+ * k-best extraction.
+ * Intermediate WordAlignmentStates are placed on a stack and/or merged down
+ * if possible.
+ * @author fhieber
+ */
+public class WordAlignmentExtractor implements WalkerFunction, DerivationVisitor {
+
+ private final Stack<WordAlignmentState> stack = new Stack<WordAlignmentState>();
+
+ /**
+ * Merges a state with the top of the stack if applicable or places it on top of the stack.
+ */
+ private void merge(final WordAlignmentState state) {
+ // if alignment state has no NTs left AND stack is not empty
+ // and parent state on stack still needs something to substitute
+ if (!stack.isEmpty()
+ && state.isComplete()) {
+ final WordAlignmentState parentState = stack.pop();
+ if (parentState.isComplete()) {
+ throw new IllegalStateException("Parent state already complete");
+ }
+ parentState.substituteIn(state);
+ merge(parentState);
+ } else {
+ stack.add(state);
+ }
+ }
+
+ /**
+ * Common entry point for WalkerFunction and DerivationVisitor.
+ */
+ private void extract(final Rule rule, final int spanStart) {
+ if (rule != null) {
+ merge(new WordAlignmentState(rule, spanStart));
+ }
+ }
+
+ /**
+ * entry for Viterbi walker. Calls word alignment extraction
+ * for best hyperedge from given node.
+ */
+ @Override
+ public void apply(HGNode node, int nodeIndex) {
+ extract(node.bestHyperedge.getRule(), node.i);
+ }
+
+ /**
+ * Visiting a node during k-best extraction is the same as
+ * apply() for Viterbi extraction but using the edge from
+ * the Derivation state.
+ */
+ @Override
+ public void before(final DerivationState state, final int level, int tailNodeIndex) {
+ extract(state.edge.getRule(), state.parentNode.i);
+ }
+
+ /**
+ * Nothing to do after visiting a node.
+ */
+ @Override
+ public void after(final DerivationState state, final int level, int tailNodeIndex) {}
+
+ /**
+ * Final word alignment without sentence markers
+ * or empty list if stack is empty.
+ * @return a final alignment list
+ */
+ public List<List<Integer>> getFinalWordAlignments() {
+ if (stack.isEmpty()) {
+ return emptyList();
+ }
+
+ if (stack.size() != 1) {
+ throw new RuntimeException(
+ String.format(
+ "Stack of WordAlignmentExtractor should contain only a single (last) element, but was size %d", stack.size()));
+ }
+
+ return stack.peek().toFinalList();
+ }
+
+ /**
+ * Returns a String representation of the (final) word alignment
+ * state on top of the stack.
+ * Empty string for empty stack.
+ */
+ @Override
+ public String toString() {
+ if (stack.isEmpty()) {
+ return "";
+ }
+
+ if (stack.size() != 1) {
+ throw new RuntimeException(
+ String.format(
+ "Stack of WordAlignmentExtractor should contain only a single (last) element, but was size %d", stack.size()));
+ }
+
+ return stack.peek().toFinalString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/WordAlignmentState.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/WordAlignmentState.java b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/WordAlignmentState.java
new file mode 100644
index 0000000..f057f23
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/WordAlignmentState.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.hypergraph;
+
+import static java.lang.Integer.MAX_VALUE;
+
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.Map;
+
+import org.apache.joshua.decoder.ff.tm.Rule;
+
+/**
+ * This class encodes a derivation state in terms of a list of alignment points.
+ * Whenever a child instance is substituted into the parent instance, we need to
+ * adjust source indexes of the alignments.
+ *
+ * @author fhieber
+ */
+public class WordAlignmentState {
+
+ /**
+ * each element in this list corresponds to a token on the target side of the
+ * rule. The values of the elements correspond to the aligned source token on
+ * the source side of the rule.
+ */
+ private List<AlignedSourceTokens> trgPoints;
+ private final int srcStart;
+ /** number of NTs we need to substitute. */
+ private int numNT;
+ /** grows with substitutions of child rules. Reaches original Rule span if substitutions are complete */
+ private int srcLength;
+
+ /**
+ * construct AlignmentState object from a virgin Rule and its source span.
+ * Determines if state is complete (if no NT present)
+ *
+ * @param rule the input Rule
+ * @param start the start index
+ */
+ public WordAlignmentState(final Rule rule, final int start) {
+ trgPoints = new LinkedList<AlignedSourceTokens>();
+ srcLength = rule.getFrench().length;
+ numNT = rule.getArity();
+ srcStart = start;
+ final Map<Integer, List<Integer>> alignmentMap = rule.getAlignmentMap();
+ final int[] nonTerminalSourcePositions = rule.getNonTerminalSourcePositions();
+ final int[] trg = rule.getEnglish();
+ // for each target index, create a TargetAlignmentPoint
+ for (int trgIndex = 0; trgIndex < trg.length; trgIndex++) {
+ final AlignedSourceTokens trgPoint = new AlignedSourceTokens();
+
+ if (trg[trgIndex] >= 0) { // this is a terminal symbol, check for alignment
+ if (alignmentMap.containsKey(trgIndex)) {
+ // add source indexes to TargetAlignmentPoint
+ for (int srcIdx : alignmentMap.get(trgIndex)) {
+ trgPoint.add(srcStart + srcIdx);
+ }
+ } else { // this target word is NULL-aligned
+ trgPoint.setNull();
+ }
+ } else { // this is a nonterminal ([X]) [actually its the (negative) index of the NT in the source]
+ trgPoint.setNonTerminal(); // mark as non-terminal
+ final int absoluteNonTerminalSourcePosition = srcStart + nonTerminalSourcePositions[Math.abs(trg[trgIndex]) - 1];
+ trgPoint.add(absoluteNonTerminalSourcePosition);
+ }
+ trgPoints.add(trgPoint);
+ }
+ }
+
+ /**
+ * if there are no more NonTerminals to substitute,
+ * this state is said to be complete
+ * @return true if complete
+ */
+ public boolean isComplete() {
+ return numNT == 0;
+ }
+
+ /**
+ * builds the final alignment string in the standard alignment format: src -
+ * trg. Sorted by trg indexes. Disregards the sentence markers.
+ * @return result string
+ */
+ public String toFinalString() {
+ final StringBuilder sb = new StringBuilder();
+ int t = 0;
+ for (AlignedSourceTokens pt : trgPoints) {
+ for (int s : pt) {
+ sb.append(String.format(" %d-%d", s-1, t-1)); // disregard sentence markers
+ }
+ t++;
+ }
+ final String result = sb.toString();
+ if (!result.isEmpty()) {
+ return result.substring(1);
+ }
+ return result;
+ }
+
+ /**
+ * builds the final alignment list.
+ * each entry in the list corresponds to a list of aligned source tokens.
+ * First and last item in trgPoints is skipped.
+ * @return a final alignment list
+ */
+ public List<List<Integer>> toFinalList() {
+ final List<List<Integer>> alignment = new ArrayList<List<Integer>>(trgPoints.size());
+ if (trgPoints.isEmpty()) {
+ return alignment;
+ }
+ final ListIterator<AlignedSourceTokens> it = trgPoints.listIterator();
+ it.next(); // skip first item (sentence marker)
+ while (it.hasNext()) {
+ final AlignedSourceTokens alignedSourceTokens = it.next();
+ if (it.hasNext()) { // if not last element in trgPoints
+ final List<Integer> newAlignedSourceTokens = new ArrayList<Integer>();
+ for (Integer sourceIndex : alignedSourceTokens) {
+ newAlignedSourceTokens.add(sourceIndex - 1); // shift by one to disregard sentence marker
+ }
+ alignment.add(newAlignedSourceTokens);
+ }
+ }
+ return alignment;
+ }
+
+ /**
+ * String representation for debugging.
+ */
+ @Override
+ public String toString() {
+ return String.format("%s , len=%d start=%d, isComplete=%s",
+ trgPoints.toString(), srcLength, srcStart, this.isComplete());
+ }
+
+ /**
+ * Substitutes a child WorldAlignmentState into this instance at the next
+ * nonterminal slot. Also shifts the indeces in this instance by the span/width of the
+ * child that is to be substituted.
+ * Substitution order is determined by the source-first traversal through the hypergraph.
+ *
+ * @param child The child
+ */
+ public void substituteIn(WordAlignmentState child) {
+ // find the index of the NonTerminal where we substitute the child targetPoints into.
+ // The correct NT is the first one on the SOURCE side.
+ // Also shift all trgPoints by the child length.
+ int substitutionIndex = 0;
+ int sourcePosition = MAX_VALUE;
+ for (final ListIterator<AlignedSourceTokens> trgPointsIterator = trgPoints.listIterator(); trgPointsIterator.hasNext();) {
+ final AlignedSourceTokens trgPoint = trgPointsIterator.next();
+ trgPoint.shiftBy(child.srcStart, child.srcLength - 1);
+ if (trgPoint.isNonTerminal() && trgPoint.get(0) < sourcePosition) {
+ sourcePosition = trgPoint.get(0);
+ substitutionIndex = trgPointsIterator.previousIndex();
+ }
+ }
+
+ // point and remove NT element determined from above
+ final ListIterator<AlignedSourceTokens> insertionIterator = trgPoints.listIterator(substitutionIndex);
+ insertionIterator.next();
+ insertionIterator.remove();
+
+ // insert child target points and set them to final.
+ for (AlignedSourceTokens childElement : child.trgPoints) {
+ childElement.setFinal();
+ insertionIterator.add(childElement);
+ }
+
+ // update length and number of non terminal slots
+ this.srcLength += child.srcLength - 1; // -1 (NT)
+ this.numNT--;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/package-info.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/package-info.java b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/package-info.java
new file mode 100644
index 0000000..05e66e2
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/package-info.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Provides implementations of hypergraph data structures
+ * and related algorithms used in extracting translation
+ * results in hierarchical phrase-based translation.
+ */
+package org.apache.joshua.decoder.hypergraph;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/io/DeNormalize.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/io/DeNormalize.java b/joshua-core/src/main/java/org/apache/joshua/decoder/io/DeNormalize.java
new file mode 100644
index 0000000..cc6e839
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/io/DeNormalize.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.io;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * Denormalize a(n English) string in a collection of ways listed below.
+ * <UL>
+ * <LI>Capitalize the first character in the string</LI>
+ * <LI>Detokenize</LI>
+ * <LI>Delete whitespace in front of periods and commas</LI>
+ * <LI>Join contractions</LI>
+ * <LI>Capitalize name titles (Mr Ms Miss Dr etc.)</LI>
+ * <LI>TODO: Handle surrounding characters ([{<"''">}])</LI>
+ * <LI>TODO: Join multi-period abbreviations (e.g. M.Phil. i.e.)</LI>
+ * <LI>TODO: Handle ambiguities like "st.", which can be an abbreviation for both "Saint" and
+ * "street"</LI>
+ * <LI>TODO: Capitalize both the title and the name of a person, e.g. Mr. Morton (named entities
+ * should be demarcated).</LI>
+ * </UL>
+ * <b>N.B.</b> These methods all assume that every translation result that will be
+ * denormalized has the following format:
+ * <UL>
+ * <LI>There is only one space between every pair of tokens</LI>
+ * <LI>There is no whitespace before the first token</LI>
+ * <LI>There is no whitespace after the final token</LI>
+ * <LI>Standard spaces are the only type of whitespace</LI>
+ * </UL>
+ */
+
+public class DeNormalize {
+
+ /**
+ * Apply all the denormalization methods to the normalized input line.
+ *
+ * @param normalized a normalized input line
+ * @return the denormalized String
+ */
+ public static String processSingleLine(String normalized) {
+ // The order in which the methods are applied could matter in some situations. E.g., a token to
+ // be matched is "phd", but if it is the first token in the line, it might have already been
+ // capitalized to "Phd" by the capitalizeFirstLetter method, and because the "phd" token won't
+ // match, "Phd" won't be corrected to "PhD".
+ String deNormalized = normalized;
+ deNormalized = capitalizeNameTitleAbbrvs(deNormalized);
+ deNormalized = replaceBracketTokens(deNormalized);
+ deNormalized = joinPunctuationMarks(deNormalized);
+ deNormalized = joinHyphen(deNormalized);
+ deNormalized = joinContractions(deNormalized);
+ deNormalized = capitalizeLineFirstLetter(deNormalized);
+ return deNormalized;
+ }
+
+ /**
+ * Capitalize the first letter of a line. This should be the last denormalization step applied to
+ * a line.
+ *
+ * @param line The single-line input string
+ * @return The input string modified as described above
+ */
+ public static String capitalizeLineFirstLetter(String line) {
+ String result = null;
+ Pattern regexp = Pattern.compile("[^\\p{Punct}\\p{Space}��]");
+ Matcher matcher = regexp.matcher(line);
+ if (matcher.find()) {
+ String match = matcher.group(0);
+ result = line.replaceFirst(match, match.toUpperCase());
+ } else {
+ result = line;
+ }
+ return result;
+ }
+
+ /**
+ * Scanning from left-to-right, a comma or period preceded by a space will become just the
+ * comma/period.
+ *
+ * @param line The single-line input string
+ * @return The input string modified as described above
+ */
+ public static String joinPunctuationMarks(String line) {
+ String result = line;
+ result = result.replace(" ,", ",");
+ result = result.replace(" ;", ";");
+ result = result.replace(" :", ":");
+ result = result.replace(" .", ".");
+ result = result.replace(" !", "!");
+ result = result.replace("� ", "�");
+ result = result.replace(" ?", "?");
+ result = result.replace("� ", "�");
+ result = result.replace(" )", ")");
+ result = result.replace(" ]", "]");
+ result = result.replace(" }", "}");
+ result = result.replace("( ", "(");
+ result = result.replace("[ ", "[");
+ result = result.replace("{ ", "{");
+ return result;
+ }
+
+ /**
+ * Scanning from left-to-right, a hyphen surrounded by a space before and after it will become
+ * just the hyphen.
+ *
+ * @param line The single-line input string
+ * @return The input string modified as described above
+ */
+ public static String joinHyphen(String line) {
+ return line.replace(" - ", "-");
+ }
+
+ /**
+ * Scanning the line from left-to-right, a contraction suffix preceded by a space will become just
+ * the contraction suffix. <br>
+ * <br>
+ * I.e., the preceding space will be deleting, joining the prefix to the suffix. <br>
+ * <br>
+ * E.g.
+ *
+ * <pre>wo n't</pre>
+ *
+ * becomes
+ *
+ * <pre>won't</pre>
+ *
+ * @param line The single-line input string
+ * @return The input string modified as described above
+ */
+ public static String joinContractions(String line) {
+ String result = line;
+ for (String suffix : new String[] {"'d", "'ll", "'m", "n't", "'re", "'s", "'ve",}) {
+ result = result.replace(" " + suffix, suffix);
+ }
+ return result;
+ }
+
+ /**
+ * Capitalize the first character of the titles of names: Mr Mrs Ms Miss Dr Prof
+ *
+ * @param line The single-line input string
+ * @return The input string modified as described above
+ */
+ public static String capitalizeNameTitleAbbrvs(String line) {
+ String result = line;
+
+ // Capitalize only the first character of certain name titles.
+ for (String title : new String[] {"dr", "miss", "mr", "mrs", "ms", "prof"}) {
+ result =
+ result.replaceAll("\\b" + title + "\\b",
+ Character.toUpperCase(title.charAt(0)) + title.substring(1));
+ }
+ // Capitalize the relevant characters of certain name titles.
+ result = result.replaceAll("\\b" + "phd" + "\\b", "PhD");
+ result = result.replaceAll("\\b" + "mphil" + "\\b", "MPhil");
+ return result;
+ }
+
+ public static String capitalizeI(String line) {
+ // Capitalize only the first character of certain name titles.
+ return line.replaceAll("\\b" + "i" + "\\b", "I");
+ }
+
+ /**
+ * Case-insensitively replace all of the character sequences that represent a bracket character.
+ *
+ * Keys are token representations of abbreviations of titles for names that capitalize more than
+ * just the first letter.<br>
+ * Bracket token sequences: -lrb- -rrb- -lsb- -rsb- -lcb- -rcb- <br>
+ * <br>
+ * See http://www.cis.upenn.edu/~treebank/tokenization.html
+ *
+ * @param line The single-line input string
+ * @return The input string modified as described above
+ */
+ public static String replaceBracketTokens(String line) {
+ String result = line;
+ result = result.replaceAll("(?iu)" + "-lrb-", "(");
+ result = result.replaceAll("(?iu)" + "-rrb-", ")");
+ result = result.replaceAll("(?iu)" + "-lsb-", "[");
+ result = result.replaceAll("(?iu)" + "-rsb-", "]");
+ result = result.replaceAll("(?iu)" + "-lcb-", "{");
+ result = result.replaceAll("(?iu)" + "-rcb-", "}");
+ return result;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/io/JSONMessage.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/io/JSONMessage.java b/joshua-core/src/main/java/org/apache/joshua/decoder/io/JSONMessage.java
new file mode 100644
index 0000000..9c3899e
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/io/JSONMessage.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.io;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+
+import org.apache.joshua.decoder.StructuredTranslation;
+import org.apache.joshua.decoder.Translation;
+
+/**
+ * Represents a JSON object returned by the server. The object has the format
+ *
+ * { data: {
+ * translations: [
+ * { annotatedSource: "",
+ * translatedText: "",
+ * raw_nbest: [
+ * { hyp: "",
+ * totalScore: 0.0, } ]
+ * tokenization: { ... }
+ * translatedTextRaw: "",
+ * nbest: [
+ * { translatedText: "",
+ * translatedTextRaw: "",
+ * tokenization: { ... } } ] } ] } }
+ *
+ * @author post
+ */
+
+public class JSONMessage {
+ public Data data = null;
+ public List<String> metadata = null;
+ public JSONMessage() {
+ metadata = new ArrayList<String>();
+ }
+
+ public class Data {
+ public List<TranslationItem> translations;
+
+ public Data() {
+ translations = new ArrayList<TranslationItem>();
+ }
+ }
+//
+// public class Metadata {
+// public String metadata = null;
+// public List<String> rules = null;
+//
+// public Metadata() {
+// rules = new ArrayList<String>();
+// }
+// }
+
+ public void addTranslation(Translation translation) {
+ String viterbi = translation.getStructuredTranslations().get(0).getTranslationString();
+
+ TranslationItem item = addTranslation(viterbi);
+
+ for (StructuredTranslation hyp: translation.getStructuredTranslations()) {
+ String text = hyp.getFormattedTranslationString();
+ float score = hyp.getTranslationScore();
+
+ item.addHypothesis(text, score);
+ }
+
+ // old string-based k-best output
+ // String[] results = translation.toString().split("\\n");
+ // if (results.length > 0) {
+ // String rawTranslation = results[0].split(" \\|\\|\\| ")[1];
+ // JSONMessage.TranslationItem item = message.addTranslation(rawTranslation);
+ //
+ // for (String result: results) {
+ // String[] tokens = result.split(" \\|\\|\\| ");
+ // String rawResult = tokens[1];
+ // float score = Float.parseFloat(tokens[3]);
+ // item.addHypothesis(rawResult, score);
+ // }
+ // }
+ }
+
+ /**
+ * Adds a new Translation to the JSON object. A Translation represents one or more hypotheses
+ * (or k-best items)
+ *
+ * @param text
+ * @return the new TranslationItem object
+ */
+ public TranslationItem addTranslation(String text) {
+ if (data == null)
+ data = new Data();
+
+ TranslationItem newItem = new TranslationItem(text);
+ data.translations.add(newItem);
+ return newItem;
+ }
+
+ public void addMetaData(String msg) {
+ this.metadata.add(msg);
+ }
+
+ public class TranslationItem {
+ public String translatedText;
+ public List<NBestItem> raw_nbest;
+
+ public TranslationItem(String value) {
+ this.translatedText = value;
+ this.raw_nbest = new ArrayList<NBestItem>();
+ }
+
+ /**
+ * Adds a new item to the translation's list of k-best items
+ *
+ * @param hyp the hypothesis
+ * @param score its score
+ */
+ public void addHypothesis(String hyp, float score) {
+ this.raw_nbest.add(new NBestItem(hyp, score));
+ }
+ }
+
+ public class NBestItem {
+ public String hyp;
+ public float totalScore;
+
+ public NBestItem(String hyp, float score) {
+ this.hyp = hyp;
+ this.totalScore = score;
+ }
+ }
+
+ public void addRule(String rule) {
+ metadata.add("custom_rule " + rule);
+ }
+
+ public String toString() {
+ Gson gson = new GsonBuilder().setPrettyPrinting().create();
+ return gson.toJson(this) + "\n";
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/io/TranslationRequestStream.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/io/TranslationRequestStream.java b/joshua-core/src/main/java/org/apache/joshua/decoder/io/TranslationRequestStream.java
new file mode 100644
index 0000000..0287688
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/io/TranslationRequestStream.java
@@ -0,0 +1,179 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.io;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.Reader;
+
+import com.google.gson.stream.JsonReader;
+
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.JoshuaConfiguration.INPUT_TYPE;
+import org.apache.joshua.decoder.segment_file.Sentence;
+
+/**
+ * This class iterates over an input stream, looking for inputs to translate. By default, it
+ * expects plain-text input, which can be plain sentences or PLF-encoded lattices. If
+ * '-input-type json' is passed to the decoder, it will instead read JSON objects from the input
+ * stream, with the following format:
+ * <pre>
+ * {
+ * "data": {
+ * "translations": [
+ * { "sourceText": "sentence to be translated" },
+ * { "sourceText": "next sentence" },
+ * { "sourceText": "@some command to run" }
+ * ]
+ * }
+ * }
+ * </pre>
+ * @author Matt Post post@cs.jhu.edu
+ * @author orluke
+ */
+public class TranslationRequestStream {
+ private final JoshuaConfiguration joshuaConfiguration;
+ private int sentenceNo = -1;
+
+ private Sentence nextSentence = null;
+
+ /* Plain text or JSON input */
+ private StreamHandler requestHandler = null;
+
+ /* Whether the request has been killed by a broken client connection. */
+ private volatile boolean isShutDown = false;
+
+ public TranslationRequestStream(BufferedReader reader, JoshuaConfiguration joshuaConfiguration) {
+ this.joshuaConfiguration = joshuaConfiguration;
+
+ if (joshuaConfiguration.input_type == INPUT_TYPE.json) {
+ this.requestHandler = new JSONStreamHandler(reader);
+ } else {
+ this.requestHandler = new PlaintextStreamHandler(reader);
+ }
+ }
+
+ private interface StreamHandler {
+ Sentence next() throws IOException;
+ }
+
+ private class JSONStreamHandler implements StreamHandler {
+
+ private JsonReader reader = null;
+ private String line = null;
+
+ public JSONStreamHandler(Reader in) {
+ reader = new JsonReader(in);
+ try {
+ reader.beginObject();
+ reader.nextName(); // "data"
+ reader.beginObject();
+ reader.nextName(); // "translations"
+ reader.beginArray();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+
+ @Override
+ public Sentence next() throws IOException {
+ line = null;
+
+ if (reader.hasNext()) {
+ reader.beginObject();
+ reader.nextName();
+ line = reader.nextString();
+ reader.endObject();
+ }
+
+ if (line == null)
+ return null;
+
+ return new Sentence(line, -1, joshuaConfiguration);
+ }
+ }
+
+ private class PlaintextStreamHandler implements StreamHandler {
+
+ private BufferedReader reader = null;
+
+ public PlaintextStreamHandler(BufferedReader in) {
+ reader = in;
+ }
+
+ @Override
+ public Sentence next() throws IOException {
+
+ String line = reader.readLine();
+
+ if (line != null) {
+ return new Sentence(line, sentenceNo, joshuaConfiguration);
+ }
+
+ return null;
+ }
+ }
+
+ public int size() {
+ return sentenceNo + 1;
+ }
+
+ /*
+ * Returns the next sentence item, then sets it to null, so that hasNext() will know to produce a
+ * new one.
+ */
+ public synchronized Sentence next() {
+ nextSentence = null;
+
+ if (isShutDown)
+ return null;
+
+ try {
+ nextSentence = requestHandler.next();
+ if (nextSentence != null) {
+ sentenceNo++;
+ nextSentence.id = sentenceNo;
+ }
+ } catch (IOException e) {
+ this.shutdown();
+ }
+
+ return nextSentence;
+ }
+
+ /**
+ * When the client socket is interrupted, we need to shut things down. On the source side, the
+ * TranslationRequest could easily have buffered a lot of lines and so will keep discovering
+ * sentences to translate, but the output Translation objects will start throwing exceptions when
+ * trying to print to the closed socket. When that happens, we call this function() so that we can
+ * tell next() to stop returning translations, which in turn will cause it to stop asking for
+ * them.
+ *
+ * Note that we don't go to the trouble of shutting down existing DecoderThreads. This would be
+ * good to do, but for the moment would require more bookkeeping than we want to do.
+ */
+
+ public void shutdown() {
+ isShutDown = true;
+ }
+
+ public boolean isShutDown() {
+ return isShutDown;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/package-info.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/package-info.java b/joshua-core/src/main/java/org/apache/joshua/decoder/package-info.java
new file mode 100644
index 0000000..af1127b
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/package-info.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * Provides infrastructure and wrapper code used relevant to
+ * hierarchical phrase-based decoding for statistical machine
+ * translation. This package does not include an implementation
+ * of any actual decoding algorithm. Rather, such code is in
+ * child packages of this package.
+ */
+package org.apache.joshua.decoder;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Candidate.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Candidate.java b/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Candidate.java
new file mode 100644
index 0000000..ee8a2a9
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Candidate.java
@@ -0,0 +1,241 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.phrase;
+
+/***
+ * A candidate is basically a cube prune state. It contains a list of hypotheses and target
+ * phrases, and an instantiated candidate is a pair of indices that index these two lists. This
+ * is the "cube prune" position.
+ */
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.joshua.corpus.Span;
+import org.apache.joshua.decoder.chart_parser.ComputeNodeResult;
+import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.hypergraph.HGNode;
+
+public class Candidate {
+
+ // the set of hypotheses that can be paired with phrases from this span
+ private List<Hypothesis> hypotheses;
+
+ // the list of target phrases gathered from a span of the input
+ private TargetPhrases phrases;
+
+ // source span of new phrase
+ public Span span;
+
+ // future cost of applying phrases to hypotheses
+ float future_delta;
+
+ // indices into the hypotheses and phrases arrays (used for cube pruning)
+ private int[] ranks;
+
+ // scoring and state information
+ private ComputeNodeResult result;
+
+ /**
+ * When candidate objects are extended, the new one is initialized with the same underlying
+ * "phrases" and "hypotheses" and "span" objects. So these all have to be equal, as well as
+ * the ranks.
+ *
+ * This is used to prevent cube pruning from adding the same candidate twice, having reached
+ * a point in the cube via different paths.
+ */
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof Candidate) {
+ Candidate other = (Candidate) obj;
+ if (hypotheses != other.hypotheses || phrases != other.phrases || span != other.span)
+ return false;
+
+ if (ranks.length != other.ranks.length)
+ return false;
+
+ for (int i = 0; i < ranks.length; i++)
+ if (ranks[i] != other.ranks[i])
+ return false;
+
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return 17 * hypotheses.size()
+ + 23 * phrases.size()
+ + 57 * span.hashCode()
+ + 117 * Arrays.hashCode(ranks);
+// return hypotheses.hashCode() * phrases.hashCode() * span.hashCode() * Arrays.hashCode(ranks);
+ }
+
+ @Override
+ public String toString() {
+ return String.format("CANDIDATE(hyp %d/%d, phr %d/%d) [%s] phrase=[%s] span=%s",
+ ranks[0], hypotheses.size(), ranks[1], phrases.size(),
+ getHypothesis(), getRule().getEnglishWords().replaceAll("\\[.*?\\] ",""), getSpan());
+ }
+
+ public Candidate(List<Hypothesis> hypotheses, TargetPhrases phrases, Span span, float delta) {
+ this.hypotheses = hypotheses;
+ this.phrases = phrases;
+ this.span = span;
+ this.future_delta = delta;
+ this.ranks = new int[] { 0, 0 };
+ }
+
+ public Candidate(List<Hypothesis> hypotheses, TargetPhrases phrases, Span span, float delta, int[] ranks) {
+ this.hypotheses = hypotheses;
+ this.phrases = phrases;
+ this.span = span;
+ this.future_delta = delta;
+ this.ranks = ranks;
+// this.score = hypotheses.get(ranks[0]).score + phrases.get(ranks[1]).getEstimatedCost();
+ }
+
+ /**
+ * Extends the cube pruning dot in both directions and returns the resulting set. Either of the
+ * results can be null if the end of their respective lists is reached.
+ *
+ * @return The neighboring candidates (possibly null)
+ */
+ public Candidate[] extend() {
+ return new Candidate[] { extendHypothesis(), extendPhrase() };
+ }
+
+ /**
+ * Extends the cube pruning dot along the dimension of existing hypotheses.
+ *
+ * @return the next candidate, or null if none
+ */
+ public Candidate extendHypothesis() {
+ if (ranks[0] < hypotheses.size() - 1) {
+ return new Candidate(hypotheses, phrases, span, future_delta, new int[] { ranks[0] + 1, ranks[1] });
+ }
+ return null;
+ }
+
+ /**
+ * Extends the cube pruning dot along the dimension of candidate target sides.
+ *
+ * @return the next Candidate, or null if none
+ */
+ public Candidate extendPhrase() {
+ if (ranks[1] < phrases.size() - 1) {
+ return new Candidate(hypotheses, phrases, span, future_delta, new int[] { ranks[0], ranks[1] + 1 });
+ }
+
+ return null;
+ }
+
+ /**
+ * Returns the input span from which the phrases for this candidates were gathered.
+ *
+ * @return the span object
+ */
+ public Span getSpan() {
+ return this.span;
+ }
+
+ /**
+ * A candidate is a (hypothesis, target phrase) pairing. The hypothesis and target phrase are
+ * drawn from a list that is indexed by (ranks[0], ranks[1]), respectively. This is a shortcut
+ * to return the hypothesis of the candidate pair.
+ *
+ * @return the hypothesis at position ranks[0]
+ */
+ public Hypothesis getHypothesis() {
+ return this.hypotheses.get(ranks[0]);
+ }
+
+ /**
+ * This returns the target side {@link org.apache.joshua.corpus.Phrase}, which is a {@link org.apache.joshua.decoder.ff.tm.Rule} object. This is just a
+ * convenience function that works by returning the phrase indexed in ranks[1].
+ *
+ * @return the phrase at position ranks[1]
+ */
+ public Rule getRule() {
+ return phrases.get(ranks[1]);
+ }
+
+ /**
+ * The hypotheses list is a list of tail pointers. This function returns the tail pointer
+ * currently selected by the value in ranks.
+ *
+ * @return a list of size one, wrapping the tail node pointer
+ */
+ public List<HGNode> getTailNodes() {
+ List<HGNode> tailNodes = new ArrayList<HGNode>();
+ tailNodes.add(getHypothesis());
+ return tailNodes;
+ }
+
+ /**
+ * Returns the bit vector of this hypothesis. The bit vector is computed by ORing the coverage
+ * vector of the tail node (hypothesis) and the source span of phrases in this candidate.
+ * @return the bit vector of this hypothesis
+ */
+ public Coverage getCoverage() {
+ Coverage cov = new Coverage(getHypothesis().getCoverage());
+ cov.set(getSpan());
+ return cov;
+ }
+
+ /**
+ * Sets the result of a candidate (TODO should just be moved to the constructor).
+ *
+ * @param result todo
+ */
+ public void setResult(ComputeNodeResult result) {
+ this.result = result;
+ }
+
+ /**
+ * This returns the sum of two costs: the HypoState cost + the transition cost. The HypoState cost
+ * is in turn the sum of two costs: the Viterbi cost of the underlying hypothesis, and the adjustment
+ * to the future score incurred by translating the words under the source phrase being added.
+ * The transition cost is the sum of new features incurred along the transition (mostly, the
+ * language model costs).
+ *
+ * The Future Cost item should probably just be implemented as another kind of feature function,
+ * but it would require some reworking of that interface, which isn't worth it.
+ *
+ * @return the sum of two costs: the HypoState cost + the transition cost
+ */
+ public float score() {
+ return getHypothesis().getScore() + future_delta + result.getTransitionCost();
+ }
+
+ public float getFutureEstimate() {
+ return getHypothesis().getScore() + future_delta;
+ }
+
+ public List<DPState> getStates() {
+ return result.getDPStates();
+ }
+
+ public ComputeNodeResult getResult() {
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/CandidateComparator.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/CandidateComparator.java b/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/CandidateComparator.java
new file mode 100644
index 0000000..322f47a
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/CandidateComparator.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.phrase;
+
+import java.util.Comparator;
+
+public class CandidateComparator implements Comparator<Candidate> {
+ @Override
+ public int compare(Candidate one, Candidate another) {
+ return Float.compare(another.score(), one.score());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Coverage.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Coverage.java b/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Coverage.java
new file mode 100644
index 0000000..2c674fc
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Coverage.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.phrase;
+
+import java.util.BitSet;
+
+import org.apache.joshua.corpus.Span;
+
+/**
+ * Represents a coverage vector. The vector is relative to a hypothesis. {firstZero} denotes the
+ * first uncovered word of the sentence, and {bits} contains the coverage vector of all the words
+ * after it, with the first zero removed.
+ */
+
+public class Coverage {
+
+ // The index of the first uncovered word
+ private int firstZero;
+
+ // Bits with the first zero removed.
+ // We also assume anything beyond this is zero due to the reordering window.
+ // Lowest bits correspond to next word.
+ private BitSet bits;
+
+ // Default bit vector length
+ private static int INITIAL_LENGTH = 10;
+
+ public Coverage() {
+ firstZero = 0;
+ bits = new BitSet(INITIAL_LENGTH);
+ }
+
+ public Coverage(int firstZero) {
+ this.firstZero = firstZero;
+ bits = new BitSet(INITIAL_LENGTH);
+ }
+
+ /**
+ * Pretty-prints the coverage vector, making a guess about the length
+ */
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(String.format("%d ", firstZero));
+
+ for (int i = 0; i < Math.max(INITIAL_LENGTH, bits.length()); i++) { // only display first 10 bits
+ sb.append(bits.get(i) ? "x" : ".");
+ }
+
+ return sb.toString();
+ }
+
+ /**
+ * Initialize a coverage vector from another Coverage vector, creating a separate object.
+ *
+ * @param other an existing coverage vector from which to create a new coverage vector
+ */
+ public Coverage(Coverage other) {
+ this.firstZero = other.firstZero;
+ this.bits = (BitSet) other.bits.clone();
+ }
+
+ /**
+ * Turns on all bits from position start to position (end - 1), that is, in the range [start .. end).
+ * This is done relative to the current coverage vector, of course, which may not start at 0.
+ *
+ * @param begin bits at start position
+ * @param end bits at end position (end - 1)
+ */
+ public void set(int begin, int end) {
+ assert compatible(begin, end);
+
+ // StringBuffer sb = new StringBuffer();
+ // sb.append(String.format("SET(%d,%d) %s", begin, end, this));
+
+ if (begin == firstZero) {
+ // A concatenation.
+ firstZero = end;
+ bits = bits.get(end - begin, Math.max(end - begin, bits.length()));
+ int firstClear = bits.nextClearBit(0);
+ if (firstClear != 0) {
+ // We might have exactly covered a gap, in which case we need to adjust shift
+ // firstZero and the bits until we reach the new end
+ firstZero += firstClear;
+ bits = bits.get(firstClear, bits.length());
+ }
+ } else {
+ // Set the bits relative to the currenS
+ bits.or(pattern(begin, end));
+ }
+
+ // sb.append(String.format(" -> %s", this));
+ // System.err.println(sb);
+ }
+
+ /**
+ * Convenience function.
+ * @param span todo
+ */
+ public final void set(Span span) {
+ set(span.start, span.end);
+ }
+
+ /**
+ * Tests whether a new range is compatible with the current coverage vector. It must be after
+ * the first uncovered word, obviously, and must not conflict with spans after the first
+ * uncovered word.
+ *
+ * @param begin the begin index (absolute)
+ * @param end the end index (absolute)
+ * @return true if the span is compatible with the coverage vector
+ */
+ public boolean compatible(int begin, int end) {
+ if (begin >= firstZero) {
+ BitSet pattern = new BitSet();
+ pattern.set(begin - firstZero, end - firstZero);
+ return ! bits.intersects(pattern);
+ }
+ return false;
+ }
+
+ /**
+ * Returns the source sentence index of the first uncovered word.
+ *
+ * @return the index
+ */
+ public int firstZero() {
+ return firstZero;
+ }
+
+ /**
+ * LeftOpen() and RightOpen() find the larger gap in which a new source phrase pair sits.
+ * When using a phrase pair covering (begin, end), the pair
+ *
+ * (LeftOpen(begin), RightOpen(end, sentence_length))
+ *
+ * provides this gap.
+
+ * Find the left bound of the gap in which the phrase [begin, ...) sits.
+ *
+ * @param begin the start index of the phrase being applied.
+ * @return todo
+ */
+ public int leftOpening(int begin) {
+ for (int i = begin - firstZero; i > 0; --i) {
+ if (bits.get(i)) {
+ assert compatible(i + firstZero + 1, begin);
+ assert !compatible(i + firstZero, begin);
+ return i + firstZero + 1;
+ }
+ }
+
+ assert compatible(firstZero, begin);
+ return firstZero;
+ }
+
+ /**
+ * LeftOpen() and RightOpen() find the larger gap in which a new source phrase pair sits.
+ * When using a phrase pair covering (begin, end), the pair
+ * <pre>
+ * (LeftOpen(begin), RightOpen(end, sentence_length))
+ * </pre>
+ *
+ * provides this gap.
+ *
+ * Finds the right bound of the enclosing gap, or the end of sentence, whichever is less.
+ * @param end end of phrase pair
+ * @param sentenceLength length of sentence
+ * @return todo
+ */
+ public int rightOpening(int end, int sentenceLength) {
+ for (int i = end - firstZero; i < Math.min(64, sentenceLength - firstZero); i++) {
+ if (bits.get(i)) {
+ return i + firstZero;
+ }
+ }
+ return sentenceLength;
+ }
+
+ /**
+ * Creates a bit vector with the same offset as the current coverage vector, flipping on
+ * bits begin..end.
+ *
+ * @param begin the begin index (absolute)
+ * @param end the end index (absolute)
+ * @return a bit vector (relative) with positions [begin..end) on
+ */
+ public BitSet pattern(int begin, int end) {
+ // System.err.println(String.format("pattern(%d,%d) %d %s %s", begin, end, firstZero, begin >= firstZero, toString()));
+ assert begin >= firstZero;
+ BitSet pattern = new BitSet(INITIAL_LENGTH);
+ pattern.set(begin - firstZero, end - firstZero);
+ return pattern;
+ }
+
+ /**
+ * Returns the underlying coverage bits.
+ *
+ * @return {@link java.util.BitSet} vector of bits
+ */
+ public BitSet getCoverage() {
+ return bits;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof Coverage) {
+ Coverage other = (Coverage) obj;
+ return getCoverage().equals(other.getCoverage()) && firstZero() == other.firstZero();
+ }
+
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return getCoverage().hashCode() * firstZero();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Future.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Future.java b/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Future.java
new file mode 100644
index 0000000..0ece4a3
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Future.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.phrase;
+
+import org.apache.joshua.util.ChartSpan;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class Future {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Future.class);
+
+ // Square matrix with half the values ignored.
+ private ChartSpan<Float> entries;
+
+ private int sentlen;
+
+ /**
+ * Computes bottom-up the best way to cover all spans of the input sentence, using the phrases
+ * that have been assembled in a {@link org.apache.joshua.decoder.phrase.PhraseChart}.
+ * Requires that there be a translation at least for every word (which can be
+ * accomplished with a pass-through grammar).
+ *
+ * @param chart an input {@link org.apache.joshua.decoder.phrase.PhraseChart}
+ */
+ public Future(PhraseChart chart) {
+
+ sentlen = chart.SentenceLength();
+ entries = new ChartSpan<Float>(sentlen + 1, Float.NEGATIVE_INFINITY);
+
+ /*
+ * The sentence is represented as a sequence of words, with the first and last words set
+ * to <s> and </s>. We start indexing at 1 because the first word (<s>) is always covered.
+ */
+ for (int begin = 1; begin <= chart.SentenceLength(); begin++) {
+ // Nothing is nothing (this is a useful concept when two phrases abut)
+ setEntry(begin, begin, 0.0f);
+ // Insert phrases
+ int max_end = Math.min(begin + chart.MaxSourcePhraseLength(), chart.SentenceLength());
+ for (int end = begin + 1; end <= max_end; end++) {
+
+ // Moses doesn't include the cost of applying </s>, so force it to zero
+ if (begin == sentlen - 1 && end == sentlen)
+ setEntry(begin, end, 0.0f);
+ else {
+ TargetPhrases phrases = chart.getRange(begin, end);
+ if (phrases != null)
+ setEntry(begin, end, phrases.get(0).getEstimatedCost());
+ }
+ }
+ }
+
+ // All the phrases are in, now do minimum dynamic programming. Lengths 0 and 1 were already handled above.
+ for (int length = 2; length <= chart.SentenceLength(); length++) {
+ for (int begin = 1; begin <= chart.SentenceLength() - length; begin++) {
+ for (int division = begin + 1; division < begin + length; division++) {
+ setEntry(begin, begin + length, Math.max(getEntry(begin, begin + length), getEntry(begin, division) + getEntry(division, begin + length)));
+ }
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ for (int i = 1; i < chart.SentenceLength(); i++) {
+ for (int j = i + 1; j < chart.SentenceLength(); j++) {
+ LOG.debug("future cost from {} to {} is {}", i - 1, j - 2, getEntry(i, j));
+ }
+ }
+ }
+ }
+
+ public float Full() {
+ // System.err.println("Future::Full(): " + Entry(1, sentlen));
+ return getEntry(1, sentlen);
+ }
+
+ /**
+ * Calculate change in rest cost when the given coverage is to be covered.
+ * @param coverage input {@link org.apache.joshua.decoder.phrase.Coverage} vector
+ * @param begin word at which to begin within a sentence
+ * @param end word at which to end within a sentence
+ * @return a float value representing a {@link Future} entry
+ */
+ public float Change(Coverage coverage, int begin, int end) {
+ int left = coverage.leftOpening(begin);
+ int right = coverage.rightOpening(end, sentlen);
+ // System.err.println(String.format("Future::Change(%s, %d, %d) left %d right %d %.3f %.3f %.3f", coverage, begin, end, left, right,
+ // Entry(left, begin), Entry(end, right), Entry(left, right)));
+ return getEntry(left, begin) + getEntry(end, right) - getEntry(left, right);
+ }
+
+ private float getEntry(int begin, int end) {
+ assert end >= begin;
+ assert end < this.sentlen;
+ return entries.get(begin, end);
+ }
+
+ private void setEntry(int begin, int end, float value) {
+ assert end >= begin;
+ assert end < this.sentlen;
+ // System.err.println(String.format("future cost from %d to %d is %.5f", begin, end, value));
+ entries.set(begin, end, value);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Header.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Header.java b/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Header.java
new file mode 100644
index 0000000..30d771c
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Header.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.phrase;
+
+// PORT: done
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Comparator;
+
+public class Header implements Comparable<Header>, Comparator<Header> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Header.class);
+
+ private float score;
+ private int arity;
+ private Note note;
+
+ protected Header() {
+ score = 0.0f;
+ arity = 0;
+ note = null;
+ }
+
+ protected Header(Header other) {
+ this.score = other.GetScore();
+ this.arity = other.GetArity();
+ this.note = other.GetNote();
+ }
+
+ protected Header(int arity) {
+ this.score = 0.0f;
+ this.arity = arity;
+ this.note = new Note();
+ }
+
+ public boolean Valid() {
+ // C++: return base_;
+ LOG.debug("Header::Valid(): {}", (note != null));
+ return note != null;
+ }
+
+ public float GetScore() {
+ return score;
+ }
+
+ public void SetScore(float score) {
+ this.score = score;
+ }
+
+ public int GetArity() { return arity; }
+
+ public Note GetNote() { return note; }
+
+ public void SetNote(Note note) { this.note = note; }
+
+ @Override
+ public int compareTo(Header other) {
+ if (this.GetScore() < other.GetScore())
+ return -1;
+ else if (this.GetScore() > other.GetScore())
+ return 1;
+ return 0;
+ }
+
+ @Override
+ public int compare(Header arg0, Header arg1) {
+ return arg0.compareTo(arg1);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Hypothesis.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Hypothesis.java b/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Hypothesis.java
new file mode 100644
index 0000000..71d3df9
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Hypothesis.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.phrase;
+
+import java.util.List;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.ff.tm.format.HieroFormatReader;
+import org.apache.joshua.decoder.hypergraph.HGNode;
+import org.apache.joshua.decoder.hypergraph.HyperEdge;
+
+/**
+ * Represents a hypothesis, a translation of some coverage of the input. Extends {@link org.apache.joshua.decoder.hypergraph.HGNode},
+ * through a bit of a hack. Whereas (i,j) represents the span of an {@link org.apache.joshua.decoder.hypergraph.HGNode}, i here is not used,
+ * and j is overloaded to denote the span of the phrase being applied. The complete coverage vector
+ * can be obtained by looking at the tail pointer and casting it.
+ *
+ * @author Kenneth Heafield
+ * @author Matt Post post@cs.jhu.edu
+ */
+public class Hypothesis extends HGNode implements Comparable<Hypothesis> {
+
+ // The hypothesis' coverage vector
+ private Coverage coverage;
+
+ public static Rule BEGIN_RULE = new HieroFormatReader().parseLine("[X] ||| <s> ||| <s> ||| ||| 0-0");
+ public static Rule END_RULE = new HieroFormatReader().parseLine("[GOAL] ||| [X,1] </s> ||| [X,1] </s> ||| ||| 0-0 1-1");
+
+ public String toString() {
+ StringBuffer sb = new StringBuffer();
+ for (DPState state: getDPStates())
+ sb.append(state);
+ String words = bestHyperedge.getRule().getEnglishWords();
+// return String.format("HYP[%s] %.5f j=%d words=%s state=%s", coverage, score, j, words, sb);
+ return String.format("HYP[%s] j=%d words=[%s] state=%s", coverage, j, words, sb);
+ }
+
+ // Initialize root hypothesis. Provide the LM's BeginSentence.
+ public Hypothesis(List<DPState> states, float futureCost) {
+ super(0, 1, Vocabulary.id("[X]"), states,
+ new HyperEdge(BEGIN_RULE, 0.0f, 0.0f, null, null), futureCost);
+ this.coverage = new Coverage(1);
+ }
+
+ public Hypothesis(Candidate cand) {
+ // TODO: sourcepath
+ super(-1, cand.span.end, Vocabulary.id("[X]"), cand.getStates(), new HyperEdge(
+ cand.getRule(), cand.getResult().getViterbiCost(), cand.getResult().getTransitionCost(),
+ cand.getTailNodes(), null), cand.score());
+ this.coverage = cand.getCoverage();
+ }
+
+ // Extend a previous hypothesis.
+ public Hypothesis(List<DPState> states, float score, Hypothesis previous, int source_end, Rule target) {
+ super(-1, source_end, -1, null, null, score);
+ this.coverage = previous.coverage;
+ }
+
+ public Coverage getCoverage() {
+ return coverage;
+ }
+
+ public Rule getRule() {
+ return bestHyperedge.getRule();
+ }
+
+ /**
+ * HGNodes (designed for chart parsing) maintain a span (i,j). We overload j
+ * here to record the index of the last translated source word.
+ *
+ * @return the int 'j' which is overloaded to denote the span of the phrase being applied
+ */
+ public int LastSourceIndex() {
+ return j;
+ }
+
+ @Override
+ public int hashCode() {
+ int hash = 0;
+ hash = 31 * LastSourceIndex() + 19 * getCoverage().hashCode();
+ if (null != dpStates && dpStates.size() > 0)
+ for (DPState dps: dpStates)
+ hash *= 57 + dps.hashCode();
+ return hash;
+ }
+
+ /**
+ * Defines equivalence in terms of recombinability. Two hypotheses are recombinable if
+ * all their DP states are the same, their coverage is the same, and they have the next soure
+ * index the same.
+ */
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof Hypothesis) {
+ Hypothesis other = (Hypothesis) obj;
+
+ if (LastSourceIndex() != other.LastSourceIndex() || ! getCoverage().equals(other.getCoverage()))
+ return false;
+
+ if (dpStates == null)
+ return (other.dpStates == null);
+
+ if (other.dpStates == null)
+ return false;
+
+ if (dpStates.size() != other.dpStates.size())
+ return false;
+
+ for (int i = 0; i < dpStates.size(); i++) {
+ if (!dpStates.get(i).equals(other.dpStates.get(i)))
+ return false;
+ }
+
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public int compareTo(Hypothesis o) {
+ // TODO: is this the order we want?
+ return Float.compare(o.getScore(), getScore());
+ }
+
+ /**
+ * Performs hypothesis recombination, incorporating the incoming hyperedges of the added
+ * hypothesis and possibly updating the cache of the best incoming hyperedge and score.
+ *
+ * @param added the equivalent hypothesis
+ */
+ public void absorb(Hypothesis added) {
+ assert(this.equals(added));
+ score = Math.max(score, added.getScore());
+ addHyperedgesInNode(added.hyperedges);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Note.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Note.java b/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Note.java
new file mode 100644
index 0000000..15b0057
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Note.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.phrase;
+
+// PORT: done
+
+public class Note {
+ public Object value;
+
+ public String toString() {
+ return value.toString();
+ }
+
+ public Note() {
+ }
+
+ public Note(Object value) {
+ this.value = value;
+ }
+
+ public Object get() {
+ return value;
+ }
+
+ public void set(Object object) {
+ this.value = object;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/PhraseChart.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/PhraseChart.java b/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/PhraseChart.java
new file mode 100644
index 0000000..9803d9b
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/PhraseChart.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.phrase;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.joshua.decoder.Decoder;
+import org.apache.joshua.decoder.ff.FeatureFunction;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.ff.tm.RuleCollection;
+import org.apache.joshua.decoder.segment_file.Sentence;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class represents a bundle of phrase tables that have been read in,
+ * reporting some stats about them. Probably could be done away with.
+ */
+public class PhraseChart {
+
+ private static final Logger LOG = LoggerFactory.getLogger(PhraseChart.class);
+
+ private int sentence_length;
+ private int max_source_phrase_length;
+
+ // Banded array: different source lengths are next to each other.
+ private List<TargetPhrases> entries;
+
+ // number of translation options
+ int numOptions = 20;
+ private List<FeatureFunction> features;
+
+ /**
+ * Create a new PhraseChart object, which represents all phrases that are
+ * applicable against the current input sentence. These phrases are extracted
+ * from all available grammars.
+ *
+ * @param tables input array of {@link org.apache.joshua.decoder.phrase.PhraseTable}'s
+ * @param features {@link java.util.List} of {@link org.apache.joshua.decoder.ff.FeatureFunction}'s
+ * @param source input to {@link org.apache.joshua.lattice.Lattice}
+ * @param num_options number of translation options (typically set to 20)
+ */
+ public PhraseChart(PhraseTable[] tables, List<FeatureFunction> features, Sentence source,
+ int num_options) {
+
+ float startTime = System.currentTimeMillis();
+
+ this.numOptions = num_options;
+ this.features = features;
+
+ max_source_phrase_length = 0;
+ for (int i = 0; i < tables.length; i++)
+ max_source_phrase_length = Math.max(max_source_phrase_length,
+ tables[i].getMaxSourcePhraseLength());
+ sentence_length = source.length();
+
+// System.err.println(String.format(
+// "PhraseChart()::Initializing chart for sentlen %d max %d from %s", sentence_length,
+// max_source_phrase_length, source));
+
+ entries = new ArrayList<TargetPhrases>();
+ for (int i = 0; i < sentence_length * max_source_phrase_length; i++)
+ entries.add(null);
+
+ // There's some unreachable ranges off the edge. Meh.
+ for (int begin = 0; begin != sentence_length; ++begin) {
+ for (int end = begin + 1; (end != sentence_length + 1)
+ && (end <= begin + max_source_phrase_length); ++end) {
+ if (source.hasPath(begin, end)) {
+ for (PhraseTable table : tables)
+ addToRange(begin, end,
+ table.getPhrases(Arrays.copyOfRange(source.getWordIDs(), begin, end)));
+ }
+
+ }
+ }
+
+ for (TargetPhrases phrases : entries) {
+ if (phrases != null)
+ phrases.finish(features, Decoder.weights, num_options);
+ }
+
+ LOG.info("Input {}: Collecting options took {} seconds", source.id(),
+ (System.currentTimeMillis() - startTime) / 1000.0f);
+
+ if (LOG.isDebugEnabled()) {
+ for (int i = 1; i < sentence_length - 1; i++) {
+ for (int j = i + 1; j < sentence_length && j <= i + max_source_phrase_length; j++) {
+ if (source.hasPath(i, j)) {
+ TargetPhrases phrases = getRange(i, j);
+ if (phrases != null) {
+ LOG.debug("{} ({}-{})", source.source(i,j), i, j);
+ for (Rule rule: phrases)
+ LOG.debug(" {} :: est={}", rule.getEnglishWords(), rule.getEstimatedCost());
+ }
+ }
+ }
+ }
+ }
+ }
+
+ public int SentenceLength() {
+ return sentence_length;
+ }
+
+ // c++: TODO: make this reflect the longest source phrase for this sentence.
+ public int MaxSourcePhraseLength() {
+ return max_source_phrase_length;
+ }
+
+ /**
+ * Maps two-dimensional span into a one-dimensional array.
+ *
+ * @param i beginning of span
+ * @param j end of span
+ * @return offset into private list of TargetPhrases
+ */
+ private int offset(int i, int j) {
+ return i * max_source_phrase_length + j - i - 1;
+ }
+
+ /**
+ * Returns phrases from all grammars that match the span.
+ *
+ * @param begin beginning of span
+ * @param end end of span
+ * @return the {@link org.apache.joshua.decoder.phrase.TargetPhrases} at the specified position in this list.
+ */
+ public TargetPhrases getRange(int begin, int end) {
+ int index = offset(begin, end);
+ // System.err.println(String.format("PhraseChart::Range(%d,%d): found %d entries",
+ // begin, end,
+ // entries.get(index) == null ? 0 : entries.get(index).size()));
+ // if (entries.get(index) != null)
+ // for (Rule phrase: entries.get(index))
+ // System.err.println(" RULE: " + phrase);
+
+ if (index < 0 || index >= entries.size() || entries.get(index) == null)
+ return null;
+
+ return entries.get(index);
+ }
+
+ /**
+ * Add a set of phrases from a grammar to the current span.
+ *
+ * @param begin beginning of span
+ * @param end end of span
+ * @param to a {@link org.apache.joshua.decoder.ff.tm.RuleCollection} to be used in scoring and sorting.
+ */
+ private void addToRange(int begin, int end, RuleCollection to) {
+ if (to != null) {
+ /*
+ * This first call to getSortedRules() is important, because it is what
+ * causes the scoring and sorting to happen. It is also a synchronized call,
+ * which is necessary because the underlying grammar gets sorted. Subsequent calls to get the
+ * rules will just return the already-sorted list. Here, we score, sort,
+ * and then trim the list to the number of translation options. Trimming provides huge
+ * performance gains --- the more common the word, the more translations options it is
+ * likely to have (often into the tens of thousands).
+ */
+ List<Rule> rules = to.getSortedRules(features);
+ if (numOptions > 0 && rules.size() > numOptions)
+ rules = rules.subList(0, numOptions);
+// to.getRules().subList(numOptions, to.getRules().size()).clear();
+
+ try {
+ int offset = offset(begin, end);
+ if (entries.get(offset) == null)
+ entries.set(offset, new TargetPhrases(rules));
+ else
+ entries.get(offset).addAll(rules);
+ } catch (java.lang.IndexOutOfBoundsException e) {
+ LOG.error("Whoops! {} [{}-{}] too long ({})", to, begin, end, entries.size());
+ LOG.error(e.getMessage(), e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/PhraseTable.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/PhraseTable.java b/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/PhraseTable.java
new file mode 100644
index 0000000..27f92ac
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/PhraseTable.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.phrase;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.ff.FeatureFunction;
+import org.apache.joshua.decoder.ff.tm.Grammar;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.ff.tm.RuleCollection;
+import org.apache.joshua.decoder.ff.tm.Trie;
+import org.apache.joshua.decoder.ff.tm.hash_based.MemoryBasedBatchGrammar;
+import org.apache.joshua.decoder.ff.tm.packed.PackedGrammar;
+
+/**
+ * Represents a phrase table, and is implemented as a wrapper around either a {@link PackedGrammar}
+ * or a {@link MemoryBasedBatchGrammar}.
+ *
+ * TODO: this should all be implemented as a two-level trie (source trie and target trie).
+ */
+public class PhraseTable implements Grammar {
+
+ private JoshuaConfiguration config;
+ private Grammar backend;
+
+ /**
+ * Chain to the super with a number of defaults. For example, we only use a single nonterminal,
+ * and there is no span limit.
+ *
+ * @param grammarFile file path parent directory
+ * @param owner used to set phrase owners
+ * @param type the grammar specification keyword (e.g., "thrax" or "moses")
+ * @param config a populated {@link org.apache.joshua.decoder.JoshuaConfiguration}
+ * @throws IOException if there is an error reading the grammar file
+ */
+ public PhraseTable(String grammarFile, String owner, String type, JoshuaConfiguration config)
+ throws IOException {
+ this.config = config;
+ int spanLimit = 0;
+
+ if (grammarFile != null && new File(grammarFile).isDirectory()) {
+ this.backend = new PackedGrammar(grammarFile, spanLimit, owner, type, config);
+ if (this.backend.getMaxSourcePhraseLength() == -1) {
+ String msg = "FATAL: Using a packed grammar for a phrase table backend requires that you "
+ + "packed the grammar with Joshua 6.0.2 or greater";
+ throw new RuntimeException(msg);
+ }
+
+ } else {
+ this.backend = new MemoryBasedBatchGrammar(type, grammarFile, owner, "[X]", spanLimit, config);
+ }
+ }
+
+ public PhraseTable(String owner, JoshuaConfiguration config) {
+ this.config = config;
+
+ this.backend = new MemoryBasedBatchGrammar(owner, config);
+ }
+
+ /**
+ * Returns the longest source phrase read. Because phrases have a dummy nonterminal prepended to
+ * them, we need to subtract 1.
+ *
+ * @return the longest source phrase read.
+ */
+ @Override
+ public int getMaxSourcePhraseLength() {
+ return this.backend.getMaxSourcePhraseLength() - 1;
+ }
+
+ /**
+ * Collect the set of target-side phrases associated with a source phrase.
+ *
+ * @param sourceWords the sequence of source words
+ * @return the rules
+ */
+ public RuleCollection getPhrases(int[] sourceWords) {
+ if (sourceWords.length != 0) {
+ Trie pointer = getTrieRoot();
+ pointer = pointer.match(Vocabulary.id("[X]"));
+ int i = 0;
+ while (pointer != null && i < sourceWords.length)
+ pointer = pointer.match(sourceWords[i++]);
+
+ if (pointer != null && pointer.hasRules()) {
+ return pointer.getRuleCollection();
+ }
+ }
+
+ return null;
+ }
+
+ /**
+ * Adds a rule to the grammar. Only supported when the backend is a MemoryBasedBatchGrammar.
+ *
+ * @param rule the rule to add
+ */
+ public void addRule(Rule rule) {
+ ((MemoryBasedBatchGrammar)backend).addRule(rule);
+ }
+
+ @Override
+ public void addOOVRules(int sourceWord, List<FeatureFunction> featureFunctions) {
+ // TODO: _OOV shouldn't be outright added, since the word might not be OOV for the LM (but now almost
+ // certainly is)
+ int targetWord = config.mark_oovs
+ ? Vocabulary.id(Vocabulary.word(sourceWord) + "_OOV")
+ : sourceWord;
+
+ int nt_i = Vocabulary.id("[X]");
+ Rule oovRule = new Rule(nt_i, new int[] { nt_i, sourceWord },
+ new int[] { -1, targetWord }, "", 1, null);
+ addRule(oovRule);
+ oovRule.estimateRuleCost(featureFunctions);
+
+// String ruleString = String.format("[X] ||| [X,1] %s ||| [X,1] %s",
+// Vocabulary.word(sourceWord), Vocabulary.word(targetWord));
+// Rule oovRule = new HieroFormatReader().parseLine(ruleString);
+// oovRule.setOwner(Vocabulary.id("oov"));
+// addRule(oovRule);
+// oovRule.estimateRuleCost(featureFunctions);
+ }
+
+ @Override
+ public Trie getTrieRoot() {
+ return backend.getTrieRoot();
+ }
+
+ @Override
+ public void sortGrammar(List<FeatureFunction> models) {
+ backend.sortGrammar(models);
+ }
+
+ @Override
+ public boolean isSorted() {
+ return backend.isSorted();
+ }
+
+ /**
+ * This should never be called.
+ */
+ @Override
+ public boolean hasRuleForSpan(int startIndex, int endIndex, int pathLength) {
+ return true;
+ }
+
+ @Override
+ public int getNumRules() {
+ return backend.getNumRules();
+ }
+
+ @Override
+ public int getOwner() {
+ return backend.getOwner();
+ }
+
+ @Override
+ public int getNumDenseFeatures() {
+ return backend.getNumDenseFeatures();
+ }
+}
[15/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/packed/small_grammar
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/packed/small_grammar b/joshua-core/src/test/java/org/apache/joshua/packed/small_grammar
new file mode 100644
index 0000000..e7ee25f
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/packed/small_grammar
@@ -0,0 +1,20000 @@
+[$+CD] ||| [$,1] 3 ||| [$,1] 3 ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=0.7621400520468967 LexprobTargetGivenSource=0.6967122467244414 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[$+CD] ||| \u0686\u06cc\u0646 3 ||| china 3 ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=0.8648618605282223 LexprobTargetGivenSource=0.9027900600445733 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[$+CD] ||| \u0686\u06cc\u0646 [CD,1] ||| china [CD,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=0.10272180848132564 LexprobTargetGivenSource=0.20607781332013186 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[$+PP] ||| % [PP,1] ||| $ [PP,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=5.225746673713201 LexprobTargetGivenSource=5.241747015059643 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=1.791759469228055 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[$+PP] ||| % \u06a9\u06d2 \u0645\u0639\u0645\u0648\u0644\u06cc \u0633\u06d2 \u0641\u0631\u0642 \u0633\u06d2 ||| $ with less difference ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=12.353420607276608 LexprobTargetGivenSource=18.48498678369073 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=4 UnalignedSource=0 UnalignedTarget=0
+[$] ||| % ||| $ ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=5.225746673713201 LexprobTargetGivenSource=5.241747015059643 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=3.1354942159291497 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=4.976733742420574 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[$] ||| 3 ||| $ ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=4.127134385045092 LexprobTargetGivenSource=4.539742380665636 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=3.1354942159291497 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=5.231108616854587 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[$] ||| [$,1] \u0633\u06d2 ||| [$,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=-0.0 LexprobTargetGivenSource=2.15539786892241 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=1 TargetPhraseGivenSource=0.6931471805599453 TargetTerminalsButNoSource=0 TargetWords=0 UnalignedSource=1 UnalignedTarget=0
+[$] ||| \u0641\u0648\u062c ||| soldiers ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=1.8827312474337816 LexprobTargetGivenSource=3.011089929208311 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=5.003946305945459 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=6.431331081933479 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[$] ||| \u0686\u06cc\u0646 ||| china ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=0.10272180848132564 LexprobTargetGivenSource=0.20607781332013186 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=6.60934924316738 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=6.6320017773956295 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[$] ||| \u0688\u0627\u0644\u0631 ||| $ ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=1.44155703979494 LexprobTargetGivenSource=1.3862943611198906 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.1253517471925912E-7 SourcePhraseGivenTarget=0.3022808718729337 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=1.7619065060783738 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[$] ||| \u0688\u0627\u0644\u0631 \u0633\u06d2 ||| $ ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=1.44155703979494 LexprobTargetGivenSource=3.5416922300423 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=3.1354942159291497 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=1.3862943611198906 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=1 UnalignedTarget=0
+[$] ||| \u0688\u0627\u0644\u0631\u0632 ||| $ ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=3.028522096376982 LexprobTargetGivenSource=1.491654876777717 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=0.1353352832366127 SourcePhraseGivenTarget=2.03688192726104 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=1.2992829841302609 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''+CC+''] ||| & quot ; [CC+'',1] ||| ' [CC+'',1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=2.6570542160672286 LexprobTargetGivenSource=9.427365955854611 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=0.1353352832366127 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''+CC+''] ||| & quot ; [CC,1] ['',2] ||| ' [CC,1] ['',2] ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=2.6570542160672286 LexprobTargetGivenSource=9.427365955854611 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''+CC+''] ||| & quot ; \u0646 & quot ; ||| ' n ' ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=6.273884275948351 LexprobTargetGivenSource=19.653239607927 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=0.1353352832366127 SourcePhraseGivenTarget=0.5108256237659907 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=0.9808292530117262 TargetTerminalsButNoSource=0 TargetWords=3 UnalignedSource=0 UnalignedTarget=0
+[''+CC+''] ||| & quot ; \u0646 ['',1] ||| ' n ['',1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=3.6168300598811225 LexprobTargetGivenSource=10.225873652072384 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[''+CC+''] ||| [''+CC,1] & quot ; ||| [''+CC,1] ' ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=2.6570542160672286 LexprobTargetGivenSource=9.427365955854611 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''+CC+''] ||| ['',1] [CC,2] & quot ; ||| ['',1] [CC,2] ' ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=2.6570542160672286 LexprobTargetGivenSource=9.427365955854611 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''+CC+''] ||| ['',1] \u0646 & quot ; ||| ['',1] n ' ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=3.616830059881123 LexprobTargetGivenSource=10.225873652072384 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=0.1353352832366127 SourcePhraseGivenTarget=0.2876820724517809 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=0.2876820724517809 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[''+CC+''] ||| ['',1] \u0646 ['',2] ||| ['',1] n ['',2] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=0.9597758438138939 LexprobTargetGivenSource=0.7985076962177716 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''+CC] ||| & quot ; [CC,1] ||| ' [CC,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=2.6570542160672286 LexprobTargetGivenSource=9.427365955854611 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=1.9459101490553135 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''+CC] ||| & quot ; \u0646 ||| ' n ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=3.6168300598811225 LexprobTargetGivenSource=10.225873652072384 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=1.6094379124341003 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[''+CC] ||| ['',1] \u0646 ||| ['',1] n ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=0.9597758438138939 LexprobTargetGivenSource=0.7985076962177716 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''+COMMA+CC] ||| [''+COMMA,1] \u0627\u0648\u0631 ||| [''+COMMA,1] and ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=0.2377656632040374 LexprobTargetGivenSource=0.21111757781868512 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''+COMMA+CC] ||| ['',1] [CC,2] ||| ['',1] , [CC,2] ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=1.0106232503834969 LexprobTargetGivenSource=-0.0 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=1 TargetWords=1 UnalignedSource=0 UnalignedTarget=1
+[''+COMMA+CC] ||| ['',1] \u0627\u0648\u0631 ||| ['',1] , and ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=1.2483889135875343 LexprobTargetGivenSource=0.21111757781868512 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=1
+[''+COMMA+CC] ||| \u06be\u06d2 [CC,1] ||| ' , [CC,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=7.745214910356445 LexprobTargetGivenSource=4.143134726391533 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=1
+[''+COMMA+CC] ||| \u06be\u06d2 [COMMA+CC,1] ||| ' [COMMA+CC,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=6.734591659972948 LexprobTargetGivenSource=4.143134726391533 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''+COMMA+CC] ||| \u06be\u06d2 \u0627\u0648\u0631 ||| ' , and ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=7.982980573560482 LexprobTargetGivenSource=4.354252304210218 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=3 UnalignedSource=0 UnalignedTarget=1
+[''+COMMA+RB] ||| [''+COMMA,1] \u062a\u0648 ||| [''+COMMA,1] then ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=0.8358389425831322 LexprobTargetGivenSource=1.4882749380860638 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''+COMMA+RB] ||| ['',1] [RB,2] ||| ['',1] , [RB,2] ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=1.0106232503834969 LexprobTargetGivenSource=-0.0 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=1 TargetWords=1 UnalignedSource=0 UnalignedTarget=1
+[''+COMMA+RB] ||| ['',1] \u062a\u0648 ||| ['',1] , then ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=1.8464621929666292 LexprobTargetGivenSource=1.4882749380860638 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=1
+[''+COMMA+RB] ||| \u0627\u062a\u0631\u06d2 [COMMA+RB,1] ||| ' [COMMA+RB,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=6.734591659972948 LexprobTargetGivenSource=2.833213344056216 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''+COMMA+RB] ||| \u0627\u062a\u0631\u06d2 [RB,1] ||| ' , [RB,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=7.745214910356445 LexprobTargetGivenSource=2.833213344056216 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=1
+[''+COMMA+RB] ||| \u0627\u062a\u0631\u06d2 \u062a\u0648 ||| ' , then ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=8.581053852939577 LexprobTargetGivenSource=4.32148828214228 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=0.6931471805599453 TargetTerminalsButNoSource=0 TargetWords=3 UnalignedSource=0 UnalignedTarget=1
+[''+COMMA] ||| & # 39 ; ||| ' , ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=8.515295833329414 LexprobTargetGivenSource=10.98738576052509 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=1.791759469228055 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=0.6931471805599453 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[''+COMMA] ||| & # 39 ; \u06a9\u06d2 ||| ' , ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=8.515295833329414 LexprobTargetGivenSource=12.797830610431983 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=1.791759469228055 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=1 UnalignedTarget=0
+[''+COMMA] ||| & quot ; ||| ' , ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=3.6676774664507255 LexprobTargetGivenSource=9.427365955854611 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=1.791759469228055 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=5.60947179518496 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=1
+[''+COMMA] ||| [''+COMMA,1] \u06a9\u06d2 ||| [''+COMMA,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=-0.0 LexprobTargetGivenSource=1.8104448499068928 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=1 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=0 UnalignedSource=1 UnalignedTarget=0
+[''+COMMA] ||| ['',1] ||| ['',1] , ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=1.0106232503834969 LexprobTargetGivenSource=-0.0 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=0.1353352832366127 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=1 TargetWords=1 UnalignedSource=0 UnalignedTarget=1
+[''+COMMA] ||| \u0627\u062a\u0631\u06d2 ||| ' , ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=7.745214910356445 LexprobTargetGivenSource=2.833213344056216 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=1.791759469228055 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=1.791759469228055 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=1
+[''+COMMA] ||| \u06be\u06d2 ||| ' , ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=7.745214910356445 LexprobTargetGivenSource=4.143134726391533 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=1.791759469228055 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=3.0910424533583156 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=1
+[''+FW+''] ||| & quot ; [FW+'',1] ||| ' [FW+'',1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=2.6570542160672286 LexprobTargetGivenSource=9.427365955854611 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''+FW+''] ||| & quot ; \u0646 & quot ; ||| ' n ' ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=6.490692008813475 LexprobTargetGivenSource=20.038085428832424 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=1.6094379124341003 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=2.0794415416798357 TargetTerminalsButNoSource=0 TargetWords=3 UnalignedSource=0 UnalignedTarget=0
+[''+FW+''] ||| ['',1] \u0646 & quot ; ||| ['',1] n ' ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=3.833637792746247 LexprobTargetGivenSource=10.610719472977813 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=1.3862943611198906 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=1.3862943611198906 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[''+IN+PRP] ||| may ||| ' as we ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=20.2301586275293 LexprobTargetGivenSource=2.0149030205422647 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=3 UnalignedSource=0 UnalignedTarget=0
+[''+JJ+''] ||| \u063a\u0627\u0632\u06cc ||| ' ghazi ' ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=10.941717055739101 LexprobTargetGivenSource=1.6471782404169475 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=2.70805020110221 TargetTerminalsButNoSource=0 TargetWords=3 UnalignedSource=0 UnalignedTarget=0
+[''+JJ+''] ||| \u063a\u06cc\u0631 \u062a\u063a\u06cc\u0631 \u067e\u0630\u06cc\u0631 ||| ' unmodified ' ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=11.207420221472105 LexprobTargetGivenSource=9.98436193938638 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=0.6931471805599453 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=2.70805020110221 TargetTerminalsButNoSource=0 TargetWords=3 UnalignedSource=0 UnalignedTarget=0
+[''+JJ+NN] ||| & quot ; [JJ+NN,1] ||| ' [JJ+NN,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=2.6570542160672286 LexprobTargetGivenSource=9.427365955854611 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=1.0986122886681098 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=2.1972245773362196 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''+JJ+NN] ||| & quot ; [JJ,1] [NN,2] ||| ' [JJ,1] [NN,2] ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=2.6570542160672286 LexprobTargetGivenSource=9.427365955854611 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=1.0986122886681098 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=2.4849066497880004 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''+JJ+NN] ||| & quot ; [JJ,1] \u062c\u06cc\u0679 ||| ' [JJ,1] jet ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=4.361802308305654 LexprobTargetGivenSource=9.65050950716882 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=0.6931471805599453 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[''+JJ+NN] ||| & quot ; \u06a9\u0631\u0627\u0686\u06cc [NN,1] ||| ' karachi [NN,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=2.694380173679915 LexprobTargetGivenSource=9.523286077733351 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=0.6931471805599453 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[''+JJ+NN] ||| & quot ; \u06a9\u0631\u0627\u0686\u06cc \u062c\u06cc\u0679 ||| ' karachi jet ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=4.399128265918341 LexprobTargetGivenSource=9.746429629047562 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=1.0986122886681098 TargetTerminalsButNoSource=0 TargetWords=3 UnalignedSource=0 UnalignedTarget=0
+[''+JJ+NN] ||| [''+JJ,1] \u062c\u06cc\u0679 ||| [''+JJ,1] jet ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=1.7047480922384253 LexprobTargetGivenSource=0.2231435513142097 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''+JJ+NN] ||| ['',1] [JJ,2] \u062c\u06cc\u0679 ||| ['',1] [JJ,2] jet ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=1.7047480922384253 LexprobTargetGivenSource=0.2231435513142097 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''+JJ+NN] ||| ['',1] \u06a9\u0631\u0627\u0686\u06cc [NN,2] ||| ['',1] karachi [NN,2] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=0.037325957612686436 LexprobTargetGivenSource=0.09592012187873925 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''+JJ+NN] ||| ['',1] \u06a9\u0631\u0627\u0686\u06cc \u062c\u06cc\u0679 ||| ['',1] karachi jet ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=1.7420740498511118 LexprobTargetGivenSource=0.3190636731929489 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[''+JJ] ||| & quot ; [JJ,1] ||| ' [JJ,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=2.6570542160672286 LexprobTargetGivenSource=9.427365955854611 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=0.36787944117144233 SourcePhraseGivenTarget=0.6931471805599453 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=2.4849066497880004 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''+JJ] ||| & quot ; \u0645\u062d\u0645\u062f ||| ' muhammad ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=2.8073364191165666 LexprobTargetGivenSource=10.09926802780082 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=0.6931471805599453 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[''+JJ] ||| & quot ; \u06a9\u0631\u0627\u0686\u06cc ||| ' karachi ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=2.694380173679915 LexprobTargetGivenSource=9.523286077733351 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=1.3862943611198906 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[''+JJ] ||| ['',1] \u0645\u062d\u0645\u062f ||| ['',1] muhammad ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=0.150282203049338 LexprobTargetGivenSource=0.671902071946209 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''+JJ] ||| ['',1] \u06a9\u0631\u0627\u0686\u06cc ||| ['',1] karachi ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=0.037325957612686436 LexprobTargetGivenSource=0.09592012187873925 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''+NN+''] ||| \u0645\u0645\u0644\u0648\u06a9 ||| ' mameluke ' ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=12.99917969070016 LexprobTargetGivenSource=2.1400661634962708 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=0.6931471805599453 TargetTerminalsButNoSource=0 TargetWords=3 UnalignedSource=0 UnalignedTarget=0
+[''+NN+:] ||| \u0645\u0631\u06af ||| ' marg-e-amboh ' ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=9.269478242065968 LexprobTargetGivenSource=3.401197381662155 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=3.912023005428146 TargetTerminalsButNoSource=0 TargetWords=3 UnalignedSource=0 UnalignedTarget=0
+[''+NN] ||| & # 39 ; & quot ; ||| ' " ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=4.235121556955648 LexprobTargetGivenSource=20.862763409845336 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[''+NN] ||| & quot ; [NN,1] ||| ' [NN,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=2.6570542160672286 LexprobTargetGivenSource=9.427365955854611 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=1.791759469228055 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=3.295836866004329 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''+NN] ||| & quot ; \u0686\u06cc\u0646 ||| ' china ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=2.7597760245485543 LexprobTargetGivenSource=9.633443769174743 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=0.6931471805599453 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[''+NN] ||| ['',1] \u0686\u06cc\u0646 ||| ['',1] china ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=0.10272180848132564 LexprobTargetGivenSource=0.20607781332013186 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''+NN] ||| [NN,1] & quot ; ||| ' [NN,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=2.6570542160672286 LexprobTargetGivenSource=9.427365955854611 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=1.791759469228055 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=3.9318256327243257 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''+NN] ||| \u0622\u0633\u0679\u0631\u06cc\u0644\u06cc\u0627 & quot ; ||| ' australia ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=2.800155059707902 LexprobTargetGivenSource=9.606477237018318 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=1.3862943611198906 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[''+NN] ||| \u0622\u0633\u0679\u0631\u06cc\u0644\u06cc\u0627 ['',1] ||| ['',1] australia ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=0.1431008436406733 LexprobTargetGivenSource=0.17911128116370645 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=0.6931471805599453 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''+NP] ||| 1839\u0621 \u0645\u06cc\u06ba \u062a\u0646\u0638\u06cc\u0645\u0627\u062a ||| ' tanzeemat ' -lrb- reforms -rrb- in 1839 ad ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=39.80623151494288 LexprobTargetGivenSource=4.210216376947244 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=0.6931471805599453 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=9 UnalignedSource=0 UnalignedTarget=0
+[''+NP] ||| 1839\u0621 \u0645\u06cc\u06ba \u062a\u0646\u0638\u06cc\u0645\u0627\u062a \u06a9\u06d2 ||| ' tanzeemat ' -lrb- reforms -rrb- in 1839 ad ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=39.80623151494288 LexprobTargetGivenSource=6.020661226854137 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=0.6931471805599453 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=9 UnalignedSource=1 UnalignedTarget=0
+[''+NP] ||| [''+NP,1] \u06a9\u06d2 ||| [''+NP,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=-0.0 LexprobTargetGivenSource=1.8104448499068928 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=1 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=0 UnalignedSource=1 UnalignedTarget=0
+[''+NP] ||| \u06c1\u0648 \u062c\u0645\u0627\u0639 ||| ' jama 'a ' ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=12.917764299214653 LexprobTargetGivenSource=8.5962034135973 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=0.6931471805599453 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=4 UnalignedSource=0 UnalignedTarget=0
+[''+RB] ||| \u0634\u0627\u0628\u0627\u0634 ||| ' bravo ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=8.344029572407049 LexprobTargetGivenSource=2.4277482359480516 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=3.1780538303479458 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[''+VBD+PP] ||| ['',1] \u0631\u06cc\u0634\u0645 [IN,2] \u0628\u0646\u06d2 \u066c ||| ['',1] made [IN,2] silk ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=5.932852871912751 LexprobTargetGivenSource=3.620002531377767 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[''+VBD+PP] ||| ['',1] \u0631\u06cc\u0634\u0645 [VBD+IN,2] \u066c ||| ['',1] [VBD+IN,2] silk ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=1.270462545594769 LexprobTargetGivenSource=0.9889133714116851 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''+VBD+PP] ||| ['',1] \u0631\u06cc\u0634\u0645 \u0633\u06d2 [VBD,2] \u066c ||| ['',1] [VBD,2] of silk ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=6.799797146405125 LexprobTargetGivenSource=5.987974000264652 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[''+VBD+PP] ||| ['',1] \u0631\u06cc\u0634\u0645 \u0633\u06d2 \u0628\u0646\u06d2 \u066c ||| ['',1] made of silk ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=11.462187472723109 LexprobTargetGivenSource=8.619063160230734 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=3 UnalignedSource=0 UnalignedTarget=0
+[''+VBD+PP] ||| \u066c [VBD+PP,1] ||| ' [VBD+PP,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=4.655150118293112 LexprobTargetGivenSource=1.0560526742493137 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''+VBD+PP] ||| \u066c \u0631\u06cc\u0634\u0645 [IN,1] [VBD,2] \u066c ||| ' [VBD,2] [IN,1] silk ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=5.925612663887881 LexprobTargetGivenSource=2.044966045660999 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[''+VBD+PP] ||| \u066c \u0631\u06cc\u0634\u0645 [IN,1] \u0628\u0646\u06d2 \u066c ||| ' made [IN,1] silk ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=10.588002990205863 LexprobTargetGivenSource=4.6760552056270805 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=3 UnalignedSource=0 UnalignedTarget=0
+[''+VBD+PP] ||| \u066c \u0631\u06cc\u0634\u0645 [VBD+IN,1] \u066c ||| ' [VBD+IN,1] silk ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=5.925612663887881 LexprobTargetGivenSource=2.044966045660999 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[''+VBD+PP] ||| \u066c \u0631\u06cc\u0634\u0645 \u0633\u06d2 [VBD,1] \u066c ||| ' [VBD,1] of silk ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=11.454947264698239 LexprobTargetGivenSource=7.044026674513966 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=3 UnalignedSource=0 UnalignedTarget=0
+[''+VBD+PP] ||| \u066c \u0631\u06cc\u0634\u0645 \u0633\u06d2 \u0628\u0646\u06d2 \u066c ||| ' made of silk ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=16.11733759101622 LexprobTargetGivenSource=9.675115834480048 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=4 UnalignedSource=0 UnalignedTarget=0
+[''+VBD] ||| \u062f\u0628\u06cc ||| ' were ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=13.637334397131541 LexprobTargetGivenSource=2.0794415416798357 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=1.6094379124341003 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[''+VBP] ||| \u0627\u0639\u0645\u0627\u0644\u0650 ||| ' say ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=11.554873225577985 LexprobTargetGivenSource=2.5649493574615367 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[''+WP+VBZ] ||| & # 39 ; \u062c\u0633 \u0646\u06d2 ||| ' whoever has ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=7.308440371472846 LexprobTargetGivenSource=30.84656679170809 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=3 UnalignedSource=0 UnalignedTarget=0
+[''+WP+VBZ] ||| [''+WP,1] \u0646\u06d2 ||| [''+WP,1] has ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=1.5081579488767312 LexprobTargetGivenSource=2.544683853823861 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''+WP] ||| & # 39 ; \u062c\u0633 ||| ' whoever ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=5.800282422596115 LexprobTargetGivenSource=28.301882937884226 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[''+WRB+DT] ||| [''+WRB,1] ||| [''+WRB,1] the ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=2.5648556967318514 LexprobTargetGivenSource=-0.0 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=1 TargetWords=1 UnalignedSource=0 UnalignedTarget=1
+[''+WRB+DT] ||| ['',1] [WRB,2] ||| ['',1] [WRB,2] the ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=2.5648556967318514 LexprobTargetGivenSource=-0.0 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=1 TargetWords=1 UnalignedSource=0 UnalignedTarget=1
+[''+WRB+DT] ||| ['',1] \u062c\u06c1\u0627\u06ba ||| ['',1] where the ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=3.404401580314737 LexprobTargetGivenSource=0.5634693572514127 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=0.6931471805599453 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=1
+[''+WRB+DT] ||| \u066c [WRB+DT,1] ||| ' [WRB+DT,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=4.655150118293112 LexprobTargetGivenSource=1.0560526742493137 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''+WRB+DT] ||| \u066c [WRB,1] ||| ' [WRB,1] the ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=7.2200058150249635 LexprobTargetGivenSource=1.0560526742493137 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=0.6931471805599453 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=1
+[''+WRB+DT] ||| \u066c \u062c\u06c1\u0627\u06ba ||| ' where the ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=8.059551698607848 LexprobTargetGivenSource=1.6195220315007264 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=0.6931471805599453 TargetTerminalsButNoSource=0 TargetWords=3 UnalignedSource=0 UnalignedTarget=1
+[''+WRB] ||| ['',1] \u062c\u06c1\u0627\u06ba ||| ['',1] where ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=0.8395458835828856 LexprobTargetGivenSource=0.5634693572514127 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=0.6931471805599453 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''+WRB] ||| \u066c [WRB,1] ||| ' [WRB,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=4.655150118293112 LexprobTargetGivenSource=1.0560526742493137 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=0.6931471805599453 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''+WRB] ||| \u066c \u062c\u06c1\u0627\u06ba ||| ' where ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=5.494696001875997 LexprobTargetGivenSource=1.6195220315007264 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=0.6931471805599453 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[''] ||| & ||| ' ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=2.5757085766132763 LexprobTargetGivenSource=3.1986731175506815 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=4.189654742026425 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=2.890371757896165 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''] ||| & quot ; ||| ' ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=2.6570542160672286 LexprobTargetGivenSource=9.427365955854611 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=3.059023205018258E-7 SourcePhraseGivenTarget=1.4170660197866443 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=2.8368830729451786 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''] ||| 39 ||| ' ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=3.402387149797744 LexprobTargetGivenSource=1.7303905228517629 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=0.006737946999085467 SourcePhraseGivenTarget=2.3978952727983707 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=1.4271163556401458 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''] ||| computer ||| ' ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=5.125153747538848 LexprobTargetGivenSource=1.9459101490553135 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=4.189654742026425 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''] ||| � ||| ' ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=4.537367082636728 LexprobTargetGivenSource=0.5753641449035618 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=4.189654742026425 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=1.0986122886681098 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''] ||| \u0627\u062a\u0631\u06d2 ||| ' ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=6.734591659972948 LexprobTargetGivenSource=2.833213344056216 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=4.189654742026425 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=1.791759469228055 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''] ||| \u0627\u0644\u063a\u06cc\u0628 ||| ' ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=6.734591659972948 LexprobTargetGivenSource=1.3862943611198906 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=4.189654742026425 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=0.6931471805599453 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''] ||| \u0644\u0641\u0638 ||| ' ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=6.041444479413003 LexprobTargetGivenSource=5.465948207931987 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=4.189654742026425 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=5.1298987149230735 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''] ||| \u066c ||| ' ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=4.655150118293112 LexprobTargetGivenSource=1.0560526742493137 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=0.36787944117144233 SourcePhraseGivenTarget=3.4965075614664802 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[''] ||| \u06a9\u06d2 ['',1] ||| ['',1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=-0.0 LexprobTargetGivenSource=1.8104448499068928 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=1 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=0 UnalignedSource=1 UnalignedTarget=0
+[''] ||| \u06a9\u06d2 \u0644\u0641\u0638 ||| ' ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=6.041444479413003 LexprobTargetGivenSource=7.27639305783888 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=4.189654742026425 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=1 UnalignedTarget=0
+[''] ||| \u06be\u06d2 ||| ' ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=6.734591659972948 LexprobTargetGivenSource=4.143134726391533 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=4.189654742026425 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=3.0910424533583156 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+.] ||| \u06c1\u06d2 \u06d4 ||| . . ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=1.2076187244107395 LexprobTargetGivenSource=3.933680012365317 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=0.6931471805599453 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=6.918695219020472 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[.+.] ||| \u06d4 ||| . . ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=0.5457061856175345 LexprobTargetGivenSource=0.629905880914893 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=0.6931471805599453 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=9.419466131522189 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[.+:+WRB] ||| -rrb- - [WRB,1] ||| . - [WRB,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=5.8324639973897385 LexprobTargetGivenSource=4.561308942321265 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=1.0986122886681098 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[.+:+WRB] ||| -rrb- - \u062c\u0628 ||| . - when ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=6.34366129928675 LexprobTargetGivenSource=4.958062109827408 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=1.0986122886681098 TargetTerminalsButNoSource=0 TargetWords=3 UnalignedSource=0 UnalignedTarget=0
+[.+:+WRB] ||| -rrb- [:+WRB,1] ||| . [:+WRB,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=5.393332343398188 LexprobTargetGivenSource=4.002124840125332 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=1.0986122886681098 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+:+WRB] ||| -rrb- [:,1] [WRB,2] ||| . [:,1] [WRB,2] ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=5.393332343398188 LexprobTargetGivenSource=4.002124840125332 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=1.0986122886681098 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+:+WRB] ||| -rrb- [:,1] \u062c\u0628 ||| . [:,1] when ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=5.9045296452952 LexprobTargetGivenSource=4.398878007631475 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=1.0986122886681098 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[.+:+WRB] ||| [.+:,1] \u062c\u0628 ||| [.+:,1] when ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=0.5111973018970117 LexprobTargetGivenSource=0.3967531675061429 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+:+WRB] ||| [.,1] - [WRB,2] ||| [.,1] - [WRB,2] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=0.43913165399155 LexprobTargetGivenSource=0.5591841021959324 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+:+WRB] ||| [.,1] - \u062c\u0628 ||| [.,1] - when ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=0.9503289558885617 LexprobTargetGivenSource=0.9559372697020754 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[.+:+WRB] ||| [.,1] [:,2] \u062c\u0628 ||| [.,1] [:,2] when ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=0.5111973018970117 LexprobTargetGivenSource=0.3967531675061429 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+:] ||| -rrb- - ||| . - ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=5.8324639973897385 LexprobTargetGivenSource=4.561308942321265 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=3.2188758248682006 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[.+:] ||| -rrb- [:,1] ||| . [:,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=5.393332343398188 LexprobTargetGivenSource=4.002124840125332 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=3.5263605246161616 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+:] ||| [.,1] - ||| [.,1] - ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=0.43913165399155 LexprobTargetGivenSource=0.5591841021959324 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+ADVP+COMMA] ||| \u060c [ADVP,1] ||| . [ADVP,1] , ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=5.439500075458905 LexprobTargetGivenSource=1.0625571634247433 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=2.3025850929940455 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[.+ADVP+COMMA] ||| \u060c \u0622\u062c ||| . even today , ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=10.193828909957217 LexprobTargetGivenSource=3.0140177430107054 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=4 UnalignedSource=0 UnalignedTarget=0
+[.+CC+COMMA] ||| [.+CC,1] ||| [.+CC,1] , ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=1.0106232503834969 LexprobTargetGivenSource=-0.0 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=0.36787944117144233 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=0.40546510810816444 TargetTerminalsButNoSource=1 TargetWords=1 UnalignedSource=0 UnalignedTarget=1
+[.+CC+COMMA] ||| [.,1] [CC,2] ||| [.,1] [CC,2] , ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=1.0106232503834969 LexprobTargetGivenSource=-0.0 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=0.36787944117144233 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=1 TargetWords=1 UnalignedSource=0 UnalignedTarget=1
+[.+CC+COMMA] ||| [.,1] \u0644\u06cc\u06a9\u0646 ||| [.,1] but , ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=1.8404177467593419 LexprobTargetGivenSource=0.20982994615806316 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=0.6931471805599453 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=1.6094379124341003 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=1
+[.+CC+COMMA] ||| [.,1] \u0645\u06af\u0631 ||| [.,1] but , ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=2.3391707892910634 LexprobTargetGivenSource=0.3025706849455316 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=0.6931471805599453 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=1.9459101490553135 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=1
+[.+CC+COMMA] ||| \u060c [CC+COMMA,1] ||| . [CC+COMMA,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=4.753531143543536 LexprobTargetGivenSource=4.40437771751069 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=0.6931471805599453 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+CC+COMMA] ||| \u060c [CC,1] ||| . [CC,1] , ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=5.764154393927033 LexprobTargetGivenSource=4.40437771751069 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=0.6931471805599453 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=5.598421958998375 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=1
+[.+CC+COMMA] ||| \u060c \u0644\u06cc\u06a9\u0646 ||| . but , ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=6.593948890302878 LexprobTargetGivenSource=4.614207663668753 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=0.6931471805599453 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=3.6109179126442243 TargetTerminalsButNoSource=0 TargetWords=3 UnalignedSource=0 UnalignedTarget=1
+[.+CC+COMMA] ||| \u0631\u06a9\u06be\u06cc [CC+COMMA,1] ||| . [CC+COMMA,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=8.127699852817772 LexprobTargetGivenSource=3.2108436531709366 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=0.6931471805599453 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+CC+COMMA] ||| \u0631\u06a9\u06be\u06cc [CC,1] ||| . [CC,1] , ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=9.13832310320127 LexprobTargetGivenSource=3.2108436531709366 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=0.6931471805599453 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=0.6931471805599453 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=1
+[.+CC+COMMA] ||| \u0631\u06a9\u06be\u06cc \u0645\u06af\u0631 ||| . but , ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=10.466870642108834 LexprobTargetGivenSource=3.513414338116468 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=0.6931471805599453 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=0.6931471805599453 TargetTerminalsButNoSource=0 TargetWords=3 UnalignedSource=0 UnalignedTarget=1
+[.+CC+DT] ||| [.+CC,1] ||| [.+CC,1] the ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=2.5648556967318514 LexprobTargetGivenSource=-0.0 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=1.0986122886681098 TargetTerminalsButNoSource=1 TargetWords=1 UnalignedSource=0 UnalignedTarget=1
+[.+CC+DT] ||| [.,1] ||| [.,1] and the ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=5.574737082561018 LexprobTargetGivenSource=-0.0 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=2.772588722239781 TargetTerminalsButNoSource=1 TargetWords=2 UnalignedSource=0 UnalignedTarget=2
+[.+CC+DT] ||| \u06d4 ||| . and the ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=5.847590175369785 LexprobTargetGivenSource=0.629905880914893 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=9.419466131522189 TargetTerminalsButNoSource=0 TargetWords=3 UnalignedSource=0 UnalignedTarget=2
+[.+CC+JJ] ||| [.+CC,1] \u067e\u0631\u062a\u06af\u0627\u0644\u06cc ||| [.+CC,1] portugal ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=3.0910424533583156 LexprobTargetGivenSource=1.3862943611198906 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+CC+JJ] ||| [.,1] [CC,2] \u067e\u0631\u062a\u06af\u0627\u0644\u06cc ||| [.,1] [CC,2] portugal ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=3.0910424533583156 LexprobTargetGivenSource=1.3862943611198906 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+CC+JJ] ||| [.,1] \u0644\u06cc\u06a9\u0646 [JJ,2] ||| [.,1] but [JJ,2] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=0.8297944963758451 LexprobTargetGivenSource=0.20982994615806316 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+CC+JJ] ||| [.,1] \u0644\u06cc\u06a9\u0646 \u067e\u0631\u062a\u06af\u0627\u0644\u06cc ||| [.,1] but portugal ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=3.9208369497341606 LexprobTargetGivenSource=1.5961243072779538 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[.+CC+JJ] ||| \u062a\u06be\u0627 [CC+JJ,1] ||| . [CC+JJ,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=6.601643549322723 LexprobTargetGivenSource=5.024452439649398 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+CC+JJ] ||| \u062a\u06be\u0627 [CC,1] [JJ,2] ||| . [CC,1] [JJ,2] ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=6.601643549322723 LexprobTargetGivenSource=5.024452439649398 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+CC+JJ] ||| \u062a\u06be\u0627 [CC,1] \u067e\u0631\u062a\u06af\u0627\u0644\u06cc ||| . [CC,1] portugal ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=9.692686002681038 LexprobTargetGivenSource=6.410746800769289 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[.+CC+JJ] ||| \u062a\u06be\u0627 \u0644\u06cc\u06a9\u0646 [JJ,1] ||| . but [JJ,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=7.431438045698568 LexprobTargetGivenSource=5.234282385807461 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[.+CC+JJ] ||| \u062a\u06be\u0627 \u0644\u06cc\u06a9\u0646 \u067e\u0631\u062a\u06af\u0627\u0644\u06cc ||| . but portugal ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=10.522480499056883 LexprobTargetGivenSource=6.620576746927352 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=3 UnalignedSource=0 UnalignedTarget=0
+[.+CC+LS] ||| 2 [CC+LS,1] ||| . [CC+LS,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=8.638525476583762 LexprobTargetGivenSource=4.43477720005941 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+CC+LS] ||| 2 [CC,1] 4 ||| . [CC,1] 4 ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=9.237904497691199 LexprobTargetGivenSource=4.892358309306589 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=1.0986122886681098 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[.+CC+LS] ||| 2 [CC,1] [LS,2] ||| . [CC,1] [LS,2] ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=8.638525476583762 LexprobTargetGivenSource=4.43477720005941 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+CC+LS] ||| 2 \u0627\u0648\u0631 4 ||| . and 4 ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=9.475670160895236 LexprobTargetGivenSource=5.103475887125274 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=1.0986122886681098 TargetTerminalsButNoSource=0 TargetWords=3 UnalignedSource=0 UnalignedTarget=0
+[.+CC+LS] ||| 2 \u0627\u0648\u0631 [LS,1] ||| . and [LS,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=8.8762911397878 LexprobTargetGivenSource=4.6458947778780955 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[.+CC+LS] ||| [.+CC,1] 4 ||| [.+CC,1] 4 ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=0.5993790211074358 LexprobTargetGivenSource=0.4575811092471784 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+CC+LS] ||| [.,1] [CC,2] 4 ||| [.,1] [CC,2] 4 ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=0.5993790211074358 LexprobTargetGivenSource=0.4575811092471784 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+CC+LS] ||| [.,1] \u0627\u0648\u0631 4 ||| [.,1] and 4 ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=0.8371446843114732 LexprobTargetGivenSource=0.6686986870658636 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[.+CC+LS] ||| [.,1] \u0627\u0648\u0631 [LS,2] ||| [.,1] and [LS,2] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=0.2377656632040374 LexprobTargetGivenSource=0.21111757781868512 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+CC+NP] ||| [.,1] \u0627\u0648\u0631 [NP,2] ||| [.,1] and [NP,2] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=0.2377656632040374 LexprobTargetGivenSource=0.21111757781868512 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+CC+NP] ||| \u06af\u0626\u06cc\u06ba [CC+NP,1] ||| . [CC+NP,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=7.657696223572036 LexprobTargetGivenSource=3.5517702401415296 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+CC+NP] ||| \u06af\u0626\u06cc\u06ba [CC,1] [NP,2] ||| . [CC,1] [NP,2] ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=7.657696223572036 LexprobTargetGivenSource=3.5517702401415296 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+CC+NP] ||| \u06af\u0626\u06cc\u06ba \u0627\u0648\u0631 [NP,1] ||| . and [NP,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=7.895461886776073 LexprobTargetGivenSource=3.7628878179602148 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[.+CC+NP] ||| \u06af\u0626\u06cc\u06ba \u0627\u0648\u0631 [NP,1] [NP+IN,2] ||| . and [NP+IN,2] [NP,1] ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=7.895461886776073 LexprobTargetGivenSource=3.7628878179602148 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[.+CC+NP] ||| \u06af\u0626\u06cc\u06ba \u0627\u0648\u0631 [NP/NN,1] \u0641\u0644\u0648\u0631\u06cc\u0688\u0627 [NP+IN,2] ||| . and [NP+IN,2] [NP/NN,1] florida ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=8.633060829906853 LexprobTargetGivenSource=4.198205889218061 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=3 UnalignedSource=0 UnalignedTarget=0
+[.+CC+NP] ||| \u06af\u0626\u06cc\u06ba \u0627\u0648\u0631 \u0645\u0634\u0631\u0642\u06cc [JJ\NP,1] [NP+IN,2] ||| . and [NP+IN,2] east [JJ\NP,1] ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=9.08033451972859 LexprobTargetGivenSource=4.877142883698778 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=3 UnalignedSource=0 UnalignedTarget=0
+[.+CC+NP] ||| \u06af\u0626\u06cc\u06ba \u0627\u0648\u0631 \u0645\u0634\u0631\u0642\u06cc \u0627\u0648\u0631 \u0645\u063a\u0631\u0628\u06cc \u0641\u0644\u0648\u0631\u06cc\u0688\u0627 \u06a9\u06cc \u0634\u0627\u06c1\u06cc \u06a9\u0627\u0644\u0648\u0646\u06cc\u0648\u06ba ||| . and the royal colonies of east and west florida ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=15.902119832541338 LexprobTargetGivenSource=9.271204325878609 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=10 UnalignedSource=0 UnalignedTarget=0
+[.+CC+PP] ||| [.,1] [CD,2] \u0645\u06cc\u06ba \u0632\u0631\u0650 ||| [.,1] and in [CD,2] ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=8.555503879286055 LexprobTargetGivenSource=1.510958927793012 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[.+CC+PP] ||| [.,1] [PP,2] \u0632\u0631\u0650 ||| [.,1] and [PP,2] ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=8.299410246344124 LexprobTargetGivenSource=1.0116009116784799 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+CC+PP] ||| [.,1] \u0627\u0648\u06312006 [CC+IN,2] ||| [.,1] [CC+IN,2] 2006 ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=3.188416617383492 LexprobTargetGivenSource=0.5596157879354228 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+CC+PP] ||| [.,1] \u0627\u0648\u06312006 [IN,2] \u0632\u0631\u0650 ||| [.,1] and [IN,2] 2006 ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=11.487826863727616 LexprobTargetGivenSource=1.5712166996139025 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[.+CC+PP] ||| [.,1] \u0627\u0648\u06312006 \u0645\u06cc\u06ba [CC,2] ||| [.,1] [CC,2] in 2006 ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=3.4445102503254237 LexprobTargetGivenSource=1.058973804049955 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[.+CC+PP] ||| [.,1] \u0627\u0648\u06312006 \u0645\u06cc\u06ba \u0632\u0631\u0650 ||| [.,1] and in 2006 ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=11.743920496669547 LexprobTargetGivenSource=2.070574715728435 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=3 UnalignedSource=0 UnalignedTarget=0
+[.+CC+PP] ||| \u06c1\u06d2 [CC+PP,1] ||| . [CC+PP,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=3.72342260920907 LexprobTargetGivenSource=3.303774131450424 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=1.0986122886681098 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+CC+PP] ||| \u06c1\u06d2 [CD,1] [CC+IN,2] ||| . [CC+IN,2] [CD,1] ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=3.72342260920907 LexprobTargetGivenSource=3.303774131450424 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+CC+PP] ||| \u06c1\u06d2 [CD,1] [IN,2] \u0632\u0631\u0650 ||| . and [IN,2] [CD,1] ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=12.022832855553194 LexprobTargetGivenSource=4.315375043128904 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[.+CC+PP] ||| \u06c1\u06d2 [CD,1] \u0645\u06cc\u06ba [CC,2] ||| . [CC,2] in [CD,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=3.9795162421510017 LexprobTargetGivenSource=3.8031321475649564 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[.+CC+PP] ||| \u06c1\u06d2 [CD,1] \u0645\u06cc\u06ba \u0632\u0631\u0650 ||| . and in [CD,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=12.278926488495125 LexprobTargetGivenSource=4.814733059243436 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=3 UnalignedSource=0 UnalignedTarget=0
+[.+CC+PP] ||| \u06c1\u06d2 [PP,1] [CC,2] ||| . [CC,2] [PP,1] ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=3.72342260920907 LexprobTargetGivenSource=3.303774131450424 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+CC+PP] ||| \u06c1\u06d2 [PP,1] \u0632\u0631\u0650 ||| . and [PP,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=12.022832855553194 LexprobTargetGivenSource=4.315375043128904 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[.+CC+PP] ||| \u06c1\u06d2 \u0627\u0648\u06312006 [CC+IN,1] ||| . [CC+IN,1] 2006 ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=6.911839226592562 LexprobTargetGivenSource=3.8633899193858467 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[.+CC+PP] ||| \u06c1\u06d2 \u0627\u0648\u06312006 [IN,1] [CC,2] ||| . [CC,2] [IN,1] 2006 ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=6.911839226592562 LexprobTargetGivenSource=3.8633899193858467 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[.+CC+PP] ||| \u06c1\u06d2 \u0627\u0648\u06312006 [IN,1] \u0632\u0631\u0650 ||| . and [IN,1] 2006 ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=15.211249472936686 LexprobTargetGivenSource=4.874990831064327 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=3 UnalignedSource=0 UnalignedTarget=0
+[.+CC+PP] ||| \u06c1\u06d2 \u0627\u0648\u06312006 \u0645\u06cc\u06ba [CC,1] ||| . [CC,1] in 2006 ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=7.167932859534494 LexprobTargetGivenSource=4.362747935500379 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=3 UnalignedSource=0 UnalignedTarget=0
+[.+CC+PP] ||| \u06c1\u06d2 \u0627\u0648\u06312006 \u0645\u06cc\u06ba \u0632\u0631\u0650 ||| . and in 2006 ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=15.467343105878617 LexprobTargetGivenSource=5.374348847178859 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=0.6931471805599453 TargetTerminalsButNoSource=0 TargetWords=4 UnalignedSource=0 UnalignedTarget=0
+[.+CC+PRP] ||| [.+CC,1] \u0627\u0633\u06d2 ||| [.+CC,1] it ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=2.5204852396783473 LexprobTargetGivenSource=0.9282332693174079 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+CC+PRP] ||| [.,1] [CC,2] \u0627\u0633\u06d2 ||| [.,1] [CC,2] it ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=2.5204852396783473 LexprobTargetGivenSource=0.9282332693174079 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+CC+PRP] ||| [.,1] \u0627\u0648\u0631 [PRP,2] ||| [.,1] and [PRP,2] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=0.2377656632040374 LexprobTargetGivenSource=0.21111757781868512 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=0.6931471805599453 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+CC+PRP] ||| [.,1] \u0627\u0648\u0631 \u0627\u0633\u06d2 ||| [.,1] and it ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=2.7582509028823847 LexprobTargetGivenSource=1.1393508471360931 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[.+CC+PRP] ||| \u060c [CC+PRP,1] ||| . [CC+PRP,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=4.753531143543536 LexprobTargetGivenSource=4.40437771751069 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=1.791759469228055 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+CC+PRP] ||| \u060c [CC,1] [PRP,2] ||| . [CC,1] [PRP,2] ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=4.753531143543536 LexprobTargetGivenSource=4.40437771751069 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=1.791759469228055 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+CC+PRP] ||| \u060c [CC,1] \u0627\u0633\u06d2 ||| . [CC,1] it ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=7.274016383221883 LexprobTargetGivenSource=5.332610986828098 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=1.6094379124341003 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[.+CC+PRP] ||| \u060c \u0627\u0648\u0631 [PRP,1] ||| . and [PRP,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=4.991296806747573 LexprobTargetGivenSource=4.615495295329375 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=1.6094379124341003 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[.+CC+PRP] ||| \u060c \u0627\u0648\u0631 \u0627\u0633\u06d2 ||| . and it ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=7.51178204642592 LexprobTargetGivenSource=5.543728564646783 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=1.3862943611198906 TargetTerminalsButNoSource=0 TargetWords=3 UnalignedSource=0 UnalignedTarget=0
+[.+CC+S] ||| [CC+S,1] \u06d4 ||| . [CC+S,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=0.27285309280876724 LexprobTargetGivenSource=0.629905880914893 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=5.58724865840025 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+CC+S] ||| [CC,1] [S,2] \u06d4 ||| . [CC,1] [S,2] ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=0.27285309280876724 LexprobTargetGivenSource=0.629905880914893 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=5.5254529391317835 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+CC+S] ||| \u0627\u0648\u0631 [S,1] [.,2] ||| [.,2] and [S,1] ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=0.2377656632040374 LexprobTargetGivenSource=0.21111757781868512 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=4.962844630259907 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+CC+S] ||| \u0627\u0648\u0631 [S,1] \u06d4 ||| . and [S,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=0.5106187560128046 LexprobTargetGivenSource=0.8410234587335781 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=5.159055299214529 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[.+CC+S] ||| \u0627\u0648\u0631 \u0627\u0646\u06c1\u06cc \u062c\u0645\u0627\u0639\u062a\u0648\u06ba \u06a9\u06cc \u062d\u06a9\u0645\u0631\u0627\u0646\u06cc \u06c1\u06d2 \u06d4 ||| . and it is these parties that rule the country ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=30.492118568109724 LexprobTargetGivenSource=8.046409128023672 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=10 UnalignedSource=0 UnalignedTarget=0
+[.+CC+S] ||| \u0627\u0648\u0631 \u0627\u064f\u0646\u06c1\u0648\u06ba \u0646\u06d2 \u0633\u0631\u0645\u0627\u06cc\u06c1 \u06a9\u0627\u0631\u06cc \u0645\u06cc\u06ba \u062a\u06cc\u0632\u06cc \u0633\u06d2 \u0627\u0636\u0627\u0641\u06c1 \u06a9\u06cc\u0627 ||| . and they increased the investment rapidly ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=1 LexprobSourceGivenTarget=19.025451972678987 LexprobTargetGivenSource=24.697132636113896 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=0.6931471805599453 TargetTerminalsButNoSource=0 TargetWords=7 UnalignedSource=0 UnalignedTarget=0
+[.+CC+VP] ||| [.+CC,1] [NN+PP,2] \u0633\u0648\u0646\u067e\u0627 ||| [.+CC,1] gave the [NN+PP,2] ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=8.082308593196558 LexprobTargetGivenSource=2.3978952727983707 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=1
+[.+CC+VP] ||| [.+CC,1] [NP+PP,2] \u0633\u0648\u0646\u067e\u0627 ||| [.+CC,1] gave [NP+PP,2] ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=5.517452896464707 LexprobTargetGivenSource=2.3978952727983707 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+CC+VP] ||| [.+CC,1] [PP,2] \u0627\u0642\u062a\u062f\u0627\u0631 \u0633\u0648\u0646\u067e\u0627 ||| [.+CC,1] gave the power [PP,2] ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=10.290160263107063 LexprobTargetGivenSource=3.67776017107153 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=3 UnalignedSource=0 UnalignedTarget=1
+[.+CC+VP] ||| [.,1] \u0627\u0648\u0631 [NN+PP,2] \u0633\u0648\u0646\u067e\u0627 ||| [.,1] and gave the [NN+PP,2] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=8.320074256400595 LexprobTargetGivenSource=2.609012850617056 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=3 UnalignedSource=0 UnalignedTarget=1
+[.+CC+VP] ||| [.,1] \u0627\u0648\u0631 [NP+PP,2] \u0633\u0648\u0646\u067e\u0627 ||| [.,1] and gave [NP+PP,2] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=5.755218559668744 LexprobTargetGivenSource=2.609012850617056 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[.+CC+VP] ||| [.,1] \u0627\u0648\u0631 [VP,2] ||| [.,1] and [VP,2] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=0.2377656632040374 LexprobTargetGivenSource=0.21111757781868512 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+CC+VP] ||| \u06a9\u06cc\u0627 [CC+VP,1] ||| . [CC+VP,1] ||| Abstract=0 Adjacent=0 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=6.47904122723039 LexprobTargetGivenSource=4.857125434540847 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+CC+VP] ||| \u06a9\u06cc\u0627 [CC,1] [NN+PP,2] \u0633\u0648\u0646\u067e\u0627 ||| . [CC,1] gave the [NN+PP,2] ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=14.561349820426948 LexprobTargetGivenSource=7.2550207073392174 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=3 UnalignedSource=0 UnalignedTarget=1
+[.+CC+VP] ||| \u06a9\u06cc\u0627 [CC,1] [NP+PP,2] \u0633\u0648\u0646\u067e\u0627 ||| . [CC,1] gave [NP+PP,2] ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=11.996494123695097 LexprobTargetGivenSource=7.2550207073392174 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[.+CC+VP] ||| \u06a9\u06cc\u0627 [CC,1] [VP,2] ||| . [CC,1] [VP,2] ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=6.47904122723039 LexprobTargetGivenSource=4.857125434540847 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=1.0 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=1 UnalignedSource=0 UnalignedTarget=0
+[.+CC+VP] ||| \u06a9\u06cc\u0627 \u0627\u0648\u0631 [NN+PP,1] [VBD+DT,2] ||| . and [VBD+DT,2] [NN+PP,1] ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0 Lexical=0 LexprobSourceGivenTarget=6.716806890434427 LexprobTargetGivenSource=5.068243012359532 Monotonic=1 PhrasePenalty=2.718 RarityPenalty=0.36787944117144233 SourcePhraseGivenTarget=-0.0 SourceTerminalsButNoTarget=0 TargetPhraseGivenSource=-0.0 TargetTerminalsButNoSource=0 TargetWords=2 UnalignedSource=0 UnalignedTarget=0
+[.+CC+VP] ||| \u06a9\u06cc\u0627 \u0627\u0648\u0631 [NN+PP,1] [VBD,2] ||| . and [VBD,2] the [NN+PP,1] ||| Abstract=0 Adjacent=1 ContainsX=0 GlueRule=0
<TRUNCATED>
[14/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/packed/test.sh
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/packed/test.sh b/joshua-core/src/test/java/org/apache/joshua/packed/test.sh
new file mode 100644
index 0000000..be6cf27
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/packed/test.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# test the vocabulary
+# javac VocabTest.java
+# java -cp .:${JOSHUA}/bin VocabTest small_packed
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/system/AlignmentMapTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/system/AlignmentMapTest.java b/joshua-core/src/test/java/org/apache/joshua/system/AlignmentMapTest.java
new file mode 100644
index 0000000..eba732a
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/system/AlignmentMapTest.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.system;
+
+import static org.junit.Assert.*;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.ff.tm.Rule;
+
+import org.junit.Before;
+import org.junit.Test;
+
+public class AlignmentMapTest {
+
+ private Rule rule1 = null;
+ private Rule rule2 = null;
+ private static Map<Integer, List<Integer>> expectedAlignmentMap = null;
+ private static final int[] expectedNonTerminalPositions = {2,5};
+
+ @Before
+ public void setUp() throws Exception {
+ Vocabulary.clear();
+ int[] sourceRhs = {Vocabulary.id("A1"),Vocabulary.id("A2"),-1,Vocabulary.id("B"),Vocabulary.id("C"),-2};
+ int[] targetRhs = {Vocabulary.id("c"),Vocabulary.id("b1"),-1,Vocabulary.id("b2"),-4,Vocabulary.id("a")};
+ int arity = 2; // 2 non terminals
+ String alignment = "0-5 1-5 3-1 3-3 4-0";
+ expectedAlignmentMap = new HashMap<Integer, List<Integer>>();
+ expectedAlignmentMap.put(0, Arrays.asList(4));
+ expectedAlignmentMap.put(5, Arrays.asList(0,1));
+ expectedAlignmentMap.put(1, Arrays.asList(3));
+ expectedAlignmentMap.put(3, Arrays.asList(3));
+ rule1 = new Rule(-1, sourceRhs, targetRhs, "", arity, alignment);
+ rule2 = new Rule(-1, sourceRhs, targetRhs, "", arity, null); // rule with no alignment
+ }
+
+ @Test
+ public void test() {
+ // test regular rule with arity 2
+ Map<Integer, List<Integer>> alignmentMap1 = rule1.getAlignmentMap();
+ assertEquals(expectedAlignmentMap, alignmentMap1);
+ int[] nonTerminalPositions1 = rule1.getNonTerminalSourcePositions();
+ assertArrayEquals(expectedNonTerminalPositions, nonTerminalPositions1);
+
+ // test rule with no alignment
+ Map<Integer, List<Integer>> alignmentMap2 = rule2.getAlignmentMap();
+ assertTrue(alignmentMap2.isEmpty());
+ int[] nonTerminalPositions2 = rule2.getNonTerminalSourcePositions();
+ assertArrayEquals(expectedNonTerminalPositions, nonTerminalPositions2);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/system/KenLmTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/system/KenLmTest.java b/joshua-core/src/test/java/org/apache/joshua/system/KenLmTest.java
new file mode 100644
index 0000000..1f032d8
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/system/KenLmTest.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.system;
+
+import static org.apache.joshua.corpus.Vocabulary.registerLanguageModel;
+import static org.apache.joshua.corpus.Vocabulary.unregisterLanguageModels;
+import static org.junit.Assert.*;
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.ff.lm.KenLM;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+/**
+ * KenLM JNI interface tests.
+ * Loads libken.{so,dylib}.
+ * If run in Eclipse, add -Djava.library.path=build/lib to JVM arguments
+ * of the run configuration.
+ */
+@Ignore("re-enable as soon as kenlm native library support will be in place")
+public class KenLmTest {
+
+ private static final String LANGUAGE_MODEL_PATH = "resources/kenlm/oilers.kenlm";
+
+ @Test
+ public void givenKenLm_whenQueryingForNgramProbability_thenProbIsCorrect() {
+ // GIVEN
+ KenLM kenLm = new KenLM(3, LANGUAGE_MODEL_PATH);
+ int[] words = Vocabulary.addAll("Wayne Gretzky");
+ registerLanguageModel(kenLm);
+
+ // WHEN
+ float probability = kenLm.prob(words);
+
+ // THEN
+ assertEquals("Found the wrong probability for 2-gram \"Wayne Gretzky\"", -0.99f, probability,
+ Float.MIN_VALUE);
+ }
+
+ @Test
+ public void givenKenLm_whenQueryingForNgramProbability_thenIdAndStringMethodsReturnTheSame() {
+ // GIVEN
+ KenLM kenLm = new KenLM(LANGUAGE_MODEL_PATH);
+ registerLanguageModel(kenLm);
+ String sentence = "Wayne Gretzky";
+ String[] words = sentence.split("\\s+");
+ int[] ids = Vocabulary.addAll(sentence);
+
+ // WHEN
+ float prob_string = kenLm.prob(words);
+ float prob_id = kenLm.prob(ids);
+
+ // THEN
+ assertEquals("ngram probabilities differ for word and id based n-gram query", prob_string, prob_id,
+ Float.MIN_VALUE);
+
+ }
+
+ @Test
+ public void givenKenLm_whenIsKnownWord_thenReturnValuesAreCorrect() {
+ KenLM kenLm = new KenLM(LANGUAGE_MODEL_PATH);
+ assertTrue(kenLm.isKnownWord("Wayne"));
+ assertFalse(kenLm.isKnownWord("Wayne2222"));
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ Vocabulary.clear();
+ unregisterLanguageModels();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ Vocabulary.clear();
+ unregisterLanguageModels();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/system/MultithreadedTranslationTests.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/system/MultithreadedTranslationTests.java b/joshua-core/src/test/java/org/apache/joshua/system/MultithreadedTranslationTests.java
new file mode 100644
index 0000000..3901f40
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/system/MultithreadedTranslationTests.java
@@ -0,0 +1,156 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.system;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+
+import org.apache.joshua.decoder.Decoder;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.Translation;
+import org.apache.joshua.decoder.Translations;
+import org.apache.joshua.decoder.io.TranslationRequestStream;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Integration test for multithreaded Joshua decoder tests. Grammar used is a
+ * toy packed grammar.
+ *
+ * @author kellens
+ */
+public class MultithreadedTranslationTests {
+
+ private JoshuaConfiguration joshuaConfig = null;
+ private Decoder decoder = null;
+ private static final String INPUT = "A K B1 U Z1 Z2 B2 C";
+ private int previousLogLevel;
+ private final static long NANO_SECONDS_PER_SECOND = 1_000_000_000;
+
+ @Before
+ public void setUp() throws Exception {
+ joshuaConfig = new JoshuaConfiguration();
+ joshuaConfig.search_algorithm = "cky";
+ joshuaConfig.mark_oovs = false;
+ joshuaConfig.pop_limit = 100;
+ joshuaConfig.use_unique_nbest = false;
+ joshuaConfig.include_align_index = false;
+ joshuaConfig.topN = 0;
+ joshuaConfig.tms.add("thrax -owner pt -maxspan 20 -path resources/wa_grammar.packed");
+ joshuaConfig.tms.add("thrax -owner glue -maxspan -1 -path resources/grammar.glue");
+ joshuaConfig.goal_symbol = "[GOAL]";
+ joshuaConfig.default_non_terminal = "[X]";
+ joshuaConfig.features.add("OOVPenalty");
+ joshuaConfig.weights.add("tm_pt_0 1");
+ joshuaConfig.weights.add("tm_pt_1 1");
+ joshuaConfig.weights.add("tm_pt_2 1");
+ joshuaConfig.weights.add("tm_pt_3 1");
+ joshuaConfig.weights.add("tm_pt_4 1");
+ joshuaConfig.weights.add("tm_pt_5 1");
+ joshuaConfig.weights.add("tm_glue_0 1");
+ joshuaConfig.weights.add("OOVPenalty 2");
+ joshuaConfig.num_parallel_decoders = 500; // This will enable 500 parallel
+ // decoders to run at once.
+ // Useful to help flush out
+ // concurrency errors in
+ // underlying
+ // data-structures.
+ this.decoder = new Decoder(joshuaConfig, ""); // Second argument
+ // (configFile)
+ // is not even used by the
+ // constructor/initialize.
+
+ previousLogLevel = Decoder.VERBOSE;
+ Decoder.VERBOSE = 0;
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ this.decoder.cleanUp();
+ this.decoder = null;
+ Decoder.VERBOSE = previousLogLevel;
+ }
+
+
+
+ // This test was created specifically to reproduce a multithreaded issue
+ // related to mapped byte array access in the PackedGrammer getAlignmentArray
+ // function.
+
+ // We'll test the decoding engine using N = 10,000 identical inputs. This
+ // should be sufficient to induce concurrent data access for many shared
+ // data structures.
+
+ @Test
+ public void givenPackedGrammar_whenNTranslationsCalledConcurrently_thenReturnNResults() throws IOException {
+ // GIVEN
+
+ int inputLines = 10000;
+ joshuaConfig.use_structured_output = true; // Enabled alignments.
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < inputLines; i++) {
+ sb.append(INPUT + "\n");
+ }
+
+ // Append a large string together to simulate N requests to the decoding
+ // engine.
+ TranslationRequestStream req = new TranslationRequestStream(
+ new BufferedReader(new InputStreamReader(new ByteArrayInputStream(sb.toString()
+ .getBytes(Charset.forName("UTF-8"))))), joshuaConfig);
+
+ ByteArrayOutputStream output = new ByteArrayOutputStream();
+
+ // WHEN
+ // Translate all spans in parallel.
+ Translations translations = this.decoder.decodeAll(req);
+
+ ArrayList<Translation> translationResults = new ArrayList<Translation>();
+
+
+ final long translationStartTime = System.nanoTime();
+ try {
+ for (Translation t: translations)
+ translationResults.add(t);
+ } finally {
+ if (output != null) {
+ try {
+ output.close();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+
+ final long translationEndTime = System.nanoTime();
+ final double pipelineLoadDurationInSeconds = (translationEndTime - translationStartTime) / ((double)NANO_SECONDS_PER_SECOND);
+ System.err.println(String.format("%.2f seconds", pipelineLoadDurationInSeconds));
+
+ // THEN
+ assertTrue(translationResults.size() == inputLines);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/system/StructuredOutputTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/system/StructuredOutputTest.java b/joshua-core/src/test/java/org/apache/joshua/system/StructuredOutputTest.java
new file mode 100644
index 0000000..f5e9d34
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/system/StructuredOutputTest.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.system;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.joshua.decoder.Decoder;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.Translation;
+import org.apache.joshua.decoder.segment_file.Sentence;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.Assert;
+
+/**
+ * Integration test for the complete Joshua decoder using a toy grammar that translates
+ * a bunch of capital letters to lowercase letters. Rules in the test grammar
+ * drop and generate additional words and simulate reordering of rules, so that
+ * proper extraction of word alignments can be tested.
+ *
+ * @author fhieber
+ */
+public class StructuredOutputTest {
+
+ private JoshuaConfiguration joshuaConfig = null;
+ private Decoder decoder = null;
+ private Translation translation = null;
+ private static final String input = "A K B1 U Z1 Z2 B2 C";
+ private static final String expectedTranslation = "a b n1 u z c1 k1 k2 k3 n1 n2 n3 c2";
+ private static final String expectedWordAlignmentString = "0-0 2-1 6-1 3-3 4-4 5-4 7-5 1-6 1-7 1-8 7-12";
+ private static final List<List<Integer>> expectedWordAlignment = Arrays.asList(
+ Arrays.asList(0), Arrays.asList(2, 6), Arrays.asList(), Arrays.asList(3),
+ Arrays.asList(4, 5), Arrays.asList(7), Arrays.asList(1),
+ Arrays.asList(1), Arrays.asList(1), Arrays.asList(), Arrays.asList(),
+ Arrays.asList(), Arrays.asList(7));
+ private static final double expectedScore = -17.0;
+
+ @Before
+ public void setUp() throws Exception {
+ joshuaConfig = new JoshuaConfiguration();
+ joshuaConfig.search_algorithm = "cky";
+ joshuaConfig.mark_oovs = false;
+ joshuaConfig.pop_limit = 100;
+ joshuaConfig.use_unique_nbest = false;
+ joshuaConfig.include_align_index = false;
+ joshuaConfig.topN = 0;
+ joshuaConfig.tms.add("thrax -owner pt -maxspan 20 -path resources/wa_grammar");
+ joshuaConfig.tms.add("thrax -owner glue -maxspan -1 -path resources/grammar.glue");
+ joshuaConfig.goal_symbol = "[GOAL]";
+ joshuaConfig.default_non_terminal = "[X]";
+ joshuaConfig.features.add("OOVPenalty");
+ joshuaConfig.weights.add("tm_pt_0 1");
+ joshuaConfig.weights.add("tm_pt_1 1");
+ joshuaConfig.weights.add("tm_pt_2 1");
+ joshuaConfig.weights.add("tm_pt_3 1");
+ joshuaConfig.weights.add("tm_pt_4 1");
+ joshuaConfig.weights.add("tm_pt_5 1");
+ joshuaConfig.weights.add("tm_glue_0 1");
+ joshuaConfig.weights.add("OOVPenalty 2");
+ decoder = new Decoder(joshuaConfig, ""); // second argument (configFile
+ // is not even used by the
+ // constructor/initialize)
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ decoder.cleanUp();
+ decoder = null;
+ translation = null;
+ }
+
+ private Translation decode(String input) {
+ Sentence sentence = new Sentence(input, 0, joshuaConfig);
+ return decoder.decode(sentence);
+ }
+
+ @Test
+ public void test() {
+
+ // test standard output
+ joshuaConfig.use_structured_output = false;
+ joshuaConfig.outputFormat = "%s | %a ";
+ translation = decode(input);
+ Assert.assertEquals(expectedTranslation + " | "
+ + expectedWordAlignmentString, translation.toString().trim());
+
+ // test structured output
+ joshuaConfig.use_structured_output = true; // set structured output creation to true
+ translation = decode(input);
+ Assert.assertEquals(expectedTranslation, translation.getStructuredTranslations().get(0).getTranslationString());
+ Assert.assertEquals(Arrays.asList(expectedTranslation.split("\\s+")),
+ translation.getStructuredTranslations().get(0).getTranslationTokens());
+ Assert.assertEquals(expectedScore, translation.getStructuredTranslations().get(0).getTranslationScore(),
+ 0.00001);
+ Assert.assertEquals(expectedWordAlignment, translation.getStructuredTranslations().get(0).getTranslationWordAlignments());
+ Assert.assertEquals(translation.getStructuredTranslations().get(0).getTranslationWordAlignments().size(), translation
+ .getStructuredTranslations().get(0).getTranslationTokens().size());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/system/StructuredTranslationTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/system/StructuredTranslationTest.java b/joshua-core/src/test/java/org/apache/joshua/system/StructuredTranslationTest.java
new file mode 100644
index 0000000..6718858
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/system/StructuredTranslationTest.java
@@ -0,0 +1,274 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.system;
+
+import static java.util.Arrays.asList;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.joshua.decoder.Decoder;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.StructuredTranslation;
+import org.apache.joshua.decoder.Translation;
+import org.apache.joshua.decoder.segment_file.Sentence;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Integration test for the complete Joshua decoder using a toy grammar that translates
+ * a bunch of capital letters to lowercase letters. Rules in the test grammar
+ * drop and generate additional words and simulate reordering of rules, so that
+ * proper extraction of word alignments and other information from the decoder
+ * can be tested.
+ *
+ * @author fhieber
+ */
+public class StructuredTranslationTest {
+
+ private JoshuaConfiguration joshuaConfig = null;
+ private Decoder decoder = null;
+ private static final String INPUT = "A K B1 U Z1 Z2 B2 C";
+ private static final String EXPECTED_TRANSLATION = "a b n1 u z c1 k1 k2 k3 n1 n2 n3 c2";
+ private static final List<String> EXPECTED_TRANSLATED_TOKENS = asList(EXPECTED_TRANSLATION.split("\\s+"));
+ private static final String EXPECTED_WORD_ALIGNMENT_STRING = "0-0 2-1 6-1 3-3 4-4 5-4 7-5 1-6 1-7 1-8 7-12";
+ private static final List<List<Integer>> EXPECTED_WORD_ALIGNMENT = asList(
+ asList(0), asList(2, 6), asList(), asList(3),
+ asList(4, 5), asList(7), asList(1),
+ asList(1), asList(1), asList(), asList(),
+ asList(), asList(7));
+ private static final double EXPECTED_SCORE = -17.0;
+ private static final Map<String,Float> EXPECTED_FEATURES = new HashMap<>();
+ private static final int EXPECTED_NBEST_LIST_SIZE = 8;
+ static {
+ EXPECTED_FEATURES.put("tm_glue_0", 1.0f);
+ EXPECTED_FEATURES.put("tm_pt_0", -3.0f);
+ EXPECTED_FEATURES.put("tm_pt_1", -3.0f);
+ EXPECTED_FEATURES.put("tm_pt_2", -3.0f);
+ EXPECTED_FEATURES.put("tm_pt_3", -3.0f);
+ EXPECTED_FEATURES.put("tm_pt_4", -3.0f);
+ EXPECTED_FEATURES.put("tm_pt_5", -3.0f);
+ EXPECTED_FEATURES.put("OOV", 7.0f);
+ EXPECTED_FEATURES.put("OOVPenalty", 0.0f);
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ joshuaConfig = new JoshuaConfiguration();
+ joshuaConfig.search_algorithm = "cky";
+ joshuaConfig.mark_oovs = false;
+ joshuaConfig.pop_limit = 100;
+ joshuaConfig.use_unique_nbest = false;
+ joshuaConfig.include_align_index = false;
+ joshuaConfig.topN = 0;
+ joshuaConfig.tms.add("thrax -owner pt -maxspan 20 -path resources/wa_grammar");
+ joshuaConfig.tms.add("thrax -owner glue -maxspan -1 -path resources/grammar.glue");
+ joshuaConfig.goal_symbol = "[GOAL]";
+ joshuaConfig.default_non_terminal = "[X]";
+ joshuaConfig.features.add("OOVPenalty");
+ joshuaConfig.weights.add("tm_pt_0 1");
+ joshuaConfig.weights.add("tm_pt_1 1");
+ joshuaConfig.weights.add("tm_pt_2 1");
+ joshuaConfig.weights.add("tm_pt_3 1");
+ joshuaConfig.weights.add("tm_pt_4 1");
+ joshuaConfig.weights.add("tm_pt_5 1");
+ joshuaConfig.weights.add("tm_glue_0 1");
+ joshuaConfig.weights.add("OOVPenalty 1");
+ decoder = new Decoder(joshuaConfig, ""); // second argument (configFile
+ // is not even used by the
+ // constructor/initialize)
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ decoder.cleanUp();
+ decoder = null;
+ }
+
+ private Translation decode(String input) {
+ Sentence sentence = new Sentence(input, 0, joshuaConfig);
+ return decoder.decode(sentence);
+ }
+
+ @Test
+ public void givenInput_whenRegularOutputFormat_thenExpectedOutput() {
+ // GIVEN
+ joshuaConfig.use_structured_output = false;
+ joshuaConfig.outputFormat = "%s | %a ";
+
+ // WHEN
+ final String translation = decode(INPUT).toString().trim();
+
+ // THEN
+ assertEquals(EXPECTED_TRANSLATION + " | " + EXPECTED_WORD_ALIGNMENT_STRING, translation);
+ }
+
+ @Test
+ public void givenInput_whenRegularOutputFormatWithTopN1_thenExpectedOutput() {
+ // GIVEN
+ joshuaConfig.use_structured_output = false;
+ joshuaConfig.outputFormat = "%s | %e | %a | %c";
+ joshuaConfig.topN = 1;
+
+ // WHEN
+ final String translation = decode(INPUT).toString().trim();
+
+ // THEN
+ assertEquals(EXPECTED_TRANSLATION + " | " + INPUT + " | " + EXPECTED_WORD_ALIGNMENT_STRING + String.format(" | %.3f", EXPECTED_SCORE),
+ translation);
+ }
+
+ @Test
+ public void givenInput_whenStructuredOutputFormatWithTopN0_thenExpectedOutput() {
+ // GIVEN
+ joshuaConfig.use_structured_output = true;
+ joshuaConfig.topN = 0;
+
+ // WHEN
+ final Translation translation = decode(INPUT);
+ final StructuredTranslation structuredTranslation = translation.getStructuredTranslations().get(0);
+ final String translationString = structuredTranslation.getTranslationString();
+ final List<String> translatedTokens = structuredTranslation.getTranslationTokens();
+ final float translationScore = structuredTranslation.getTranslationScore();
+ final List<List<Integer>> wordAlignment = structuredTranslation.getTranslationWordAlignments();
+ final Map<String,Float> translationFeatures = structuredTranslation.getTranslationFeatures();
+
+ // THEN
+ assertTrue(translation.getStructuredTranslations().size() == 1);
+ assertEquals(EXPECTED_TRANSLATION, translationString);
+ assertEquals(EXPECTED_TRANSLATED_TOKENS, translatedTokens);
+ assertEquals(EXPECTED_SCORE, translationScore, 0.00001);
+ assertEquals(EXPECTED_WORD_ALIGNMENT, wordAlignment);
+ assertEquals(wordAlignment.size(), translatedTokens.size());
+ assertEquals(EXPECTED_FEATURES.entrySet(), translationFeatures.entrySet());
+ }
+
+ @Test
+ public void givenInput_whenStructuredOutputFormatWithTopN1_thenExpectedOutput() {
+ // GIVEN
+ joshuaConfig.use_structured_output = true;
+ joshuaConfig.topN = 1;
+
+ // WHEN
+ final Translation translation = decode(INPUT);
+ final List<StructuredTranslation> structuredTranslations = translation.getStructuredTranslations();
+ final StructuredTranslation structuredTranslation = structuredTranslations.get(0);
+ final String translationString = structuredTranslation.getTranslationString();
+ final List<String> translatedTokens = structuredTranslation.getTranslationTokens();
+ final float translationScore = structuredTranslation.getTranslationScore();
+ final List<List<Integer>> wordAlignment = structuredTranslation.getTranslationWordAlignments();
+ final Map<String,Float> translationFeatures = structuredTranslation.getTranslationFeatures();
+
+ // THEN
+ assertTrue(structuredTranslations.size() == 1);
+ assertEquals(EXPECTED_TRANSLATION, translationString);
+ assertEquals(EXPECTED_TRANSLATED_TOKENS, translatedTokens);
+ assertEquals(EXPECTED_SCORE, translationScore, 0.00001);
+ assertEquals(EXPECTED_WORD_ALIGNMENT, wordAlignment);
+ assertEquals(wordAlignment.size(), translatedTokens.size());
+ assertEquals(EXPECTED_FEATURES.entrySet(), translationFeatures.entrySet());
+ }
+
+ @Test
+ public void givenInput_whenStructuredOutputFormatWithKBest_thenExpectedOutput() {
+ // GIVEN
+ joshuaConfig.use_structured_output = true;
+ joshuaConfig.topN = 100;
+
+ // WHEN
+ final Translation translation = decode(INPUT);
+ final List<StructuredTranslation> structuredTranslations = translation.getStructuredTranslations();
+ final StructuredTranslation viterbiTranslation = structuredTranslations.get(0);
+ final StructuredTranslation lastKBest = structuredTranslations.get(structuredTranslations.size() - 1);
+
+ // THEN
+ assertEquals(structuredTranslations.size(), EXPECTED_NBEST_LIST_SIZE);
+ assertTrue(structuredTranslations.size() > 1);
+ assertEquals(EXPECTED_TRANSLATION, viterbiTranslation.getTranslationString());
+ assertEquals(EXPECTED_TRANSLATED_TOKENS, viterbiTranslation.getTranslationTokens());
+ assertEquals(EXPECTED_SCORE, viterbiTranslation.getTranslationScore(), 0.00001);
+ assertEquals(EXPECTED_WORD_ALIGNMENT, viterbiTranslation.getTranslationWordAlignments());
+ assertEquals(EXPECTED_FEATURES.entrySet(), viterbiTranslation.getTranslationFeatures().entrySet());
+ // last entry in KBEST is all input words untranslated, should have 8 OOVs.
+ assertEquals(INPUT, lastKBest.getTranslationString());
+ assertEquals(-800.0, lastKBest.getTranslationFeatures().get("OOVPenalty"), 0.0001);
+
+ }
+
+ @Test
+ public void givenEmptyInput_whenStructuredOutputFormat_thenEmptyOutput() {
+ // GIVEN
+ joshuaConfig.use_structured_output = true;
+
+ // WHEN
+ final Translation translation = decode("");
+ final StructuredTranslation structuredTranslation = translation.getStructuredTranslations().get(0);
+ final String translationString = structuredTranslation.getTranslationString();
+ final List<String> translatedTokens = structuredTranslation.getTranslationTokens();
+ final float translationScore = structuredTranslation.getTranslationScore();
+ final List<List<Integer>> wordAlignment = structuredTranslation.getTranslationWordAlignments();
+
+ // THEN
+ assertEquals("", translationString);
+ assertTrue(translatedTokens.isEmpty());
+ assertEquals(0, translationScore, 0.00001);
+ assertTrue(wordAlignment.isEmpty());
+ }
+
+ @Test
+ public void givenOOVInput_whenStructuredOutputFormat_thenOOVOutput() {
+ // GIVEN
+ joshuaConfig.use_structured_output = true;
+ final String input = "gabarbl";
+
+ // WHEN
+ final Translation translation = decode(input);
+ final StructuredTranslation structuredTranslation = translation.getStructuredTranslations().get(0);
+ final String translationString = structuredTranslation.getTranslationString();
+ final List<String> translatedTokens = structuredTranslation.getTranslationTokens();
+ final float translationScore = structuredTranslation.getTranslationScore();
+ final List<List<Integer>> wordAlignment = structuredTranslation.getTranslationWordAlignments();
+
+ // THEN
+ assertEquals(input, translationString);
+ assertTrue(translatedTokens.contains(input));
+ assertEquals(-99.0, translationScore, 0.00001);
+ assertTrue(wordAlignment.contains(asList(0)));
+ }
+
+ @Test
+ public void givenEmptyInput_whenRegularOutputFormat_thenNewlineOutput() {
+ // GIVEN
+ joshuaConfig.use_structured_output = false;
+ joshuaConfig.outputFormat = "%s";
+
+ // WHEN
+ final Translation translation = decode("");
+ final String translationString = translation.toString();
+
+ // THEN
+ assertEquals("\n", translationString);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/ui/tree_visualizer/tree/TreeTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/ui/tree_visualizer/tree/TreeTest.java b/joshua-core/src/test/java/org/apache/joshua/ui/tree_visualizer/tree/TreeTest.java
new file mode 100644
index 0000000..55e8f56
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/ui/tree_visualizer/tree/TreeTest.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.ui.tree_visualizer.tree;
+
+import java.util.List;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class TreeTest {
+ @Test(expectedExceptions = { IllegalArgumentException.class })
+ public void ctor_EmptyString_IllegalArgument() {
+ Tree tree = new Tree("");
+ Assert.assertEquals(tree.size(), 0);
+ }
+
+ @Test(expectedExceptions = { IllegalArgumentException.class })
+ public void ctor_TooFewCloseParens_IllegalArgument() {
+ Tree tree = new Tree("(A{0-1} foo");
+ Assert.assertEquals(tree.size(), 0);
+ }
+
+ @Test
+ public void simpleTree_correctSize() {
+ Tree tree = new Tree("(A{0-1} foo)");
+ Assert.assertEquals(tree.size(), 2);
+ }
+
+ @Test
+ public void simpleTree_correctRoot() {
+ Tree tree = new Tree("(A{0-1} foo)");
+ Tree.Node root = tree.root();
+ Assert.assertEquals(root.label(), "A");
+ Assert.assertEquals(root.sourceStartIndex(), 0);
+ Assert.assertEquals(root.sourceEndIndex(), 1);
+ Assert.assertEquals(root.children().size(), 1);
+ }
+
+ @Test
+ public void simpleTree_correctLeaf() {
+ Tree tree = new Tree("(A{0-1} foo)");
+ Tree.Node leaf = tree.root().children().get(0);
+ Assert.assertEquals(leaf.label(), "foo");
+ Assert.assertEquals(leaf.sourceStartIndex(), -1);
+ Assert.assertEquals(leaf.sourceEndIndex(), -1);
+ Assert.assertEquals(leaf.children().size(), 0);
+ }
+
+ @Test
+ public void simpleTree_toString() {
+ Tree tree = new Tree("(A{0-1} foo)");
+ Assert.assertEquals(tree.toString(), "(A{0-1} foo)");
+ }
+
+ @Test
+ public void trickyTree_children() {
+ Tree tree = new Tree("(A{0-2} foo (B{1-2} bar))");
+ List<Tree.Node> children = tree.root().children();
+ Assert.assertEquals(children.size(), 2);
+ Tree.Node foo = children.get(0);
+ Assert.assertEquals(foo.label(), "foo");
+ Assert.assertTrue(foo.isLeaf());
+ Assert.assertEquals(foo.sourceStartIndex(), -1);
+ Assert.assertEquals(foo.sourceEndIndex(), -1);
+ Tree.Node b = children.get(1);
+ Assert.assertEquals(b.label(), "B");
+ Assert.assertEquals(b.children().size(), 1);
+ Assert.assertFalse(b.isLeaf());
+ Assert.assertEquals(b.sourceStartIndex(), 1);
+ Assert.assertEquals(b.sourceEndIndex(), 2);
+ }
+
+ @Test
+ public void SourceStartComparator() {
+ Tree tree = new Tree("(A{0-2} foo (B{1-2} bar))");
+ Tree.Node a = tree.root();
+ Tree.Node b = a.children().get(1);
+ Tree.NodeSourceStartComparator cmp = new Tree.NodeSourceStartComparator();
+ Assert.assertTrue(cmp.compare(a, b) < 0);
+ }
+
+ @Test
+ public void SourceStartComparator_LeafSmallerThanAllInternals() {
+ Tree tree = new Tree("(A{0-2} foo (B{1-2} bar))");
+ Tree.Node a = tree.root();
+ Tree.Node foo = a.children().get(0);
+ Tree.Node b = a.children().get(1);
+ Tree.Node bar = b.children().get(0);
+ Tree.NodeSourceStartComparator cmp = new Tree.NodeSourceStartComparator();
+ Assert.assertTrue(cmp.compare(foo, a) < 0);
+ Assert.assertTrue(cmp.compare(foo, b) < 0);
+ Assert.assertTrue(cmp.compare(bar, a) < 0);
+ Assert.assertTrue(cmp.compare(bar, b) < 0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/util/BitsTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/util/BitsTest.java b/joshua-core/src/test/java/org/apache/joshua/util/BitsTest.java
new file mode 100644
index 0000000..50704dc
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/util/BitsTest.java
@@ -0,0 +1,187 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+/**
+ * Unit tests for doing bit twiddling.
+ *
+ * @author Lane Schwartz
+ */
+public class BitsTest {
+
+ @Test
+ public void positiveLowBitsLongEncoding() {
+
+ int[] highs = {Integer.MIN_VALUE, -1234567890, -1, 0, 1, 1234567890, Integer.MAX_VALUE};
+
+ for (int high : highs) {
+ for (int low=0, step=(Integer.MAX_VALUE/754); low>=0 && low<=Integer.MAX_VALUE; low+=step) {
+
+ Assert.assertTrue(step > 0);
+ Assert.assertTrue(low >= 0);
+
+ long encoded = Bits.encodeAsLong(high, low);
+
+ Assert.assertEquals(Bits.decodeHighBits(encoded), high);
+ Assert.assertEquals(Bits.decodeLowBits(encoded), low);
+ }
+ }
+
+ }
+
+ @Test
+ public void negativeLowBitsLongEncoding() {
+
+ int[] highs = {Integer.MIN_VALUE, -1234567890, -1, 0, 1, 1234567890, Integer.MAX_VALUE};
+
+ for (int high : highs) {
+ for (int low=Integer.MIN_VALUE, step=(Integer.MAX_VALUE/754); low<=0 && low>=Integer.MIN_VALUE; low-=step) {
+
+ Assert.assertTrue(step > 0);
+ Assert.assertTrue(low <= 0);
+
+ long encoded = Bits.encodeAsLong(high, low);
+
+ Assert.assertEquals(Bits.decodeHighBits(encoded), high);
+ Assert.assertEquals(Bits.decodeLowBits(encoded), low);
+ }
+ }
+ }
+
+
+ @Test
+ public void positiveHighBitsLongEncoding() {
+
+ int[] lows = {Integer.MIN_VALUE, -1234567890, -1, 0, 1, 1234567890, Integer.MAX_VALUE};
+
+ for (int low : lows) {
+ for (int high=0, step=(Integer.MAX_VALUE/754); high>=0 && high<=Integer.MAX_VALUE; high+=step) {
+
+ Assert.assertTrue(step > 0);
+ Assert.assertTrue(high >= 0);
+
+ long encoded = Bits.encodeAsLong(high, low);
+
+ Assert.assertEquals(Bits.decodeHighBits(encoded), high);
+ Assert.assertEquals(Bits.decodeLowBits(encoded), low);
+ }
+ }
+ }
+
+ @Test
+ public void negativeHighBitsLongEncoding() {
+
+ int[] lows = {Integer.MIN_VALUE, -1234567890, -1, 0, 1, 1234567890, Integer.MAX_VALUE};
+
+ for (int low : lows) {
+ for (int high=Integer.MIN_VALUE, step=(Integer.MAX_VALUE/754); high<=0 && high>=Integer.MIN_VALUE; high-=step) {
+
+ Assert.assertTrue(step > 0);
+ Assert.assertTrue(high <= 0);
+
+ long encoded = Bits.encodeAsLong(high, low);
+
+ Assert.assertEquals(Bits.decodeHighBits(encoded), high);
+ Assert.assertEquals(Bits.decodeLowBits(encoded), low);
+ }
+ }
+ }
+
+
+ @Test
+ public void positiveLowBitsIntEncoding() {
+
+ short[] highs = {Short.MIN_VALUE, -12345, -1, 0, 1, 12345, Short.MAX_VALUE};
+
+ for (short high : highs) {
+ for (short low=0, step=(Short.MAX_VALUE/75); low>=0 && low<=Short.MAX_VALUE; low+=step) {
+
+ Assert.assertTrue(step > 0);
+ Assert.assertTrue(low >= 0);
+
+ int encoded = Bits.encodeAsInt(high, low);
+
+ Assert.assertEquals(Bits.decodeHighBits(encoded), high);
+ Assert.assertEquals(Bits.decodeLowBits(encoded), low);
+ }
+ }
+
+ }
+
+ @Test
+ public void negativeLowBitsIntEncoding() {
+
+ short[] highs = {Short.MIN_VALUE, -12345, -1, 0, 1, 12345, Short.MAX_VALUE};
+
+ for (short high : highs) {
+ for (short low=0, step=(Short.MAX_VALUE/75); low>=0 && low>=Short.MIN_VALUE; low-=step) {
+
+ Assert.assertTrue(step > 0);
+ Assert.assertTrue(low <= 0);
+
+ int encoded = Bits.encodeAsInt(high, low);
+
+ Assert.assertEquals(Bits.decodeHighBits(encoded), high);
+ Assert.assertEquals(Bits.decodeLowBits(encoded), low);
+ }
+ }
+ }
+
+
+ @Test
+ public void positiveHighBitsIntEncoding() {
+
+ short[] lows = {Short.MIN_VALUE, -12345, -1, 0, 1, 12345, Short.MAX_VALUE};
+
+ for (short low : lows) {
+ for (short high=0, step=(Short.MAX_VALUE/75); high>=0 && high<=Short.MAX_VALUE; high+=step) {
+
+ Assert.assertTrue(step > 0);
+ Assert.assertTrue(high >= 0);
+
+ int encoded = Bits.encodeAsInt(high, low);
+
+ Assert.assertEquals(Bits.decodeHighBits(encoded), high);
+ Assert.assertEquals(Bits.decodeLowBits(encoded), low);
+ }
+ }
+ }
+
+ @Test
+ public void negativeHighBitsIntEncoding() {
+
+ short[] lows = {Short.MIN_VALUE, -12345, -1, 0, 1, 12345, Short.MAX_VALUE};
+
+ for (short low : lows) {
+ for (short high=0, step=(Short.MAX_VALUE/75); high>=0 && high>=Short.MIN_VALUE; high-=step) {
+
+ Assert.assertTrue(step > 0);
+ Assert.assertTrue(high <= 0);
+
+ int encoded = Bits.encodeAsInt(high, low);
+
+ Assert.assertEquals(Bits.decodeHighBits(encoded), high);
+ Assert.assertEquals(Bits.decodeLowBits(encoded), low);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/util/CacheTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/util/CacheTest.java b/joshua-core/src/test/java/org/apache/joshua/util/CacheTest.java
new file mode 100644
index 0000000..53b8eb2
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/util/CacheTest.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class CacheTest {
+
+ @Test
+ public void test() {
+
+ Cache<String,Integer> cache = new Cache<String,Integer>(5);
+
+ cache.put("a", 1);
+ cache.put("b", 2);
+ cache.put("c", 3);
+ cache.put("d", 4);
+ cache.put("e", 5);
+
+ Assert.assertTrue(cache.containsKey("a"));
+ Assert.assertTrue(cache.containsKey("b"));
+ Assert.assertTrue(cache.containsKey("c"));
+ Assert.assertTrue(cache.containsKey("d"));
+ Assert.assertTrue(cache.containsKey("e"));
+
+ // Access the "a" element in the cache
+ cache.get("a");
+
+ // Now add a new element that exceeds the capacity of the cache
+ cache.put("f", 6);
+
+ Assert.assertTrue(cache.containsKey("a"));
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/util/CountsTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/util/CountsTest.java b/joshua-core/src/test/java/org/apache/joshua/util/CountsTest.java
new file mode 100644
index 0000000..e6a20a4
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/util/CountsTest.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+/**
+ * Unit tests for Counts class.
+ *
+ * @author Lane Schwartz
+ */
+public class CountsTest {
+
+ @Test
+ public void verifyCounts() {
+
+ Counts<Integer,Integer> counts = new Counts<Integer,Integer>();
+
+ int maxA = 100;
+ int maxB = 100;
+
+ // Increment counts
+ for (int a=0; a<maxA; a++) {
+ for (int b=0; b<maxB; b++) {
+
+ for (int n=0, times=b%10; n<=times; n++) {
+ counts.incrementCount(a,b);
+ counts.incrementCount(null, b);
+ }
+
+ }
+
+ for (int n=0, times=10-a%10; n<times; n++) {
+ counts.incrementCount(a,null);
+ }
+ }
+
+ // Verify co-occurrence counts
+ for (int a=0; a<maxA; a++) {
+ for (int b=0; b<maxB; b++) {
+ int expected = b%10 + 1;
+ Assert.assertEquals(counts.getCount(a, b), expected);
+ Assert.assertEquals(counts.getCount(null, b), maxA*expected);
+ }
+
+ int expected = 10 - a%10;
+ Assert.assertEquals(counts.getCount(a, null), expected);
+ }
+
+ // Verify totals for B counts
+ for (int b=0; b<maxB; b++) {
+ int expected = maxA * 2 * (b%10 + 1);
+ Assert.assertEquals(counts.getCount(b), expected);
+ }
+
+ // Verify probabilities
+ for (int a=0; a<maxA; a++) {
+ for (int b=0; b<maxB; b++) {
+ float expected = 1.0f / (maxA*2);
+ Assert.assertEquals(counts.getProbability(a, b), expected);
+ Assert.assertEquals(counts.getProbability(null, b), 0.5f);
+ }
+
+ int aCounter = 0;
+ for (int b=0; b<maxB; b++) {
+ for (int n=0, times=b%10; n<=times; n++) {
+ aCounter++;
+ }
+ }
+ for (int n=0, times=10-a%10; n<times; n++) {
+ aCounter++;
+ }
+
+ float nullExpected = (float) (10-a%10) / (float) (aCounter);
+ Assert.assertEquals(counts.getReverseProbability(null, a), nullExpected);
+
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/util/FormatUtilsTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/util/FormatUtilsTest.java b/joshua-core/src/test/java/org/apache/joshua/util/FormatUtilsTest.java
new file mode 100644
index 0000000..84b418b
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/util/FormatUtilsTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.util;
+
+import static org.apache.joshua.util.FormatUtils.cleanNonTerminal;
+import static org.apache.joshua.util.FormatUtils.escapeSpecialSymbols;
+import static org.apache.joshua.util.FormatUtils.isNonterminal;
+import static org.apache.joshua.util.FormatUtils.ensureNonTerminalBrackets;
+import static org.apache.joshua.util.FormatUtils.stripNonTerminalIndex;
+import static org.apache.joshua.util.FormatUtils.unescapeSpecialSymbols;
+import static org.junit.Assert.*;
+
+import org.junit.Test;
+
+public class FormatUtilsTest {
+
+ @Test
+ public void givenTokens_whenIsNonTerminal_thenTokensCorrectlyClassified() {
+ assertTrue(isNonterminal("[X]"));
+ assertTrue(isNonterminal("[X,1]"));
+ assertFalse(isNonterminal("[]"));
+ assertFalse(isNonterminal("[X)"));
+ }
+
+ @Test
+ public void givenTokens_whenCleanNonTerminal_thenCorrectlyCleaned() {
+ assertEquals(cleanNonTerminal("[GOAL]"), "GOAL");
+ assertEquals(cleanNonTerminal("[X]"), "X");
+ assertEquals(cleanNonTerminal("[X,1]"), "X");
+ assertEquals(cleanNonTerminal("bla"), "bla");
+ assertEquals(cleanNonTerminal("[bla"), "[bla");
+ }
+
+ @Test
+ public void givenTokens_whenStripNonTerminalIndex_thenCorrectlyStripped() {
+ assertEquals(stripNonTerminalIndex("[X,1]"), "[X]");
+ assertEquals(stripNonTerminalIndex("[X,114]"), "[X]");
+ assertEquals(stripNonTerminalIndex("[X,]"), "[X]");
+ assertEquals(stripNonTerminalIndex("[X]"), "[X]");
+ assertEquals(stripNonTerminalIndex("[X"), "[[X]");
+ }
+
+ @Test
+ public void givenTokens_whenMarkup_thenCorrectMarkup() {
+ assertEquals(ensureNonTerminalBrackets("X"), "[X]");
+ }
+
+ @Test
+ public void givenSpecialSymbols_whenEscapeSpecialSymbols_thenCorrectlyEscaped() {
+ assertEquals(escapeSpecialSymbols("[ ] | ["), "-lsb- -rsb- -pipe- -lsb-");
+ }
+
+ @Test
+ public void givenEscapedSpecialSymbols_whenUnEscapeSpecialSymbols_thenCorrectlyUnEscaped() {
+ assertEquals(unescapeSpecialSymbols("-lsb- -rsb- -pipe- -lsb-"), "[ ] | [");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/util/io/BinaryTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/util/io/BinaryTest.java b/joshua-core/src/test/java/org/apache/joshua/util/io/BinaryTest.java
new file mode 100644
index 0000000..6739b8b
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/util/io/BinaryTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util.io;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+public class BinaryTest {
+
+ @Test
+ public void externalizeVocabulary() throws IOException, ClassNotFoundException {
+
+ Set<String> words = new HashSet<String>();
+
+ for (char c1='a'; c1<='z'; c1++) {
+ words.add(new String(new char[]{c1}));
+ for (char c2='a'; c2<='z'; c2++) {
+ words.add(new String(new char[]{c1,c2}));
+ }
+ }
+
+ Vocabulary vocab = new Vocabulary();
+ vocab.addAll(words.toArray(new String[words.size()]));
+
+ try {
+
+ File tempFile = File.createTempFile(BinaryTest.class.getName(), "vocab");
+ FileOutputStream outputStream = new FileOutputStream(tempFile);
+ @SuppressWarnings({ "unused", "resource" })
+ ObjectOutput out = new BinaryOut(outputStream, true);
+ vocab.write(tempFile.toString());
+
+ @SuppressWarnings("resource")
+ ObjectInput in = new BinaryIn(tempFile.getAbsolutePath(), Vocabulary.class);
+ Object o = in.readObject();
+ Assert.assertTrue(o instanceof Vocabulary);
+
+ Vocabulary newVocab = (Vocabulary) o;
+
+ Assert.assertNotNull(newVocab);
+ Assert.assertEquals(newVocab.size(), vocab.size());
+
+ Assert.assertTrue(newVocab.equals(vocab));
+
+ } catch (SecurityException e) {
+ Assert.fail("Operating system is unable to create a temp file required by this unit test: " + e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/zmert/BLEUTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/zmert/BLEUTest.java b/joshua-core/src/test/java/org/apache/joshua/zmert/BLEUTest.java
new file mode 100644
index 0000000..9423d88
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/zmert/BLEUTest.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.zmert;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.util.Scanner;
+
+import org.apache.joshua.metrics.BLEU;
+import org.apache.joshua.metrics.EvaluationMetric;
+import org.testng.Assert;
+import org.testng.annotations.Parameters;
+import org.testng.annotations.Test;
+
+/**
+ * Unit tests for BLEU class.
+ *
+ * @author Lane Schwartz
+ * @version $LastChangedDate$
+ */
+public class BLEUTest {
+
+ @Test
+ public void metricName() {
+
+ // Setup the EvaluationMetric class
+ EvaluationMetric.set_numSentences(0);
+ EvaluationMetric.set_refsPerSen(1);
+ EvaluationMetric.set_refSentences(null);
+
+ BLEU bleu = new BLEU();
+
+ Assert.assertEquals(bleu.get_metricName(), "BLEU");
+
+ }
+
+ @Test
+ public void defaultConstructor() {
+
+ // Setup the EvaluationMetric class
+ EvaluationMetric.set_numSentences(0);
+ EvaluationMetric.set_refsPerSen(1);
+ EvaluationMetric.set_refSentences(null);
+
+ BLEU bleu = new BLEU();
+
+ // Default constructor should use a maximum n-gram length of 4
+ Assert.assertEquals(bleu.getMaxGramLength(), 4);
+
+ // Default constructor should use the closest reference
+ Assert.assertEquals(bleu.getEffLengthMethod(), BLEU.EffectiveLengthMethod.CLOSEST);
+
+ }
+
+ @Test
+ public void simpleTest() {
+
+ String ref = "this is the fourth chromosome whose sequence has been completed to date . it comprises more than 87 million pairs of dna .";
+ String test = "this is the fourth chromosome to be fully sequenced up till now and it comprises of over 87 million pairs of deoxyribonucleic acid ( dna ) .";
+
+ // refSentences[i][r] stores the r'th reference of the i'th sentence
+ String[][] refSentences = new String[1][1];
+ refSentences[0][0] = ref;
+
+ EvaluationMetric.set_numSentences(1);
+ EvaluationMetric.set_refsPerSen(1);
+ EvaluationMetric.set_refSentences(refSentences);
+
+ BLEU bleu = new BLEU();
+
+ // testSentences[i] stores the candidate translation for the i'th sentence
+ String[] testSentences = new String[1];
+ testSentences[0] = test;
+ try {
+ // Check BLEU score matches
+ double actualScore = bleu.score(testSentences);
+ double expectedScore = 0.2513;
+ double acceptableScoreDelta = 0.00001f;
+
+ Assert.assertEquals(actualScore, expectedScore, acceptableScoreDelta);
+
+ // Check sufficient statistics match
+ int[] actualSS = bleu.suffStats(testSentences);
+ int[] expectedSS = {14,27,8,26,5,25,3,24,27,23};
+
+ Assert.assertEquals(actualSS[0], expectedSS[0], 0); // 1-gram matches
+ Assert.assertEquals(actualSS[1], expectedSS[1], 0); // 1-gram total
+ Assert.assertEquals(actualSS[2], expectedSS[2], 0); // 2-gram matches
+ Assert.assertEquals(actualSS[3], expectedSS[3], 0); // 2-gram total
+ Assert.assertEquals(actualSS[4], expectedSS[4], 0); // 3-gram matches
+ Assert.assertEquals(actualSS[5], expectedSS[5], 0); // 3-gram total
+ Assert.assertEquals(actualSS[6], expectedSS[6], 0); // 4-gram matches
+ Assert.assertEquals(actualSS[7], expectedSS[7], 0); // 4-gram total
+ Assert.assertEquals(actualSS[8], expectedSS[8], 0); // candidate length
+ Assert.assertEquals(actualSS[9], expectedSS[9], 0); // reference length
+ } catch (Exception e) {
+ Assert.fail();
+ }
+ }
+
+ @Parameters({"referenceFile","testFile"})
+ @Test(enabled=false)
+ public void fileTest(String referenceFile, String testFile) throws FileNotFoundException {
+
+ //TODO You can now read in the files, and do something useful with them.
+
+ @SuppressWarnings("resource")
+ Scanner refScanner = new Scanner(new File(referenceFile));
+
+ while (refScanner.hasNextLine()) {
+
+ @SuppressWarnings("unused")
+ String refLine = refScanner.nextLine();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/hiero/.gitignore
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/hiero/.gitignore b/joshua-core/src/test/resources/bn-en/hiero/.gitignore
new file mode 100644
index 0000000..1710208
--- /dev/null
+++ b/joshua-core/src/test/resources/bn-en/hiero/.gitignore
@@ -0,0 +1,4 @@
+diff
+log
+output
+output.scores
[19/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/quantization/BooleanQuantizer.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/quantization/BooleanQuantizer.java b/joshua-core/src/main/java/org/apache/joshua/util/quantization/BooleanQuantizer.java
new file mode 100644
index 0000000..accd933
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/quantization/BooleanQuantizer.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util.quantization;
+
+import java.nio.ByteBuffer;
+
+/**
+ * Standard quantizer for boolean types.
+ *
+ * @author jg
+ *
+ */
+public class BooleanQuantizer extends StatelessQuantizer {
+
+ public final float read(ByteBuffer stream, int position) {
+ return 1.0f;
+ }
+
+ public final void write(ByteBuffer stream, float value) {}
+
+ @Override
+ public String getKey() {
+ return "boolean";
+ }
+
+ public final int size() {
+ return 0;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/quantization/Quantizer.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/quantization/Quantizer.java b/joshua-core/src/main/java/org/apache/joshua/util/quantization/Quantizer.java
new file mode 100644
index 0000000..33a4e9a
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/quantization/Quantizer.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util.quantization;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public interface Quantizer {
+
+ public float read(ByteBuffer stream, int position);
+
+ public void write(ByteBuffer stream, float value);
+
+ public void initialize();
+
+ public void add(float key);
+
+ public void finalize();
+
+ public String getKey();
+
+ public void writeState(DataOutputStream out) throws IOException;
+
+ public void readState(DataInputStream in) throws IOException;
+
+ public int size();
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/quantization/QuantizerConfiguration.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/quantization/QuantizerConfiguration.java b/joshua-core/src/main/java/org/apache/joshua/util/quantization/QuantizerConfiguration.java
new file mode 100644
index 0000000..f4765f9
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/quantization/QuantizerConfiguration.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util.quantization;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.joshua.corpus.Vocabulary;
+
+public class QuantizerConfiguration {
+
+ private static final Quantizer DEFAULT;
+
+ private ArrayList<Quantizer> quantizers;
+ private Map<Integer, Integer> quantizerByFeatureId;
+
+ static {
+ DEFAULT = new BooleanQuantizer();
+ }
+
+ public QuantizerConfiguration() {
+ quantizers = new ArrayList<Quantizer>();
+ quantizerByFeatureId = new HashMap<Integer, Integer>();
+ }
+
+ public void add(String quantizer_key, List<Integer> feature_ids) {
+ Quantizer q = QuantizerFactory.get(quantizer_key);
+ quantizers.add(q);
+ int index = quantizers.size() - 1;
+ for (int feature_id : feature_ids)
+ quantizerByFeatureId.put(feature_id, index);
+ }
+
+ public void initialize() {
+ for (Quantizer q : quantizers)
+ q.initialize();
+ }
+
+ public void finalize() {
+ for (Quantizer q : quantizers)
+ q.finalize();
+ }
+
+ public final Quantizer get(int feature_id) {
+ Integer index = quantizerByFeatureId.get(feature_id);
+ return (index != null ? quantizers.get(index) : DEFAULT);
+ }
+
+ public void read(String file_name) throws IOException {
+ quantizers.clear();
+ quantizerByFeatureId.clear();
+
+ File quantizer_file = new File(file_name);
+ DataInputStream in_stream =
+ new DataInputStream(new BufferedInputStream(new FileInputStream(quantizer_file)));
+ int num_quantizers = in_stream.readInt();
+ quantizers.ensureCapacity(num_quantizers);
+ for (int i = 0; i < num_quantizers; i++) {
+ String key = in_stream.readUTF();
+ Quantizer q = QuantizerFactory.get(key);
+ q.readState(in_stream);
+ quantizers.add(q);
+ }
+ int num_mappings = in_stream.readInt();
+ for (int i = 0; i < num_mappings; i++) {
+ String feature_name = in_stream.readUTF();
+ int feature_id = Vocabulary.id(feature_name);
+ int quantizer_index = in_stream.readInt();
+ if (quantizer_index >= num_quantizers) {
+ throw new RuntimeException("Error deserializing QuanitzerConfig. " + "Feature "
+ + feature_name + " referring to quantizer " + quantizer_index + " when only "
+ + num_quantizers + " known.");
+ }
+ this.quantizerByFeatureId.put(feature_id, quantizer_index);
+ }
+ in_stream.close();
+ }
+
+ public void write(String file_name) throws IOException {
+ File vocab_file = new File(file_name);
+ DataOutputStream out_stream =
+ new DataOutputStream(new BufferedOutputStream(new FileOutputStream(vocab_file)));
+ out_stream.writeInt(quantizers.size());
+ for (int index = 0; index < quantizers.size(); index++)
+ quantizers.get(index).writeState(out_stream);
+ out_stream.writeInt(quantizerByFeatureId.size());
+ for (int feature_id : quantizerByFeatureId.keySet()) {
+ out_stream.writeUTF(Vocabulary.word(feature_id));
+ out_stream.writeInt(quantizerByFeatureId.get(feature_id));
+ }
+ out_stream.close();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/quantization/QuantizerFactory.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/quantization/QuantizerFactory.java b/joshua-core/src/main/java/org/apache/joshua/util/quantization/QuantizerFactory.java
new file mode 100644
index 0000000..687b1da
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/quantization/QuantizerFactory.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util.quantization;
+
+
+public class QuantizerFactory {
+
+ public static Quantizer get(String key) {
+ if ("boolean".equals(key)) {
+ return new BooleanQuantizer();
+
+// } else if ("byte".equals(key)) {
+// return new ByteQuantizer();
+//
+// } else if ("char".equals(key)) {
+// return new CharQuantizer();
+//
+// } else if ("short".equals(key)) {
+// return new ShortQuantizer();
+//
+// } else if ("float".equals(key)) {
+// return new FloatQuantizer();
+//
+// } else if ("int".equals(key)) {
+// return new IntQuantizer();
+//
+// } else if ("8bit".equals(key)) {
+// return new EightBitQuantizer();
+
+ } else {
+ throw new RuntimeException("Unknown quantizer type: " + key);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/quantization/StatelessQuantizer.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/quantization/StatelessQuantizer.java b/joshua-core/src/main/java/org/apache/joshua/util/quantization/StatelessQuantizer.java
new file mode 100644
index 0000000..e81e945
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/quantization/StatelessQuantizer.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util.quantization;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+abstract class StatelessQuantizer implements Quantizer {
+
+ public void initialize() {}
+
+ public void add(float key) {}
+
+ public void finalize() {}
+
+ public void writeState(DataOutputStream out) throws IOException {
+ out.writeUTF(getKey());
+ }
+
+ public void readState(DataInputStream in) throws IOException {}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/quantization/package-info.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/quantization/package-info.java b/joshua-core/src/main/java/org/apache/joshua/util/quantization/package-info.java
new file mode 100644
index 0000000..2418577
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/quantization/package-info.java
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util.quantization;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/zmert/IntermediateOptimizer.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/zmert/IntermediateOptimizer.java b/joshua-core/src/main/java/org/apache/joshua/zmert/IntermediateOptimizer.java
new file mode 100644
index 0000000..7cde513
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/zmert/IntermediateOptimizer.java
@@ -0,0 +1,991 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.zmert;
+
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.text.DecimalFormat;
+import java.util.Date;
+import java.util.Iterator;
+import java.util.TreeMap;
+import java.util.TreeSet;
+import java.util.Vector;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.Semaphore;
+
+import org.apache.joshua.metrics.EvaluationMetric;
+
+public class IntermediateOptimizer implements Runnable {
+ /* non-static data members */
+ private int j;
+ private Semaphore blocker;
+ private Vector<String> threadOutput;
+ private String strToPrint;
+
+ private double[] initialLambda;
+ private double[] finalLambda;
+ private int[][] best1Cand_suffStats;
+ private double[] finalScore;
+ private int[] candCount;
+ private double[][][] featVal_array;
+ private ConcurrentHashMap<Integer, int[]>[] suffStats_array;
+
+ /* static data members */
+ private final static DecimalFormat f4 = new DecimalFormat("###0.0000");
+ private final static double NegInf = (-1.0 / 0.0);
+ private final static double PosInf = (+1.0 / 0.0);
+
+ private static int numSentences;
+ private static int numDocuments;
+ private static int[] docOfSentence;
+ private static int docSubset_firstRank;
+ private static int docSubset_lastRank;
+ private static boolean optimizeSubset;
+ private static int numParams;
+ private static double[] normalizationOptions;
+ private static boolean[] isOptimizable;
+ private static double[] minThValue;
+ private static double[] maxThValue;
+ private static boolean oneModificationPerIteration;
+ private static EvaluationMetric evalMetric;
+ private static String metricName;
+ private static String metricName_display;
+ private static int suffStatsCount;
+ private static String tmpDirPrefix;
+ private static int verbosity;
+
+ public static void set_MERTparams(int in_numSentences, int in_numDocuments,
+ int[] in_docOfSentence, int[] in_docSubsetInfo, int in_numParams,
+ double[] in_normalizationOptions, boolean[] in_isOptimizable, double[] in_minThValue,
+ double[] in_maxThValue, boolean in_oneModificationPerIteration,
+ EvaluationMetric in_evalMetric, String in_tmpDirPrefix, int in_verbosity) {
+ numSentences = in_numSentences;
+ numDocuments = in_numDocuments;
+ docOfSentence = in_docOfSentence;
+
+ docSubset_firstRank = in_docSubsetInfo[1];
+ docSubset_lastRank = in_docSubsetInfo[2];
+ if (in_docSubsetInfo[3] != numDocuments)
+ optimizeSubset = true;
+ else
+ optimizeSubset = false;
+
+ numParams = in_numParams;
+ normalizationOptions = in_normalizationOptions;
+ isOptimizable = in_isOptimizable;
+ minThValue = in_minThValue;
+ maxThValue = in_maxThValue;
+ oneModificationPerIteration = in_oneModificationPerIteration;
+ evalMetric = in_evalMetric;
+ metricName = evalMetric.get_metricName();
+ metricName_display = metricName;
+ if (numDocuments > 1) metricName_display = "doc-level " + metricName;
+ suffStatsCount = evalMetric.get_suffStatsCount();
+ tmpDirPrefix = in_tmpDirPrefix;
+ verbosity = in_verbosity;
+ }
+
+ public IntermediateOptimizer(int in_j, Semaphore in_blocker, Vector<String> in_threadOutput,
+ double[] in_initialLambda, double[] in_finalLambda, int[][] in_best1Cand_suffStats,
+ double[] in_finalScore, int[] in_candCount, double[][][] in_featVal_array,
+ ConcurrentHashMap<Integer, int[]>[] in_suffStats_array) {
+ j = in_j;
+ blocker = in_blocker;
+ threadOutput = in_threadOutput;
+ strToPrint = "";
+
+ initialLambda = in_initialLambda;
+ finalLambda = in_finalLambda;
+ best1Cand_suffStats = in_best1Cand_suffStats;
+ finalScore = in_finalScore;
+ candCount = in_candCount;
+ featVal_array = in_featVal_array;
+ suffStats_array = in_suffStats_array;
+ }
+
+ // private TreeMap<Double,TreeMap> thresholdsForParam(int c, int[] candCount, double[][][]
+ // featVal_array, double[] currLambda, TreeSet<Integer>[] indicesOfInterest)
+ private void set_thresholdsForParam(TreeMap<Double, TreeMap<Integer, int[]>> thresholdsAll,
+ int c, double[] currLambda, TreeSet<Integer>[] indicesOfInterest) {
+ /*
+ * TreeMap[] thresholds = new TreeMap[numSentences]; // thresholds[i] stores thresholds for the
+ * cth parameter obtained by // processing the candidates of sentence i. It not only stores the
+ * // thresholds themselves, but also a triple of {i,from,to}, where from/to // are indices that
+ * characterize the 1-best switch at this threshold.
+ *
+ * for (int i = 0; i < numSentences; ++i) { thresholds[i] = new TreeMap<Double,int[]>(); }
+ */
+
+ // TreeMap<Double,int[]> thresholds = new TreeMap<Double,int[]>();
+
+ // Find threshold points
+ // TreeMap<Double,TreeMap> thresholdsAll = new TreeMap<Double,TreeMap>();
+ thresholdsAll.clear();
+
+ for (int i = 0; i < numSentences; ++i) {
+ // find threshold points contributed by ith sentence
+
+ // println("Processing sentence #" + i,4);
+
+ int numCandidates = candCount[i];
+ // aka simply K
+
+ double[] slope = new double[numCandidates];
+ // will be h_c from candidatesInfo
+ // repeated here for easy access
+ double[] offset = new double[numCandidates];
+ // SUM_j!=c currLambda_j*h_j(x)
+
+ int minSlopeIndex = -1; // index of line with steepest descent...
+ double minSlope = PosInf; // ...and its slope...
+ double offset_minSlope = NegInf; // ...and its offset (needed to break ties)
+
+ int maxSlopeIndex = -1; // index of line with steepest ascent...
+ double maxSlope = NegInf; // ...and its slope...
+ double offset_maxSlope = NegInf; // ...and its offset (needed to break ties)
+
+ double bestScore_left = NegInf; // these are used if the min/max values are
+ double bestScore_right = NegInf; // not neg/pos infinity
+
+ for (int k = 0; k < numCandidates; ++k) {
+ slope[k] = featVal_array[c][i][k];
+
+ offset[k] = 0.0;
+ for (int c2 = 1; c2 <= numParams; ++c2) {
+ if (c2 != c) {
+ offset[k] += currLambda[c2] * featVal_array[c2][i][k];
+ }
+ }
+
+ // debugging
+ // println("@ (i,k)=(" + i + "," + k + "), "
+ // + "slope = " + slope[k] + "; offset = " + offset[k],4);
+
+ if (minThValue[c] == NegInf) {
+ if (slope[k] < minSlope || (slope[k] == minSlope && offset[k] > offset_minSlope)) {
+ minSlopeIndex = k;
+ minSlope = slope[k];
+ offset_minSlope = offset[k];
+ }
+ } else {
+ double score = offset[k] + ((minThValue[c] - 0.1) * slope[k]);
+ if (score > bestScore_left || (score == bestScore_left && slope[k] > minSlope)) {
+ minSlopeIndex = k;
+ minSlope = slope[k];
+ bestScore_left = score;
+ }
+ }
+
+ if (maxThValue[c] == PosInf) {
+ if (slope[k] > maxSlope || (slope[k] == maxSlope && offset[k] > offset_maxSlope)) {
+ maxSlopeIndex = k;
+ maxSlope = slope[k];
+ offset_maxSlope = offset[k];
+ }
+ } else {
+ double score = offset[k] + ((maxThValue[c] + 0.1) * slope[k]);
+ if (score > bestScore_right || (score == bestScore_right && slope[k] < maxSlope)) {
+ maxSlopeIndex = k;
+ maxSlope = slope[k];
+ bestScore_right = score;
+ }
+ }
+ }
+
+ // debugging
+ // println("minSlope is @ k = " + minSlopeIndex + ": slope " + minSlope
+ // + " (offset " + offset_minSlope + ")",4);
+ // println("maxSlope is @ k = " + maxSlopeIndex + ": slope " + maxSlope
+ // + " (offset " + offset_maxSlope + ")",4);
+
+
+ // some lines can be eliminated: the ones that have a lower offset
+ // than some other line with the same slope.
+ // That is, for any k1 and k2:
+ // if slope[k1] = slope[k2] and offset[k1] > offset[k2],
+ // then k2 can be eliminated.
+ // (This is actually important to do as it eliminates a bug.)
+ // HashSet<Integer> discardedIndices = indicesToDiscard(slope,offset);
+
+
+ // println("Extracting thresholds[(i,c)=(" + i + "," + c + ")]",4);
+
+ int currIndex = minSlopeIndex;
+ // As we traverse the currLambda_c dimension, the "winner" candidate will
+ // change at intersection points. currIndex tells us which candidate is
+ // the winner in the interval currently under investigation.
+
+ // We traverse the lambda_c dimension starting at -Inf. The line with
+ // steepest descent is the winner as lambda_c -> -Inf, so we initialize
+ // currIndex to minSlopeIndex to reflect that fact.
+
+ // Similarly, the winner as lambda_c -> +Inf is the line with the
+ // steepest *ascent* (i.e. max slope), and so we continue finding
+ // intersection points until we hit that line.
+
+ // Notice that we didn't have to investigate the entire space (-Inf,+Inf)
+ // if the parameter's range is more restricted than that. That is why, in
+ // the loop above, the "left-most" winner is not necessarily the one with
+ // the steepest descent (though it will be if minThValue[c] is -Inf).
+ // And similarly, the "right-most" winner is not necessarily the one with
+ // the steepest ascent (though it will be if minThValue[c] is +Inf). The
+ // point of doing this is to avoid extracting thresholds that will end up
+ // being discarded anyway due to range constraints, thus saving us a little
+ // bit of time.
+
+ int last_new_k = -1;
+
+ while (currIndex != maxSlopeIndex) {
+
+ if (currIndex < 0) break;
+ // Due to rounding errors, the index identified as maxSlopeIndex above
+ // might be different from the one this loop expects, in which case
+ // it won't be found and currIndex remains -1. So if currIndex is -1
+ // a rounding error happened, which is cool since we can just break.
+
+ // print("cI=" + currIndex + " ",4);
+
+ // find the candidate whose line is the first to intersect the current
+ // line. ("first" meaning with an intersection point that has the
+ // lowest possible lambda_c value.)
+
+ double nearestIntersectionPoint = PosInf;
+ int nearestIntersectingLineIndex = -1;
+
+ for (int k = 0; k < numCandidates; ++k) {
+ // if (slope[k] > slope[currIndex] && !discardedIndices.contains(k)) {
+ if (slope[k] > slope[currIndex]) {
+ // only higher-sloped lines will intersect the current line
+ // (If we didn't have discardedIndices a bug would creep up here.)
+
+ // find intersection point ip_k
+ double ip_k = (offset[k] - offset[currIndex]) / (slope[currIndex] - slope[k]);
+ if (ip_k < nearestIntersectionPoint) {
+ nearestIntersectionPoint = ip_k;
+ nearestIntersectingLineIndex = k;
+ }
+ }
+ }
+
+ // print("ip=" + f4.format(nearestIntersectionPoint) + " ",4);
+
+ if (nearestIntersectionPoint > minThValue[c] && nearestIntersectionPoint < maxThValue[c]) {
+
+ int[] th_info = {currIndex, nearestIntersectingLineIndex};
+ last_new_k = nearestIntersectingLineIndex;
+
+ indicesOfInterest[i].add(currIndex); // old_k
+ // indicesOfInterest_all[i].add(currIndex); // old_k ***/
+
+ if (!thresholdsAll.containsKey(nearestIntersectionPoint)) {
+ TreeMap<Integer, int[]> A = new TreeMap<Integer, int[]>();
+ A.put(i, th_info);
+ thresholdsAll.put(nearestIntersectionPoint, A);
+ } else {
+ TreeMap<Integer, int[]> A = thresholdsAll.get(nearestIntersectionPoint);
+ if (!A.containsKey(i)) {
+ A.put(i, th_info);
+ } else {
+ int[] old_th_info = A.get(i);
+ old_th_info[1] = th_info[1]; // replace the existing new_k
+ A.put(i, th_info);
+ }
+ thresholdsAll.put(nearestIntersectionPoint, A);
+ }
+ /*
+ * if (!thresholds.containsKey(nearestIntersectionPoint)) {
+ * thresholds.put(nearestIntersectionPoint,th_info); // i.e., at lambda_c = nIP, the
+ * (index of the) 1-best changes // from currIndex to nearestIntersectingLineIndex (which
+ * is // indicated in th_info) } else { // extremely rare, but causes problem if it does
+ * occur // in essence, just replace the new_k of the existing th_info int[] old_th_info =
+ * (int[])thresholds.get(nearestIntersectionPoint); old_th_info[1] = th_info[1];
+ * thresholds.put(nearestIntersectionPoint,old_th_info); // When does this happen? If two
+ * consecutive intersection points are so close // to each other so as to appear as having
+ * the same value. For instance, assume // we have two intersection points ip1 and ip2
+ * corresponding to two transitions, // one from k_a to k_b, and the other from k_b to
+ * k_c. It might be the case // that ip2-ip1 is extremeley small, so that the ip2 entry
+ * would actually REPLACE // the ip1 entry. This would be bad.
+ *
+ * // Instead, we pretend that k_b never happened, and just assume there is a single //
+ * intersection point, ip (which equals whatever value Java calculates for ip1 // and
+ * ip2), with a corresponding transition of k_a to k_c. }
+ */
+ } // if (in-range)
+
+ currIndex = nearestIntersectingLineIndex;
+
+ } // end while (currIndex != maxSlopeIndex)
+
+ if (last_new_k != -1) {
+ indicesOfInterest[i].add(last_new_k); // last new_k
+ // indicesOfInterest_all[i].add(last_new_k); // last new_k ***/
+ }
+
+ // println("cI=" + currIndex + "(=? " + maxSlopeIndex + " = mxSI)",4);
+
+ // now thresholds has the values for lambda_c at which score changes
+ // based on the candidates for the ith sentence
+
+ // println("",4);
+
+ /*
+ * Iterator<Double> It = (thresholds.keySet()).iterator(); int[] th_info = null; while
+ * (It.hasNext()) { // process intersection points contributed by this sentence double ip =
+ * It.next(); if (ip > minThValue[c] && ip < maxThValue[c]) { th_info = thresholds.get(ip); if
+ * (!thresholdsAll.containsKey(ip)) { TreeMap A = new TreeMap(); A.put(i,th_info);
+ * thresholdsAll.put(ip,A); } else { // not frequent, but does happen (when same intersection
+ * point // corresponds to a candidate switch for more than one i) TreeMap A =
+ * thresholdsAll.get(ip); A.put(i,th_info); thresholdsAll.put(ip,A); }
+ *
+ * // if (useDisk == 2) { // th_info[0] = old_k, th_info[1] = new_k
+ * indicesOfInterest[i].add(th_info[0]); // }
+ *
+ * } // if (in-range)
+ *
+ * } // while (It.hasNext())
+ */
+
+ /*
+ * // if (useDisk == 2 && th_info != null) { if (th_info != null) { // new_k from the last
+ * th_info (previous new_k already appear as the next old_k)
+ * indicesOfInterest[i].add(th_info[1]); }
+ */
+
+ // thresholds.clear();
+
+ } // for (i)
+
+ // now thresholdsAll has the values for lambda_c at which score changes
+ // based on the candidates for *all* the sentences (that satisfy
+ // range constraints).
+ // Each lambda_c value maps to a Vector of th_info. An overwhelming majority
+ // of these Vectors are of size 1.
+
+ // indicesOfInterest[i] tells us which candidates for the ith sentence need
+ // to be read from the merged decoder output file.
+
+ if (thresholdsAll.size() != 0) {
+ double smallest_th = thresholdsAll.firstKey();
+ double largest_th = thresholdsAll.lastKey();
+ println("# extracted thresholds: " + thresholdsAll.size(), 2);
+ println("Smallest extracted threshold: " + smallest_th, 2);
+ println("Largest extracted threshold: " + largest_th, 2);
+
+ if (maxThValue[c] != PosInf) {
+ thresholdsAll.put(maxThValue[c], null);
+ } else {
+ thresholdsAll.put((thresholdsAll.lastKey() + 0.1), null);
+ }
+ }
+
+ // return thresholdsAll;
+
+ } // TreeMap<Double,TreeMap> thresholdsForParam (int c)
+
+ private double[] line_opt(TreeMap<Double, TreeMap<Integer, int[]>> thresholdsAll,
+ int[] indexOfCurrBest, int c, double[] lambda) {
+ println("Line-optimizing lambda[" + c + "]...", 3);
+
+ double[] bestScoreInfo = new double[2];
+ // to be returned: [0] will store the best lambda, and [1] will store its score
+
+ if (thresholdsAll.size() == 0) {
+ // no thresholds extracted! Possible in theory...
+ // simply return current value for this parameter
+ println("No thresholds extracted! Returning this parameter's current value...", 2);
+
+ bestScoreInfo[0] = lambda[c];
+ bestScoreInfo[1] = evalMetric.worstPossibleScore();
+
+ return bestScoreInfo;
+ }
+
+ double smallest_th = thresholdsAll.firstKey();
+ double largest_th = thresholdsAll.lastKey();
+ println("Minimum threshold: " + smallest_th, 3);
+ println("Maximum threshold: " + largest_th, 3);
+
+ double[] temp_lambda = new double[1 + numParams];
+ System.arraycopy(lambda, 1, temp_lambda, 1, numParams);
+
+ double ip_prev = 0.0, ip_curr = 0.0;
+
+ if (minThValue[c] != NegInf) {
+ temp_lambda[c] = (minThValue[c] + smallest_th) / 2.0;
+ ip_curr = minThValue[c];
+ } else {
+ temp_lambda[c] = smallest_th - 0.05;
+ ip_curr = smallest_th - 0.1;
+ }
+
+
+
+ int[][] suffStats = new int[numSentences][suffStatsCount];
+ // suffStats[i][s] stores the contribution to the sth sufficient
+ // statistic from the candidate for the ith sentence (the candidate
+ // indicated by indexOfCurrBest[i]).
+
+ int[][] suffStats_doc = new int[numDocuments][suffStatsCount];
+ // suffStats_doc[doc][s] := SUM_i suffStats[i][s], over sentences in the doc'th document
+ // i.e. treat each document as a mini corpus
+ // (if not doing document-level optimization, all sentences will belong in a single
+ // document: the 1st one, indexed 0)
+
+ // initialize document SS
+ for (int doc = 0; doc < numDocuments; ++doc) {
+ for (int s = 0; s < suffStatsCount; ++s) {
+ suffStats_doc[doc][s] = 0;
+ }
+ }
+
+ // Now, set suffStats[][], and increment suffStats_doc[][]
+ for (int i = 0; i < numSentences; ++i) {
+ suffStats[i] = suffStats_array[i].get(indexOfCurrBest[i]);
+
+ for (int s = 0; s < suffStatsCount; ++s) {
+ suffStats_doc[docOfSentence[i]][s] += suffStats[i][s];
+ }
+ }
+
+
+
+ double bestScore = 0.0;
+ if (optimizeSubset)
+ bestScore = evalMetric.score(suffStats_doc, docSubset_firstRank, docSubset_lastRank);
+ else
+ bestScore = evalMetric.score(suffStats_doc);
+ double bestLambdaVal = temp_lambda[c];
+ double nextLambdaVal = bestLambdaVal;
+ println("At lambda[" + c + "] = " + bestLambdaVal + "," + "\t" + metricName_display + " = "
+ + bestScore + " (*)", 3);
+
+ Iterator<Double> It = (thresholdsAll.keySet()).iterator();
+ if (It.hasNext()) {
+ ip_curr = It.next();
+ }
+
+ while (It.hasNext()) {
+ ip_prev = ip_curr;
+ ip_curr = It.next();
+ nextLambdaVal = (ip_prev + ip_curr) / 2.0;
+
+ TreeMap<Integer, int[]> th_info_M = thresholdsAll.get(ip_prev);
+ Iterator<Integer> It2 = (th_info_M.keySet()).iterator();
+ while (It2.hasNext()) {
+ int i = It2.next();
+ // i.e. the 1-best for the i'th sentence changes at this threshold value
+ int docOf_i = docOfSentence[i];
+
+ int[] th_info = th_info_M.get(i);
+ @SuppressWarnings("unused")
+ int old_k = th_info[0]; // should be equal to indexOfCurrBest[i]
+ int new_k = th_info[1];
+
+ for (int s = 0; s < suffStatsCount; ++s) {
+ suffStats_doc[docOf_i][s] -= suffStats[i][s]; // subtract stats for candidate old_k
+ }
+
+ indexOfCurrBest[i] = new_k;
+ suffStats[i] = suffStats_array[i].get(indexOfCurrBest[i]); // update the SS for the i'th
+ // sentence
+
+ for (int s = 0; s < suffStatsCount; ++s) {
+ suffStats_doc[docOf_i][s] += suffStats[i][s]; // add stats for candidate new_k
+ }
+
+ }
+
+ double nextTestScore = 0.0;
+ if (optimizeSubset)
+ nextTestScore = evalMetric.score(suffStats_doc, docSubset_firstRank, docSubset_lastRank);
+ else
+ nextTestScore = evalMetric.score(suffStats_doc);
+
+ print("At lambda[" + c + "] = " + nextLambdaVal + "," + "\t" + metricName_display + " = "
+ + nextTestScore, 3);
+
+ if (evalMetric.isBetter(nextTestScore, bestScore)) {
+ bestScore = nextTestScore;
+ bestLambdaVal = nextLambdaVal;
+ print(" (*)", 3);
+ }
+
+ println("", 3);
+
+ } // while (It.hasNext())
+
+ println("", 3);
+
+ // what is the purpose of this block of code ?????????????????????
+ /*
+ * if (maxThValue[c] != PosInf) { nextLambdaVal = (largest_th + maxThValue[c]) / 2.0; } else {
+ * nextLambdaVal = largest_th + 0.05; }
+ */
+ // ???????????????????????????????????????????????????????????????
+
+ /*************************************************/
+ /*************************************************/
+
+ bestScoreInfo[0] = bestLambdaVal;
+ bestScoreInfo[1] = bestScore;
+
+ return bestScoreInfo;
+
+ } // double[] line_opt(int c)
+
+ private void set_suffStats_array(TreeSet<Integer>[] indicesOfInterest) {
+ int candsOfInterestCount = 0;
+ for (int i = 0; i < numSentences; ++i) {
+ candsOfInterestCount += indicesOfInterest[i].size();
+ // candsOfInterestCount_all += indicesOfInterest_all[i].size(); ****/
+ }
+ println("Processing merged stats file; extracting SS " + "for " + candsOfInterestCount
+ + " candidates of interest.", 2);
+ // println("(*_all: " + candsOfInterestCount_all + ")",2); *****/
+
+
+ try {
+
+ // process the merged sufficient statistics file, and read (and store) the
+ // stats for candidates of interest
+ BufferedReader inFile =
+ new BufferedReader(new FileReader(tmpDirPrefix + "temp.stats.merged"));
+ String candidate_suffStats;
+
+ for (int i = 0; i < numSentences; ++i) {
+ int numCandidates = candCount[i];
+
+ int currCand = 0;
+ Iterator<Integer> It = indicesOfInterest[i].iterator();
+
+ while (It.hasNext()) {
+ int nextIndex = It.next();
+
+ // skip candidates until you get to the nextIndex'th candidate
+ while (currCand < nextIndex) {
+ inFile.readLine();
+ ++currCand;
+ }
+
+ // now currCand == nextIndex, and the next line in inFile
+ // contains the sufficient statistics we want
+
+ candidate_suffStats = inFile.readLine();
+ ++currCand;
+
+ String[] suffStats_str = candidate_suffStats.split("\\s+");
+
+ int[] suffStats = new int[suffStatsCount];
+
+ for (int s = 0; s < suffStatsCount; ++s) {
+ suffStats[s] = Integer.parseInt(suffStats_str[s]);
+ }
+
+ suffStats_array[i].put(nextIndex, suffStats);
+
+ }
+
+ // skip the rest of ith sentence's candidates
+ while (currCand < numCandidates) {
+ inFile.readLine();
+ ++currCand;
+ }
+
+ } // for (i)
+
+ inFile.close();
+
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ } // set_suffStats_array(HashMap[] suffStats_array, TreeSet[] indicesOfInterest, Vector[]
+ // candidates)
+
+ private double L_norm(double[] A, double pow) {
+ // calculates the L-pow norm of A[]
+ // NOTE: this calculation ignores A[0]
+ double sum = 0.0;
+ for (int i = 1; i < A.length; ++i) {
+ sum += Math.pow(Math.abs(A[i]), pow);
+ }
+ return Math.pow(sum, 1 / pow);
+ }
+
+ private int[] initial_indexOfCurrBest(double[] temp_lambda, TreeSet<Integer>[] indicesOfInterest) {
+ int[] indexOfCurrBest = new int[numSentences];
+ // As we traverse lambda_c, indexOfCurrBest indicates which is the
+ // current best candidate.
+
+ // initialize indexOfCurrBest[]
+
+ for (int i = 0; i < numSentences; ++i) {
+ int numCandidates = candCount[i];
+
+ double max = NegInf;
+ int indexOfMax = -1;
+ for (int k = 0; k < numCandidates; ++k) {
+ double score = 0;
+
+ for (int c2 = 1; c2 <= numParams; ++c2) {
+ score += temp_lambda[c2] * featVal_array[c2][i][k];
+ }
+ if (score > max) {
+ max = score;
+ indexOfMax = k;
+ }
+ }
+
+ indexOfCurrBest[i] = indexOfMax;
+
+ // if (useDisk == 2) {
+ // add indexOfCurrBest[i] to indicesOfInterest
+ indicesOfInterest[i].add(indexOfMax);
+ // indicesOfInterest_all[i].add(indexOfMax);
+ // }
+
+ }
+
+ return indexOfCurrBest;
+
+ } // int[] initial_indexOfCurrBest (int c)
+
+ private double[] bestParamToChange(TreeMap<Double, TreeMap<Integer, int[]>>[] thresholdsAll,
+ int lastChanged_c, double[] currLambda) {
+ int c_best = 0; // which parameter to change?
+ double bestLambdaVal = 0.0;
+ double bestScore;
+ if (evalMetric.getToBeMinimized()) {
+ bestScore = evalMetric.worstPossibleScore() + 1.0;
+ } else {
+ bestScore = evalMetric.worstPossibleScore() - 1.0;
+ }
+
+
+
+ // prep for line_opt
+
+ TreeSet<Integer>[] indicesOfInterest = null;
+ // indicesOfInterest[i] tells us which candidates for the ith sentence need
+ // to be read from the merged decoder output file.
+
+ // if (useDisk == 2) {
+ @SuppressWarnings("unchecked")
+ TreeSet<Integer>[] temp_TSA = new TreeSet[numSentences];
+ indicesOfInterest = temp_TSA;
+ for (int i = 0; i < numSentences; ++i) {
+ indicesOfInterest[i] = new TreeSet<Integer>();
+ }
+ // }
+
+ int[][] indexOfCurrBest = new int[1 + numParams][numSentences];
+
+ for (int c = 1; c <= numParams; ++c) {
+ if (!isOptimizable[c]) {
+ println("Not investigating lambda[j=" + j + "][" + c + "].", 2);
+ } else {
+ if (c != lastChanged_c) {
+ println("Investigating lambda[j=" + j + "][" + c + "]...", 2);
+ // thresholdsAll[c] =
+ // thresholdsForParam(c,candCount,featVal_array,currLambda,indicesOfInterest);
+ set_thresholdsForParam(thresholdsAll[c], c, currLambda, indicesOfInterest);
+ } else {
+ println("Keeping thresholds for lambda[j=" + j + "][" + c + "] from previous step.", 2);
+ }
+ // now thresholdsAll has the values for lambda_c at which score changes
+ // based on the candidates for *all* the sentences (that satisfy
+ // range constraints).
+ // Each lambda_c value maps to a Vector of th_info. An overwhelming majority
+ // of these Vectors are of size 1.
+
+ if (thresholdsAll[c].size() != 0) {
+
+ double[] temp_lambda = new double[1 + numParams];
+ System.arraycopy(currLambda, 1, temp_lambda, 1, numParams);
+
+ double smallest_th = thresholdsAll[c].firstKey();
+
+ if (minThValue[c] != NegInf) {
+ temp_lambda[c] = (minThValue[c] + smallest_th) / 2.0;
+ } else {
+ temp_lambda[c] = smallest_th - 0.05;
+ }
+
+ indexOfCurrBest[c] = initial_indexOfCurrBest(temp_lambda, indicesOfInterest);
+ }
+ }
+
+ println("", 2);
+
+ }
+
+
+
+ // if (useDisk == 2) {
+
+ set_suffStats_array(indicesOfInterest);
+
+ // } // if (useDisk == 2)
+
+
+
+ for (int c = 1; c <= numParams; ++c) {
+ // investigate currLambda[j][c]
+
+ if (isOptimizable[c]) {
+ double[] bestScoreInfo_c = line_opt(thresholdsAll[c], indexOfCurrBest[c], c, currLambda);
+ // get best score and its lambda value
+
+ double bestLambdaVal_c = bestScoreInfo_c[0];
+ double bestScore_c = bestScoreInfo_c[1];
+
+ if (evalMetric.isBetter(bestScore_c, bestScore)) {
+ c_best = c;
+ bestLambdaVal = bestLambdaVal_c;
+ bestScore = bestScore_c;
+ }
+
+ } // if (!isOptimizable[c])
+
+ }
+
+
+
+ // delete according to indicesOfInterest
+
+ // printMemoryUsage();
+
+ // if (useDisk == 2) {
+
+ for (int i = 0; i < numSentences; ++i) {
+
+ indicesOfInterest[i].clear();
+
+ }
+
+ // }
+
+ // cleanupMemory();
+ // printMemoryUsage();
+ // println("",2);
+
+
+
+ double[] c_best_info = {c_best, bestLambdaVal, bestScore};
+ return c_best_info;
+
+ } // double[] bestParamToChange(int j, double[] currLambda)
+
+ private void normalizeLambda(double[] origLambda) {
+ // private String[] normalizationOptions;
+ // How should a lambda[] vector be normalized (before decoding)?
+ // nO[0] = 0: no normalization
+ // nO[0] = 1: scale so that parameter nO[2] has absolute value nO[1]
+ // nO[0] = 2: scale so that the maximum absolute value is nO[1]
+ // nO[0] = 3: scale so that the minimum absolute value is nO[1]
+ // nO[0] = 4: scale so that the L-nO[1] norm equals nO[2]
+
+ int normalizationMethod = (int) normalizationOptions[0];
+ double scalingFactor = 1.0;
+ if (normalizationMethod == 0) {
+
+ scalingFactor = 1.0;
+
+ } else if (normalizationMethod == 1) {
+
+ int c = (int) normalizationOptions[2];
+ scalingFactor = normalizationOptions[1] / Math.abs(origLambda[c]);
+
+ } else if (normalizationMethod == 2) {
+
+ double maxAbsVal = -1;
+ int maxAbsVal_c = 0;
+ for (int c = 1; c <= numParams; ++c) {
+ if (Math.abs(origLambda[c]) > maxAbsVal) {
+ maxAbsVal = Math.abs(origLambda[c]);
+ maxAbsVal_c = c;
+ }
+ }
+ scalingFactor = normalizationOptions[1] / Math.abs(origLambda[maxAbsVal_c]);
+
+ } else if (normalizationMethod == 3) {
+
+ double minAbsVal = PosInf;
+ int minAbsVal_c = 0;
+ for (int c = 1; c <= numParams; ++c) {
+ if (Math.abs(origLambda[c]) < minAbsVal) {
+ minAbsVal = Math.abs(origLambda[c]);
+ minAbsVal_c = c;
+ }
+ }
+ scalingFactor = normalizationOptions[1] / Math.abs(origLambda[minAbsVal_c]);
+
+ } else if (normalizationMethod == 4) {
+
+ double pow = normalizationOptions[1];
+ double norm = L_norm(origLambda, pow);
+ scalingFactor = normalizationOptions[2] / norm;
+
+ }
+
+ for (int c = 1; c <= numParams; ++c) {
+ origLambda[c] *= scalingFactor;
+ }
+
+ }
+
+ private void real_run() {
+ @SuppressWarnings("unchecked")
+ TreeMap<Double, TreeMap<Integer, int[]>>[] thresholdsAll = new TreeMap[1 + numParams];
+ thresholdsAll[0] = null;
+ for (int c = 1; c <= numParams; ++c) {
+ if (isOptimizable[c]) {
+ thresholdsAll[c] = new TreeMap<Double, TreeMap<Integer, int[]>>();
+ } else {
+ thresholdsAll[c] = null;
+ }
+ }
+
+
+ // cleanupMemory();
+
+ println("+++ Optimization of lambda[j=" + j + "] starting @ " + (new Date()) + " +++", 1);
+
+ double[] currLambda = new double[1 + numParams];
+ System.arraycopy(initialLambda, 1, currLambda, 1, numParams);
+
+ int[][] best1Cand_suffStats_doc = new int[numDocuments][suffStatsCount];
+ for (int doc = 0; doc < numDocuments; ++doc) {
+ for (int s = 0; s < suffStatsCount; ++s) {
+ best1Cand_suffStats_doc[doc][s] = 0;
+ }
+ }
+
+ for (int i = 0; i < numSentences; ++i) {
+ for (int s = 0; s < suffStatsCount; ++s) {
+ best1Cand_suffStats_doc[docOfSentence[i]][s] += best1Cand_suffStats[i][s];
+ }
+ }
+
+ double initialScore = 0.0;
+ if (optimizeSubset)
+ initialScore =
+ evalMetric.score(best1Cand_suffStats_doc, docSubset_firstRank, docSubset_lastRank);
+ else
+ initialScore = evalMetric.score(best1Cand_suffStats_doc);
+
+ println("Initial lambda[j=" + j + "]: " + lambdaToString(initialLambda), 1);
+ println("(Initial " + metricName_display + "[j=" + j + "]: " + initialScore + ")", 1);
+ println("", 1);
+ finalScore[j] = initialScore;
+
+ int c_best = 0; // which param to change?
+ double bestLambdaVal = 0; // what value to change to?
+ double bestScore = 0; // what score would be achieved?
+
+ while (true) {
+
+ double[] c_best_info = bestParamToChange(thresholdsAll, c_best, currLambda);
+ // we pass in c_best because we don't need
+ // to recalculate thresholds for it
+ c_best = (int) c_best_info[0]; // which param to change?
+ bestLambdaVal = c_best_info[1]; // what value to change to?
+ bestScore = c_best_info[2]; // what score would be achieved?
+
+ // now c_best is the parameter giving the most gain
+
+ if (evalMetric.isBetter(bestScore, finalScore[j])) {
+ println(
+ "*** Changing lambda[j=" + j + "][" + c_best + "] from "
+ + f4.format(currLambda[c_best]) + " (" + metricName_display + ": "
+ + f4.format(finalScore[j]) + ") to " + f4.format(bestLambdaVal) + " ("
+ + metricName_display + ": " + f4.format(bestScore) + ") ***", 2);
+ println("*** Old lambda[j=" + j + "]: " + lambdaToString(currLambda) + " ***", 2);
+ currLambda[c_best] = bestLambdaVal;
+ finalScore[j] = bestScore;
+ println("*** New lambda[j=" + j + "]: " + lambdaToString(currLambda) + " ***", 2);
+ println("", 2);
+ } else {
+ println("*** Not changing any weight in lambda[j=" + j + "] ***", 2);
+ println("*** lambda[j=" + j + "]: " + lambdaToString(currLambda) + " ***", 2);
+ println("", 2);
+ break; // exit while (true) loop
+ }
+
+ if (oneModificationPerIteration) {
+ break;
+ } // exit while (true) loop
+
+ } // while (true)
+
+ // now currLambda is the optimized weight vector on the current candidate list
+ // (corresponding to initialLambda)
+
+ System.arraycopy(currLambda, 1, finalLambda, 1, numParams);
+ normalizeLambda(finalLambda);
+ // check if a lambda is outside its threshold range
+ for (int c = 1; c <= numParams; ++c) {
+ if (finalLambda[c] < minThValue[c] || finalLambda[c] > maxThValue[c]) {
+ println(
+ "Warning: after normalization, final lambda[j=" + j + "][" + c + "]="
+ + f4.format(finalLambda[c]) + " is outside its critical value range.", 2);
+ }
+ }
+ println("Final lambda[j=" + j + "]: " + lambdaToString(finalLambda), 1);
+ println("(Final " + metricName_display + "[j=" + j + "]: " + finalScore[j] + ")", 1);
+ println("", 1);
+
+ blocker.release();
+ }
+
+ public void run() {
+ real_run();
+ if (!strToPrint.equals("")) {
+ threadOutput.add(strToPrint);
+ }
+ }
+
+ private void println(String str, int priority) {
+ if (priority <= verbosity) println(str);
+ }
+
+ private void print(String str, int priority) {
+ if (priority <= verbosity) print(str);
+ }
+
+ private void println(String str) {
+ threadOutput.add(strToPrint + str);
+ strToPrint = "";
+ }
+
+ private void print(String str) {
+ strToPrint += str;
+ }
+
+ private String lambdaToString(double[] lambdaA) {
+ String retStr = "{";
+ for (int c = 1; c <= numParams - 1; ++c) {
+ retStr += "" + lambdaA[c] + ", ";
+ }
+ retStr += "" + lambdaA[numParams] + "}";
+
+ return retStr;
+ }
+}
[35/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/HGNode.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/HGNode.java b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/HGNode.java
new file mode 100644
index 0000000..695cad5
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/HGNode.java
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.hypergraph;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+
+/**
+ * this class implement Hypergraph node (i.e., HGNode); also known as Item in parsing.
+ *
+ * @author Zhifei Li, zhifei.work@gmail.com
+ * @author Juri Ganitkevitch, juri@cs.jhu.edu
+ */
+
+// TODO: handle the case that the Hypergraph only maintains the one-best tree
+
+public class HGNode {
+
+ public int i, j;
+
+ // this is the symbol like: NP, VP, and so on
+ public int lhs;
+
+ // each hyperedge is an "and" node
+ public List<HyperEdge> hyperedges = null;
+
+ // used in pruning, compute_item, and transit_to_goal
+ public HyperEdge bestHyperedge = null;
+
+ // the key is the state id; remember the state required by each model, for example, edge-ngrams
+ // for LM model
+ protected List<DPState> dpStates;
+
+ private Signature signature = null;
+// private int hash = 0;
+
+ protected float score = 0.0f;
+
+ // ===============================================================
+ // Constructors
+ // ===============================================================
+
+ public HGNode(int i, int j, int lhs, List<DPState> dpStates, HyperEdge hyperEdge,
+ float pruningEstimate) {
+ this.lhs = lhs;
+ this.i = i;
+ this.j = j;
+ this.dpStates = dpStates;
+ this.score = pruningEstimate;
+ addHyperedgeInNode(hyperEdge);
+ }
+
+ // used by disk hg
+ public HGNode(int i, int j, int lhs, List<HyperEdge> hyperedges, HyperEdge bestHyperedge,
+ List<DPState> states) {
+ this.i = i;
+ this.j = j;
+ this.lhs = lhs;
+ this.hyperedges = hyperedges;
+ this.bestHyperedge = bestHyperedge;
+ this.dpStates = states;
+ }
+
+ // ===============================================================
+ // Methods
+ // ===============================================================
+
+ public float getScore() {
+ return this.score;
+ }
+
+ /**
+ * Adds the hyperedge to the list of incoming hyperedges (i.e., ways to form this node), creating
+ * the list if necessary. We then update the cache of the best incoming hyperedge via a call to
+ * the (obscurely named) semiringPlus().
+ * @param hyperEdge the {@link org.apache.joshua.decoder.hypergraph.HyperEdge} to add
+ * to the list of incoming hyperedges
+ */
+ public void addHyperedgeInNode(HyperEdge hyperEdge) {
+ if (hyperEdge != null) {
+ if (null == hyperedges)
+ hyperedges = new ArrayList<HyperEdge>();
+ hyperedges.add(hyperEdge);
+ // Update the cache of this node's best incoming edge.
+ semiringPlus(hyperEdge);
+ }
+ }
+
+ /**
+ * Convenience function to add a list of hyperedges one at a time.
+ * @param hyperedges a {@link java.util.List} of {@link org.apache.joshua.decoder.hypergraph.HyperEdge}'s
+ * to add to the current HGNode.
+ */
+ public void addHyperedgesInNode(List<HyperEdge> hyperedges) {
+ for (HyperEdge hyperEdge : hyperedges)
+ addHyperedgeInNode(hyperEdge);
+ }
+
+ /**
+ * Updates the cache of the best incoming hyperedge.
+ * @param hyperEdge an incoming {{@link org.apache.joshua.decoder.hypergraph.HyperEdge}
+ */
+ public void semiringPlus(HyperEdge hyperEdge) {
+ if (null == bestHyperedge || bestHyperedge.getBestDerivationScore() < hyperEdge.getBestDerivationScore()) {
+ bestHyperedge = hyperEdge;
+ }
+ }
+
+ public List<DPState> getDPStates() {
+ return dpStates;
+ }
+
+ public DPState getDPState(int i) {
+ if (null == this.dpStates) {
+ return null;
+ } else {
+ return this.dpStates.get(i);
+ }
+ }
+
+ public Signature signature() {
+ if (signature == null)
+ signature = new Signature();
+ return signature;
+ }
+
+ /*
+ * Including hashCode() and equals() directly in the class causes problems, because the
+ * virtual node table (in KBestExtractor) does not combine HGNodes.
+ */
+// @Override
+// public int hashCode() {
+// if (hash == 0) {
+// hash = 31 * lhs + 2399 * i + 7853 * j;
+// if (null != dpStates && dpStates.size() > 0)
+// for (DPState dps : dpStates)
+// hash = hash * 19 + dps.hashCode();
+// }
+// return hash;
+// }
+//
+// @Override
+// public boolean equals(Object other) {
+// if (other instanceof HGNode) {
+// HGNode that = (HGNode) other;
+// if (lhs != that.lhs)
+// return false;
+// if (i != that.i || j != that.j)
+// return false;
+// if (bestHyperedge == null && that.bestHyperedge != null)
+// return false;
+// if (bestHyperedge != null && that.bestHyperedge == null)
+// return false;
+// if (score != that.score)
+// return false;
+// if (dpStates == null)
+// return (that.dpStates == null);
+// if (that.dpStates == null)
+// return false;
+// if (dpStates.size() != that.dpStates.size())
+// return false;
+// for (int i = 0; i < dpStates.size(); i++) {
+// if (!dpStates.get(i).equals(that.dpStates.get(i)))
+// return false;
+// }
+// return true;
+// }
+// return false;
+// }
+
+ /***
+ * We have different purposes when hashing HGNodes. For dynamic programming, we want to establish
+ * equivalency based on dynamic programming state, but when doing k-best extraction, we need
+ * to maintain a separate entry for every object. The Signature class provides a way to hash
+ * based on the dynamic programming state.
+ */
+ public class Signature {
+ // Cached hash code.
+ private int hash = 0;
+
+ @Override
+ public int hashCode() {
+ if (hash == 0) {
+ hash = 31 * lhs;
+ if (null != dpStates && dpStates.size() > 0)
+ for (DPState dps : dpStates)
+ hash = hash * 19 + dps.hashCode();
+ }
+ return hash;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other instanceof Signature) {
+ HGNode that = ((Signature) other).node();
+ if (lhs != that.lhs)
+ return false;
+ if (i != that.i || j != that.j)
+ return false;
+ if (dpStates == null)
+ return (that.dpStates == null);
+ if (that.dpStates == null)
+ return false;
+ if (dpStates.size() != that.dpStates.size())
+ return false;
+ for (int i = 0; i < dpStates.size(); i++) {
+ if (!dpStates.get(i).equals(that.dpStates.get(i)))
+ return false;
+ }
+ return true;
+ }
+ return false;
+ }
+
+ public String toString() {
+ return String.format("%d", hashCode());
+ }
+
+ public HGNode node() {
+ return HGNode.this;
+ }
+ }
+
+ /*
+ * this will called by the sorting in Cell.ensureSorted()
+ */
+ // sort by estTotalLogP: for pruning purpose
+ public int compareTo(HGNode anotherItem) {
+ throw new RuntimeException("HGNode, compare functiuon should never be called");
+ /*
+ * if (this.estTotalLogP > anotherItem.estTotalLogP) { return -1; } else if (this.estTotalLogP
+ * == anotherItem.estTotalLogP) { return 0; } else { return 1; }
+ */
+
+ }
+
+ /**
+ * This sorts nodes by span, useful when dumping the hypergraph.
+ */
+ public static Comparator<HGNode> spanComparator = new Comparator<HGNode>() {
+ public int compare(HGNode item1, HGNode item2) {
+ int span1 = item1.j - item1.i;
+ int span2 = item2.j - item2.i;
+ if (span1 < span2)
+ return -1;
+ else if (span1 > span2)
+ return 1;
+ else if (item1.i < item2.i)
+ return -1;
+ else if (item1.i > item2.i)
+ return 1;
+ return 0;
+ }
+ };
+
+ public static Comparator<HGNode> inverseLogPComparator = new Comparator<HGNode>() {
+ public int compare(HGNode item1, HGNode item2) {
+ float logp1 = item1.score;
+ float logp2 = item2.score;
+ if (logp1 > logp2) {
+ return -1;
+ } else if (logp1 == logp2) {
+ return 0;
+ } else {
+ return 1;
+ }
+ }
+ };
+
+ /**
+ * natural order
+ * */
+ public static Comparator<HGNode> logPComparator = new Comparator<HGNode>() {
+ public int compare(HGNode item1, HGNode item2) {
+ float logp1 = item1.score;
+ float logp2 = item2.score;
+ if (logp1 > logp2) {
+ return 1;
+ } else if (logp1 == logp2) {
+ return 0;
+ } else {
+ return -1;
+ }
+ }
+ };
+
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+
+ sb.append(String.format("%s (%d,%d) score=%.5f", Vocabulary.word(lhs), i, j,
+ bestHyperedge.getBestDerivationScore()));
+ if (dpStates != null)
+ for (DPState state : dpStates)
+ sb.append(" <" + state + ">");
+
+ // if (this.hyperedges != null) {
+ // sb.append(" hyperedges: " + hyperedges.size());
+ // for (HyperEdge edge: hyperedges) {
+ // sb.append("\n\t" + edge.getRule() + " ||| pathcost=" + edge.getSourcePath() + " ref="+
+ // Integer.toHexString(edge.hashCode()));
+ // }
+ // }
+
+ // sb.append("\n\ttransition score = " + bestHyperedge.getTransitionLogP(true));
+ return sb.toString();
+ }
+
+ public List<HyperEdge> getHyperEdges() {
+ return this.hyperedges;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/HyperEdge.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/HyperEdge.java b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/HyperEdge.java
new file mode 100644
index 0000000..b188650
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/HyperEdge.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.hypergraph;
+
+import java.util.List;
+
+import org.apache.joshua.decoder.chart_parser.SourcePath;
+import org.apache.joshua.decoder.ff.tm.Rule;
+
+/**
+ * this class implement Hyperedge
+ *
+ * @author Zhifei Li, zhifei.work@gmail.com
+ * @author Matt Post post@cs.jhu.edu
+ */
+
+public class HyperEdge {
+
+ /**
+ * the 1-best logP of all possible derivations: best logP of ant hgnodes + transitionlogP
+ **/
+ private float bestDerivationScore = Float.NEGATIVE_INFINITY;
+
+ /**
+ * this remembers the stateless + non_stateless logP assocated with the rule (excluding the
+ * best-logP from ant nodes)
+ * */
+ private float transitionScore;
+
+ private Rule rule;
+
+ private SourcePath srcPath = null;
+
+ /**
+ * If antNodes is null, then this edge corresponds to a rule with zero arity. Aslo, the nodes
+ * appear in the list as per the index of the Foreign side non-terminal
+ * */
+ private List<HGNode> tailNodes = null;
+
+ public HyperEdge(Rule rule, float bestDerivationScore, float transitionScore,
+ List<HGNode> tailNodes, SourcePath srcPath) {
+ this.bestDerivationScore = bestDerivationScore;
+ this.transitionScore = transitionScore;
+ this.rule = rule;
+ this.tailNodes = tailNodes;
+ this.srcPath = srcPath;
+ }
+
+ public Rule getRule() {
+ return rule;
+ }
+
+ public float getBestDerivationScore() {
+ return bestDerivationScore;
+ }
+
+ public SourcePath getSourcePath() {
+ return srcPath;
+ }
+
+ public List<HGNode> getTailNodes() {
+ return tailNodes;
+ }
+
+ public float getTransitionLogP(boolean forceCompute) {
+ if (forceCompute) {
+ float res = bestDerivationScore;
+ if (tailNodes != null) for (HGNode tailNode : tailNodes) {
+ res += tailNode.bestHyperedge.bestDerivationScore;
+ }
+ transitionScore = res;
+ }
+ return transitionScore;
+ }
+
+ public void setTransitionLogP(float transitionLogP) {
+ this.transitionScore = transitionLogP;
+ }
+
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(this.rule);
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/HyperGraph.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/HyperGraph.java b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/HyperGraph.java
new file mode 100644
index 0000000..499d4f3
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/HyperGraph.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.hypergraph;
+
+import java.io.IOException;
+import java.io.PrintWriter;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.chart_parser.ComputeNodeResult;
+import org.apache.joshua.decoder.ff.FeatureFunction;
+import org.apache.joshua.decoder.ff.FeatureVector;
+import org.apache.joshua.decoder.hypergraph.ForestWalker.TRAVERSAL;
+import org.apache.joshua.decoder.segment_file.Sentence;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * this class implement (1) HyperGraph-related data structures (Item and Hyper-edges)
+ *
+ * Note: to seed the kbest extraction, each deduction should have the best_cost properly set. We do
+ * not require any list being sorted
+ *
+ * @author Zhifei Li, zhifei.work@gmail.com
+ */
+public class HyperGraph {
+
+ private static final Logger LOG = LoggerFactory.getLogger(HyperGraph.class);
+
+ // pointer to goal HGNode
+ public HGNode goalNode = null;
+
+ public int numNodes = -1;
+ public int numEdges = -1;
+ public Sentence sentence = null;
+
+ public HyperGraph(HGNode goalNode, int numNodes, int numEdges, Sentence sentence) {
+ this.goalNode = goalNode;
+ this.numNodes = numNodes;
+ this.numEdges = numEdges;
+ this.sentence = sentence;
+ }
+
+ public void count() {
+ new ForestWalker().walk(this.goalNode, new HyperGraphCounter(this));
+ }
+
+ public int sentID() {
+ return sentence.id();
+ }
+
+ public int sentLen() {
+ return sentence.length();
+ }
+
+ private class HyperGraphCounter implements WalkerFunction {
+
+ private HyperGraph hg = null;
+ private HashSet<HGNode> nodesVisited = null;
+
+ public HyperGraphCounter(HyperGraph hg) {
+ this.hg = hg;
+ this.hg.numNodes = 0;
+ this.hg.numEdges = 0;
+ this.nodesVisited = new HashSet<HGNode>();
+ }
+
+ @Override
+ public void apply(HGNode node, int index) {
+ if (! nodesVisited.contains(node)) {
+ if (node.bestHyperedge.getRule() != null) {
+ hg.numNodes++;
+ if (node.hyperedges != null)
+ hg.numEdges += node.hyperedges.size();
+ }
+ }
+ }
+ }
+
+ private class HyperGraphDumper implements WalkerFunction {
+
+ private int node_number = 1;
+ private List<FeatureFunction> model = null;
+ private PrintWriter out = null;
+
+ private HashMap<HGNode, Integer> nodeMap;
+
+ public HyperGraphDumper(PrintWriter out, List<FeatureFunction> model) {
+ this.out = out;
+ this.model = model;
+ this.nodeMap = new HashMap<HGNode, Integer>();
+ }
+
+ @Override
+ public void apply(HGNode node, int index) {
+ if (! nodeMap.containsKey(node)) { // Make sure each node is listed only once
+ nodeMap.put(node, this.node_number);
+
+ if (node.hyperedges.size() != 0 && node.bestHyperedge.getRule() != null) {
+ out.println(this.node_number);
+ for (HyperEdge e: node.hyperedges) {
+ if (e.getRule() != null) {
+ for (int id: e.getRule().getEnglish()) {
+ if (id < 0) {
+ out.print(String.format("[%d] ", nodeMap.get(e.getTailNodes().get(-id-1))));
+ } else {
+ out.print(String.format("%s ", Vocabulary.word(id)));
+ }
+ }
+
+ FeatureVector edgeFeatures = ComputeNodeResult.computeTransitionFeatures(
+ model, e, node.i, node.j, sentence);
+ out.println(String.format("||| %s", edgeFeatures));
+ }
+ }
+ }
+
+ this.node_number++;
+ }
+ }
+ }
+
+ /**
+ * Dump the hypergraph to the specified file.
+ *
+ * @param fileName local file path
+ * @param model {@link java.util.List} of {@link org.apache.joshua.decoder.ff.FeatureFunction}'s
+ */
+ public void dump(String fileName, List<FeatureFunction> model) {
+ try ( PrintWriter out = new PrintWriter(fileName, "UTF-8") ) {
+ count();
+ out.println("# target ||| features");
+ out.println(String.format("%d %d", numNodes, numEdges));
+ new ForestWalker(TRAVERSAL.POSTORDER).walk(this.goalNode, new HyperGraphDumper(out, model));
+ } catch (IOException e) {
+ LOG.error("Can't dump hypergraph to file '{}'", fileName);
+ LOG.error(e.getMessage(), e);
+ }
+ }
+
+ public float bestScore() {
+ return this.goalNode.bestHyperedge.getBestDerivationScore();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/HyperGraphPruning.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/HyperGraphPruning.java b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/HyperGraphPruning.java
new file mode 100644
index 0000000..27f5525
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/HyperGraphPruning.java
@@ -0,0 +1,176 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.hypergraph;
+
+import java.util.HashMap;
+
+import org.apache.joshua.corpus.Vocabulary;
+
+/**
+ * during the pruning process, many Item/Deductions may not be explored at all due to the early-stop
+ * in pruning_deduction
+ *
+ * @author Zhifei Li, zhifei.work@gmail.com
+ * @version $LastChangedDate$
+ */
+public class HyperGraphPruning extends TrivialInsideOutside {
+
+ HashMap<HGNode, Boolean> processedNodesTbl = new HashMap<HGNode, Boolean>();
+ double bestLogProb;// viterbi unnormalized log prob in the hypergraph
+
+ boolean ViterbiPruning = false;// Viterbi or Posterior pruning
+
+ boolean fixThresholdPruning = true;
+ double THRESHOLD_GENERAL = 10;// if the merit is worse than the best_log_prob by this number, then
+ // prune
+ double THRESHOLD_GLUE = 10;// if the merit is worse than the best_log_prob by this number, then
+ // prune
+
+ int numSurvivedEdges = 0;
+ int numSurvivedNodes = 0;
+
+ int glueGrammarOwner = 0;// TODO
+
+
+ public HyperGraphPruning(boolean fixThreshold, double thresholdGeneral, double thresholdGlue) {
+ fixThresholdPruning = fixThreshold;
+ THRESHOLD_GENERAL = thresholdGeneral;
+ THRESHOLD_GLUE = thresholdGlue;
+ glueGrammarOwner = Vocabulary.id("glue");// TODO
+ }
+
+ public void clearState() {
+ processedNodesTbl.clear();
+ super.clearState();
+ }
+
+
+ // ######################### pruning here ##############
+ public void pruningHG(HyperGraph hg) {
+
+ runInsideOutside(hg, 2, 1, 1.0);// viterbi-max, log-semiring
+
+ if (fixThresholdPruning) {
+ pruningHGHelper(hg);
+ super.clearState();
+ } else {
+ throw new RuntimeException("wrong call");
+ }
+ }
+
+ private void pruningHGHelper(HyperGraph hg) {
+
+ this.bestLogProb = getLogNormalizationConstant();// set the best_log_prob
+
+ numSurvivedEdges = 0;
+ numSurvivedNodes = 0;
+ processedNodesTbl.clear();
+ pruningNode(hg.goalNode);
+
+ // clear up
+ processedNodesTbl.clear();
+
+ System.out.println("Item suvived ratio: " + numSurvivedNodes * 1.0 / hg.numNodes + " = "
+ + numSurvivedNodes + "/" + hg.numNodes);
+ System.out.println("Deduct suvived ratio: " + numSurvivedEdges * 1.0 / hg.numEdges + " = "
+ + numSurvivedEdges + "/" + hg.numEdges);
+ }
+
+
+ private void pruningNode(HGNode it) {
+
+ if (processedNodesTbl.containsKey(it)) return;
+
+ processedNodesTbl.put(it, true);
+ boolean shouldSurvive = false;
+
+ // ### recursive call on each deduction
+ for (int i = 0; i < it.hyperedges.size(); i++) {
+ HyperEdge dt = it.hyperedges.get(i);
+ boolean survived = pruningEdge(dt, it);// deduction-specifc operation
+ if (survived) {
+ shouldSurvive = true; // at least one deduction survive
+ } else {
+ it.hyperedges.remove(i);
+ i--;
+ }
+ }
+ // TODO: now we simply remove the pruned deductions, but in general, we may want to update the
+ // variables mainted in the item (e.g., best_deduction); this depends on the pruning method used
+
+ /*
+ * by defintion: "should_surive==false" should be impossible, since if I got called, then my
+ * upper-deduction must survive, then i will survive because there must be one way to reach me
+ * from lower part in order for my upper-deduction survive
+ */
+ if (!shouldSurvive) {
+ throw new RuntimeException("item explored but does not survive");
+ // TODO: since we always keep the best_deduction, this should never be true
+ } else {
+ numSurvivedNodes++;
+ }
+ }
+
+
+ // if survive, return true
+ // best-deduction is always kept
+ private boolean pruningEdge(HyperEdge dt, HGNode parent) {
+
+ /**
+ * TODO: theoretically, if an item is get called, then its best deduction should always be kept
+ * even just by the threshold-checling. In reality, due to precision of Double, the
+ * threshold-checking may not be perfect
+ */
+ if (dt != parent.bestHyperedge) { // best deduction should always survive if the Item is get
+ // called
+ // ### prune?
+ if (shouldPruneHyperedge(dt, parent)) {
+ return false; // early stop
+ }
+ }
+
+ // ### still survive, recursive call all my ant-items
+ if (null != dt.getTailNodes()) {
+ for (HGNode ant_it : dt.getTailNodes()) {
+ pruningNode(ant_it); // recursive call on each ant item, note: the ant_it will not be pruned
+ // as I need it
+ }
+ }
+
+ // ### if get to here, then survive; remember: if I survive, then my upper-item must survive
+ numSurvivedEdges++;
+ return true; // survive
+ }
+
+ private boolean shouldPruneHyperedge(HyperEdge dt, HGNode parent) {
+
+ // ### get merit
+ double postLogProb = getEdgeUnormalizedPosteriorLogProb(dt, parent);
+
+
+ if (dt.getRule() != null && dt.getRule().getOwner() == glueGrammarOwner
+ && dt.getRule().getArity() == 2) { // specicial rule: S->S X
+ // TODO
+ return (postLogProb - this.bestLogProb < THRESHOLD_GLUE);
+ } else {
+ return (postLogProb - this.bestLogProb < THRESHOLD_GENERAL);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/KBestExtractor.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/KBestExtractor.java b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/KBestExtractor.java
new file mode 100644
index 0000000..8fc55df
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/KBestExtractor.java
@@ -0,0 +1,1052 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.hypergraph;
+
+import static org.apache.joshua.util.FormatUtils.unescapeSpecialSymbols;
+import static org.apache.joshua.util.FormatUtils.removeSentenceMarkers;
+import static java.util.Collections.emptyList;
+
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.PriorityQueue;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.BLEU;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.ff.FeatureFunction;
+import org.apache.joshua.decoder.ff.FeatureVector;
+import org.apache.joshua.decoder.ff.fragmentlm.Tree;
+import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.io.DeNormalize;
+import org.apache.joshua.decoder.segment_file.Sentence;
+import org.apache.joshua.decoder.segment_file.Token;
+import org.apache.joshua.util.FormatUtils;
+import org.apache.joshua.decoder.StructuredTranslation;
+import org.apache.joshua.decoder.StructuredTranslationFactory;
+
+/**
+ * <p>This class implements lazy k-best extraction on a hyper-graph.</p>
+ *
+ * <p>K-best extraction over hypergraphs is a little hairy, but is best understood in the following
+ * manner. Imagine a hypergraph, which is composed of nodes connected by hyperedges. A hyperedge has
+ * exactly one parent node and 1 or more tail nodes, corresponding to the rank of the rule that gave
+ * rise to the hyperedge. Each node has 1 or more incoming hyperedges.</p>
+ *
+ * <p>K-best extraction works in the following manner. A derivation is a set of nodes and hyperedges
+ * that leads from the root node down and exactly covers the source-side sentence. To define a
+ * derivation, we start at the root node, choose one of its incoming hyperedges, and then recurse to
+ * the tail (or antecedent) nodes of that hyperedge, where we continually make the same decision.</p>
+ *
+ * <p>Each hypernode has its hyperedges sorted according to their model score. To get the best
+ * (Viterbi) derivation, we simply recursively follow the best hyperedge coming in to each
+ * hypernode.</p>
+ *
+ * <p>How do we get the second-best derivation? It is defined by changing exactly one of the decisions
+ * about which hyperedge to follow in the recursion. Somewhere, we take the second-best. Similarly,
+ * the third-best derivation makes a single change from the second-best: either making another
+ * (differnt) second-best choice somewhere along the 1-best derivation, or taking the third-best
+ * choice at the same spot where the second-best derivation took the second-best choice. And so on.</p>
+ *
+ * <p>This class uses two classes that encode the necessary meta-information. The first is the
+ * DerivationState class. It roughly corresponds to a hyperedge, and records, for each of that
+ * hyperedge's tail nodes, which-best to take. So for a hyperedge with three tail nodes, the 1-best
+ * derivation will be (1,1,1), the second-best will be one of (2,1,1), (1,2,1), or (1,1,2), the
+ * third best will be one of</p>
+ *
+ * <code>(3,1,1), (2,2,1), (1,1,3)</code>
+ *
+ * <p>and so on.</p>
+ *
+ * <p>The configuration parameter `output-format` controls what exactly is extracted from the forest.
+ * See documentation for that below. Note that Joshua does not store individual feature values while
+ * decoding, but only the cost of each edge (in the form of a float). Therefore, if you request
+ * the features values (`%f` in `output-format`), the feature functions must be replayed, which
+ * is expensive.</p>
+ *
+ * <p>The configuration parameter `top-n` controls how many items are returned. If this is set to 0,
+ * k-best extraction should be turned off entirely.</p>
+ *
+ * @author Zhifei Li, zhifei.work@gmail.com
+ * @author Matt Post post@cs.jhu.edu
+ */
+public class KBestExtractor {
+ private final JoshuaConfiguration joshuaConfiguration;
+ private final String outputFormat;
+ private final HashMap<HGNode, VirtualNode> virtualNodesTable = new HashMap<HGNode, VirtualNode>();
+
+ // static final String rootSym = JoshuaConfiguration.goal_symbol;
+ static final String rootSym = "ROOT";
+ static final int rootID = Vocabulary.id(rootSym);
+
+ private enum Side {
+ SOURCE, TARGET
+ };
+
+ /* Whether to extract only unique strings */
+ private final boolean extractUniqueNbest;
+
+ /* Which side to output (source or target) */
+ private final Side defaultSide;
+
+ /* The input sentence */
+ private final Sentence sentence;
+
+ /* The weights being used to score the forest */
+ private final FeatureVector weights;
+
+ /* The feature functions */
+ private final List<FeatureFunction> featureFunctions;
+
+ /* BLEU statistics of the references */
+ private BLEU.References references = null;
+
+ public KBestExtractor(
+ Sentence sentence,
+ List<FeatureFunction> featureFunctions,
+ FeatureVector weights,
+ boolean isMonolingual,
+ JoshuaConfiguration joshuaConfiguration) {
+
+ this.featureFunctions = featureFunctions;
+
+ this.joshuaConfiguration = joshuaConfiguration;
+ this.outputFormat = this.joshuaConfiguration.outputFormat;
+ this.extractUniqueNbest = joshuaConfiguration.use_unique_nbest;
+
+ this.weights = weights;
+ this.defaultSide = (isMonolingual ? Side.SOURCE : Side.TARGET);
+ this.sentence = sentence;
+
+ if (joshuaConfiguration.rescoreForest) {
+ references = new BLEU.References(sentence.references());
+ }
+ }
+
+ /**
+ * Returns the kth derivation.
+ *
+ * You may need to reset_state() before you call this function for the first time.
+ *
+ * @param node the node to start at
+ * @param k the kth best derivation (indexed from 1)
+ * @return the derivation object
+ */
+ public DerivationState getKthDerivation(HGNode node, int k) {
+ final VirtualNode virtualNode = getVirtualNode(node);
+ return virtualNode.lazyKBestExtractOnNode(this, k);
+ }
+
+ /**
+ * Returns the k-th Structured Translation.
+ *
+ * @param node The node to extract from
+ * @param k The (1-indexed) index of the item wanted
+ * @return a StructuredTranslation object
+ */
+ public StructuredTranslation getKthStructuredTranslation(HGNode node, int k) {
+ StructuredTranslation result = null;
+ final DerivationState derivationState = getKthDerivation(node, k);
+ if (derivationState != null) {
+ result = StructuredTranslationFactory.fromKBestDerivation(sentence, derivationState);
+ }
+ return result;
+ }
+
+ /**
+ * This is an entry point for extracting k-best hypotheses as StructuredTranslation objects.
+ * It computes all of them and returning a list of StructuredTranslation objects.
+ * These objects hold all translation information (string, tokens, features, alignments, score).
+ *
+ * @param hg the hypergraph to extract from
+ * @param topN how many to extract
+ * @return list of StructuredTranslation objects, empty if there is no HyperGraph goal node.
+ */
+ public List<StructuredTranslation> KbestExtractOnHG(HyperGraph hg, int topN) {
+ resetState();
+ if (hg == null || hg.goalNode == null) {
+ return emptyList();
+ }
+ final List<StructuredTranslation> kbest = new ArrayList<>(topN);
+ for (int k = 1; k <= topN; k++) {
+ StructuredTranslation translation = getKthStructuredTranslation(hg.goalNode, k);
+ if (translation == null) {
+ break;
+ }
+ kbest.add(translation);
+ }
+ return kbest;
+ }
+
+ /**
+ * Compute the string that is output from the decoder, using the "output-format" config file
+ * parameter as a template.
+ *
+ * You may need to {@link org.apache.joshua.decoder.hypergraph.KBestExtractor#resetState()}
+ * before you call this function for the first time.
+ *
+ * @param node todo
+ * @param k todo
+ * @return todo
+ */
+ public String getKthHyp(HGNode node, int k) {
+
+ String outputString = null;
+ DerivationState derivationState = getKthDerivation(node, k);
+ if (derivationState != null) {
+ // ==== read the kbest from each hgnode and convert to output format
+ String hypothesis = maybeProjectCase(
+ unescapeSpecialSymbols(
+ removeSentenceMarkers(
+ derivationState.getHypothesis())), derivationState);
+
+
+ /*
+ * To save space, the decoder only stores the model cost,
+ * no the individual feature values.
+ * If you want to output them, you have to replay them.
+ */
+
+ FeatureVector features = new FeatureVector();
+ if (outputFormat.contains("%f") || outputFormat.contains("%d"))
+ features = derivationState.getFeatures();
+
+ outputString = outputFormat
+ .replace("%k", Integer.toString(k))
+ .replace("%s", hypothesis)
+ .replace("%S", DeNormalize.processSingleLine(hypothesis))
+ // TODO (kellens): Fix the recapitalization here
+ .replace("%i", Integer.toString(sentence.id()))
+ .replace("%f", joshuaConfiguration.moses ? features.mosesString() : features.toString())
+ .replace("%c", String.format("%.3f", derivationState.cost));
+
+ if (outputFormat.contains("%t")) {
+ outputString = outputString.replace("%t", derivationState.getTree());
+ }
+
+ if (outputFormat.contains("%e")) {
+ outputString = outputString.replace("%e", removeSentenceMarkers(derivationState.getHypothesis(Side.SOURCE)));
+ }
+
+ /* %d causes a derivation with rules one per line to be output */
+ if (outputFormat.contains("%d")) {
+ outputString = outputString.replace("%d", derivationState.getDerivation());
+ }
+
+ /* %a causes output of word level alignments between input and output hypothesis */
+ if (outputFormat.contains("%a")) {
+ outputString = outputString.replace("%a", derivationState.getWordAlignment());
+ }
+
+ }
+
+ return outputString;
+ }
+
+ // =========================== end kbestHypergraph
+
+ /**
+ * If requested, projects source-side lettercase to target, and appends the alignment from
+ * to the source-side sentence in ||s.
+ *
+ * @param hypothesis todo
+ * @param state todo
+ * @return source-side lettercase to target, and appends the alignment from to the source-side sentence in ||s
+ */
+ private String maybeProjectCase(String hypothesis, DerivationState state) {
+ String output = hypothesis;
+
+ if (joshuaConfiguration.project_case) {
+ String[] tokens = hypothesis.split("\\s+");
+ List<List<Integer>> points = state.getWordAlignmentList();
+ for (int i = 0; i < points.size(); i++) {
+ List<Integer> target = points.get(i);
+ for (int source: target) {
+ Token token = sentence.getTokens().get(source + 1); // skip <s>
+ String annotation = "";
+ if (token != null && token.getAnnotation("lettercase") != null)
+ annotation = token.getAnnotation("lettercase");
+ if (source != 0 && annotation.equals("upper"))
+ tokens[i] = FormatUtils.capitalize(tokens[i]);
+ else if (annotation.equals("all-upper"))
+ tokens[i] = tokens[i].toUpperCase();
+ }
+ }
+
+ output = String.join(" ", tokens);
+ }
+
+ return output;
+ }
+
+ /**
+ * Convenience function for k-best extraction that prints to STDOUT.
+ * @param hg the {@link org.apache.joshua.decoder.hypergraph.HyperGraph} from which to extract
+ * @param topN the number of k
+ * @throws IOException if there is an error writing the extraction
+ */
+ public void lazyKBestExtractOnHG(HyperGraph hg, int topN) throws IOException {
+ lazyKBestExtractOnHG(hg, topN, new BufferedWriter(new OutputStreamWriter(System.out)));
+ }
+
+ /**
+ * This is the entry point for extracting k-best hypotheses. It computes all of them, writing
+ * the results to the BufferedWriter passed in. If you want intermediate access to the k-best
+ * derivations, you'll want to call getKthHyp() or getKthDerivation() directly.
+ *
+ * The number of derivations that are looked for is controlled by the `top-n` parameter.
+ * Note that when `top-n` is set to 0, k-best extraction is disabled entirely, and only things
+ * like the viterbi string and the model score are available to the decoder. Since k-best
+ * extraction involves the recomputation of features to get the component values, turning off
+ * that extraction saves a lot of time when only the 1-best string is desired.
+ *
+ * @param hg the hypergraph to extract from
+ * @param topN how many to extract
+ * @param out object to write to
+ * @throws IOException if there is an error writing the extraction
+ */
+ public void lazyKBestExtractOnHG(HyperGraph hg, int topN, BufferedWriter out) throws IOException {
+
+ resetState();
+
+ if (null == hg.goalNode)
+ return;
+
+ for (int k = 1; k <= topN; k++) {
+ String hypStr = getKthHyp(hg.goalNode, k);
+ if (null == hypStr)
+ break;
+
+ out.write(hypStr);
+ out.write("\n");
+ out.flush();
+ }
+ }
+
+ /**
+ * This clears the virtualNodesTable, which maintains a list of virtual nodes. This should be
+ * called in between forest rescorings.
+ */
+ public void resetState() {
+ virtualNodesTable.clear();
+ }
+
+ /**
+ * Returns the {@link org.apache.joshua.decoder.hypergraph.KBestExtractor.VirtualNode}
+ * corresponding to an {@link org.apache.joshua.decoder.hypergraph.HGNode}.
+ * If no such VirtualNode exists, it is created.
+ *
+ * @param hgnode from which we wish to create a
+ * {@link org.apache.joshua.decoder.hypergraph.KBestExtractor.VirtualNode}
+ * @return the corresponding {@link org.apache.joshua.decoder.hypergraph.KBestExtractor.VirtualNode}
+ */
+ private VirtualNode getVirtualNode(HGNode hgnode) {
+ VirtualNode virtualNode = virtualNodesTable.get(hgnode);
+ if (null == virtualNode) {
+ virtualNode = new VirtualNode(hgnode);
+ virtualNodesTable.put(hgnode, virtualNode);
+ }
+ return virtualNode;
+ }
+
+ /**
+ * This class is essentially a wrapper around an HGNode, annotating it with information needed to
+ * record which hypotheses have been explored from this point. There is one virtual node for
+ * each HGNode in the underlying hypergraph. This VirtualNode maintains information about the
+ * k-best derivations from that point on, retaining the derivations computed so far and a priority
+ * queue of candidates.
+ */
+ private class VirtualNode {
+
+ // The node being annotated.
+ HGNode node = null;
+
+ // sorted ArrayList of DerivationState, in the paper is: D(^) [v]
+ public List<DerivationState> nbests = new ArrayList<DerivationState>();
+
+ // remember frontier states, best-first; in the paper, it is called cand[v]
+ private PriorityQueue<DerivationState> candHeap = null;
+
+ // Remember which DerivationState has been explored (positions in the hypercube). This allows
+ // us to avoid duplicated states that are reached from different places of expansion, e.g.,
+ // position (2,2) can be reached be extending (1,2) and (2,1).
+ private HashSet<DerivationState> derivationTable = null;
+
+ // This records unique *strings* at each item, used for unique-nbest-string extraction.
+ private HashSet<String> uniqueStringsTable = null;
+
+ public VirtualNode(HGNode it) {
+ this.node = it;
+ }
+
+ /**
+ * This returns a DerivationState corresponding to the kth-best derivation rooted at this node.
+ *
+ * @param kbestExtractor todo
+ * @param k (indexed from one)
+ * @return the k-th best (1-indexed) hypothesis, or null if there are no more.
+ */
+ // return: the k-th hyp or null; k is started from one
+ private DerivationState lazyKBestExtractOnNode(KBestExtractor kbestExtractor, int k) {
+ if (nbests.size() >= k) { // no need to continue
+ return nbests.get(k - 1);
+ }
+
+ // ### we need to fill in the l_nest in order to get k-th hyp
+ DerivationState derivationState = null;
+
+ /*
+ * The first time this is called, the heap of candidates (the frontier of the cube) is
+ * uninitialized. This recursive call will seed the candidates at each node.
+ */
+ if (null == candHeap) {
+ getCandidates(kbestExtractor);
+ }
+
+ /*
+ * Now build the kbest list by repeatedly popping the best candidate and then placing all
+ * extensions of that hypothesis back on the candidates list.
+ */
+ int tAdded = 0; // sanity check
+ while (nbests.size() < k) {
+ if (candHeap.size() > 0) {
+ derivationState = candHeap.poll();
+ // derivation_tbl.remove(res.get_signature());//TODO: should remove? note that two state
+ // may be tied because the cost is the same
+ if (extractUniqueNbest) {
+ // We pass false for extract_nbest_tree because we want; to check that the hypothesis
+ // *strings* are unique, not the trees.
+ final String res_str = derivationState.getHypothesis();
+
+ if (!uniqueStringsTable.contains(res_str)) {
+ nbests.add(derivationState);
+ uniqueStringsTable.add(res_str);
+ }
+ } else {
+ nbests.add(derivationState);
+ }
+
+ // Add all extensions of this hypothesis to the candidates list.
+ lazyNext(kbestExtractor, derivationState);
+
+ // debug: sanity check
+ tAdded++;
+ // this is possible only when extracting unique nbest
+ if (!extractUniqueNbest && tAdded > 1) {
+ throw new RuntimeException("In lazyKBestExtractOnNode, add more than one time, k is "
+ + k);
+ }
+ } else {
+ break;
+ }
+ }
+ if (nbests.size() < k) {
+ derivationState = null;// in case we do not get to the depth of k
+ }
+ // debug: sanity check
+ // if (l_nbest.size() >= k && l_nbest.get(k-1) != res) {
+ // throw new RuntimeException("In lazy_k_best_extract, ranking is not correct ");
+ // }
+
+ return derivationState;
+ }
+
+ /**
+ * This function extends the current hypothesis, adding each extended item to the list of
+ * candidates (assuming they have not been added before). It does this by, in turn, extending
+ * each of the tail node items.
+ *
+ * @param kbestExtractor
+ * @param previousState
+ */
+ private void lazyNext(KBestExtractor kbestExtractor, DerivationState previousState) {
+ /* If there are no tail nodes, there is nothing to do. */
+ if (null == previousState.edge.getTailNodes())
+ return;
+
+ /* For each tail node, create a new state candidate by "sliding" that item one position. */
+ for (int i = 0; i < previousState.edge.getTailNodes().size(); i++) {
+ /* Create a new virtual node that is a copy of the current node */
+ HGNode tailNode = (HGNode) previousState.edge.getTailNodes().get(i);
+ VirtualNode virtualTailNode = kbestExtractor.getVirtualNode(tailNode);
+ // Copy over the ranks.
+ int[] newRanks = new int[previousState.ranks.length];
+ for (int c = 0; c < newRanks.length; c++) {
+ newRanks[c] = previousState.ranks[c];
+ }
+ // Now increment/slide the current tail node by one
+ newRanks[i] = previousState.ranks[i] + 1;
+
+ // Create a new state so we can see if it's new. The cost will be set below if it is.
+ DerivationState nextState = new DerivationState(previousState.parentNode,
+ previousState.edge, newRanks, 0.0f, previousState.edgePos);
+
+ // Don't add the state to the list of candidates if it's already been added.
+ if (!derivationTable.contains(nextState)) {
+ // Make sure that next candidate exists
+ virtualTailNode.lazyKBestExtractOnNode(kbestExtractor, newRanks[i]);
+ // System.err.println(String.format(" newRanks[%d] = %d and tail size %d", i,
+ // newRanks[i], virtualTailNode.nbests.size()));
+ if (newRanks[i] <= virtualTailNode.nbests.size()) {
+ // System.err.println("NODE: " + this.node);
+ // System.err.println(" tail is " + virtualTailNode.node);
+ float cost = previousState.getModelCost()
+ - virtualTailNode.nbests.get(previousState.ranks[i] - 1).getModelCost()
+ + virtualTailNode.nbests.get(newRanks[i] - 1).getModelCost();
+ nextState.setCost(cost);
+
+ if (joshuaConfiguration.rescoreForest)
+ nextState.bleu = nextState.computeBLEU();
+
+ candHeap.add(nextState);
+ derivationTable.add(nextState);
+
+ // System.err.println(String.format(" LAZYNEXT(%s", nextState));
+ }
+ }
+ }
+ }
+
+ /**
+ * this is the seeding function, for example, it will get down to the leaf, and sort the
+ * terminals get a 1best from each hyperedge, and add them into the heap_cands
+ *
+ * @param kbestExtractor
+ */
+ private void getCandidates(KBestExtractor kbestExtractor) {
+ /* The list of candidates extending from this (virtual) node. */
+ candHeap = new PriorityQueue<DerivationState>(11, new DerivationStateComparator());
+
+ /*
+ * When exploring the cube frontier, there are multiple paths to each candidate. For example,
+ * going down 1 from grid position (2,1) is the same as going right 1 from grid position
+ * (1,2). To avoid adding states more than once, we keep a list of derivation states we have
+ * already added to the candidates heap.
+ *
+ * TODO: these should really be keyed on the states themselves instead of a string
+ * representation of them.
+ */
+ derivationTable = new HashSet<DerivationState>();
+
+ /*
+ * A Joshua configuration option allows the decoder to output only unique strings. In that
+ * case, we keep an list of the frontiers of derivation states extending from this node.
+ */
+ if (extractUniqueNbest) {
+ uniqueStringsTable = new HashSet<String>();
+ }
+
+ /*
+ * Get the single-best derivation along each of the incoming hyperedges, and add the lot of
+ * them to the priority queue of candidates in the form of DerivationState objects.
+ *
+ * Note that since the hyperedges are not sorted according to score, the first derivation
+ * computed here may not be the best. But since the loop over all hyperedges seeds the entire
+ * candidates list with the one-best along each of them, when the candidate heap is polled
+ * afterwards, we are guaranteed to have the best one.
+ */
+ int pos = 0;
+ for (HyperEdge edge : node.hyperedges) {
+ DerivationState bestState = getBestDerivation(kbestExtractor, node, edge, pos);
+ // why duplicate, e.g., 1 2 + 1 0 == 2 1 + 0 1 , but here we should not get duplicate
+ if (!derivationTable.contains(bestState)) {
+ candHeap.add(bestState);
+ derivationTable.add(bestState);
+ } else { // sanity check
+ throw new RuntimeException(
+ "get duplicate derivation in get_candidates, this should not happen"
+ + "\nsignature is " + bestState + "\nl_hyperedge size is "
+ + node.hyperedges.size());
+ }
+ pos++;
+ }
+
+ // TODO: if tem.size is too large, this may cause unnecessary computation, we comment the
+ // segment to accommodate the unique nbest extraction
+ /*
+ * if(tem.size()>global_n){ heap_cands=new PriorityQueue<DerivationState>(new DerivationStateComparator()); for(int i=1;
+ * i<=global_n; i++) heap_cands.add(tem.poll()); }else heap_cands=tem;
+ */
+ }
+
+ // get my best derivation, and recursively add 1best for all my children, used by get_candidates
+ // only
+ /**
+ * This computes the best derivation along a particular hyperedge. It is only called by
+ * getCandidates() to initialize the candidates priority queue at each (virtual) node.
+ *
+ * @param kbestExtractor
+ * @param parentNode
+ * @param hyperEdge
+ * @param edgePos
+ * @return an object representing the best derivation from this node
+ */
+ private DerivationState getBestDerivation(KBestExtractor kbestExtractor, HGNode parentNode,
+ HyperEdge hyperEdge, int edgePos) {
+ int[] ranks;
+ float cost = 0.0f;
+
+ /*
+ * There are two cases: (1) leaf nodes and (2) internal nodes. A leaf node is represented by a
+ * hyperedge with no tail nodes.
+ */
+ if (hyperEdge.getTailNodes() == null) {
+ ranks = null;
+
+ } else {
+ // "ranks" records which derivation to take at each of the tail nodes. Ranks are 1-indexed.
+ ranks = new int[hyperEdge.getTailNodes().size()];
+
+ /* Initialize the one-best at each tail node. */
+ for (int i = 0; i < hyperEdge.getTailNodes().size(); i++) { // children is ready
+ ranks[i] = 1;
+ VirtualNode childVirtualNode = kbestExtractor.getVirtualNode(hyperEdge.getTailNodes()
+ .get(i));
+ // recurse
+ childVirtualNode.lazyKBestExtractOnNode(kbestExtractor, ranks[i]);
+ }
+ }
+ cost = (float) hyperEdge.getBestDerivationScore();
+
+ DerivationState state = new DerivationState(parentNode, hyperEdge, ranks, cost, edgePos);
+ if (joshuaConfiguration.rescoreForest)
+ state.bleu = state.computeBLEU();
+
+ return state;
+ }
+ };
+
+ /**
+ * A DerivationState describes which path to follow through the hypergraph. For example, it
+ * might say to use the 1-best from the first tail node, the 9th-best from the second tail node,
+ * and so on. This information is represented recursively through a chain of DerivationState
+ * objects. This function follows that chain, extracting the information according to a number
+ * of parameters, and returning results to a string, and also (optionally) accumulating the
+ * feature values into the passed-in FeatureVector.
+ */
+
+ // each DerivationState roughly corresponds to a hypothesis
+ public class DerivationState {
+ /* The edge ("e" in the paper) */
+ public HyperEdge edge;
+
+ /* The edge's parent node */
+ public HGNode parentNode;
+
+ /*
+ * This state's position in its parent node's list of incoming hyperedges (used in signature
+ * calculation)
+ */
+ public int edgePos;
+
+ /*
+ * The rank item to select from each of the incoming tail nodes ("j" in the paper, an ArrayList
+ * of size |e|)
+ */
+ public int[] ranks;
+
+ /*
+ * The cost of the hypothesis, including a weighted BLEU score, if any.
+ */
+ private float cost;
+
+ private float bleu = 0.0f;
+
+ /*
+ * The BLEU sufficient statistics associated with the edge's derivation. Note that this is a
+ * function of the complete derivation headed by the edge, i.e., all the particular
+ * subderivations of edges beneath it. That is why it must be contained in DerivationState
+ * instead of in the HyperEdge itself.
+ */
+ BLEU.Stats stats = null;
+
+ public DerivationState(HGNode pa, HyperEdge e, int[] r, float c, int pos) {
+ parentNode = pa;
+ edge = e;
+ ranks = r;
+ cost = c;
+ edgePos = pos;
+ bleu = 0.0f;
+ }
+
+ /**
+ * Computes a scaled approximate BLEU from the accumulated statistics. We know the number of
+ * words; to compute the effective reference length, we take the real reference length statistic
+ * and scale it by the percentage of the input sentence that is consumed, based on the
+ * assumption that the total number of words in the hypothesis scales linearly with the input
+ * sentence span.
+ *
+ * @return float representing {@link org.apache.joshua.decoder.BLEU} score
+ */
+ public float computeBLEU() {
+ if (stats == null) {
+ float percentage = 1.0f * (parentNode.j - parentNode.i) / (sentence.length());
+ // System.err.println(String.format("computeBLEU: (%d - %d) / %d = %f", parentNode.j,
+ // parentNode.i, sentence.length(), percentage));
+ stats = BLEU.compute(edge, percentage, references);
+
+ if (edge.getTailNodes() != null) {
+ for (int id = 0; id < edge.getTailNodes().size(); id++) {
+ stats.add(getChildDerivationState(edge, id).stats);
+ }
+ }
+ }
+
+ return BLEU.score(stats);
+ }
+
+ public void setCost(float cost2) {
+ this.cost = cost2;
+ }
+
+ /**
+ * Returns the model cost. This is obtained by subtracting off the incorporated BLEU score (if
+ * used).
+ *
+ * @return float representing model cost
+ */
+ public float getModelCost() {
+ return this.cost;
+ }
+
+ /**
+ * Returns the model cost plus the BLEU score.
+ *
+ * @return float representing model cost plus the BLEU score
+ */
+ public float getCost() {
+ return cost - weights.getSparse("BLEU") * bleu;
+ }
+
+ public String toString() {
+ StringBuilder sb = new StringBuilder(String.format("DS[[ %s (%d,%d)/%d ||| ",
+ Vocabulary.word(parentNode.lhs), parentNode.i, parentNode.j, edgePos));
+ sb.append("ranks=[ ");
+ if (ranks != null)
+ for (int i = 0; i < ranks.length; i++)
+ sb.append(ranks[i] + " ");
+ sb.append("] ||| " + String.format("%.5f ]]", cost));
+ return sb.toString();
+ }
+
+ public boolean equals(Object other) {
+ if (other instanceof DerivationState) {
+ DerivationState that = (DerivationState) other;
+ if (edgePos == that.edgePos) {
+ if (ranks != null && that.ranks != null) {
+ if (ranks.length == that.ranks.length) {
+ for (int i = 0; i < ranks.length; i++)
+ if (ranks[i] != that.ranks[i])
+ return false;
+ return true;
+ }
+ }
+ }
+ }
+
+ return false;
+ }
+
+ /**
+ * DerivationState objects are unique to each VirtualNode, so the unique identifying information
+ * only need contain the edge position and the ranks.
+ * @return hashof the edge position and ranks
+ */
+ public int hashCode() {
+ int hash = edgePos;
+ if (ranks != null) {
+ for (int i = 0; i < ranks.length; i++)
+ hash = hash * 53 + i;
+ }
+
+ return hash;
+ }
+
+ /**
+ * Visits every state in the derivation in a depth-first order.
+ * @param visitor todo
+ * @return todo
+ */
+ private DerivationVisitor visit(DerivationVisitor visitor) {
+ return visit(visitor, 0, 0);
+ }
+
+ private DerivationVisitor visit(DerivationVisitor visitor, int indent, int tailNodeIndex) {
+
+ visitor.before(this, indent, tailNodeIndex);
+
+ final Rule rule = edge.getRule();
+ final List<HGNode> tailNodes = edge.getTailNodes();
+
+ if (rule == null) {
+ getChildDerivationState(edge, 0).visit(visitor, indent + 1, 0);
+ } else {
+ if (tailNodes != null) {
+ for (int index = 0; index < tailNodes.size(); index++) {
+ getChildDerivationState(edge, index).visit(visitor, indent + 1, index);
+ }
+ }
+ }
+
+ visitor.after(this, indent, tailNodeIndex);
+
+ return visitor;
+ }
+
+ public String getWordAlignment() {
+ return visit(new WordAlignmentExtractor()).toString();
+ }
+
+ public List<List<Integer>> getWordAlignmentList() {
+ final WordAlignmentExtractor visitor = new WordAlignmentExtractor();
+ visit(visitor);
+ return visitor.getFinalWordAlignments();
+ }
+
+ public String getTree() {
+ return visit(new TreeExtractor()).toString();
+ }
+
+ public String getHypothesis() {
+ return getHypothesis(defaultSide);
+ }
+
+ private String getHypothesis(final Side side) {
+ return visit(new OutputStringExtractor(side.equals(Side.SOURCE))).toString();
+ }
+
+ public FeatureVector getFeatures() {
+ final FeatureVectorExtractor extractor = new FeatureVectorExtractor(featureFunctions, sentence);
+ visit(extractor);
+ return extractor.getFeatures();
+ }
+
+ public String getDerivation() {
+ return visit(new DerivationExtractor()).toString();
+ }
+
+ /**
+ * Helper function for navigating the hierarchical list of DerivationState objects. This
+ * function looks up the VirtualNode corresponding to the HGNode pointed to by the edge's
+ * {tailNodeIndex}th tail node.
+ *
+ * @param edge todo
+ * @param tailNodeIndex todo
+ * @return todo
+ */
+ public DerivationState getChildDerivationState(HyperEdge edge, int tailNodeIndex) {
+ HGNode child = edge.getTailNodes().get(tailNodeIndex);
+ VirtualNode virtualChild = getVirtualNode(child);
+ return virtualChild.nbests.get(ranks[tailNodeIndex] - 1);
+ }
+
+ } // end of Class DerivationState
+
+ public static class DerivationStateComparator implements Comparator<DerivationState> {
+ // natural order by cost
+ public int compare(DerivationState one, DerivationState another) {
+ if (one.getCost() > another.getCost()) {
+ return -1;
+ } else if (one.getCost() == another.getCost()) {
+ return 0;
+ } else {
+ return 1;
+ }
+ }
+ }
+
+ /**
+ * This interface provides a generic way to do things at each stage of a derivation. The
+ * DerivationState::visit() function visits every node in a derivation and calls the
+ * DerivationVisitor functions both before and after it visits each node. This provides a common
+ * way to do different things to the tree (e.g., extract its words, assemble a derivation, and so
+ * on) without having to rewrite the node-visiting code.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ */
+ public interface DerivationVisitor {
+ /**
+ * Called before each node's children are visited.
+ *
+ * @param state the derivation state
+ * @param level the tree depth
+ * @param tailNodeIndex the tailNodeIndex corresponding to state
+ */
+ void before(DerivationState state, int level, int tailNodeIndex);
+
+ /**
+ * Called after a node's children have been visited.
+ *
+ * @param state the derivation state
+ * @param level the tree depth
+ * @param tailNodeIndex the tailNodeIndex corresponding to state
+ */
+ void after(DerivationState state, int level, int tailNodeIndex);
+ }
+
+ /**
+ * Assembles a Penn treebank format tree for a given derivation.
+ */
+ public class TreeExtractor implements DerivationVisitor {
+
+ /* The tree being built. */
+ private Tree tree;
+
+ public TreeExtractor() {
+ tree = null;
+ }
+
+ /**
+ * Before visiting the children, find the fragment representation for the current rule,
+ * and merge it into the tree we're building.
+ */
+ @Override
+ public void before(DerivationState state, int indent, int tailNodeIndex) {
+ HyperEdge edge = state.edge;
+ Rule rule = edge.getRule();
+
+ // Skip the special top-level rule
+ if (rule == null) {
+ return;
+ }
+
+ String lhs = Vocabulary.word(rule.getLHS());
+ String unbracketedLHS = lhs.substring(1, lhs.length() - 1);
+
+ /* Find the fragment corresponding to this flattened rule in the fragment map; if it's not
+ * there, just pretend it's a depth-one rule.
+ */
+ Tree fragment = Tree.getFragmentFromYield(rule.getEnglishWords());
+ if (fragment == null) {
+ String subtree = String.format("(%s{%d-%d} %s)", unbracketedLHS,
+ state.parentNode.i, state.parentNode.j,
+ quoteTerminals(rule.getEnglishWords()));
+ fragment = Tree.fromString(subtree);
+ }
+
+ merge(fragment);
+ }
+
+ /**
+ * Quotes just the terminals in the yield of a tree, represented as a string. This is to force
+ * compliance with the Tree class, which interprets all non-quoted strings as nonterminals.
+ *
+ * @param words a string of words representing a rule's yield
+ * @return
+ */
+ private String quoteTerminals(String words) {
+ StringBuilder quotedWords = new StringBuilder();
+ for (String word: words.split("\\s+"))
+ if (word.startsWith("[") && word.endsWith("]"))
+ quotedWords.append(String.format("%s ", word));
+ else
+ quotedWords.append(String.format("\"%s\" ", word));
+
+ return quotedWords.substring(0, quotedWords.length() - 1);
+ }
+
+ @Override
+ public void after(DerivationState state, int indent, int tailNodeIndex) {
+ // do nothing
+ }
+
+ public String toString() {
+ return tree.unquotedString();
+ }
+
+ /**
+ * Either set the root of the tree or merge this tree by grafting it onto the first nonterminal
+ * in the yield of the parent tree.
+ *
+ * @param fragment
+ */
+ private void merge(Tree fragment) {
+ if (tree == null) {
+ tree = fragment;
+ } else {
+ Tree parent = tree.getNonterminalYield().get(0);
+ parent.setLabel(Vocabulary.word(fragment.getLabel()));
+ parent.setChildren(fragment.getChildren());
+ }
+ }
+ }
+
+ /**
+ * Assembles an informative version of the derivation. Each rule is printed as it is encountered.
+ * Don't try to parse this output; make something that writes out JSON or something, instead.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ */
+ public class DerivationExtractor implements DerivationVisitor {
+
+ StringBuffer sb;
+
+ public DerivationExtractor() {
+ sb = new StringBuffer();
+ }
+
+ @Override
+ public void before(DerivationState state, int indent, int tailNodeIndex) {
+
+ HyperEdge edge = state.edge;
+ Rule rule = edge.getRule();
+
+ if (rule != null) {
+
+ for (int i = 0; i < indent * 2; i++)
+ sb.append(" ");
+
+ final FeatureVectorExtractor extractor = new FeatureVectorExtractor(featureFunctions, sentence);
+ extractor.before(state, indent, tailNodeIndex);
+ final FeatureVector transitionFeatures = extractor.getFeatures();
+
+ // sb.append(rule).append(" ||| " + features + " ||| " +
+ // KBestExtractor.this.weights.innerProduct(features));
+ sb.append(String.format("%d-%d", state.parentNode.i, state.parentNode.j));
+ sb.append(" ||| " + Vocabulary.word(rule.getLHS()) + " -> "
+ + Vocabulary.getWords(rule.getFrench()) + " /// " + rule.getEnglishWords());
+ sb.append(" |||");
+ for (DPState dpState : state.parentNode.getDPStates()) {
+ sb.append(" " + dpState);
+ }
+ sb.append(" ||| " + transitionFeatures);
+ sb.append(" ||| " + weights.innerProduct(transitionFeatures));
+ if (rule.getAlignment() != null)
+ sb.append(" ||| " + Arrays.toString(rule.getAlignment()));
+ sb.append("\n");
+ }
+ }
+
+ public String toString() {
+ return sb.toString();
+ }
+
+ @Override
+ public void after(DerivationState state, int level, int tailNodeIndex) {}
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/OutputStringExtractor.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/OutputStringExtractor.java b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/OutputStringExtractor.java
new file mode 100644
index 0000000..f20e063
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/OutputStringExtractor.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.hypergraph;
+
+import static java.lang.Math.min;
+import static org.apache.joshua.corpus.Vocabulary.getWords;
+
+import java.util.Stack;
+
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.hypergraph.KBestExtractor.DerivationState;
+import org.apache.joshua.decoder.hypergraph.KBestExtractor.DerivationVisitor;
+import org.apache.joshua.util.FormatUtils;
+
+public class OutputStringExtractor implements WalkerFunction, DerivationVisitor {
+
+ public OutputStringExtractor(final boolean extractSource) {
+ this.extractSource = extractSource;
+ }
+
+ private Stack<OutputString> outputStringStack = new Stack<>();
+ private final boolean extractSource;
+
+ @Override
+ public void apply(HGNode node, int nodeIndex) {
+ apply(node.bestHyperedge.getRule(), nodeIndex);
+ }
+
+ /**
+ * Visiting a node during k-best extraction is the same as
+ * apply() for Viterbi extraction but using the edge from
+ * the Derivation state.
+ */
+ @Override
+ public void before(final DerivationState state, int level, int tailNodeIndex) {
+ apply(state.edge.getRule(), tailNodeIndex);
+ }
+
+ private void apply(Rule rule, int nodeIndex) {
+ if (rule != null) {
+ final int[] words = extractSource ? rule.getFrench() : rule.getEnglish();
+ merge(new OutputString(words, rule.getArity(), nodeIndex));
+ }
+ }
+
+ /** Nothing to do */
+ @Override
+ public void after(DerivationState state, int level, int tailNodeIndex) {}
+
+ private static int getSourceNonTerminalPosition(final int[] words, int nonTerminalIndex) {
+ int nonTerminalsSeen = 0;
+ for (int i = 0; i < words.length; i++) {
+ if (FormatUtils.isNonterminal(words[i])) {
+ nonTerminalsSeen++;
+ if (nonTerminalsSeen == nonTerminalIndex) {
+ return i;
+ }
+ }
+ }
+ throw new RuntimeException(
+ String.format(
+ "Can not find %s-th non terminal in source ids: %s. This should not happen!",
+ nonTerminalIndex,
+ arrayToString(words)));
+ }
+
+ /**
+ * Returns the position of the nonTerminalIndex-th nonTerminal words.
+ * Non-terminals on target sides of rules are indexed by
+ * their order on the source side, e.g. '-1', '-2',
+ * Thus, if index==0 we return the index of '-1'.
+ * For index==1, we return index of '-2'
+ */
+ private static int getTargetNonTerminalPosition(int[] words, int nonTerminalIndex) {
+ for (int pos = 0; pos < words.length; pos++) {
+ if (FormatUtils.isNonterminal(words[pos]) && -(words[pos] + 1) == nonTerminalIndex) {
+ return pos;
+ }
+ }
+ throw new RuntimeException(
+ String.format(
+ "Can not find %s-th non terminal in target ids: %s. This should not happen!",
+ nonTerminalIndex,
+ arrayToString(words)));
+ }
+
+ private static String arrayToString(int[] ids) {
+ StringBuilder sb = new StringBuilder();
+ for (int i : ids) {
+ sb.append(i + " ");
+ }
+ return sb.toString().trim();
+ }
+
+ private void substituteNonTerminal(
+ final OutputString parentState,
+ final OutputString childState) {
+ int mergePosition;
+ if (extractSource) {
+ /* correct nonTerminal is given by the tailNodePosition of the childState (zero-index, thus +1) and
+ * current parentState's arity. If the parentState has already filled one of two available slots,
+ * we need to use the remaining one, even if childState refers to the second slot.
+ */
+ mergePosition = getSourceNonTerminalPosition(
+ parentState.words, min(childState.tailNodePosition + 1, parentState.arity));
+ } else {
+ mergePosition = getTargetNonTerminalPosition(
+ parentState.words, childState.tailNodePosition);
+ }
+ parentState.substituteNonTerminalAtPosition(childState.words, mergePosition);
+ }
+
+ private void merge(final OutputString state) {
+ if (!outputStringStack.isEmpty()
+ && state.arity == 0) {
+ if (outputStringStack.peek().arity == 0) {
+ throw new IllegalStateException("Parent OutputString has arity of 0. Cannot merge.");
+ }
+ final OutputString parent = outputStringStack.pop();
+ substituteNonTerminal(parent, state);
+ merge(parent);
+ } else {
+ outputStringStack.add(state);
+ }
+ }
+
+ @Override
+ public String toString() {
+ if (outputStringStack.isEmpty()) {
+ return "";
+ }
+
+ if (outputStringStack.size() != 1) {
+ throw new IllegalStateException(
+ String.format(
+ "Stack should contain only a single (last) element, but was size %d", outputStringStack.size()));
+ }
+ return getWords(outputStringStack.pop().words);
+ }
+
+ /** Stores necessary information to obtain an output string on source or target side */
+ private class OutputString {
+
+ private int[] words;
+ private int arity;
+ private final int tailNodePosition;
+
+ private OutputString(int[] words, int arity, int tailNodePosition) {
+ this.words = words;
+ this.arity = arity;
+ this.tailNodePosition = tailNodePosition;
+ }
+
+ /**
+ * Merges child words into this at the correct
+ * non terminal position of this.
+ * The correct position is determined by the tailNodePosition
+ * of child and the arity of this.
+ */
+ private void substituteNonTerminalAtPosition(final int[] words, final int position) {
+ assert(FormatUtils.isNonterminal(this.words[position]));
+ final int[] result = new int[words.length + this.words.length - 1];
+ int resultIndex = 0;
+ for (int i = 0; i < position; i++) {
+ result[resultIndex++] = this.words[i];
+ }
+ for (int i = 0; i < words.length; i++) {
+ result[resultIndex++] = words[i];
+ }
+ for (int i = position + 1; i < this.words.length; i++) {
+ result[resultIndex++] = this.words[i];
+ }
+ // update words and reduce arity of this OutputString
+ this.words = result;
+ arity--;
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/StringToTreeConverter.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/StringToTreeConverter.java b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/StringToTreeConverter.java
new file mode 100644
index 0000000..f393a01
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/StringToTreeConverter.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.hypergraph;
+
+import java.util.Stack;
+
+// example: (ROOT ([S] ([X] ([X] scientists completed ([X] for ([X] ([X] chromosome) related to ([X]
+// early ([X] OOV))))) sequencing)))
+
+public class StringToTreeConverter {
+
+ static private final String beginSymbol = "(b";
+ static private final String nodeSymbol = "node";
+
+ HyperGraph convert(String inputStr) {
+
+ HyperGraph tree = null;
+
+ Stack<String> stack = new Stack<String>();
+ for (int i = 0; i < inputStr.length(); i++) {
+ char curChar = inputStr.charAt(i);
+
+ if (curChar == ')' && inputStr.charAt(i - 1) != ' ') {// end of a rule
+ StringBuffer ruleString = new StringBuffer();
+
+ while (stack.empty() == false) {
+ String cur = stack.pop();
+ if (cur.equals(beginSymbol)) {// stop
+ // setup a node
+ // HGNode(int i, int j, int lhs, HashMap<Integer,DPState> dpStates, HyperEdge
+ // initHyperedge, double estTotalLogP)
+ // public HyperEdge(Rule rule, double bestDerivationLogP, Double transitionLogP,
+ // List<HGNode> antNodes, SourcePath srcPath)
+ // public Rule(int lhs, int[] sourceRhs, int[] targetRhs, float[]
+ // featureScores, int arity, int owner, float latticeCost, int ruleID)
+
+
+ stack.add(nodeSymbol);// TODO: should be lHS+id
+ break;
+ } else if (cur.equals(nodeSymbol)) {
+
+ } else {
+ ruleString.append(cur);
+ }
+ }
+ } else if (curChar == '(' && inputStr.charAt(i + 1) != ' ') {// begin of a rule
+ stack.add(beginSymbol);
+ } else {
+ stack.add("" + curChar);
+ }
+ }
+
+
+
+ return tree;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/TrivialInsideOutside.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/TrivialInsideOutside.java b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/TrivialInsideOutside.java
new file mode 100644
index 0000000..67be0c1
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/TrivialInsideOutside.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.hypergraph;
+
+/**
+ * @author Zhifei Li, zhifei.work@gmail.com
+ * @version $LastChangedDate$
+ */
+
+public class TrivialInsideOutside extends DefaultInsideOutside {
+ // used by inside-outside estimation
+ protected double getHyperedgeLogProb(HyperEdge dt, HGNode parent_it) {
+ return dt.getTransitionLogP(false);// TODO this is very bad in terms of computation
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/ViterbiExtractor.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/ViterbiExtractor.java b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/ViterbiExtractor.java
new file mode 100644
index 0000000..734e0aa
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/ViterbiExtractor.java
@@ -0,0 +1,178 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.hypergraph;
+
+import static java.util.Collections.emptyList;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.joshua.decoder.ff.FeatureFunction;
+import org.apache.joshua.decoder.ff.FeatureVector;
+import org.apache.joshua.decoder.segment_file.Sentence;
+
+/**
+ * @author Zhifei Li, zhifei.work@gmail.com
+ * @author Matt Post post@cs.jhu.edu
+ */
+
+public class ViterbiExtractor {
+
+ /**
+ * This function recursively visits the nodes of the Viterbi derivation in a depth-first
+ * traversal, applying the walker to each of the nodes. It provides a more general framework for
+ * implementing operations on a tree.
+ *
+ * @param node the node to start viterbi traversal from
+ * @param walker an implementation of the WalkerFunction interface, to be applied to each node in
+ * the tree
+ * @param nodeIndex the tail node index of the given node. This allows implementations of the
+ * WalkerFunction to associate nonTerminals with the index of node in the outgoing edges
+ * list of tail nodes.
+ */
+ public static void viterbiWalk(
+ final HGNode node,
+ final WalkerFunction walker,
+ final int nodeIndex) {
+ // apply the walking function to the node
+ walker.apply(node, nodeIndex);
+ // recurse on the anterior nodes of the best hyperedge in source order
+ final HyperEdge bestEdge = node.bestHyperedge;
+ final List<HGNode> tailNodes = bestEdge.getTailNodes();
+ if (tailNodes != null) {
+ for (int tailNodeIndex = 0; tailNodeIndex < tailNodes.size(); tailNodeIndex++) {
+ viterbiWalk(tailNodes.get(tailNodeIndex), walker, tailNodeIndex);
+ }
+ }
+ }
+
+ public static void viterbiWalk(final HGNode node, final WalkerFunction walker) {
+ viterbiWalk(node, walker, 0);
+ }
+
+ /**
+ * Returns the Viterbi translation of the Hypergraph (includes sentence markers)
+ * @param hg a {@link org.apache.joshua.decoder.hypergraph.HyperGraph} we wish to
+ * obtain a Viterbi translation for
+ * @return a String Viterbi translation
+ */
+ public static String getViterbiString(final HyperGraph hg) {
+ if (hg == null)
+ return "";
+
+ final WalkerFunction viterbiOutputStringWalker = new OutputStringExtractor(false);
+ viterbiWalk(hg.goalNode, viterbiOutputStringWalker);
+ return viterbiOutputStringWalker.toString();
+ }
+
+ /**
+ * Returns the Viterbi feature vector
+ * @param hg a {@link org.apache.joshua.decoder.hypergraph.HyperGraph} we wish to
+ * obtain a Viterbi features for
+ * @param featureFunctions a {@link java.util.List} of {@link org.apache.joshua.decoder.ff.FeatureFunction}'s
+ * @param sentence {@link org.apache.joshua.lattice.Lattice} input
+ * @return a Viterbi {@link org.apache.joshua.decoder.ff.FeatureVector}
+ */
+ public static FeatureVector getViterbiFeatures(
+ final HyperGraph hg,
+ final List<FeatureFunction> featureFunctions,
+ final Sentence sentence) {
+ if (hg == null)
+ return new FeatureVector();
+
+ final FeatureVectorExtractor extractor = new FeatureVectorExtractor(
+ featureFunctions, sentence);
+ viterbiWalk(hg.goalNode, extractor);
+ return extractor.getFeatures();
+ }
+
+ /**
+ * Returns the Viterbi Word Alignments as String.
+ * @param hg input {@link org.apache.joshua.decoder.hypergraph.HyperGraph}
+ * @return the Viterbi Word Alignments as String
+ */
+ public static String getViterbiWordAlignments(final HyperGraph hg) {
+ if (hg == null)
+ return "";
+
+ final WordAlignmentExtractor wordAlignmentWalker = new WordAlignmentExtractor();
+ viterbiWalk(hg.goalNode, wordAlignmentWalker);
+ return wordAlignmentWalker.toString();
+ }
+
+ /**
+ * Returns the Viterbi Word Alignments as list of lists (target-side).
+ * @param hg input {@link org.apache.joshua.decoder.hypergraph.HyperGraph}
+ * @return a {@link java.util.List} of Viterbi Word Alignments
+ */
+ public static List<List<Integer>> getViterbiWordAlignmentList(final HyperGraph hg) {
+ if (hg == null)
+ return emptyList();
+
+ final WordAlignmentExtractor wordAlignmentWalker = new WordAlignmentExtractor();
+ viterbiWalk(hg.goalNode, wordAlignmentWalker);
+ return wordAlignmentWalker.getFinalWordAlignments();
+ }
+
+ /**
+ * find 1best hypergraph
+ * @param hg_in input {@link org.apache.joshua.decoder.hypergraph.HyperGraph}
+ * @return new best {@link org.apache.joshua.decoder.hypergraph.HyperGraph}
+ */
+ public static HyperGraph getViterbiTreeHG(HyperGraph hg_in) {
+ HyperGraph res =
+ new HyperGraph(cloneNodeWithBestHyperedge(hg_in.goalNode), -1, -1, null);
+ // TODO: number of items/deductions
+ get1bestTreeNode(res.goalNode);
+ return res;
+ }
+
+ private static void get1bestTreeNode(HGNode it) {
+ HyperEdge dt = it.bestHyperedge;
+ if (null != dt.getTailNodes()) {
+ for (int i = 0; i < dt.getTailNodes().size(); i++) {
+ HGNode antNode = dt.getTailNodes().get(i);
+ HGNode newNode = cloneNodeWithBestHyperedge(antNode);
+ dt.getTailNodes().set(i, newNode);
+ get1bestTreeNode(newNode);
+ }
+ }
+ }
+
+ // TODO: tbl_states
+ private static HGNode cloneNodeWithBestHyperedge(HGNode inNode) {
+ List<HyperEdge> hyperedges = new ArrayList<HyperEdge>(1);
+ HyperEdge cloneEdge = cloneHyperedge(inNode.bestHyperedge);
+ hyperedges.add(cloneEdge);
+ return new HGNode(inNode.i, inNode.j, inNode.lhs, hyperedges, cloneEdge, inNode.getDPStates());
+ }
+
+
+ private static HyperEdge cloneHyperedge(HyperEdge inEdge) {
+ List<HGNode> antNodes = null;
+ if (null != inEdge.getTailNodes()) {
+ antNodes = new ArrayList<HGNode>(inEdge.getTailNodes());// l_ant_items will be changed in
+ // get_1best_tree_item
+ }
+ HyperEdge res =
+ new HyperEdge(inEdge.getRule(), inEdge.getBestDerivationScore(), inEdge.getTransitionLogP(false),
+ antNodes, inEdge.getSourcePath());
+ return res;
+ }
+}
[20/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/encoding/Analyzer.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/encoding/Analyzer.java b/joshua-core/src/main/java/org/apache/joshua/util/encoding/Analyzer.java
new file mode 100644
index 0000000..ad2910c
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/encoding/Analyzer.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util.encoding;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.TreeMap;
+
+import org.apache.joshua.util.io.LineReader;
+
+public class Analyzer {
+
+ private TreeMap<Float, Integer> histogram;
+ private int total;
+
+ public Analyzer() {
+ histogram = new TreeMap<Float, Integer>();
+ initialize();
+ }
+
+ public void initialize() {
+ histogram.clear();
+ // TODO: drop zero bucket; we won't encode zero-valued features anyway.
+ histogram.put(0.0f, 0);
+ total = 0;
+ }
+
+ public void add(float key) {
+ if (histogram.containsKey(key))
+ histogram.put(key, histogram.get(key) + 1);
+ else
+ histogram.put(key, 1);
+ total++;
+ }
+
+ public float[] quantize(int num_bits) {
+ float[] buckets = new float[1 << num_bits];
+
+ // We make sure that 0.0f always has its own bucket, so the bucket
+ // size is determined excluding the zero values.
+ int size = (total - histogram.get(0.0f)) / (buckets.length - 1);
+ buckets[0] = 0.0f;
+
+ int old_size = -1;
+ while (old_size != size) {
+ int sum = 0;
+ int count = buckets.length - 1;
+ for (float key : histogram.keySet()) {
+ int entry_count = histogram.get(key);
+ if (entry_count < size && key != 0)
+ sum += entry_count;
+ else
+ count--;
+ }
+ old_size = size;
+ size = sum / count;
+ }
+
+ float last_key = Float.MAX_VALUE;
+
+ int index = 1;
+ int count = 0;
+ float sum = 0.0f;
+
+ int value;
+ for (float key : histogram.keySet()) {
+ value = histogram.get(key);
+ // Special bucket termination cases: zero boundary and histogram spikes.
+ if (key == 0 || (last_key < 0 && key > 0) || (value >= size)) {
+ // If the count is not 0, i.e. there were negative values, we should
+ // not bucket them with the positive ones. Close out the bucket now.
+ if (count != 0 && index < buckets.length - 2) {
+ buckets[index++] = (float) sum / count;
+ count = 0;
+ sum = 0;
+ }
+ if (key == 0)
+ continue;
+ }
+ count += value;
+ sum += key * value;
+ // Check if the bucket is full.
+ if (count >= size && index < buckets.length - 2) {
+ buckets[index++] = (float) sum / count;
+ count = 0;
+ sum = 0;
+ }
+ last_key = key;
+ }
+ if (count > 0 && index < buckets.length - 1)
+ buckets[index++] = (float) sum / count;
+
+ float[] shortened = new float[index];
+ for (int i = 0; i < shortened.length; ++i)
+ shortened[i] = buckets[i];
+ return shortened;
+ }
+
+ public boolean isBoolean() {
+ for (float value : histogram.keySet())
+ if (value != 0 && value != 1)
+ return false;
+ return true;
+ }
+
+ public boolean isByte() {
+ for (float value : histogram.keySet())
+ if (Math.ceil(value) != value || value < Byte.MIN_VALUE || value > Byte.MAX_VALUE)
+ return false;
+ return true;
+ }
+
+ public boolean isShort() {
+ for (float value : histogram.keySet())
+ if (Math.ceil(value) != value || value < Short.MIN_VALUE || value > Short.MAX_VALUE)
+ return false;
+ return true;
+ }
+
+ public boolean isChar() {
+ for (float value : histogram.keySet())
+ if (Math.ceil(value) != value || value < Character.MIN_VALUE || value > Character.MAX_VALUE)
+ return false;
+ return true;
+ }
+
+ public boolean isInt() {
+ for (float value : histogram.keySet())
+ if (Math.ceil(value) != value)
+ return false;
+ return true;
+ }
+
+ public boolean is8Bit() {
+ return (histogram.keySet().size() <= 256);
+ }
+
+ public FloatEncoder inferUncompressedType() {
+ if (isBoolean())
+ return PrimitiveFloatEncoder.BOOLEAN;
+ if (isByte())
+ return PrimitiveFloatEncoder.BYTE;
+ if (is8Bit())
+ return new EightBitQuantizer(this.quantize(8));
+ if (isChar())
+ return PrimitiveFloatEncoder.CHAR;
+ if (isShort())
+ return PrimitiveFloatEncoder.SHORT;
+ if (isInt())
+ return PrimitiveFloatEncoder.INT;
+ return PrimitiveFloatEncoder.FLOAT;
+ }
+
+ public FloatEncoder inferType(int bits) {
+ if (isBoolean())
+ return PrimitiveFloatEncoder.BOOLEAN;
+ if (isByte())
+ return PrimitiveFloatEncoder.BYTE;
+ if (bits == 8 || is8Bit())
+ return new EightBitQuantizer(this.quantize(8));
+ // TODO: Could add sub-8-bit encoding here (or larger).
+ if (isChar())
+ return PrimitiveFloatEncoder.CHAR;
+ if (isShort())
+ return PrimitiveFloatEncoder.SHORT;
+ if (isInt())
+ return PrimitiveFloatEncoder.INT;
+ return PrimitiveFloatEncoder.FLOAT;
+ }
+
+ public String toString(String label) {
+ StringBuilder sb = new StringBuilder();
+ for (float val : histogram.keySet())
+ sb.append(label + "\t" + String.format("%.5f", val) + "\t" + histogram.get(val) + "\n");
+ return sb.toString();
+ }
+
+ public static void main(String[] args) throws IOException {
+ LineReader reader = new LineReader(args[0]);
+ ArrayList<Float> s = new ArrayList<Float>();
+
+ System.out.println("Initialized.");
+ while (reader.hasNext())
+ s.add(Float.parseFloat(reader.next().trim()));
+ System.out.println("Data read.");
+ int n = s.size();
+ byte[] c = new byte[n];
+ ByteBuffer b = ByteBuffer.wrap(c);
+ Analyzer q = new Analyzer();
+
+ q.initialize();
+ for (int i = 0; i < n; i++)
+ q.add(s.get(i));
+ EightBitQuantizer eb = new EightBitQuantizer(q.quantize(8));
+ System.out.println("Quantizer learned.");
+
+ for (int i = 0; i < n; i++)
+ eb.write(b, s.get(i));
+ b.rewind();
+ System.out.println("Quantization complete.");
+
+ float avg_error = 0;
+ float error = 0;
+ int count = 0;
+ for (int i = -4; i < n - 4; i++) {
+ float coded = eb.read(b, i);
+ if (s.get(i + 4) != 0) {
+ error = Math.abs(s.get(i + 4) - coded);
+ avg_error += error;
+ count++;
+ }
+ }
+ avg_error /= count;
+ System.out.println("Evaluation complete.");
+
+ System.out.println("Average quanitization error over " + n + " samples is: " + avg_error);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/encoding/EightBitQuantizer.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/encoding/EightBitQuantizer.java b/joshua-core/src/main/java/org/apache/joshua/util/encoding/EightBitQuantizer.java
new file mode 100644
index 0000000..5876d4f
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/encoding/EightBitQuantizer.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util.encoding;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public class EightBitQuantizer implements FloatEncoder {
+
+ private float[] buckets;
+
+ public EightBitQuantizer() {
+ this.buckets = new float[256];
+ }
+
+ public EightBitQuantizer(float[] buckets) {
+ if (buckets.length > 256)
+ throw new RuntimeException("Incompatible number of buckets: " + buckets.length);
+ this.buckets = buckets;
+ }
+
+ @Override
+ public final float read(ByteBuffer stream, int position) {
+ byte index = stream.get(position + EncoderConfiguration.ID_SIZE);
+ return buckets[index + 128];
+ }
+
+ @Override
+ public final void write(ByteBuffer stream, float val) {
+ byte index = -128;
+
+ // We search for the bucket best matching the value. Only zeroes will be
+ // mapped to the zero bucket.
+ if (val != 0 && buckets.length > 1) {
+ int t = 1;
+ int b = buckets.length - 1;
+ while ((b - t) > 1) {
+ int half = (t + b) / 2;
+ if (val >= buckets[half])
+ t = half;
+ if (val <= buckets[half])
+ b = half;
+ }
+ index = (byte) ((Math.abs(buckets[t] - val) > (Math.abs(buckets[b] - val)) ? b : t) - 128);
+ }
+ stream.put(index);
+ }
+
+ @Override
+ public String getKey() {
+ return "8bit";
+ }
+
+ @Override
+ public void writeState(DataOutputStream out) throws IOException {
+ out.writeUTF(getKey());
+ out.writeInt(buckets.length);
+ for (int i = 0; i < buckets.length; i++)
+ out.writeFloat(buckets[i]);
+ }
+
+ @Override
+ public void readState(DataInputStream in) throws IOException {
+ int length = in.readInt();
+ buckets = new float[length];
+ for (int i = 0; i < buckets.length; i++)
+ buckets[i] = in.readFloat();
+ }
+
+ @Override
+ public final int size() {
+ return 1;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/encoding/EncoderConfiguration.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/encoding/EncoderConfiguration.java b/joshua-core/src/main/java/org/apache/joshua/util/encoding/EncoderConfiguration.java
new file mode 100644
index 0000000..28b013f
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/encoding/EncoderConfiguration.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util.encoding;
+
+import java.io.BufferedInputStream;
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.joshua.corpus.Vocabulary;
+
+public class EncoderConfiguration {
+
+ public static int ID_SIZE = 4;
+
+ private IntEncoder idEncoder;
+ private int[] innerToOuter;
+ private FloatEncoder[] encoderById;
+ private FloatEncoder[] encoders;
+
+ private Map<Integer, Integer> outerToInner;
+
+ private boolean labeled;
+
+ private int numDenseFeatures = 0;
+
+ public EncoderConfiguration() {
+ this.outerToInner = new HashMap<Integer, Integer>();
+ }
+
+ public int getNumDenseFeatures() {
+ return numDenseFeatures;
+ }
+
+ public int getNumFeatures() {
+ return encoders.length;
+ }
+
+ public void load(String file_name) throws IOException {
+ File encoding_file = new File(file_name);
+ BufferedInputStream buf_stream = new BufferedInputStream(new FileInputStream(encoding_file));
+ DataInputStream in_stream = new DataInputStream(buf_stream);
+
+ String id_key = in_stream.readUTF();
+ idEncoder = EncoderFactory.getIntEncoder(id_key);
+ idEncoder.readState(in_stream);
+ ID_SIZE = idEncoder.size();
+ labeled = in_stream.readBoolean();
+
+ int num_encoders = in_stream.readInt();
+ encoders = new FloatEncoder[num_encoders];
+ for (int i = 0; i < num_encoders; i++) {
+ String key = in_stream.readUTF();
+ FloatEncoder e = EncoderFactory.getFloatEncoder(key);
+ e.readState(in_stream);
+ encoders[i] = e;
+ }
+ int num_features = in_stream.readInt();
+ encoderById = new FloatEncoder[num_features];
+ innerToOuter = new int[num_features];
+ for (int i = 0; i < num_features; i++) {
+ int outer_id;
+ if (labeled) {
+ String feature_name = in_stream.readUTF();
+ outer_id = Vocabulary.id(feature_name);
+ try {
+ Integer.parseInt(feature_name);
+ numDenseFeatures++;
+ } catch (NumberFormatException e) {}
+ } else {
+ outer_id = in_stream.readInt();
+ }
+ int inner_id = in_stream.readInt();
+ int encoder_index = in_stream.readInt();
+ if (encoder_index >= num_encoders) {
+ throw new RuntimeException("Error deserializing EncoderConfig. " + "Feature "
+ + (labeled ? Vocabulary.word(outer_id) : outer_id) + " referring to encoder "
+ + encoder_index + " when only " + num_encoders + " known.");
+ }
+ encoderById[inner_id] = encoders[encoder_index];
+ innerToOuter[inner_id] = outer_id;
+ }
+ in_stream.close();
+
+ outerToInner.clear();
+ for (int i = 0; i < innerToOuter.length; ++i)
+ outerToInner.put(innerToOuter[i], i);
+ }
+
+ public FloatEncoder encoder(int inner_id) {
+ return encoderById[inner_id];
+ }
+
+ public int readId(ByteBuffer buffer, int pos) {
+ return idEncoder.read(buffer, pos);
+ }
+
+ public int outerId(int inner_id) {
+ return innerToOuter[inner_id];
+ }
+
+ public int innerId(int outer_id) {
+ return outerToInner.get(outer_id);
+ }
+
+ public boolean isLabeled() {
+ return labeled;
+ }
+
+ /**
+ * For now, this just loads a configuration and prints out the number of features.
+ *
+ * @param args an input configuration file
+ */
+ public static void main(String[] args) {
+ String grammar_dir = null;
+ try {
+ grammar_dir = args[0];
+
+ EncoderConfiguration encoding = new EncoderConfiguration();
+ encoding.load(grammar_dir + File.separator + "encoding");
+ int num_features = encoding.getNumFeatures();
+ System.out.println(String.format("num_features = %d", encoding.getNumFeatures()));
+
+ for (int feature_id = 0; feature_id < num_features; feature_id++) {
+ if (Vocabulary.size() == 1) {
+ System.out.println(String.format("feature: %d", feature_id));
+ } else {
+ String name = Vocabulary.word(encoding.outerId(feature_id));
+ System.out.println(String.format("feature: %s", name));
+ }
+ }
+
+ } catch (ArrayIndexOutOfBoundsException e) {
+ throw new RuntimeException("Usage: EncoderConfiguration <packed_directory>");
+ } catch (IOException e) {
+ throw new RuntimeException(String.format("* FATAL: can't find file %s/encoding", grammar_dir));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/encoding/EncoderFactory.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/encoding/EncoderFactory.java b/joshua-core/src/main/java/org/apache/joshua/util/encoding/EncoderFactory.java
new file mode 100644
index 0000000..a1f93d0
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/encoding/EncoderFactory.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util.encoding;
+
+public class EncoderFactory {
+
+ public static FloatEncoder getFloatEncoder(String key) {
+ FloatEncoder encoder = PrimitiveFloatEncoder.get(key.toUpperCase());
+ if (encoder != null) {
+ return encoder;
+ } else if ("8bit".equals(key)) {
+ return new EightBitQuantizer();
+ } else {
+ throw new RuntimeException("Unknown FloatEncoder type: " + key.toUpperCase());
+ }
+ }
+
+ public static IntEncoder getIntEncoder(String key) {
+ IntEncoder encoder = PrimitiveIntEncoder.get(key.toUpperCase());
+ if (encoder != null) {
+ return encoder;
+ } else {
+ throw new RuntimeException("Unknown IntEncoder type: " + key.toUpperCase());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/encoding/FeatureTypeAnalyzer.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/encoding/FeatureTypeAnalyzer.java b/joshua-core/src/main/java/org/apache/joshua/util/encoding/FeatureTypeAnalyzer.java
new file mode 100644
index 0000000..504859f
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/encoding/FeatureTypeAnalyzer.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util.encoding;
+
+import java.io.BufferedOutputStream;
+import java.io.DataOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.util.io.LineReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class FeatureTypeAnalyzer {
+
+ private static final Logger LOG = LoggerFactory.getLogger(FeatureTypeAnalyzer.class);
+
+ private ArrayList<FeatureType> types;
+
+ private Map<Integer, Integer> featureToType;
+
+ private Map<Integer, Integer> featureIdMap;
+
+ // Is the feature setup labeled.
+ private boolean labeled;
+
+ // Is the encoder configuration open for new features (that are not assumed boolean)?
+ private boolean open;
+
+ public FeatureTypeAnalyzer() {
+ this(false);
+ }
+
+ public FeatureTypeAnalyzer(boolean open) {
+ this.open = open;
+ this.types = new ArrayList<FeatureType>();
+ this.featureToType = new HashMap<Integer, Integer>();
+ this.featureIdMap = new HashMap<Integer, Integer>();
+ }
+
+ public void readConfig(String config_filename) throws IOException {
+ LineReader reader = new LineReader(config_filename);
+ while (reader.hasNext()) {
+ // Clean up line, chop comments off and skip if the result is empty.
+ String line = reader.next().trim();
+ if (line.indexOf('#') != -1)
+ line = line.substring(0, line.indexOf('#'));
+ if (line.isEmpty())
+ continue;
+ String[] fields = line.split("[\\s]+");
+
+ if ("encoder".equals(fields[0])) {
+ // Adding an encoder to the mix.
+ if (fields.length < 3) {
+ throw new RuntimeException("Incomplete encoder line in config.");
+ }
+ String encoder_key = fields[1];
+ ArrayList<Integer> feature_ids = new ArrayList<Integer>();
+ for (int i = 2; i < fields.length; i++)
+ feature_ids.add(Vocabulary.id(fields[i]));
+ addFeatures(encoder_key, feature_ids);
+ }
+ }
+ }
+
+ public void addFeatures(String encoder_key, List<Integer> feature_ids) {
+ int index = addType(encoder_key);
+ for (int feature_id : feature_ids)
+ featureToType.put(feature_id, index);
+ }
+
+ private int addType(String encoder_key) {
+ FeatureType ft = new FeatureType(encoder_key);
+ int index = types.indexOf(ft);
+ if (index < 0) {
+ types.add(ft);
+ return types.size() - 1;
+ }
+ return index;
+ }
+
+ private int addType() {
+ types.add(new FeatureType());
+ return types.size() - 1;
+ }
+
+ public void observe(int feature_id, float value) {
+ Integer type_id = featureToType.get(feature_id);
+ if (type_id == null && open) {
+ type_id = addType();
+ featureToType.put(feature_id, type_id);
+ }
+ if (type_id != null)
+ types.get(type_id).observe(value);
+ }
+
+ // Inspects the collected histograms, inferring actual type of feature. Then replaces the
+ // analyzer, if present, with the most compact applicable type.
+ public void inferTypes(boolean labeled) {
+ for (FeatureType ft : types) {
+ ft.inferUncompressedType();
+ }
+ if (LOG.isInfoEnabled()) {
+ for (int id : featureToType.keySet()) {
+ LOG.info("Type inferred: {} is {}", (labeled ? Vocabulary.word(id) : "Feature " + id),
+ types.get(featureToType.get(id)).encoder.getKey());
+ }
+ }
+ }
+
+ public void buildFeatureMap() {
+ int[] known_features = new int[featureToType.keySet().size()];
+ int i = 0;
+ for (int f : featureToType.keySet())
+ known_features[i++] = f;
+ Arrays.sort(known_features);
+
+ featureIdMap.clear();
+ for (i = 0; i < known_features.length; ++i)
+ featureIdMap.put(known_features[i], i);
+ }
+
+ public int getRank(int feature_id) {
+ return featureIdMap.get(feature_id);
+ }
+
+ public IntEncoder getIdEncoder() {
+ int num_features = featureIdMap.size();
+ if (num_features <= Byte.MAX_VALUE)
+ return PrimitiveIntEncoder.BYTE;
+ else if (num_features <= Character.MAX_VALUE)
+ return PrimitiveIntEncoder.CHAR;
+ else
+ return PrimitiveIntEncoder.INT;
+ }
+
+ public void write(String file_name) throws IOException {
+ File out_file = new File(file_name);
+ BufferedOutputStream buf_stream = new BufferedOutputStream(new FileOutputStream(out_file));
+ DataOutputStream out_stream = new DataOutputStream(buf_stream);
+
+ buildFeatureMap();
+
+ getIdEncoder().writeState(out_stream);
+ out_stream.writeBoolean(labeled);
+ out_stream.writeInt(types.size());
+ for (int index = 0; index < types.size(); index++)
+ types.get(index).encoder.writeState(out_stream);
+
+ out_stream.writeInt(featureToType.size());
+ for (int feature_id : featureToType.keySet()) {
+ if (labeled)
+ out_stream.writeUTF(Vocabulary.word(feature_id));
+ else
+ out_stream.writeInt(feature_id);
+ out_stream.writeInt(featureIdMap.get(feature_id));
+ out_stream.writeInt(featureToType.get(feature_id));
+ }
+ out_stream.close();
+ }
+
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ for (int feature_id : featureToType.keySet()) {
+ sb.append(types.get(featureToType.get(feature_id)).analyzer.toString(Vocabulary.word(feature_id)));
+ }
+ return sb.toString();
+ }
+
+ public boolean isLabeled() {
+ return labeled;
+ }
+
+ public void setLabeled(boolean labeled) {
+ this.labeled = labeled;
+ }
+
+ class FeatureType {
+ FloatEncoder encoder;
+ Analyzer analyzer;
+ int bits;
+
+ FeatureType() {
+ encoder = null;
+ analyzer = new Analyzer();
+ bits = -1;
+ }
+
+ FeatureType(String key) {
+ // either throws or returns non-null
+ FloatEncoder e = EncoderFactory.getFloatEncoder(key);
+ encoder = e;
+ analyzer = null;
+ bits = -1;
+ }
+
+ void inferUncompressedType() {
+ if (encoder != null)
+ return;
+ encoder = analyzer.inferUncompressedType();
+ analyzer = null;
+ }
+
+ void inferType() {
+ if (encoder != null)
+ return;
+ encoder = analyzer.inferType(bits);
+ analyzer = null;
+ }
+
+ void observe(float value) {
+ if (analyzer != null)
+ analyzer.add(value);
+ }
+
+ public boolean equals(Object t) {
+ if (t != null && t instanceof FeatureType) {
+ FeatureType that = (FeatureType) t;
+ if (this.encoder != null) {
+ return this.encoder.equals(that.encoder);
+ } else {
+ if (that.encoder != null)
+ return false;
+ if (this.analyzer != null)
+ return this.analyzer.equals(that.analyzer);
+ }
+ }
+ return false;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/encoding/FloatEncoder.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/encoding/FloatEncoder.java b/joshua-core/src/main/java/org/apache/joshua/util/encoding/FloatEncoder.java
new file mode 100644
index 0000000..5121ea2
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/encoding/FloatEncoder.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util.encoding;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public interface FloatEncoder {
+
+ public float read(ByteBuffer stream, int position);
+
+ public void write(ByteBuffer stream, float value);
+
+ public String getKey();
+
+ public void writeState(DataOutputStream out) throws IOException;
+
+ public void readState(DataInputStream in) throws IOException;
+
+ public int size();
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/encoding/IntEncoder.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/encoding/IntEncoder.java b/joshua-core/src/main/java/org/apache/joshua/util/encoding/IntEncoder.java
new file mode 100644
index 0000000..a8917f7
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/encoding/IntEncoder.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util.encoding;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public interface IntEncoder {
+
+ public int read(ByteBuffer stream, int position);
+
+ public void write(ByteBuffer stream, int value);
+
+ public String getKey();
+
+ public void writeState(DataOutputStream out) throws IOException;
+
+ public void readState(DataInputStream in) throws IOException;
+
+ public int size();
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/encoding/PrimitiveFloatEncoder.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/encoding/PrimitiveFloatEncoder.java b/joshua-core/src/main/java/org/apache/joshua/util/encoding/PrimitiveFloatEncoder.java
new file mode 100644
index 0000000..d5015f2
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/encoding/PrimitiveFloatEncoder.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util.encoding;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public enum PrimitiveFloatEncoder implements FloatEncoder {
+
+ BYTE("byte", 1) {
+ public final float read(ByteBuffer stream, int position) {
+ return (float) stream.get(position + EncoderConfiguration.ID_SIZE);
+ }
+
+ public final void write(ByteBuffer stream, float value) {
+ stream.put((byte) value);
+ }
+ },
+
+ BOOLEAN("boolean", 0) {
+ public final float read(ByteBuffer stream, int position) {
+ return 1.0f;
+ }
+
+ public final void write(ByteBuffer stream, float value) {
+ }
+ },
+
+ CHAR("char", 2) {
+ public final float read(ByteBuffer stream, int position) {
+ return (float) stream.getChar(position + EncoderConfiguration.ID_SIZE);
+ }
+
+ public final void write(ByteBuffer stream, float value) {
+ stream.putChar((char) value);
+ }
+ },
+
+ FLOAT("float", 4) {
+ public final float read(ByteBuffer stream, int position) {
+ return stream.getFloat(position + EncoderConfiguration.ID_SIZE);
+ }
+
+ public final void write(ByteBuffer stream, float value) {
+ stream.putFloat(value);
+ }
+ },
+
+ INT("int", 4) {
+ public final float read(ByteBuffer stream, int position) {
+ return (float) stream.getInt(position + EncoderConfiguration.ID_SIZE);
+ }
+
+ public final void write(ByteBuffer stream, float value) {
+ stream.putInt((int) value);
+ }
+ },
+
+ SHORT("short", 2) {
+ public final float read(ByteBuffer stream, int position) {
+ return (float) stream.getShort(position + EncoderConfiguration.ID_SIZE);
+ }
+
+ public final void write(ByteBuffer stream, float value) {
+ stream.putShort((short) value);
+ }
+ };
+
+ private final String key;
+ private final int size;
+
+ private PrimitiveFloatEncoder(String k, int s) {
+ key = k;
+ size = s;
+ }
+
+ @Override
+ public String getKey() {
+ return key;
+ }
+
+ @Override
+ public int size() {
+ return size;
+ }
+
+ public static PrimitiveFloatEncoder get(String k) {
+ PrimitiveFloatEncoder encoder;
+ try {
+ encoder = valueOf(k);
+ } catch (IllegalArgumentException e) {
+ return null;
+ }
+ return encoder;
+ }
+
+ @Override
+ public void readState(DataInputStream in) throws IOException {
+ }
+
+ @Override
+ public void writeState(DataOutputStream out) throws IOException {
+ out.writeUTF(getKey());
+ }
+
+ @Override
+ public abstract float read(ByteBuffer stream, int position);
+
+ @Override
+ public abstract void write(ByteBuffer stream, float value);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/encoding/PrimitiveIntEncoder.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/encoding/PrimitiveIntEncoder.java b/joshua-core/src/main/java/org/apache/joshua/util/encoding/PrimitiveIntEncoder.java
new file mode 100644
index 0000000..42f6053
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/encoding/PrimitiveIntEncoder.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util.encoding;
+
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+public enum PrimitiveIntEncoder implements IntEncoder {
+
+ // TODO: the inconsistency with FloatEncoders is dangerous.
+ BYTE("byte", 1) {
+ public final int read(ByteBuffer stream, int position) {
+ return (int) stream.get(position);
+ }
+
+ public final void write(ByteBuffer stream, int value) {
+ stream.put((byte) value);
+ }
+ },
+
+ CHAR("char", 2) {
+ public final int read(ByteBuffer stream, int position) {
+ return (int) stream.getChar(position);
+ }
+
+ public final void write(ByteBuffer stream, int value) {
+ stream.putChar((char) value);
+ }
+ },
+
+ INT("int", 4) {
+ public final int read(ByteBuffer stream, int position) {
+ return (int) stream.getInt(position);
+ }
+
+ public final void write(ByteBuffer stream, int value) {
+ stream.putInt((int) value);
+ }
+ },
+
+ SHORT("short", 2) {
+ public final int read(ByteBuffer stream, int position) {
+ return (int) stream.getShort(position);
+ }
+
+ public final void write(ByteBuffer stream, int value) {
+ stream.putShort((short) value);
+ }
+ };
+
+ private final String key;
+ private final int size;
+
+ private PrimitiveIntEncoder(String k, int s) {
+ key = k;
+ size = s;
+ }
+
+ @Override
+ public String getKey() {
+ return key;
+ }
+
+ @Override
+ public int size() {
+ return size;
+ }
+
+ public static PrimitiveIntEncoder get(String k) {
+ PrimitiveIntEncoder encoder;
+ try {
+ encoder = valueOf(k);
+ } catch (IllegalArgumentException e) {
+ return null;
+ }
+ return encoder;
+ }
+
+ @Override
+ public void readState(DataInputStream in) throws IOException {
+ }
+
+ @Override
+ public void writeState(DataOutputStream out) throws IOException {
+ out.writeUTF(getKey());
+ }
+
+ @Override
+ public abstract int read(ByteBuffer stream, int position);
+
+ @Override
+ public abstract void write(ByteBuffer stream, int value);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/encoding/VariableQuantizer.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/encoding/VariableQuantizer.java b/joshua-core/src/main/java/org/apache/joshua/util/encoding/VariableQuantizer.java
new file mode 100644
index 0000000..afa3f69
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/encoding/VariableQuantizer.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util.encoding;
+
+public class VariableQuantizer {
+
+ private final byte[] bytes;
+ private int byteOffset;
+ private int bitOffset;
+
+ /**
+ * @param bytes bytes from which this will read bits. Bits will be read from the first byte first.
+ * Bits are read within a byte from most-significant to least-significant bit.
+ */
+ public VariableQuantizer(byte[] bytes) {
+ this.bytes = bytes;
+ }
+
+ /**
+ * @return index of next bit in current byte which would be read by the next call to
+ * {@link #readBits(int)}.
+ */
+ public int getBitOffset() {
+ return bitOffset;
+ }
+
+ /**
+ * @return index of next byte in input byte array which would be read by the next call to
+ * {@link #readBits(int)}.
+ */
+ public int getByteOffset() {
+ return byteOffset;
+ }
+
+ /**
+ * @param numBits number of bits to read
+ * @return int representing the bits read. The bits will appear as the least-significant bits of
+ * the int
+ * @throws IllegalArgumentException if numBits isn't in [1,32] or more than is available
+ */
+ public int readBits(int numBits) {
+ if (numBits < 1 || numBits > 32 || numBits > available()) {
+ throw new IllegalArgumentException(String.valueOf(numBits));
+ }
+
+ int result = 0;
+
+ // First, read remainder from current byte
+ if (bitOffset > 0) {
+ int bitsLeft = 8 - bitOffset;
+ int toRead = numBits < bitsLeft ? numBits : bitsLeft;
+ int bitsToNotRead = bitsLeft - toRead;
+ int mask = (0xFF >> (8 - toRead)) << bitsToNotRead;
+ result = (bytes[byteOffset] & mask) >> bitsToNotRead;
+ numBits -= toRead;
+ bitOffset += toRead;
+ if (bitOffset == 8) {
+ bitOffset = 0;
+ byteOffset++;
+ }
+ }
+
+ // Next read whole bytes
+ if (numBits > 0) {
+ while (numBits >= 8) {
+ result = (result << 8) | (bytes[byteOffset] & 0xFF);
+ byteOffset++;
+ numBits -= 8;
+ }
+
+ // Finally read a partial byte
+ if (numBits > 0) {
+ int bitsToNotRead = 8 - numBits;
+ int mask = (0xFF >> bitsToNotRead) << bitsToNotRead;
+ result = (result << numBits) | ((bytes[byteOffset] & mask) >> bitsToNotRead);
+ bitOffset += numBits;
+ }
+ }
+
+ return result;
+ }
+
+ /**
+ * @return number of bits that can be read successfully
+ */
+ public int available() {
+ return 8 * (bytes.length - byteOffset) - bitOffset;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/io/BinaryIn.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/io/BinaryIn.java b/joshua-core/src/main/java/org/apache/joshua/util/io/BinaryIn.java
new file mode 100644
index 0000000..9483e3e
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/io/BinaryIn.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util.io;
+
+import java.io.DataInput;
+import java.io.Externalizable;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectStreamConstants;
+import java.io.RandomAccessFile;
+
+public class BinaryIn<E extends Externalizable> extends RandomAccessFile implements DataInput, ObjectInput {
+
+ private final Class<E> type;
+
+ public BinaryIn(String filename, Class<E> type) throws FileNotFoundException {
+ super(filename, "r");
+ this.type = type;
+ }
+
+ public int available() throws IOException {
+ long pos = getFilePointer();
+ long length = length();
+ long bytesAvailable = length - pos;
+ if (bytesAvailable > Integer.MAX_VALUE) {
+ return Integer.MAX_VALUE;
+ } else {
+ return (int) bytesAvailable;
+ }
+ }
+
+ public E readObject() throws ClassNotFoundException, IOException {
+
+ int b = peek();
+ if (b == ObjectStreamConstants.TC_NULL) {
+ return null;
+ } else {
+ E obj;
+ try {
+ obj = type.newInstance();
+ obj.readExternal(this);
+ return obj;
+ } catch (InstantiationException e) {
+ throw new RuntimeException(e);
+ } catch (IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ public long skip(long n) throws IOException {
+
+ long bytesSkipped = 0;
+
+ while (n > 0) {
+ if (n > Integer.MAX_VALUE) {
+ bytesSkipped += skipBytes(Integer.MAX_VALUE);
+ n -= Integer.MAX_VALUE;
+ } else {
+ bytesSkipped = skipBytes((int) n);
+ n = 0;
+ }
+ }
+
+ return bytesSkipped;
+ }
+
+ private int peek() throws IOException {
+ long pos = getFilePointer();
+ int b = read();
+ seek(pos);
+ return b;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/io/BinaryOut.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/io/BinaryOut.java b/joshua-core/src/main/java/org/apache/joshua/util/io/BinaryOut.java
new file mode 100644
index 0000000..8383053
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/io/BinaryOut.java
@@ -0,0 +1,505 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util.io;
+
+import java.io.Closeable;
+import java.io.DataOutput;
+import java.io.Externalizable;
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.Flushable;
+import java.io.IOException;
+import java.io.ObjectOutput;
+import java.io.ObjectOutputStream;
+import java.io.ObjectStreamConstants;
+import java.io.OutputStream;
+import java.io.UTFDataFormatException;
+
+/**
+ * A BinaryOut writes data to an output stream in raw binary form. Each data type is converted to
+ * byte representation.
+ * <p>
+ * Unlike ObjectOutputStream, no extra Java meta-data is written to the stream.
+ *
+ * @author Lane Schwartz
+ * @see ObjectOutputStream
+ * @see Externalizable
+ */
+public class BinaryOut implements DataOutput, ObjectOutput, Flushable, Closeable {
+
+
+ public final int BITS_PER_BYTE = 8;
+
+ public final int BOOLEAN_SIZE = 1;
+ public final int BYTE_SIZE = 1;
+ public final int CHAR_SIZE = 2;
+ public final int SHORT_SIZE = 2;
+ public final int FLOAT_SIZE = 4;
+ public final int INT_SIZE = 4;
+ public final int DOUBLE_SIZE = 8;
+ public final int LONG_SIZE = 8;
+
+ private final OutputStream out;
+
+ private int bufferPosition;
+ private static final int BUFFER_SIZE = 1024;
+ private final byte[] buffer;
+ private final char[] charBuffer;
+ private final utf8CharRange[] charSizeBuffer;
+ private final boolean writeObjects;
+
+ public BinaryOut(File file) throws FileNotFoundException, IOException {
+ this(new FileOutputStream(file), true);
+ }
+
+ public BinaryOut(String filename) throws FileNotFoundException, IOException {
+ this(new File(filename));
+ }
+
+ public BinaryOut(OutputStream out, boolean writeObjects) throws IOException {
+ this.out = out;
+ this.buffer = new byte[BUFFER_SIZE];
+ this.charBuffer = new char[BUFFER_SIZE];
+ this.charSizeBuffer = new utf8CharRange[BUFFER_SIZE];
+ this.bufferPosition = 0;
+ this.writeObjects = writeObjects;
+ }
+
+ public void close() throws IOException {
+ flush();
+ out.close();
+ }
+
+ /**
+ * Ensures that the buffer has at least enough space available to hold <code>size</code>
+ * additional bytes.
+ * <p>
+ * If necessary, the current contents of the buffer will be written to the underlying output
+ * stream.
+ *
+ * @param size the size of the buffer
+ * @throws IOException if there is an error determining the current size
+ */
+ protected void prepareBuffer(int size) throws IOException {
+ if (bufferPosition > 0 && bufferPosition >= BUFFER_SIZE - size) {
+
+ writeBuffer();
+
+ }
+ }
+
+ protected void writeBuffer() throws IOException {
+ if (bufferPosition > 0) {
+ out.write(buffer, 0, bufferPosition);
+ bufferPosition = 0;
+ }
+ }
+
+ public void flush() throws IOException {
+ writeBuffer();
+ out.flush();
+ }
+
+ public void write(int b) throws IOException {
+ writeBuffer();
+ out.write(b);
+ }
+
+ public void write(byte[] b) throws IOException {
+ writeBuffer();
+ out.write(b);
+ }
+
+ public void write(byte[] b, int off, int len) throws IOException {
+ writeBuffer();
+ out.write(b, off, len);
+ }
+
+
+ public void writeObject(Object obj) throws IOException {
+
+ if (writeObjects) {
+ if (obj == null) {
+
+ write(ObjectStreamConstants.TC_NULL);
+
+ } else if (obj instanceof String) {
+
+ String s = (String) obj;
+ long bytesRequired = utfBytesRequired(s);
+ boolean forceLongHeader = (bytesRequired > Short.MAX_VALUE);
+
+ writeUTF(s, bytesRequired, forceLongHeader);
+
+ } else if (obj instanceof Externalizable) {
+
+ Externalizable e = (Externalizable) obj;
+
+ e.writeExternal(this);
+
+ } else {
+
+ throw new RuntimeException("Object is not Externalizable: " + obj.toString());
+
+ }
+ }
+ }
+
+ public void writeBoolean(boolean v) throws IOException {
+ prepareBuffer(BOOLEAN_SIZE);
+ if (v) {
+ buffer[bufferPosition] = 0x01;
+ } else {
+ buffer[bufferPosition] = 0x00;
+ }
+ bufferPosition += BOOLEAN_SIZE;
+ }
+
+ public void writeByte(int v) throws IOException {
+ prepareBuffer(BYTE_SIZE);
+ buffer[bufferPosition] = (byte) v;
+ bufferPosition += BYTE_SIZE;
+ }
+
+ public void writeBytes(String s) throws IOException {
+ int charsRemaining = s.length();
+
+ while (charsRemaining > 0) {
+
+ int bytesAvailableInBuffer = (BUFFER_SIZE - 1) - bufferPosition;
+ int charsAvailableInBuffer = bytesAvailableInBuffer;
+
+ if (charsAvailableInBuffer > charsRemaining) {
+ charsAvailableInBuffer = charsRemaining;
+ }
+
+ int charStart = 0;
+
+ if (charsAvailableInBuffer > 0) {
+
+ // Copy characters into the character buffer
+ s.getChars(charStart, charStart + charsAvailableInBuffer, charBuffer, 0);
+
+ // Iterate over each character in the character buffer
+ for (int charIndex = 0; charIndex < charsAvailableInBuffer; charIndex++) {
+
+ // Put the low-order byte for the current character into the byte buffer
+ buffer[bufferPosition] = (byte) charBuffer[charIndex];
+
+ bufferPosition += BYTE_SIZE;
+
+ }
+
+ charsRemaining -= charsAvailableInBuffer;
+
+ } else {
+ writeBuffer();
+ }
+ }
+ }
+
+ public void writeChar(int v) throws IOException {
+ prepareBuffer(CHAR_SIZE);
+
+ for (int offset = 0, mask = ((CHAR_SIZE - 1) * BITS_PER_BYTE); offset < CHAR_SIZE && mask >= 0; offset++, mask -=
+ BITS_PER_BYTE) {
+
+ buffer[bufferPosition + offset] = (byte) (v >>> mask);
+
+ }
+
+ bufferPosition += CHAR_SIZE;
+ }
+
+ public void writeChars(String s) throws IOException {
+
+ int charsRemaining = s.length();
+
+ while (charsRemaining > 0) {
+
+ int bytesAvailableInBuffer = (BUFFER_SIZE - 1) - bufferPosition;
+ int charsAvailableInBuffer = bytesAvailableInBuffer / CHAR_SIZE;
+
+ if (charsAvailableInBuffer > charsRemaining) {
+ charsAvailableInBuffer = charsRemaining;
+ }
+
+ int charStart = 0;
+
+ if (charsAvailableInBuffer > 0) {
+
+ // Copy characters into the character buffer
+ s.getChars(charStart, charStart + charsAvailableInBuffer, charBuffer, 0);
+
+ // Iterate over each character in the character buffer
+ for (int charIndex = 0; charIndex < charsAvailableInBuffer; charIndex++) {
+
+ // Put the bytes for the current character into the byte buffer
+ for (int offset = 0, mask = (CHAR_SIZE * BITS_PER_BYTE); offset < CHAR_SIZE && mask >= 0; offset++, mask -=
+ BITS_PER_BYTE) {
+
+ buffer[bufferPosition + offset] = (byte) (charBuffer[charIndex] >>> mask);
+ }
+
+ bufferPosition += CHAR_SIZE;
+
+ }
+
+ charsRemaining -= charsAvailableInBuffer;
+
+ } else {
+ writeBuffer();
+ }
+ }
+
+ }
+
+ public void writeDouble(double v) throws IOException {
+ prepareBuffer(DOUBLE_SIZE);
+
+ long l = Double.doubleToLongBits(v);
+
+ for (int offset = 0, mask = ((DOUBLE_SIZE - 1) * BITS_PER_BYTE); offset < DOUBLE_SIZE
+ && mask >= 0; offset++, mask -= BITS_PER_BYTE) {
+
+ buffer[bufferPosition + offset] = (byte) (l >>> mask);
+
+ }
+
+ bufferPosition += DOUBLE_SIZE;
+ }
+
+ public void writeFloat(float v) throws IOException {
+ prepareBuffer(FLOAT_SIZE);
+
+ int i = Float.floatToIntBits(v);
+
+ for (int offset = 0, mask = ((FLOAT_SIZE - 1) * BITS_PER_BYTE); offset < FLOAT_SIZE
+ && mask >= 0; offset++, mask -= BITS_PER_BYTE) {
+
+ buffer[bufferPosition + offset] = (byte) (i >>> mask);
+
+ }
+
+ bufferPosition += FLOAT_SIZE;
+ }
+
+ public void writeInt(int v) throws IOException {
+ prepareBuffer(INT_SIZE);
+
+ for (int offset = 0, mask = ((INT_SIZE - 1) * BITS_PER_BYTE); offset < INT_SIZE && mask >= 0; offset++, mask -=
+ BITS_PER_BYTE) {
+
+ buffer[bufferPosition + offset] = (byte) (v >>> mask);
+
+ }
+
+ bufferPosition += INT_SIZE;
+ }
+
+ public void writeLong(long v) throws IOException {
+ prepareBuffer(LONG_SIZE);
+
+ for (int offset = 0, mask = ((LONG_SIZE - 1) * BITS_PER_BYTE); offset < LONG_SIZE && mask >= 0; offset++, mask -=
+ LONG_SIZE) {
+
+ buffer[bufferPosition + offset] = (byte) (v >>> mask);
+
+ }
+
+ bufferPosition += LONG_SIZE;
+ }
+
+ public void writeShort(int v) throws IOException {
+ prepareBuffer(SHORT_SIZE);
+
+ for (int offset = 0, mask = ((SHORT_SIZE - 1) * BITS_PER_BYTE); offset < SHORT_SIZE
+ && mask >= 0; offset++, mask -= BITS_PER_BYTE) {
+
+ buffer[bufferPosition + offset] = (byte) (v >>> mask);
+
+ }
+
+ bufferPosition += SHORT_SIZE;
+ }
+
+ private long utfBytesRequired(String str) {
+
+ long bytesRequired = 0;
+
+ // Calculate the number of bytes required
+ for (int charStart = 0, charsRemaining = str.length(); charsRemaining > 0;) {
+
+ int charsToCopy = ((charsRemaining < charBuffer.length) ? charsRemaining : charBuffer.length);
+
+ int charEnd = charStart + charsToCopy;
+
+
+ // Copy characters into the character buffer
+ str.getChars(charStart, charEnd, charBuffer, 0);
+
+ // Iterate over each character in the character buffer
+ for (int charIndex = 0; charIndex < charsToCopy; charIndex++) {
+
+ char c = charBuffer[charIndex];
+
+ if (c >= '\u0001' && c <= '\u007f') {
+ charSizeBuffer[charIndex] = utf8CharRange.ONE_BYTE;
+ bytesRequired += 1;
+ // } else if ((c>='\u0080' && c<='\u07ff') || c=='\u0000') {
+ } else if (c < '\u0800') {
+ charSizeBuffer[charIndex] = utf8CharRange.TWO_BYTES;
+ bytesRequired += 2;
+ } else {
+ charSizeBuffer[charIndex] = utf8CharRange.THREE_BYTES;
+ bytesRequired += 3;
+ }
+
+ }
+
+ charStart = charEnd;
+ charsRemaining -= charsToCopy;
+
+ }
+
+ return bytesRequired;
+ }
+
+ public void writeUTF(String str) throws IOException {
+
+ // Calculate the number of bytes required to encode the string
+ long bytesRequired = utfBytesRequired(str);
+
+ writeUTF(str, bytesRequired, false);
+ }
+
+
+
+ private void writeUTF(String str, long bytesRequired, boolean forceLongHeader) throws IOException {
+
+ if (forceLongHeader) {
+ writeLong(bytesRequired);
+ } else {
+ // Attempt to write the number of bytes required to encode this string.
+ //
+ // Because the size of the string is encoded as a short,
+ // only strings that require no more than Short.MAX_VALUE bytes can be encoded.
+ if (bytesRequired > Short.MAX_VALUE) {
+ throw new UTFDataFormatException(
+ "Unable to successfully encode strings that require more than " + Short.MAX_VALUE
+ + " bytes. Encoding the provided string would require " + bytesRequired + " bytes.");
+ } else {
+ writeShort((short) bytesRequired);
+ }
+ }
+
+ int numChars = str.length();
+ int charsRemaining = numChars;
+
+
+ int charStart = 0;
+ int charEnd = numChars;
+
+ while (charsRemaining > 0) {
+
+ // Get the number of empty bytes available in the buffer
+ int bytesAvailableInBuffer = (BUFFER_SIZE - 1) - bufferPosition;
+
+ // Calculate the number of characters that
+ // can be encoded in the remaining buffer space.
+ int bytesToUse = 0;
+ for (int charIndex = charStart; charIndex < numChars; charIndex++) {
+ int bytesNeeded;
+ switch (charSizeBuffer[charIndex]) {
+ case ONE_BYTE:
+ bytesNeeded = 1;
+ break;
+ case TWO_BYTES:
+ bytesNeeded = 2;
+ break;
+ case THREE_BYTES:
+ default:
+ bytesNeeded = 3;
+ break;
+ }
+
+ if (bytesToUse + bytesNeeded > bytesAvailableInBuffer) {
+ charEnd = charIndex;
+ break;
+ } else {
+ bytesToUse += bytesNeeded;
+ }
+ }
+
+
+ // Write character data to the byte buffer
+ int charsAvailableInBuffer = charEnd - charStart;
+ int charsToCopy = charEnd - charStart;
+
+ if (charsToCopy > 0) {
+
+ // Copy characters into the character buffer
+ str.getChars(charStart, charEnd, charBuffer, 0);
+
+ // Iterate over each character in the character buffer
+ for (int charIndex = 0; charIndex < charsAvailableInBuffer; charIndex++) {
+
+ char c = charBuffer[charIndex];
+
+ switch (charSizeBuffer[charIndex]) {
+
+ case ONE_BYTE: {
+ buffer[bufferPosition++] = (byte) c;
+ break;
+ }
+
+ case TWO_BYTES: {
+ buffer[bufferPosition++] = (byte) (0xc0 | (0x1f & (c >> 6)));
+ buffer[bufferPosition++] = (byte) (0x80 | (0x3f & c));
+ break;
+ }
+
+ case THREE_BYTES: {
+ buffer[bufferPosition++] = (byte) (0xe0 | (0x0f & (c >> 12)));
+ buffer[bufferPosition++] = (byte) (0x80 | (0x3f & (c >> 6)));
+ buffer[bufferPosition++] = (byte) (0x80 | (0x3f & c));
+ break;
+ }
+ }
+
+ }
+
+ charsRemaining -= charsToCopy;
+ charStart = charEnd;
+ charEnd = numChars;
+
+ } else {
+ writeBuffer();
+ }
+
+ }
+
+ }
+
+ private static enum utf8CharRange {
+ ONE_BYTE, TWO_BYTES, THREE_BYTES
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/io/IndexedReader.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/io/IndexedReader.java b/joshua-core/src/main/java/org/apache/joshua/util/io/IndexedReader.java
new file mode 100644
index 0000000..f357e55
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/io/IndexedReader.java
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util.io;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+
+/**
+ * Wraps a reader with "line" index information.
+ *
+ * @author wren ng thornton wren@users.sourceforge.net
+ * @version $LastChangedDate: 2009-03-26 15:06:57 -0400 (Thu, 26 Mar 2009) $
+ */
+public class IndexedReader<E> implements Reader<E> {
+
+ /** A name for the type of elements the reader produces. */
+ private final String elementName;
+
+ /** The number of elements the reader has delivered so far. */
+ private int lineNumber;
+
+ /** The underlying reader. */
+ private final Reader<E> reader;
+
+ public IndexedReader(String elementName, Reader<E> reader) {
+ this.elementName = elementName;
+ this.lineNumber = 0;
+ this.reader = reader;
+ }
+
+ /**
+ * Return the number of elements delivered so far.
+ * @return integer representing the number of elements delivered so far
+ */
+ public int index() {
+ return this.lineNumber;
+ }
+
+
+ /**
+ * Wrap an IOException's message with the index when it occured.
+ * @param oldError the old {@link java.io.IOException} we wish to wrap
+ * @return the new wrapped {@link java.io.IOException}
+ */
+ public IOException wrapIOException(IOException oldError) {
+ IOException newError =
+ new IOException("At " + this.elementName + " " + this.lineNumber + ": "
+ + oldError.getMessage());
+ newError.initCause(oldError);
+ return newError;
+ }
+
+ // ===============================================================
+ // Reader
+ // ===============================================================
+
+ /**
+ * Delegated to the underlying reader.
+ * @return true if the reader is ready
+ * @throws IOException if there is an error determining readiness
+ */
+ @Override
+ public boolean ready() throws IOException {
+ try {
+ return this.reader.ready();
+ } catch (IOException oldError) {
+ throw wrapIOException(oldError);
+ }
+ }
+
+
+ /**
+ * Delegated to the underlying reader. Note that we do not have a <code>finalize()</code> method;
+ * however, when we fall out of scope, the underlying reader will too, so its finalizer may be
+ * called. For correctness, be sure to manually close all readers.
+ */
+ public void close() throws IOException {
+ try {
+ this.reader.close();
+ } catch (IOException oldError) {
+ throw wrapIOException(oldError);
+ }
+ }
+
+
+ /** Delegated to the underlying reader. */
+ public E readLine() throws IOException {
+ E line;
+ try {
+ line = this.reader.readLine();
+ } catch (IOException oldError) {
+ throw wrapIOException(oldError);
+ }
+ ++this.lineNumber;
+ return line;
+ }
+
+
+ // ===============================================================
+ // Iterable -- because sometimes Java can be very stupid
+ // ===============================================================
+
+ /** Return self as an iterator. */
+ public Iterator<E> iterator() {
+ return this;
+ }
+
+
+ // ===============================================================
+ // Iterator
+ // ===============================================================
+
+ /** Delegated to the underlying reader. */
+ public boolean hasNext() {
+ return this.reader.hasNext();
+ }
+
+
+ /** Delegated to the underlying reader. */
+ public E next() throws NoSuchElementException {
+ E line = this.reader.next();
+ // Let exceptions out, we'll wrap any errors a closing time.
+
+ ++this.lineNumber;
+ return line;
+ }
+
+
+ /**
+ * If the underlying reader supports removal, then so do we. Note that the {@link #index()} method
+ * returns the number of elements delivered to the client, so removing an element from the
+ * underlying collection does not affect that number.
+ */
+ public void remove() throws UnsupportedOperationException {
+ this.reader.remove();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/io/LineReader.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/io/LineReader.java b/joshua-core/src/main/java/org/apache/joshua/util/io/LineReader.java
new file mode 100644
index 0000000..5122994
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/io/LineReader.java
@@ -0,0 +1,368 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util.io;
+
+import java.io.BufferedReader;
+import java.io.FileDescriptor;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.File;
+import java.nio.charset.Charset;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import java.util.zip.GZIPInputStream;
+
+import org.apache.joshua.decoder.Decoder;
+
+/**
+ * This class provides an Iterator interface to a BufferedReader. This covers the most common
+ * use-cases for reading from files without ugly code to check whether we got a line or not.
+ *
+ * @author wren ng thornton wren@users.sourceforge.net
+ * @author Matt Post post@cs.jhu.edu
+ */
+public class LineReader implements Reader<String> {
+
+ /*
+ * Note: charset name is case-agnostic "UTF-8" is the canonical name "UTF8", "unicode-1-1-utf-8"
+ * are aliases Java doesn't distinguish utf8 vs UTF-8 like Perl does
+ */
+ private static final Charset FILE_ENCODING = Charset.forName("UTF-8");
+
+ /*
+ * The reader and its underlying input stream. We need to keep a hold of the underlying
+ * input stream so that we can query how many raw bytes it's read (for a generic progress
+ * meter that works across GZIP'ed and plain text files).
+ */
+ private BufferedReader reader;
+ private ProgressInputStream rawStream;
+
+ private String buffer;
+ private IOException error;
+
+ private int lineno = 0;
+
+ private boolean display_progress = false;
+
+ private int progress = 0;
+
+ // ===============================================================
+ // Constructors and destructors
+ // ===============================================================
+
+ /**
+ * Opens a file for iterating line by line. The special "-" filename can be used to specify
+ * STDIN. GZIP'd files are tested for automatically.
+ *
+ * @param filename the file to be opened ("-" for STDIN)
+ * @throws IOException if there is an error reading the input file
+ */
+ public LineReader(String filename) throws IOException {
+
+ display_progress = (Decoder.VERBOSE >= 1);
+
+ progress = 0;
+
+ InputStream stream = null;
+ long totalBytes = -1;
+ if (filename.equals("-")) {
+ rawStream = null;
+ stream = new FileInputStream(FileDescriptor.in);
+ } else {
+ totalBytes = new File(filename).length();
+ rawStream = new ProgressInputStream(new FileInputStream(filename), totalBytes);
+
+ try {
+ stream = new GZIPInputStream(rawStream);
+ } catch (Exception e) {
+ // GZIP ate a byte, so reset
+ rawStream.close();
+ stream = rawStream = new ProgressInputStream(new FileInputStream(filename), totalBytes);
+ }
+ }
+
+ this.reader = new BufferedReader(new InputStreamReader(stream, FILE_ENCODING));
+ }
+
+ public LineReader(String filename, boolean show_progress) throws IOException {
+ this(filename);
+ display_progress = (Decoder.VERBOSE >= 1 && show_progress);
+ }
+
+
+ /**
+ * Wraps an InputStream for iterating line by line. Stream encoding is assumed to be UTF-8.
+ * @param in an {@link java.io.InputStream} to wrap and iterate over line by line
+ */
+ public LineReader(InputStream in) {
+ this.reader = new BufferedReader(new InputStreamReader(in, FILE_ENCODING));
+ display_progress = false;
+ }
+
+ /**
+ * Chain to the underlying {@link ProgressInputStream}.
+ *
+ * @return an integer from 0..100, indicating how much of the file has been read.
+ */
+ public int progress() {
+ return rawStream == null ? 0 : rawStream.progress();
+ }
+
+ /**
+ * This method will close the file handle, and will raise any exceptions that occured during
+ * iteration. The method is idempotent, and all calls after the first are no-ops (unless the
+ * thread was interrupted or killed). For correctness, you <b>must</b> call this method before the
+ * object falls out of scope.
+ * @throws IOException if there is an error closing the file handler
+ */
+ public void close() throws IOException {
+
+ this.buffer = null; // Just in case it's a large string
+
+ if (null != this.reader) {
+ try {
+ // We assume the wrappers will percolate this down.
+ this.reader.close();
+
+ } catch (IOException e) {
+ // We need to trash our cached error for idempotence.
+ // Presumably the closing error is the more important
+ // one to throw.
+ this.error = null;
+ throw e;
+
+ } finally {
+ this.reader = null;
+ }
+ }
+
+ if (null != this.error) {
+ IOException e = this.error;
+ this.error = null;
+ throw e;
+ }
+ }
+
+
+ /**
+ * We attempt to avoid leaking file descriptors if you fail to call close before the object falls
+ * out of scope. However, the language spec makes <b>no guarantees</b> about timeliness of garbage
+ * collection. It is a bug to rely on this method to release the resources. Also, the garbage
+ * collector will discard any exceptions that have queued up, without notifying the application in
+ * any way.
+ *
+ * Having a finalizer means the JVM can't do "fast allocation" of LineReader objects (or
+ * subclasses). This isn't too important due to disk latency, but may be worth noting.
+ *
+ * @see <a
+ * href="http://java2go.blogspot.com/2007/09/javaone-2007-performance-tips-2-finish.html">Performance
+ * Tips</a>
+ * @see <a
+ * href="http://www.javaworld.com/javaworld/jw-06-1998/jw-06-techniques.html?page=1">Techniques</a>
+ */
+ protected void finalize() throws Throwable {
+ try {
+ this.close();
+ } catch (IOException e) {
+ // Do nothing. The GC will discard the exception
+ // anyways, but it may cause us to linger on the heap.
+ } finally {
+ super.finalize();
+ }
+ }
+
+
+
+ // ===============================================================
+ // Reader
+ // ===============================================================
+
+ // Copied from interface documentation.
+ /** Determine if the reader is ready to read a line. */
+ public boolean ready() throws IOException {
+ return this.reader.ready();
+ }
+
+
+ /**
+ * This method is like next() except that it throws the IOException directly. If there are no
+ * lines to be read then null is returned.
+ */
+ public String readLine() throws IOException {
+ if (this.hasNext()) {
+ String line = this.buffer;
+ this.buffer = null;
+ return line;
+
+ } else {
+ if (null != this.error) {
+ IOException e = this.error;
+ this.error = null;
+ throw e;
+ }
+ return null;
+ }
+ }
+
+
+ // ===============================================================
+ // Iterable -- because sometimes Java can be very stupid
+ // ===============================================================
+
+ /** Return self as an iterator. */
+ public Iterator<String> iterator() {
+ return this;
+ }
+
+
+ // ===============================================================
+ // Iterator
+ // ===============================================================
+
+ // Copied from interface documentation.
+ /**
+ * Returns <code>true</code> if the iteration has more elements. (In other words, returns
+ * <code>true</code> if <code>next</code> would return an element rather than throwing an
+ * exception.)
+ */
+ public boolean hasNext() {
+ if (null != this.buffer) {
+ return true;
+
+ } else if (null != this.error) {
+ return false;
+
+ } else {
+ // We're not allowed to throw IOException from within Iterator
+ try {
+ this.buffer = this.reader.readLine();
+ } catch (IOException e) {
+ this.buffer = null;
+ this.error = e;
+ return false;
+ }
+ return (null != this.buffer);
+ }
+ }
+
+
+ /**
+ * Return the next line of the file. If an error is encountered, NoSuchElementException is thrown.
+ * The actual IOException encountered will be thrown later, when the LineReader is closed. Also if
+ * there is no line to be read then NoSuchElementException is thrown.
+ */
+ public String next() throws NoSuchElementException {
+ if (this.hasNext()) {
+ if (display_progress) {
+ int newProgress = (reader != null) ? progress() : 100;
+// System.err.println(String.format("OLD %d NEW %d", progress, newProgress));
+
+ if (newProgress > progress) {
+ for (int i = progress + 1; i <= newProgress; i++)
+ if (i == 97) {
+ System.err.print("1");
+ } else if (i == 98) {
+ System.err.print("0");
+ } else if (i == 99) {
+ System.err.print("0");
+ } else if (i == 100) {
+ System.err.println("%");
+ } else if (i % 10 == 0) {
+ System.err.print(String.format("%d", i));
+ System.err.flush();
+ } else if ((i - 1) % 10 == 0)
+ ; // skip at 11 since 10, 20, etc take two digits
+ else {
+ System.err.print(".");
+ System.err.flush();
+ }
+ progress = newProgress;
+ }
+ }
+
+ String line = this.buffer;
+ this.lineno++;
+ this.buffer = null;
+ return line;
+ } else {
+ throw new NoSuchElementException();
+ }
+ }
+
+ /* Get the line number of the last line that was returned */
+ public int lineno() {
+ return this.lineno;
+ }
+
+ /** Unsupported. */
+ public void remove() throws UnsupportedOperationException {
+ throw new UnsupportedOperationException();
+ }
+
+
+ /**
+ * Iterates over all lines, ignoring their contents, and returns the count of lines. If some lines
+ * have already been read, this will return the count of remaining lines. Because no lines will
+ * remain after calling this method, we implicitly call close.
+ *
+ * @return the number of lines read
+ * @throws IOException if there is an error reading lines
+ */
+ public int countLines() throws IOException {
+ int lines = 0;
+
+ while (this.hasNext()) {
+ this.next();
+ lines++;
+ }
+ this.close();
+
+ return lines;
+ }
+
+ /**
+ * Example usage code.
+ * @param args an input file
+ */
+ public static void main(String[] args) {
+ if (1 != args.length) {
+ System.out.println("Usage: java LineReader filename");
+ System.exit(1);
+ }
+
+ try {
+
+ LineReader in = new LineReader(args[0]);
+ try {
+ for (String line : in) {
+
+ System.out.println(line);
+
+ }
+ } finally {
+ in.close();
+ }
+
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/io/NullReader.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/io/NullReader.java b/joshua-core/src/main/java/org/apache/joshua/util/io/NullReader.java
new file mode 100644
index 0000000..f833f00
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/io/NullReader.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util.io;
+
+import java.io.IOException;
+
+import org.apache.joshua.util.NullIterator;
+
+
+/**
+ * This class provides a null-object Reader. This is primarily useful for when you may or may not
+ * have a {@link Reader}, and you don't want to check for null all the time. All operations are
+ * no-ops.
+ *
+ * @author wren ng thornton wren@users.sourceforge.net
+ * @version $LastChangedDate: 2009-03-26 15:06:57 -0400 (Thu, 26 Mar 2009) $
+ */
+public class NullReader<E> extends NullIterator<E> implements Reader<E> {
+
+ // ===============================================================
+ // Constructors and destructors
+ // ===============================================================
+
+ // TODO: use static factory method and singleton?
+ public NullReader() {}
+
+ /** A no-op. */
+ public void close() throws IOException {}
+
+
+ // ===============================================================
+ // Reader
+ // ===============================================================
+
+ /**
+ * Always returns true. Is this correct? What are the semantics of ready()? We're always capable
+ * of delivering nothing, but we're never capable of delivering anything...
+ */
+ public boolean ready() {
+ return true;
+ }
+
+ /** Always returns null. */
+ public E readLine() throws IOException {
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/io/ProgressInputStream.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/io/ProgressInputStream.java b/joshua-core/src/main/java/org/apache/joshua/util/io/ProgressInputStream.java
new file mode 100644
index 0000000..075c0b3
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/io/ProgressInputStream.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util.io;
+
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+/**
+ * Generic progress meter for reading files (compressed or not). Pass it the raw input file stream
+ * and it will keep track for you.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ */
+public class ProgressInputStream extends FilterInputStream {
+
+ private long totalBytes = -1;
+ private long bytesRead = 0;
+
+ protected ProgressInputStream(InputStream in, long totalBytes) {
+ super(in);
+
+ this.totalBytes = totalBytes;
+ }
+
+ @Override
+ public int read() throws IOException {
+ int value = super.read();
+ bytesRead += 1;
+ return value;
+ }
+
+ @Override
+ public int read(byte[] b) throws IOException {
+ int value = super.read(b);
+ bytesRead += value;
+ return value;
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException {
+ int value = super.read(b, off, len);
+ bytesRead += value;
+ return value;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ super.reset();
+ bytesRead = 0;
+ }
+
+ @Override
+ public long skip(long bytesRead) throws IOException {
+ long skip = super.skip(bytesRead);
+ bytesRead += skip;
+ return skip;
+ }
+
+ /**
+ * @return progress through the file, as an integer (0..100).
+ */
+ public int progress() {
+ return (int)(100.0 * (float)bytesRead / (float)totalBytes);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/io/Reader.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/io/Reader.java b/joshua-core/src/main/java/org/apache/joshua/util/io/Reader.java
new file mode 100644
index 0000000..cab6d74
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/io/Reader.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util.io;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+/**
+ * Common interface for Reader type objects.
+ *
+ * @author wren ng thornton wren@users.sourceforge.net
+ * @version $LastChangedDate: 2009-03-26 15:06:57 -0400 (Thu, 26 Mar 2009) $
+ */
+public interface Reader<E> extends Iterable<E>, Iterator<E> {
+
+ /**
+ * Close the reader, freeing all resources.
+ * @throws IOException if there is an error closing the reader instance
+ */
+ void close() throws IOException;
+
+ /**
+ * Determine if the reader is ready to read a line.
+ * @return true if it is ready
+ * @throws IOException if there is an error whilst determining if the reader if ready
+ */
+ boolean ready() throws IOException;
+
+ /**
+ * Read a "line" and return an object representing it.
+ * @return an object representing a single line
+ * @throws IOException if there is an error reading lines
+ */
+ E readLine() throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/io/package-info.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/io/package-info.java b/joshua-core/src/main/java/org/apache/joshua/util/io/package-info.java
new file mode 100644
index 0000000..d7ea475
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/io/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * Provides common utility classes for IO.
+ */
+package org.apache.joshua.util.io;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/package-info.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/package-info.java b/joshua-core/src/main/java/org/apache/joshua/util/package-info.java
new file mode 100644
index 0000000..2dedb37
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * Provides common utility classes.
+ */
+package org.apache.joshua.util;
[42/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/Cell.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/Cell.java b/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/Cell.java
new file mode 100644
index 0000000..10b9200
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/Cell.java
@@ -0,0 +1,294 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.chart_parser;
+
+import static com.google.common.base.Preconditions.checkNotNull;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.Map.Entry;
+
+import org.apache.joshua.decoder.ff.FeatureFunction;
+import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.hypergraph.HGNode;
+import org.apache.joshua.decoder.hypergraph.HyperEdge;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * this class implement functions: (1) combine small itesm into larger ones using rules, and create
+ * items and hyper-edges to construct a hyper-graph, (2) evaluate model score for items, (3)
+ * cube-pruning Note: Bin creates Items, but not all Items will be used in the hyper-graph
+ *
+ * @author Matt Post <po...@cs.jhu.edu>
+ * @author Zhifei Li, <zh...@gmail.com>
+ */
+class Cell {
+
+ // ===============================================================
+ // Static fields
+ // ===============================================================
+ private static final Logger LOG = LoggerFactory.getLogger(Cell.class);
+
+
+ // The chart this cell belongs to
+ private Chart chart = null;
+
+ // The top-level (goal) symbol
+ private int goalSymbol;
+
+ // to maintain uniqueness of nodes
+ private HashMap<HGNode.Signature, HGNode> nodesSigTbl = new LinkedHashMap<HGNode.Signature, HGNode>();
+
+ // signature by lhs
+ private Map<Integer, SuperNode> superNodesTbl = new HashMap<Integer, SuperNode>();
+
+ /**
+ * sort values in nodesSigTbl, we need this list when necessary
+ */
+ private List<HGNode> sortedNodes = null;
+
+
+ // ===============================================================
+ // Constructor
+ // ===============================================================
+
+ public Cell(Chart chart, int goalSymID) {
+ this.chart = chart;
+ this.goalSymbol = goalSymID;
+ }
+
+ public Cell(Chart chart, int goal_sym_id, int constraint_symbol_id) {
+ this(chart, goal_sym_id);
+ }
+
+ // ===============================================================
+ // Package-protected methods
+ // ===============================================================
+
+ public Set<Integer> getKeySet() {
+ return superNodesTbl.keySet();
+ }
+
+ public SuperNode getSuperNode(int lhs) {
+ return superNodesTbl.get(lhs);
+ }
+
+ /**
+ * This function loops over all items in the top-level bin (covering the input sentence from
+ * <s> ... </s>), looking for items with the goal LHS. For each of these,
+ * add all the items with GOAL_SYM state into the goal bin the goal bin has only one Item, which
+ * itself has many hyperedges only "goal bin" should call this function
+ */
+ // note that the input bin is bin[0][n], not the goal bin
+ boolean transitToGoal(Cell bin, List<FeatureFunction> featureFunctions, int sentenceLength) {
+ this.sortedNodes = new ArrayList<HGNode>();
+ HGNode goalItem = null;
+
+ for (HGNode antNode : bin.getSortedNodes()) {
+ if (antNode.lhs == this.goalSymbol) {
+ float logP = antNode.bestHyperedge.getBestDerivationScore();
+ List<HGNode> antNodes = new ArrayList<HGNode>();
+ antNodes.add(antNode);
+
+ float finalTransitionLogP = ComputeNodeResult.computeFinalCost(featureFunctions, antNodes,
+ 0, sentenceLength, null, this.chart.getSentence());
+
+ List<HGNode> previousItems = new ArrayList<HGNode>();
+ previousItems.add(antNode);
+
+ HyperEdge dt = new HyperEdge(null, logP + finalTransitionLogP, finalTransitionLogP,
+ previousItems, null);
+
+ if (null == goalItem) {
+ goalItem = new HGNode(0, sentenceLength + 1, this.goalSymbol, null, dt, logP
+ + finalTransitionLogP);
+ this.sortedNodes.add(goalItem);
+ } else {
+ goalItem.addHyperedgeInNode(dt);
+ }
+ } // End if item.lhs == this.goalSymID
+ } // End foreach Item in bin.get_sorted_items()
+
+ int itemsInGoalBin = getSortedNodes().size();
+ if (1 != itemsInGoalBin) {
+ LOG.error("the goal_bin does not have exactly one item");
+ return false;
+ }
+
+ return true;
+ }
+
+ /**
+ * a note about pruning: when a hyperedge gets created, it first needs to pass through
+ * shouldPruneEdge filter. Then, if it does not trigger a new node (i.e. will be merged to an old
+ * node), then does not trigger pruningNodes. If it does trigger a new node (either because its
+ * signature is new or because its logP is better than the old node's logP), then it will trigger
+ * pruningNodes, which might causes *other* nodes got pruned as well
+ * */
+
+ /**
+ * Creates a new hyperedge and adds it to the chart, subject to pruning. The logic of this
+ * function is as follows: if the pruner permits the edge to be added, we build the new edge,
+ * which ends in an HGNode. If this is the first time we've built an HGNode for this point in the
+ * graph, it gets added automatically. Otherwise, we add the hyperedge to the existing HGNode,
+ * possibly updating the HGNode's cache of the best incoming hyperedge.
+ *
+ * @return the new hypernode, or null if the cell was pruned.
+ */
+ HGNode addHyperEdgeInCell(ComputeNodeResult result, Rule rule, int i, int j, List<HGNode> ants,
+ SourcePath srcPath, boolean noPrune) {
+
+// System.err.println(String.format("ADD_EDGE(%d-%d): %s", i, j, rule.getRuleString()));
+// if (ants != null) {
+// for (int xi = 0; xi < ants.size(); xi++) {
+// System.err.println(String.format(" -> TAIL %s", ants.get(xi)));
+// }
+// }
+
+ List<DPState> dpStates = result.getDPStates();
+ float pruningEstimate = result.getPruningEstimate();
+ float transitionLogP = result.getTransitionCost();
+ float finalizedTotalLogP = result.getViterbiCost();
+
+ /**
+ * Here, the edge has passed pre-pruning. The edge will be added to the chart in one of three
+ * ways:
+ *
+ * 1. If there is no existing node, a new one gets created and the edge is its only incoming
+ * hyperedge.
+ *
+ * 2. If there is an existing node, the edge will be added to its list of incoming hyperedges,
+ * possibly taking place as the best incoming hyperedge for that node.
+ */
+
+ HyperEdge hyperEdge = new HyperEdge(rule, finalizedTotalLogP, transitionLogP, ants, srcPath);
+ HGNode newNode = new HGNode(i, j, rule.getLHS(), dpStates, hyperEdge, pruningEstimate);
+
+ /**
+ * each node has a list of hyperedges, need to check whether the node is already exist, if
+ * yes, just add the hyperedges, this may change the best logP of the node
+ * */
+ HGNode oldNode = this.nodesSigTbl.get(newNode.signature());
+ if (null != oldNode) { // have an item with same states, combine items
+ this.chart.nMerged++;
+
+ /**
+ * the position of oldItem in this.heapItems may change, basically, we should remove the
+ * oldItem, and re-insert it (linear time), this is too expense)
+ **/
+ if (newNode.getScore() > oldNode.getScore()) { // merge old to new: semiring plus
+
+ newNode.addHyperedgesInNode(oldNode.hyperedges);
+ // This will update the HashMap, so that the oldNode is destroyed.
+ addNewNode(newNode);
+ } else {// merge new to old, does not trigger pruningItems
+ oldNode.addHyperedgesInNode(newNode.hyperedges);
+ }
+
+ } else { // first time item
+ this.chart.nAdded++; // however, this item may not be used in the future due to pruning in
+ // the hyper-graph
+ addNewNode(newNode);
+ }
+
+ return newNode;
+ }
+
+ List<HGNode> getSortedNodes() {
+ ensureSorted();
+ return this.sortedNodes;
+ }
+
+ Map<Integer, SuperNode> getSortedSuperItems() {
+ ensureSorted();
+ return this.superNodesTbl;
+ }
+
+ // ===============================================================
+ // Private Methods
+ // ===============================================================
+
+ /**
+ * two cases this function gets called (1) a new hyperedge leads to a non-existing node signature
+ * (2) a new hyperedge's signature matches an old node's signature, but the best-logp of old node
+ * is worse than the new hyperedge's logP
+ * */
+ private void addNewNode(HGNode node) {
+ this.nodesSigTbl.put(node.signature(), node); // add/replace the item
+ this.sortedNodes = null; // reset the list
+
+// System.err.println(String.format("** NEW NODE %s %d %d", Vocabulary.word(node.lhs), node.i, node.j));
+
+ // since this.sortedItems == null, this is not necessary because we will always call
+ // ensure_sorted to reconstruct the this.tableSuperItems
+ // add a super-items if necessary
+ SuperNode si = this.superNodesTbl.get(node.lhs);
+ if (null == si) {
+ si = new SuperNode(node.lhs);
+ this.superNodesTbl.put(node.lhs, si);
+ }
+ si.nodes.add(node);// TODO what about the dead items?
+ }
+
+ /**
+ * get a sorted list of Nodes in the cell, and also make sure the list of node in any SuperItem is
+ * sorted, this will be called only necessary, which means that the list is not always sorted,
+ * mainly needed for goal_bin and cube-pruning
+ */
+ private void ensureSorted() {
+ if (null == this.sortedNodes) {
+
+ // get sortedNodes.
+ this.sortedNodes = new ArrayList<>(this.nodesSigTbl.size());
+ for (HGNode node : this.nodesSigTbl.values()) {
+ this.sortedNodes.add(node);
+ }
+
+ // sort the node in an decreasing-LogP order
+ this.sortedNodes.sort(HGNode.inverseLogPComparator);
+
+ // TODO: we cannot create new SuperItem here because the DotItem link to them.
+ // Thus, we clear nodes from existing SuperNodes
+ for (SuperNode superNode : this.superNodesTbl.values()) {
+ superNode.nodes.clear();
+ }
+
+ for (HGNode node : this.sortedNodes) {
+ SuperNode superNode = this.superNodesTbl.get(node.lhs);
+ checkNotNull(superNode, "Does not have super Item, have to exist");
+ superNode.nodes.add(node);
+ }
+
+ // Remove SuperNodes who may not contain any nodes anymore due to pruning
+ for (Iterator<Entry<Integer, SuperNode>> it = this.superNodesTbl.entrySet().iterator(); it.hasNext(); ) {
+ Entry<Integer, SuperNode> entry = it.next();
+ if (entry.getValue().nodes.isEmpty()) {
+ it.remove();
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/Chart.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/Chart.java b/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/Chart.java
new file mode 100644
index 0000000..d0cd96b
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/Chart.java
@@ -0,0 +1,746 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.chart_parser;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.PriorityQueue;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.chart_parser.DotChart.DotNode;
+import org.apache.joshua.decoder.ff.FeatureFunction;
+import org.apache.joshua.decoder.ff.SourceDependentFF;
+import org.apache.joshua.decoder.ff.tm.AbstractGrammar;
+import org.apache.joshua.decoder.ff.tm.Grammar;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.ff.tm.RuleCollection;
+import org.apache.joshua.decoder.ff.tm.Trie;
+import org.apache.joshua.decoder.ff.tm.hash_based.MemoryBasedBatchGrammar;
+import org.apache.joshua.decoder.hypergraph.HGNode;
+import org.apache.joshua.decoder.hypergraph.HyperGraph;
+import org.apache.joshua.decoder.segment_file.Sentence;
+import org.apache.joshua.decoder.segment_file.Token;
+import org.apache.joshua.lattice.Arc;
+import org.apache.joshua.lattice.Lattice;
+import org.apache.joshua.lattice.Node;
+import org.apache.joshua.util.ChartSpan;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Chart class this class implements chart-parsing: (1) seeding the chart (2)
+ * cky main loop over bins, (3) identify applicable rules in each bin
+ *
+ * Note: the combination operation will be done in Cell
+ *
+ * Signatures of class: Cell: i, j SuperNode (used for CKY check): i,j, lhs
+ * HGNode ("or" node): i,j, lhs, edge ngrams HyperEdge ("and" node)
+ *
+ * index of sentences: start from zero index of cell: cell (i,j) represent span
+ * of words indexed [i,j-1] where i is in [0,n-1] and j is in [1,n]
+ *
+ * @author Zhifei Li, zhifei.work@gmail.com
+ * @author Matt Post post@cs.jhu.edu
+ */
+
+public class Chart {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Chart.class);
+ private final JoshuaConfiguration config;
+ // ===========================================================
+ // Statistics
+ // ===========================================================
+
+ /**
+ * how many items have been pruned away because its cost is greater than the
+ * cutoff in calling chart.add_deduction_in_chart()
+ */
+ int nMerged = 0;
+ int nAdded = 0;
+ int nDotitemAdded = 0; // note: there is no pruning in dot-item
+
+ public Sentence getSentence() {
+ return this.sentence;
+ }
+
+ // ===============================================================
+ // Private instance fields (maybe could be protected instead)
+ // ===============================================================
+ private ChartSpan<Cell> cells; // note that in some cell, it might be null
+ private int sourceLength;
+ private List<FeatureFunction> featureFunctions;
+ private Grammar[] grammars;
+ private DotChart[] dotcharts; // each grammar should have a dotchart associated with it
+ private Cell goalBin;
+ private int goalSymbolID = -1;
+ private Lattice<Token> inputLattice;
+
+ private Sentence sentence = null;
+// private SyntaxTree parseTree;
+ private StateConstraint stateConstraint;
+
+
+ // ===============================================================
+ // Constructors
+ // ===============================================================
+
+ /*
+ * TODO: Once the Segment interface is adjusted to provide a Lattice<String>
+ * for the sentence() method, we should just accept a Segment instead of the
+ * sentence, segmentID, and constraintSpans parameters. We have the symbol
+ * table already, so we can do the integerization here instead of in
+ * DecoderThread. GrammarFactory.getGrammarForSentence will want the
+ * integerized sentence as well, but then we'll need to adjust that interface
+ * to deal with (non-trivial) lattices too. Of course, we get passed the
+ * grammars too so we could move all of that into here.
+ */
+
+ public Chart(Sentence sentence, List<FeatureFunction> featureFunctions, Grammar[] grammars,
+ String goalSymbol, JoshuaConfiguration config) {
+ this.config = config;
+ this.inputLattice = sentence.getLattice();
+ this.sourceLength = inputLattice.size() - 1;
+ this.featureFunctions = featureFunctions;
+
+ this.sentence = sentence;
+
+ // TODO: OOV handling no longer handles parse tree input (removed after
+ // commit 748eb69714b26dd67cba8e7c25a294347603bede)
+// this.parseTree = null;
+// if (sentence instanceof ParsedSentence)
+// this.parseTree = ((ParsedSentence) sentence).syntaxTree();
+//
+ this.cells = new ChartSpan<Cell>(sourceLength, null);
+
+ this.goalSymbolID = Vocabulary.id(goalSymbol);
+ this.goalBin = new Cell(this, this.goalSymbolID);
+
+ /* Create the grammars, leaving space for the OOV grammar. */
+ this.grammars = new Grammar[grammars.length + 1];
+ for (int i = 0; i < grammars.length; i++)
+ this.grammars[i + 1] = grammars[i];
+
+ MemoryBasedBatchGrammar oovGrammar = new MemoryBasedBatchGrammar("oov", this.config);
+ AbstractGrammar.addOOVRules(oovGrammar, sentence.getLattice(), featureFunctions,
+ this.config.true_oovs_only);
+ this.grammars[0] = oovGrammar;
+
+ // each grammar will have a dot chart
+ this.dotcharts = new DotChart[this.grammars.length];
+ for (int i = 0; i < this.grammars.length; i++)
+ this.dotcharts[i] = new DotChart(this.inputLattice, this.grammars[i], this);
+
+ // Begin to do initialization work
+
+ stateConstraint = null;
+ if (sentence.target() != null)
+ // stateConstraint = new StateConstraint(sentence.target());
+ stateConstraint = new StateConstraint(Vocabulary.START_SYM + " " + sentence.target() + " "
+ + Vocabulary.STOP_SYM);
+
+ /* Find the SourceDependent feature and give it access to the sentence. */
+ for (FeatureFunction ff : this.featureFunctions)
+ if (ff instanceof SourceDependentFF)
+ ((SourceDependentFF) ff).setSource(sentence);
+
+ LOG.debug("Finished seeding chart.");
+ }
+
+ /**
+ * Manually set the goal symbol ID. The constructor expects a String
+ * representing the goal symbol, but there may be time (say, for example, in
+ * the second pass of a synchronous parse) where we want to set the goal
+ * symbol to a particular ID (regardless of String representation).
+ * <p>
+ * This method should be called before expanding the chart, as chart expansion
+ * depends on the goal symbol ID.
+ *
+ * @param i the id of the goal symbol to use
+ */
+ public void setGoalSymbolID(int i) {
+ this.goalSymbolID = i;
+ this.goalBin = new Cell(this, i);
+ return;
+ }
+
+ // ===============================================================
+ // The primary method for filling in the chart
+ // ===============================================================
+
+ /**
+ * Construct the hypergraph with the help from DotChart using cube pruning.
+ * Cube pruning occurs at the span level, with all completed rules from the
+ * dot chart competing against each other; that is, rules with different
+ * source sides *and* rules sharing a source side but with different target
+ * sides are all in competition with each other.
+ *
+ * Terminal rules are added to the chart directly.
+ *
+ * Rules with nonterminals are added to the list of candidates. The candidates
+ * list is seeded with the list of all rules and, for each nonterminal in the
+ * rule, the 1-best tail node for that nonterminal and subspan. If the maximum
+ * arity of a rule is R, then the dimension of the hypercube is R + 1, since
+ * the first dimension is used to record the rule.
+ */
+ private void completeSpan(int i, int j) {
+
+ /* STEP 1: create the heap, and seed it with all of the candidate states */
+ PriorityQueue<CubePruneState> candidates = new PriorityQueue<CubePruneState>();
+
+ /*
+ * Look at all the grammars, seeding the chart with completed rules from the
+ * DotChart
+ */
+ for (int g = 0; g < grammars.length; g++) {
+ if (!grammars[g].hasRuleForSpan(i, j, inputLattice.distance(i, j))
+ || null == dotcharts[g].getDotCell(i, j))
+ continue;
+
+ // for each rule with applicable rules
+ for (DotNode dotNode : dotcharts[g].getDotCell(i, j).getDotNodes()) {
+ RuleCollection ruleCollection = dotNode.getRuleCollection();
+ if (ruleCollection == null)
+ continue;
+
+ List<Rule> rules = ruleCollection.getSortedRules(this.featureFunctions);
+ SourcePath sourcePath = dotNode.getSourcePath();
+
+ if (null == rules || rules.size() == 0)
+ continue;
+
+ if (ruleCollection.getArity() == 0) {
+ /*
+ * The total number of arity-0 items (pre-terminal rules) that we add
+ * is controlled by num_translation_options in the configuration.
+ *
+ * We limit the translation options per DotNode; that is, per LHS.
+ */
+ int numTranslationsAdded = 0;
+
+ /* Terminal productions are added directly to the chart */
+ for (Rule rule : rules) {
+
+ if (config.num_translation_options > 0
+ && numTranslationsAdded >= config.num_translation_options) {
+ break;
+ }
+
+ ComputeNodeResult result = new ComputeNodeResult(this.featureFunctions, rule, null, i,
+ j, sourcePath, this.sentence);
+
+ if (stateConstraint == null || stateConstraint.isLegal(result.getDPStates())) {
+ getCell(i, j).addHyperEdgeInCell(result, rule, i, j, null, sourcePath, true);
+ numTranslationsAdded++;
+ }
+ }
+ } else {
+ /* Productions with rank > 0 are subject to cube pruning */
+
+ Rule bestRule = rules.get(0);
+
+ List<HGNode> currentTailNodes = new ArrayList<HGNode>();
+ List<SuperNode> superNodes = dotNode.getAntSuperNodes();
+ for (SuperNode si : superNodes) {
+ currentTailNodes.add(si.nodes.get(0));
+ }
+
+ /*
+ * `ranks` records the current position in the cube. the 0th index is
+ * the rule, and the remaining indices 1..N correspond to the tail
+ * nodes (= nonterminals in the rule). These tail nodes are
+ * represented by SuperNodes, which group together items with the same
+ * nonterminal but different DP state (e.g., language model state)
+ */
+ int[] ranks = new int[1 + superNodes.size()];
+ Arrays.fill(ranks, 1);
+
+ ComputeNodeResult result = new ComputeNodeResult(featureFunctions, bestRule,
+ currentTailNodes, i, j, sourcePath, sentence);
+ CubePruneState bestState = new CubePruneState(result, ranks, rules, currentTailNodes,
+ dotNode);
+ candidates.add(bestState);
+ }
+ }
+ }
+
+ applyCubePruning(i, j, candidates);
+ }
+
+ /**
+ * Applies cube pruning over a span.
+ *
+ * @param i
+ * @param j
+ * @param stateConstraint
+ * @param candidates
+ */
+ private void applyCubePruning(int i, int j, PriorityQueue<CubePruneState> candidates) {
+
+ // System.err.println(String.format("CUBEPRUNE: %d-%d with %d candidates",
+ // i, j, candidates.size()));
+ // for (CubePruneState cand: candidates) {
+ // System.err.println(String.format(" CAND " + cand));
+ // }
+
+ /*
+ * There are multiple ways to reach each point in the cube, so short-circuit
+ * that.
+ */
+ HashSet<CubePruneState> visitedStates = new HashSet<CubePruneState>();
+
+ int popLimit = config.pop_limit;
+ int popCount = 0;
+ while (candidates.size() > 0 && ((++popCount <= popLimit) || popLimit == 0)) {
+ CubePruneState state = candidates.poll();
+
+ DotNode dotNode = state.getDotNode();
+ List<Rule> rules = state.rules;
+ SourcePath sourcePath = dotNode.getSourcePath();
+ List<SuperNode> superNodes = dotNode.getAntSuperNodes();
+
+ /*
+ * Add the hypothesis to the chart. This can only happen if (a) we're not
+ * doing constrained decoding or (b) we are and the state is legal.
+ */
+ if (stateConstraint == null || stateConstraint.isLegal(state.getDPStates())) {
+ getCell(i, j).addHyperEdgeInCell(state.computeNodeResult, state.getRule(), i, j,
+ state.antNodes, sourcePath, true);
+ }
+
+ /*
+ * Expand the hypothesis by walking down a step along each dimension of
+ * the cube, in turn. k = 0 means we extend the rule being used; k > 0
+ * expands the corresponding tail node.
+ */
+
+ for (int k = 0; k < state.ranks.length; k++) {
+
+ /* Copy the current ranks, then extend the one we're looking at. */
+ int[] nextRanks = new int[state.ranks.length];
+ System.arraycopy(state.ranks, 0, nextRanks, 0, state.ranks.length);
+ nextRanks[k]++;
+
+ /*
+ * We might have reached the end of something (list of rules or tail
+ * nodes)
+ */
+ if (k == 0
+ && (nextRanks[k] > rules.size() || (config.num_translation_options > 0 && nextRanks[k] > config.num_translation_options)))
+ continue;
+ else if ((k != 0 && nextRanks[k] > superNodes.get(k - 1).nodes.size()))
+ continue;
+
+ /* Use the updated ranks to assign the next rule and tail node. */
+ Rule nextRule = rules.get(nextRanks[0] - 1);
+ // HGNode[] nextAntNodes = new HGNode[state.antNodes.size()];
+ List<HGNode> nextAntNodes = new ArrayList<HGNode>(state.antNodes.size());
+ for (int x = 0; x < state.ranks.length - 1; x++)
+ nextAntNodes.add(superNodes.get(x).nodes.get(nextRanks[x + 1] - 1));
+
+ /* Create the next state. */
+ CubePruneState nextState = new CubePruneState(new ComputeNodeResult(featureFunctions,
+ nextRule, nextAntNodes, i, j, sourcePath, this.sentence), nextRanks, rules,
+ nextAntNodes, dotNode);
+
+ /* Skip states that have been explored before. */
+ if (visitedStates.contains(nextState))
+ continue;
+
+ visitedStates.add(nextState);
+ candidates.add(nextState);
+ }
+ }
+ }
+
+ /* Create a priority queue of candidates for each span under consideration */
+ private PriorityQueue<CubePruneState>[] allCandidates;
+
+ private ArrayList<SuperNode> nodeStack;
+
+ /**
+ * Translates the sentence using the CKY+ variation proposed in
+ * "A CYK+ Variant for SCFG Decoding Without A Dot Chart" (Sennrich, SSST
+ * 2014).
+ */
+ private int i = -1;
+
+ public HyperGraph expandSansDotChart() {
+ for (i = sourceLength - 1; i >= 0; i--) {
+ allCandidates = new PriorityQueue[sourceLength - i + 2];
+ for (int id = 0; id < allCandidates.length; id++)
+ allCandidates[id] = new PriorityQueue<CubePruneState>();
+
+ nodeStack = new ArrayList<SuperNode>();
+
+ for (int j = i + 1; j <= sourceLength; j++) {
+ if (!sentence.hasPath(i, j))
+ continue;
+
+ for (int g = 0; g < this.grammars.length; g++) {
+ // System.err.println(String.format("\n*** I=%d J=%d GRAMMAR=%d", i, j, g));
+
+ if (j == i + 1) {
+ /* Handle terminals */
+ Node<Token> node = sentence.getNode(i);
+ for (Arc<Token> arc : node.getOutgoingArcs()) {
+ int word = arc.getLabel().getWord();
+ // disallow lattice decoding for now
+ assert arc.getHead().id() == j;
+ Trie trie = this.grammars[g].getTrieRoot().match(word);
+ if (trie != null && trie.hasRules())
+ addToChart(trie, j, false);
+ }
+ } else {
+ /* Recurse for non-terminal case */
+ consume(this.grammars[g].getTrieRoot(), i, j - 1);
+ }
+ }
+
+ // Now that we've accumulated all the candidates, apply cube pruning
+ applyCubePruning(i, j, allCandidates[j - i]);
+
+ // Add unary nodes
+ addUnaryNodes(this.grammars, i, j);
+ }
+ }
+
+ // transition_final: setup a goal item, which may have many deductions
+ if (null == this.cells.get(0, sourceLength)
+ || !this.goalBin.transitToGoal(this.cells.get(0, sourceLength), this.featureFunctions,
+ this.sourceLength)) {
+ LOG.warn("Input {}: Parse failure (either no derivations exist or pruning is too aggressive",
+ sentence.id());
+ return null;
+ }
+
+ return new HyperGraph(this.goalBin.getSortedNodes().get(0), -1, -1, this.sentence);
+ }
+
+ /**
+ * Recursively consumes the trie, following input nodes, finding applicable
+ * rules and adding them to bins for each span for later cube pruning.
+ *
+ * @param dotNode data structure containing information about what's been
+ * already matched
+ * @param l extension point we're looking at
+ *
+ */
+ private void consume(Trie trie, int j, int l) {
+ /*
+ * 1. If the trie node has any rules, we can add them to the collection
+ *
+ * 2. Next, look at all the outgoing nonterminal arcs of the trie node. If
+ * any of them match an existing chart item, then we know we can extend
+ * (i,j) to (i,l). We then recurse for all m from l+1 to n (the end of the
+ * sentence)
+ *
+ * 3. We also try to match terminals if (j + 1 == l)
+ */
+
+ // System.err.println(String.format("CONSUME %s / %d %d %d", dotNode,
+ // dotNode.begin(), dotNode.end(), l));
+
+ // Try to match terminals
+ if (inputLattice.distance(j, l) == 1) {
+ // Get the current sentence node, and explore all outgoing arcs, since we
+ // might be decoding
+ // a lattice. For sentence decoding, this is trivial: there is only one
+ // outgoing arc.
+ Node<Token> inputNode = sentence.getNode(j);
+ for (Arc<Token> arc : inputNode.getOutgoingArcs()) {
+ int word = arc.getLabel().getWord();
+ Trie nextTrie;
+ if ((nextTrie = trie.match(word)) != null) {
+ // add to chart item over (i, l)
+ addToChart(nextTrie, arc.getHead().id(), i == j);
+ }
+ }
+ }
+
+ // Now try to match nonterminals
+ Cell cell = cells.get(j, l);
+ if (cell != null) {
+ for (int id : cell.getKeySet()) { // for each supernode (lhs), see if you
+ // can match a trie
+ Trie nextTrie = trie.match(id);
+ if (nextTrie != null) {
+ SuperNode superNode = cell.getSuperNode(id);
+ nodeStack.add(superNode);
+ addToChart(nextTrie, superNode.end(), i == j);
+ nodeStack.remove(nodeStack.size() - 1);
+ }
+ }
+ }
+ }
+
+ /**
+ * Adds all rules at a trie node to the chart, unless its a unary rule. A
+ * unary rule is the first outgoing arc of a grammar's root trie. For
+ * terminals, these are added during the seeding stage; for nonterminals,
+ * these confuse cube pruning and can result in infinite loops, and are
+ * handled separately (see addUnaryNodes());
+ *
+ * @param trie the grammar node
+ * @param isUnary whether the rules at this dotnode are unary
+ */
+ private void addToChart(Trie trie, int j, boolean isUnary) {
+
+ // System.err.println(String.format("ADD TO CHART %s unary=%s", dotNode,
+ // isUnary));
+
+ if (!isUnary && trie.hasRules()) {
+ DotNode dotNode = new DotNode(i, j, trie, new ArrayList<SuperNode>(nodeStack), null);
+
+ addToCandidates(dotNode);
+ }
+
+ for (int l = j + 1; l <= sentence.length(); l++)
+ consume(trie, j, l);
+ }
+
+ /**
+ * Record the completed rule with backpointers for later cube-pruning.
+ *
+ * @param width
+ * @param rules
+ * @param tailNodes
+ */
+ private void addToCandidates(DotNode dotNode) {
+ // System.err.println(String.format("ADD TO CANDIDATES %s AT INDEX %d",
+ // dotNode, dotNode.end() - dotNode.begin()));
+
+ // TODO: one entry per rule, or per rule instantiation (rule together with
+ // unique matching of input)?
+ List<Rule> rules = dotNode.getRuleCollection().getSortedRules(featureFunctions);
+ Rule bestRule = rules.get(0);
+ List<SuperNode> superNodes = dotNode.getAntSuperNodes();
+
+ List<HGNode> tailNodes = new ArrayList<HGNode>();
+ for (SuperNode superNode : superNodes)
+ tailNodes.add(superNode.nodes.get(0));
+
+ int[] ranks = new int[1 + superNodes.size()];
+ Arrays.fill(ranks, 1);
+
+ ComputeNodeResult result = new ComputeNodeResult(featureFunctions, bestRule, tailNodes,
+ dotNode.begin(), dotNode.end(), dotNode.getSourcePath(), sentence);
+ CubePruneState seedState = new CubePruneState(result, ranks, rules, tailNodes, dotNode);
+
+ allCandidates[dotNode.end() - dotNode.begin()].add(seedState);
+ }
+
+ /**
+ * This function performs the main work of decoding.
+ *
+ * @return the hypergraph containing the translated sentence.
+ */
+ public HyperGraph expand() {
+
+ for (int width = 1; width <= sourceLength; width++) {
+ for (int i = 0; i <= sourceLength - width; i++) {
+ int j = i + width;
+ if (LOG.isDebugEnabled())
+ LOG.debug("Processing span ({}, {})", i, j);
+
+ /* Skips spans for which no path exists (possible in lattices). */
+ if (inputLattice.distance(i, j) == Float.POSITIVE_INFINITY) {
+ continue;
+ }
+
+ /*
+ * 1. Expand the dot through all rules. This is a matter of (a) look for
+ * rules over (i,j-1) that need the terminal at (j-1,j) and looking at
+ * all split points k to expand nonterminals.
+ */
+ if (LOG.isDebugEnabled())
+ LOG.debug("Expanding cell");
+ for (int k = 0; k < this.grammars.length; k++) {
+ /**
+ * Each dotChart can act individually (without consulting other
+ * dotCharts) because it either consumes the source input or the
+ * complete nonTerminals, which are both grammar-independent.
+ **/
+ this.dotcharts[k].expandDotCell(i, j);
+ }
+
+ /*
+ * 2. The regular CKY part: add completed items onto the chart via cube
+ * pruning.
+ */
+ if (LOG.isDebugEnabled())
+ LOG.debug("Adding complete items into chart");
+ completeSpan(i, j);
+
+ /* 3. Process unary rules. */
+ if (LOG.isDebugEnabled())
+ LOG.debug("Adding unary items into chart");
+ addUnaryNodes(this.grammars, i, j);
+
+ // (4)=== in dot_cell(i,j), add dot-nodes that start from the /complete/
+ // superIterms in
+ // chart_cell(i,j)
+ if (LOG.isDebugEnabled())
+ LOG.debug("Initializing new dot-items that start from complete items in this cell");
+ for (int k = 0; k < this.grammars.length; k++) {
+ if (this.grammars[k].hasRuleForSpan(i, j, inputLattice.distance(i, j))) {
+ this.dotcharts[k].startDotItems(i, j);
+ }
+ }
+
+ /*
+ * 5. Sort the nodes in the cell.
+ *
+ * Sort the nodes in this span, to make them usable for future
+ * applications of cube pruning.
+ */
+ if (null != this.cells.get(i, j)) {
+ this.cells.get(i, j).getSortedNodes();
+ }
+ }
+ }
+
+ logStatistics();
+
+ // transition_final: setup a goal item, which may have many deductions
+ if (null == this.cells.get(0, sourceLength)
+ || !this.goalBin.transitToGoal(this.cells.get(0, sourceLength), this.featureFunctions,
+ this.sourceLength)) {
+ LOG.warn("Input {}: Parse failure (either no derivations exist or pruning is too aggressive",
+ sentence.id());
+ return null;
+ }
+
+ if (LOG.isDebugEnabled())
+ LOG.debug("Finished expand");
+ return new HyperGraph(this.goalBin.getSortedNodes().get(0), -1, -1, this.sentence);
+ }
+
+ /**
+ * Get the requested cell, creating the entry if it doesn't already exist.
+ *
+ * @param i span start
+ * @param j span end
+ * @return the cell item
+ */
+ public Cell getCell(int i, int j) {
+ assert i >= 0;
+ assert i <= sentence.length();
+ assert i <= j;
+ if (cells.get(i, j) == null)
+ cells.set(i, j, new Cell(this, goalSymbolID));
+
+ return cells.get(i, j);
+ }
+
+ // ===============================================================
+ // Private methods
+ // ===============================================================
+
+ private void logStatistics() {
+ if (LOG.isDebugEnabled())
+ LOG.debug("Input {}: Chart: added {} merged {} dot-items added: {}",
+ this.sentence.id(), this.nAdded, this.nMerged, this.nDotitemAdded);
+ }
+
+ /**
+ * Handles expansion of unary rules. Rules are expanded in an agenda-based
+ * manner to avoid constructing infinite unary chains. Assumes a triangle
+ * inequality of unary rule expansion (e.g., A -> B will always be cheaper
+ * than A -> C -> B), which is not a true assumption.
+ *
+ * @param grammars A list of the grammars for the sentence
+ * @param i
+ * @param j
+ * @return the number of nodes added
+ */
+ private int addUnaryNodes(Grammar[] grammars, int i, int j) {
+
+ Cell chartBin = this.cells.get(i, j);
+ if (null == chartBin) {
+ return 0;
+ }
+ int qtyAdditionsToQueue = 0;
+ ArrayList<HGNode> queue = new ArrayList<HGNode>(chartBin.getSortedNodes());
+ HashSet<Integer> seen_lhs = new HashSet<Integer>();
+
+ if (LOG.isDebugEnabled())
+ LOG.debug("Adding unary to [{}, {}]", i, j);
+
+ while (queue.size() > 0) {
+ HGNode node = queue.remove(0);
+ seen_lhs.add(node.lhs);
+
+ for (Grammar gr : grammars) {
+ if (!gr.hasRuleForSpan(i, j, inputLattice.distance(i, j)))
+ continue;
+
+ /*
+ * Match against the node's LHS, and then make sure the rule collection
+ * has unary rules
+ */
+ Trie childNode = gr.getTrieRoot().match(node.lhs);
+ if (childNode != null && childNode.getRuleCollection() != null
+ && childNode.getRuleCollection().getArity() == 1) {
+
+ ArrayList<HGNode> antecedents = new ArrayList<HGNode>();
+ antecedents.add(node);
+
+ List<Rule> rules = childNode.getRuleCollection().getSortedRules(this.featureFunctions);
+ for (Rule rule : rules) { // for each unary rules
+
+ ComputeNodeResult states = new ComputeNodeResult(this.featureFunctions, rule,
+ antecedents, i, j, new SourcePath(), this.sentence);
+ HGNode resNode = chartBin.addHyperEdgeInCell(states, rule, i, j, antecedents,
+ new SourcePath(), true);
+
+ if (LOG.isDebugEnabled())
+ LOG.debug("{}", rule);
+ if (null != resNode && !seen_lhs.contains(resNode.lhs)) {
+ queue.add(resNode);
+ qtyAdditionsToQueue++;
+ }
+ }
+ }
+ }
+ }
+ return qtyAdditionsToQueue;
+ }
+
+ /***
+ * Add a terminal production (X -> english phrase) to the hypergraph.
+ *
+ * @param i the start index
+ * @param j stop index
+ * @param rule the terminal rule applied
+ * @param srcPath the source path cost
+ */
+ public void addAxiom(int i, int j, Rule rule, SourcePath srcPath) {
+ if (null == this.cells.get(i, j)) {
+ this.cells.set(i, j, new Cell(this, this.goalSymbolID));
+ }
+
+ this.cells.get(i, j).addHyperEdgeInCell(
+ new ComputeNodeResult(this.featureFunctions, rule, null, i, j, srcPath, sentence), rule, i,
+ j, null, srcPath, false);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/ComputeNodeResult.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/ComputeNodeResult.java b/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/ComputeNodeResult.java
new file mode 100644
index 0000000..9833734
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/ComputeNodeResult.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.chart_parser;
+
+import java.util.ArrayList;
+
+import java.util.List;
+
+import org.apache.joshua.decoder.Decoder;
+import org.apache.joshua.decoder.ff.StatefulFF;
+import org.apache.joshua.decoder.ff.FeatureFunction;
+import org.apache.joshua.decoder.ff.FeatureVector;
+import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.hypergraph.HGNode;
+import org.apache.joshua.decoder.hypergraph.HyperEdge;
+import org.apache.joshua.decoder.segment_file.Sentence;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class computes the cost of applying a rule.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ * @author Zhifei Li, zhifei.work@gmail.com
+ */
+
+public class ComputeNodeResult {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ComputeNodeResult.class);
+
+ // The cost incurred by the rule itself (and all associated feature functions)
+ private float transitionCost;
+
+ // transitionCost + the Viterbi costs of the tail nodes.
+ private float viterbiCost;
+
+ // viterbiCost + a future estimate (outside cost estimate).
+ private float pruningCostEstimate;
+
+ // The StateComputer objects themselves serve as keys.
+ private List<DPState> dpStates;
+
+ /**
+ * Computes the new state(s) that are produced when applying the given rule to the list of tail
+ * nodes. Also computes a range of costs of doing so (the transition cost, the total (Viterbi)
+ * cost, and a score that includes a future cost estimate).
+ *
+ * Old version that doesn't use the derivation state.
+ * @param featureFunctions {@link java.util.List} of {@link org.apache.joshua.decoder.ff.FeatureFunction}'s
+ * @param rule {@link org.apache.joshua.decoder.ff.tm.Rule} to use when computing th node result
+ * @param tailNodes {@link java.util.List} of {@link org.apache.joshua.decoder.hypergraph.HGNode}'s
+ * @param i todo
+ * @param j todo
+ * @param sourcePath information about a path taken through the source lattice
+ * @param sentence the lattice input
+ */
+ public ComputeNodeResult(List<FeatureFunction> featureFunctions, Rule rule, List<HGNode> tailNodes,
+ int i, int j, SourcePath sourcePath, Sentence sentence) {
+
+ // The total Viterbi cost of this edge. This is the Viterbi cost of the tail nodes, plus
+ // whatever costs we incur applying this rule to create a new hyperedge.
+ float viterbiCost = 0.0f;
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("ComputeNodeResult():");
+ LOG.debug("-> RULE {}", rule);
+ }
+
+ /*
+ * Here we sum the accumulated cost of each of the tail nodes. The total cost of the new
+ * hyperedge (the inside or Viterbi cost) is the sum of these nodes plus the cost of the
+ * transition. Note that this could and should all be generalized to whatever semiring is being
+ * used.
+ */
+ if (null != tailNodes) {
+ for (HGNode item : tailNodes) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("-> item.bestedge: {}", item);
+ LOG.debug("-> TAIL NODE {}", item);
+ }
+ viterbiCost += item.bestHyperedge.getBestDerivationScore();
+ }
+ }
+
+ List<DPState> allDPStates = new ArrayList<DPState>();
+
+ // The transition cost is the new cost incurred by applying this rule
+ float transitionCost = 0.0f;
+
+ // The future cost estimate is a heuristic estimate of the outside cost of this edge.
+ float futureCostEstimate = 0.0f;
+
+ /*
+ * We now iterate over all the feature functions, computing their cost and their expected future
+ * cost.
+ */
+ for (FeatureFunction feature : featureFunctions) {
+ FeatureFunction.ScoreAccumulator acc = feature.new ScoreAccumulator();
+
+ DPState newState = feature.compute(rule, tailNodes, i, j, sourcePath, sentence, acc);
+ transitionCost += acc.getScore();
+
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("FEATURE {} = {} * {} = {}", feature.getName(),
+ acc.getScore() / Decoder.weights.getSparse(feature.getName()),
+ Decoder.weights.getSparse(feature.getName()), acc.getScore());
+ }
+
+ if (feature.isStateful()) {
+ futureCostEstimate += feature.estimateFutureCost(rule, newState, sentence);
+ allDPStates.add(((StatefulFF)feature).getStateIndex(), newState);
+ }
+ }
+ viterbiCost += transitionCost;
+ if (LOG.isDebugEnabled())
+ LOG.debug("-> COST = {}", transitionCost);
+ // Set the final results.
+ this.pruningCostEstimate = viterbiCost + futureCostEstimate;
+ this.viterbiCost = viterbiCost;
+ this.transitionCost = transitionCost;
+ this.dpStates = allDPStates;
+ }
+
+ /**
+ * This is called from {@link org.apache.joshua.decoder.chart_parser.Cell}
+ * when making the final transition to the goal state.
+ * This is done to allow feature functions to correct for partial estimates, since
+ * they now have the knowledge that the whole sentence is complete. Basically, this
+ * is only used by LanguageModelFF, which does not score partial n-grams, and therefore
+ * needs to correct for this when a short sentence ends. KenLMFF corrects for this by
+ * always scoring partial hypotheses, and subtracting off the partial score when longer
+ * context is available. This would be good to do for the LanguageModelFF feature function,
+ * too: it makes search better (more accurate at the beginning, for example), and would
+ * also do away with the need for the computeFinal* class of functions (and hooks in
+ * the feature function interface).
+ *
+ * @param featureFunctions {@link java.util.List} of {@link org.apache.joshua.decoder.ff.FeatureFunction}'s
+ * @param tailNodes {@link java.util.List} of {@link org.apache.joshua.decoder.hypergraph.HGNode}'s
+ * @param i todo
+ * @param j todo
+ * @param sourcePath information about a path taken through the source lattice
+ * @param sentence the lattice input
+ * @return the final cost for the Node
+ */
+ public static float computeFinalCost(List<FeatureFunction> featureFunctions,
+ List<HGNode> tailNodes, int i, int j, SourcePath sourcePath, Sentence sentence) {
+
+ float cost = 0;
+ for (FeatureFunction ff : featureFunctions) {
+ cost += ff.computeFinalCost(tailNodes.get(0), i, j, sourcePath, sentence);
+ }
+ return cost;
+ }
+
+ public static FeatureVector computeTransitionFeatures(List<FeatureFunction> featureFunctions,
+ HyperEdge edge, int i, int j, Sentence sentence) {
+
+ // Initialize the set of features with those that were present with the rule in the grammar.
+ FeatureVector featureDelta = new FeatureVector();
+
+ // === compute feature logPs
+ for (FeatureFunction ff : featureFunctions) {
+ // A null rule signifies the final transition.
+ if (edge.getRule() == null)
+ featureDelta.add(ff.computeFinalFeatures(edge.getTailNodes().get(0), i, j, edge.getSourcePath(), sentence));
+ else {
+ featureDelta.add(ff.computeFeatures(edge.getRule(), edge.getTailNodes(), i, j, edge.getSourcePath(), sentence));
+ }
+ }
+
+ return featureDelta;
+ }
+
+ public float getPruningEstimate() {
+ return this.pruningCostEstimate;
+ }
+
+ /**
+ * The complete cost of the Viterbi derivation at this point
+ * @return float representing cost
+ */
+ public float getViterbiCost() {
+ return this.viterbiCost;
+ }
+
+ public float getBaseCost() {
+ return getViterbiCost() - getTransitionCost();
+ }
+
+ /**
+ * The cost incurred by this edge alone
+ *
+ * @return float representing cost
+ */
+ public float getTransitionCost() {
+ return this.transitionCost;
+ }
+
+ public List<DPState> getDPStates() {
+ return this.dpStates;
+ }
+
+ public void printInfo() {
+ System.out.println("scores: " + transitionCost + "; " + viterbiCost + "; "
+ + pruningCostEstimate);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/CubePruneState.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/CubePruneState.java b/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/CubePruneState.java
new file mode 100644
index 0000000..d57a6a2
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/CubePruneState.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.chart_parser;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.joshua.decoder.hypergraph.HGNode;
+import org.apache.joshua.decoder.chart_parser.DotChart.DotNode;
+import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+import org.apache.joshua.decoder.ff.tm.Rule;
+
+// ===============================================================
+// CubePruneState class
+// ===============================================================
+public class CubePruneState implements Comparable<CubePruneState> {
+ int[] ranks;
+ ComputeNodeResult computeNodeResult;
+ List<HGNode> antNodes;
+ List<Rule> rules;
+ private DotNode dotNode;
+
+ public CubePruneState(ComputeNodeResult score, int[] ranks, List<Rule> rules, List<HGNode> antecedents, DotNode dotNode) {
+ this.computeNodeResult = score;
+ this.ranks = ranks;
+ this.rules = rules;
+ this.antNodes = antecedents;
+ this.dotNode = dotNode;
+ }
+
+ /**
+ * This returns the list of DP states associated with the result.
+ *
+ * @return
+ */
+ List<DPState> getDPStates() {
+ return this.computeNodeResult.getDPStates();
+ }
+
+ Rule getRule() {
+ return this.rules.get(this.ranks[0]-1);
+ }
+
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("STATE ||| rule=" + getRule() + " inside cost = " + computeNodeResult.getViterbiCost()
+ + " estimate = " + computeNodeResult.getPruningEstimate());
+ return sb.toString();
+ }
+
+ public void setDotNode(DotNode node) {
+ this.dotNode = node;
+ }
+
+ public DotNode getDotNode() {
+ return this.dotNode;
+ }
+
+ public boolean equals(Object obj) {
+ if (obj == null)
+ return false;
+ if (!this.getClass().equals(obj.getClass()))
+ return false;
+ CubePruneState state = (CubePruneState) obj;
+ if (state.ranks.length != ranks.length)
+ return false;
+ for (int i = 0; i < ranks.length; i++)
+ if (state.ranks[i] != ranks[i])
+ return false;
+ if (getDotNode() != state.getDotNode())
+ return false;
+
+ return true;
+ }
+
+ public int hashCode() {
+ int hash = (dotNode != null) ? dotNode.hashCode() : 0;
+ hash += Arrays.hashCode(ranks);
+
+ return hash;
+ }
+
+ /**
+ * Compares states by ExpectedTotalLogP, allowing states to be sorted according to their inverse
+ * order (high-prob first).
+ */
+ public int compareTo(CubePruneState another) {
+ if (this.computeNodeResult.getPruningEstimate() < another.computeNodeResult
+ .getPruningEstimate()) {
+ return 1;
+ } else if (this.computeNodeResult.getPruningEstimate() == another.computeNodeResult
+ .getPruningEstimate()) {
+ return 0;
+ } else {
+ return -1;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/DotChart.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/DotChart.java b/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/DotChart.java
new file mode 100644
index 0000000..71c4f03
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/DotChart.java
@@ -0,0 +1,476 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.chart_parser;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.ff.tm.Grammar;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.ff.tm.RuleCollection;
+import org.apache.joshua.decoder.ff.tm.Trie;
+import org.apache.joshua.decoder.segment_file.Token;
+import org.apache.joshua.lattice.Arc;
+import org.apache.joshua.lattice.Lattice;
+import org.apache.joshua.lattice.Node;
+import org.apache.joshua.util.ChartSpan;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * The DotChart handles Earley-style implicit binarization of translation rules.
+ *
+ * The {@link DotNode} object represents the (possibly partial) application of a synchronous rule.
+ * The implicit binarization is maintained with a pointer to the {@link Trie} node in the grammar,
+ * for easy retrieval of the next symbol to be matched. At every span (i,j) of the input sentence,
+ * every incomplete DotNode is examined to see whether it (a) needs a terminal and matches against
+ * the final terminal of the span or (b) needs a nonterminal and matches against a completed
+ * nonterminal in the main chart at some split point (k,j).
+ *
+ * Once a rule is completed, it is entered into the {@link DotChart}. {@link DotCell} objects are
+ * used to group completed DotNodes over a span.
+ *
+ * There is a separate DotChart for every grammar.
+ *
+ * @author Zhifei Li, <zh...@gmail.com>
+ * @author Matt Post <po...@cs.jhu.edu>
+ * @author Kristy Hollingshead Seitz
+ */
+class DotChart {
+
+ // ===============================================================
+ // Static fields
+ // ===============================================================
+
+ private static final Logger LOG = LoggerFactory.getLogger(DotChart.class);
+
+
+ // ===============================================================
+ // Package-protected instance fields
+ // ===============================================================
+ /**
+ * Two-dimensional chart of cells. Some cells might be null. This could definitely be represented
+ * more efficiently, since only the upper half of this triangle is every used.
+ */
+ private ChartSpan<DotCell> dotcells;
+
+ public DotCell getDotCell(int i, int j) {
+ return dotcells.get(i, j);
+ }
+
+ // ===============================================================
+ // Private instance fields (maybe could be protected instead)
+ // ===============================================================
+
+ /**
+ * CKY+ style parse chart in which completed span entries are stored.
+ */
+ private Chart dotChart;
+
+ /**
+ * Translation grammar which contains the translation rules.
+ */
+ private Grammar pGrammar;
+
+ /* Length of input sentence. */
+ private final int sentLen;
+
+ /* Represents the input sentence being translated. */
+ private final Lattice<Token> input;
+
+ // ===============================================================
+ // Constructors
+ // ===============================================================
+
+ // TODO: Maybe this should be a non-static inner class of Chart. That would give us implicit
+ // access to all the arguments of this constructor. Though we would need to take an argument, i,
+ // to know which Chart.this.grammars[i] to use.
+
+ /**
+ * Constructs a new dot chart from a specified input lattice, a translation grammar, and a parse
+ * chart.
+ *
+ * @param input A lattice which represents an input sentence.
+ * @param grammar A translation grammar.
+ * @param chart A CKY+ style chart in which completed span entries are stored.
+ */
+ public DotChart(Lattice<Token> input, Grammar grammar, Chart chart) {
+
+ this.dotChart = chart;
+ this.pGrammar = grammar;
+ this.input = input;
+ this.sentLen = input.size();
+ this.dotcells = new ChartSpan<DotCell>(sentLen, null);
+
+ seed();
+ }
+
+ /**
+ * Add initial dot items: dot-items pointer to the root of the grammar trie.
+ */
+ void seed() {
+ for (int j = 0; j <= sentLen - 1; j++) {
+ if (pGrammar.hasRuleForSpan(j, j, input.distance(j, j))) {
+ if (null == pGrammar.getTrieRoot()) {
+ throw new RuntimeException("trie root is null");
+ }
+ addDotItem(pGrammar.getTrieRoot(), j, j, null, null, new SourcePath());
+ }
+ }
+ }
+
+ /**
+ * This function computes all possible expansions of all rules over the provided span (i,j). By
+ * expansions, we mean the moving of the dot forward (from left to right) over a nonterminal or
+ * terminal symbol on the rule's source side.
+ *
+ * There are two kinds of expansions:
+ *
+ * <ol>
+ * <li>Expansion over a nonterminal symbol. For this kind of expansion, a rule has a dot
+ * immediately prior to a source-side nonterminal. The main Chart is consulted to see whether
+ * there exists a completed nonterminal with the same label. If so, the dot is advanced.
+ *
+ * Discovering nonterminal expansions is a matter of enumerating all split points k such that i <
+ * k and k < j. The nonterminal symbol must exist in the main Chart over (k,j).
+ *
+ * <li>Expansion over a terminal symbol. In this case, expansion is a simple matter of determing
+ * whether the input symbol at position j (the end of the span) matches the next symbol in the
+ * rule. This is equivalent to choosing a split point k = j - 1 and looking for terminal symbols
+ * over (k,j). Note that phrases in the input rule are handled one-by-one as we consider longer
+ * spans.
+ * </ol>
+ */
+ void expandDotCell(int i, int j) {
+ if (LOG.isDebugEnabled())
+ LOG.debug("Expanding dot cell ({}, {})", i, j);
+
+ /*
+ * (1) If the dot is just to the left of a non-terminal variable, we look for theorems or axioms
+ * in the Chart that may apply and extend the dot position. We look for existing axioms over all
+ * spans (k,j), i < k < j.
+ */
+ for (int k = i + 1; k < j; k++) {
+ extendDotItemsWithProvedItems(i, k, j, false);
+ }
+
+ /*
+ * (2) If the the dot-item is looking for a source-side terminal symbol, we simply match against
+ * the input sentence and advance the dot.
+ */
+ Node<Token> node = input.getNode(j - 1);
+ for (Arc<Token> arc : node.getOutgoingArcs()) {
+
+ int last_word = arc.getLabel().getWord();
+ int arc_len = arc.getHead().getNumber() - arc.getTail().getNumber();
+
+ // int last_word=foreign_sent[j-1]; // input.getNode(j-1).getNumber(); //
+
+ if (null != dotcells.get(i, j - 1)) {
+ // dotitem in dot_bins[i][k]: looking for an item in the right to the dot
+
+
+ for (DotNode dotNode : dotcells.get(i, j - 1).getDotNodes()) {
+
+ // String arcWord = Vocabulary.word(last_word);
+ // Assert.assertFalse(arcWord.endsWith("]"));
+ // Assert.assertFalse(arcWord.startsWith("["));
+ // logger.info("DotChart.expandDotCell: " + arcWord);
+
+ // List<Trie> child_tnodes = ruleMatcher.produceMatchingChildTNodesTerminalevel(dotNode,
+ // last_word);
+
+ List<Trie> child_tnodes = null;
+
+ Trie child_node = dotNode.trieNode.match(last_word);
+ if (null != child_node) {
+ addDotItem(child_node, i, j - 1 + arc_len, dotNode.antSuperNodes, null,
+ dotNode.srcPath.extend(arc));
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * note: (i,j) is a non-terminal, this cannot be a cn-side terminal, which have been handled in
+ * case2 of dotchart.expand_cell add dotitems that start with the complete super-items in
+ * cell(i,j)
+ */
+ void startDotItems(int i, int j) {
+ extendDotItemsWithProvedItems(i, i, j, true);
+ }
+
+ // ===============================================================
+ // Private methods
+ // ===============================================================
+
+ /**
+ * Attempt to combine an item in the dot chart with an item in the main chart to create a new item
+ * in the dot chart. The DotChart item is a {@link DotNode} begun at position i with the dot
+ * currently at position k, that is, a partially-applied rule.
+ *
+ * In other words, this method looks for (proved) theorems or axioms in the completed chart that
+ * may apply and extend the dot position.
+ *
+ * @param i Start index of a dot chart item
+ * @param k End index of a dot chart item; start index of a completed chart item
+ * @param j End index of a completed chart item
+ * @param skipUnary if true, don't extend unary rules
+ */
+ private void extendDotItemsWithProvedItems(int i, int k, int j, boolean skipUnary) {
+ if (this.dotcells.get(i, k) == null || this.dotChart.getCell(k, j) == null) {
+ return;
+ }
+
+ // complete super-items (items over the same span with different LHSs)
+ List<SuperNode> superNodes = new ArrayList<SuperNode>(this.dotChart.getCell(k, j)
+ .getSortedSuperItems().values());
+
+ /* For every partially complete item over (i,k) */
+ for (DotNode dotNode : dotcells.get(i, k).dotNodes) {
+ /* For every completed nonterminal in the main chart */
+ for (SuperNode superNode : superNodes) {
+
+ // String arcWord = Vocabulary.word(superNode.lhs);
+ // logger.info("DotChart.extendDotItemsWithProvedItems: " + arcWord);
+ // Assert.assertTrue(arcWord.endsWith("]"));
+ // Assert.assertTrue(arcWord.startsWith("["));
+
+ /*
+ * Regular Expression matching allows for a regular-expression style rules in the grammar,
+ * which allows for a very primitive treatment of morphology. This is an advanced,
+ * undocumented feature that introduces a complexity, in that the next "word" in the grammar
+ * rule might match more than one outgoing arc in the grammar trie.
+ */
+ Trie child_node = dotNode.getTrieNode().match(superNode.lhs);
+ if (child_node != null) {
+ if ((!skipUnary) || (child_node.hasExtensions())) {
+ addDotItem(child_node, i, j, dotNode.getAntSuperNodes(), superNode, dotNode
+ .getSourcePath().extendNonTerminal());
+ }
+ }
+ }
+ }
+ }
+
+ /*
+ * We introduced the ability to have regular expressions in rules for matching against terminals.
+ * For example, you could have the rule
+ *
+ * <pre> [X] ||| l?s herman?s ||| siblings </pre>
+ *
+ * When this is enabled for a grammar, we need to test against *all* (positive) outgoing arcs of
+ * the grammar trie node to see if any of them match, and then return the whole set. This is quite
+ * expensive, which is why you should only enable regular expressions for small grammars.
+ */
+
+ private ArrayList<Trie> matchAll(DotNode dotNode, int wordID) {
+ ArrayList<Trie> trieList = new ArrayList<>();
+ HashMap<Integer, ? extends Trie> childrenTbl = dotNode.trieNode.getChildren();
+
+ if (childrenTbl != null && wordID >= 0) {
+ // get all the extensions, map to string, check for *, build regexp
+ for (Map.Entry<Integer, ? extends Trie> entry : childrenTbl.entrySet()) {
+ Integer arcID = entry.getKey();
+ if (arcID == wordID) {
+ trieList.add(entry.getValue());
+ } else {
+ String arcWord = Vocabulary.word(arcID);
+ if (Vocabulary.word(wordID).matches(arcWord)) {
+ trieList.add(entry.getValue());
+ }
+ }
+ }
+ }
+ return trieList;
+ }
+
+
+ /**
+ * Creates a {@link DotNode} and adds it into the {@link DotChart} at the correct place. These
+ * are (possibly incomplete) rule applications.
+ *
+ * @param tnode the trie node pointing to the location ("dot") in the grammar trie
+ * @param i
+ * @param j
+ * @param antSuperNodesIn the supernodes representing the rule's tail nodes
+ * @param curSuperNode the lefthand side of the rule being created
+ * @param srcPath the path taken through the input lattice
+ */
+ private void addDotItem(Trie tnode, int i, int j, ArrayList<SuperNode> antSuperNodesIn,
+ SuperNode curSuperNode, SourcePath srcPath) {
+ ArrayList<SuperNode> antSuperNodes = new ArrayList<SuperNode>();
+ if (antSuperNodesIn != null) {
+ antSuperNodes.addAll(antSuperNodesIn);
+ }
+ if (curSuperNode != null) {
+ antSuperNodes.add(curSuperNode);
+ }
+
+ DotNode item = new DotNode(i, j, tnode, antSuperNodes, srcPath);
+ if (dotcells.get(i, j) == null) {
+ dotcells.set(i, j, new DotCell());
+ }
+ dotcells.get(i, j).addDotNode(item);
+ dotChart.nDotitemAdded++;
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Add a dotitem in cell ({}, {}), n_dotitem={}, {}", i, j,
+ dotChart.nDotitemAdded, srcPath);
+
+ RuleCollection rules = tnode.getRuleCollection();
+ if (rules != null) {
+ for (Rule r : rules.getRules()) {
+ // System.out.println("rule: "+r.toString());
+ LOG.debug("{}", r);
+ }
+ }
+ }
+ }
+
+ // ===============================================================
+ // Package-protected classes
+ // ===============================================================
+
+ /**
+ * A DotCell groups together DotNodes that have been applied over a particular span. A DotNode, in
+ * turn, is a partially-applied grammar rule, represented as a pointer into the grammar trie
+ * structure.
+ */
+ static class DotCell {
+
+ // Package-protected fields
+ private List<DotNode> dotNodes = new ArrayList<DotNode>();
+
+ public List<DotNode> getDotNodes() {
+ return dotNodes;
+ }
+
+ private void addDotNode(DotNode dt) {
+ /*
+ * if(l_dot_items==null) l_dot_items= new ArrayList<DotItem>();
+ */
+ dotNodes.add(dt);
+ }
+ }
+
+ /**
+ * A DotNode represents the partial application of a rule rooted to a particular span (i,j). It
+ * maintains a pointer to the trie node in the grammar for efficient mapping.
+ */
+ static class DotNode {
+
+ private int i, j;
+ private Trie trieNode = null;
+
+ /* A list of grounded (over a span) nonterminals that have been crossed in traversing the rule */
+ private ArrayList<SuperNode> antSuperNodes = null;
+
+ /* The source lattice cost of applying the rule */
+ private SourcePath srcPath;
+
+ @Override
+ public String toString() {
+ int size = 0;
+ if (trieNode != null && trieNode.getRuleCollection() != null)
+ size = trieNode.getRuleCollection().getRules().size();
+ return String.format("DOTNODE i=%d j=%d #rules=%d #tails=%d", i, j, size, antSuperNodes.size());
+ }
+
+ /**
+ * Initialize a dot node with the span, grammar trie node, list of supernode tail pointers, and
+ * the lattice sourcepath.
+ *
+ * @param i
+ * @param j
+ * @param trieNode
+ * @param antSuperNodes
+ * @param srcPath
+ */
+ public DotNode(int i, int j, Trie trieNode, ArrayList<SuperNode> antSuperNodes, SourcePath srcPath) {
+ this.i = i;
+ this.j = j;
+ this.trieNode = trieNode;
+ this.antSuperNodes = antSuperNodes;
+ this.srcPath = srcPath;
+ }
+
+ public boolean equals(Object obj) {
+ if (obj == null)
+ return false;
+ if (!this.getClass().equals(obj.getClass()))
+ return false;
+ DotNode state = (DotNode) obj;
+
+ /*
+ * Technically, we should be comparing the span inforamtion as well, but that would require us
+ * to store it, increasing memory requirements, and we should be able to guarantee that we
+ * won't be comparing DotNodes across spans.
+ */
+ // if (this.i != state.i || this.j != state.j)
+ // return false;
+
+ if (this.trieNode != state.trieNode)
+ return false;
+
+ return true;
+ }
+
+ /**
+ * Technically the hash should include the span (i,j), but since DotNodes are grouped by span,
+ * this isn't necessary, and we gain something by not having to store the span.
+ */
+ public int hashCode() {
+ return this.trieNode.hashCode();
+ }
+
+ // convenience function
+ public boolean hasRules() {
+ return getTrieNode().getRuleCollection() != null && getTrieNode().getRuleCollection().getRules().size() != 0;
+ }
+
+ public RuleCollection getRuleCollection() {
+ return getTrieNode().getRuleCollection();
+ }
+
+ public Trie getTrieNode() {
+ return trieNode;
+ }
+
+ public SourcePath getSourcePath() {
+ return srcPath;
+ }
+
+ public ArrayList<SuperNode> getAntSuperNodes() {
+ return antSuperNodes;
+ }
+
+ public int begin() {
+ return i;
+ }
+
+ public int end() {
+ return j;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/SourcePath.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/SourcePath.java b/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/SourcePath.java
new file mode 100644
index 0000000..1d96149
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/SourcePath.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.chart_parser;
+
+import org.apache.joshua.decoder.segment_file.Token;
+import org.apache.joshua.lattice.Arc;
+
+/**
+ * This class represents information about a path taken through the source lattice.
+ *
+ * <p>This implementation only tracks the source path cost which is assumed to be a scalar value.
+ * If you need multiple values, or want to recover more detailed path statistics, you'll need
+ * to update this code.
+ */
+public class SourcePath {
+
+ private final float pathCost;
+
+ public SourcePath() {
+ pathCost = 0.0f;
+ }
+
+ private SourcePath(float cost) {
+ pathCost = cost;
+ }
+
+ public float getPathCost() {
+ return pathCost;
+ }
+
+ public SourcePath extend(Arc<Token> srcEdge) {
+ float tcost = (float) srcEdge.getCost();
+ if (tcost == 0.0)
+ return this;
+ else
+ return new SourcePath(pathCost + (float) srcEdge.getCost());
+ }
+
+ public SourcePath extendNonTerminal() {
+ return this;
+ }
+
+ public String toString() {
+ return "SourcePath.cost=" + pathCost;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/StateConstraint.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/StateConstraint.java b/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/StateConstraint.java
new file mode 100644
index 0000000..d21ceca
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/StateConstraint.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.chart_parser;
+
+import java.util.Collection;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+import org.apache.joshua.decoder.ff.state_maintenance.NgramDPState;
+
+/**
+ * This class provides constraints on the sorts of states that are permitted in the chart. Its
+ * original motivation was to be used as a means of doing forced decoding, which is accomplished by
+ * forcing all n-gram states that are created to match the target string.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ *
+ */
+public class StateConstraint {
+ private String target = null;
+
+ public StateConstraint(String target) {
+ this.target = " <s> " + target + " </s> ";
+ }
+
+ /**
+ * Determines if all of the states passed in are legal in light of the input that was passed
+ * earlier. Currently only defined for n-gram states.
+ *
+ * @param dpStates {@link java.util.Collection} of {@link org.apache.joshua.decoder.ff.state_maintenance.DPState}'s
+ * @return whether the states are legal in light of the target side sentence
+ */
+ public boolean isLegal(Collection<DPState> dpStates) {
+ /*
+ * Iterate over all the state-contributing objects associated with the new state, querying
+ * n-gram ones (of which there is probably only one), allowing them to veto the move.
+ */
+ for (DPState dpState : dpStates) {
+ if (dpState instanceof NgramDPState) {
+ // Build a regular expression out of the state context.
+ String leftWords = " "
+ + Vocabulary.getWords(((NgramDPState) dpState).getLeftLMStateWords()) + " ";
+ String rightWords = " "
+ + Vocabulary.getWords(((NgramDPState) dpState).getRightLMStateWords()) + " ";
+
+ int leftPos = this.target.indexOf(leftWords);
+ int rightPos = this.target.lastIndexOf(rightWords);
+
+ boolean legal = (leftPos != -1 && leftPos <= rightPos);
+// System.err.println(String.format(" isLegal(%s @ %d,%s @ %d) = %s", leftWords, leftPos,
+// rightWords, rightPos, legal));
+
+ return legal;
+ }
+ }
+
+ return true;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/SuperNode.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/SuperNode.java b/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/SuperNode.java
new file mode 100644
index 0000000..a7c6e34
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/SuperNode.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.chart_parser;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.joshua.decoder.hypergraph.HGNode;
+
+/**
+ * Represents a list of items in the hypergraph that have the same left-hand side but may have
+ * different LM states.
+ *
+ * @author Zhifei Li
+ */
+class SuperNode {
+
+ /** Common left-hand side state. */
+ final int lhs;
+
+ /**
+ * List of hypergraph nodes, each of which has its own language model state.
+ */
+ final List<HGNode> nodes;
+
+ /**
+ * All nodes in a SuperNode have the same start and end points, so we pick the first one and
+ * return it.
+ *
+ * @return
+ */
+ public int end() {
+ return nodes.get(0).j;
+ }
+
+
+ /**
+ * Constructs a super item defined by a common left-hand side.
+ *
+ * @param lhs Left-hand side token
+ */
+ public SuperNode(int lhs) {
+ this.lhs = lhs;
+ this.nodes = new ArrayList<HGNode>();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/package-info.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/package-info.java b/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/package-info.java
new file mode 100644
index 0000000..8bf73ba
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/chart_parser/package-info.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * Provides an implementation of a hierarchical phrase-based
+ * decoder for statistical machine translation. The code in
+ * this package is based largely on algorithms from Chiang (2007).
+ */
+package org.apache.joshua.decoder.chart_parser;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/ArityPhrasePenalty.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/ArityPhrasePenalty.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/ArityPhrasePenalty.java
new file mode 100644
index 0000000..ae273b7
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/ArityPhrasePenalty.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff;
+
+import java.util.List;
+
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.hypergraph.HGNode;
+import org.apache.joshua.decoder.segment_file.Sentence;
+import org.apache.joshua.decoder.chart_parser.SourcePath;
+import org.apache.joshua.corpus.Vocabulary;
+
+/**
+ * This feature function counts rules from a particular grammar (identified by the owner) having an
+ * arity within a specific range. It expects three parameters upon initialization: the owner, the
+ * minimum arity, and the maximum arity.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ * @author Zhifei Li zhifei.work@gmail.com
+ */
+public class ArityPhrasePenalty extends StatelessFF {
+
+ // when the rule.arity is in the range, then this feature is activated
+ private final int owner;
+ private final int minArity;
+ private final int maxArity;
+
+ public ArityPhrasePenalty(final FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, "ArityPenalty", args, config);
+
+ this.owner = Vocabulary.id(parsedArgs.get("owner"));
+ this.minArity = Integer.parseInt(parsedArgs.get("min-arity"));
+ this.maxArity = Integer.parseInt(parsedArgs.get("max-arity"));
+ }
+
+ /**
+ * Returns 1 if the arity penalty feature applies to the current rule.
+ */
+ private int isEligible(final Rule rule) {
+ if (this.owner == rule.getOwner() && rule.getArity() >= this.minArity
+ && rule.getArity() <= this.maxArity)
+ return 1;
+
+ return 0;
+ }
+
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+ acc.add(name, isEligible(rule));
+
+ return null;
+ }
+}
[02/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/samt/grammar.glue
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/samt/grammar.glue b/joshua-core/src/test/resources/bn-en/samt/grammar.glue
new file mode 100644
index 0000000..d6667cd
--- /dev/null
+++ b/joshua-core/src/test/resources/bn-en/samt/grammar.glue
@@ -0,0 +1,5673 @@
+[GOAL] ||| <s> ||| <s> ||| 0
+[GOAL] ||| [GOAL,1] [SYM+RB+DT,2] ||| [GOAL,1] [SYM+RB+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+NN+IN,2] ||| [GOAL,1] [:+NN+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+.,2] ||| [GOAL,1] [CC+NP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [RP+IN+DT,2] ||| [GOAL,1] [RP+IN+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+:,2] ||| [GOAL,1] [CC+NP+:,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+PP+DT,2] ||| [GOAL,1] [PP+PP+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+VBP+DT,2] ||| [GOAL,1] [PP+VBP+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VBZ+WRB,2] ||| [GOAL,1] [NN+VBZ+WRB,2] ||| -1
+[GOAL] ||| [GOAL,1] [POS+CC,2] ||| [GOAL,1] [POS+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [NX+COMMA+ADJP,2] ||| [GOAL,1] [NX+COMMA+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+FW,2] ||| [GOAL,1] [:+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+RB+.,2] ||| [GOAL,1] [PP+RB+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+IN+OOV,2] ||| [GOAL,1] [COMMA+IN+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+:+NP,2] ||| [GOAL,1] [CD+:+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+PP+.,2] ||| [GOAL,1] [VBN+PP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+MD,2] ||| [GOAL,1] [CC+NP+MD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+EX,2] ||| [GOAL,1] [NN+EX,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+NN+NP,2] ||| [GOAL,1] [SYM+NN+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJS+WDT,2] ||| [GOAL,1] [JJS+WDT,2] ||| -1
+[GOAL] ||| [GOAL,1] [OOV+COMMA+OOV,2] ||| [GOAL,1] [OOV+COMMA+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+CD+.,2] ||| [GOAL,1] [CD+CD+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+VP+.,2] ||| [GOAL,1] [CC+VP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+RB,2] ||| [GOAL,1] [CD+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+.,2] ||| [GOAL,1] [NP+VBD+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+RB+ADVP,2] ||| [GOAL,1] [COMMA+RB+ADVP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+RB+VP,2] ||| [GOAL,1] [NN+RB+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+:,2] ||| [GOAL,1] [NP+VBD+:,2] ||| -1
+[GOAL] ||| [GOAL,1] [.,2] ||| [GOAL,1] [.,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+SYM,2] ||| [GOAL,1] [RB+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+FW+SYM,2] ||| [GOAL,1] [PP+FW+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJS+NN+VBZ,2] ||| [GOAL,1] [JJS+NN+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+VBG+COMMA,2] ||| [GOAL,1] [CC+VBG+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR+COMMA+VB,2] ||| [GOAL,1] [SBAR+COMMA+VB,2] ||| -1
+[GOAL] ||| [GOAL,1] [QP+PRP+$,2] ||| [GOAL,1] [QP+PRP+$,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+NN,2] ||| [GOAL,1] [CC+NP+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VB+RB,2] ||| [GOAL,1] [NN+VB+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP/'',2] ||| [GOAL,1] [ADJP/'',2] ||| -1
+[GOAL] ||| [GOAL,1] [JJS+NN+VBP,2] ||| [GOAL,1] [JJS+NN+VBP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+PP+CC,2] ||| [GOAL,1] [PP+PP+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [:,2] ||| [GOAL,1] [:,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+RB+NP,2] ||| [GOAL,1] [NNS+RB+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+VBP+DT,2] ||| [GOAL,1] [PRP+VBP+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+FRAG,2] ||| [GOAL,1] [NN+FRAG,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+FW,2] ||| [GOAL,1] [NN+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+JJ+CD,2] ||| [GOAL,1] [NP+JJ+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+JJ+CC,2] ||| [GOAL,1] [NP+JJ+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+DT+ADJP,2] ||| [GOAL,1] [VBD+DT+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [.+OOV+JJ,2] ||| [GOAL,1] [.+OOV+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+CD,2] ||| [GOAL,1] [NN+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+:+NN,2] ||| [GOAL,1] [VBD+:+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+COMMA+IN,2] ||| [GOAL,1] [PP+COMMA+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+VBZ+CD,2] ||| [GOAL,1] [PRP+VBZ+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+:+NP,2] ||| [GOAL,1] [VBD+:+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [OOV\S,2] ||| [GOAL,1] [OOV\S,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+NP,2] ||| [GOAL,1] [CC+NP+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+CC,2] ||| [GOAL,1] [NN+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [OOV\X,2] ||| [GOAL,1] [OOV\X,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+IN+CC,2] ||| [GOAL,1] [NN+IN+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [OOV+SYM+RB,2] ||| [GOAL,1] [OOV+SYM+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [.+DT,2] ||| [GOAL,1] [.+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NN+WP$,2] ||| [GOAL,1] [JJ+NN+WP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [S,2] ||| [GOAL,1] [S,2] ||| -1
+[GOAL] ||| [GOAL,1] [FRAG+COMMA+DT,2] ||| [GOAL,1] [FRAG+COMMA+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+IN+CD,2] ||| [GOAL,1] [NN+IN+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+PRP+VBD,2] ||| [GOAL,1] [ADVP+PRP+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+RB+DT,2] ||| [GOAL,1] [RB+RB+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+RB+CD,2] ||| [GOAL,1] [SYM+RB+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+OOV+COMMA,2] ||| [GOAL,1] [VB+OOV+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+CD+NNP,2] ||| [GOAL,1] [DT+CD+NNP,2] ||| -1
+[GOAL] ||| [GOAL,1] [X,2] ||| [GOAL,1] [X,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP/S,2] ||| [GOAL,1] [PP/S,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+TO,2] ||| [GOAL,1] [CD+TO,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+:+PP,2] ||| [GOAL,1] [CD+:+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+DT,2] ||| [GOAL,1] [:+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNP+CC+IN,2] ||| [GOAL,1] [NNP+CC+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+:+VBP,2] ||| [GOAL,1] [PP+:+VBP,2] ||| -1
+[GOAL] ||| [GOAL,1] [S+CC+RB,2] ||| [GOAL,1] [S+CC+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+VBD+SYM,2] ||| [GOAL,1] [NNS+VBD+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+:+VP,2] ||| [GOAL,1] [PP+:+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+SYM,2] ||| [GOAL,1] [PP+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+PP,2] ||| [GOAL,1] [CC+NP+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+VBN+DT,2] ||| [GOAL,1] [VBD+VBN+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [OOV+POS,2] ||| [GOAL,1] [OOV+POS,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+DT,2] ||| [GOAL,1] [NN+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+COMMA+CD,2] ||| [GOAL,1] [PP+COMMA+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+CD,2] ||| [GOAL,1] [:+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [POS+NX+CC,2] ||| [GOAL,1] [POS+NX+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+COMMA+CC,2] ||| [GOAL,1] [PP+COMMA+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+NX,2] ||| [GOAL,1] [CD+NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+CC,2] ||| [GOAL,1] [:+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBG\SINV,2] ||| [GOAL,1] [VBG\SINV,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+NP+NP,2] ||| [GOAL,1] [ADVP+NP+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+MD+VB,2] ||| [GOAL,1] [CC+MD+VB,2] ||| -1
+[GOAL] ||| [GOAL,1] [WRB+VBZ,2] ||| [GOAL,1] [WRB+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [FW+VP+'',2] ||| [GOAL,1] [FW+VP+'',2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VBP+NNS,2] ||| [GOAL,1] [NN+VBP+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+VBZ+DT,2] ||| [GOAL,1] [PRP+VBZ+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+CD,2] ||| [GOAL,1] [VBD+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+COMMA+ADVP,2] ||| [GOAL,1] [RB+COMMA+ADVP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+IN,2] ||| [GOAL,1] [NN+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+NP,2] ||| [GOAL,1] [CD+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+NN,2] ||| [GOAL,1] [CD+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [TO+VB+DT,2] ||| [GOAL,1] [TO+VB+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+SYM+FW,2] ||| [GOAL,1] [:+SYM+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+CC,2] ||| [GOAL,1] [VBD+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+JJ,2] ||| [GOAL,1] [NN+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR+COMMA+RB,2] ||| [GOAL,1] [SBAR+COMMA+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+COMMA+PRP$,2] ||| [GOAL,1] [PP+COMMA+PRP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [POS+WDT+VP,2] ||| [GOAL,1] [POS+WDT+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+SYM,2] ||| [GOAL,1] [NP+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+PP+WRB,2] ||| [GOAL,1] [ADVP+PP+WRB,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+:+DT,2] ||| [GOAL,1] [VBP+:+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+NN+JJ,2] ||| [GOAL,1] [SYM+NN+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NNS+.,2] ||| [GOAL,1] [CC+NNS+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [TO+VB+CC,2] ||| [GOAL,1] [TO+VB+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+COMMA+SYM,2] ||| [GOAL,1] [PP+COMMA+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VP+CC,2] ||| [GOAL,1] [COMMA+VP+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+RB,2] ||| [GOAL,1] [CC+NP+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [NX+OOV,2] ||| [GOAL,1] [NX+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+VBP+JJ,2] ||| [GOAL,1] [PP+VBP+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+COMMA+SYM,2] ||| [GOAL,1] [NP+COMMA+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+MD,2] ||| [GOAL,1] [CD+MD,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NNS+:,2] ||| [GOAL,1] [CC+NNS+:,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+VBP+.,2] ||| [GOAL,1] [NNS+VBP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [TO+VBG,2] ||| [GOAL,1] [TO+VBG,2] ||| -1
+[GOAL] ||| [GOAL,1] [OOV+NNS+COMMA,2] ||| [GOAL,1] [OOV+NNS+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+X,2] ||| [GOAL,1] [NP+VBD+X,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+$+PRN,2] ||| [GOAL,1] [PRP+$+PRN,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+PP+VP,2] ||| [GOAL,1] [ADVP+PP+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+NN+CC,2] ||| [GOAL,1] [VBD+NN+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ\NP,2] ||| [GOAL,1] [JJ\NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+DT,2] ||| [GOAL,1] [VBD+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [''+COMMA,2] ||| [GOAL,1] [''+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+RB+VP,2] ||| [GOAL,1] [NNS+RB+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+WDT+VBZ,2] ||| [GOAL,1] [PP+WDT+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+DT+ADJP,2] ||| [GOAL,1] [NP+DT+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [''+NN+COMMA,2] ||| [GOAL,1] [''+NN+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBZ+OOV,2] ||| [GOAL,1] [VBZ+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+VBP+:,2] ||| [GOAL,1] [NNS+VBP+:,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB\INTJ,2] ||| [GOAL,1] [VB\INTJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+PP,2] ||| [GOAL,1] [CD+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [TO+VBN,2] ||| [GOAL,1] [TO+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+VBP+SYM,2] ||| [GOAL,1] [SYM+VBP+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+:+FW,2] ||| [GOAL,1] [VBP+:+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBP+VBG,2] ||| [GOAL,1] [NP+VBP+VBG,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+WDT,2] ||| [GOAL,1] [JJ+WDT,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NX+CD,2] ||| [GOAL,1] [CC+NX+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NX+CC,2] ||| [GOAL,1] [CC+NX+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+ADJP+JJ,2] ||| [GOAL,1] [NN+ADJP+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+VBD+VBN,2] ||| [GOAL,1] [PP+VBD+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [WP+VBG+IN,2] ||| [GOAL,1] [WP+VBG+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+NN+VBZ,2] ||| [GOAL,1] [SYM+NN+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+VBN+TO,2] ||| [GOAL,1] [RB+VBN+TO,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+JJR+JJ,2] ||| [GOAL,1] [VBP+JJR+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [.+JJ,2] ||| [GOAL,1] [.+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+NN+NN,2] ||| [GOAL,1] [:+NN+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+ADJP+COMMA,2] ||| [GOAL,1] [COMMA+ADJP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+NP+ADVP,2] ||| [GOAL,1] [COMMA+NP+ADVP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ\NX,2] ||| [GOAL,1] [JJ\NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+COMMA+DT,2] ||| [GOAL,1] [PP+COMMA+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+VBZ+NN,2] ||| [GOAL,1] [JJ+VBZ+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+VBZ+NP,2] ||| [GOAL,1] [JJ+VBZ+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [OOV+CC+VBN,2] ||| [GOAL,1] [OOV+CC+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBP+VBN,2] ||| [GOAL,1] [NP+VBP+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [POS+JJ,2] ||| [GOAL,1] [POS+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+SBAR+.,2] ||| [GOAL,1] [COMMA+SBAR+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+NP+COMMA,2] ||| [GOAL,1] [VB+NP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+NP+IN,2] ||| [GOAL,1] [ADVP+NP+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBN+IN,2] ||| [GOAL,1] [NP+VBN+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+NP+:,2] ||| [GOAL,1] [NP+NP+:,2] ||| -1
+[GOAL] ||| [GOAL,1] [UCP+NNS,2] ||| [GOAL,1] [UCP+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+COMMA+PP,2] ||| [GOAL,1] [JJ+COMMA+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+JJ,2] ||| [GOAL,1] [CD+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+NN+VBZ,2] ||| [GOAL,1] [PP+NN+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+DT+PRP$,2] ||| [GOAL,1] [CC+DT+PRP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+VBN+PP,2] ||| [GOAL,1] [RB+VBN+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [FRAG+COMMA+NP,2] ||| [GOAL,1] [FRAG+COMMA+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+MD,2] ||| [GOAL,1] [NN+MD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+NP+.,2] ||| [GOAL,1] [NP+NP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+DT+RB,2] ||| [GOAL,1] [CC+DT+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+RBR+NP,2] ||| [GOAL,1] [NN+RBR+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [LS+SYM,2] ||| [GOAL,1] [LS+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+DT,2] ||| [GOAL,1] [CC+NP+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+JJ+JJR,2] ||| [GOAL,1] [JJ+JJ+JJR,2] ||| -1
+[GOAL] ||| [GOAL,1] [CONJP+NP+IN,2] ||| [GOAL,1] [CONJP+NP+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+VP+COMMA,2] ||| [GOAL,1] [JJ+VP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+RB+NP,2] ||| [GOAL,1] [RB+RB+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+IN,2] ||| [GOAL,1] [CD+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBN+JJ,2] ||| [GOAL,1] [NP+VBN+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBZ+CD+IN,2] ||| [GOAL,1] [VBZ+CD+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+NX,2] ||| [GOAL,1] [NN+NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [LST+OOV+JJ,2] ||| [GOAL,1] [LST+OOV+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+VBP+NN,2] ||| [GOAL,1] [PP+VBP+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+VBP+NP,2] ||| [GOAL,1] [PP+VBP+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [TO+VB+OOV,2] ||| [GOAL,1] [TO+VB+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP+COMMA+VBZ,2] ||| [GOAL,1] [ADJP+COMMA+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+''+JJ,2] ||| [GOAL,1] [:+''+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [''+NN+WDT,2] ||| [GOAL,1] [''+NN+WDT,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+JJ+NNS,2] ||| [GOAL,1] [COMMA+JJ+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJR+COMMA,2] ||| [GOAL,1] [JJR+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [POS+VP+.,2] ||| [GOAL,1] [POS+VP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [.+CD+JJ,2] ||| [GOAL,1] [.+CD+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+JJ+NNS,2] ||| [GOAL,1] [IN+JJ+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+COMMA+NP,2] ||| [GOAL,1] [PP+COMMA+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [FW+WRB,2] ||| [GOAL,1] [FW+WRB,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+COMMA,2] ||| [GOAL,1] [IN+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [X/CD,2] ||| [GOAL,1] [X/CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+COMMA+NN,2] ||| [GOAL,1] [PP+COMMA+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+NNS+VBD,2] ||| [GOAL,1] [NP+NNS+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+NN,2] ||| [GOAL,1] [NN+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+NP,2] ||| [GOAL,1] [NN+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+VBD+OOV,2] ||| [GOAL,1] [PRP+VBD+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP+NN+JJR,2] ||| [GOAL,1] [ADJP+NN+JJR,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP,2] ||| [GOAL,1] [ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+CC+NN,2] ||| [GOAL,1] [VB+CC+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+:+FW,2] ||| [GOAL,1] [JJ+:+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+COMMA,2] ||| [GOAL,1] [NN+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+POS,2] ||| [GOAL,1] [NN+POS,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR+COMMA+NP,2] ||| [GOAL,1] [SBAR+COMMA+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [WRB+FW+CC,2] ||| [GOAL,1] [WRB+FW+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+COMMA+RB,2] ||| [GOAL,1] [PP+COMMA+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+FW+FW,2] ||| [GOAL,1] [VBP+FW+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR+COMMA+NN,2] ||| [GOAL,1] [SBAR+COMMA+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [WRB+NNS,2] ||| [GOAL,1] [WRB+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+NN+IN,2] ||| [GOAL,1] [SYM+NN+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+FW,2] ||| [GOAL,1] [CC+NP+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [TO+VB+JJ,2] ||| [GOAL,1] [TO+VB+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+:+JJ,2] ||| [GOAL,1] [ADVP+:+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+WRB,2] ||| [GOAL,1] [JJ+WRB,2] ||| -1
+[GOAL] ||| [GOAL,1] [''+ADJP+NNS,2] ||| [GOAL,1] [''+ADJP+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+COMMA+QP,2] ||| [GOAL,1] [PP+COMMA+QP,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+OOV,2] ||| [GOAL,1] [DT+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+IN+JJ,2] ||| [GOAL,1] [NN+IN+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NX+FW,2] ||| [GOAL,1] [CC+NX+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VB+COMMA,2] ||| [GOAL,1] [COMMA+VB+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP+JJ+ADJP,2] ||| [GOAL,1] [ADJP+JJ+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [TO+VB+IN,2] ||| [GOAL,1] [TO+VB+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+SBAR+VP,2] ||| [GOAL,1] [NN+SBAR+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+ADJP+NN,2] ||| [GOAL,1] [NN+ADJP+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP+VBZ+CD,2] ||| [GOAL,1] [ADJP+VBZ+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [.+NP+COMMA,2] ||| [GOAL,1] [.+NP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+IN+VBG,2] ||| [GOAL,1] [NN+IN+VBG,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+TO+CC,2] ||| [GOAL,1] [JJ+TO+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+''+NP,2] ||| [GOAL,1] [COMMA+''+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+WRB+NP,2] ||| [GOAL,1] [NP+WRB+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN\VP,2] ||| [GOAL,1] [IN\VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+NN+VBD,2] ||| [GOAL,1] [PP+NN+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+DT+NNS,2] ||| [GOAL,1] [NN+DT+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+IN+IN,2] ||| [GOAL,1] [NN+IN+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+.+``,2] ||| [GOAL,1] [NP+.+``,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+NN+NNS,2] ||| [GOAL,1] [SYM+NN+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP/SYM,2] ||| [GOAL,1] [NP/SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+COMMA+PP,2] ||| [GOAL,1] [PP+COMMA+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [WRB+NNS+IN,2] ||| [GOAL,1] [WRB+NNS+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+VBZ+CD,2] ||| [GOAL,1] [JJ+VBZ+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+RB,2] ||| [GOAL,1] [NN+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+ADJP+.,2] ||| [GOAL,1] [RB+ADJP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+JJ+JJ,2] ||| [GOAL,1] [VBN+JJ+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+NNS+COMMA,2] ||| [GOAL,1] [NN+NNS+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBG+TO+DT,2] ||| [GOAL,1] [VBG+TO+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+PP+ADVP,2] ||| [GOAL,1] [VBD+PP+ADVP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+COMMA,2] ||| [GOAL,1] [VB+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+TO+DT,2] ||| [GOAL,1] [JJ+TO+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+JJ+RBS,2] ||| [GOAL,1] [JJ+JJ+RBS,2] ||| -1
+[GOAL] ||| [GOAL,1] [MD\S,2] ||| [GOAL,1] [MD\S,2] ||| -1
+[GOAL] ||| [GOAL,1] [QP+NNS,2] ||| [GOAL,1] [QP+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+QP,2] ||| [GOAL,1] [NN+QP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VBN+NNS,2] ||| [GOAL,1] [COMMA+VBN+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+OOV+CC,2] ||| [GOAL,1] [NN+OOV+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+NP+VP,2] ||| [GOAL,1] [RB+NP+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+QP+JJ,2] ||| [GOAL,1] [PP+QP+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+RB+JJ,2] ||| [GOAL,1] [RB+RB+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+SBAR+RB,2] ||| [GOAL,1] [COMMA+SBAR+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+NN+CC,2] ||| [GOAL,1] [SYM+NN+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+NN+CD,2] ||| [GOAL,1] [SYM+NN+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+IN+CD,2] ||| [GOAL,1] [NNS+IN+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [WRB+COMMA+SYM,2] ||| [GOAL,1] [WRB+COMMA+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [.+CD,2] ||| [GOAL,1] [.+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+''+.,2] ||| [GOAL,1] [JJ+''+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NX+JJ,2] ||| [GOAL,1] [CC+NX+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+FW,2] ||| [GOAL,1] [CD+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+COMMA+VBZ,2] ||| [GOAL,1] [JJ+COMMA+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+IN+DT,2] ||| [GOAL,1] [NN+IN+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN\QP,2] ||| [GOAL,1] [IN\QP,2] ||| -1
+[GOAL] ||| [GOAL,1] [X/FW,2] ||| [GOAL,1] [X/FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+''+NP,2] ||| [GOAL,1] [:+''+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+WDT,2] ||| [GOAL,1] [IN+WDT,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+:+CC,2] ||| [GOAL,1] [CD+:+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NN+WRB,2] ||| [GOAL,1] [JJ+NN+WRB,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR+COMMA+IN,2] ||| [GOAL,1] [SBAR+COMMA+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBN+NN,2] ||| [GOAL,1] [NP+VBN+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+IN+DT,2] ||| [GOAL,1] [NNS+IN+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBN+NP,2] ||| [GOAL,1] [NP+VBN+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+JJ,2] ||| [GOAL,1] [CC+NP+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJR\NP,2] ||| [GOAL,1] [JJR\NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+IN,2] ||| [GOAL,1] [CC+NP+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+VBP+RB,2] ||| [GOAL,1] [PP+VBP+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NX+IN,2] ||| [GOAL,1] [CC+NX+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [VP/NNS,2] ||| [GOAL,1] [VP/NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+RB+IN,2] ||| [GOAL,1] [RB+RB+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+COMMA+JJ,2] ||| [GOAL,1] [PP+COMMA+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+VBZ+DT,2] ||| [GOAL,1] [JJ+VBZ+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+QP+JJ,2] ||| [GOAL,1] [RB+QP+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+RB+SYM,2] ||| [GOAL,1] [JJ+RB+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+ADVP+COMMA,2] ||| [GOAL,1] [COMMA+ADVP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [CONJP+NP,2] ||| [GOAL,1] [CONJP+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN\PP,2] ||| [GOAL,1] [IN\PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+SYM+NP,2] ||| [GOAL,1] [:+SYM+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [.+CD+NN,2] ||| [GOAL,1] [.+CD+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+NNS+COMMA,2] ||| [GOAL,1] [NNS+NNS+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+ADJP,2] ||| [GOAL,1] [SYM+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+COMMA+NX,2] ||| [GOAL,1] [JJ+COMMA+NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [VP+PRP+NN,2] ||| [GOAL,1] [VP+PRP+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+SBAR+TO,2] ||| [GOAL,1] [COMMA+SBAR+TO,2] ||| -1
+[GOAL] ||| [GOAL,1] [TO+VB+NP,2] ||| [GOAL,1] [TO+VB+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NNS,2] ||| [GOAL,1] [JJ+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR+COMMA+JJ,2] ||| [GOAL,1] [SBAR+COMMA+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+OOV+NNS,2] ||| [GOAL,1] [COMMA+OOV+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [$+PRN+PRP,2] ||| [GOAL,1] [$+PRN+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+DT+NNS,2] ||| [GOAL,1] [CC+DT+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+COMMA+NP,2] ||| [GOAL,1] [JJ+COMMA+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN\VP,2] ||| [GOAL,1] [NN\VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+COMMA+NN,2] ||| [GOAL,1] [JJ+COMMA+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NNP,2] ||| [GOAL,1] [JJ+NNP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VP+VBZ,2] ||| [GOAL,1] [COMMA+VP+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+NP+ADJP,2] ||| [GOAL,1] [IN+NP+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+VP+.,2] ||| [GOAL,1] [ADVP+VP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+WRB+NN,2] ||| [GOAL,1] [JJ+WRB+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+PP+SYM,2] ||| [GOAL,1] [NN+PP+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VP+VBP,2] ||| [GOAL,1] [COMMA+VP+VBP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+PRP+NNS,2] ||| [GOAL,1] [COMMA+PRP+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+SYM+QP,2] ||| [GOAL,1] [:+SYM+QP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNP+SYM+JJ,2] ||| [GOAL,1] [NNP+SYM+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [TO+VB+NN,2] ||| [GOAL,1] [TO+VB+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NX+COMMA+NNS,2] ||| [GOAL,1] [NX+COMMA+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VB+CC,2] ||| [GOAL,1] [NN+VB+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+JJ+.,2] ||| [GOAL,1] [RB+JJ+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+IN+FW,2] ||| [GOAL,1] [NN+IN+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+COMMA+OOV,2] ||| [GOAL,1] [NN+COMMA+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VP+VBD,2] ||| [GOAL,1] [COMMA+VP+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+:+IN,2] ||| [GOAL,1] [JJ+:+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+PP+IN,2] ||| [GOAL,1] [PP+PP+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+VBZ+WRB,2] ||| [GOAL,1] [DT+VBZ+WRB,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBN+PP,2] ||| [GOAL,1] [NP+VBN+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+X,2] ||| [GOAL,1] [CC+NP+X,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+VP+COMMA,2] ||| [GOAL,1] [VBP+VP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+:+DT,2] ||| [GOAL,1] [CD+:+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+CD+COMMA,2] ||| [GOAL,1] [NN+CD+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+PP,2] ||| [GOAL,1] [NN+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+S+COMMA,2] ||| [GOAL,1] [IN+S+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [LS+SYM+JJ,2] ||| [GOAL,1] [LS+SYM+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+WRB+JJ,2] ||| [GOAL,1] [NP+WRB+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+NNS+COMMA,2] ||| [GOAL,1] [IN+NNS+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+''+JJ,2] ||| [GOAL,1] [COMMA+''+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+:+JJ,2] ||| [GOAL,1] [JJ+:+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [``+SYM+SYM,2] ||| [GOAL,1] [``+SYM+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+RB,2] ||| [GOAL,1] [NP+VBD+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP\NX,2] ||| [GOAL,1] [ADJP\NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+PRP,2] ||| [GOAL,1] [NNS+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [WHPP+NP+VBZ,2] ||| [GOAL,1] [WHPP+NP+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+X+.,2] ||| [GOAL,1] [SYM+X+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+NNS+NP,2] ||| [GOAL,1] [CD+NNS+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VP,2] ||| [GOAL,1] [NN+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+PRN,2] ||| [GOAL,1] [NNS+PRN,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+NNS+NN,2] ||| [GOAL,1] [CD+NNS+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+OOV+SYM,2] ||| [GOAL,1] [CC+OOV+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP\NP,2] ||| [GOAL,1] [ADJP\NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NNS+COMMA,2] ||| [GOAL,1] [CC+NNS+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+''+RB,2] ||| [GOAL,1] [:+''+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+VBG+NNS,2] ||| [GOAL,1] [IN+VBG+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+SYM+RB,2] ||| [GOAL,1] [:+SYM+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [MD+NP+VP,2] ||| [GOAL,1] [MD+NP+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP+JJR,2] ||| [GOAL,1] [ADJP+JJR,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+VBN+SYM,2] ||| [GOAL,1] [RB+VBN+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+CC+VBG,2] ||| [GOAL,1] [NP+CC+VBG,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+DT+NP,2] ||| [GOAL,1] [IN+DT+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+WHNP+PP,2] ||| [GOAL,1] [NN+WHNP+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+DT+NN,2] ||| [GOAL,1] [IN+DT+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+NP+VBP,2] ||| [GOAL,1] [CD+NP+VBP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+NNS,2] ||| [GOAL,1] [CD+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+COMMA,2] ||| [GOAL,1] [CC+NP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [S+WDT+PRP,2] ||| [GOAL,1] [S+WDT+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+MD+CD,2] ||| [GOAL,1] [NP+MD+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [WHPP+NP,2] ||| [GOAL,1] [WHPP+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+CC+VBD,2] ||| [GOAL,1] [NP+CC+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [WHPP+NN,2] ||| [GOAL,1] [WHPP+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+VBN+COMMA,2] ||| [GOAL,1] [CC+VBN+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+JJ+VBN,2] ||| [GOAL,1] [NN+JJ+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+PDT+DT,2] ||| [GOAL,1] [RB+PDT+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [X/JJ,2] ||| [GOAL,1] [X/JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+VP+.,2] ||| [GOAL,1] [VB+VP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+CC+VBZ,2] ||| [GOAL,1] [NP+CC+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+WP$,2] ||| [GOAL,1] [COMMA+WP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+COMMA+PRP,2] ||| [GOAL,1] [ADVP+COMMA+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [MD+VBZ,2] ||| [GOAL,1] [MD+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [X\S,2] ||| [GOAL,1] [X\S,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+CC+VBP,2] ||| [GOAL,1] [NP+CC+VBP,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR+COMMA+DT,2] ||| [GOAL,1] [SBAR+COMMA+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [WDT+S+.,2] ||| [GOAL,1] [WDT+S+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+MD+IN,2] ||| [GOAL,1] [CC+MD+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+VBN+RB,2] ||| [GOAL,1] [VBD+VBN+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+:+DT,2] ||| [GOAL,1] [PP+:+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBZ+NP+RBS,2] ||| [GOAL,1] [VBZ+NP+RBS,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VB,2] ||| [GOAL,1] [NN+VB,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+VBP+TO,2] ||| [GOAL,1] [PRP+VBP+TO,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+TO+CD,2] ||| [GOAL,1] [CD+TO+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+QP,2] ||| [GOAL,1] [NP+VBD+QP,2] ||| -1
+[GOAL] ||| [GOAL,1] [X\X,2] ||| [GOAL,1] [X\X,2] ||| -1
+[GOAL] ||| [GOAL,1] [S/SBAR,2] ||| [GOAL,1] [S/SBAR,2] ||| -1
+[GOAL] ||| [GOAL,1] [.+ADJP,2] ||| [GOAL,1] [.+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+FW+CC,2] ||| [GOAL,1] [SYM+FW+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+''+QP,2] ||| [GOAL,1] [:+''+QP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+JJ+NN,2] ||| [GOAL,1] [VBN+JJ+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+NN+.,2] ||| [GOAL,1] [DT+NN+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+IN+JJ,2] ||| [GOAL,1] [NNS+IN+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+:+NN,2] ||| [GOAL,1] [JJ+:+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR+PRP,2] ||| [GOAL,1] [SBAR+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+PP+VP,2] ||| [GOAL,1] [PP+PP+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+ADJP+VP,2] ||| [GOAL,1] [NN+ADJP+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+WHADVP,2] ||| [GOAL,1] [COMMA+WHADVP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+:+NP,2] ||| [GOAL,1] [JJ+:+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP$+VP,2] ||| [GOAL,1] [PRP$+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+TO,2] ||| [GOAL,1] [NN+TO,2] ||| -1
+[GOAL] ||| [GOAL,1] [TO+VB+QP,2] ||| [GOAL,1] [TO+VB+QP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+WRB,2] ||| [GOAL,1] [COMMA+WRB,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NN+COMMA,2] ||| [GOAL,1] [CC+NN+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+VBP,2] ||| [GOAL,1] [CC+NP+VBP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+VBN,2] ||| [GOAL,1] [CC+NP+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP/VP,2] ||| [GOAL,1] [PP/VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NX+NX+COMMA,2] ||| [GOAL,1] [NX+NX+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+NP+VBD,2] ||| [GOAL,1] [NNS+NP+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP/SBAR,2] ||| [GOAL,1] [ADJP/SBAR,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP+S+.,2] ||| [GOAL,1] [ADJP+S+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+NP+ADJP,2] ||| [GOAL,1] [COMMA+NP+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP+IN+NP,2] ||| [GOAL,1] [ADJP+IN+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+PP,2] ||| [GOAL,1] [NP+VBD+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+CD,2] ||| [GOAL,1] [CD+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+CC,2] ||| [GOAL,1] [CD+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB\VP,2] ||| [GOAL,1] [VB\VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+VBZ,2] ||| [GOAL,1] [CC+NP+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+VBZ+RB,2] ||| [GOAL,1] [PRP+VBZ+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [POS+JJ+CC,2] ||| [GOAL,1] [POS+JJ+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+CC+JJR,2] ||| [GOAL,1] [JJ+CC+JJR,2] ||| -1
+[GOAL] ||| [GOAL,1] [OOV+FW+.,2] ||| [GOAL,1] [OOV+FW+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+JJ+COMMA,2] ||| [GOAL,1] [VBN+JJ+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+DT,2] ||| [GOAL,1] [CD+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBZ+CD+NN,2] ||| [GOAL,1] [VBZ+CD+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+VBN+TO,2] ||| [GOAL,1] [VBD+VBN+TO,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+ADJP+COMMA,2] ||| [GOAL,1] [PP+ADJP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+VBD,2] ||| [GOAL,1] [CC+NP+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+:+FW,2] ||| [GOAL,1] [PP+:+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+VBG,2] ||| [GOAL,1] [CC+NP+VBG,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+OOV,2] ||| [GOAL,1] [COMMA+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+ADVP+.,2] ||| [GOAL,1] [JJ+ADVP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VBP+JJ,2] ||| [GOAL,1] [NN+VBP+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+NP+PRP,2] ||| [GOAL,1] [COMMA+NP+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NN+PRP$,2] ||| [GOAL,1] [JJ+NN+PRP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+NX+COMMA,2] ||| [GOAL,1] [NP+NX+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+VBP+:,2] ||| [GOAL,1] [DT+VBP+:,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+DT+RBS,2] ||| [GOAL,1] [COMMA+DT+RBS,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+NNS+.,2] ||| [GOAL,1] [CD+NNS+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+S+COMMA,2] ||| [GOAL,1] [:+S+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [WDT+PRP+VBD,2] ||| [GOAL,1] [WDT+PRP+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+VB,2] ||| [GOAL,1] [NP+VBD+VB,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+VP,2] ||| [GOAL,1] [NP+VBD+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [WHPP+JJ,2] ||| [GOAL,1] [WHPP+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+DT+JJ,2] ||| [GOAL,1] [IN+DT+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+POS,2] ||| [GOAL,1] [NNS+POS,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+FW+FW,2] ||| [GOAL,1] [SYM+FW+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [X/NP,2] ||| [GOAL,1] [X/NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+CC+PRP,2] ||| [GOAL,1] [COMMA+CC+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [X/NN,2] ||| [GOAL,1] [X/NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+:+VB,2] ||| [GOAL,1] [VBP+:+VB,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+VP+.,2] ||| [GOAL,1] [DT+VP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+SYM+WP,2] ||| [GOAL,1] [:+SYM+WP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VBZ+NP,2] ||| [GOAL,1] [COMMA+VBZ+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+RB+CC,2] ||| [GOAL,1] [NNS+RB+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [NX+NX+OOV,2] ||| [GOAL,1] [NX+NX+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+WHPP+CD,2] ||| [GOAL,1] [RB+WHPP+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+NNS+VBZ,2] ||| [GOAL,1] [IN+NNS+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+OOV,2] ||| [GOAL,1] [NN+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+NNS+RB,2] ||| [GOAL,1] [CD+NNS+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+PP+RB,2] ||| [GOAL,1] [PP+PP+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+VBZ+.,2] ||| [GOAL,1] [SYM+VBZ+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+OOV+JJ,2] ||| [GOAL,1] [NN+OOV+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS\S,2] ||| [GOAL,1] [NNS\S,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+CC+COMMA,2] ||| [GOAL,1] [NP+CC+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+JJ+VBN,2] ||| [GOAL,1] [DT+JJ+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+RB+PRP,2] ||| [GOAL,1] [NN+RB+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+IN+NN,2] ||| [GOAL,1] [NNS+IN+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+WP$+JJ,2] ||| [GOAL,1] [NNS+WP$+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN\NX,2] ||| [GOAL,1] [NN\NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+OOV,2] ||| [GOAL,1] [CC+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [WRB+QP+IN,2] ||| [GOAL,1] [WRB+QP+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBZ+:+NN,2] ||| [GOAL,1] [VBZ+:+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR+COMMA+CC,2] ||| [GOAL,1] [SBAR+COMMA+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+CONJP+DT,2] ||| [GOAL,1] [NN+CONJP+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+RB+NP,2] ||| [GOAL,1] [SYM+RB+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+COMMA+RB,2] ||| [GOAL,1] [VBN+COMMA+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+IN+NP,2] ||| [GOAL,1] [NNS+IN+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+TO,2] ||| [GOAL,1] [NP+VBD+TO,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+OOV,2] ||| [GOAL,1] [VBD+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+JJ+OOV,2] ||| [GOAL,1] [:+JJ+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJS+NNS,2] ||| [GOAL,1] [JJS+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+COMMA+'',2] ||| [GOAL,1] [NN+COMMA+'',2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+JJ+VP,2] ||| [GOAL,1] [NNS+JJ+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+NN+COMMA,2] ||| [GOAL,1] [NNS+NN+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+JJ+VBD,2] ||| [GOAL,1] [NN+JJ+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+TO+NP,2] ||| [GOAL,1] [JJ+TO+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VB+ADVP,2] ||| [GOAL,1] [COMMA+VB+ADVP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+WP,2] ||| [GOAL,1] [NN+WP,2] ||| -1
+[GOAL] ||| [GOAL,1] [FW+ADJP,2] ||| [GOAL,1] [FW+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VBP+NN,2] ||| [GOAL,1] [NN+VBP+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [VP+PRP+VP,2] ||| [GOAL,1] [VP+PRP+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN\NP,2] ||| [GOAL,1] [NN\NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [LST+JJ,2] ||| [GOAL,1] [LST+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VBP+NP,2] ||| [GOAL,1] [NN+VBP+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+WRB,2] ||| [GOAL,1] [ADVP+WRB,2] ||| -1
+[GOAL] ||| [GOAL,1] [.+PRP+RB,2] ||| [GOAL,1] [.+PRP+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [WHADVP,2] ||| [GOAL,1] [WHADVP,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR+RB+DT,2] ||| [GOAL,1] [SBAR+RB+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [NX+.,2] ||| [GOAL,1] [NX+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+JJ,2] ||| [GOAL,1] [NP+VBD+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+VP,2] ||| [GOAL,1] [CC+NP+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VP+RB+DT,2] ||| [GOAL,1] [VP+RB+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+NNS,2] ||| [GOAL,1] [SYM+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+VBP+RBR,2] ||| [GOAL,1] [NNS+VBP+RBR,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+QP+NNS,2] ||| [GOAL,1] [IN+QP+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [MD+VB+NNP,2] ||| [GOAL,1] [MD+VB+NNP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VBP+CC,2] ||| [GOAL,1] [NN+VBP+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VBP+CD,2] ||| [GOAL,1] [NN+VBP+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+VP+COMMA,2] ||| [GOAL,1] [VBD+VP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [S\FRAG,2] ||| [GOAL,1] [S\FRAG,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+OOV+.,2] ||| [GOAL,1] [CC+OOV+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [NX+:,2] ||| [GOAL,1] [NX+:,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+X+SYM,2] ||| [GOAL,1] [SYM+X+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+IN+JJ,2] ||| [GOAL,1] [VBN+IN+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+OOV+CC,2] ||| [GOAL,1] [VBP+OOV+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP/PP,2] ||| [GOAL,1] [PP/PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+IN,2] ||| [GOAL,1] [NP+VBD+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+COMMA+SBAR,2] ||| [GOAL,1] [NNS+COMMA+SBAR,2] ||| -1
+[GOAL] ||| [GOAL,1] [X/RB,2] ||| [GOAL,1] [X/RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+IN+S,2] ||| [GOAL,1] [VBN+IN+S,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+CC+PRP,2] ||| [GOAL,1] [NN+CC+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+NN+NNS,2] ||| [GOAL,1] [IN+NN+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [OOV,2] ||| [GOAL,1] [OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRN+PRP+NNP,2] ||| [GOAL,1] [PRN+PRP+NNP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NX+''+NX,2] ||| [GOAL,1] [NX+''+NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+JJ+COMMA,2] ||| [GOAL,1] [COMMA+JJ+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+NNS+VP,2] ||| [GOAL,1] [CD+NNS+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+:+JJ,2] ||| [GOAL,1] [VBP+:+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP+RB+ADJP,2] ||| [GOAL,1] [ADJP+RB+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+PRP$+RBS,2] ||| [GOAL,1] [IN+PRP$+RBS,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+WRB,2] ||| [GOAL,1] [IN+WRB,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+VBN+IN,2] ||| [GOAL,1] [VBD+VBN+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [FW+VP+CC,2] ||| [GOAL,1] [FW+VP+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+JJ+COMMA,2] ||| [GOAL,1] [VB+JJ+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+VBN+JJ,2] ||| [GOAL,1] [VBD+VBN+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [PDT+PRP$,2] ||| [GOAL,1] [PDT+PRP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+WRB+DT,2] ||| [GOAL,1] [COMMA+WRB+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NNS+WDT,2] ||| [GOAL,1] [JJ+NNS+WDT,2] ||| -1
+[GOAL] ||| [GOAL,1] [NX+''+NN,2] ||| [GOAL,1] [NX+''+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+JJ+COMMA,2] ||| [GOAL,1] [CC+JJ+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [WHNP\S,2] ||| [GOAL,1] [WHNP\S,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+VBZ+IN,2] ||| [GOAL,1] [PRP+VBZ+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP/NP,2] ||| [GOAL,1] [PP/NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [''+COMMA+CC,2] ||| [GOAL,1] [''+COMMA+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [RP+IN+NP,2] ||| [GOAL,1] [RP+IN+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+VBZ+COMMA,2] ||| [GOAL,1] [DT+VBZ+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [.+NN+VP,2] ||| [GOAL,1] [.+NN+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP/NX,2] ||| [GOAL,1] [PP/NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+JJ+IN,2] ||| [GOAL,1] [NNS+JJ+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP$+NN,2] ||| [GOAL,1] [PRP$+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP$+NP,2] ||| [GOAL,1] [PRP$+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+IN+PRP$,2] ||| [GOAL,1] [CD+IN+PRP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+NN+WRB,2] ||| [GOAL,1] [IN+NN+WRB,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB\NP,2] ||| [GOAL,1] [VB\NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VB+JJS,2] ||| [GOAL,1] [COMMA+VB+JJS,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+:+NN,2] ||| [GOAL,1] [PP+:+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+PP+VBZ,2] ||| [GOAL,1] [CC+PP+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+:+NP,2] ||| [GOAL,1] [PP+:+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+OOV+NN,2] ||| [GOAL,1] [NN+OOV+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNP+COMMA,2] ||| [GOAL,1] [NNP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP/NN,2] ||| [GOAL,1] [PP/NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+CC+'',2] ||| [GOAL,1] [:+CC+'',2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+OOV+NP,2] ||| [GOAL,1] [NN+OOV+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+PP+VBP,2] ||| [GOAL,1] [CC+PP+VBP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+VBZ+JJ,2] ||| [GOAL,1] [PRP+VBZ+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+ADJP+.,2] ||| [GOAL,1] [COMMA+ADJP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+COMMA+NP,2] ||| [GOAL,1] [VBN+COMMA+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [RBS+NN,2] ||| [GOAL,1] [RBS+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+MD+CD,2] ||| [GOAL,1] [JJ+MD+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+JJ+JJ,2] ||| [GOAL,1] [NNS+JJ+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+RB+SYM,2] ||| [GOAL,1] [CD+RB+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+RB+IN,2] ||| [GOAL,1] [NNS+RB+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+RB+WDT,2] ||| [GOAL,1] [COMMA+RB+WDT,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+VP,2] ||| [GOAL,1] [ADVP+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+WRB+NP,2] ||| [GOAL,1] [NNS+WRB+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [POS+JJ+JJ,2] ||| [GOAL,1] [POS+JJ+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP\S,2] ||| [GOAL,1] [ADVP\S,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+CC+S,2] ||| [GOAL,1] [COMMA+CC+S,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+PRP$,2] ||| [GOAL,1] [RB+PRP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+VBP+IN,2] ||| [GOAL,1] [PRP+VBP+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [X/PP,2] ||| [GOAL,1] [X/PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+FW,2] ||| [GOAL,1] [NP+VBD+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NNS+POS,2] ||| [GOAL,1] [CC+NNS+POS,2] ||| -1
+[GOAL] ||| [GOAL,1] [OOV+COMMA+JJ,2] ||| [GOAL,1] [OOV+COMMA+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+WRB,2] ||| [GOAL,1] [CC+WRB,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+RB+JJ,2] ||| [GOAL,1] [NNS+RB+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+RB+VBD,2] ||| [GOAL,1] [CC+RB+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP/CD,2] ||| [GOAL,1] [ADJP/CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [S/SYM,2] ||| [GOAL,1] [S/SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+NP,2] ||| [GOAL,1] [NP+VBD+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+NN,2] ||| [GOAL,1] [NP+VBD+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+SYM+ADJP,2] ||| [GOAL,1] [NP+SYM+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+CC+ADJP,2] ||| [GOAL,1] [NNS+CC+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+PRP+VBD,2] ||| [GOAL,1] [PP+PRP+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+RB+VBZ,2] ||| [GOAL,1] [CC+RB+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VBP+VBN,2] ||| [GOAL,1] [NN+VBP+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [INTJ\FRAG,2] ||| [GOAL,1] [INTJ\FRAG,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+NNS+COMMA,2] ||| [GOAL,1] [VBP+NNS+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+RB+VBP,2] ||| [GOAL,1] [CC+RB+VBP,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+DT+RB,2] ||| [GOAL,1] [IN+DT+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+IN+NNS,2] ||| [GOAL,1] [NP+IN+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+IN+WRB,2] ||| [GOAL,1] [VB+IN+WRB,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+RB+VBN,2] ||| [GOAL,1] [CC+RB+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+:+NP,2] ||| [GOAL,1] [VBP+:+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+:+NN,2] ||| [GOAL,1] [VBP+:+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+:+PRP$,2] ||| [GOAL,1] [VBP+:+PRP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [''+RBS,2] ||| [GOAL,1] [''+RBS,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP,2] ||| [GOAL,1] [ADVP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+IN+DT,2] ||| [GOAL,1] [VBN+IN+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+PP,2] ||| [GOAL,1] [ADVP+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+PRP+VBZ,2] ||| [GOAL,1] [PP+PRP+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+JJ+VBN,2] ||| [GOAL,1] [IN+JJ+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+ADJP+ADVP,2] ||| [GOAL,1] [DT+ADJP+ADVP,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR+VBZ+VBG,2] ||| [GOAL,1] [SBAR+VBZ+VBG,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+VBN+IN,2] ||| [GOAL,1] [CC+VBN+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [WRB+SYM+CC,2] ||| [GOAL,1] [WRB+SYM+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBZ+CD+TO,2] ||| [GOAL,1] [VBZ+CD+TO,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP/POS,2] ||| [GOAL,1] [PP/POS,2] ||| -1
+[GOAL] ||| [GOAL,1] [RP+NP+.,2] ||| [GOAL,1] [RP+NP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP/DT,2] ||| [GOAL,1] [ADJP/DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+VBN+NP,2] ||| [GOAL,1] [VBD+VBN+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [''+COMMA+FW,2] ||| [GOAL,1] [''+COMMA+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [X/VP,2] ||| [GOAL,1] [X/VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+ADVP+COMMA,2] ||| [GOAL,1] [NN+ADVP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBG+NP+COMMA,2] ||| [GOAL,1] [VBG+NP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+COMMA+IN,2] ||| [GOAL,1] [VB+COMMA+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VBD+SYM,2] ||| [GOAL,1] [NN+VBD+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+NP+DT,2] ||| [GOAL,1] [NP+NP+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [WHPP+PRP,2] ||| [GOAL,1] [WHPP+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NN+WHPP,2] ||| [GOAL,1] [JJ+NN+WHPP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJR+SYM,2] ||| [GOAL,1] [JJR+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+VBZ+NP,2] ||| [GOAL,1] [PRP+VBZ+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VBP+DT,2] ||| [GOAL,1] [NN+VBP+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+RB,2] ||| [GOAL,1] [ADVP+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+RB+COMMA,2] ||| [GOAL,1] [SYM+RB+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+VBZ+NN,2] ||| [GOAL,1] [PRP+VBZ+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [VP+:+SYM,2] ||| [GOAL,1] [VP+:+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+ADVP,2] ||| [GOAL,1] [CC+ADVP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+VBN+QP,2] ||| [GOAL,1] [VBD+VBN+QP,2] ||| -1
+[GOAL] ||| [GOAL,1] [POS+WDT,2] ||| [GOAL,1] [POS+WDT,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP+VBD+NP,2] ||| [GOAL,1] [ADJP+VBD+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+JJ+SYM,2] ||| [GOAL,1] [SYM+JJ+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+VP+CC,2] ||| [GOAL,1] [PRP+VP+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB\X,2] ||| [GOAL,1] [RB\X,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP/RB,2] ||| [GOAL,1] [PP/RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP+ADJP,2] ||| [GOAL,1] [ADJP+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+JJ+NX,2] ||| [GOAL,1] [NNS+JJ+NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+VP+:,2] ||| [GOAL,1] [:+VP+:,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+WRB+NN,2] ||| [GOAL,1] [VBN+WRB+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [POS+JJ+NN,2] ||| [GOAL,1] [POS+JJ+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+NN+COMMA,2] ||| [GOAL,1] [RB+NN+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB\S,2] ||| [GOAL,1] [RB\S,2] ||| -1
+[GOAL] ||| [GOAL,1] [OOV+COMMA+NX,2] ||| [GOAL,1] [OOV+COMMA+NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+JJ+NP,2] ||| [GOAL,1] [NNS+JJ+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+JJ+NN,2] ||| [GOAL,1] [NNS+JJ+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP/PRP,2] ||| [GOAL,1] [PP/PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+JJR+.,2] ||| [GOAL,1] [NN+JJR+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP/QP,2] ||| [GOAL,1] [PP/QP,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+VP+.,2] ||| [GOAL,1] [:+VP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+OOV,2] ||| [GOAL,1] [IN+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+NP+CC,2] ||| [GOAL,1] [NP+NP+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+:+PRP$,2] ||| [GOAL,1] [PP+:+PRP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBZ+:+VB,2] ||| [GOAL,1] [VBZ+:+VB,2] ||| -1
+[GOAL] ||| [GOAL,1] [WRB+PRP+VP,2] ||| [GOAL,1] [WRB+PRP+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+VBN+PP,2] ||| [GOAL,1] [VBD+VBN+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+WRB+FW,2] ||| [GOAL,1] [COMMA+WRB+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+WDT+VBN,2] ||| [GOAL,1] [COMMA+WDT+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+COMMA,2] ||| [GOAL,1] [NP+VBD+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+VBZ+RB,2] ||| [GOAL,1] [CC+VBZ+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+WDT+VBP,2] ||| [GOAL,1] [COMMA+WDT+VBP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+CD+IN,2] ||| [GOAL,1] [NN+CD+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+CD+NNS,2] ||| [GOAL,1] [RB+CD+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+NNS+CC,2] ||| [GOAL,1] [IN+NNS+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+OOV+JJ,2] ||| [GOAL,1] [VBP+OOV+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+CC+PP,2] ||| [GOAL,1] [:+CC+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+NP,2] ||| [GOAL,1] [ADVP+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+SYM+SYM,2] ||| [GOAL,1] [NN+SYM+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [FW+NN+COMMA,2] ||| [GOAL,1] [FW+NN+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+JJ+VBN,2] ||| [GOAL,1] [CC+JJ+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+JJ+VBP,2] ||| [GOAL,1] [CC+JJ+VBP,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+NN,2] ||| [GOAL,1] [ADVP+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+NN+COMMA,2] ||| [GOAL,1] [NN+NN+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+WHPP+NP,2] ||| [GOAL,1] [NP+WHPP+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+JJ+VBD,2] ||| [GOAL,1] [NP+JJ+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+ADVP,2] ||| [GOAL,1] [NP+ADVP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+WDT+VBZ,2] ||| [GOAL,1] [COMMA+WDT+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+CD+JJ,2] ||| [GOAL,1] [NN+CD+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NX+.,2] ||| [GOAL,1] [CC+NX+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+TO,2] ||| [GOAL,1] [VB+TO,2] ||| -1
+[GOAL] ||| [GOAL,1] [VP+RB+NP,2] ||| [GOAL,1] [VP+RB+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+FW+JJ,2] ||| [GOAL,1] [RB+FW+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+JJ+VBG,2] ||| [GOAL,1] [CC+JJ+VBG,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+JJ+VBD,2] ||| [GOAL,1] [CC+JJ+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [MD+VBZ+DT,2] ||| [GOAL,1] [MD+VBZ+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR+TO,2] ||| [GOAL,1] [SBAR+TO,2] ||| -1
+[GOAL] ||| [GOAL,1] [FW+X+NP,2] ||| [GOAL,1] [FW+X+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+WDT+VBD,2] ||| [GOAL,1] [COMMA+WDT+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP+NNS+CC,2] ||| [GOAL,1] [ADJP+NNS+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+RB+NP,2] ||| [GOAL,1] [CC+RB+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+WRB+JJ,2] ||| [GOAL,1] [COMMA+WRB+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+JJ+JJ,2] ||| [GOAL,1] [:+JJ+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+VBD+ADJP,2] ||| [GOAL,1] [PRP+VBD+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+RB+NN,2] ||| [GOAL,1] [CC+RB+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+$+.,2] ||| [GOAL,1] [PRP+$+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+WDT,2] ||| [GOAL,1] [CD+WDT,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+'',2] ||| [GOAL,1] [:+'',2] ||| -1
+[GOAL] ||| [GOAL,1] [:+CC+NP,2] ||| [GOAL,1] [:+CC+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+NP+PRP,2] ||| [GOAL,1] [IN+NP+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+SYM+JJ,2] ||| [GOAL,1] [RB+SYM+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN\S,2] ||| [GOAL,1] [NN\S,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+IN+PRP,2] ||| [GOAL,1] [COMMA+IN+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+PRP$+JJ,2] ||| [GOAL,1] [VBD+PRP$+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+''+JJ,2] ||| [GOAL,1] [CC+''+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+S+COMMA,2] ||| [GOAL,1] [COMMA+S+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [FRAG/NN,2] ||| [GOAL,1] [FRAG/NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN\X,2] ||| [GOAL,1] [NN\X,2] ||| -1
+[GOAL] ||| [GOAL,1] [FRAG/NP,2] ||| [GOAL,1] [FRAG/NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NNS+NP,2] ||| [GOAL,1] [CC+NNS+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+''+RBS,2] ||| [GOAL,1] [IN+''+RBS,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNP+VBZ,2] ||| [GOAL,1] [NNP+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+JJ+NNS,2] ||| [GOAL,1] [DT+JJ+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NNS+NN,2] ||| [GOAL,1] [CC+NNS+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+WHNP,2] ||| [GOAL,1] [NNS+WHNP,2] ||| -1
+[GOAL] ||| [GOAL,1] [EX+VP,2] ||| [GOAL,1] [EX+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VP\SINV,2] ||| [GOAL,1] [VP\SINV,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+VBZ+OOV,2] ||| [GOAL,1] [NNS+VBZ+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+COMMA+NP,2] ||| [GOAL,1] [VB+COMMA+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+RP+NP,2] ||| [GOAL,1] [VBD+RP+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+RB+COMMA,2] ||| [GOAL,1] [:+RB+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+NN+VBD,2] ||| [GOAL,1] [COMMA+NN+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+VP,2] ||| [GOAL,1] [VB+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+NP+CC,2] ||| [GOAL,1] [RB+NP+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NNS+VP,2] ||| [GOAL,1] [JJ+NNS+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+CC+PRP$,2] ||| [GOAL,1] [VBN+CC+PRP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+NN+VBG,2] ||| [GOAL,1] [COMMA+NN+VBG,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBZ+NNS,2] ||| [GOAL,1] [NP+VBZ+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR+VP,2] ||| [GOAL,1] [SBAR+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [''+JJ+NNS,2] ||| [GOAL,1] [''+JJ+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+NN+VBN,2] ||| [GOAL,1] [COMMA+NN+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJS+NN+PP,2] ||| [GOAL,1] [JJS+NN+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+WP$+JJ,2] ||| [GOAL,1] [COMMA+WP$+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+NN+VBP,2] ||| [GOAL,1] [COMMA+NN+VBP,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+MD+CD,2] ||| [GOAL,1] [IN+MD+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+DT+RBS,2] ||| [GOAL,1] [VBP+DT+RBS,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+CD+COMMA,2] ||| [GOAL,1] [CC+CD+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VB+CC,2] ||| [GOAL,1] [COMMA+VB+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+NN+VBZ,2] ||| [GOAL,1] [COMMA+NN+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+SBAR+CC,2] ||| [GOAL,1] [VBN+SBAR+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [WHADJP+VBZ,2] ||| [GOAL,1] [WHADJP+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NP+JJR,2] ||| [GOAL,1] [JJ+NP+JJR,2] ||| -1
+[GOAL] ||| [GOAL,1] [FW+NP+VBD,2] ||| [GOAL,1] [FW+NP+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP$+NN+COMMA,2] ||| [GOAL,1] [PRP$+NN+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP$+NN+CC,2] ||| [GOAL,1] [PRP$+NN+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+NNS+CC,2] ||| [GOAL,1] [CD+NNS+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+VBG+DT,2] ||| [GOAL,1] [VBP+VBG+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+NP+NP,2] ||| [GOAL,1] [SYM+NP+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+CC+RB,2] ||| [GOAL,1] [DT+CC+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+NN+COMMA,2] ||| [GOAL,1] [COMMA+NN+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+CD,2] ||| [GOAL,1] [JJ+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+PRP,2] ||| [GOAL,1] [DT+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+NP,2] ||| [GOAL,1] [VB+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+CD,2] ||| [GOAL,1] [NP+VBD+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+JJR+JJ,2] ||| [GOAL,1] [COMMA+JJR+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NX+COMMA,2] ||| [GOAL,1] [JJ+NX+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+RB+PRP,2] ||| [GOAL,1] [COMMA+RB+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+OOV+COMMA,2] ||| [GOAL,1] [JJ+OOV+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+''+DT,2] ||| [GOAL,1] [CC+''+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+VBZ,2] ||| [GOAL,1] [JJ+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+CC,2] ||| [GOAL,1] [JJ+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+CD+NN,2] ||| [GOAL,1] [NN+CD+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+EX+VP,2] ||| [GOAL,1] [NNS+EX+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+WHPP,2] ||| [GOAL,1] [NNS+WHPP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NNS+PP,2] ||| [GOAL,1] [CC+NNS+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+JJ,2] ||| [GOAL,1] [ADVP+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+WP$+JJ,2] ||| [GOAL,1] [NP+WP$+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+NP+WP$,2] ||| [GOAL,1] [VBP+NP+WP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+VBN,2] ||| [GOAL,1] [JJ+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+IN,2] ||| [GOAL,1] [ADVP+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+VBG+PRP,2] ||| [GOAL,1] [CC+VBG+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+RB+RB,2] ||| [GOAL,1] [CC+RB+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+DT,2] ||| [GOAL,1] [JJ+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+VBG,2] ||| [GOAL,1] [JJ+VBG,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+PP+COMMA,2] ||| [GOAL,1] [COMMA+PP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] ['',2] ||| [GOAL,1] ['',2] ||| -1
+[GOAL] ||| [GOAL,1] [VP+SYM,2] ||| [GOAL,1] [VP+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [VP/FRAG,2] ||| [GOAL,1] [VP/FRAG,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NNS+RB,2] ||| [GOAL,1] [CC+NNS+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+PP,2] ||| [GOAL,1] [VB+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+VBP,2] ||| [GOAL,1] [JJ+VBP,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+DT+ADJP,2] ||| [GOAL,1] [IN+DT+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+WRB+NN,2] ||| [GOAL,1] [COMMA+WRB+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+WRB+NP,2] ||| [GOAL,1] [COMMA+WRB+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+NNS+DT,2] ||| [GOAL,1] [CD+NNS+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+NX+COMMA,2] ||| [GOAL,1] [:+NX+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+WHNP+VBD,2] ||| [GOAL,1] [NN+WHNP+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+VBD,2] ||| [GOAL,1] [JJ+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+VBD+VBN,2] ||| [GOAL,1] [PRP+VBD+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VBG+NP,2] ||| [GOAL,1] [COMMA+VBG+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+MD+VB,2] ||| [GOAL,1] [NN+MD+VB,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+JJ+DT,2] ||| [GOAL,1] [:+JJ+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [FRAG/JJ,2] ||| [GOAL,1] [FRAG/JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+DT,2] ||| [GOAL,1] [NP+VBD+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBZ+NP+SBAR,2] ||| [GOAL,1] [VBZ+NP+SBAR,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+JJ+ADVP,2] ||| [GOAL,1] [COMMA+JJ+ADVP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+CD+VBZ,2] ||| [GOAL,1] [JJ+CD+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+NN+COMMA,2] ||| [GOAL,1] [SYM+NN+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+NP+IN,2] ||| [GOAL,1] [NP+NP+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+ADJP+CC,2] ||| [GOAL,1] [DT+ADJP+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+NNS,2] ||| [GOAL,1] [DT+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+WRB+QP,2] ||| [GOAL,1] [COMMA+WRB+QP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+RB+VB,2] ||| [GOAL,1] [CC+RB+VB,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+WHPP,2] ||| [GOAL,1] [NP+VBD+WHPP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+VBP+JJ,2] ||| [GOAL,1] [JJ+VBP+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+CD+COMMA,2] ||| [GOAL,1] [IN+CD+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NNS+TO,2] ||| [GOAL,1] [CC+NNS+TO,2] ||| -1
+[GOAL] ||| [GOAL,1] [WHNP+VBZ+NP,2] ||| [GOAL,1] [WHNP+VBZ+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+JJ+PRP$,2] ||| [GOAL,1] [NNS+JJ+PRP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+FW,2] ||| [GOAL,1] [JJ+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VBP+.,2] ||| [GOAL,1] [NN+VBP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBG+OOV+NN,2] ||| [GOAL,1] [VBG+OOV+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+NP+JJ,2] ||| [GOAL,1] [NP+NP+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [POS+NNS,2] ||| [GOAL,1] [POS+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VBG+COMMA,2] ||| [GOAL,1] [COMMA+VBG+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+PP+NN,2] ||| [GOAL,1] [NNS+PP+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+SYM+NP,2] ||| [GOAL,1] [RB+SYM+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [EX+.,2] ||| [GOAL,1] [EX+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [FW+NNS+.,2] ||| [GOAL,1] [FW+NNS+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+PP+NP,2] ||| [GOAL,1] [NNS+PP+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+DT+S,2] ||| [GOAL,1] [COMMA+DT+S,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+RB,2] ||| [GOAL,1] [VB+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+VBZ+VBN,2] ||| [GOAL,1] [NNS+VBZ+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+OOV+OOV,2] ||| [GOAL,1] [SYM+OOV+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP/VB,2] ||| [GOAL,1] [NP/VB,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+VBZ+IN,2] ||| [GOAL,1] [CC+VBZ+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+MD+VB,2] ||| [GOAL,1] [SYM+MD+VB,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+EX,2] ||| [GOAL,1] [ADVP+EX,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VBG+CC,2] ||| [GOAL,1] [COMMA+VBG+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+POS+JJS,2] ||| [GOAL,1] [NN+POS+JJS,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+PRP$,2] ||| [GOAL,1] [NN+PRP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+RP+IN,2] ||| [GOAL,1] [VBN+RP+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NNS+DT,2] ||| [GOAL,1] [CC+NNS+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJS+NN+IN,2] ||| [GOAL,1] [JJS+NN+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+.,2] ||| [GOAL,1] [ADVP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+SYM+.,2] ||| [GOAL,1] [NN+SYM+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+JJ+COMMA,2] ||| [GOAL,1] [RB+JJ+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+:,2] ||| [GOAL,1] [ADVP+:,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+PRP+DT,2] ||| [GOAL,1] [VBD+PRP+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+S+VBG,2] ||| [GOAL,1] [:+S+VBG,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+JJ+TO,2] ||| [GOAL,1] [IN+JJ+TO,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+ADJP,2] ||| [GOAL,1] [CC+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [S\UCP,2] ||| [GOAL,1] [S\UCP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VBG+JJ,2] ||| [GOAL,1] [NN+VBG+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN\FRAG,2] ||| [GOAL,1] [NN\FRAG,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB\SINV,2] ||| [GOAL,1] [RB\SINV,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+ADJP+JJ,2] ||| [GOAL,1] [DT+ADJP+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [FRAG+VP+JJ,2] ||| [GOAL,1] [FRAG+VP+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [S+COMMA+JJ,2] ||| [GOAL,1] [S+COMMA+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+.+COMMA,2] ||| [GOAL,1] [NN+.+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+ADVP+DT,2] ||| [GOAL,1] [COMMA+ADVP+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+:+'',2] ||| [GOAL,1] [VBD+:+'',2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+RB+VP,2] ||| [GOAL,1] [CC+RB+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+NP+PP,2] ||| [GOAL,1] [NP+NP+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBZ+VBG+NN,2] ||| [GOAL,1] [VBZ+VBG+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [FW+VBP+DT,2] ||| [GOAL,1] [FW+VBP+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBZ+VBG+NP,2] ||| [GOAL,1] [VBZ+VBG+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+JJR+CC,2] ||| [GOAL,1] [RB+JJR+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP/VP,2] ||| [GOAL,1] [NP/VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+X+:,2] ||| [GOAL,1] [COMMA+X+:,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJS+NN+JJ,2] ||| [GOAL,1] [JJS+NN+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBZ+NP+JJS,2] ||| [GOAL,1] [VBZ+NP+JJS,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+PRP,2] ||| [GOAL,1] [PRP+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VBG+IN,2] ||| [GOAL,1] [NN+VBG+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBG+WP$+JJ,2] ||| [GOAL,1] [VBG+WP$+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+NP+VP,2] ||| [GOAL,1] [DT+NP+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+MD+IN,2] ||| [GOAL,1] [NN+MD+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VP+.,2] ||| [GOAL,1] [NN+VP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+VBZ+NP,2] ||| [GOAL,1] [CD+VBZ+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+SYM+S,2] ||| [GOAL,1] [NN+SYM+S,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+VBZ+.,2] ||| [GOAL,1] [JJ+VBZ+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NNS+NN,2] ||| [GOAL,1] [JJ+NNS+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJR+IN+CD,2] ||| [GOAL,1] [JJR+IN+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NNS+NNS,2] ||| [GOAL,1] [JJ+NNS+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NNS+NP,2] ||| [GOAL,1] [JJ+NNS+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+DT,2] ||| [GOAL,1] [ADVP+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+IN,2] ||| [GOAL,1] [JJ+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [FRAG/VP,2] ||| [GOAL,1] [FRAG/VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+COMMA+VB,2] ||| [GOAL,1] [VB+COMMA+VB,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+PRP+NN,2] ||| [GOAL,1] [CD+PRP+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+DT+DT,2] ||| [GOAL,1] [IN+DT+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+RB+WRB,2] ||| [GOAL,1] [CC+RB+WRB,2] ||| -1
+[GOAL] ||| [GOAL,1] [''+CD+NN,2] ||| [GOAL,1] [''+CD+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VP+:,2] ||| [GOAL,1] [NN+VP+:,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+FW+.,2] ||| [GOAL,1] [COMMA+FW+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+CC,2] ||| [GOAL,1] [ADVP+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP/SINV,2] ||| [GOAL,1] [NP/SINV,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+VBZ+JJ,2] ||| [GOAL,1] [CC+VBZ+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+SYM+:,2] ||| [GOAL,1] [NN+SYM+:,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNP+:+NX,2] ||| [GOAL,1] [NNP+:+NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [WHNP+VBD,2] ||| [GOAL,1] [WHNP+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+DT+VBN,2] ||| [GOAL,1] [IN+DT+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [WHNP+VBP,2] ||| [GOAL,1] [WHNP+VBP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+JJ,2] ||| [GOAL,1] [JJ+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+NNS+VBZ,2] ||| [GOAL,1] [VBN+NNS+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+DT+VBP,2] ||| [GOAL,1] [IN+DT+VBP,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+JJ+PP,2] ||| [GOAL,1] [:+JJ+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+NP+SYM,2] ||| [GOAL,1] [PP+NP+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+NN,2] ||| [GOAL,1] [VB+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP\S,2] ||| [GOAL,1] [PP\S,2] ||| -1
+[GOAL] ||| [GOAL,1] [WHNP+VBZ,2] ||| [GOAL,1] [WHNP+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR+NP,2] ||| [GOAL,1] [SBAR+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+''+CC,2] ||| [GOAL,1] [VB+''+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [NX+JJ+JJ,2] ||| [GOAL,1] [NX+JJ+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+NP+NP,2] ||| [GOAL,1] [NP+NP+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+WDT,2] ||| [GOAL,1] [CC+WDT,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+NP+NN,2] ||| [GOAL,1] [NP+NP+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+RP+DT,2] ||| [GOAL,1] [VBD+RP+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [.+``,2] ||| [GOAL,1] [.+``,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBG+VBN,2] ||| [GOAL,1] [VBG+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP\X,2] ||| [GOAL,1] [PP\X,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+CD+DT,2] ||| [GOAL,1] [NN+CD+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+DT+VBZ,2] ||| [GOAL,1] [IN+DT+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [WP+NNS,2] ||| [GOAL,1] [WP+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [TO+PRP$,2] ||| [GOAL,1] [TO+PRP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB\FRAG,2] ||| [GOAL,1] [RB\FRAG,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+IN+'',2] ||| [GOAL,1] [NN+IN+'',2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+DT+CD,2] ||| [GOAL,1] [IN+DT+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NNS+PP,2] ||| [GOAL,1] [JJ+NNS+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+NNS+CC,2] ||| [GOAL,1] [RB+NNS+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+CC+NNS,2] ||| [GOAL,1] [NP+CC+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [VP+CC+WHPP,2] ||| [GOAL,1] [VP+CC+WHPP,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+FW+FW,2] ||| [GOAL,1] [RB+FW+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+MD+X,2] ||| [GOAL,1] [NN+MD+X,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR/NNP,2] ||| [GOAL,1] [SBAR/NNP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+WDT,2] ||| [GOAL,1] [COMMA+WDT,2] ||| -1
+[GOAL] ||| [GOAL,1] [FRAG/PP,2] ||| [GOAL,1] [FRAG/PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR/NNS,2] ||| [GOAL,1] [SBAR/NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+WHPP,2] ||| [GOAL,1] [NN+WHPP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NNS+WP$,2] ||| [GOAL,1] [JJ+NNS+WP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [S+PRP,2] ||| [GOAL,1] [S+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJS+CC+RB,2] ||| [GOAL,1] [JJS+CC+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+ADJP+NN,2] ||| [GOAL,1] [DT+ADJP+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+JJ+RB,2] ||| [GOAL,1] [IN+JJ+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+NN+NNS,2] ||| [GOAL,1] [COMMA+NN+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+JJ+NN,2] ||| [GOAL,1] [:+JJ+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NNS+IN,2] ||| [GOAL,1] [CC+NNS+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [LST+NP+:,2] ||| [GOAL,1] [LST+NP+:,2] ||| -1
+[GOAL] ||| [GOAL,1] [S+COMMA+NN,2] ||| [GOAL,1] [S+COMMA+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+DT+OOV,2] ||| [GOAL,1] [CC+DT+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+COMMA+PRP,2] ||| [GOAL,1] [NN+COMMA+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+NX+VBN,2] ||| [GOAL,1] [NN+NX+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NX+COMMA+OOV,2] ||| [GOAL,1] [NX+COMMA+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJS+NN+NN,2] ||| [GOAL,1] [JJS+NN+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+:+PRP,2] ||| [GOAL,1] [NN+:+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR+IN+SYM,2] ||| [GOAL,1] [SBAR+IN+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [S+COMMA+NP,2] ||| [GOAL,1] [S+COMMA+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP/S,2] ||| [GOAL,1] [ADVP/S,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VBZ+DT,2] ||| [GOAL,1] [COMMA+VBZ+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+:+PRP$,2] ||| [GOAL,1] [NP+:+PRP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBZ+JJ+COMMA,2] ||| [GOAL,1] [VBZ+JJ+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NX+:,2] ||| [GOAL,1] [CC+NX+:,2] ||| -1
+[GOAL] ||| [GOAL,1] [''+ADJP,2] ||| [GOAL,1] [''+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+MD,2] ||| [GOAL,1] [JJ+MD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+ADJP,2] ||| [GOAL,1] [NN+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR+RB,2] ||| [GOAL,1] [SBAR+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+COMMA+S,2] ||| [GOAL,1] [PP+COMMA+S,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NNS+JJ,2] ||| [GOAL,1] [CC+NNS+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [S+COMMA+RB,2] ||| [GOAL,1] [S+COMMA+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+''+NP,2] ||| [GOAL,1] [CC+''+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PDT,2] ||| [GOAL,1] [PDT,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+NNS+IN,2] ||| [GOAL,1] [CD+NNS+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+JJ+NP,2] ||| [GOAL,1] [IN+JJ+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+JJ+NN,2] ||| [GOAL,1] [IN+JJ+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+VBG+DT,2] ||| [GOAL,1] [NN+VBG+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+JJ+NX,2] ||| [GOAL,1] [IN+JJ+NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+COMMA+WDT,2] ||| [GOAL,1] [JJ+COMMA+WDT,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+WHNP,2] ||| [GOAL,1] [NN+WHNP,2] ||| -1
+[GOAL] ||| [GOAL,1] [S+COMMA+PP,2] ||| [GOAL,1] [S+COMMA+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+IN,2] ||| [GOAL,1] [VB+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+NN+VBD,2] ||| [GOAL,1] [CD+NN+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NN,2] ||| [GOAL,1] [JJ+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NP,2] ||| [GOAL,1] [JJ+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+NP+VP,2] ||| [GOAL,1] [SYM+NP+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+PP+NNS,2] ||| [GOAL,1] [CC+PP+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+PP+'',2] ||| [GOAL,1] [PP+PP+'',2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+OOV,2] ||| [GOAL,1] [JJ+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+IN+PRP,2] ||| [GOAL,1] [:+IN+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+VBP+PRP$,2] ||| [GOAL,1] [NNS+VBP+PRP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+VBZ+NP,2] ||| [GOAL,1] [CC+VBZ+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [WRB+RB+SYM,2] ||| [GOAL,1] [WRB+RB+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP+SYM,2] ||| [GOAL,1] [ADJP+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NNS+RB,2] ||| [GOAL,1] [JJ+NNS+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+NP+COMMA,2] ||| [GOAL,1] [COMMA+NP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+NNS+JJ,2] ||| [GOAL,1] [CD+NNS+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [FW+NN+SYM,2] ||| [GOAL,1] [FW+NN+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VBG+DT,2] ||| [GOAL,1] [COMMA+VBG+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+JJ,2] ||| [GOAL,1] [VB+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+NP+WDT,2] ||| [GOAL,1] [VBP+NP+WDT,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+NN+CC,2] ||| [GOAL,1] [ADVP+NN+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [NX+JJ+NN,2] ||| [GOAL,1] [NX+JJ+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NX,2] ||| [GOAL,1] [JJ+NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [MD+CD+:,2] ||| [GOAL,1] [MD+CD+:,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+PP+VP,2] ||| [GOAL,1] [NNS+PP+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [POS+NNS+CC,2] ||| [GOAL,1] [POS+NNS+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+FW+SBAR,2] ||| [GOAL,1] [SYM+FW+SBAR,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN\NP,2] ||| [GOAL,1] [IN\NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+WDT+VBZ,2] ||| [GOAL,1] [NN+WDT+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP/NN,2] ||| [GOAL,1] [NP/NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP/NP,2] ||| [GOAL,1] [NP/NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+NN+VB,2] ||| [GOAL,1] [NNS+NN+VB,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+SBAR+COMMA,2] ||| [GOAL,1] [PP+SBAR+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+WHPP,2] ||| [GOAL,1] [VBN+WHPP,2] ||| -1
+[GOAL] ||| [GOAL,1] [OOV+SYM+CC,2] ||| [GOAL,1] [OOV+SYM+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+CC+VP,2] ||| [GOAL,1] [COMMA+CC+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [S+VBZ+VBG,2] ||| [GOAL,1] [S+VBZ+VBG,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+CC+PRP,2] ||| [GOAL,1] [PP+CC+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+NNS+COMMA,2] ||| [GOAL,1] [VBN+NNS+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+COMMA,2] ||| [GOAL,1] [VBN+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP/S,2] ||| [GOAL,1] [NP/S,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+COMMA+SBAR,2] ||| [GOAL,1] [NN+COMMA+SBAR,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+WDT+VBD,2] ||| [GOAL,1] [NN+WDT+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+PP,2] ||| [GOAL,1] [JJ+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [QP+MD,2] ||| [GOAL,1] [QP+MD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP/X,2] ||| [GOAL,1] [NP/X,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+EX+VP,2] ||| [GOAL,1] [PP+EX+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NNS+CC,2] ||| [GOAL,1] [JJ+NNS+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+SYM+SYM,2] ||| [GOAL,1] [NNS+SYM+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+OOV+SYM,2] ||| [GOAL,1] [NN+OOV+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP/NX,2] ||| [GOAL,1] [NP/NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+NNS,2] ||| [GOAL,1] [COMMA+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+JJ+JJ,2] ||| [GOAL,1] [CD+JJ+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+NNP,2] ||| [GOAL,1] [COMMA+NNP,2] ||| -1
+[GOAL] ||| [GOAL,1] [WRB+NP+VBD,2] ||| [GOAL,1] [WRB+NP+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+ADJP+.,2] ||| [GOAL,1] [:+ADJP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NNS+DT,2] ||| [GOAL,1] [JJ+NNS+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [WRB+NP+VBP,2] ||| [GOAL,1] [WRB+NP+VBP,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+DT+MD,2] ||| [GOAL,1] [:+DT+MD,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+NP+WP$,2] ||| [GOAL,1] [COMMA+NP+WP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+:+OOV,2] ||| [GOAL,1] [IN+:+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+S+.,2] ||| [GOAL,1] [NP+S+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+JJ+TO,2] ||| [GOAL,1] [NP+JJ+TO,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+FW+.,2] ||| [GOAL,1] [CD+FW+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [QP+NN,2] ||| [GOAL,1] [QP+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+JJS,2] ||| [GOAL,1] [NP+JJS,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+JJ+MD,2] ||| [GOAL,1] [IN+JJ+MD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+''+VBD,2] ||| [GOAL,1] [NN+''+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+MD+CD,2] ||| [GOAL,1] [NN+MD+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+JJR,2] ||| [GOAL,1] [NP+JJR,2] ||| -1
+[GOAL] ||| [GOAL,1] [S+COMMA+CC,2] ||| [GOAL,1] [S+COMMA+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+NN+SBAR,2] ||| [GOAL,1] [NN+NN+SBAR,2] ||| -1
+[GOAL] ||| [GOAL,1] [S+COMMA+CD,2] ||| [GOAL,1] [S+COMMA+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NN+ADJP,2] ||| [GOAL,1] [JJ+NN+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+COMMA+JJ,2] ||| [GOAL,1] [JJ+COMMA+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+PP+SYM,2] ||| [GOAL,1] [NP+PP+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+:+X,2] ||| [GOAL,1] [PP+:+X,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP/PP,2] ||| [GOAL,1] [NP/PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [WP$\SBAR,2] ||| [GOAL,1] [WP$\SBAR,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+DT+JJ,2] ||| [GOAL,1] [:+DT+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+CC+SYM,2] ||| [GOAL,1] [SYM+CC+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [CONJP+JJ,2] ||| [GOAL,1] [CONJP+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+CD+ADJP,2] ||| [GOAL,1] [SYM+CD+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [S+COMMA+DT,2] ||| [GOAL,1] [S+COMMA+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJS+NNS+CC,2] ||| [GOAL,1] [JJS+NNS+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+POS+.,2] ||| [GOAL,1] [NNS+POS+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+IN+PRP$,2] ||| [GOAL,1] [COMMA+IN+PRP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+COMMA+IN,2] ||| [GOAL,1] [ADVP+COMMA+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+DT+IN,2] ||| [GOAL,1] [CC+DT+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [RP+DT+NN,2] ||| [GOAL,1] [RP+DT+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+VBD+NNS,2] ||| [GOAL,1] [PP+VBD+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+VBZ+DT,2] ||| [GOAL,1] [VBN+VBZ+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+COMMA+IN,2] ||| [GOAL,1] [JJ+COMMA+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+WRB+DT,2] ||| [GOAL,1] [NP+WRB+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR/NN,2] ||| [GOAL,1] [SBAR/NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+OOV+COMMA,2] ||| [GOAL,1] [COMMA+OOV+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+NN+.,2] ||| [GOAL,1] [PRP+NN+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+NP+SYM,2] ||| [GOAL,1] [NP+NP+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [FW+.+JJ,2] ||| [GOAL,1] [FW+.+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+JJ+JJ,2] ||| [GOAL,1] [IN+JJ+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NX+ADJP,2] ||| [GOAL,1] [NX+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [S+:+VBD,2] ||| [GOAL,1] [S+:+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+RB,2] ||| [GOAL,1] [JJ+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [QP+PP,2] ||| [GOAL,1] [QP+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+NP+NN,2] ||| [GOAL,1] [DT+NP+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+PP+COMMA,2] ||| [GOAL,1] [JJ+PP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+NP+WHNP,2] ||| [GOAL,1] [RB+NP+WHNP,2] ||| -1
+[GOAL] ||| [GOAL,1] [X+COMMA,2] ||| [GOAL,1] [X+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VB+TO,2] ||| [GOAL,1] [COMMA+VB+TO,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP/.,2] ||| [GOAL,1] [NP/.,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR/NX,2] ||| [GOAL,1] [SBAR/NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+OOV,2] ||| [GOAL,1] [CD+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+NP+VP,2] ||| [GOAL,1] [NP+NP+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP/:,2] ||| [GOAL,1] [NP/:,2] ||| -1
+[GOAL] ||| [GOAL,1] [WRB+LS+SYM,2] ||| [GOAL,1] [WRB+LS+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+NNS+CC,2] ||| [GOAL,1] [PP+NNS+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+DT+JJ,2] ||| [GOAL,1] [CC+DT+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR/NP,2] ||| [GOAL,1] [SBAR/NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+CC+JJ,2] ||| [GOAL,1] [VBP+CC+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+QP,2] ||| [GOAL,1] [JJ+QP,2] ||| -1
+[GOAL] ||| [GOAL,1] [QP+IN,2] ||| [GOAL,1] [QP+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [FW+SYM+CC,2] ||| [GOAL,1] [FW+SYM+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [TO+CD+NX,2] ||| [GOAL,1] [TO+CD+NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+CC+CD,2] ||| [GOAL,1] [:+CC+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [VP+:+S,2] ||| [GOAL,1] [VP+:+S,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+JJ+VB,2] ||| [GOAL,1] [:+JJ+VB,2] ||| -1
+[GOAL] ||| [GOAL,1] [.+SYM,2] ||| [GOAL,1] [.+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VB+VB,2] ||| [GOAL,1] [COMMA+VB+VB,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+NP+WP$,2] ||| [GOAL,1] [RB+NP+WP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+CC+OOV,2] ||| [GOAL,1] [NN+CC+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+NNS+VP,2] ||| [GOAL,1] [IN+NNS+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [OOV+OOV+SYM,2] ||| [GOAL,1] [OOV+OOV+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [TO+RB+VB,2] ||| [GOAL,1] [TO+RB+VB,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+PRP,2] ||| [GOAL,1] [NN+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+NAC,2] ||| [GOAL,1] [DT+NAC,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+NX+CC,2] ||| [GOAL,1] [PP+NX+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+PRN,2] ||| [GOAL,1] [NN+PRN,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+NP+RB,2] ||| [GOAL,1] [RB+NP+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+CD+COMMA,2] ||| [GOAL,1] [JJ+CD+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+TO,2] ||| [GOAL,1] [JJ+TO,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR+PRP$,2] ||| [GOAL,1] [SBAR+PRP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+COMMA+VB,2] ||| [GOAL,1] [PP+COMMA+VB,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+S+.,2] ||| [GOAL,1] [:+S+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+COMMA+FW,2] ||| [GOAL,1] [JJ+COMMA+FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBG+.,2] ||| [GOAL,1] [VBG+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [CONJP+IN,2] ||| [GOAL,1] [CONJP+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR/PP,2] ||| [GOAL,1] [SBAR/PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+OOV,2] ||| [GOAL,1] [NNS+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+WRB+NP,2] ||| [GOAL,1] [PP+WRB+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+SBAR,2] ||| [GOAL,1] [PP+SBAR,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+COMMA+VP,2] ||| [GOAL,1] [PP+COMMA+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [RP+PP,2] ||| [GOAL,1] [RP+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN\SINV,2] ||| [GOAL,1] [IN\SINV,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+NNS,2] ||| [GOAL,1] [VBD+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP+RB+.,2] ||| [GOAL,1] [ADJP+RB+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [TO+CD+NN,2] ||| [GOAL,1] [TO+CD+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP+NNS+VP,2] ||| [GOAL,1] [ADJP+NNS+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NNP,2] ||| [GOAL,1] [CC+NNP,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+JJ+NNS,2] ||| [GOAL,1] [:+JJ+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NP+CC,2] ||| [GOAL,1] [CC+NP+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+RB+CD,2] ||| [GOAL,1] [CC+RB+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+NN+VBZ,2] ||| [GOAL,1] [IN+NN+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJR+NN+VBN,2] ||| [GOAL,1] [JJR+NN+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [TO+VB+ADJP,2] ||| [GOAL,1] [TO+VB+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADVP+COMMA+JJ,2] ||| [GOAL,1] [ADVP+COMMA+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+''+COMMA,2] ||| [GOAL,1] [NN+''+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NNS,2] ||| [GOAL,1] [CC+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+FW+.,2] ||| [GOAL,1] [PP+FW+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [QP+JJ,2] ||| [GOAL,1] [QP+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP/RB,2] ||| [GOAL,1] [NP/RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP$+ADJP+NN,2] ||| [GOAL,1] [PRP$+ADJP+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+VBZ,2] ||| [GOAL,1] [NNS+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBZ+:,2] ||| [GOAL,1] [NP+VBZ+:,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+:,2] ||| [GOAL,1] [VB+:,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+NX+COMMA,2] ||| [GOAL,1] [DT+NX+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+VBD,2] ||| [GOAL,1] [NNS+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [.+PRP,2] ||| [GOAL,1] [.+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+VBG,2] ||| [GOAL,1] [NNS+VBG,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+VBP,2] ||| [GOAL,1] [NNS+VBP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP$+ADJP,2] ||| [GOAL,1] [PRP$+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP\NP,2] ||| [GOAL,1] [NP\NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBZ+.,2] ||| [GOAL,1] [NP+VBZ+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+.,2] ||| [GOAL,1] [VB+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR/RB,2] ||| [GOAL,1] [SBAR/RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+NNS+CC,2] ||| [GOAL,1] [CC+NNS+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+VP,2] ||| [GOAL,1] [JJ+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+S+CC,2] ||| [GOAL,1] [CC+S+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NNS+IN,2] ||| [GOAL,1] [JJ+NNS+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+NN+.,2] ||| [GOAL,1] [VBN+NN+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+VBN,2] ||| [GOAL,1] [NNS+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+COMMA+DT,2] ||| [GOAL,1] [JJ+COMMA+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [VP+VBP+DT,2] ||| [GOAL,1] [VP+VBP+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP\NX,2] ||| [GOAL,1] [NP\NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+RB+DT,2] ||| [GOAL,1] [CC+RB+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNP+:+CD,2] ||| [GOAL,1] [NNP+:+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+FW+.,2] ||| [GOAL,1] [NNS+FW+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+VB,2] ||| [GOAL,1] [JJ+VB,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBD+SYM,2] ||| [GOAL,1] [NP+VBD+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+RB+VBZ,2] ||| [GOAL,1] [PP+RB+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [TO+CD+PP,2] ||| [GOAL,1] [TO+CD+PP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+DT+NN,2] ||| [GOAL,1] [CC+DT+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+NNS+JJ,2] ||| [GOAL,1] [JJ+NNS+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+ADJP+COMMA,2] ||| [GOAL,1] [NNS+ADJP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+CC+OOV,2] ||| [GOAL,1] [COMMA+CC+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+DT+NP,2] ||| [GOAL,1] [CC+DT+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+CD+NNS,2] ||| [GOAL,1] [JJ+CD+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [RP+NP,2] ||| [GOAL,1] [RP+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+RB+VBD,2] ||| [GOAL,1] [PP+RB+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+'',2] ||| [GOAL,1] [RB+'',2] ||| -1
+[GOAL] ||| [GOAL,1] [NP/FW,2] ||| [GOAL,1] [NP/FW,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+NP+CC,2] ||| [GOAL,1] [DT+NP+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+SYM+SYM,2] ||| [GOAL,1] [CD+SYM+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+OOV+NN,2] ||| [GOAL,1] [RB+OOV+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+CC+NP,2] ||| [GOAL,1] [COMMA+CC+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+CC+NN,2] ||| [GOAL,1] [COMMA+CC+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR/CD,2] ||| [GOAL,1] [SBAR/CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+COMMA+CC,2] ||| [GOAL,1] [JJ+COMMA+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [QP+VBP,2] ||| [GOAL,1] [QP+VBP,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+CC+NX,2] ||| [GOAL,1] [COMMA+CC+NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [QP+COMMA+CD,2] ||| [GOAL,1] [QP+COMMA+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+NNS+IN,2] ||| [GOAL,1] [IN+NNS+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+COMMA+CD,2] ||| [GOAL,1] [JJ+COMMA+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+JJ+CD,2] ||| [GOAL,1] [CD+JJ+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [CD+JJ+CC,2] ||| [GOAL,1] [CD+JJ+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [''+JJ+NN,2] ||| [GOAL,1] [''+JJ+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+VBZ+NP,2] ||| [GOAL,1] [VBN+VBZ+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+PP+DT,2] ||| [GOAL,1] [:+PP+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [RP+NP+COMMA,2] ||| [GOAL,1] [RP+NP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [QP\S,2] ||| [GOAL,1] [QP\S,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBP+:+SYM,2] ||| [GOAL,1] [VBP+:+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+WRB+PRP,2] ||| [GOAL,1] [COMMA+WRB+PRP,2] ||| -1
+[GOAL] ||| [GOAL,1] [.+NN,2] ||| [GOAL,1] [.+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NX+NNS+NNS,2] ||| [GOAL,1] [NX+NNS+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [.+NP,2] ||| [GOAL,1] [.+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+NX+COMMA,2] ||| [GOAL,1] [NN+NX+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+WHPP+DT,2] ||| [GOAL,1] [NP+WHPP+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [FRAG/CD,2] ||| [GOAL,1] [FRAG/CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+NNS+IN,2] ||| [GOAL,1] [PP+NNS+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+NX+NN,2] ||| [GOAL,1] [PP+NX+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NX+CC+OOV,2] ||| [GOAL,1] [NX+CC+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+ADJP+.,2] ||| [GOAL,1] [NN+ADJP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [POS,2] ||| [GOAL,1] [POS,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+VB+JJ,2] ||| [GOAL,1] [COMMA+VB+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [PDT+DT,2] ||| [GOAL,1] [PDT+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [QP+VP,2] ||| [GOAL,1] [QP+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+NP+COMMA,2] ||| [GOAL,1] [:+NP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+JJ+DT,2] ||| [GOAL,1] [IN+JJ+DT,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+COMMA,2] ||| [GOAL,1] [CC+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+VBZ+RB,2] ||| [GOAL,1] [JJ+VBZ+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [.+MD,2] ||| [GOAL,1] [.+MD,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+NN+NN,2] ||| [GOAL,1] [NNS+NN+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+NP+JJ,2] ||| [GOAL,1] [SYM+NP+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBZ+PDT,2] ||| [GOAL,1] [VBZ+PDT,2] ||| -1
+[GOAL] ||| [GOAL,1] [NNS+NN+NX,2] ||| [GOAL,1] [NNS+NN+NX,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+CC+MD,2] ||| [GOAL,1] [COMMA+CC+MD,2] ||| -1
+[GOAL] ||| [GOAL,1] [FW+NP+.,2] ||| [GOAL,1] [FW+NP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [OOV+SYM+JJ,2] ||| [GOAL,1] [OOV+SYM+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [ADJP+NNS+IN,2] ||| [GOAL,1] [ADJP+NNS+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+JJ+ADJP,2] ||| [GOAL,1] [CC+JJ+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+:+OOV,2] ||| [GOAL,1] [NP+:+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+PRP$+JJ,2] ||| [GOAL,1] [CC+PRP$+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+JJ+UCP,2] ||| [GOAL,1] [IN+JJ+UCP,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+SYM+NP,2] ||| [GOAL,1] [DT+SYM+NP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+RBS,2] ||| [GOAL,1] [NP+RBS,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+JJ+NNP,2] ||| [GOAL,1] [NN+JJ+NNP,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+JJ+NNS,2] ||| [GOAL,1] [NN+JJ+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+NNS+POS,2] ||| [GOAL,1] [NN+NNS+POS,2] ||| -1
+[GOAL] ||| [GOAL,1] [VP+VBP+RB,2] ||| [GOAL,1] [VP+VBP+RB,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+JJ+CC,2] ||| [GOAL,1] [IN+JJ+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBG+NN+COMMA,2] ||| [GOAL,1] [VBG+NN+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+JJ+NN,2] ||| [GOAL,1] [NP+JJ+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+JJ+CD,2] ||| [GOAL,1] [IN+JJ+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP$+NN+JJ,2] ||| [GOAL,1] [PRP$+NN+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+NP+IN,2] ||| [GOAL,1] [SYM+NP+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [VB+DT+NN,2] ||| [GOAL,1] [VB+DT+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [JJ+COMMA+NNS,2] ||| [GOAL,1] [JJ+COMMA+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [S/RRC,2] ||| [GOAL,1] [S/RRC,2] ||| -1
+[GOAL] ||| [GOAL,1] [VP+SYM+ADJP,2] ||| [GOAL,1] [VP+SYM+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [CC+RB+IN,2] ||| [GOAL,1] [CC+RB+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+NP+JJ,2] ||| [GOAL,1] [RB+NP+JJ,2] ||| -1
+[GOAL] ||| [GOAL,1] [QP+VBZ,2] ||| [GOAL,1] [QP+VBZ,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+PRN+CC,2] ||| [GOAL,1] [NN+PRN+CC,2] ||| -1
+[GOAL] ||| [GOAL,1] [PRP+COMMA,2] ||| [GOAL,1] [PRP+COMMA,2] ||| -1
+[GOAL] ||| [GOAL,1] [:+DT+CD,2] ||| [GOAL,1] [:+DT+CD,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+PRP$,2] ||| [GOAL,1] [COMMA+PRP$,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+VBP+NNS,2] ||| [GOAL,1] [NP+VBP+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+JJ+ADJP,2] ||| [GOAL,1] [COMMA+JJ+ADJP,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBD+TO+SYM,2] ||| [GOAL,1] [VBD+TO+SYM,2] ||| -1
+[GOAL] ||| [GOAL,1] [WRB,2] ||| [GOAL,1] [WRB,2] ||| -1
+[GOAL] ||| [GOAL,1] [SYM+OOV,2] ||| [GOAL,1] [SYM+OOV,2] ||| -1
+[GOAL] ||| [GOAL,1] [SBAR/EX,2] ||| [GOAL,1] [SBAR/EX,2] ||| -1
+[GOAL] ||| [GOAL,1] [NN+ADVP,2] ||| [GOAL,1] [NN+ADVP,2] ||| -1
+[GOAL] ||| [GOAL,1] [X/.,2] ||| [GOAL,1] [X/.,2] ||| -1
+[GOAL] ||| [GOAL,1] [COMMA+JJ+VBN,2] ||| [GOAL,1] [COMMA+JJ+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+NNP,2] ||| [GOAL,1] [IN+NNP,2] ||| -1
+[GOAL] ||| [GOAL,1] [POS+VBD,2] ||| [GOAL,1] [POS+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+NP+IN,2] ||| [GOAL,1] [RB+NP+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+S,2] ||| [GOAL,1] [NP+S,2] ||| -1
+[GOAL] ||| [GOAL,1] [NP+X,2] ||| [GOAL,1] [NP+X,2] ||| -1
+[GOAL] ||| [GOAL,1] [IN+NNS,2] ||| [GOAL,1] [IN+NNS,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+JJR+NN,2] ||| [GOAL,1] [PP+JJR+NN,2] ||| -1
+[GOAL] ||| [GOAL,1] [POS+VBN,2] ||| [GOAL,1] [POS+VBN,2] ||| -1
+[GOAL] ||| [GOAL,1] [VBN+VP+.,2] ||| [GOAL,1] [VBN+VP+.,2] ||| -1
+[GOAL] ||| [GOAL,1] [DT+PRP+VBD,2] ||| [GOAL,1] [DT+PRP+VBD,2] ||| -1
+[GOAL] ||| [GOAL,1] [RB+EX+VP,2] ||| [GOAL,1] [RB+EX+VP,2] ||| -1
+[GOAL] ||| [GOAL,1] [WHNP+VBZ+TO,2] ||| [GOAL,1] [WHNP+VBZ+TO,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+VBN+IN,2] ||| [GOAL,1] [PP+VBN+IN,2] ||| -1
+[GOAL] ||| [GOAL,1] [PP+SYM+SYM,2] ||| [GOAL,1] [PP+SYM+SYM,2] ||| -1
+[GOAL] ||| [GOAL
<TRUNCATED>
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/samt/grammar.gz
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/samt/grammar.gz b/joshua-core/src/test/resources/bn-en/samt/grammar.gz
new file mode 100644
index 0000000..01a7d92
Binary files /dev/null and b/joshua-core/src/test/resources/bn-en/samt/grammar.gz differ
[04/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/packed/output.scores.gold
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/packed/output.scores.gold b/joshua-core/src/test/resources/bn-en/packed/output.scores.gold
new file mode 100644
index 0000000..fd63d12
--- /dev/null
+++ b/joshua-core/src/test/resources/bn-en/packed/output.scores.gold
@@ -0,0 +1,862 @@
+ rabindranath was born in kolkata is a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| -230.669
+ rabindranath was born in a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| -230.837
+ rabindranath was born in kolkata is one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| -231.593
+ rabindranath was born in kolkata in a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| -231.977
+ rabindranath was born in kolkata a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| -232.054
+ rabindranath was born in the a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| -232.473
+ in recent times of india with united states relationship between improved . ||| -26.695
+ in recent times india with united states relationship between improved . ||| -27.174
+ in recent times of india with the united states relationship between improved . ||| -27.329
+ in recent times of india with united states relation between improved . ||| -27.367
+ in recent times in india with united states relationship between improved . ||| -27.695
+ in recent times india with the united states relationship between improved . ||| -27.807
+ in recent times india with united states relation between improved . ||| -27.845
+ in recent times of india with the united states relation between improved . ||| -28.000
+ in recent times of india with united states relation improved . ||| -28.037
+ in recent times in india with united states relation between improved . ||| -28.367
+ mathematics is science language . ||| -12.270
+ mathematics so science language . ||| -12.290
+ mathematics hence science language . ||| -14.648
+ mathematics that is why science language . ||| -14.684
+ mathematics that science language . ||| -14.686
+ mathematics is science languages . ||| -14.843
+ mathematics so science languages . ||| -14.864
+ mathematics is reasons language . ||| -14.908
+ from this can easily it can be understood that this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf will be \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| -329.615
+ from this easily it can be understood that this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf will be \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| -329.869
+ from this can easily it can be understood that this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf would be \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| -330.539
+ from this can easily it can be understood that this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf will \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| -330.631
+ from this easily it can be understood that this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf would be \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| -330.793
+ from this can easily it can be understood that in this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf will be \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| -330.878
+ from this easily it can be understood that this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf will \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| -330.886
+ from this can easily it can be understood that these \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf will be \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| -331.119
+ from this easily it can be understood that in this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf will be \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| -331.132
+ from this can easily it can be understood that the \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf will be \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| -331.188
+ in the same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match from this novel . ||| -244.715
+ in the same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match this novel from . ||| -244.892
+ the same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match this novel from . ||| -245.063
+ on the same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match this novel from . ||| -245.074
+ in the same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 presage match this novel from . ||| -245.208
+ in the same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match in this novel from . ||| -245.411
+ in the same time with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match from this novel . ||| -245.823
+ in the same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match this novel to . ||| -245.993
+ in the same time with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match this novel from . ||| -246.000
+ in the same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match this novel from the . ||| -246.051
+ \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 majority in the . ||| -222.050
+ \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 majority in . ||| -223.836
+ \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 majority , in the . ||| -224.380
+ \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 majority in the goods . ||| -224.789
+ \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 seikh mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 majority in the . ||| -224.989
+ \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 majority in the . . ||| -224.991
+ \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 majority in the of . ||| -225.383
+ \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 majority in the , . ||| -225.396
+ \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his work with to that . ||| -439.395
+ \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his work with can and pajama . ||| -439.526
+ \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his work with to and pajama . ||| -440.155
+ \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his work with can that . ||| -440.188
+ \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 of his work with to that . ||| -440.260
+ \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his works with to that . ||| -440.280
+ \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 in his work with to that . ||| -440.347
+ \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 of his work with can and pajama . ||| -440.391
+ \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his works with can and pajama . ||| -440.410
+ \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 in his work with can and pajama . ||| -440.477
+ task apart from \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character is , but these are very is not . ||| -252.878
+ task apart from \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character there are , but these are very is not . ||| -252.999
+ task other than \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character is , but these are very is not . ||| -253.123
+ task apart from \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character are , but these are very is not . ||| -253.207
+ task other than \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character there are , but these are very is not . ||| -253.244
+ task apart from \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character there is , but these are very is not . ||| -253.351
+ task apart from \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some of linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character is , but these are very is not . ||| -253.526
+ task apart from \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some of linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character there is , but these are very is not . ||| -253.999
+ task apart from \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character is , but these are very common not . ||| -254.604
+ task apart from \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character there are , but these are very common not . ||| -254.725
+ it social situation in \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the vagina . ||| -264.205
+ it social situation in \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with the \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the vagina . ||| -264.435
+ it social situation in view \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the vagina . ||| -264.473
+ it social situation in view of \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the vagina . ||| -264.567
+ it social situation in view \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with the \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the vagina . ||| -264.703
+ this social situation in \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the vagina . ||| -264.755
+ it social situation in view of \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with the \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the vagina . ||| -264.796
+ it social situation in \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the in the . ||| -264.875
+ this social situation in \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with the \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the vagina . ||| -264.985
+ it social situation in \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rules of the vagina . ||| -264.988
+ \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 annual \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf meter ||| -324.551
+ \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 arrange \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf meter ||| -324.635
+ \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 annual \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf meters ||| -324.722
+ \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 arrange \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf meters ||| -324.806
+ \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 annual \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf metres ||| -326.046
+ \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 arrange \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf metres ||| -326.131
+ \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 annual \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf above ||| -327.121
+ \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 arrange \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf above ||| -327.206
+ in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in the city in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f as the national conference was he main speech -lrb- keynote speech -rrb- on the . ||| -675.013
+ in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in the city in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f as the national conference was he main speech -lrb- keynote speech -rrb- to the . ||| -675.262
+ in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in the city in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f the national conference was he main speech -lrb- keynote speech -rrb- on the . ||| -675.282
+ in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in the city in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f the national conference was he main speech -lrb- keynote speech -rrb- to the . ||| -675.530
+ in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in the city in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f as the national conference was he the speech -lrb- keynote speech -rrb- on the . ||| -675.766
+ in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in the city in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f as the national conference was he main speech -lrb- keynote speech -rrb- , the . ||| -675.800
+ in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in the city in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f party national conference was he main speech -lrb- keynote speech -rrb- on the . ||| -675.864
+ in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in the city in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f as the national conference was he the main speech -lrb- keynote speech -rrb- on the . ||| -675.940
+ in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in the city in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f as the national conference was he the speech -lrb- keynote speech -rrb- to the . ||| -676.015
+ in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in the city in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f the national conference was he the speech -lrb- keynote speech -rrb- on the . ||| -676.035
+ population on the power distribution of east pakistan where due to the west pakistan " one unit theory , " is a \u0985\u09ad\u09bf\u09a8\u09ac they started to the , where the whole of west pakistan is a province considered as the . ||| -199.719
+ population on the power distribution of east pakistan where due to the west pakistan " one unit theory , " is a \u0985\u09ad\u09bf\u09a8\u09ac they started to the , where the whole of west pakistan a province considered as the . ||| -199.743
+ population on the power distribution of east pakistan where due to the west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac they started to the , where the whole of west pakistan is a province considered as the . ||| -199.975
+ population on the power distribution of east pakistan where due to the west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac they started to the , where the whole of west pakistan a province considered as the . ||| -199.999
+ population on the power distribution of east pakistan where due to the west pakistan " one unit theory , " is a \u0985\u09ad\u09bf\u09a8\u09ac they started to the , where the west pakistan is a province considered as the . ||| -200.182
+ population on the power distribution of east pakistan where due to the west pakistan " one unit theory , " is a \u0985\u09ad\u09bf\u09a8\u09ac they started to the , where the west pakistan a province considered as the . ||| -200.206
+ population based on this distribution of east pakistan where due to the west pakistan " one unit theory , " is a \u0985\u09ad\u09bf\u09a8\u09ac they started to the , where the west pakistan is a province considered as the . ||| -200.259
+ population on the power distribution of east pakistan where due to the west pakistan , " one unit theory , " is a \u0985\u09ad\u09bf\u09a8\u09ac they started to the , where the whole of west pakistan is a province considered as the . ||| -200.280
+ population based on this distribution of east pakistan where due to the west pakistan " one unit theory , " is a \u0985\u09ad\u09bf\u09a8\u09ac they started to the , where the west pakistan a province considered as the . ||| -200.283
+ population on the power distribution of east pakistan where due to the west pakistan , " one unit theory , " is a \u0985\u09ad\u09bf\u09a8\u09ac they started to the , where the whole of west pakistan a province considered as the . ||| -200.304
+ the \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 theory ||| -113.022
+ mathematical \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 theory ||| -113.373
+ \u2022 \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 theory ||| -113.486
+ the \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 theory of ||| -115.266
+ the theory of \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 ||| -115.559
+ mathematical \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 theory of ||| -115.616
+ \u2022 \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 theory of ||| -115.729
+ the \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 is ||| -117.032
+ external links of ||| -6.764
+ relation with outside of ||| -7.040
+ external link of ||| -7.543
+ external communication of ||| -7.776
+ external connections of ||| -8.099
+ external connection of ||| -8.379
+ external link by ||| -8.600
+ external communication by ||| -8.928
+ external links by ||| -9.099
+ external connections by ||| -9.281
+ tata communication foreign this \u09a8\u09bf\u0997\u09ae limited building , it is the telecommunication system is a is one of the main providers ||| -159.669
+ tata communicationer foreign this \u09a8\u09bf\u0997\u09ae limited building , it is the telecommunication system is a is one of the main providers ||| -159.792
+ tata communication foreign this \u09a8\u09bf\u0997\u09ae limited building , it is the city telecommunication system is a is one of the main providers ||| -160.578
+ tata communication foreign this \u09a8\u09bf\u0997\u09ae limited building , it is in the telecommunication system is a is one of the main providers ||| -160.606
+ tata communicationer foreign this \u09a8\u09bf\u0997\u09ae limited building , it is the city telecommunication system is a is one of the main providers ||| -160.701
+ tata communication foreign this \u09a8\u09bf\u0997\u09ae limited building , it is the telecommunication system is a one of the main providers ||| -161.029
+ tata communication foreign this \u09a8\u09bf\u0997\u09ae limited building , it is in the city telecommunication system is a is one of the main providers ||| -161.034
+ tata communication foreign this \u09a8\u09bf\u0997\u09ae limited building , it is the telecommunication system of a is one of the main providers ||| -161.049
+ tata communicationer foreign this \u09a8\u09bf\u0997\u09ae limited building , it is the telecommunication system is a one of the main providers ||| -161.152
+ tata communicationer foreign this \u09a8\u09bf\u0997\u09ae limited building , it is in the city telecommunication system is a is one of the main providers ||| -161.157
+ he that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national election \u099c\u09af\u09bc\u09c0 in and was elected as the 44th president of the united states . ||| -342.060
+ he that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national election \u099c\u09af\u09bc\u09c0 in and the 44th president of the united states was elected as . ||| -342.499
+ he that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national election \u099c\u09af\u09bc\u09c0 in and the 44th president of the united states elected as . ||| -342.884
+ he that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national election \u099c\u09af\u09bc\u09c0 in the 44th president of the united states and was elected as . ||| -343.137
+ he that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national election \u099c\u09af\u09bc\u09c0 was and the 44th president of the united states was elected as . ||| -343.290
+ he in that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national election \u099c\u09af\u09bc\u09c0 in and the 44th president of the united states was elected as . ||| -343.354
+ he that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national election \u099c\u09af\u09bc\u09c0 to and the 44th president of the united states was elected as . ||| -343.597
+ he that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national election \u099c\u09af\u09bc\u09c0 was and was elected as the 44th president of the united states . ||| -343.600
+ he that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national election \u099c\u09af\u09bc\u09c0 the and the 44th president of the united states was elected as . ||| -343.619
+ he that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 of national election \u099c\u09af\u09bc\u09c0 in and was elected as the 44th president of the united states . ||| -343.622
+ many of the indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be from the upper stage ||| -337.568
+ many of the indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be from upper stage ||| -337.736
+ many indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be from the upper stage ||| -338.457
+ many of the \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be from the upper stage ||| -338.537
+ many indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be from upper stage ||| -338.625
+ many of the \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be from upper stage ||| -338.705
+ many of the indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be to upper stage ||| -338.832
+ many of indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be from upper stage ||| -339.032
+ many of the \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be to upper stage ||| -339.800
+ britain writers written drama , novels , stories and recently scripts in the \u0986\u09a6\u09c3\u09a4 . ||| -144.560
+ britain writers written drama , novels , stories and recently scripts in \u0986\u09a6\u09c3\u09a4 . ||| -144.892
+ britain writers written drama , novels , stories and recently script in the \u0986\u09a6\u09c3\u09a4 . ||| -144.986
+ britain writers written drama , novels , stories and in the recent scripts in the \u0986\u09a6\u09c3\u09a4 . ||| -145.290
+ britain writers of written drama , novels , stories and recently scripts in the \u0986\u09a6\u09c3\u09a4 . ||| -145.319
+ britain writers written drama , novels , stories and recently script in \u0986\u09a6\u09c3\u09a4 . ||| -145.319
+ britain writers of the drama , novels , stories and recently scripts in the \u0986\u09a6\u09c3\u09a4 . ||| -145.402
+ britain writers written drama , novels , stories and in the recent scripts in \u0986\u09a6\u09c3\u09a4 . ||| -145.623
+ britain writers of written drama , novels , stories and recently scripts in \u0986\u09a6\u09c3\u09a4 . ||| -145.651
+ britain writers written drama , novels , stories and in the recent script in the \u0986\u09a6\u09c3\u09a4 . ||| -145.717
+ 1919 , on may month it saogat magazine was published in the . ||| -29.082
+ 1919 on may month it saogat magazine was published in the . ||| -29.173
+ 1919 , on may month it saogat magazine was published . ||| -29.196
+ 1919 on may month it saogat magazine was published . ||| -29.287
+ 1919 , on may month it is saogat magazine was published in the . ||| -29.459
+ 1919 on may month it is saogat magazine was published in the . ||| -29.550
+ 1919 , on may month it is saogat magazine was published . ||| -29.572
+ 1919 on may month it is saogat magazine was published . ||| -29.663
+ 1919 , on may month it saogat magazine was published in . ||| -29.879
+ 1919 on may month it saogat magazine was published in . ||| -29.971
+ in 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium was arranged . ||| -460.093
+ 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium was arranged . ||| -460.244
+ in year 2005 \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium was arranged . ||| -460.557
+ in 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association of the tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium was arranged . ||| -460.712
+ 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association of the tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium was arranged . ||| -460.863
+ in 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium is the . ||| -461.115
+ 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium is the . ||| -461.266
+ in 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association of the tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium is the . ||| -461.734
+ in 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium arranged in the . ||| -461.872
+ 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association of the tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium is the . ||| -461.885
+ to prevent this several measures are taken . ||| -21.333
+ to prevent this several measures are in the . ||| -23.640
+ to prevent this several measures are the . ||| -23.669
+ to prevent this several measures are . ||| -23.707
+ to prevent this several measures are in . ||| -24.099
+ to avoid this possibility several measures are taken . ||| -24.504
+ \u09e7\u09ef\u09ec\u09ec on 5 february \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 of \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 a national was held in . ||| -342.312
+ \u09e7\u09ef\u09ec\u09ec on 5 february \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 opposition \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 a national was held in . ||| -342.432
+ \u09e7\u09ef\u09ec\u09ec on february 5 \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 of \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 a national was held in . ||| -342.506
+ \u09e7\u09ef\u09ec\u09ec on 5 february \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 against \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 a national was held in . ||| -342.592
+ \u09e7\u09ef\u09ec\u09ec on february 5 \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 opposition \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 a national was held in . ||| -342.627
+ \u09e7\u09ef\u09ec\u09ec on 5th february \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 of \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 a national was held in . ||| -342.668
+ \u09e7\u09ef\u09ec\u09ec on february 5 \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 against \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 a national was held in . ||| -342.787
+ \u09e7\u09ef\u09ec\u09ec on 5th february \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 against \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 a national was held in . ||| -342.948
+ \u09e7\u09ef\u09ec\u09ec , on 5 february \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 of \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 a national was held in . ||| -343.049
+ \u09e7\u09ef\u09ec\u09ec on 5 february \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 of \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 is a national was held in . ||| -343.128
+ bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in in the . ||| -348.887
+ bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in took the . ||| -349.144
+ bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in adopted . ||| -349.389
+ bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in took . ||| -349.516
+ bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in in . ||| -349.627
+ bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in adopted the . ||| -350.110
+ the bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in in the . ||| -350.210
+ bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and the islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in in the . ||| -350.262
+ the bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in took the . ||| -350.466
+ bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and the islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in took the . ||| -350.518
+ subject category : encyclopedia ||| 0.287
+ subject class : encyclopedia ||| -0.402
+ subject matter : encyclopedia ||| -0.446
+ subject : encyclopedia ||| -0.476
+ category : encyclopedia ||| -0.771
+ russia , france and israel is the main \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 country and defense sub country . ||| -136.409
+ russia , france and israel is the main \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 state and defense sub country . ||| -136.785
+ russia , france and israel is the main \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 country and the defense sub country . ||| -137.263
+ russia , france and israel is the main \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 country and defence sub country . ||| -137.272
+ russia , france and israel is the main \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 country and defense subsidiary country . ||| -138.633
+ russia , france and israel is the main \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 country and defense coordinating country . ||| -138.974
+ russia , france and the israel is the main \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 country and defense sub country . ||| -139.126
+ russia , france and israel is the main \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 country and defense sub country of . ||| -139.429
+ russia , france and of israel is the main \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 country and defense sub country . ||| -139.441
+ russia , france and the israel is the main \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 state and defense sub country . ||| -139.502
+ this is our known as an imaginary unit which with the help of mathematics formed the real number set from the complex number \u09b8\u09c7\u099f\u09c7 par with the . ||| -155.661
+ this is our known as imaginary unit which with the help of mathematics formed the real number set from the complex number \u09b8\u09c7\u099f\u09c7 par with the . ||| -155.793
+ this is our known as an imaginary unit which with the help of mathematics formed the real number set from complex number \u09b8\u09c7\u099f\u09c7 par with the . ||| -156.117
+ this is our known as an imaginary unit which with the help of mathematics formed the real number set from the complex numbers \u09b8\u09c7\u099f\u09c7 par with the . ||| -156.225
+ this is our known as imaginary unit which with the help of mathematics formed the real number set from complex number \u09b8\u09c7\u099f\u09c7 par with the . ||| -156.249
+ this is our known as an imaginary unit which with the help of mathematics formed the real number set from the complex number \u09b8\u09c7\u099f\u09c7 par with . ||| -156.297
+ this is our known as imaginary unit which with the help of mathematics formed the real number set from the complex numbers \u09b8\u09c7\u099f\u09c7 par with the . ||| -156.357
+ this is our known imaginary unit which with the help of mathematics formed the real number set from the complex number \u09b8\u09c7\u099f\u09c7 par with the . ||| -156.402
+ this is our known as imaginary unit which with the help of mathematics formed the real number set from the complex number \u09b8\u09c7\u099f\u09c7 par with . ||| -156.430
+ this is our known as an imaginary unit which with the help of mathematics formed the real number set from complex number \u09b8\u09c7\u099f\u09c7 par with . ||| -156.753
+ <address> ||| -21.727
+ " \u09a0\u09bf\u0995\u09be\u09a8\u09be " ||| -126.861
+ <district>kumilla</district> \u09a0\u09bf\u0995\u09be\u09a8\u09be " ||| -127.198
+ " \u09a0\u09bf\u0995\u09be\u09a8\u09be > ||| -127.412
+ < \u09a0\u09bf\u0995\u09be\u09a8\u09be " ||| -127.534
+ <district>kumilla</district> \u09a0\u09bf\u0995\u09be\u09a8\u09be > ||| -127.749
+ september ||| 0.260
+ september . ||| -3.522
+ \u09b8\u09c7\u09aa\u09cd\u099f\u09c7\u09ae\u09cd\u09ac\u09b0 ||| -104.738
+ this theory from , in the \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 against can not be , rather it can be supported . ||| -147.375
+ this theory from , in the big \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 against can not be , rather it can be supported . ||| -147.379
+ this theory from , in the \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 against can not be , but it can be supported . ||| -147.384
+ this theory from , in the big \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 against can not be , but it can be supported . ||| -147.388
+ in this theory from , in the big \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 against can not be , rather it can be supported . ||| -147.623
+ in this theory from , in the big \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 against can not be , but it can be supported . ||| -147.632
+ this theory from however , the \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 against can not be , rather it can be supported . ||| -147.916
+ this theory from however , the \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 against can not be , but it can be supported . ||| -147.925
+ this theory from however , the big \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 against can not be , rather it can be supported . ||| -147.965
+ this theory from however , the big \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 against can not be , but it can be supported . ||| -147.974
+ agricultural in production france is the most important country , it is mainly \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af , wine , cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and in the whole world export . ||| -265.370
+ agricultural in production france is the most important country ; it is mainly \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af , wine , cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and in the whole world export . ||| -265.446
+ agricultural in production france is the most important country , it is mainly \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af , wine , cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and the whole world export . ||| -266.013
+ agricultural in production france is the most important country ; it is mainly \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af , wine , cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and the whole world export . ||| -266.089
+ agricultural in production france is the most important country , it is mainly \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af , the wine , cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and in the whole world export . ||| -266.264
+ agricultural in production france is the most important country ; it is mainly \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af , the wine , cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and in the whole world export . ||| -266.340
+ agricultural in production france is the most important country , it is basically \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af , wine , cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and in the whole world export . ||| -266.392
+ agricultural in production france is the most important country ; it is \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af , wine , cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and in the whole world export . ||| -266.490
+ agricultural in production france is the most important country , it is mainly \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af , why , cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and in the whole world export . ||| -266.548
+ agricultural in production france is the most important country ; it is mainly \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af , why , cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and in the whole world export . ||| -266.623
+ their in mathematics \u09aa\u09be\u099f\u09c0\u0997\u09a3\u09bf\u09a4\u09c7\u09b0 person was . ||| -118.675
+ their in mathematics \u09aa\u09be\u099f\u09c0\u0997\u09a3\u09bf\u09a4\u09c7\u09b0 priority was . ||| -119.589
+ their in mathematics \u09aa\u09be\u099f\u09c0\u0997\u09a3\u09bf\u09a4\u09c7\u09b0 were was . ||| -119.755
+ in their in mathematics \u09aa\u09be\u099f\u09c0\u0997\u09a3\u09bf\u09a4\u09c7\u09b0 person was . ||| -119.974
+ their in mathematics \u09aa\u09be\u099f\u09c0\u0997\u09a3\u09bf\u09a4\u09c7\u09b0 are was . ||| -120.094
+ they in mathematics \u09aa\u09be\u099f\u09c0\u0997\u09a3\u09bf\u09a4\u09c7\u09b0 person was . ||| -120.273
+ of their in mathematics \u09aa\u09be\u099f\u09c0\u0997\u09a3\u09bf\u09a4\u09c7\u09b0 person was . ||| -120.375
+ their in mathematics \u09aa\u09be\u099f\u09c0\u0997\u09a3\u09bf\u09a4\u09c7\u09b0 dominance was . ||| -120.399
+ their in mathematics \u09aa\u09be\u099f\u09c0\u0997\u09a3\u09bf\u09a4\u09c7\u09b0 in was . ||| -120.487
+ deshgulo are : france , call , make noise , china , belgium , switzerland , germany , denmark , sweden , austria , chekoslovakia , argentina , italy , norway , \u09b9\u09be\u0999\u09cd\u0997\u09c7\u09b0\u09c0 , yugoslavia , bulgaria , rumania , \u0997\u09cd\u09b0\u09c0\u09b8 , egypt , singapore , indonesia , \u09a5\u09be\u0987\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 , japan , burma , \u09b9\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 , the soviet russia , iran , iraq and sri lanka . ||| -605.242
+ deshgulo are : france , call , make noise , china , belgium , switzerland , germany , denmark , sweden , austria , chekoslovakia , argentina , italy , in norway , \u09b9\u09be\u0999\u09cd\u0997\u09c7\u09b0\u09c0 , yugoslavia , bulgaria , rumania , \u0997\u09cd\u09b0\u09c0\u09b8 , egypt , singapore , indonesia , \u09a5\u09be\u0987\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 , japan , burma , \u09b9\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 , the soviet russia , iran , iraq and sri lanka . ||| -605.394
+ deshgulo are : france , call , make noise , china , belgium , switzerland , germany , denmark , sweden , austria , chekoslovakia , argentina , italy , norway , \u09b9\u09be\u0999\u09cd\u0997\u09c7\u09b0\u09c0 , yugoslavia , bulgaria , romania , \u0997\u09cd\u09b0\u09c0\u09b8 , egypt , singapore , indonesia , \u09a5\u09be\u0987\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 , japan , burma , \u09b9\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 , the soviet russia , iran , iraq and sri lanka . ||| -605.592
+ deshgulo are : france , call , make noise , china , belgium , switzerland , germany , denmark , in sweden , austria , chekoslovakia , argentina , italy , norway , \u09b9\u09be\u0999\u09cd\u0997\u09c7\u09b0\u09c0 , yugoslavia , bulgaria , rumania , \u0997\u09cd\u09b0\u09c0\u09b8 , egypt , singapore , indonesia , \u09a5\u09be\u0987\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 , japan , burma , \u09b9\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 , the soviet russia , iran , iraq and sri lanka . ||| -605.619
+ deshgulo are : france , call , make noise , china , belgium , switzerland , germany , denmark , sweden , austria , chekoslovakia , argentina , italy , in norway , \u09b9\u09be\u0999\u09cd\u0997\u09c7\u09b0\u09c0 , yugoslavia , bulgaria , romania , \u0997\u09cd\u09b0\u09c0\u09b8 , egypt , singapore , indonesia , \u09a5\u09be\u0987\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 , japan , burma , \u09b9\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 , the soviet russia , iran , iraq and sri lanka . ||| -605.745
+ deshgulo are : france , call , make noise , china , belgium , switzerland , germany , denmark , in sweden , austria , chekoslovakia , argentina , italy , in norway , \u09b9\u09be\u0999\u09cd\u0997\u09c7\u09b0\u09c0 , yugoslavia , bulgaria , rumania , \u0997\u09cd\u09b0\u09c0\u09b8 , egypt , singapore , indonesia , \u09a5\u09be\u0987\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 , japan , burma , \u09b9\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 , the soviet russia , iran , iraq and sri lanka . ||| -605.771
+ deshgulo are : france , call , make noise , china , belgium , switzerland , germany , denmark , sweden , austria , chekoslovakia , argentina , italy , norway , \u09b9\u09be\u0999\u09cd\u0997\u09c7\u09b0\u09c0 , and yugoslavia , bulgaria , rumania , \u0997\u09cd\u09b0\u09c0\u09b8 , egypt , singapore , indonesia , \u09a5\u09be\u0987\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 , japan , burma , \u09b9\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 , the soviet russia , iran , iraq and sri lanka . ||| -605.951
+ deshgulo are : france , call , make noise , china , belgium , switzerland , germany , denmark , in sweden , austria , chekoslovakia , argentina , italy , norway , \u09b9\u09be\u0999\u09cd\u0997\u09c7\u09b0\u09c0 , yugoslavia , bulgaria , romania , \u0997\u09cd\u09b0\u09c0\u09b8 , egypt , singapore , indonesia , \u09a5\u09be\u0987\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 , japan , burma , \u09b9\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 , the soviet russia , iran , iraq and sri lanka . ||| -605.969
+ deshgulo are : france , call , make noise , china , belgium , switzerland , germany , denmark , sweden , austria , chekoslovakia , argentina , italy , norway , \u09b9\u09be\u0999\u09cd\u0997\u09c7\u09b0\u09c0 , yugoslavia , bulgaria , rumania , \u0997\u09cd\u09b0\u09c0\u09b8 , egypt , singapore , indonesia , \u09a5\u09be\u0987\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 , japan , burma , \u09b9\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 , the soviet russia , iran , iraq and after visiting srilanka . ||| -605.983
+ deshgulo are : france , call , make noise , china , belgium , switzerland , germany , denmark , in sweden , austria , chekoslovakia , argentina , italy , in norway , \u09b9\u09be\u0999\u09cd\u0997\u09c7\u09b0\u09c0 , yugoslavia , bulgaria , romania , \u0997\u09cd\u09b0\u09c0\u09b8 , egypt , singapore , indonesia , \u09a5\u09be\u0987\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 , japan , burma , \u09b9\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 , the soviet russia , iran , iraq and sri lanka . ||| -606.122
+ this \u09ac\u09cd\u09af\u09be\u09b8\u09bf\u09b2\u09bf\u0995\u09be\u09b0 places now bank of england is situated . ||| -119.423
+ in this \u09ac\u09cd\u09af\u09be\u09b8\u09bf\u09b2\u09bf\u0995\u09be\u09b0 places now bank of england is situated . ||| -119.722
+ this \u09ac\u09cd\u09af\u09be\u09b8\u09bf\u09b2\u09bf\u0995\u09be\u09b0 places is now bank of england is situated . ||| -119.957
+ this \u09ac\u09cd\u09af\u09be\u09b8\u09bf\u09b2\u09bf\u0995\u09be\u09b0 places now bank of england is located . ||| -120.301
+ this \u09ac\u09cd\u09af\u09be\u09b8\u09bf\u09b2\u09bf\u0995\u09be\u09b0 places now bank of england is situated in . ||| -120.554
+ this \u09ac\u09cd\u09af\u09be\u09b8\u09bf\u09b2\u09bf\u0995\u09be\u09b0 places now bank of england is situated at . ||| -120.695
+ this \u09ac\u09cd\u09af\u09be\u09b8\u09bf\u09b2\u09bf\u0995\u09be\u09b0 places now bank of england is located in . ||| -120.727
+ this \u09ac\u09cd\u09af\u09be\u09b8\u09bf\u09b2\u09bf\u0995\u09be\u09b0 places is now bank of england is located . ||| -120.834
+ in this \u09ac\u09cd\u09af\u09be\u09b8\u09bf\u09b2\u09bf\u0995\u09be\u09b0 places now bank of england is situated in . ||| -120.852
+ this \u09ac\u09cd\u09af\u09be\u09b8\u09bf\u09b2\u09bf\u0995\u09be\u09b0 places are bank of england is situated in . ||| -121.016
+ in the north of this country \u09ac\u09bf\u09b8\u09cd\u0995\u09be\u0987 sub-sea , on the south \u099c\u09bf\u09ac\u09cd\u09b0\u09be\u09b2\u09cd\u099f\u09be\u09b0 strait , \u09aa\u09cd\u09b0\u09a3\u09be\u09b2\u09c0\u09b0 in the south \u09ae\u09b0\u0995\u09cd\u0995\u09cb , west and the towards the atlantic ocean . ||| -461.835
+ in the north of this country \u09ac\u09bf\u09b8\u09cd\u0995\u09be\u0987 sub-sea , on the south \u099c\u09bf\u09ac\u09cd\u09b0\u09be\u09b2\u09cd\u099f\u09be\u09b0 strait , \u09aa\u09cd\u09b0\u09a3\u09be\u09b2\u09c0\u09b0 in the south \u09ae\u09b0\u0995\u09cd\u0995\u09cb , west and the side of atlantic ocean . ||| -462.322
+ in the north of this country \u09ac\u09bf\u09b8\u09cd\u0995\u09be\u0987 sub-sea , on the south \u099c\u09bf\u09ac\u09cd\u09b0\u09be\u09b2\u09cd\u099f\u09be\u09b0 strait , \u09aa\u09cd\u09b0\u09a3\u09be\u09b2\u09c0\u09b0 in the south \u09ae\u09b0\u0995\u09cd\u0995\u09cb , west and the of atlantic ocean . ||| -462.361
+ in the north of this country \u09ac\u09bf\u09b8\u09cd\u0995\u09be\u0987 sub-sea , on the south \u099c\u09bf\u09ac\u09cd\u09b0\u09be\u09b2\u09cd\u099f\u09be\u09b0 strait , \u09aa\u09cd\u09b0\u09a3\u09be\u09b2\u09c0\u09b0 in the south \u09ae\u09b0\u0995\u09cd\u0995\u09cb , west and the of the atlantic ocean . ||| -462.532
+ in the north of this country \u09ac\u09bf\u09b8\u09cd\u0995\u09be\u0987 sub-sea , on the south \u099c\u09bf\u09ac\u09cd\u09b0\u09be\u09b2\u09cd\u099f\u09be\u09b0 strait , \u09aa\u09cd\u09b0\u09a3\u09be\u09b2\u09c0\u09b0 in the south \u09ae\u09b0\u0995\u09cd\u0995\u09cb , west and the side of the atlantic ocean . ||| -462.573
+ on the north of the country is the \u09ac\u09bf\u09b8\u09cd\u0995\u09be\u0987 sub-sea , on the south \u099c\u09bf\u09ac\u09cd\u09b0\u09be\u09b2\u09cd\u099f\u09be\u09b0 strait , \u09aa\u09cd\u09b0\u09a3\u09be\u09b2\u09c0\u09b0 in the south \u09ae\u09b0\u0995\u09cd\u0995\u09cb , west and the towards the atlantic ocean . ||| -462.634
+ in the north of this country \u09ac\u09bf\u09b8\u09cd\u0995\u09be\u0987 sub-sea , on the south \u099c\u09bf\u09ac\u09cd\u09b0\u09be\u09b2\u09cd\u099f\u09be\u09b0 strait , \u09aa\u09cd\u09b0\u09a3\u09be\u09b2\u09c0\u09b0 in the south \u09ae\u09b0\u0995\u09cd\u0995\u09cb , west and the the atlantic ocean . ||| -462.712
+ in the north of this country \u09ac\u09bf\u09b8\u09cd\u0995\u09be\u0987 sub-sea , on the south \u099c\u09bf\u09ac\u09cd\u09b0\u09be\u09b2\u09cd\u099f\u09be\u09b0 strait , \u09aa\u09cd\u09b0\u09a3\u09be\u09b2\u09c0\u09b0 in \u09ae\u09b0\u0995\u09cd\u0995\u09cb , west and the towards the atlantic ocean . ||| -462.825
+ in the north of this country \u09ac\u09bf\u09b8\u09cd\u0995\u09be\u0987 sub-sea , on the south \u099c\u09bf\u09ac\u09cd\u09b0\u09be\u09b2\u09cd\u099f\u09be\u09b0 strait , \u09aa\u09cd\u09b0\u09a3\u09be\u09b2\u09c0\u09b0 in the south \u09ae\u09b0\u0995\u09cd\u0995\u09cb , west and the on the atlantic ocean . ||| -462.833
+ in the north of this country \u09ac\u09bf\u09b8\u09cd\u0995\u09be\u0987 sub-sea , on the south \u099c\u09bf\u09ac\u09cd\u09b0\u09be\u09b2\u09cd\u099f\u09be\u09b0 strait , \u09aa\u09cd\u09b0\u09a3\u09be\u09b2\u09c0\u09b0 the \u09ae\u09b0\u0995\u09cd\u0995\u09cb , west and the towards the atlantic ocean . ||| -462.991
+ apart from this situation it can be understood that a \u09ae\u09c1\u09b9\u09c2\u09b0\u09cd\u09a4\u09c7 of the united nations of the decision \u09a8\u09c7\u09ac\u09be\u09b0 \u0985\u0995\u09cd\u09b7\u09ae\u09a4\u09be . ||| -349.945
+ apart from that , this situation it can be understood that a \u09ae\u09c1\u09b9\u09c2\u09b0\u09cd\u09a4\u09c7 of the united nations of the decision \u09a8\u09c7\u09ac\u09be\u09b0 \u0985\u0995\u09cd\u09b7\u09ae\u09a4\u09be . ||| -350.476
+ apart from this situation it can be understood a \u09ae\u09c1\u09b9\u09c2\u09b0\u09cd\u09a4\u09c7 of the united nations of the decision \u09a8\u09c7\u09ac\u09be\u09b0 \u0985\u0995\u09cd\u09b7\u09ae\u09a4\u09be . ||| -350.589
+ apart from this situation it can be understood that a \u09ae\u09c1\u09b9\u09c2\u09b0\u09cd\u09a4\u09c7 of the united nations the decision \u09a8\u09c7\u09ac\u09be\u09b0 \u0985\u0995\u09cd\u09b7\u09ae\u09a4\u09be . ||| -350.844
+ apart from that , this situation it can be understood a \u09ae\u09c1\u09b9\u09c2\u09b0\u09cd\u09a4\u09c7 of the united nations of the decision \u09a8\u09c7\u09ac\u09be\u09b0 \u0985\u0995\u09cd\u09b7\u09ae\u09a4\u09be . ||| -351.121
+ apart from that , for this situation it can be understood that a \u09ae\u09c1\u09b9\u09c2\u09b0\u09cd\u09a4\u09c7 of the united nations of the decision \u09a8\u09c7\u09ac\u09be\u09b0 \u0985\u0995\u09cd\u09b7\u09ae\u09a4\u09be . ||| -351.262
+ apart from these , this situation it can be understood that a \u09ae\u09c1\u09b9\u09c2\u09b0\u09cd\u09a4\u09c7 of the united nations of the decision \u09a8\u09c7\u09ac\u09be\u09b0 \u0985\u0995\u09cd\u09b7\u09ae\u09a4\u09be . ||| -351.267
+ apart from this situation it can be understood that a \u09ae\u09c1\u09b9\u09c2\u09b0\u09cd\u09a4\u09c7 of united nation of the decision \u09a8\u09c7\u09ac\u09be\u09b0 \u0985\u0995\u09cd\u09b7\u09ae\u09a4\u09be . ||| -351.327
+ apart from this situation it can be understood that a \u09ae\u09c1\u09b9\u09c2\u09b0\u09cd\u09a4\u09c7 of the united nations fast decision \u09a8\u09c7\u09ac\u09be\u09b0 \u0985\u0995\u09cd\u09b7\u09ae\u09a4\u09be . ||| -351.427
+ apart from this situation it can be understood that a \u09ae\u09c1\u09b9\u09c2\u09b0\u09cd\u09a4\u09c7 of the united nations rapidly decision \u09a8\u09c7\u09ac\u09be\u09b0 \u0985\u0995\u09cd\u09b7\u09ae\u09a4\u09be . ||| -351.449
+ \u0995\u09be\u09b0\u09cd\u09b2 there is work through it by the . ||| -129.293
+ \u0995\u09be\u09b0\u09cd\u09b2 there is work through it is controlled by . ||| -130.169
+ \u0995\u09be\u09b0\u09cd\u09b2 there is work through it controlled by the . ||| -130.397
+ \u0995\u09be\u09b0\u09cd\u09b2 there is work through it by . ||| -130.442
+ \u0995\u09be\u09b0\u09cd\u09b2 there is work through this controlled by the . ||| -130.455
+ \u0995\u09be\u09b0\u09cd\u09b2 there is only through it by the . ||| -130.463
+ \u0995\u09be\u09b0\u09cd\u09b2 there is work through this by the . ||| -130.546
+ \u0995\u09be\u09b0\u09cd\u09b2 place there is work through it by the . ||| -130.628
+ \u0995\u09be\u09b0\u09cd\u09b2 there is work through this are conducted by the . ||| -130.739
+ \u0995\u09be\u09b0\u09cd\u09b2 there is work through it are conducted by the . ||| -131.107
+ the subject of sometimes puran -lrb- from , sometimes in the middle age love story from , sometimes again today 's social and political \u0998\u099f\u09a8\u09be\u09ac\u09b2\u09bf from accepted . ||| -165.025
+ the subject of sometimes puran -lrb- from , sometimes in the middle age love story from , sometimes again today 's social and political \u0998\u099f\u09a8\u09be\u09ac\u09b2\u09bf taken from . ||| -165.876
+ the subject of sometimes puran -lrb- from , sometimes in the middle age love story from , sometimes again today 's social and political \u0998\u099f\u09a8\u09be\u09ac\u09b2\u09bf from the accepted . ||| -165.884
+ the subject of some puran -lrb- from , sometimes in the middle age love story from , sometimes again today 's social and political \u0998\u099f\u09a8\u09be\u09ac\u09b2\u09bf from accepted . ||| -166.021
+ the subject of sometimes puran -lrb- from , sometimes in the middle age love story to , sometimes again today 's social and political \u0998\u099f\u09a8\u09be\u09ac\u09b2\u09bf from accepted . ||| -166.116
+ the subject of sometimes puran -lrb- from , sometimes in the middle age love story from , sometimes again today 's social and political \u0998\u099f\u09a8\u09be\u09ac\u09b2\u09bf to accepted . ||| -166.120
+ the subject of sometimes puran -lrb- from , sometimes in middle age love story from , sometimes again today 's social and political \u0998\u099f\u09a8\u09be\u09ac\u09b2\u09bf from accepted . ||| -166.162
+ the subject of sometimes puran -lrb- from , sometimes in the middle age love story from the , sometimes again today 's social and political \u0998\u099f\u09a8\u09be\u09ac\u09b2\u09bf from accepted . ||| -166.174
+ the subject of sometimes puran -lrb- from , sometimes in the middle age love story from , sometimes again at the present age social and political \u0998\u099f\u09a8\u09be\u09ac\u09b2\u09bf from accepted . ||| -166.244
+ the subject of sometime puran -lrb- from , sometimes in the middle age love story from , sometimes again today 's social and political \u0998\u099f\u09a8\u09be\u09ac\u09b2\u09bf from accepted . ||| -166.279
+ three measure on the basis of the universe that age is found that is almost \u09e7\u09e9.\u09ed � \u09e6.\u09e8 billion years . ||| -354.717
+ three measure on the basis of the universe that the age is found that is almost \u09e7\u09e9.\u09ed � \u09e6.\u09e8 billion years . ||| -355.390
+ three measure on the basis of the universe that age is found that are almost \u09e7\u09e9.\u09ed � \u09e6.\u09e8 billion years . ||| -355.566
+ three measure on the basis of the universe that age is found that is about \u09e7\u09e9.\u09ed � \u09e6.\u09e8 billion years . ||| -355.918
+ three measure on the basis of the universe the age is found that is almost \u09e7\u09e9.\u09ed � \u09e6.\u09e8 billion years . ||| -355.947
+ three measure on the basis of the universe that age found in that is almost \u09e7\u09e9.\u09ed � \u09e6.\u09e8 billion years . ||| -356.235
+ three measure on the basis of the universe that the age is found that are almost \u09e7\u09e9.\u09ed � \u09e6.\u09e8 billion years . ||| -356.239
+ three measure on the basis of the universe that age is found in the almost \u09e7\u09e9.\u09ed � \u09e6.\u09e8 billion years . ||| -356.484
+ three measure on the basis of the universe that age is found that is around \u09e7\u09e9.\u09ed � \u09e6.\u09e8 billion years . ||| -356.558
+ three measure on the basis of the universe that the age is found that is about \u09e7\u09e9.\u09ed � \u09e6.\u09e8 billion years . ||| -356.591
+ \u0995\u09be\u099b\u09c7\u0987 there are east russia , which \u0993\u0996\u099f\u09b8\u09cd\u0995 sea and japan sea on the other can be is situated . ||| -242.737
+ \u0995\u09be\u099b\u09c7\u0987 there are east russia , which is \u0993\u0996\u099f\u09b8\u09cd\u0995 sea and japan sea on the other can be is situated . ||| -243.137
+ \u0995\u09be\u099b\u09c7\u0987 there are east russia , which \u0993\u0996\u099f\u09b8\u09cd\u0995 sea and japan sea on the other can be situated in . ||| -243.852
+ \u0995\u09be\u099b\u09c7\u0987 there are east russia , which \u0993\u0996\u099f\u09b8\u09cd\u0995 sea and japan sea on the other can be is situated in . ||| -243.867
+ \u0995\u09be\u099b\u09c7\u0987 there are east russia , which \u0993\u0996\u099f\u09b8\u09cd\u0995 sea and japan sea on the other can be situated in the . ||| -243.900
+ \u0995\u09be\u099b\u09c7\u0987 there are east russia , which \u0993\u0996\u099f\u09b8\u09cd\u0995 sea and japan sea on the other can be is situated at . ||| -244.008
+ \u0995\u09be\u099b\u09c7\u0987 there are east russia , which \u0993\u0996\u099f\u09b8\u09cd\u0995 sea and japan sea on the other can be situated . ||| -244.031
+ \u0995\u09be\u099b\u09c7\u0987 there are east russia , which \u0993\u0996\u099f\u09b8\u09cd\u0995 sea and japan sea on the other can be is located in . ||| -244.041
+ \u0995\u09be\u099b\u09c7\u0987 there are east russia , which \u0993\u0996\u099f\u09b8\u09cd\u0995 sea and japan sea on the other can is situated in . ||| -244.052
+ \u0995\u09be\u099b\u09c7\u0987 are east russia , which \u0993\u0996\u099f\u09b8\u09cd\u0995 sea and japan sea on the other can be is situated in . ||| -244.075
+ in kolkata is located in the national library of the leading public \u09b2\u09be\u0987\u09ac\u09cd\u09b0\u09c7\u09b0\u09bf . ||| -123.312
+ in kolkata is located in the national library of the country was public \u09b2\u09be\u0987\u09ac\u09cd\u09b0\u09c7\u09b0\u09bf . ||| -123.654
+ in kolkata is located in the national library of the country leading public \u09b2\u09be\u0987\u09ac\u09cd\u09b0\u09c7\u09b0\u09bf . ||| -123.951
+ in kolkata is located in the national library the leading public \u09b2\u09be\u0987\u09ac\u09cd\u09b0\u09c7\u09b0\u09bf . ||| -124.144
+ in kolkata is located in the national library , is the leading public \u09b2\u09be\u0987\u09ac\u09cd\u09b0\u09c7\u09b0\u09bf . ||| -124.779
+ in kolkata is located in the national library of the country is leading public \u09b2\u09be\u0987\u09ac\u09cd\u09b0\u09c7\u09b0\u09bf . ||| -125.068
+ in kolkata is located in the national library countries leading public \u09b2\u09be\u0987\u09ac\u09cd\u09b0\u09c7\u09b0\u09bf . ||| -125.571
+ in kolkata is located in the national library of the country , the pioneering public \u09b2\u09be\u0987\u09ac\u09cd\u09b0\u09c7\u09b0\u09bf . ||| -125.601
+ in kolkata is located in the national library the country leading public \u09b2\u09be\u0987\u09ac\u09cd\u09b0\u09c7\u09b0\u09bf . ||| -125.684
+ in kolkata is located in the national library of the leading public \u09b2\u09be\u0987\u09ac\u09cd\u09b0\u09c7\u09b0\u09bf . . ||| -126.253
+ \u099b\u09a4\u09cd\u09b0\u09be\u0995\u09ac\u09bf\u09a6\u09cd\u09af\u09be ||| -104.738
+ \u09b0\u09be\u09b7\u09cd\u099f\u09cd\u09b0\u09b8\u0999\u09cd\u0998\u09c7\u09b0 secretary general \u09ac\u09be\u09a8 ki moon ||| -222.227
+ \u09b0\u09be\u09b7\u09cd\u099f\u09cd\u09b0\u09b8\u0999\u09cd\u0998\u09c7\u09b0 general secretary \u09ac\u09be\u09a8 ki moon ||| -222.626
+ \u09b0\u09be\u09b7\u09cd\u099f\u09cd\u09b0\u09b8\u0999\u09cd\u0998\u09c7\u09b0 chief secretary \u09ac\u09be\u09a8 ki moon ||| -224.534
+ \u09b0\u09be\u09b7\u09cd\u099f\u09cd\u09b0\u09b8\u0999\u09cd\u0998\u09c7\u09b0 secretary general \u09ac\u09be\u09a8 what moon ||| -225.003
+ \u09b0\u09be\u09b7\u09cd\u099f\u09cd\u09b0\u09b8\u0999\u09cd\u0998\u09c7\u09b0 general secretary \u09ac\u09be\u09a8 what moon ||| -225.402
+ \u09b0\u09be\u09b7\u09cd\u099f\u09cd\u09b0\u09b8\u0999\u09cd\u0998\u09c7\u09b0 chief secretary of \u09ac\u09be\u09a8 ki moon ||| -226.183
+ \u09b0\u09be\u09b7\u09cd\u099f\u09cd\u09b0\u09b8\u0999\u09cd\u0998\u09c7\u09b0 chief secretary \u09ac\u09be\u09a8 what moon ||| -226.565
+ \u09b0\u09be\u09b7\u09cd\u099f\u09cd\u09b0\u09b8\u0999\u09cd\u0998\u09c7\u09b0 secretary general \u09ac\u09be\u09a8 what salt ||| -226.669
+ \u09ae\u09bf\u09a8\u09bf\u0995\u09cd\u09b8\u09c7\u09b0 of india was smells \u099f\u09be\u09a8\u09c7\u09a8\u09ac\u09ae , a famous operating system designing \u09aa\u09cd\u09b0\u09b6\u09bf\u0995\u09cd\u09b7\u0995 . ||| -344.577
+ \u09ae\u09bf\u09a8\u09bf\u0995\u09cd\u09b8\u09c7\u09b0 of india was smells \u099f\u09be\u09a8\u09c7\u09a8\u09ac\u09ae , a famous operating system design \u09aa\u09cd\u09b0\u09b6\u09bf\u0995\u09cd\u09b7\u0995 . ||| -344.611
+ \u09ae\u09bf\u09a8\u09bf\u0995\u09cd\u09b8\u09c7\u09b0 of india was smells \u099f\u09be\u09a8\u09c7\u09a8\u09ac\u09ae , in a famous operating system designing \u09aa\u09cd\u09b0\u09b6\u09bf\u0995\u09cd\u09b7\u0995 . ||| -344.995
+ \u09ae\u09bf\u09a8\u09bf\u0995\u09cd\u09b8\u09c7\u09b0 of india was smells \u099f\u09be\u09a8\u09c7\u09a8\u09ac\u09ae , in a famous operating system design \u09aa\u09cd\u09b0\u09b6\u09bf\u0995\u09cd\u09b7\u0995 . ||| -345.029
+ \u09ae\u09bf\u09a8\u09bf\u0995\u09cd\u09b8\u09c7\u09b0 of was smells \u099f\u09be\u09a8\u09c7\u09a8\u09ac\u09ae , a famous operating system designing \u09aa\u09cd\u09b0\u09b6\u09bf\u0995\u09cd\u09b7\u0995 . ||| -345.466
+ \u09ae\u09bf\u09a8\u09bf\u0995\u09cd\u09b8\u09c7\u09b0 of was smells \u099f\u09be\u09a8\u09c7\u09a8\u09ac\u09ae , a famous operating system design \u09aa\u09cd\u09b0\u09b6\u09bf\u0995\u09cd\u09b7\u0995 . ||| -345.499
+ \u09ae\u09bf\u09a8\u09bf\u0995\u09cd\u09b8\u09c7\u09b0 of was smells \u099f\u09be\u09a8\u09c7\u09a8\u09ac\u09ae , in a famous operating system designing \u09aa\u09cd\u09b0\u09b6\u09bf\u0995\u09cd\u09b7\u0995 . ||| -345.884
+ \u09ae\u09bf\u09a8\u09bf\u0995\u09cd\u09b8\u09c7\u09b0 of were smells \u099f\u09be\u09a8\u09c7\u09a8\u09ac\u09ae , a famous operating system designing \u09aa\u09cd\u09b0\u09b6\u09bf\u0995\u09cd\u09b7\u0995 . ||| -345.906
+ \u09ae\u09bf\u09a8\u09bf\u0995\u09cd\u09b8\u09c7\u09b0 of was smells \u099f\u09be\u09a8\u09c7\u09a8\u09ac\u09ae , in a famous operating system design \u09aa\u09cd\u09b0\u09b6\u09bf\u0995\u09cd\u09b7\u0995 . ||| -345.918
+ \u09ae\u09bf\u09a8\u09bf\u0995\u09cd\u09b8\u09c7\u09b0 of were smells \u099f\u09be\u09a8\u09c7\u09a8\u09ac\u09ae , a famous operating system design \u09aa\u09cd\u09b0\u09b6\u09bf\u0995\u09cd\u09b7\u0995 . ||| -345.940
+ the \u099f\u09be\u0987\u09ae\u09cd of 's of \u0987\u09a8\u09cd\u09a1\u09bf\u09af\u09bc\u09be-\u09a4\u09c7 written in the " it is absurd to compare it with any other indian cinema ... pather panchali is pure cinema " -lrb- " it other by the indian films with compared to unreal ... pather panchali are pure film " -rrb- . ||| -1735.945
+ the \u099f\u09be\u0987\u09ae\u09cd of 's of \u0987\u09a8\u09cd\u09a1\u09bf\u09af\u09bc\u09be-\u09a4\u09c7 written in the " it is absurd to compare it with any other indian cinema ... pather panchali is pure cinema " -lrb- " it in other by the indian films with compared to unreal ... pather panchali are pure film " -rrb- . ||| -1736.149
+ the \u099f\u09be\u0987\u09ae\u09cd of 's of \u0987\u09a8\u09cd\u09a1\u09bf\u09af\u09bc\u09be-\u09a4\u09c7 written in that " it is absurd to compare it with any other indian cinema ... pather panchali is pure cinema " -lrb- " it other by the indian films with compared to unreal ... pather panchali are pure film " -rrb- . ||| -1736.151
+ the \u099f\u09be\u0987\u09ae\u09cd of 's of \u0987\u09a8\u09cd\u09a1\u09bf\u09af\u09bc\u09be-\u09a4\u09c7 written in the " it is absurd to compare it with any other indian cinema ... pather panchali is pure cinema " -lrb- " it is the by the indian films with compared to unreal ... pather panchali are pure film " -rrb- . ||| -1736.297
+ the \u099f\u09be\u0987\u09ae\u09cd of 's of \u0987\u09a8\u09cd\u09a1\u09bf\u09af\u09bc\u09be-\u09a4\u09c7 written in that " it is absurd to compare it with any other indian cinema ... pather panchali is pure cinema " -lrb- " it in other by the indian films with compared to unreal ... pather panchali are pure film " -rrb- . ||| -1736.355
+ the \u099f\u09be\u0987\u09ae\u09cd of 's of \u0987\u09a8\u09cd\u09a1\u09bf\u09af\u09bc\u09be-\u09a4\u09c7 written in the " it is absurd to compare it with any other indian cinema ... pather panchali is pure cinema , " -lrb- " it other by the indian films with compared to unreal ... pather panchali are pure film " -rrb- . ||| -1736.363
+ the \u099f\u09be\u0987\u09ae\u09cd of 's of \u0987\u09a8\u09cd\u09a1\u09bf\u09af\u09bc\u09be-\u09a4\u09c7 written in the " it is absurd to compare it with any other indian cinema ... pather panchali is pure cinema " -lrb- " it other by the indian films with compared to unreal ... pather panchali is pure film " -rrb- . ||| -1736.461
+ the \u099f\u09be\u0987\u09ae\u09cd of 's of \u0987\u09a8\u09cd\u09a1\u09bf\u09af\u09bc\u09be-\u09a4\u09c7 written in that " it is absurd to compare it with any other indian cinema ... pather panchali is pure cinema " -lrb- " it is the by the indian films with compared to unreal ... pather panchali are pure film " -rrb- . ||| -1736.503
+ the \u099f\u09be\u0987\u09ae\u09cd of 's of \u0987\u09a8\u09cd\u09a1\u09bf\u09af\u09bc\u09be-\u09a4\u09c7 written in that , " it is absurd to compare it with any other indian cinema ... pather panchali is pure cinema " -lrb- " it other by the indian films with compared to unreal ... pather panchali are pure film " -rrb- . ||| -1736.517
+ the \u099f\u09be\u0987\u09ae\u09cd of 's of \u0987\u09a8\u09cd\u09a1\u09bf\u09af\u09bc\u09be-\u09a4\u09c7 written in the " it is absurd to compare it with any other indian cinema ... pather panchali is pure cinema ' -lrb- " it other by the indian films with compared to unreal ... pather panchali are pure film " -rrb- . ||| -1736.518
+ after this 1953 , in the month of may nazrul and \u09aa\u09cd\u09b0\u09ae\u09c0\u09b2\u09be \u09a6\u09c7\u09ac\u09c0\u0995\u09c7 \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u09be\u09b0 for london sent to . ||| -345.817
+ after this 1953 on may month nazrul and \u09aa\u09cd\u09b0\u09ae\u09c0\u09b2\u09be \u09a6\u09c7\u09ac\u09c0\u0995\u09c7 \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u09be\u09b0 for london sent to . ||| -345.874
+ after that , 1953 may month nazrul and \u09aa\u09cd\u09b0\u09ae\u09c0\u09b2\u09be \u09a6\u09c7\u09ac\u09c0\u0995\u09c7 \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u09be\u09b0 for london sent to . ||| -345.956
+ after that 1953 on may month nazrul and \u09aa\u09cd\u09b0\u09ae\u09c0\u09b2\u09be \u09a6\u09c7\u09ac\u09c0\u0995\u09c7 \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u09be\u09b0 for london sent to . ||| -346.040
+ after this 1953 , in the month of may nazrul and \u09aa\u09cd\u09b0\u09ae\u09c0\u09b2\u09be \u09a6\u09c7\u09ac\u09c0\u0995\u09c7 \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u09be\u09b0 for london sent to the . ||| -346.057
+ after this 1953 on may month nazrul and \u09aa\u09cd\u09b0\u09ae\u09c0\u09b2\u09be \u09a6\u09c7\u09ac\u09c0\u0995\u09c7 \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u09be\u09b0 for london sent to the . ||| -346.113
+ after this 1953 , in the month of may najrul and \u09aa\u09cd\u09b0\u09ae\u09c0\u09b2\u09be \u09a6\u09c7\u09ac\u09c0\u0995\u09c7 \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u09be\u09b0 for london sent to . ||| -346.160
+ after that , 1953 may month nazrul and \u09aa\u09cd\u09b0\u09ae\u09c0\u09b2\u09be \u09a6\u09c7\u09ac\u09c0\u0995\u09c7 \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u09be\u09b0 for london sent to the . ||| -346.195
+ after this 1953 on may month najrul and \u09aa\u09cd\u09b0\u09ae\u09c0\u09b2\u09be \u09a6\u09c7\u09ac\u09c0\u0995\u09c7 \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u09be\u09b0 for london sent to . ||| -346.217
+ after that 1953 on may month nazrul and \u09aa\u09cd\u09b0\u09ae\u09c0\u09b2\u09be \u09a6\u09c7\u09ac\u09c0\u0995\u09c7 \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u09be\u09b0 for london sent to the . ||| -346.279
+ the southern and the east there is \u09ac\u09bf\u09b8\u09cd\u09a4\u09c0\u09b0\u09cd\u09a3 plain , west and on the north there are \u09b0\u09c1\u0995\u09cd\u09b7 mountain and mountain . ||| -250.818
+ the southern and the east there is \u09ac\u09bf\u09b8\u09cd\u09a4\u09c0\u09b0\u09cd\u09a3 land , west and on the north there are \u09b0\u09c1\u0995\u09cd\u09b7 mountain and mountain . ||| -250.835
+ the southern and the east there is \u09ac\u09bf\u09b8\u09cd\u09a4\u09c0\u09b0\u09cd\u09a3 plain , west and on the north there is \u09b0\u09c1\u0995\u09cd\u09b7 mountain and mountain . ||| -251.000
+ the southern and the east there is \u09ac\u09bf\u09b8\u09cd\u09a4\u09c0\u09b0\u09cd\u09a3 land , west and on the north there is \u09b0\u09c1\u0995\u09cd\u09b7 mountain and mountain . ||| -251.017
+ the southern and the east there is \u09ac\u09bf\u09b8\u09cd\u09a4\u09c0\u09b0\u09cd\u09a3 plain , west and in the north there are \u09b0\u09c1\u0995\u09cd\u09b7 mountain and mountain . ||| -251.127
+ the southern and the east there is \u09ac\u09bf\u09b8\u09cd\u09a4\u09c0\u09b0\u09cd\u09a3 land , west and in the north there are \u09b0\u09c1\u0995\u09cd\u09b7 mountain and mountain . ||| -251.145
+ the south and the east there is \u09ac\u09bf\u09b8\u09cd\u09a4\u09c0\u09b0\u09cd\u09a3 land , west and on the north there are \u09b0\u09c1\u0995\u09cd\u09b7 mountain and mountain . ||| -251.255
+ the southern and the east there is \u09ac\u09bf\u09b8\u09cd\u09a4\u09c0\u09b0\u09cd\u09a3 plain , west and in the north there is \u09b0\u09c1\u0995\u09cd\u09b7 mountain and mountain . ||| -251.309
+ the southern and the east there is \u09ac\u09bf\u09b8\u09cd\u09a4\u09c0\u09b0\u09cd\u09a3 plain , west and on the north are \u09b0\u09c1\u0995\u09cd\u09b7 mountain and mountain . ||| -251.317
+ the southern and the east there is \u09ac\u09bf\u09b8\u09cd\u09a4\u09c0\u09b0\u09cd\u09a3 land , west and in the north there is \u09b0\u09c1\u0995\u09cd\u09b7 mountain and mountain . ||| -251.327
+ \u099f\u09cd\u09b0\u09c7\u09a1\u09ae\u09be\u09b0\u09cd\u0995 ||| -104.738
+ \u09b2\u09b0\u09cd\u09a1 \u0993\u09af\u09bc\u09c7\u09b2\u09c7\u09b8\u09b2\u09bf\u09b0 -lrb- \u0997\u09ad\u09b0\u09cd\u09a8\u09b0-\u099c\u09c7\u09a8\u09be\u09b0\u09c7\u09b2 \u09e7\u09ed\u09ef\u09ed-\u09e7\u09ee\u09e6\u09eb -rrb- during his of the city are the ghotechilo ||| -450.843
+ \u09b2\u09b0\u09cd\u09a1 \u0993\u09af\u09bc\u09c7\u09b2\u09c7\u09b8\u09b2\u09bf\u09b0 -lrb- \u0997\u09ad\u09b0\u09cd\u09a8\u09b0-\u099c\u09c7\u09a8\u09be\u09b0\u09c7\u09b2 \u09e7\u09ed\u09ef\u09ed-\u09e7\u09ee\u09e6\u09eb -rrb- during his of the city are the already happened ||| -450.967
+ \u09b2\u09b0\u09cd\u09a1 \u0993\u09af\u09bc\u09c7\u09b2\u09c7\u09b8\u09b2\u09bf\u09b0 -lrb- \u0997\u09ad\u09b0\u09cd\u09a8\u09b0-\u099c\u09c7\u09a8\u09be\u09b0\u09c7\u09b2 \u09e7\u09ed\u09ef\u09ed-\u09e7\u09ee\u09e6\u09eb -rrb- during his of the city are increased ghotechilo ||| -451.585
+ \u09b2\u09b0\u09cd\u09a1 \u0993\u09af\u09bc\u09c7\u09b2\u09c7\u09b8\u09b2\u09bf\u09b0 -lrb- \u0997\u09ad\u09b0\u09cd\u09a8\u09b0-\u099c\u09c7\u09a8\u09be\u09b0\u09c7\u09b2 \u09e7\u09ed\u09ef\u09ed-\u09e7\u09ee\u09e6\u09eb -rrb- during his city are the ghotechilo ||| -451.690
+ \u09b2\u09b0\u09cd\u09a1 \u0993\u09af\u09bc\u09c7\u09b2\u09c7\u09b8\u09b2\u09bf\u09b0 -lrb- \u0997\u09ad\u09b0\u09cd\u09a8\u09b0-\u099c\u09c7\u09a8\u09be\u09b0\u09c7\u09b2 \u09e7\u09ed\u09ef\u09ed-\u09e7\u09ee\u09e6\u09eb -rrb- during his city are the already happened ||| -451.813
+ \u09b2\u09b0\u09cd\u09a1 \u0993\u09af\u09bc\u09c7\u09b2\u09c7\u09b8\u09b2\u09bf\u09b0 -lrb- \u0997\u09ad\u09b0\u09cd\u09a8\u09b0-\u099c\u09c7\u09a8\u09be\u09b0\u09c7\u09b2 \u09e7\u09ed\u09ef\u09ed-\u09e7\u09ee\u09e6\u09eb -rrb- during his in the city are the ghotechilo ||| -452.090
+ \u09b2\u09b0\u09cd\u09a1 \u0993\u09af\u09bc\u09c7\u09b2\u09c7\u09b8\u09b2\u09bf\u09b0 -lrb- \u0997\u09ad\u09b0\u09cd\u09a8\u09b0-\u099c\u09c7\u09a8\u09be\u09b0\u09c7\u09b2 \u09e7\u09ed\u09ef\u09ed-\u09e7\u09ee\u09e6\u09eb -rrb- during his of the city are increased already happened ||| -452.202
+ \u09b2\u09b0\u09cd\u09a1 \u0993\u09af\u09bc\u09c7\u09b2\u09c7\u09b8\u09b2\u09bf\u09b0 -lrb- \u0997\u09ad\u09b0\u09cd\u09a8\u09b0-\u099c\u09c7\u09a8\u09be\u09b0\u09c7\u09b2 \u09e7\u09ed\u09ef\u09ed-\u09e7\u09ee\u09e6\u09eb -rrb- during his in the city are the already happened ||| -452.213
+ \u09b2\u09b0\u09cd\u09a1 \u0993\u09af\u09bc\u09c7\u09b2\u09c7\u09b8\u09b2\u09bf\u09b0 -lrb- the \u0997\u09ad\u09b0\u09cd\u09a8\u09b0-\u099c\u09c7\u09a8\u09be\u09b0\u09c7\u09b2 \u09e7\u09ed\u09ef\u09ed-\u09e7\u09ee\u09e6\u09eb -rrb- during his of the city are the ghotechilo ||| -452.856
+ \u09b2\u09b0\u09cd\u09a1 \u0993\u09af\u09bc\u09c7\u09b2\u09c7\u09b8\u09b2\u09bf\u09b0 -lrb- the \u0997\u09ad\u09b0\u09cd\u09a8\u09b0-\u099c\u09c7\u09a8\u09be\u09b0\u09c7\u09b2 \u09e7\u09ed\u09ef\u09ed-\u09e7\u09ee\u09e6\u09eb -rrb- during his of the city are the already happened ||| -452.980
+ many important and real problem to solve complex number apariharza ||| -31.095
+ many important and real problem to solve complex number extremely necessary ||| -31.408
+ many important and real problem to solve complex number inevitable ||| -31.447
+ many important and real problem to for complex number extremely necessary ||| -31.534
+ many important and real problem to for complex number inevitable ||| -31.572
+ many important and real problems to solve complex number apariharza ||| -31.603
+ many important and real problems to solve complex number extremely necessary ||| -31.917
+ in many important and real problem to solve complex number extremely necessary ||| -31.920
+ many important and real problems to solve complex number inevitable ||| -31.955
+ many important and real problems to for complex number extremely necessary ||| -32.043
+ the big bang is a famous result are , in the state of the universe so and the recent state from the separate . ||| -56.424
+ the big bang is a famous result are , in the state of the universe so and the recent situation from the separate . ||| -56.531
+ the big bang is a famous result are , in the state of the universe so and recent state from the separate . ||| -56.639
+ the big bang is a famous result is , in the state of the universe so and the recent state from the separate . ||| -56.704
+ the big bang is a famous result is , in the state of the universe so and the recent situation from the separate . ||| -56.811
+ the big bang is a famous result is , in the state of the universe so and recent state from the separate . ||| -56.919
+ the big bang is a famous result are , in the state so and the recent state from the separate . ||| -56.945
+ the big bang is a famous result are , in the state so and the recent situation from the separate . ||| -57.052
+ the big bang is a famous result are , in the state of the universe so and the recent state from the different . ||| -57.133
+ the big bang is a famous result are , in the state so and recent state from the separate . ||| -57.160
+ windows \u09ae\u09bf\u09b2\u09bf\u09a8\u09bf\u09af\u09bc\u09be\u09ae ||| -106.565
+ of windows \u09ae\u09bf\u09b2\u09bf\u09a8\u09bf\u09af\u09bc\u09be\u09ae ||| -110.367
+ the windows \u09ae\u09bf\u09b2\u09bf\u09a8\u09bf\u09af\u09bc\u09be\u09ae ||| -113.483
+ of the windows \u09ae\u09bf\u09b2\u09bf\u09a8\u09bf\u09af\u09bc\u09be\u09ae ||| -114.357
+ \u0989\u0987\u09a8\u09cd\u09a1\u09cb\u099c \u09ae\u09bf\u09b2\u09bf\u09a8\u09bf\u09af\u09bc\u09be\u09ae ||| -209.243
+ rabindranath , in more than one \u09b6\u09c8\u09b2\u09c0 \u0986\u09af\u09bc\u09a4\u09cd\u09a4\u09cd\u09ac was . ||| -223.409
+ rabindranath , in more than one \u09b6\u09c8\u09b2\u09c0 \u0986\u09af\u09bc\u09a4\u09cd\u09a4\u09cd\u09ac by . ||| -223.899
+ rabindranath , in more than one \u09b6\u09c8\u09b2\u09c0 \u0986\u09af\u09bc\u09a4\u09cd\u09a4\u09cd\u09ac a . ||| -223.979
+ rabindranath however , more than one \u09b6\u09c8\u09b2\u09c0 \u0986\u09af\u09bc\u09a4\u09cd\u09a4\u09cd\u09ac was . ||| -224.217
+ rabindranath , in more than one \u09b6\u09c8\u09b2\u09c0 \u0986\u09af\u09bc\u09a4\u09cd\u09a4\u09cd\u09ac was being . ||| -224.221
+ rabindranath , more than one \u09b6\u09c8\u09b2\u09c0 \u0986\u09af\u09bc\u09a4\u09cd\u09a4\u09cd\u09ac was . ||| -224.482
+ rabindranath , in many \u09b6\u09c8\u09b2\u09c0 \u0986\u09af\u09bc\u09a4\u09cd\u09a4\u09cd\u09ac was . ||| -224.552
+ rabindranath , in a number of \u09b6\u09c8\u09b2\u09c0 \u0986\u09af\u09bc\u09a4\u09cd\u09a4\u09cd\u09ac was . ||| -224.584
+ however , rabindranath more than one \u09b6\u09c8\u09b2\u09c0 \u0986\u09af\u09bc\u09a4\u09cd\u09a4\u09cd\u09ac was . ||| -224.720
+ rabindranath , in multiple \u09b6\u09c8\u09b2\u09c0 \u0986\u09af\u09bc\u09a4\u09cd\u09a4\u09cd\u09ac was . ||| -224.729
+ labor economy . ||| -2.036
+ labour economics ||| -2.248
+ labor economy ||| -2.358
+ labor economics ||| -4.475
+ labour economy ||| -4.807
+ labor economy of ||| -5.625
+ britain at the same time in the world of the and his economic power was . ||| -28.193
+ britain at some point of time the world of the and his economic power was . ||| -28.201
+ britain at the same time in the world of the main and his economic power was . ||| -28.256
+ britain at some point of time the world of the main and his economic power was . ||| -28.263
+ britain at some point of time the world the and his economic power was . ||| -28.308
+ britain at some point of time in the world of the and his economic power was . ||| -28.467
+ britain at some point of time in the world of the main and his economic power was . ||| -28.530
+ britain at a time in the world of the and his economic power was . ||| -28.540
+ britain at the same time in the world the and his economic power was . ||| -28.715
+ britain at the same time in the world of the and his economical power was . ||| -29.089
+ the military rule movement against and the \u0997\u09cb\u09b7\u09cd\u09a0\u09c0\u0997\u09a4 was against protest and movement of the bengalis independence on \u09a7\u09be\u09ac\u09bf\u09a4 to for he widely praised . ||| -279.788
+ the military rule movement against and the \u0997\u09cb\u09b7\u09cd\u09a0\u09c0\u0997\u09a4 was protest against and movement of the bengalis independence on \u09a7\u09be\u09ac\u09bf\u09a4 to for he widely praised . ||| -279.948
+ the military rule movement against and pakistan \u0997\u09cb\u09b7\u09cd\u09a0\u09c0\u0997\u09a4 was against protest and movement of the bengalis independence on \u09a7\u09be\u09ac\u09bf\u09a4 to for he widely praised . ||| -279.988
+ the military rule movement against and the \u0997\u09cb\u09b7\u09cd\u09a0\u09c0\u0997\u09a4 was against protest and of the bengalis movement independence on \u09a7\u09be\u09ac\u09bf\u09a4 to for he widely praised . ||| -279.993
+ the military rule movement against and the \u0997\u09cb\u09b7\u09cd\u09a0\u09c0\u0997\u09a4 was against protest and of the bengalis movement of on \u09a7\u09be\u09ac\u09bf\u09a4 to for he widely praised . ||| -280.102
+ the military rule movement against and the \u0997\u09cb\u09b7\u09cd\u09a0\u09c0\u0997\u09a4 was against protest and of the bengalis movement of independence on \u09a7\u09be\u09ac\u09bf\u09a4 to for he widely praised . ||| -280.103
+ the military rule movement against and pakistan \u0997\u09cb\u09b7\u09cd\u09a0\u09c0\u0997\u09a4 was protest against and movement of the bengalis independence on \u09a7\u09be\u09ac\u09bf\u09a4 to for he widely praised . ||| -280.147
+ the military rule movement against and the \u0997\u09cb\u09b7\u09cd\u09a0\u09c0\u0997\u09a4 was protest against and of the bengalis movement independence on \u09a7\u09be\u09ac\u09bf\u09a4 to for he widely praised . ||| -280.152
+ the military rule movement against and pakistan \u0997\u09cb\u09b7\u09cd\u09a0\u09c0\u0997\u09a4 was against protest and of the bengalis movement independence on \u09a7\u09be\u09ac\u09bf\u09a4 to for he widely praised . ||| -280.192
+ the military rule movement against and the \u0997\u09cb\u09b7\u09cd\u09a0\u09c0\u0997\u09a4 inequality first protest against and movement of the bengalis independence on \u09a7\u09be\u09ac\u09bf\u09a4 to for he widely praised . ||| -280.219
+ is mentioned here is required that were internet and world wide web \u2019 s synonymous word to be but actually \u09b6\u09ac\u09cd\u09a6\u09a6\u09cd\u09ac\u09af\u09bc different subject to the . ||| -187.414
+ here mentioned is that were internet and world wide web \u2019 s synonymous word to be but actually \u09b6\u09ac\u09cd\u09a6\u09a6\u09cd\u09ac\u09af\u09bc different subject to the . ||| -187.625
+ is mentioned here is required that were internet and world wide web \u2019 s synonymous word as considered as but actually \u09b6\u09ac\u09cd\u09a6\u09a6\u09cd\u09ac\u09af\u09bc different subject to the . ||| -187.629
+ here mentioned is that were internet and world wide web \u2019 s synonymous word as considered as but actually \u09b6\u09ac\u09cd\u09a6\u09a6\u09cd\u09ac\u09af\u09bc different subject to the . ||| -187.840
+ is mentioned here is required that were internet and world wide web \u2019 s synonymous word to be but actually \u09b6\u09ac\u09cd\u09a6\u09a6\u09cd\u09ac\u09af\u09bc different subject to . ||| -187.840
+ is mentioned here is required that were internet and world wide web \u2019 s synonymous word to be but actually \u09b6\u09ac\u09cd\u09a6\u09a6\u09cd\u09ac\u09af\u09bc different subject indicates the . ||| -187.927
+ here mentioned is required that were internet and world wide web \u2019 s synonymous word to be but actually \u09b6\u09ac\u09cd\u09a6\u09a6\u09cd\u09ac\u09af\u09bc different subject to the . ||| -187.968
+ here mentioned is that were internet and world wide web \u2019 s synonymous word to be but actually \u09b6\u09ac\u09cd\u09a6\u09a6\u09cd\u09ac\u09af\u09bc different subject to . ||| -188.050
+ is mentioned here is required that were internet and world wide web \u2019 s synonymous word as considered as but actually \u09b6\u09ac\u09cd\u09a6\u09a6\u09cd\u09ac\u09af\u09bc different subject to . ||| -188.055
+ is mentioned here is required that were internet and world wide web \u2019 s synonymous word to be though actually \u09b6\u09ac\u09cd\u09a6\u09a6\u09cd\u09ac\u09af\u09bc different subject to the . ||| -188.063
+ . z is related polar co-ordinate two is r = . ||| -38.983
+ . z is related polar co-ordinate two are r = . ||| -39.102
+ . z its related polar co-ordinate two is r = . ||| -39.400
+ . z its related polar co-ordinate two are r = . ||| -39.519
+ . z it related polar co-ordinate two are r = . ||| -39.683
+ . z it 's related polar co-ordinate two are r = . ||| -39.693
+ . z the related polar co-ordinate two are r = . ||| -39.807
+ . z is the related polar co-ordinate two is r = . ||| -40.038
+ . z is the related polar co-ordinate two are r = . ||| -40.546
+ november ||| 0.257
+ november . ||| -3.433
+ november of ||| -5.024
+ 1972 \u0996\u09cd\u09b0\u09c0\u09b8\u09cd\u099f\u09be\u09ac\u09cd\u09a6\u09c7 then \u0986\u09b0\u09aa\u09be\u09a8\u09c7\u099f\u09c7 first \u0987\u09b2\u09c7\u0995\u09cd\u099f\u09cd\u09b0\u09a8\u09bf\u0995 mail to to . ||| -345.087
+ 1972 \u0996\u09cd\u09b0\u09c0\u09b8\u09cd\u099f\u09be\u09ac\u09cd\u09a6\u09c7 then \u0986\u09b0\u09aa\u09be\u09a8\u09c7\u099f\u09c7 first \u0987\u09b2\u09c7\u0995\u09cd\u099f\u09cd\u09b0\u09a8\u09bf\u0995 mail to be the . ||| -345.088
+ 1972 \u0996\u09cd\u09b0\u09c0\u09b8\u09cd\u099f\u09be\u09ac\u09cd\u09a6\u09c7 then \u0986\u09b0\u09aa\u09be\u09a8\u09c7\u099f\u09c7 first ever \u0987\u09b2\u09c7\u0995\u09cd\u099f\u09cd\u09b0\u09a8\u09bf\u0995 mail to to . ||| -345.346
+ 1972 \u0996\u09cd\u09b0\u09c0\u09b8\u09cd\u099f\u09be\u09ac\u09cd\u09a6\u09c7 then \u0986\u09b0\u09aa\u09be\u09a8\u09c7\u099f\u09c7 first ever \u0987\u09b2\u09c7\u0995\u09cd\u099f\u09cd\u09b0\u09a8\u09bf\u0995 mail to be the . ||| -345.348
+ 1972 \u0996\u09cd\u09b0\u09c0\u09b8\u09cd\u099f\u09be\u09ac\u09cd\u09a6\u09c7 " \u0986\u09b0\u09aa\u09be\u09a8\u09c7\u099f\u09c7 first \u0987\u09b2\u09c7\u0995\u09cd\u099f\u09cd\u09b0\u09a8\u09bf\u0995 mail to to . ||| -345.404
+ 1972 \u0996\u09cd\u09b0\u09c0\u09b8\u09cd\u099f\u09be\u09ac\u09cd\u09a6\u09c7 " \u0986\u09b0\u09aa\u09be\u09a8\u09c7\u099f\u09c7 first \u0987\u09b2\u09c7\u0995\u09cd\u099f\u09cd\u09b0\u09a8\u09bf\u0995 mail to be the . ||| -345.406
+ 1972 \u0996\u09cd\u09b0\u09c0\u09b8\u09cd\u099f\u09be\u09ac\u09cd\u09a6\u09c7 then \u0986\u09b0\u09aa\u09be\u09a8\u09c7\u099f\u09c7 first \u0987\u09b2\u09c7\u0995\u09cd\u099f\u09cd\u09b0\u09a8\u09bf\u0995 mail to be . ||| -345.424
+ 1972 \u0996\u09cd\u09b0\u09c0\u09b8\u09cd\u099f\u09be\u09ac\u09cd\u09a6\u09c7 then professor \u0986\u09b0\u09aa\u09be\u09a8\u09c7\u099f\u09c7 first \u0987\u09b2\u09c7\u0995\u09cd\u099f\u09cd\u09b0\u09a8\u09bf\u0995 mail to to . ||| -345.504
+ 1972 \u0996\u09cd\u09b0\u09c0\u09b8\u09cd\u099f\u09be\u09ac\u09cd\u09a6\u09c7 then professor \u0986\u09b0\u09aa\u09be\u09a8\u09c7\u099f\u09c7 first \u0987\u09b2\u09c7\u0995\u09cd\u099f\u09cd\u09b0\u09a8\u09bf\u0995 mail to be the . ||| -345.505
+ 1972 \u0996\u09cd\u09b0\u09c0\u09b8\u09cd\u099f\u09be\u09ac\u09cd\u09a6\u09c7 previously called \u0986\u09b0\u09aa\u09be\u09a8\u09c7\u099f\u09c7 first \u0987\u09b2\u09c7\u0995\u09cd\u099f\u09cd\u09b0\u09a8\u09bf\u0995 mail to to . ||| -345.587
+ \u099c\u09c0\u09ac science that \u09b6\u09be\u0996\u09be\u09af\u09bc fungi and its practical subject about to the he was \u099b\u09a4\u09cd\u09b0\u09be\u0995\u09ac\u09bf\u09a6\u09cd\u09af\u09be -lrb- mycology -rrb- ||| -464.793
+ \u099c\u09c0\u09ac science that \u09b6\u09be\u0996\u09be\u09af\u09bc fungi and its practical issues about to the he was \u099b\u09a4\u09cd\u09b0\u09be\u0995\u09ac\u09bf\u09a6\u09cd\u09af\u09be -lrb- mycology -rrb- ||| -464.821
+ \u099c\u09c0\u09ac science that \u09b6\u09be\u0996\u09be\u09af\u09bc fungi and its practical subject about the to he was \u099b\u09a4\u09cd\u09b0\u09be\u0995\u09ac\u09bf\u09a6\u09cd\u09af\u09be -lrb- mycology -rrb- ||| -464.935
+ \u099c\u09c0\u09ac science that \u09b6\u09be\u0996\u09be\u09af\u09bc fungi and its practical issues about the to he was \u099b\u09a4\u09cd\u09b0\u09be\u0995\u09ac\u09bf\u09a6\u09cd\u09af\u09be -lrb- mycology -rrb- ||| -464.963
+ \u099c\u09c0\u09ac science that \u09b6\u09be\u0996\u09be\u09af\u09bc fungi and its practical issue about to the he was \u099b\u09a4\u09cd\u09b0\u09be\u0995\u09ac\u09bf\u09a6\u09cd\u09af\u09be -lrb- mycology -rrb- ||| -465.138
+ \u099c\u09c0\u09ac science that \u09b6\u09be\u0996\u09be\u09af\u09bc fungi and its practical issues with about to he was \u099b\u09a4\u09cd\u09b0\u09be\u0995\u09ac\u09bf\u09a6\u09cd\u09af\u09be -lrb- mycology -rrb- ||| -465.198
+ \u099c\u09c0\u09ac science that \u09b6\u09be\u0996\u09be\u09af\u09bc fungi and the practical subject about to the he was \u099b\u09a4\u09cd\u09b0\u09be\u0995\u09ac\u09bf\u09a6\u09cd\u09af\u09be -lrb- mycology -rrb- ||| -465.199
+ \u099c\u09c0\u09ac science that \u09b6\u09be\u0996\u09be\u09af\u09bc fungi and the practical issues about to the he was \u099b\u09a4\u09cd\u09b0\u09be\u0995\u09ac\u09bf\u09a6\u09cd\u09af\u09be -lrb- mycology -rrb- ||| -465.227
+ \u099c\u09c0\u09ac science that \u09b6\u09be\u0996\u09be\u09af\u09bc fungi and its practical issue about the to he was \u099b\u09a4\u09cd\u09b0\u09be\u0995\u09ac\u09bf\u09a6\u09cd\u09af\u09be -lrb- mycology -rrb- ||| -465.280
+ \u099c\u09c0\u09ac science that \u09b6\u09be\u0996\u09be\u09af\u09bc fungus and its practical subject about to the he was \u099b\u09a4\u09cd\u09b0\u09be\u0995\u09ac\u09bf\u09a6\u09cd\u09af\u09be -lrb- mycology -rrb- ||| -465.303
+ water river from \u0989\u09a0\u09be\u09a8\u09cb it was a few purs \u09a6\u09bf\u09af\u09bc\u09c7- \u09a6\u09a1\u09bc\u09bf and \u09ac\u09be\u09b2\u099f\u09bf\u09b0 through water animal by can of world in a method . ||| -479.857
+ water river from \u0989\u09a0\u09be\u09a8\u09cb it was a few purs \u09a6\u09bf\u09af\u09bc\u09c7- \u09a6\u09a1\u09bc\u09bf and \u09ac\u09be\u09b2\u099f\u09bf\u09b0 through the water animal by can of world in a method . ||| -479.921
+ water river from \u0989\u09a0\u09be\u09a8\u09cb it was a few purs \u09a6\u09bf\u09af\u09bc\u09c7- \u09a6\u09a1\u09bc\u09bf and \u09ac\u09be\u09b2\u099f\u09bf\u09b0 through water animal by can of world a method . ||| -479.965
+ water river from \u0989\u09a0\u09be\u09a8\u09cb it was a few purs \u09a6\u09bf\u09af\u09bc\u09c7- \u09a6\u09a1\u09bc\u09bf and \u09ac\u09be\u09b2\u099f\u09bf\u09b0 through water animal by can world in a method . ||| -479.974
+ water river from \u0989\u09a0\u09be\u09a8\u09cb it was a few purs \u09a6\u09bf\u09af\u09bc\u09c7- \u09a6\u09a1\u09bc\u09bf and \u09ac\u09be\u09b2\u099f\u09bf\u09b0 through water animal by used of world in a method . ||| -480.007
+ water river from \u0989\u09a0\u09be\u09a8\u09cb it was a few purs \u09a6\u09bf\u09af\u09bc\u09c7- \u09a6\u09a1\u09bc\u09bf and \u09ac\u09be\u09b2\u099f\u09bf\u09b0 through the water animal by can of world a method . ||| -480.029
+ water river from \u0989\u09a0\u09be\u09a8\u09cb it was a few purs \u09a6\u09bf\u09af\u09bc\u09c7- \u09a6\u09a1\u09bc\u09bf and \u09ac\u09be\u09b2\u099f\u09bf\u09b0 through water animal by can bhola 's a method . ||| -480.029
+ water river from \u0989\u09a0\u09be\u09a8\u09cb it was a few purs \u09a6\u09bf\u09af\u09bc\u09c7- \u09a6\u09a1\u09bc\u09bf and \u09ac\u09be\u09b2\u099f\u09bf\u09b0 through the water animal by can world in a method . ||| -480.038
+ water river from \u0989\u09a0\u09be\u09a8\u09cb it was a few purs \u09a6\u09bf\u09af\u09bc\u09c7- \u09a6\u09a1\u09bc\u09bf and \u09ac\u09be\u09b2\u099f\u09bf\u09b0 through the water animal by used of world in a method . ||| -480.071
+ water river from \u0989\u09a0\u09be\u09a8\u09cb it was a few purs \u09a6\u09bf\u09af\u09bc\u09c7- \u09a6\u09a1\u09bc\u09bf and \u09ac\u09be\u09b2\u099f\u09bf\u09b0 through water animal by can world a method . ||| -480.081
+ among these there tribal dance , lokuj dance , classical dance , etc . ||| -37.853
+ among these there are tribal dance , lokuj dance , classical dance , etc . ||| -38.404
+ among these there tribal dance , influences impact dance , classical dance , etc . ||| -38.447
+ among them are tribal dance , lokuj dance , classical dance , etc . ||| -38.605
+ among these there tribal dance , lokuj dance , the classical dance , etc . ||| -38.918
+ among these there are tribal dance , influences impact dance , classical dance , etc . ||| -38.997
+ among these there are tribal dance , lokuj dance , the classical dance , etc . ||| -39.468
+ the oldest literature at first \u09ae\u09cc\u0996\u09bf\u0995\u09ad\u09be\u09ac\u09c7 and later written form is the . ||| -132.632
+ the oldest literature at first \u09ae\u09cc\u0996\u09bf\u0995\u09ad\u09be\u09ac\u09c7 and later written form is in the . ||| -132.742
+ the oldest literature at first \u09ae\u09cc\u0996\u09bf\u0995\u09ad\u09be\u09ac\u09c7 and later written form is . ||| -132.785
+ the oldest literature at first \u09ae\u09cc\u0996\u09bf\u0995\u09ad\u09be\u09ac\u09c7 and later written in the form is the . ||| -133.224
+ the oldest literature at first \u09ae\u09cc\u0996\u09bf\u0995\u09ad\u09be\u09ac\u09c7 and later written form is in . ||| -133.319
+ the oldest literature at first \u09ae\u09cc\u0996\u09bf\u0995\u09ad\u09be\u09ac\u09c7 and later written in the form is in the . ||| -133.334
+ the oldest literature at first \u09ae\u09cc\u0996\u09bf\u0995\u09ad\u09be\u09ac\u09c7 and later written in the form is . ||| -133.376
+ the oldest literature at first \u09ae\u09cc\u0996\u09bf\u0995\u09ad\u09be\u09ac\u09c7 and later written form the . ||| -133.443
+ the oldest literature at first \u09ae\u09cc\u0996\u09bf\u0995\u09ad\u09be\u09ac\u09c7 and later written in the . ||| -133.812
+ the oldest literature at first \u09ae\u09cc\u0996\u09bf\u0995\u09ad\u09be\u09ac\u09c7 and later written in the form is in . ||| -133.910
+ in 1989 , the \u0997\u09a3\u09b6\u09a4\u09cd\u09b0\u09c1 in the film his conducted \u09a4\u09c1\u09b2\u09a8\u09be\u09ae\u09c2\u09b2\u0995\u09ad\u09be\u09ac\u09c7 \u09a6\u09c1\u09b0\u09cd\u09ac\u09b2 and this \u09a6\u09c0\u09b0\u09cd\u0998\u09a6\u09bf\u09a8\u09c7\u09b0 \u0985\u09b8\u09c1\u09b8\u09cd\u09a5\u09a4\u09be\u09b6\u09c7\u09b7\u09c7 back had come after of satyajit 's films , \u09aa\u09c1\u09a8\u09b0\u09cd\u09aa\u09cd\u09b0\u099a\u09c7\u09b7\u09cd\u099f\u09be a is considered as . ||| -699.161
+ in 1989 , the \u0997\u09a3\u09b6\u09a4\u09cd\u09b0\u09c1 in this film his conducted \u09a4\u09c1\u09b2\u09a8\u09be\u09ae\u09c2\u09b2\u0995\u09ad\u09be\u09ac\u09c7 \u09a6\u09c1\u09b0\u09cd\u09ac\u09b2 and this \u09a6\u09c0\u09b0\u09cd\u0998\u09a6\u09bf\u09a8\u09c7\u09b0 \u0985\u09b8\u09c1\u09b8\u09cd\u09a5\u09a4\u09be\u09b6\u09c7\u09b7\u09c7 back had come after of satyajit 's films , \u09aa\u09c1\u09a8\u09b0\u09cd\u09aa\u09cd\u09b0\u099a\u09c7\u09b7\u09cd\u099f\u09be a is considered as . ||| -699.174
+ in 1989 , the \u0997\u09a3\u09b6\u09a4\u09cd\u09b0\u09c1 in the film his conducted \u09a4\u09c1\u09b2\u09a8\u09be\u09ae\u09c2\u09b2\u0995\u09ad\u09be\u09ac\u09c7 \u09a6\u09c1\u09b0\u09cd\u09ac\u09b2 and it \u09a6\u09c0\u09b0\u09cd\u0998\u09a6\u09bf\u09a8\u09c7\u09b0 \u0985\u09b8\u09c1\u09b8\u09cd\u09a5\u09a4\u09be\u09b6\u09c7\u09b7\u09c7 back had come after of satyajit 's films , \u09aa\u09c1\u09a8\u09b0\u09cd\u09aa\u09cd\u09b0\u099a\u09c7\u09b7\u09cd\u099f\u09be a is considered as . ||| -699.330
+ in 1989 , the \u0997\u09a3\u09b6\u09a4\u09cd\u09b0\u09c1 in this film his conducted \u09a4\u09c1\u09b2\u09a8\u09be\u09ae\u09c2\u09b2\u0995\u09ad\u09be\u09ac\u09c7 \u09a6\u09c1\u09b0\u09cd\u09ac\u09b2 and it \u09a6\u09c0\u09b0\u09cd\u0998\u09a6\u09bf\u09a8\u09c7\u09b0 \u0985\u09b8\u09c1\u09b8\u09cd\u09a5\u09a4\u09be\u09b6\u09c7\u09b7\u09c7 back had come after of satyajit 's films , \u09aa\u09c1\u09a8\u09b0\u09cd\u09aa\u09cd\u09b0\u099a\u09c7\u09b7\u09cd\u099f\u09be a is considered as . ||| -699.343
+ in 1989 , the \u0997\u09a3\u09b6\u09a4\u09cd\u09b0\u09c1 in the film of his conducted \u09a4\u09c1\u09b2\u09a8\u09be\u09ae\u09c2\u09b2\u0995\u09ad\u09be\u09ac\u09c7 \u09a6\u09c1\u09b0\u09cd\u09ac\u09b2 and this \u09a6\u09c0\u09b0\u09cd\u0998\u09a6\u09bf\u09a8\u09c7\u09b0 \u0985\u09b8\u09c1\u09b8\u09cd\u09a5\u09a4\u09be\u09b6\u09c7\u09b7\u09c7 back had come after of satyajit 's films , \u09aa\u09c1\u09a8\u09b0\u09cd\u09aa\u09cd\u09b0\u099a\u09c7\u09b7\u09cd\u099f\u09be a is considered as . ||| -699.580
+ in 1989 , the \u0997\u09a3\u09b6\u09a4\u09cd\u09b0\u09c1 in the film his conducted \u09a4\u09c1\u09b2\u09a8\u09be\u09ae\u09c2\u09b2\u0995\u09ad\u09be\u09ac\u09c7 \u09a6\u09c1\u09b0\u09cd\u09ac\u09b2 and this \u09a6\u09c0\u09b0\u09cd\u0998\u09a6\u09bf\u09a8\u09c7\u09b0 \u0985\u09b8\u09c1\u09b8\u09cd\u09a5\u09a4\u09be\u09b6\u09c7\u09b7\u09c7 back had come after of satyajit 's films building \u09aa\u09c1\u09a8\u09b0\u09cd\u09aa\u09cd\u09b0\u099a\u09c7\u09b7\u09cd\u099f\u09be a is considered as . ||| -699.725
+ in 1989 , the \u0997\u09a3\u09b6\u09a4\u09cd\u09b0\u09c1 in this film of his conducted \u09a4\u09c1\u09b2\u09a8\u09be\u09ae\u09c2\u09b2\u0995\u09ad\u09be\u09ac\u09c7 \u09a6\u09c1\u09b0\u09cd\u09ac\u09b2 and this \u09a6\u09c0\u09b0\u09cd\u0998\u09a6\u09bf\u09a8\u09c7\u09b0 \u0985\u09b8\u09c1\u09b8\u09cd\u09a5\u09a4\u09be\u09b6\u09c7\u09b7\u09c7 back had come after of satyajit 's films , \u09aa\u09c1\u09a8\u09b0\u09cd\u09aa\u09cd\u09b0\u099a\u09c7\u09b7\u09cd\u099f\u09be a is considered as . ||| -699.727
+ in 1989 , the \u0997\u09a3\u09b6\u09a4\u09cd\u09b0\u09c1 in this film his conducted \u09a4\u09c1\u09b2\u09a8\u09be\u09ae\u09c2\u09b2\u0995\u09ad\u09be\u09ac\u09c7 \u09a6\u09c1\u09b0\u09cd\u09ac\u09b2 and this \u09a6\u09c0\u09b0\u09cd\u0998\u09a6\u09bf\u09a8\u09c7\u09b0 \u0985\u09b8\u09c1\u09b8\u09cd\u09a5\u09a4\u09be\u09b6\u09c7\u09b7\u09c7 back had come after of satyajit 's films building \u09aa\u09c1\u09a8\u09b0\u09cd\u09aa\u09cd\u09b0\u099a\u09c7\u09b7\u09cd\u099f\u09be a is considered as . ||| -699.739
+ in 1989 , the \u0997\u09a3\u09b6\u09a4\u09cd\u09b0\u09c1 in the film of his conducted \u09a4\u09c1\u09b2\u09a8\u09be\u09ae\u09c2\u09b2\u0995\u09ad\u09be\u09ac\u09c7 \u09a6\u09c1\u09b0\u09cd\u09ac\u09b2 and it \u09a6\u09c0\u09b0\u09cd\u0998\u09a6\u09bf\u09a8\u09c7\u09b0 \u0985\u09b8\u09c1\u09b8\u09cd\u09a5\u09a4\u09be\u09b6\u09c7\u09b7\u09c7 back had come after of satyajit 's films , \u09aa\u09c1\u09a8\u09b0\u09cd\u09aa\u09cd\u09b0\u099a\u09c7\u09b7\u09cd\u099f\u09be a is considered as . ||| -699.749
+ in 1989 , the \u0997\u09a3\u09b6\u09a4\u09cd\u09b0\u09c1 in the film his between \u09a4\u09c1\u09b2\u09a8\u09be\u09ae\u09c2\u09b2\u0995\u09ad\u09be\u09ac\u09c7 \u09a6\u09c1\u09b0\u09cd\u09ac\u09b2 and this \u09a6\u09c0\u09b0\u09cd\u0998\u09a6\u09bf\u09a8\u09c7\u09b0 \u0985\u09b8\u09c1\u09b8\u09cd\u09a5\u09a4\u09be\u09b6\u09c7\u09b7\u09c7 back had come after of satyajit 's films , \u09aa\u09c1\u09a8\u09b0\u09cd\u09aa\u09cd\u09b0\u099a\u09c7\u09b7\u09cd\u099f\u09be a is considered as . ||| -699.773
+ the \u09ac\u09b2\u09ac\u09bf\u09a6\u09cd\u09af\u09be ||| -110.735
+ mathematical \u09ac\u09b2\u09ac\u09bf\u09a6\u09cd\u09af\u09be ||| -111.085
+ \u2022 \u09ac\u09b2\u09ac\u09bf\u09a6\u09cd\u09af\u09be ||| -111.198
+ . \u09ac\u09b2\u09ac\u09bf\u09a6\u09cd\u09af\u09be ||| -114.066
+ mathematical theory . \u09ac\u09b2\u09ac\u09bf\u09a6\u09cd\u09af\u09be ||| -115.120
+ the mathematical \u09ac\u09b2\u09ac\u09bf\u09a6\u09cd\u09af\u09be ||| -116.452
+ of the mathematical \u09ac\u09b2\u09ac\u09bf\u09a6\u09cd\u09af\u09be ||| -117.545
+ other \u09b8\u09cd\u09ac\u09a4\u09cd\u09a4\u09cd\u09ac-\u09b8\u0982\u09b0\u0995\u09cd\u09b7\u09bf\u09a4 operating system for windows and mac os to linux in different different . ||| -133.431
+ other \u09b8\u09cd\u09ac\u09a4\u09cd\u09a4\u09cd\u09ac-\u09b8\u0982\u09b0\u0995\u09cd\u09b7\u09bf\u09a4 operating system like windows and mac os to linux in different different . ||| -133.448
+ other \u09b8\u09cd\u09ac\u09a4\u09cd\u09a4\u09cd\u09ac-\u09b8\u0982\u09b0\u0995\u09cd\u09b7\u09bf\u09a4 operating systems like windows and mac os to linux in different different . ||| -133.827
+ other \u09b8\u09cd\u09ac\u09a4\u09cd\u09a4\u09cd\u09ac-\u09b8\u0982\u09b0\u0995\u09cd\u09b7\u09bf\u09a4 operating system for windows and mac os to linux in different separate . ||| -134.040
+ other \u09b8\u09cd\u09ac\u09a4\u09cd\u09a4\u09cd\u09ac-\u09b8\u0982\u09b0\u0995\u09cd\u09b7\u09bf\u09a4 operating system like windows and mac os to linux in different separate . ||| -134.057
+ the other \u09b8\u09cd\u09ac\u09a4\u09cd\u09a4\u09cd\u09ac-\u09b8\u0982\u09b0\u0995\u09cd\u09b7\u09bf\u09a4 operating system like windows and mac os to linux in different different . ||| -134.130
+ other \u09b8\u09cd\u09ac\u09a4\u09cd\u09a4\u09cd\u09ac-\u09b8\u0982\u09b0\u0995\u09cd\u09b7\u09bf\u09a4 operating system for windows and mac os to the linux in different different . ||| -134.224
+ other \u09b8\u09cd\u09ac\u09a4\u09cd\u09a4\u09cd\u09ac-\u09b8\u0982\u09b0\u0995\u09cd\u09b7\u09bf\u09a4 operating system like windows and mac os to the linux in different different . ||| -134.241
+ other \u09b8\u09cd\u09ac\u09a4\u09cd\u09a4\u09cd\u09ac-\u09b8\u0982\u09b0\u0995\u09cd\u09b7\u09bf\u09a4 operating system for windows and mac os from the linux in different different . ||| -134.254
+ other \u09b8\u09cd\u09ac\u09a4\u09cd\u09a4\u09cd\u09ac-\u09b8\u0982\u09b0\u0995\u09cd\u09b7\u09bf\u09a4 operating system like windows and mac os from the linux in different different . ||| -134.270
+ asia \u099f\u09be\u0987\u09ae\u09b8\u09c7\u09b0 \u09ad\u09be\u09b7\u09cd\u09af according to , ||| -218.686
+ asia \u099f\u09be\u0987\u09ae\u09b8\u09c7\u09b0 \u09ad\u09be\u09b7\u09cd\u09af according to the , ||| -219.408
+ asia \u099f\u09be\u0987\u09ae\u09b8\u09c7\u09b0 \u09ad\u09be\u09b7\u09cd\u09af to the , ||| -219.724
+ asia \u099f\u09be\u0987\u09ae\u09b8\u09c7\u09b0 \u09ad\u09be\u09b7\u09cd\u09af to , ||| -220.516
+ the \u099f\u09be\u0987\u09ae\u09b8\u09c7\u09b0 \u09ad\u09be\u09b7\u09cd\u09af according to , ||| -220.701
+ asia \u099f\u09be\u0987\u09ae\u09b8\u09c7\u09b0 \u09ad\u09be\u09b7\u09cd\u09af according to , the ||| -220.728
+ asia \u099f\u09be\u0987\u09ae\u09b8\u09c7\u09b0 \u09ad\u09be\u09b7\u09cd\u09af according to , and ||| -220.812
+ asia \u099f\u09be\u0987\u09ae\u09b8\u09c7\u09b0 \u09ad\u09be\u09b7\u09cd\u09af according to the , the ||| -221.449
+ asia \u099f\u09be\u0987\u09ae\u09b8\u09c7\u09b0 \u09ad\u09be\u09b7\u09cd\u09af according to , in ||| -221.531
+ asia \u099f\u09be\u0987\u09ae\u09b8\u09c7\u09b0 \u09ad\u09be\u09b7\u09cd\u09af according to the , and ||| -221.534
+ open source or open source -lrb- open source -rrb- the money is computer software the source code or main \u09b8\u09be\u0982\u0995\u09c7\u09a4\u09bf\u0995 language open way \u09ac\u09bf\u09a4\u09b0\u09a3 to . ||| -471.566
+ open source or open source -lrb- open source -rrb- , the money is computer software the source code or main \u09b8\u09be\u0982\u0995\u09c7\u09a4\u09bf\u0995 language open way \u09ac\u09bf\u09a4\u09b0\u09a3 to . ||| -471.638
+ open source or open source -lrb- open source -rrb- the money is computer software of the source code or main \u09b8\u09be\u0982\u0995\u09c7\u09a4\u09bf\u0995 language open way \u09ac\u09bf\u09a4\u09b0\u09a3 to . ||| -471.644
+ open source or open source -lrb- open source -rrb- of the money is computer software the source code or main \u09b8\u09be\u0982\u0995\u09c7\u09a4\u09bf\u0995 language open way \u09ac\u09bf\u09a4\u09b0\u09a3 to . ||| -471.711
+ open source or open source -lrb- open source -rrb- the money is computer software the source code or main \u09b8\u09be\u0982\u0995\u09c7\u09a4\u09bf\u0995 language free way \u09ac\u09bf\u09a4\u09b0\u09a3 to . ||| -471.716
+ open source or open source -lrb- open source -rrb- , the money is computer software of the source code or main \u09b8\u09be\u0982\u0995\u09c7\u09a4\u09bf\u0995 language open way \u09ac\u09bf\u09a4\u09b0\u09a3 to . ||| -471.717
+ open source or open source -lrb- open source -rrb- , the money is computer software the source code or main \u09b8\u09be\u0982\u0995\u09c7\u09a4\u09bf\u0995 language free way \u09ac\u09bf\u09a4\u09b0\u09a3 to . ||| -471.789
+ open source or open source -lrb- open source -rrb- of the money is computer software of the source code or main \u09b8\u09be\u0982\u0995\u09c7\u09a4\u09bf\u0995 language open way \u09ac\u09bf\u09a4\u09b0\u09a3 to . ||| -471.790
+ open source or open source -lrb- open source -rrb- the money is computer software of the source code or main \u09b8\u09be\u0982\u0995\u09c7\u09a4\u09bf\u0995 language free way \u09ac\u09bf\u09a4\u09b0\u09a3 to . ||| -471.794
+ open source or open source -lrb- open source -rrb- in the money is computer software the source code or main \u09b8\u09be\u0982\u0995\u09c7\u09a4\u09bf\u0995 language open way \u09ac\u09bf\u09a4\u09b0\u09a3 to . ||| -471.850
+ bangladesh \u0985\u09a8\u09b2\u09be\u0987\u09a8\u09c7 dhaka ||| -109.639
+ bangladesh \u0985\u09a8\u09b2\u09be\u0987\u09a8\u09c7 of dhaka ||| -110.190
+ the bangladesh \u0985\u09a8\u09b2\u09be\u0987\u09a8\u09c7 dhaka ||| -110.961
+ the bangladesh \u0985\u09a8\u09b2\u09be\u0987\u09a8\u09c7 of dhaka ||| -111.512
+ the first world war germany \u09b9\u09c7\u09b0\u09c7 can be . ||| -113.950
+ the first world war of germany \u09b9\u09c7\u09b0\u09c7 can be . ||| -114.276
+ the first world war germany \u09b9\u09c7\u09b0\u09c7 can be seen . ||| -114.953
+ of the first world war germany \u09b9\u09c7\u09b0\u09c7 can be . ||| -115.018
+ the first world war of germany \u09b9\u09c7\u09b0\u09c7 can be seen . ||| -115.280
+ the first world war german \u09b9\u09c7\u09b0\u09c7 can be . ||| -115.333
+ of the first world war of germany \u09b9\u09c7\u09b0\u09c7 can be . ||| -115.345
+ the first world war germany \u09b9\u09c7\u09b0\u09c7 that can be . ||| -115.434
+ the first world war conquered \u09b9\u09c7\u09b0\u09c7 can be . ||| -115.553
+ first world war germany \u09b9\u09c7\u09b0\u09c7 can be . ||| -115.694
+ but this subject is to understand for even research to going on . ||| -34.990
+ but this subject is to understand for even research progress going on . ||| -35.178
+ but this subject is to understand for at present also research to going on . ||| -35.277
+ but in this matter is to understand for even research to going on . ||| -35.392
+ but this subject is to understand for today also research to going on . ||| -35.410
+ but this subject is to understand for at present also research progress going on . ||| -35.464
+ but in this subject is to understand for even research to going on . ||| -35.590
+ but this subject is to understand for presently research to going on . ||| -35.623
+ but in this matter is to understand for at present also research to going on . ||| -35.678
+ but at this subject is to understand for even research to going on . ||| -35.785
+ \u09b8\u09c1\u09aa\u09be\u09b0\u098f\u0987\u099a ||| -104.738
+ he was the military forces for \u0986\u09a8\u09ab\u09bf\u099f was declared . ||| -122.214
+ he was the military forces to \u0986\u09a8\u09ab\u09bf\u099f was declared . ||| -122.669
+ he was military forces for \u0986\u09a8\u09ab\u09bf\u099f was declared . ||| -122.670
+ he was the military forces for \u0986\u09a8\u09ab\u09bf\u099f declared was . ||| -122.742
+ he was the military forces for \u0986\u09a8\u09ab\u09bf\u099f declared the . ||| -123.074
+ he was military forces to \u0986\u09a8\u09ab\u09bf\u099f was declared . ||| -123.125
+ he was the military forces to \u0986\u09a8\u09ab\u09bf\u099f declared was . ||| -123.198
+ he was military forces for \u0986\u09a8\u09ab\u09bf\u099f declared was . ||| -123.198
+ he was armed forces for \u0986\u09a8\u09ab\u09bf\u099f was declared . ||| -123.208
+ he was the military forces for \u0986\u09a8\u09ab\u09bf\u099f declared in . ||| -123.629
+ bhutto \u098f\u09cd\u09af\u09be\u09b8\u09c7\u09ae\u09cd\u09ac\u09b2\u09bf \u09ac\u09af\u09bc\u0995\u099f to 2.5 with declared in the , yahya khan mujib was to form the government to \u099c\u09be\u09a8\u09be\u09b2\u09c7 he that government by \u09a8\u09c7\u09ac\u09c7\u09a8 not . ||| -492.585
+ bhutto \u098f\u09cd\u09af\u09be\u09b8\u09c7\u09ae\u09cd\u09ac\u09b2\u09bf \u09ac\u09af\u09bc\u0995\u099f to 2.5 with announced that the , yahya khan mujib was to form the government to \u099c\u09be\u09a8\u09be\u09b2\u09c7 he that government by \u09a8\u09c7\u09ac\u09c7\u09a8 not . ||| -492.686
+ bhutto \u098f\u09cd\u09af\u09be\u09b8\u09c7\u09ae\u09cd\u09ac\u09b2\u09bf \u09ac\u09af\u09bc\u0995\u099f to 2.5 to declared in the , yahya khan mujib was to form the government to \u099c\u09be\u09a8\u09be\u09b2\u09c7 he that government by \u09a8\u09c7\u09ac\u09c7\u09a8 not . ||| -492.687
+ bhutto \u098f\u09cd\u09af\u09be\u09b8\u09c7\u09ae\u09cd\u09ac\u09b2\u09bf \u09ac\u09af\u09bc\u0995\u099f to 2.5 with declared in the , yahya khan mujib was to form the government on to \u099c\u09be\u09a8\u09be\u09b2\u09c7 he that government by \u09a8\u09c7\u09ac\u09c7\u09a8 not . ||| -492.743
+ bhutto \u098f\u09cd\u09af\u09be\u09b8\u09c7\u09ae\u09cd\u09ac\u09b2\u09bf \u09ac\u09af\u09bc\u0995\u099f to 2.5 with declared in the , yahya khan mujib was to form the government for \u099c\u09be\u09a8\u09be\u09b2\u09c7 he that government by \u09a8\u09c7\u09ac\u09c7\u09a8 not . ||| -492.760
+ bhutto \u098f\u09cd\u09af\u09be\u09b8\u09c7\u09ae\u09cd\u09ac\u09b2\u09bf \u09ac\u09af\u09bc\u0995\u099f to 2.5 with announced that that , yahya khan mujib was to form the government to \u099c\u09be\u09a8\u09be\u09b2\u09c7 he that government by \u09a8\u09c7\u09ac\u09c7\u09a8 not . ||| -492.771
+ bhutto \u098f\u09cd\u09af\u09be\u09b8\u09c7\u09ae\u09cd\u09ac\u09b2\u09bf \u09ac\u09af\u09bc\u0995\u099f to 2.5 with announced that the , yahya khan mujib was to form the government on to \u099c\u09be\u09a8\u09be\u09b2\u09c7 he that government by \u09a8\u09c7\u09ac\u09c7\u09a8 not . ||| -492.843
+ bhutto \u098f\u09cd\u09af\u09be\u09b8\u09c7\u09ae\u09cd\u09ac\u09b2\u09bf \u09ac\u09af\u09bc\u0995\u099f to 2.5 to declared in the , yahya khan mujib was to form the government on to \u099c\u09be\u09a8\u09be\u09b2\u09c7 he that government by \u09a8\u09c7\u09ac\u09c7\u09a8 not . ||| -492.844
+ bhutto \u098f\u09cd\u09af\u09be\u09b8\u09c7\u09ae\u09cd\u09ac\u09b2\u09bf \u09ac\u09af\u09bc\u0995\u099f to 2.5 with announced that the , yahya khan mujib was to form the government for \u099c\u09be\u09a8\u09be\u09b2\u09c7 he that government by \u09a8\u09c7\u09ac\u09c7\u09a8 not . ||| -492.861
+ bhutto \u098f\u09cd\u09af\u09be\u09b8\u09c7\u09ae\u09cd\u09ac\u09b2\u09bf \u09ac\u09af\u09bc\u0995\u099f to 2.5 to declared in the , yahya khan mujib was to form the government for \u099c\u09be\u09a8\u09be\u09b2\u09c7 he that government by \u09a8\u09c7\u09ac\u09c7\u09a8 not . ||| -492.862
+ and the computer words money \u0997\u09a3\u09a8\u09be\u0995\u09be\u09b0\u09c0 machine . ||| -220.733
+ the computer words money \u0997\u09a3\u09a8\u09be\u0995\u09be\u09b0\u09c0 machine . ||| -220.783
+ and the computer word meaning \u0997\u09a3\u09a8\u09be\u0995\u09be\u09b0\u09c0 machine . ||| -221.198
+ and computer words money \u0997\u09a3\u09a8\u09be\u0995\u09be\u09b0\u09c0 machine . ||| -221.210
+ the computer word meaning \u0997\u09a3\u09a8\u09be\u0995\u09be\u09b0\u09c0 machine . ||| -221.248
+ and for computer words money \u0997\u09a3\u09a8\u09be\u0995\u09be\u09b0\u09c0 machine . ||| -221.763
+ and the computer words means \u0997\u09a3\u09a8\u09be\u0995\u09be\u09b0\u09c0 machine . ||| -221.835
+ the computer words means \u0997\u09a3\u09a8\u09be\u0995\u09be\u09b0\u09c0 machine . ||| -221.885
+ and the computer word money \u0997\u09a3\u09a8\u09be\u0995\u09be\u09b0\u09c0 machine . ||| -221.886
+ and the computer words money \u0997\u09a3\u09a8\u09be\u0995\u09be\u09b0\u09c0 machines . ||| -222.136
+ \u09e7\u09ed\u09ed\u09ec on 4th july this colonies are a independence notice \u099c\u09be\u09b0\u09bf . ||| -240.825
+ \u09e7\u09ed\u09ed\u09ec on 4th july this colonies are a independence of notice \u099c\u09be\u09b0\u09bf . ||| -241.490
+ \u09e7\u09ed\u09ed\u09ec on 4th july this colonies are a independence written proclamation \u099c\u09be\u09b0\u09bf . ||| -241.514
+ \u09e7\u09ed\u09ed\u09ec on 4th july in this colonies are a independence notice \u099c\u09be\u09b0\u09bf . ||| -241.523
+ \u09e7\u09ed\u09ed\u09ec , on 4th july this colonies are a independence notice \u099c\u09be\u09b0\u09bf . ||| -241.562
+ \u09e7\u09ed\u09ed\u09ec on 4th july this constituents a independence notice \u099c\u09be\u09b0\u09bf . ||| -241.593
+ \u09e7\u09ed\u09ed\u09ec on 4th july of this colonies are a independence notice \u099c\u09be\u09b0\u09bf . ||| -241.692
+ \u09e7\u09ed\u09ed\u09ec on 4th july this colonies are a independence notice \u099c\u09be\u09b0\u09bf the . ||| -241.990
+ \u09e7\u09ed\u09ed\u09ec on 4th july , this colonies are a independence notice \u099c\u09be\u09b0\u09bf . ||| -242.101
+ \u09e7\u09ed\u09ed\u09ec on 4th july this colonies are a independence notice \u099c\u09be\u09b0\u09bf by the . ||| -242.148
+ germany -lrb- in german : deutschland , \u09a1\u09af\u09bc\u099a\u09cd of \u09b2\u09be\u09a8\u09cd\u099f\u09cd of , pronounced [ d\u0254\u028ft\u0283lant ] -rrb- , the central is a country of europe . ||| -467.519
+ germany -lrb- in german : deutschland , \u09a1\u09af\u09bc\u099a\u09cd of \u09b2\u09be\u09a8\u09cd\u099f\u09cd of , pronounced [ d\u0254\u028ft\u0283lant ] -rrb- , the is a country of europe . ||| -467.593
+ germany -lrb- in german : deutschland , \u09a1\u09af\u09bc\u099a\u09cd of \u09b2\u09be\u09a8\u09cd\u099f\u09cd of , pronounced [ d\u0254\u028ft\u0283lant ] -rrb- , central is a country of europe . ||| -467.745
+ germany -lrb- in german : deutschland , \u09a1\u09af\u09bc\u099a\u09cd the \u09b2\u09be\u09a8\u09cd\u099f\u09cd of , pronounced [ d\u0254\u028ft\u0283lant ] -rrb- , the central is a country of europe . ||| -467.814
+ germany -lrb- in german : deutschland , \u09a1\u09af\u09bc\u099a\u09cd of \u09b2\u09be\u09a8\u09cd\u099f\u09cd the , pronounced [ d\u0254\u028ft\u0283lant ] -rrb- , the central is a country of europe . ||| -467.819
+ germany -lrb- in german : deutschland , \u09a1\u09af\u09bc\u099a\u09cd the \u09b2\u09be\u09a8\u09cd\u099f\u09cd of , pronounced [ d\u0254\u028ft\u0283lant ] -rrb- , the is a country of europe . ||| -467.887
+ germany -lrb- in german : deutschland , \u09a1\u09af\u09bc\u099a\u09cd of \u09b2\u09be\u09a8\u09cd\u099f\u09cd the , pronounced [ d\u0254\u028ft\u0283lant ] -rrb- , the is a country of europe . ||| -467.892
+ germany -lrb- in german : deutschland , \u09a1\u09af\u09bc\u099a\u09cd of \u09b2\u09be\u09a8\u09cd\u099f\u09cd of , pronounced [ d\u0254\u028ft\u0283lant ] -rrb- , middle is a country of europe . ||| -467.986
+ germany -lrb- in german : deutschland , \u09a1\u09af\u09bc\u099a\u09cd of \u09b2\u09be\u09a8\u09cd\u099f\u09cd of , pronounced [ d\u0254\u028ft\u0283lant ] -rrb- , the middle is a country of europe . ||| -468.038
+ germany -lrb- in german : deutschland , \u09a1\u09af\u09bc\u099a\u09cd the \u09b2\u09be\u09a8\u09cd\u099f\u09cd of , pronounced [ d\u0254\u028ft\u0283lant ] -rrb- , central is a country of europe . ||| -468.039
+ \u0996\u09cd\u09b0\u09bf\u09b8\u09cd\u099f\u09a7\u09b0\u09cd\u09ae russia the main religion . ||| -113.803
+ \u0996\u09cd\u09b0\u09bf\u09b8\u09cd\u099f\u09a7\u09b0\u09cd\u09ae of russia the main religion . ||| -113.989
+ \u0996\u09cd\u09b0\u09bf\u09b8\u09cd\u099f\u09a7\u09b0\u09cd\u09ae russia the religion . ||| -115.327
+ \u0996\u09cd\u09b0\u09bf\u09b8\u09cd\u099f\u09a7\u09b0\u09cd\u09ae russia the main religion of . ||| -116.365
+ \u0996\u09cd\u09b0\u09bf\u09b8\u09cd\u099f\u09a7\u09b0\u09cd\u09ae of russia the main religion of . ||| -116.551
+ \u0996\u09cd\u09b0\u09bf\u09b8\u09cd\u099f\u09a7\u09b0\u09cd\u09ae the main religion . ||| -116.584
+ \u0996\u09cd\u09b0\u09bf\u09b8\u09cd\u099f\u09a7\u09b0\u09cd\u09ae russia the religion of . ||| -116.600
+ \u0996\u09cd\u09b0\u09bf\u09b8\u09cd\u099f\u09a7\u09b0\u09cd\u09ae of russia the religion of . ||| -116.786
+ but \u0997\u09b2\u09a6\u09c7\u09b0 education \u09b0\u09cb\u09ae\u09be\u09a8\u09c0\u0995\u09b0\u09a3\u09c7\u09b0 \u0997\u09a4\u09bf was slow down . ||| -325.728
+ but \u0997\u09b2\u09a6\u09c7\u09b0 the \u09b0\u09cb\u09ae\u09be\u09a8\u09c0\u0995\u09b0\u09a3\u09c7\u09b0 \u0997\u09a4\u09bf was slow down . ||| -326.536
+ but \u0997\u09b2\u09a6\u09c7\u09b0 the educational \u09b0\u09cb\u09ae\u09be\u09a8\u09c0\u0995\u09b0\u09a3\u09c7\u09b0 \u0997\u09a4\u09bf was slow down . ||| -326.920
+ but \u0997\u09b2\u09a6\u09c7\u09b0 's education \u09b0\u09cb\u09ae\u09be\u09a8\u09c0\u0995\u09b0\u09a3\u09c7\u09b0 \u0997\u09a4\u09bf was slow down . ||| -327.161
+ but \u0997\u09b2\u09a6\u09c7\u09b0 of education \u09b0\u09cb\u09ae\u09be\u09a8\u09c0\u0995\u09b0\u09a3\u09c7\u09b0 \u0997\u09a4\u09bf was slow down . ||| -327.169
+ but \u0997\u09b2\u09a6\u09c7\u09b0 education in \u09b0\u09cb\u09ae\u09be\u09a8\u09c0\u0995\u09b0\u09a3\u09c7\u09b0 \u0997\u09a4\u09bf was slow down . ||| -327.194
+ but \u0997\u09b2\u09a6\u09c7\u09b0 education \u09b0\u09cb\u09ae\u09be\u09a8\u09c0\u0995\u09b0\u09a3\u09c7\u09b0 \u0997\u09a4\u09bf was too slow . ||| -327.746
+ but \u0997\u09b2\u09a6\u09c7\u09b0 education \u09b0\u09cb\u09ae\u09be\u09a8\u09c0\u0995\u09b0\u09a3\u09c7\u09b0 \u0997\u09a4\u09bf was the slow down . ||| -327.833
+ but \u0997\u09b2\u09a6\u09c7\u09b0 education \u09b0\u09cb\u09ae\u09be\u09a8\u09c0\u0995\u09b0\u09a3\u09c7\u09b0 \u0997\u09a4\u09bf was become slow . ||| -327.983
+ subject category : gnu foundation ||| -6.756
+ subject class : the gnu foundation ||| -7.225
+ subject class : gnu foundation ||| -7.445
+ subject matter : gnu foundation ||| -7.489
+ subject category : the gnu foundation ||| -7.497
+ subject : gnu foundation ||| -7.519
+ economic policy and revenue \u09a8\u09c0\u09a4\u09bf\u0995\u09c7\u0993 it study . ||| -127.108
+ economic policy and revenue \u09a8\u09c0\u09a4\u09bf\u0995\u09c7\u0993 it study of the . ||| -127.422
+ economic policy and tax \u09a8\u09c0\u09a4\u09bf\u0995\u09c7\u0993 it study . ||| -127.668
+ economic policy and tax \u09a8\u09c0\u09a4\u09bf\u0995\u09c7\u0993 it study of the . ||| -127.982
+ economic policy and tax revenue \u09a8\u09c0\u09a4\u09bf\u0995\u09c7\u0993 it study . ||| -128.096
+ economic policy and revenue \u09a8\u09c0\u09a4\u09bf\u0995\u09c7\u0993 it study the . ||| -128.352
+ financial policy and revenue \u09a8\u09c0\u09a4\u09bf\u0995\u09c7\u0993 it study . ||| -128.370
+ economic policy and tax revenue \u09a8\u09c0\u09a4\u09bf\u0995\u09c7\u0993 it study of the . ||| -128.410
+ economic policy and revenue \u09a8\u09c0\u09a4\u09bf\u0995\u09c7\u0993 this study . ||| -128.457
+ economic policy and revenue \u09a8\u09c0\u09a4\u09bf\u0995\u09c7\u0993 it study by the . ||| -128.634
+ among these there are : would have \u09a4\u09cb\u09ae\u09be\u09b0 will get seen , \u0993\u09b0\u09c7 at this any \u09b8\u09cd\u09a8\u09c7\u09b9-\u09b8\u09c1\u09b0\u09a7\u09c1\u09a8\u09c0 . ||| -346.223
+ among these there are : would have \u09a4\u09cb\u09ae\u09be\u09b0 will get seen , \u0993\u09b0\u09c7 this any \u09b8\u09cd\u09a8\u09c7\u09b9-\u09b8\u09c1\u09b0\u09a7\u09c1\u09a8\u09c0 . ||| -346.260
+ among these there are : may be \u09a4\u09cb\u09ae\u09be\u09b0 will get seen , \u0993\u09b0\u09c7 at this any \u09b8\u09cd\u09a8\u09c7\u09b9-\u09b8\u09c1\u09b0\u09a7\u09c1\u09a8\u09c0 . ||| -346.280
+ among these there are : may be \u09a4\u09cb\u09ae\u09be\u09b0 will get seen , \u0993\u09b0\u09c7 this any \u09b8\u09cd\u09a8\u09c7\u09b9-\u09b8\u09c1\u09b0\u09a7\u09c1\u09a8\u09c0 . ||| -346.317
+ among these there are : would have \u09a4\u09cb\u09ae\u09be\u09b0 get seen , \u0993\u09b0\u09c7 at this any \u09b8\u09cd\u09a8\u09c7\u09b9-\u09b8\u09c1\u09b0\u09a7\u09c1\u09a8\u09c0 . ||| -346.361
+ among these there are : would have \u09a4\u09cb\u09ae\u09be\u09b0 get seen , \u0993\u09b0\u09c7 this any \u09b8\u09cd\u09a8\u09c7\u09b9-\u09b8\u09c1\u09b0\u09a7\u09c1\u09a8\u09c0 . ||| -346.398
+ among these there are : may be \u09a4\u09cb\u09ae\u09be\u09b0 get seen , \u0993\u09b0\u09c7 at this any \u09b8\u09cd\u09a8\u09c7\u09b9-\u09b8\u09c1\u09b0\u09a7\u09c1\u09a8\u09c0 . ||| -346.418
+ among these there are : may be \u09a4\u09cb\u09ae\u09be\u09b0 get seen , \u0993\u09b0\u09c7 this any \u09b8\u09cd\u09a8\u09c7\u09b9-\u09b8\u09c1\u09b0\u09a7\u09c1\u09a8\u09c0 . ||| -346.455
+ among these there are : would have \u09a4\u09cb\u09ae\u09be\u09b0 will get seen , \u0993\u09b0\u09c7 for this any \u09b8\u09cd\u09a8\u09c7\u09b9-\u09b8\u09c1\u09b0\u09a7\u09c1\u09a8\u09c0 . ||| -346.560
+ among them are : would have \u09a4\u09cb\u09ae\u09be\u09b0 will get seen , \u0993\u09b0\u09c7 at this any \u09b8\u09cd\u09a8\u09c7\u09b9-\u09b8\u09c1\u09b0\u09a7\u09c1\u09a8\u09c0 . ||| -346.567
+ on 23rd april 1992 satyajit died . ||| -17.735
+ on 23rd april 1992 satyajit expired . ||| -18.257
+ on 23rd april 1992 satyajit passed away . ||| -19.284
+ on 23rd april 1992 satyajit died . . ||| -21.617
+ he died on 23rd april 1992 . ||| -21.773
+ in this time nazrul 's medical \u09b0\u09bf\u09aa\u09cb\u09b0\u09cd\u099f earning famous \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u0995\u09a6\u09c7\u09b0 to send to . ||| -238.748
+ in this time nazrul 's medical \u09b0\u09bf\u09aa\u09cb\u09b0\u09cd\u099f stay famous \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u0995\u09a6\u09c7\u09b0 to send to . ||| -238.899
+ this time nazrul 's medical \u09b0\u09bf\u09aa\u09cb\u09b0\u09cd\u099f earning famous \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u0995\u09a6\u09c7\u09b0 to send to . ||| -239.024
+ in this time nazrul medical \u09b0\u09bf\u09aa\u09cb\u09b0\u09cd\u099f earning famous \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u0995\u09a6\u09c7\u09b0 to send to . ||| -239.101
+ in this time nazrul 's medical \u09b0\u09bf\u09aa\u09cb\u09b0\u09cd\u099f earning famous \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u0995\u09a6\u09c7\u09b0 to send . ||| -239.116
+ this time nazrul 's medical \u09b0\u09bf\u09aa\u09cb\u09b0\u09cd\u099f stay famous \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u0995\u09a6\u09c7\u09b0 to send to . ||| -239.175
+ during this time nazrul 's medical \u09b0\u09bf\u09aa\u09cb\u09b0\u09cd\u099f earning famous \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u0995\u09a6\u09c7\u09b0 to send to . ||| -239.209
+ at this time nazrul 's medical \u09b0\u09bf\u09aa\u09cb\u09b0\u09cd\u099f earning famous \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u0995\u09a6\u09c7\u09b0 to send to . ||| -239.234
+ in this time nazrul medical \u09b0\u09bf\u09aa\u09cb\u09b0\u09cd\u099f stay famous \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u0995\u09a6\u09c7\u09b0 to send to . ||| -239.252
+ in this time najrul 's medical \u09b0\u09bf\u09aa\u09cb\u09b0\u09cd\u099f earning famous \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u0995\u09a6\u09c7\u09b0 to send to . ||| -239.262
+ acted other than at different times rani \u09ae\u09c1\u0996\u09be\u09b0\u09cd\u099c\u09c0 different \u09a6\u09be\u09a4\u09ac\u09cd\u09af are connected with \u09a5\u09c7\u0995\u09c7\u099b\u09c7\u09a8 . ||| -347.110
+ acted apart from at different times rani \u09ae\u09c1\u0996\u09be\u09b0\u09cd\u099c\u09c0 different \u09a6\u09be\u09a4\u09ac\u09cd\u09af are connected with \u09a5\u09c7\u0995\u09c7\u099b\u09c7\u09a8 . ||| -347.165
+ acted apart at different times rani \u09ae\u09c1\u0996\u09be\u09b0\u09cd\u099c\u09c0 different \u09a6\u09be\u09a4\u09ac\u09cd\u09af are connected with \u09a5\u09c7\u0995\u09c7\u099b\u09c7\u09a8 . ||| -347.335
+ acted , at different times rani \u09ae\u09c1\u0996\u09be\u09b0\u09cd\u099c\u09c0 different \u09a6\u09be\u09a4\u09ac\u09cd\u09af are connected with \u09a5\u09c7\u0995\u09c7\u099b\u09c7\u09a8 . ||| -347.592
+ acted apart from different times rani \u09ae\u09c1\u0996\u09be\u09b0\u09cd\u099c\u09c0 different \u09a6\u09be\u09a4\u09ac\u09cd\u09af are connected with \u09a5\u09c7\u0995\u09c7\u099b\u09c7\u09a8 . ||| -347.594
+ acted apart from different time rani \u09ae\u09c1\u0996\u09be\u09b0\u09cd\u099c\u09c0 different \u09a6\u09be\u09a4\u09ac\u09cd\u09af are connected with \u09a5\u09c7\u0995\u09c7\u099b\u09c7\u09a8 . ||| -347.830
+ acted in addition at different times rani \u09ae\u09c1\u0996\u09be\u09b0\u09cd\u099c\u09c0 different \u09a6\u09be\u09a4\u09ac\u09cd\u09af are connected with \u09a5\u09c7\u0995\u09c7\u099b\u09c7\u09a8 . ||| -347.893
+ acted without at different times rani \u09ae\u09c1\u0996\u09be\u09b0\u09cd\u099c\u09c0 different \u09a6\u09be\u09a4\u09ac\u09cd\u09af are connected with \u09a5\u09c7\u0995\u09c7\u099b\u09c7\u09a8 . ||| -348.081
+ acted other than at different times rani \u09ae\u09c1\u0996\u09be\u09b0\u09cd\u099c\u09c0 of different \u09a6\u09be\u09a4\u09ac\u09cd\u09af are connected with \u09a5\u09c7\u0995\u09c7\u099b\u09c7\u09a8 . ||| -348.245
+ acted other than at different times rani \u09ae\u09c1\u0996\u09be\u09b0\u09cd\u099c\u09c0 different \u09a6\u09be\u09a4\u09ac\u09cd\u09af offices connected with \u09a5\u09c7\u0995\u09c7\u099b\u09c7\u09a8 . ||| -349.825
+ bengali literature and culture of his special contribution on \u09b8\u09cd\u09ac\u09c0\u0995\u09c3\u09a4\u09bf\u09b8\u09cd\u09ac\u09b0\u09c1\u09aa year on 9th december , dhaka university he was honorary \u09a1\u09bf.\u09b2\u09bf\u099f \u0989\u09aa\u09be\u09a7\u09bf\u09a4\u09c7 awarded with the . ||| -368.897
+ bengali literature and culture in his special contribution on \u09b8\u09cd\u09ac\u09c0\u0995\u09c3\u09a4\u09bf\u09b8\u09cd\u09ac\u09b0\u09c1\u09aa year on 9th december , dhaka university he was honorary \u09a1\u09bf.\u09b2\u09bf\u099f \u0989\u09aa\u09be\u09a7\u09bf\u09a4\u09c7 awarded with the . ||| -369.012
+ bengali literature and culture his special contribution on \u09b8\u09cd\u09ac\u09c0\u0995\u09c3\u09a4\u09bf\u09b8\u09cd\u09ac\u09b0\u09c1\u09aa year on 9th december , dhaka university he was honorary \u09a1\u09bf.\u09b2\u09bf\u099f \u0989\u09aa\u09be\u09a7\u09bf\u09a4\u09c7 awarded with the . ||| -369.196
+ bengali literature and culture , his special contribution on \u09b8\u09cd\u09ac\u09c0\u0995\u09c3\u09a4\u09bf\u09b8\u09cd\u09ac\u09b0\u09c1\u09aa year on 9th december , dhaka university he was honorary \u09a1\u09bf.\u09b2\u09bf\u099f \u0989\u09aa\u09be\u09a7\u09bf\u09a4\u09c7 awarded with the . ||| -369.213
+ bengali literature and culture of his special contribution on \u09b8\u09cd\u09ac\u09c0\u0995\u09c3\u09a4\u09bf\u09b8\u09cd\u09ac\u09b0\u09c1\u09aa year on 9 december , dhaka university he was honorary \u09a1\u09bf.\u09b2\u09bf\u099f \u0989\u09aa\u09be\u09a7\u09bf\u09a4\u09c7 awarded with the . ||| -369.296
+ the bengali literature and culture of his special contribution on \u09b8\u09cd\u09ac\u09c0\u0995\u09c3\u09a4\u09bf\u09b8\u09cd\u09ac\u09b0\u09c1\u09aa year on 9th december , dhaka university he was honorary \u09a1\u09bf.\u09b2\u09bf\u099f \u0989\u09aa\u09be\u09a7\u09bf\u09a4\u09c7 awarded with the . ||| -369.384
+ bengali literature and culture in his special contribution on \u09b8\u09cd\u09ac\u09c0\u0995\u09c3\u09a4\u09bf\u09b8\u09cd\u09ac\u09b0\u09c1\u09aa year on 9 december , dhaka university he was honorary \u09a1\u09bf.\u09b2\u09bf\u099f \u0989\u09aa\u09be\u09a7\u09bf\u09a4\u09c7 awarded with the . ||| -369.411
+ the bengali literature and culture in his special contribution on \u09b8\u09cd\u09ac\u09c0\u0995\u09c3\u09a4\u09bf\u09b8\u09cd\u09ac\u09b0\u09c1\u09aa year on 9th december , dhaka university he was honorary \u09a1\u09bf.\u09b2\u09bf\u099f \u0989\u09aa\u09be\u09a7\u09bf\u09a4\u09c7 awarded with the . ||| -369.499
+ bengali literature and culture his special contribution on \u09b8\u09cd\u09ac\u09c0\u0995\u09c3\u09a4\u09bf\u09b8\u09cd\u09ac\u09b0\u09c1\u09aa year on 9 december , dhaka university he was honorary \u09a1\u09bf.\u09b2\u09bf\u099f \u0989\u09aa\u09be\u09a7\u09bf\u09a4\u09c7 awarded with the . ||| -369.595
+ bengali literature and culture , his special contribution on \u09b8\u09cd\u09ac\u09c0\u0995\u09c3\u09a4\u09bf\u09b8\u09cd\u09ac\u09b0\u09c1\u09aa year on 9 december , dhaka university he was honorary \u09a1\u09bf.\u09b2\u09bf\u099f \u0989\u09aa\u09be\u09a7\u09bf\u09a4\u09c7 awarded with the . ||| -369.612
+ in kolkata durga puja of the city is one of the tourism \u0986\u0995\u09b0\u09cd\u09b7\u09a3\u0993 is also a reason ||| -136.580
+ kolkata 's durga puja of the city is one of the tourism \u0986\u0995\u09b0\u09cd\u09b7\u09a3\u0993 is also a reason ||| -136.704
+ kolkata durga puja of the city is one of the tourism \u0986\u0995\u09b0\u09cd\u09b7\u09a3\u0993 is also a reason ||| -136.923
+ in the durga puja of the city is one of the tourism \u0986\u0995\u09b0\u09cd\u09b7\u09a3\u0993 is also a reason ||| -137.284
+ in kolkata durga puja of the city is one of the tourists spot \u0986\u0995\u09b0\u09cd\u09b7\u09a3\u0993 is also a reason ||| -137.331
+ in kolkata durga puja in the city is one of the tourism \u0986\u0995\u09b0\u09cd\u09b7\u09a3\u0993 is also a reason ||| -138.172
+ kolkata 's durga puja in the city is one of the tourism \u0986\u0995\u09b0\u09cd\u09b7\u09a3\u0993 is also a reason ||| -138.296
+ in kolkata durga puja in the city is one of the tourists spot \u0986\u0995\u09b0\u09cd\u09b7\u09a3\u0993 is also a reason ||| -138.923
+ but many of east german started living in economically reach and democratic west germany , east germany government built a wall in berlin in 1961 and reinforced its boundaries . ||| -64.035
+ but many people of east german started living in economically reach and democratic west germany , east germany government built a wall in berlin in 1961 and reinforced its boundaries . ||| -64.412
+ but many of the east german started living in economically reach and democratic west germany , east germany government built a wall in berlin in 1961 and reinforced its boundaries . ||| -64.422
+ but many people of east germany started living in economically reach and democratic west germany , east germany government built a wall in berlin in 1961 and reinforced its boundaries . ||| -64.469
+ but many of east germany started living in economically reach and democratic west germany , east germany government built a wall in berlin in 1961 and reinforced its boundaries . ||| -65.162
+ but many of east german started living in economically reach and democratic west germany , east germany government built a wall in berlin in 1961 of the and its boundaries . ||| -66.368
+ but many of east german started living in economically reach and democratic west germany , east germany government built a wall in berlin in 1961 and reinforced its boundaries of . ||| -66.416
+ but many people of east german started living in economically reach and democratic west germany , east germany government built a wall in berlin in 1961 of the and its boundaries . ||| -66.744
+ but many of the east german started living in economically reach and democratic west germany , east germany government built a wall in berlin in 1961 of the and its boundaries . ||| -66.755
+ but many people of east german started living in economically reach and democratic west germany , east germany government built a wall in berlin in 1961 and reinforced its boundaries of . ||| -66.793
+ the first one is the film \u09a8\u09bf\u09b0\u09cd\u09ac\u09be\u0995 first seven \u09ae\u09bf\u09a8\u09bf\u099f , which \u099a\u09be\u09b0\u09c1\u09b0 \u098f\u0995\u0998\u09c7\u09af\u09bc\u09c7\u09ae\u09bf life shown them , and the second one is " of the garden \u09a6\u09cb\u09b2\u09a8\u09be\u09b0 scene " , where \u099a\u09be\u09b0\u09c1 \u0985\u09ae\u09b2\u09c7\u09b0 for his love \u09ae\u09c1\u0996\u09cb\u09ae\u09c1\u0996\u09bf . ||| -914.735
+ the first one is the film \u09a8\u09bf\u09b0\u09cd\u09ac\u09be\u0995 first seven \u09ae\u09bf\u09a8\u09bf\u099f , which \u099a\u09be\u09b0\u09c1\u09b0 \u098f\u0995\u0998\u09c7\u09af\u09bc\u09c7\u09ae\u09bf life shown them , and the second one is " the \u09a6\u09cb\u09b2\u09a8\u09be\u09b0 scene " , where \u099a\u09be\u09b0\u09c1 \u0985\u09ae\u09b2\u09c7\u09b0 for his love \u09ae\u09c1\u0996\u09cb\u09ae\u09c1\u0996\u09bf . ||| -914.852
+ the first one is the film \u09a8\u09bf\u09b0\u09cd\u09ac\u09be\u0995 first seven \u09ae\u09bf\u09a8\u09bf\u099f , which \u099a\u09be\u09b0\u09c1\u09b0 \u098f\u0995\u0998\u09c7\u09af\u09bc\u09c7\u09ae\u09bf life shown them , and the second one is " of the garden \u09a6\u09cb\u09b2\u09a8\u09be\u09b0 scene of " , where \u099a\u09be\u09b0\u09c1 \u0985\u09ae\u09b2\u09c7\u09b0 for his love \u09ae\u09c1\u0996\u09cb\u09ae\u09c1\u0996\u09bf . ||| -915.015
+ the first one is the film \u09a8\u09bf\u09b0\u09cd\u09ac\u09be\u0995 first seven \u09ae\u09bf\u09a8\u09bf\u099f , which \u099a\u09be\u09b0\u09c1\u09b0 \u098f\u0995\u0998\u09c7\u09af\u09bc\u09c7\u09ae\u09bf life shown lift , and the second one is " of the garden \u09a6\u09cb\u09b2\u09a8\u09be\u09b0 scene " , where \u099a\u09be\u09b0\u09c1 \u0985\u09ae\u09b2\u09c7\u09b0 for his love \u09ae\u09c1\u0996\u09cb\u09ae\u09c1\u0996\u09bf . ||| -915.060
+ the first one is the film \u09a8\u09bf\u09b0\u09cd\u09ac\u09be\u0995 first seven \u09ae\u09bf\u09a8\u09bf\u099f , which \u099a\u09be\u09b0\u09c1\u09b0 \u098f\u0995\u0998\u09c7\u09af\u09bc\u09c7\u09ae\u09bf life shown them , and the second one is " garden \u09a6\u09cb\u09b2\u09a8\u09be\u09b0 scene " , where \u099a\u09be\u09b0\u09c1 \u0985\u09ae\u09b2\u09c7\u09b0 for his love \u09ae\u09c1\u0996\u09cb\u09ae\u09c1\u0996\u09bf . ||| -915.091
+ the first one is the film \u09a8\u09bf\u09b0\u09cd\u09ac\u09be\u0995 first seven \u09ae\u09bf\u09a8\u09bf\u099f , which \u099a\u09be\u09b0\u09c1\u09b0 \u098f\u0995\u0998\u09c7\u09af\u09bc\u09c7\u09ae\u09bf life shown them , and the second one is " the \u09a6\u09cb\u09b2\u09a8\u09be\u09b0 scene of " , where \u099a\u09be\u09b0\u09c1 \u0985\u09ae\u09b2\u09c7\u09b0 for his love \u09ae\u09c1\u0996\u09cb\u09ae\u09c1\u0996\u09bf . ||| -915.131
+ the first one is the film \u09a8\u09bf\u09b0\u09cd\u09ac\u09be\u0995 first seven \u09ae\u09bf\u09a8\u09bf\u099f , which is \u099a\u09be\u09b0\u09c1\u09b0 \u098f\u0995\u0998\u09c7\u09af\u09bc\u09c7\u09ae\u09bf life shown them , and the second one is " of the garden \u09a6\u09cb\u09b2\u09a8\u09be\u09b0 scene " , where \u099a\u09be\u09b0\u09c1 \u0985\u09ae\u09b2\u09c7\u09b0 for his love \u09ae\u09c1\u0996\u09cb\u09ae\u09c1\u0996\u09bf . ||| -915.136
+ the first one is the film \u09a8\u09bf\u09b0\u09cd\u09ac\u09be\u0995 first seven \u09ae\u09bf\u09a8\u09bf\u099f , which \u099a\u09be\u09b0\u09c1\u09b0 \u098f\u0995\u0998\u09c7\u09af\u09bc\u09c7\u09ae\u09bf life shown lift , and the second one is " the \u09a6\u09cb\u09b2\u09a8\u09be\u09b0 scene " , where \u099a\u09be\u09b0\u09c1 \u0985\u09ae\u09b2\u09c7\u09b0 for his love \u09ae\u09c1\u0996\u09cb\u09ae\u09c1\u0996\u09bf . ||| -915.176
+ the first one is the film \u09a8\u09bf\u09b0\u09cd\u09ac\u09be\u0995 the seven \u09ae\u09bf\u09a8\u09bf\u099f , which \u099a\u09be\u09b0\u09c1\u09b0 \u098f\u0995\u0998\u09c7\u09af\u09bc\u09c7\u09ae\u09bf life shown them , and the second one is " of the garden \u09a6\u09cb\u09b2\u09a8\u09be\u09b0 scene " , where \u099a\u09be\u09b0\u09c1 \u0985\u09ae\u09b2\u09c7\u09b0 for his love \u09ae\u09c1\u0996\u09cb\u09ae\u09c1\u0996\u09bf . ||| -915.244
+ the first one is the film \u09a8\u09bf\u09b0\u09cd\u09ac\u09be\u0995 first seven \u09ae\u09bf\u09a8\u09bf\u099f , which is \u099a\u09be\u09b0\u09c1\u09b0 \u098f\u0995\u0998\u09c7\u09af\u09bc\u09c7\u09ae\u09bf life shown them , and the second one is " the \u09a6\u09cb\u09b2\u09a8\u09be\u09b0 scene " , where \u099a\u09be\u09b0\u09c1 \u0985\u09ae\u09b2\u09c7\u09b0 for his love \u09ae\u09c1\u0996\u09cb\u09ae\u09c1\u0996\u09bf . ||| -915.252
+ 18th century the a group of \u099a\u09bf\u09a8\u09cd\u09a4\u09be\u09ac\u09bf\u09a6 and writer income and production \u099a\u0995\u09cd\u09b0\u09be\u0995\u09be\u09b0 current through the economic thoughts development it . ||| -261.213
+ 18th century the a group of \u099a\u09bf\u09a8\u09cd\u09a4\u09be\u09ac\u09bf\u09a6 and writer income and production \u099a\u0995\u09cd\u09b0\u09be\u0995\u09be\u09b0 current through the economic thoughts development of it . ||| -261.415
+ 18th century the a group of \u099a\u09bf\u09a8\u09cd\u09a4\u09be\u09ac\u09bf\u09a6 and writer income and production \u099a\u0995\u09cd\u09b0\u09be\u0995\u09be\u09b0 current through economic thoughts development it . ||| -261.573
+ 18th century the a group of \u099a\u09bf\u09a8\u09cd\u09a4\u09be\u09ac\u09bf\u09a6 and writer income and production \u099a\u0995\u09cd\u09b0\u09be\u0995\u09be\u09b0 current through the economic thoughts development people . ||| -261.583
+ 18th century the a group of \u099a\u09bf\u09a8\u09cd\u09a4\u09be\u09ac\u09bf\u09a6 and writer of income and production \u099a\u0995\u09cd\u09b0\u09be\u0995\u09be\u09b0 current through the economic thoughts development it . ||| -261.743
+ 18th century the a group of \u099a\u09bf\u09a8\u09cd\u09a4\u09be\u09ac\u09bf\u09a6 and writer income and production \u099a\u0995\u09cd\u09b0\u09be\u0995\u09be\u09b0 current through economic thoughts development of it . ||| -261.775
+ the 18th century a group of \u099a\u09bf\u09a8\u09cd\u09a4\u09be\u09ac\u09bf\u09a6 and writer income and production \u099a\u0995\u09cd\u09b0\u09be\u0995\u09be\u09b0 current through the economic thoughts development it . ||| -261.841
+ 18th century a group of \u099a\u09bf\u09a8\u09cd\u09a4\u09be\u09ac\u09bf\u09a6 and writer income and production \u099a\u0995\u09cd\u09b0\u09be\u0995\u09be\u09b0 current through the economic thoughts development it . ||| -261.873
+ 18th century the a group of \u099a\u09bf\u09a8\u09cd\u09a4\u09be\u09ac\u09bf\u09a6 and writer income and production \u099a\u0995\u09cd\u09b0\u09be\u0995\u09be\u09b0 current through economic thoughts development people . ||| -261.944
+ 18th century the a group of \u099a\u09bf\u09a8\u09cd\u09a4\u09be\u09ac\u09bf\u09a6 and writer of income and production \u099a\u0995\u09cd\u09b0\u09be\u0995\u09be\u09b0 current through the economic thoughts development of it . ||| -261.946
+ the arousal activities before penetrating male organ into vagina is called foreplay . ||| -25.161
+ the arousal activities before penetrating male organ into vagina is called the foreplay . ||| -26.500
+ the arousal activities before penetrating male organ into vagina is called as the foreplay . ||| -27.117
+ the arousal activities before penetrating male organ into vagina is called as foreplay . ||| -27.359
+ the arousal activities before penetrating male organ into
<TRUNCATED>
[11/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/hiero/output.gold
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/hiero/output.gold b/joshua-core/src/test/resources/bn-en/hiero/output.gold
new file mode 100644
index 0000000..57c1655
--- /dev/null
+++ b/joshua-core/src/test/resources/bn-en/hiero/output.gold
@@ -0,0 +1,805 @@
+0 ||| rabindranath was born in kolkata a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| lm_0=-27.184 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-6.000 tm_pt_3=-0.000 tm_pt_4=-5.000 tm_pt_5=-31.718 tm_pt_6=-15.214 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.001 tm_pt_10=-10.850 tm_pt_11=-0.000 tm_pt_12=-6.748 tm_pt_13=-0.000 tm_pt_14=-9.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=7.000 WordPenalty=-5.646 OOVPenalty=2.000 ||| -227.217
+0 ||| rabindranath was born in kolkata in a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| lm_0=-28.078 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-6.000 tm_pt_3=-0.000 tm_pt_4=-5.000 tm_pt_5=-36.789 tm_pt_6=-15.727 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.001 tm_pt_10=-9.614 tm_pt_11=-0.000 tm_pt_12=-9.341 tm_pt_13=-0.000 tm_pt_14=-10.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=7.000 WordPenalty=-6.080 OOVPenalty=2.000 ||| -228.133
+0 ||| rabindranath was born in kolkata one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| lm_0=-27.935 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-6.000 tm_pt_3=-0.000 tm_pt_4=-5.000 tm_pt_5=-30.948 tm_pt_6=-16.212 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.001 tm_pt_10=-9.310 tm_pt_11=-0.000 tm_pt_12=-7.369 tm_pt_13=-0.000 tm_pt_14=-9.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=7.000 WordPenalty=-5.646 OOVPenalty=2.000 ||| -228.160
+0 ||| rabindranath born in the a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| lm_0=-26.980 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-5.000 tm_pt_3=-0.000 tm_pt_4=-5.000 tm_pt_5=-32.737 tm_pt_6=-16.092 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.000 tm_pt_10=-12.188 tm_pt_11=-0.000 tm_pt_12=-3.876 tm_pt_13=-0.000 tm_pt_14=-8.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=7.000 WordPenalty=-5.212 OOVPenalty=2.000 ||| -228.388
+0 ||| rabindranath born in the one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| lm_0=-27.320 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-5.000 tm_pt_3=-0.000 tm_pt_4=-5.000 tm_pt_5=-31.967 tm_pt_6=-17.090 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.000 tm_pt_10=-10.648 tm_pt_11=-0.000 tm_pt_12=-4.497 tm_pt_13=-0.000 tm_pt_14=-8.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=7.000 WordPenalty=-5.212 OOVPenalty=2.000 ||| -228.823
+0 ||| rabindranath 's birth was the a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| lm_0=-29.996 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-6.000 tm_pt_3=-0.000 tm_pt_4=-6.000 tm_pt_5=-33.336 tm_pt_6=-14.613 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.000 tm_pt_10=-10.783 tm_pt_11=-0.000 tm_pt_12=-4.047 tm_pt_13=-0.000 tm_pt_14=-9.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=8.000 WordPenalty=-5.646 OOVPenalty=2.000 ||| -228.862
+0 ||| rabindranath was born in kolkata in \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| lm_0=-26.673 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-6.000 tm_pt_3=-0.000 tm_pt_4=-5.000 tm_pt_5=-34.098 tm_pt_6=-16.835 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.001 tm_pt_10=-12.831 tm_pt_11=-0.000 tm_pt_12=-8.966 tm_pt_13=-0.000 tm_pt_14=-9.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=7.000 WordPenalty=-5.646 OOVPenalty=2.000 ||| -228.892
+0 ||| rabindranath 's birth in the a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| lm_0=-28.839 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-6.000 tm_pt_3=-0.000 tm_pt_4=-6.000 tm_pt_5=-35.470 tm_pt_6=-15.731 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.000 tm_pt_10=-12.256 tm_pt_11=-0.000 tm_pt_12=-5.179 tm_pt_13=-0.000 tm_pt_14=-9.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=8.000 WordPenalty=-5.646 OOVPenalty=2.000 ||| -229.004
+0 ||| rabindranath was born in kolkata a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 \u09aa\u09b0\u09bf\u09ac\u09be\u09b0\u09c7 . ||| lm_0=-27.046 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-6.000 tm_pt_3=-0.000 tm_pt_4=-5.000 tm_pt_5=-14.175 tm_pt_6=-6.655 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-0.001 tm_pt_10=-10.859 tm_pt_11=-0.000 tm_pt_12=-6.751 tm_pt_13=-0.000 tm_pt_14=-7.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=8.000 WordPenalty=-5.212 OOVPenalty=3.000 ||| -321.208
+0 ||| rabindranath was born in kolkata in a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 \u09aa\u09b0\u09bf\u09ac\u09be\u09b0\u09c7 . ||| lm_0=-27.940 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-6.000 tm_pt_3=-0.000 tm_pt_4=-5.000 tm_pt_5=-19.247 tm_pt_6=-7.168 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-0.001 tm_pt_10=-9.624 tm_pt_11=-0.000 tm_pt_12=-9.344 tm_pt_13=-0.000 tm_pt_14=-8.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=8.000 WordPenalty=-5.646 OOVPenalty=3.000 ||| -322.124
+1 ||| recently with united states with the relationship between improved . ||| lm_0=-21.908 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-5.000 tm_pt_3=-0.000 tm_pt_4=-4.000 tm_pt_5=-25.015 tm_pt_6=-15.386 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.742 tm_pt_10=-4.668 tm_pt_11=-0.000 tm_pt_12=-2.927 tm_pt_13=-0.000 tm_pt_14=-10.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=4.000 WordPenalty=-5.212 ||| -21.022
+1 ||| recently with united states with the relation between improved . ||| lm_0=-21.608 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-5.000 tm_pt_3=-0.000 tm_pt_4=-4.000 tm_pt_5=-25.910 tm_pt_6=-15.828 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.742 tm_pt_10=-5.584 tm_pt_11=-0.000 tm_pt_12=-2.927 tm_pt_13=-0.000 tm_pt_14=-10.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=4.000 WordPenalty=-5.212 ||| -21.210
+1 ||| in recent times india with united states relationship between improved . ||| lm_0=-23.087 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-6.000 tm_pt_3=-0.000 tm_pt_4=-6.000 tm_pt_5=-31.818 tm_pt_6=-14.884 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-2.375 tm_pt_10=-7.301 tm_pt_11=-0.000 tm_pt_12=-5.105 tm_pt_13=-0.000 tm_pt_14=-11.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=6.000 WordPenalty=-5.646 ||| -22.055
+1 ||| recently india with united states relationship between improved . ||| lm_0=-22.178 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-6.000 tm_pt_3=-0.000 tm_pt_4=-6.000 tm_pt_5=-19.105 tm_pt_6=-15.289 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.742 tm_pt_10=-8.805 tm_pt_11=-0.000 tm_pt_12=-4.412 tm_pt_13=-0.000 tm_pt_14=-9.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=6.000 WordPenalty=-4.777 ||| -22.129
+1 ||| in recent times with united states with the relationship between improved . ||| lm_0=-23.586 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-5.000 tm_pt_3=-0.000 tm_pt_4=-3.000 tm_pt_5=-37.728 tm_pt_6=-14.981 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-3.375 tm_pt_10=-2.471 tm_pt_11=-0.000 tm_pt_12=-2.927 tm_pt_13=-0.000 tm_pt_14=-12.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=3.000 WordPenalty=-6.080 ||| -22.439
+1 ||| recently with united states with the relation improved . ||| lm_0=-21.512 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-5.000 tm_pt_3=-0.000 tm_pt_4=-4.000 tm_pt_5=-21.571 tm_pt_6=-15.946 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.742 tm_pt_10=-6.748 tm_pt_11=-0.000 tm_pt_12=-2.927 tm_pt_13=-0.000 tm_pt_14=-9.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=4.000 WordPenalty=-4.777 ||| -22.538
+1 ||| in recent times in india with united states relationship between improved . ||| lm_0=-23.598 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-6.000 tm_pt_3=-0.000 tm_pt_4=-6.000 tm_pt_5=-37.182 tm_pt_6=-15.449 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-2.377 tm_pt_10=-7.480 tm_pt_11=-0.000 tm_pt_12=-7.334 tm_pt_13=-0.000 tm_pt_14=-12.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=6.000 WordPenalty=-6.080 ||| -22.811
+2 ||| mathematics so science language . ||| lm_0=-15.141 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-4.000 tm_pt_3=-0.000 tm_pt_4=-4.000 tm_pt_5=-6.483 tm_pt_6=-3.387 tm_pt_7=-4.000 tm_pt_8=-10.872 tm_pt_9=-0.002 tm_pt_10=-3.378 tm_pt_11=-0.000 tm_pt_12=-1.626 tm_pt_13=-0.000 tm_pt_14=-5.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=4.000 WordPenalty=-3.040 ||| -10.471
+2 ||| mathematics is science language . ||| lm_0=-12.890 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-4.000 tm_pt_3=-0.000 tm_pt_4=-4.000 tm_pt_5=-10.375 tm_pt_6=-3.926 tm_pt_7=-4.000 tm_pt_8=-10.872 tm_pt_9=-0.052 tm_pt_10=-8.326 tm_pt_11=-0.000 tm_pt_12=-3.330 tm_pt_13=-0.000 tm_pt_14=-5.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=4.000 WordPenalty=-3.040 ||| -10.543
+2 ||| mathematics that science language . ||| lm_0=-14.001 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-4.000 tm_pt_3=-0.000 tm_pt_4=-4.000 tm_pt_5=-9.625 tm_pt_6=-3.926 tm_pt_7=-4.000 tm_pt_8=-10.872 tm_pt_9=-0.052 tm_pt_10=-7.607 tm_pt_11=-0.000 tm_pt_12=-3.330 tm_pt_13=-0.000 tm_pt_14=-5.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=4.000 WordPenalty=-3.040 ||| -11.587
+2 ||| science mathematics that language . ||| lm_0=-14.271 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-4.000 tm_pt_3=-0.000 tm_pt_4=-3.000 tm_pt_5=-9.625 tm_pt_6=-3.926 tm_pt_7=-4.000 tm_pt_8=-10.872 tm_pt_9=-0.138 tm_pt_10=-2.832 tm_pt_11=-0.000 tm_pt_12=-1.486 tm_pt_13=-0.000 tm_pt_14=-5.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=2.000 WordPenalty=-3.040 ||| -12.065
+3 ||| from this it can be understood easily that this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf will be \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| lm_0=-35.973 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-7.000 tm_pt_3=-0.000 tm_pt_4=-6.000 tm_pt_5=-21.761 tm_pt_6=-8.823 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-1.503 tm_pt_10=-4.103 tm_pt_11=-0.000 tm_pt_12=-2.603 tm_pt_13=-0.000 tm_pt_14=-12.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=9.000 WordPenalty=-7.383 OOVPenalty=3.000 ||| -322.982
+3 ||| it can be understood from this easily that this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf will be \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| lm_0=-35.158 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-7.000 tm_pt_3=-0.000 tm_pt_4=-5.000 tm_pt_5=-21.761 tm_pt_6=-8.823 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-1.503 tm_pt_10=-4.876 tm_pt_11=-0.000 tm_pt_12=-2.198 tm_pt_13=-0.000 tm_pt_14=-12.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=8.000 WordPenalty=-7.383 OOVPenalty=3.000 ||| -322.995
+3 ||| from this it can be understood easily that in this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf will be \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| lm_0=-37.367 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-6.000 tm_pt_3=-0.000 tm_pt_4=-5.000 tm_pt_5=-25.417 tm_pt_6=-9.365 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.639 tm_pt_10=-2.794 tm_pt_11=-0.000 tm_pt_12=-1.915 tm_pt_13=-0.000 tm_pt_14=-13.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=8.000 WordPenalty=-7.817 OOVPenalty=3.000 ||| -322.996
+3 ||| it can be understood from this easily that in this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf will be \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| lm_0=-36.552 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-6.000 tm_pt_3=-0.000 tm_pt_4=-4.000 tm_pt_5=-25.417 tm_pt_6=-9.365 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.639 tm_pt_10=-3.567 tm_pt_11=-0.000 tm_pt_12=-1.510 tm_pt_13=-0.000 tm_pt_14=-13.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=7.000 WordPenalty=-7.817 OOVPenalty=3.000 ||| -323.010
+3 ||| it can be understood from this can easily that this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf will be \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| lm_0=-35.539 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-7.000 tm_pt_3=-0.000 tm_pt_4=-5.000 tm_pt_5=-25.797 tm_pt_6=-8.970 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-2.368 tm_pt_10=-3.490 tm_pt_11=-0.000 tm_pt_12=-3.296 tm_pt_13=-0.000 tm_pt_14=-13.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=8.000 WordPenalty=-7.817 OOVPenalty=3.000 ||| -323.095
+3 ||| it can be understood from this can easily that in this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf will be \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| lm_0=-36.933 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-6.000 tm_pt_3=-0.000 tm_pt_4=-4.000 tm_pt_5=-29.453 tm_pt_6=-9.512 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-2.503 tm_pt_10=-2.181 tm_pt_11=-0.000 tm_pt_12=-2.609 tm_pt_13=-0.000 tm_pt_14=-14.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=7.000 WordPenalty=-8.252 OOVPenalty=3.000 ||| -323.109
+3 ||| from this it will be it can be understood that in this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| lm_0=-33.528 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-6.000 tm_pt_3=-0.000 tm_pt_4=-3.000 tm_pt_5=-29.319 tm_pt_6=-10.259 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.386 tm_pt_10=-5.844 tm_pt_11=-0.000 tm_pt_12=-3.695 tm_pt_13=-0.000 tm_pt_14=-13.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=5.000 WordPenalty=-7.817 OOVPenalty=3.000 ||| -323.442
+3 ||| from this it it can be understood that this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf will be \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| lm_0=-33.961 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-7.000 tm_pt_3=-0.000 tm_pt_4=-5.000 tm_pt_5=-25.663 tm_pt_6=-9.717 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-1.386 tm_pt_10=-6.898 tm_pt_11=-0.000 tm_pt_12=-3.493 tm_pt_13=-0.000 tm_pt_14=-12.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=8.000 WordPenalty=-7.383 OOVPenalty=3.000 ||| -323.453
+3 ||| from this it will be it can be understood that the \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| lm_0=-31.634 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-6.000 tm_pt_3=-0.000 tm_pt_4=-3.000 tm_pt_5=-28.998 tm_pt_6=-11.009 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.389 tm_pt_10=-5.844 tm_pt_11=-0.000 tm_pt_12=-4.457 tm_pt_13=-0.000 tm_pt_14=-12.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=5.000 WordPenalty=-7.383 OOVPenalty=3.000 ||| -323.498
+3 ||| from this it it can be understood that in this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf will be \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| lm_0=-35.355 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-6.000 tm_pt_3=-0.000 tm_pt_4=-4.000 tm_pt_5=-29.319 tm_pt_6=-10.259 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.405 tm_pt_10=-5.633 tm_pt_11=-0.000 tm_pt_12=-3.196 tm_pt_13=-0.000 tm_pt_14=-13.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=7.000 WordPenalty=-7.817 OOVPenalty=3.000 ||| -323.521
+4 ||| the same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match from this novel . ||| lm_0=-39.669 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-9.000 tm_pt_3=-0.000 tm_pt_4=-8.000 tm_pt_5=-21.437 tm_pt_6=-10.032 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-3.005 tm_pt_10=-12.116 tm_pt_11=-0.000 tm_pt_12=-4.580 tm_pt_13=-0.000 tm_pt_14=-11.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=10.000 WordPenalty=-6.514 OOVPenalty=2.000 ||| -236.836
+4 ||| the same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 presage match from this novel . ||| lm_0=-39.944 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-9.000 tm_pt_3=-0.000 tm_pt_4=-8.000 tm_pt_5=-20.338 tm_pt_6=-10.032 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-3.005 tm_pt_10=-12.116 tm_pt_11=-0.000 tm_pt_12=-4.580 tm_pt_13=-0.000 tm_pt_14=-11.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=10.000 WordPenalty=-6.514 OOVPenalty=2.000 ||| -236.965
+4 ||| in the same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match from this novel . ||| lm_0=-39.886 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-9.000 tm_pt_3=-0.000 tm_pt_4=-8.000 tm_pt_5=-27.312 tm_pt_6=-10.335 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-3.008 tm_pt_10=-12.419 tm_pt_11=-0.000 tm_pt_12=-6.068 tm_pt_13=-0.000 tm_pt_14=-12.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=10.000 WordPenalty=-6.949 OOVPenalty=2.000 ||| -237.005
+4 ||| in the same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 presage match from this novel . ||| lm_0=-40.160 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-9.000 tm_pt_3=-0.000 tm_pt_4=-8.000 tm_pt_5=-26.214 tm_pt_6=-10.335 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-3.008 tm_pt_10=-12.419 tm_pt_11=-0.000 tm_pt_12=-6.068 tm_pt_13=-0.000 tm_pt_14=-12.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=10.000 WordPenalty=-6.949 OOVPenalty=2.000 ||| -237.135
+4 ||| in the same along with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match from this novel . ||| lm_0=-40.830 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-9.000 tm_pt_3=-0.000 tm_pt_4=-7.000 tm_pt_5=-29.608 tm_pt_6=-10.928 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-3.010 tm_pt_10=-10.951 tm_pt_11=-0.000 tm_pt_12=-7.562 tm_pt_13=-0.000 tm_pt_14=-13.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=9.000 WordPenalty=-7.383 OOVPenalty=2.000 ||| -237.997
+4 ||| in the same along with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 presage match from this novel . ||| lm_0=-41.105 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-9.000 tm_pt_3=-0.000 tm_pt_4=-7.000 tm_pt_5=-28.509 tm_pt_6=-10.928 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-3.010 tm_pt_10=-10.951 tm_pt_11=-0.000 tm_pt_12=-7.562 tm_pt_13=-0.000 tm_pt_14=-13.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=9.000 WordPenalty=-7.383 OOVPenalty=2.000 ||| -238.127
+4 ||| the same with of bengal earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match from this novel . ||| lm_0=-43.356 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-9.000 tm_pt_3=-0.000 tm_pt_4=-8.000 tm_pt_5=-23.639 tm_pt_6=-10.334 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-3.010 tm_pt_10=-6.029 tm_pt_11=-0.000 tm_pt_12=-4.734 tm_pt_13=-0.000 tm_pt_14=-12.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=10.000 WordPenalty=-6.949 OOVPenalty=2.000 ||| -238.469
+4 ||| the same with of bengal earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 presage match from this novel . ||| lm_0=-43.631 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-9.000 tm_pt_3=-0.000 tm_pt_4=-8.000 tm_pt_5=-22.541 tm_pt_6=-10.334 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-3.010 tm_pt_10=-6.029 tm_pt_11=-0.000 tm_pt_12=-4.734 tm_pt_13=-0.000 tm_pt_14=-12.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=10.000 WordPenalty=-6.949 OOVPenalty=2.000 ||| -238.599
+4 ||| in the same with of bengal earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match from this novel . ||| lm_0=-43.573 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-9.000 tm_pt_3=-0.000 tm_pt_4=-8.000 tm_pt_5=-29.515 tm_pt_6=-10.637 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-3.012 tm_pt_10=-6.332 tm_pt_11=-0.000 tm_pt_12=-6.222 tm_pt_13=-0.000 tm_pt_14=-13.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=10.000 WordPenalty=-7.383 OOVPenalty=2.000 ||| -238.638
+4 ||| the same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match from this novels . ||| lm_0=-41.826 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-9.000 tm_pt_3=-0.000 tm_pt_4=-8.000 tm_pt_5=-20.389 tm_pt_6=-9.586 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-3.005 tm_pt_10=-10.960 tm_pt_11=-0.000 tm_pt_12=-4.213 tm_pt_13=-0.000 tm_pt_14=-11.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=10.000 WordPenalty=-6.514 OOVPenalty=2.000 ||| -238.721
+5 ||| \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 majority in the . ||| lm_0=-25.263 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-7.000 tm_pt_3=-0.000 tm_pt_4=-7.000 tm_pt_5=-16.310 tm_pt_6=-6.695 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-0.289 tm_pt_10=-10.344 tm_pt_11=-0.000 tm_pt_12=-2.428 tm_pt_13=-0.000 tm_pt_14=-8.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=9.000 WordPenalty=-5.212 OOVPenalty=2.000 ||| -217.895
+5 ||| \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 majority that the . ||| lm_0=-27.702 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-6.000 tm_pt_3=-0.000 tm_pt_4=-6.000 tm_pt_5=-16.018 tm_pt_6=-7.571 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.271 tm_pt_10=-5.619 tm_pt_11=-0.000 tm_pt_12=-1.161 tm_pt_13=-0.000 tm_pt_14=-8.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=8.000 WordPenalty=-5.212 OOVPenalty=2.000 ||| -220.097
+5 ||| \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 majority where the . ||| lm_0=-28.188 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-6.000 tm_pt_3=-0.000 tm_pt_4=-5.000 tm_pt_5=-15.032 tm_pt_6=-9.180 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-0.406 tm_pt_10=-2.153 tm_pt_11=-0.000 tm_pt_12=-0.468 tm_pt_13=-0.000 tm_pt_14=-8.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=7.000 WordPenalty=-5.212 OOVPenalty=2.000 ||| -220.171
+5 ||| \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 single absolute majority in the . ||| lm_0=-28.599 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-7.000 tm_pt_3=-0.000 tm_pt_4=-7.000 tm_pt_5=-21.593 tm_pt_6=-7.414 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-1.154 tm_pt_10=-9.044 tm_pt_11=-0.000 tm_pt_12=-3.527 tm_pt_13=-0.000 tm_pt_14=-10.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=9.000 WordPenalty=-6.080 OOVPenalty=2.000 ||| -220.174
+5 ||| \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 the . majority in ||| lm_0=-30.418 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-6.000 tm_pt_3=-0.000 tm_pt_4=-5.000 tm_pt_5=-16.310 tm_pt_6=-6.695 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.271 tm_pt_10=-2.153 tm_pt_11=-0.000 tm_pt_12=-0.468 tm_pt_13=-0.000 tm_pt_14=-8.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=6.000 WordPenalty=-5.212 OOVPenalty=2.000 ||| -224.035
+6 ||| \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his work with to that . ||| lm_0=-33.425 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-6.000 tm_pt_3=-0.000 tm_pt_4=-6.000 tm_pt_5=-22.509 tm_pt_6=-11.163 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.370 tm_pt_10=-18.845 tm_pt_11=-0.000 tm_pt_12=-2.681 tm_pt_13=-0.000 tm_pt_14=-6.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=10.000 WordPenalty=-5.212 OOVPenalty=4.000 ||| -432.357
+6 ||| \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his work with can and pajama . ||| lm_0=-38.085 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-5.000 tm_pt_3=-0.000 tm_pt_4=-5.000 tm_pt_5=-21.424 tm_pt_6=-9.427 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-1.503 tm_pt_10=-10.841 tm_pt_11=-0.000 tm_pt_12=-2.832 tm_pt_13=-0.000 tm_pt_14=-7.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=9.000 WordPenalty=-5.646 OOVPenalty=4.000 ||| -433.284
+6 ||| \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his work with to and pajama . ||| lm_0=-37.294 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-5.000 tm_pt_3=-0.000 tm_pt_4=-5.000 tm_pt_5=-24.691 tm_pt_6=-10.758 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-1.370 tm_pt_10=-12.675 tm_pt_11=-0.000 tm_pt_12=-1.984 tm_pt_13=-0.000 tm_pt_14=-7.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=9.000 WordPenalty=-5.646 OOVPenalty=4.000 ||| -433.453
+6 ||| \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his work with can that . ||| lm_0=-35.292 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-6.000 tm_pt_3=-0.000 tm_pt_4=-6.000 tm_pt_5=-19.242 tm_pt_6=-9.832 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.503 tm_pt_10=-17.011 tm_pt_11=-0.000 tm_pt_12=-3.528 tm_pt_13=-0.000 tm_pt_14=-6.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=10.000 WordPenalty=-5.212 OOVPenalty=4.000 ||| -433.520
+6 ||| \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his work with that can . ||| lm_0=-33.973 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-6.000 tm_pt_3=-0.000 tm_pt_4=-5.000 tm_pt_5=-19.242 tm_pt_6=-9.832 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-2.368 tm_pt_10=-17.376 tm_pt_11=-0.000 tm_pt_12=-3.305 tm_pt_13=-0.000 tm_pt_14=-6.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=9.000 WordPenalty=-5.212 OOVPenalty=4.000 ||| -433.577
+6 ||| \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his work with a that . ||| lm_0=-33.108 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-6.000 tm_pt_3=-0.000 tm_pt_4=-5.000 tm_pt_5=-24.229 tm_pt_6=-13.109 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.418 tm_pt_10=-18.986 tm_pt_11=-0.000 tm_pt_12=-2.612 tm_pt_13=-0.000 tm_pt_14=-6.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=9.000 WordPenalty=-5.212 OOVPenalty=4.000 ||| -433.974
+6 ||| \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his work with a and pajama . ||| lm_0=-36.186 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-5.000 tm_pt_3=-0.000 tm_pt_4=-4.000 tm_pt_5=-26.411 tm_pt_6=-12.704 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-1.418 tm_pt_10=-12.815 tm_pt_11=-0.000 tm_pt_12=-1.915 tm_pt_13=-0.000 tm_pt_14=-7.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=8.000 WordPenalty=-5.646 OOVPenalty=4.000 ||| -434.091
+6 ||| \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 in his work with to that . ||| lm_0=-34.398 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-6.000 tm_pt_3=-0.000 tm_pt_4=-6.000 tm_pt_5=-28.289 tm_pt_6=-11.756 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.420 tm_pt_10=-18.922 tm_pt_11=-0.000 tm_pt_12=-6.557 tm_pt_13=-0.000 tm_pt_14=-7.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=10.000 WordPenalty=-5.646 OOVPenalty=4.000 ||| -434.368
+6 ||| \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 in his work with can and pajama . ||| lm_0=-39.058 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-5.000 tm_pt_3=-0.000 tm_pt_4=-5.000 tm_pt_5=-27.204 tm_pt_6=-10.020 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-1.553 tm_pt_10=-10.918 tm_pt_11=-0.000 tm_pt_12=-6.708 tm_pt_13=-0.000 tm_pt_14=-8.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=9.000 WordPenalty=-6.080 OOVPenalty=4.000 ||| -435.295
+6 ||| \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 in his work with to and pajama . ||| lm_0=-38.267 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-5.000 tm_pt_3=-0.000 tm_pt_4=-5.000 tm_pt_5=-30.471 tm_pt_6=-11.351 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-1.420 tm_pt_10=-12.751 tm_pt_11=-0.000 tm_pt_12=-5.861 tm_pt_13=-0.000 tm_pt_14=-8.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=9.000 WordPenalty=-6.080 OOVPenalty=4.000 ||| -435.464
+7 ||| other than task \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character there are but these are very is not . ||| lm_0=-50.273 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-13.000 tm_pt_3=-0.000 tm_pt_4=-12.000 tm_pt_5=-43.711 tm_pt_6=-15.780 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-1.889 tm_pt_10=-13.889 tm_pt_11=-0.000 tm_pt_12=-8.044 tm_pt_13=-0.000 tm_pt_14=-17.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=14.000 WordPenalty=-9.120 OOVPenalty=2.000 ||| -246.114
+7 ||| other than task \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and some more linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character there are but these are very is not . ||| lm_0=-50.299 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-13.000 tm_pt_3=-0.000 tm_pt_4=-11.000 tm_pt_5=-41.728 tm_pt_6=-14.487 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-1.892 tm_pt_10=-13.969 tm_pt_11=-0.000 tm_pt_12=-8.217 tm_pt_13=-0.000 tm_pt_14=-17.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=13.000 WordPenalty=-9.120 OOVPenalty=2.000 ||| -246.373
+7 ||| other than task \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character is but these are very is not . ||| lm_0=-47.882 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-13.000 tm_pt_3=-0.000 tm_pt_4=-12.000 tm_pt_5=-43.252 tm_pt_6=-16.128 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-1.889 tm_pt_10=-17.521 tm_pt_11=-0.000 tm_pt_12=-9.306 tm_pt_13=-0.000 tm_pt_14=-16.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=14.000 WordPenalty=-8.686 OOVPenalty=2.000 ||| -246.508
+7 ||| other than task \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and some more linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character is but these are very is not . ||| lm_0=-47.907 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-13.000 tm_pt_3=-0.000 tm_pt_4=-11.000 tm_pt_5=-41.269 tm_pt_6=-14.835 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-1.892 tm_pt_10=-17.601 tm_pt_11=-0.000 tm_pt_12=-9.480 tm_pt_13=-0.000 tm_pt_14=-16.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=13.000 WordPenalty=-8.686 OOVPenalty=2.000 ||| -246.767
+7 ||| other than task \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character there is but these are very is not . ||| lm_0=-50.416 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-13.000 tm_pt_3=-0.000 tm_pt_4=-12.000 tm_pt_5=-44.789 tm_pt_6=-15.888 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-1.889 tm_pt_10=-14.025 tm_pt_11=-0.000 tm_pt_12=-8.613 tm_pt_13=-0.000 tm_pt_14=-17.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=14.000 WordPenalty=-9.120 OOVPenalty=2.000 ||| -246.771
+7 ||| other than task \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and some more linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character there is but these are very is not . ||| lm_0=-50.441 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-13.000 tm_pt_3=-0.000 tm_pt_4=-11.000 tm_pt_5=-42.805 tm_pt_6=-14.595 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-1.892 tm_pt_10=-14.105 tm_pt_11=-0.000 tm_pt_12=-8.786 tm_pt_13=-0.000 tm_pt_14=-17.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=13.000 WordPenalty=-9.120 OOVPenalty=2.000 ||| -247.030
+7 ||| other than task \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and more some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character there are but these are very is not . ||| lm_0=-52.085 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-13.000 tm_pt_3=-0.000 tm_pt_4=-12.000 tm_pt_5=-41.728 tm_pt_6=-14.487 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-1.889 tm_pt_10=-13.108 tm_pt_11=-0.000 tm_pt_12=-7.196 tm_pt_13=-0.000 tm_pt_14=-17.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=14.000 WordPenalty=-9.120 OOVPenalty=2.000 ||| -247.032
+7 ||| other than task \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and more some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character is but these are very is not . ||| lm_0=-49.693 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-13.000 tm_pt_3=-0.000 tm_pt_4=-12.000 tm_pt_5=-41.269 tm_pt_6=-14.835 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-1.889 tm_pt_10=-16.740 tm_pt_11=-0.000 tm_pt_12=-8.459 tm_pt_13=-0.000 tm_pt_14=-16.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=14.000 WordPenalty=-8.686 OOVPenalty=2.000 ||| -247.426
+7 ||| other than task \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character there are but these are very not common . ||| lm_0=-51.792 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-13.000 tm_pt_3=-0.000 tm_pt_4=-12.000 tm_pt_5=-40.998 tm_pt_6=-17.207 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-1.889 tm_pt_10=-11.943 tm_pt_11=-0.000 tm_pt_12=-8.044 tm_pt_13=-0.000 tm_pt_14=-17.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=14.000 WordPenalty=-9.120 OOVPenalty=2.000 ||| -247.458
+7 ||| other than task \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character there are but these very is not . ||| lm_0=-50.979 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-13.000 tm_pt_3=-0.000 tm_pt_4=-12.000 tm_pt_5=-37.874 tm_pt_6=-15.430 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-1.871 tm_pt_10=-14.919 tm_pt_11=-0.000 tm_pt_12=-7.014 tm_pt_13=-0.000 tm_pt_14=-16.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=14.000 WordPenalty=-8.686 OOVPenalty=2.000 ||| -247.572
+8 ||| it social situation in \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the fingers . ||| lm_0=-47.090 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-13.000 tm_pt_3=-0.000 tm_pt_4=-13.000 tm_pt_5=-48.497 tm_pt_6=-24.838 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-3.665 tm_pt_10=-34.032 tm_pt_11=-0.000 tm_pt_12=-10.582 tm_pt_13=-0.000 tm_pt_14=-15.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=15.000 WordPenalty=-8.252 OOVPenalty=2.000 ||| -256.668
+8 ||| it social situation in \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the is . ||| lm_0=-46.563 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-14.000 tm_pt_3=-0.000 tm_pt_4=-14.000 tm_pt_5=-51.579 tm_pt_6=-22.823 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-3.298 tm_pt_10=-38.843 tm_pt_11=-0.000 tm_pt_12=-10.620 tm_pt_13=-0.000 tm_pt_14=-15.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=16.000 WordPenalty=-8.252 OOVPenalty=2.000 ||| -257.068
+8 ||| this social situation in \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the fingers . ||| lm_0=-46.329 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-13.000 tm_pt_3=-0.000 tm_pt_4=-13.000 tm_pt_5=-50.091 tm_pt_6=-25.785 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-3.665 tm_pt_10=-35.965 tm_pt_11=-0.000 tm_pt_12=-11.214 tm_pt_13=-0.000 tm_pt_14=-15.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=15.000 WordPenalty=-8.252 OOVPenalty=2.000 ||| -257.080
+8 ||| it social situation in \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the in the . ||| lm_0=-46.908 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-14.000 tm_pt_3=-0.000 tm_pt_4=-14.000 tm_pt_5=-58.491 tm_pt_6=-22.780 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-3.432 tm_pt_10=-38.119 tm_pt_11=-0.000 tm_pt_12=-11.601 tm_pt_13=-0.000 tm_pt_14=-16.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=16.000 WordPenalty=-8.686 OOVPenalty=2.000 ||| -257.139
+8 ||| this social situation in \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the is . ||| lm_0=-45.803 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-14.000 tm_pt_3=-0.000 tm_pt_4=-14.000 tm_pt_5=-53.173 tm_pt_6=-23.770 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-3.298 tm_pt_10=-40.776 tm_pt_11=-0.000 tm_pt_12=-11.253 tm_pt_13=-0.000 tm_pt_14=-15.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=16.000 WordPenalty=-8.252 OOVPenalty=2.000 ||| -257.480
+8 ||| it social situation in \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the the . ||| lm_0=-46.235 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-14.000 tm_pt_3=-0.000 tm_pt_4=-14.000 tm_pt_5=-52.547 tm_pt_6=-22.440 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-3.346 tm_pt_10=-40.842 tm_pt_11=-0.000 tm_pt_12=-11.313 tm_pt_13=-0.000 tm_pt_14=-15.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=16.000 WordPenalty=-8.252 OOVPenalty=2.000 ||| -257.520
+8 ||| this social situation in \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the in the . ||| lm_0=-46.147 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-14.000 tm_pt_3=-0.000 tm_pt_4=-14.000 tm_pt_5=-60.085 tm_pt_6=-23.727 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-3.432 tm_pt_10=-40.052 tm_pt_11=-0.000 tm_pt_12=-12.233 tm_pt_13=-0.000 tm_pt_14=-16.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=16.000 WordPenalty=-8.686 OOVPenalty=2.000 ||| -257.551
+8 ||| it social situation in \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rules of the fingers . ||| lm_0=-47.651 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-13.000 tm_pt_3=-0.000 tm_pt_4=-13.000 tm_pt_5=-47.899 tm_pt_6=-25.426 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-3.793 tm_pt_10=-33.665 tm_pt_11=-0.000 tm_pt_12=-11.275 tm_pt_13=-0.000 tm_pt_14=-15.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=15.000 WordPenalty=-8.252 OOVPenalty=2.000 ||| -257.705
+8 ||| it social situation in \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with the \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the fingers . ||| lm_0=-47.945 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-13.000 tm_pt_3=-0.000 tm_pt_4=-13.000 tm_pt_5=-56.067 tm_pt_6=-24.656 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-4.529 tm_pt_10=-32.026 tm_pt_11=-0.000 tm_pt_12=-11.680 tm_pt_13=-0.000 tm_pt_14=-16.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=15.000 WordPenalty=-8.686 OOVPenalty=2.000 ||| -257.756
+8 ||| this social situation in \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the the . ||| lm_0=-45.474 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-14.000 tm_pt_3=-0.000 tm_pt_4=-14.000 tm_pt_5=-54.141 tm_pt_6=-23.387 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-3.346 tm_pt_10=-42.775 tm_pt_11=-0.000 tm_pt_12=-11.946 tm_pt_13=-0.000 tm_pt_14=-15.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=16.000 WordPenalty=-8.252 OOVPenalty=2.000 ||| -257.932
+9 ||| \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 annual \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf meters ||| lm_0=-26.340 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-2.000 tm_pt_3=-0.000 tm_pt_4=-2.000 tm_pt_5=-3.372 tm_pt_6=-3.054 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.368 tm_pt_10=-2.140 tm_pt_11=-0.000 tm_pt_12=-1.263 tm_pt_13=-0.000 tm_pt_14=-2.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=5.000 WordPenalty=-3.040 OOVPenalty=3.000 ||| -321.077
+9 ||| \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 annual \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf meter ||| lm_0=-26.316 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-2.000 tm_pt_3=-0.000 tm_pt_4=-2.000 tm_pt_5=-3.885 tm_pt_6=-2.821 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.368 tm_pt_10=-2.140 tm_pt_11=-0.000 tm_pt_12=-1.337 tm_pt_13=-0.000 tm_pt_14=-2.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=5.000 WordPenalty=-3.040 OOVPenalty=3.000 ||| -321.092
+9 ||| \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 arrange \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf meters ||| lm_0=-26.532 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-2.000 tm_pt_3=-0.000 tm_pt_4=-2.000 tm_pt_5=-2.916 tm_pt_6=-3.748 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-1.000 tm_pt_10=0.000 tm_pt_11=-0.000 tm_pt_12=-1.956 tm_pt_13=-0.000 tm_pt_14=-2.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=5.000 WordPenalty=-3.040 OOVPenalty=3.000 ||| -321.673
+9 ||| \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 arrange \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf meter ||| lm_0=-26.509 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-2.000 tm_pt_3=-0.000 tm_pt_4=-2.000 tm_pt_5=-3.430 tm_pt_6=-3.514 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-1.000 tm_pt_10=0.000 tm_pt_11=-0.000 tm_pt_12=-2.030 tm_pt_13=-0.000 tm_pt_14=-2.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=5.000 WordPenalty=-3.040 OOVPenalty=3.000 ||| -321.689
+9 ||| \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 annual \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf metres ||| lm_0=-26.771 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-2.000 tm_pt_3=-0.000 tm_pt_4=-2.000 tm_pt_5=-3.608 tm_pt_6=-4.389 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.503 tm_pt_10=-2.140 tm_pt_11=-0.000 tm_pt_12=-2.803 tm_pt_13=-0.000 tm_pt_14=-2.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=5.000 WordPenalty=-3.040 OOVPenalty=3.000 ||| -322.768
+9 ||| \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 arrange \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf metres ||| lm_0=-26.963 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-2.000 tm_pt_3=-0.000 tm_pt_4=-2.000 tm_pt_5=-3.153 tm_pt_6=-5.083 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-1.135 tm_pt_10=0.000 tm_pt_11=-0.000 tm_pt_12=-3.497 tm_pt_13=-0.000 tm_pt_14=-2.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=5.000 WordPenalty=-3.040 OOVPenalty=3.000 ||| -323.365
+10 ||| on the year 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in the city in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f the national he was the main speech -lrb- keynote speech -rrb- on the . ||| lm_0=-67.116 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-14.000 tm_pt_3=-0.000 tm_pt_4=-13.000 tm_pt_5=-67.047 tm_pt_6=-19.159 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-2.603 tm_pt_10=-33.759 tm_pt_11=-0.000 tm_pt_12=-12.980 tm_pt_13=-0.000 tm_pt_14=-20.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=19.000 WordPenalty=-12.160 OOVPenalty=6.000 ||| -664.033
+10 ||| on the year 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in the city in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f the national he was the main speech -lrb- keynote speech -rrb- to the . ||| lm_0=-66.839 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-14.000 tm_pt_3=-0.000 tm_pt_4=-13.000 tm_pt_5=-68.125 tm_pt_6=-19.313 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-2.572 tm_pt_10=-34.760 tm_pt_11=-0.000 tm_pt_12=-12.757 tm_pt_13=-0.000 tm_pt_14=-20.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=19.000 WordPenalty=-12.160 OOVPenalty=6.000 ||| -664.104
+10 ||| on the year 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in the city in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f the national he was the main speech -lrb- keynote speech -rrb- , the . ||| lm_0=-65.985 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-14.000 tm_pt_3=-0.000 tm_pt_4=-13.000 tm_pt_5=-69.257 tm_pt_6=-18.853 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-2.921 tm_pt_10=-37.503 tm_pt_11=-0.000 tm_pt_12=-13.673 tm_pt_13=-0.000 tm_pt_14=-20.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=19.000 WordPenalty=-12.160 OOVPenalty=6.000 ||| -664.430
+10 ||| on the year 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in the city in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f as the national he was the main speech -lrb- keynote speech -rrb- on the . ||| lm_0=-68.204 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-14.000 tm_pt_3=-0.000 tm_pt_4=-12.000 tm_pt_5=-72.655 tm_pt_6=-19.643 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-3.468 tm_pt_10=-29.762 tm_pt_11=-0.000 tm_pt_12=-11.034 tm_pt_13=-0.000 tm_pt_14=-21.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=18.000 WordPenalty=-12.595 OOVPenalty=6.000 ||| -664.633
+10 ||| on the year 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in the city in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f as the national he was the main speech -lrb- keynote speech -rrb- to the . ||| lm_0=-67.926 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-14.000 tm_pt_3=-0.000 tm_pt_4=-12.000 tm_pt_5=-73.733 tm_pt_6=-19.797 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-3.436 tm_pt_10=-30.763 tm_pt_11=-0.000 tm_pt_12=-10.811 tm_pt_13=-0.000 tm_pt_14=-21.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=18.000 WordPenalty=-12.595 OOVPenalty=6.000 ||| -664.703
+10 ||| on the year 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in the city in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f as the national he was the main speech -lrb- keynote speech -rrb- , the . ||| lm_0=-67.073 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-14.000 tm_pt_3=-0.000 tm_pt_4=-12.000 tm_pt_5=-74.865 tm_pt_6=-19.338 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-3.786 tm_pt_10=-33.506 tm_pt_11=-0.000 tm_pt_12=-11.727 tm_pt_13=-0.000 tm_pt_14=-21.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=18.000 WordPenalty=-12.595 OOVPenalty=6.000 ||| -665.029
+10 ||| on the year 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 held in the city \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f as the national he was the main speech -lrb- keynote speech -rrb- on the . ||| lm_0=-69.138 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-14.000 tm_pt_3=-0.000 tm_pt_4=-11.000 tm_pt_5=-67.606 tm_pt_6=-19.452 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-3.835 tm_pt_10=-24.841 tm_pt_11=-0.000 tm_pt_12=-12.045 tm_pt_13=-0.000 tm_pt_14=-21.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=17.000 WordPenalty=-12.595 OOVPenalty=6.000 ||| -665.105
+10 ||| on the year 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 held in the city as the national \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f he was the main speech -lrb- keynote speech -rrb- on the . ||| lm_0=-68.551 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-14.000 tm_pt_3=-0.000 tm_pt_4=-11.000 tm_pt_5=-67.606 tm_pt_6=-19.452 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-3.204 tm_pt_10=-19.038 tm_pt_11=-0.000 tm_pt_12=-14.273 tm_pt_13=-0.000 tm_pt_14=-21.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=15.000 WordPenalty=-12.595 OOVPenalty=6.000 ||| -665.144
+10 ||| on the year 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in the city in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f the national was he main speech -lrb- keynote speech -rrb- on the . ||| lm_0=-70.024 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-14.000 tm_pt_3=-0.000 tm_pt_4=-14.000 tm_pt_5=-61.423 tm_pt_6=-18.853 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-2.603 tm_pt_10=-30.229 tm_pt_11=-0.000 tm_pt_12=-9.211 tm_pt_13=-0.000 tm_pt_14=-19.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=20.000 WordPenalty=-11.726 OOVPenalty=6.000 ||| -665.170
+10 ||| on the year 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 held in the city \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f as the national he was the main speech -lrb- keynote speech -rrb- to the . ||| lm_0=-68.860 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-14.000 tm_pt_3=-0.000 tm_pt_4=-11.000 tm_pt_5=-68.685 tm_pt_6=-19.606 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-3.804 tm_pt_10=-25.842 tm_pt_11=-0.000 tm_pt_12=-11.822 tm_pt_13=-0.000 tm_pt_14=-21.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=17.000 WordPenalty=-12.595 OOVPenalty=6.000 ||| -665.175
+11 ||| based on power distribution of population east pakistan where to west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac they started the where the whole of west pakistan a province was considered as . ||| lm_0=-78.142 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-25.000 tm_pt_3=-0.000 tm_pt_4=-21.000 tm_pt_5=-84.616 tm_pt_6=-43.203 tm_pt_7=-25.000 tm_pt_8=-67.950 tm_pt_9=-4.966 tm_pt_10=-42.706 tm_pt_11=-0.000 tm_pt_12=-17.287 tm_pt_13=-0.000 tm_pt_14=-34.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=21.000 WordPenalty=-16.069 OOVPenalty=1.000 ||| -187.213
+11 ||| based on power distribution of population east pakistan where to west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac they started the where the whole of west pakistan a province was considered as a . ||| lm_0=-78.822 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-25.000 tm_pt_3=-0.000 tm_pt_4=-21.000 tm_pt_5=-89.085 tm_pt_6=-44.156 tm_pt_7=-25.000 tm_pt_8=-67.950 tm_pt_9=-4.966 tm_pt_10=-42.706 tm_pt_11=-0.000 tm_pt_12=-17.287 tm_pt_13=-0.000 tm_pt_14=-35.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=21.000 WordPenalty=-16.503 OOVPenalty=1.000 ||| -187.302
+11 ||| based on power distribution of population east pakistan where to west pakistan " one unit theory , " is a \u0985\u09ad\u09bf\u09a8\u09ac they started the where the whole of west pakistan a province was considered as . ||| lm_0=-79.014 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-25.000 tm_pt_3=-0.000 tm_pt_4=-21.000 tm_pt_5=-85.933 tm_pt_6=-40.936 tm_pt_7=-25.000 tm_pt_8=-67.950 tm_pt_9=-4.967 tm_pt_10=-43.105 tm_pt_11=-0.000 tm_pt_12=-21.202 tm_pt_13=-0.000 tm_pt_14=-35.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=21.000 WordPenalty=-16.503 OOVPenalty=1.000 ||| -187.326
+11 ||| based on power distribution of population east pakistan where to west pakistan " one unit theory , " is a \u0985\u09ad\u09bf\u09a8\u09ac they started the where the whole of west pakistan a province was considered as a . ||| lm_0=-79.694 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-25.000 tm_pt_3=-0.000 tm_pt_4=-21.000 tm_pt_5=-90.402 tm_pt_6=-41.889 tm_pt_7=-25.000 tm_pt_8=-67.950 tm_pt_9=-4.967 tm_pt_10=-43.105 tm_pt_11=-0.000 tm_pt_12=-21.202 tm_pt_13=-0.000 tm_pt_14=-36.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=21.000 WordPenalty=-16.937 OOVPenalty=1.000 ||| -187.415
+11 ||| based on power distribution of population east pakistan where to west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac started with the where the whole of west pakistan a province was considered as . ||| lm_0=-76.593 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-25.000 tm_pt_3=-0.000 tm_pt_4=-20.000 tm_pt_5=-86.009 tm_pt_6=-43.203 tm_pt_7=-25.000 tm_pt_8=-67.950 tm_pt_9=-5.284 tm_pt_10=-41.754 tm_pt_11=-0.000 tm_pt_12=-17.170 tm_pt_13=-0.000 tm_pt_14=-34.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=19.000 WordPenalty=-16.069 OOVPenalty=1.000 ||| -187.500
+11 ||| based on power distribution of population east pakistan where to west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac started with the where the whole of west pakistan a province was considered as a . ||| lm_0=-77.273 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-25.000 tm_pt_3=-0.000 tm_pt_4=-20.000 tm_pt_5=-90.478 tm_pt_6=-44.156 tm_pt_7=-25.000 tm_pt_8=-67.950 tm_pt_9=-5.284 tm_pt_10=-41.754 tm_pt_11=-0.000 tm_pt_12=-17.170 tm_pt_13=-0.000 tm_pt_14=-35.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=19.000 WordPenalty=-16.503 OOVPenalty=1.000 ||| -187.589
+11 ||| based on power distribution of population east pakistan where to west pakistan " one unit theory , " is a \u0985\u09ad\u09bf\u09a8\u09ac started with the where the whole of west pakistan a province was considered as . ||| lm_0=-77.465 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-25.000 tm_pt_3=-0.000 tm_pt_4=-20.000 tm_pt_5=-87.325 tm_pt_6=-40.936 tm_pt_7=-25.000 tm_pt_8=-67.950 tm_pt_9=-5.285 tm_pt_10=-42.153 tm_pt_11=-0.000 tm_pt_12=-21.084 tm_pt_13=-0.000 tm_pt_14=-35.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=19.000 WordPenalty=-16.503 OOVPenalty=1.000 ||| -187.612
+11 ||| based on power distribution of population east pakistan where to west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac they started to the where the whole of west pakistan a province was considered as . ||| lm_0=-78.485 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-25.000 tm_pt_3=-0.000 tm_pt_4=-21.000 tm_pt_5=-87.967 tm_pt_6=-43.714 tm_pt_7=-25.000 tm_pt_8=-67.950 tm_pt_9=-4.968 tm_pt_10=-42.173 tm_pt_11=-0.000 tm_pt_12=-20.847 tm_pt_13=-0.000 tm_pt_14=-35.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=21.000 WordPenalty=-16.503 OOVPenalty=1.000 ||| -187.644
+11 ||| based on power distribution of population east pakistan where to west pakistan " one unit theory , " is a \u0985\u09ad\u09bf\u09a8\u09ac started with the where the whole of west pakistan a province was considered as a . ||| lm_0=-78.145 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-25.000 tm_pt_3=-0.000 tm_pt_4=-20.000 tm_pt_5=-91.794 tm_pt_6=-41.889 tm_pt_7=-25.000 tm_pt_8=-67.950 tm_pt_9=-5.285 tm_pt_10=-42.153 tm_pt_11=-0.000 tm_pt_12=-21.084 tm_pt_13=-0.000 tm_pt_14=-36.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=19.000 WordPenalty=-16.937 OOVPenalty=1.000 ||| -187.701
+11 ||| based on power distribution of population east pakistan where to west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac they started to the where the whole of west pakistan a province was considered as a . ||| lm_0=-79.165 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-25.000 tm_pt_3=-0.000 tm_pt_4=-21.000 tm_pt_5=-92.436 tm_pt_6=-44.667 tm_pt_7=-25.000 tm_pt_8=-67.950 tm_pt_9=-4.968 tm_pt_10=-42.173 tm_pt_11=-0.000 tm_pt_12=-20.847 tm_pt_13=-0.000 tm_pt_14=-36.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=21.000 WordPenalty=-16.937 OOVPenalty=1.000 ||| -187.733
+12 ||| the \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 theory ||| lm_0=-11.112 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-2.000 tm_pt_3=-0.000 tm_pt_4=-2.000 tm_pt_5=-9.309 tm_pt_6=-3.988 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.368 tm_pt_10=-7.916 tm_pt_11=-0.000 tm_pt_12=-1.316 tm_pt_13=-0.000 tm_pt_14=-2.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=3.000 WordPenalty=-2.171 OOVPenalty=1.000 ||| -110.415
+12 ||| the theory of \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 ||| lm_0=-10.781 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-2.000 tm_pt_3=-0.000 tm_pt_4=-1.000 tm_pt_5=-9.931 tm_pt_6=-3.988 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.369 tm_pt_10=-8.459 tm_pt_11=-0.000 tm_pt_12=-4.332 tm_pt_13=-0.000 tm_pt_14=-3.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=2.000 WordPenalty=-2.606 OOVPenalty=1.000 ||| -110.598
+12 ||| mathematical \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 theory ||| lm_0=-12.665 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-2.000 tm_pt_3=-0.000 tm_pt_4=-2.000 tm_pt_5=-2.869 tm_pt_6=-2.890 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-1.000 tm_pt_10=-4.888 tm_pt_11=-0.000 tm_pt_12=-2.010 tm_pt_13=-0.000 tm_pt_14=-2.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=3.000 WordPenalty=-2.171 OOVPenalty=1.000 ||| -110.707
+12 ||| \u2022 \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 theory ||| lm_0=-14.217 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-2.000 tm_pt_3=-0.000 tm_pt_4=-2.000 tm_pt_5=-1.046 tm_pt_6=-5.241 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.368 tm_pt_10=-0.422 tm_pt_11=-0.000 tm_pt_12=-1.316 tm_pt_13=-0.000 tm_pt_14=-2.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=3.000 WordPenalty=-2.171 OOVPenalty=1.000 ||| -111.160
+12 ||| . \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 theory ||| lm_0=-12.758 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-2.000 tm_pt_3=-0.000 tm_pt_4=-1.000 tm_pt_5=-7.900 tm_pt_6=-2.990 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.018 tm_pt_10=-3.641 tm_pt_11=-0.000 tm_pt_12=-1.712 tm_pt_13=-0.000 tm_pt_14=-2.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=2.000 WordPenalty=-2.171 OOVPenalty=1.000 ||| -111.550
+12 ||| mathematical theory of \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 ||| lm_0=-13.625 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-2.000 tm_pt_3=-0.000 tm_pt_4=-1.000 tm_pt_5=-3.491 tm_pt_6=-2.890 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-1.001 tm_pt_10=-5.431 tm_pt_11=-0.000 tm_pt_12=-5.026 tm_pt_13=-0.000 tm_pt_14=-3.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=2.000 WordPenalty=-2.606 OOVPenalty=1.000 ||| -112.488
+13 ||| external links of ||| lm_0=-6.986 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-2.000 tm_pt_3=-0.000 tm_pt_4=-2.000 tm_pt_5=-7.390 tm_pt_6=-2.729 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.018 tm_pt_10=-5.285 tm_pt_11=-0.000 tm_pt_12=-1.611 tm_pt_13=-0.000 tm_pt_14=-3.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=2.000 WordPenalty=-2.171 ||| -4.318
+13 ||| external link of ||| lm_0=-7.533 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-2.000 tm_pt_3=-0.000 tm_pt_4=-2.000 tm_pt_5=-8.091 tm_pt_6=-2.871 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.018 tm_pt_10=-5.285 tm_pt_11=-0.000 tm_pt_12=-2.767 tm_pt_13=-0.000 tm_pt_14=-3.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=2.000 WordPenalty=-2.171 ||| -5.587
+13 ||| outer link of ||| lm_0=-7.249 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-2.000 tm_pt_3=-0.000 tm_pt_4=-2.000 tm_pt_5=-8.247 tm_pt_6=-3.617 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.018 tm_pt_10=-5.285 tm_pt_11=-0.000 tm_pt_12=-3.202 tm_pt_13=-0.000 tm_pt_14=-3.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=2.000 WordPenalty=-2.171 ||| -5.674
+13 ||| external communication of ||| lm_0=-7.692 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-2.000 tm_pt_3=-0.000 tm_pt_4=-2.000 tm_pt_5=-8.265 tm_pt_6=-2.886 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.018 tm_pt_10=-5.285 tm_pt_11=-0.000 tm_pt_12=-2.555 tm_pt_13=-0.000 tm_pt_14=-3.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=2.000 WordPenalty=-2.171 ||| -5.747
+13 ||| outer communication of ||| lm_0=-7.553 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-2.000 tm_pt_3=-0.000 tm_pt_4=-2.000 tm_pt_5=-8.420 tm_pt_6=-3.648 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.018 tm_pt_10=-5.285 tm_pt_11=-0.000 tm_pt_12=-3.297 tm_pt_13=-0.000 tm_pt_14=-3.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=2.000 WordPenalty=-2.171 ||| -6.128
+14 ||| tata communicationer " foreign sanchar nigam limited building this is one of the main providers in telecommunication system of the city ||| lm_0=-48.982 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-10.000 tm_pt_3=-0.000 tm_pt_4=-6.000 tm_pt_5=-53.147 tm_pt_6=-19.625 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-5.390 tm_pt_10=-6.792 tm_pt_11=-0.000 tm_pt_12=-8.795 tm_pt_13=-0.000 tm_pt_14=-21.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=5.000 WordPenalty=-9.989 ||| -49.965
+14 ||| tata communicationer " foreign sanchar nigam limited building it is the telecommunication system is a one of the main providers ||| lm_0=-50.838 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-12.000 tm_pt_3=-0.000 tm_pt_4=-10.000 tm_pt_5=-48.689 tm_pt_6=-15.499 tm_pt_7=-12.000 tm_pt_8=-32.616 tm_pt_9=-3.371 tm_pt_10=-12.911 tm_pt_11=-0.000 tm_pt_12=-11.430 tm_pt_13=-0.000 tm_pt_14=-20.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=10.000 WordPenalty=-9.554 ||| -50.012
+14 ||| tata communication " foreign sanchar nigam limited building this is one of the main providers in telecommunication system of the city ||| lm_0=-47.728 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-10.000 tm_pt_3=-0.000 tm_pt_4=-6.000 tm_pt_5=-57.701 tm_pt_6=-19.625 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-5.390 tm_pt_10=-9.928 tm_pt_11=-0.000 tm_pt_12=-8.795 tm_pt_13=-0.000 tm_pt_14=-21.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=5.000 WordPenalty=-9.989 ||| -50.098
+14 ||| tata communicationer " foreign sanchar nigam limited building it in the city telecommunication system is a one of the main providers ||| lm_0=-53.506 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-12.000 tm_pt_3=-0.000 tm_pt_4=-11.000 tm_pt_5=-51.907 tm_pt_6=-15.695 tm_pt_7=-12.000 tm_pt_8=-32.616 tm_pt_9=-3.390 tm_pt_10=-9.448 tm_pt_11=-0.000 tm_pt_12=-12.069 tm_pt_13=-0.000 tm_pt_14=-21.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=11.000 WordPenalty=-9.989 ||| -50.441
+14 ||| tata communicationer " foreign sanchar nigam limited building this is one of the main providers in telecommunication system in the city ||| lm_0=-49.750 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-10.000 tm_pt_3=-0.000 tm_pt_4=-6.000 tm_pt_5=-58.246 tm_pt_6=-19.941 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-4.525 tm_pt_10=-5.981 tm_pt_11=-0.000 tm_pt_12=-7.696 tm_pt_13=-0.000 tm_pt_14=-21.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=5.000 WordPenalty=-9.989 ||| -50.456
+14 ||| tata communication " foreign sanchar nigam limited building this is one of the main providers in telecommunication system in the city ||| lm_0=-48.496 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-10.000 tm_pt_3=-0.000 tm_pt_4=-6.000 tm_pt_5=-62.800 tm_pt_6=-19.941 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-4.525 tm_pt_10=-9.117 tm_pt_11=-0.000 tm_pt_12=-7.696 tm_pt_13=-0.000 tm_pt_14=-21.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=5.000 WordPenalty=-9.989 ||| -50.589
+14 ||| tata communicationer " foreign sanchar nigam limited building it telecommunication system of the city is a one of the main providers ||| lm_0=-52.157 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-11.000 tm_pt_3=-0.000 tm_pt_4=-9.000 tm_pt_5=-47.201 tm_pt_6=-15.973 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-5.371 tm_pt_10=-9.098 tm_pt_11=-0.000 tm_pt_12=-12.632 tm_pt_13=-0.000 tm_pt_14=-21.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=9.000 WordPenalty=-9.989 ||| -50.709
+14 ||| tata communicationer " foreign sanchar nigam limited building it is the telecommunication system a one of the main providers ||| lm_0=-51.411 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-12.000 tm_pt_3=-0.000 tm_pt_4=-10.000 tm_pt_5=-44.990 tm_pt_6=-14.929 tm_pt_7=-12.000 tm_pt_8=-32.616 tm_pt_9=-3.371 tm_pt_10=-12.900 tm_pt_11=-0.000 tm_pt_12=-8.564 tm_pt_13=-0.000 tm_pt_14=-19.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=10.000 WordPenalty=-9.120 ||| -50.735
+14 ||| tata communicationer " foreign sanchar nigam limited building this is one of the main providers in telecommunication system of the city is ||| lm_0=-50.316 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-10.000 tm_pt_3=-0.000 tm_pt_4=-6.000 tm_pt_5=-59.703 tm_pt_6=-20.004 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-5.390 tm_pt_10=-5.694 tm_pt_11=-0.000 tm_pt_12=-8.795 tm_pt_13=-0.000 tm_pt_14=-22.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=5.000 WordPenalty=-10.423 ||| -50.780
+14 ||| tata communicationer " foreign sanchar nigam limited building it is in the city telecommunication system is a one of the main providers ||| lm_0=-54.678 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-12.000 tm_pt_3=-0.000 tm_pt_4=-11.000 tm_pt_5=-55.761 tm_pt_6=-15.961 tm_pt_7=-12.000 tm_pt_8=-32.616 tm_pt_9=-3.390 tm_pt_10=-8.882 tm_pt_11=-0.000 tm_pt_12=-12.784 tm_pt_13=-0.000 tm_pt_14=-22.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=11.000 WordPenalty=-10.423 ||| -50.893
+15 ||| he that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national assembly in election won all and was elected as the 44th president of the united states . ||| lm_0=-49.317 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-10.000 tm_pt_3=-0.000 tm_pt_4=-8.000 tm_pt_5=-60.569 tm_pt_6=-16.864 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-2.376 tm_pt_10=-7.564 tm_pt_11=-0.000 tm_pt_12=-5.624 tm_pt_13=-0.000 tm_pt_14=-21.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=8.000 WordPenalty=-10.857 OOVPenalty=2.000 ||| -241.090
+15 ||| he that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national assembly in the won all and was elected as the 44th president of the united states . ||| lm_0=-46.628 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-10.000 tm_pt_3=-0.000 tm_pt_4=-8.000 tm_pt_5=-67.250 tm_pt_6=-18.027 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-2.383 tm_pt_10=-13.421 tm_pt_11=-0.000 tm_pt_12=-6.471 tm_pt_13=-0.000 tm_pt_14=-21.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=8.000 WordPenalty=-10.857 OOVPenalty=2.000 ||| -241.262
+15 ||| in the same year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national assembly in election won all and was elected as the 44th president of the united states . ||| lm_0=-45.230 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-10.000 tm_pt_3=-0.000 tm_pt_4=-8.000 tm_pt_5=-72.283 tm_pt_6=-20.726 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-3.008 tm_pt_10=-13.960 tm_pt_11=-0.000 tm_pt_12=-9.950 tm_pt_13=-0.000 tm_pt_14=-22.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=8.000 WordPenalty=-11.292 OOVPenalty=2.000 ||| -241.368
+15 ||| in the same year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national assembly in the won all and was elected as the 44th president of the united states . ||| lm_0=-42.540 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-10.000 tm_pt_3=-0.000 tm_pt_4=-8.000 tm_pt_5=-78.964 tm_pt_6=-21.889 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-3.015 tm_pt_10=-19.816 tm_pt_11=-0.000 tm_pt_12=-10.797 tm_pt_13=-0.000 tm_pt_14=-22.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=8.000 WordPenalty=-11.292 OOVPenalty=2.000 ||| -241.540
+15 ||| he that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 assembly in election of national won all and was elected as the 44th president of the united states . ||| lm_0=-48.663 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-10.000 tm_pt_3=-0.000 tm_pt_4=-7.000 tm_pt_5=-61.191 tm_pt_6=-16.864 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-3.376 tm_pt_10=-8.412 tm_pt_11=-0.000 tm_pt_12=-8.117 tm_pt_13=-0.000 tm_pt_14=-22.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=7.000 WordPenalty=-11.292 OOVPenalty=2.000 ||| -241.586
+15 ||| he that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 assembly in national election won all and was elected as the 44th president of the united states . ||| lm_0=-49.232 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-10.000 tm_pt_3=-0.000 tm_pt_4=-7.000 tm_pt_5=-60.569 tm_pt_6=-16.864 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-2.376 tm_pt_10=-7.362 tm_pt_11=-0.000 tm_pt_12=-5.408 tm_pt_13=-0.000 tm_pt_14=-21.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=7.000 WordPenalty=-10.857 OOVPenalty=2.000 ||| -241.820
+15 ||| in the same year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 assembly in election of national won all and was elected as the 44th president of the united states . ||| lm_0=-44.575 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-10.000 tm_pt_3=-0.000 tm_pt_4=-7.000 tm_pt_5=-72.906 tm_pt_6=-20.726 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-4.008 tm_pt_10=-14.807 tm_pt_11=-0.000 tm_pt_12=-12.442 tm_pt_13=-0.000 tm_pt_14=-23.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=7.000 WordPenalty=-11.726 OOVPenalty=2.000 ||| -241.865
+15 ||| he that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 assembly the national election won all and was elected as the 44th president of the united states . ||| lm_0=-48.445 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-10.000 tm_pt_3=-0.000 tm_pt_4=-7.000 tm_pt_5=-62.362 tm_pt_6=-17.724 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-2.387 tm_pt_10=-8.613 tm_pt_11=-0.000 tm_pt_12=-5.591 tm_pt_13=-0.000 tm_pt_14=-21.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=7.000 WordPenalty=-10.857 OOVPenalty=2.000 ||| -241.881
+15 ||| in the year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national assembly in election won all and was elected as the 44th president of the united states . ||| lm_0=-44.737 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-11.000 tm_pt_3=-0.000 tm_pt_4=-9.000 tm_pt_5=-67.702 tm_pt_6=-20.073 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.008 tm_pt_10=-17.796 tm_pt_11=-0.000 tm_pt_12=-9.831 tm_pt_13=-0.000 tm_pt_14=-21.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=9.000 WordPenalty=-10.857 OOVPenalty=2.000 ||| -241.915
+15 ||| in that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national assembly in election won all and was elected as the 44th president of the united states . ||| lm_0=-46.393 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-10.000 tm_pt_3=-0.000 tm_pt_4=-8.000 tm_pt_5=-65.673 tm_pt_6=-20.144 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-2.376 tm_pt_10=-11.763 tm_pt_11=-0.000 tm_pt_12=-9.257 tm_pt_13=-0.000 tm_pt_14=-21.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=8.000 WordPenalty=-10.857 OOVPenalty=2.000 ||| -241.944
+16 ||| many indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be from upper stage ||| lm_0=-35.728 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-5.000 tm_pt_3=-0.000 tm_pt_4=-5.000 tm_pt_5=-14.556 tm_pt_6=-12.125 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.000 tm_pt_10=-2.670 tm_pt_11=-0.000 tm_pt_12=-0.912 tm_pt_13=-0.000 tm_pt_14=-6.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=8.000 WordPenalty=-4.777 OOVPenalty=3.000 ||| -332.122
+16 ||| many indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be from the upper stage ||| lm_0=-35.506 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-5.000 tm_pt_3=-0.000 tm_pt_4=-5.000 tm_pt_5=-20.408 tm_pt_6=-12.744 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.000 tm_pt_10=-3.443 tm_pt_11=-0.000 tm_pt_12=-4.470 tm_pt_13=-0.000 tm_pt_14=-7.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=8.000 WordPenalty=-5.212 OOVPenalty=3.000 ||| -332.706
+16 ||| many indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 from \u099f\u09c7\u0995\u09cd\u09b8\u09be upper stage ||| lm_0=-35.728 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-5.000 tm_pt_3=-0.000 tm_pt_4=-4.000 tm_pt_5=-14.556 tm_pt_6=-12.125 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.000 tm_pt_10=-3.002 tm_pt_11=-0.000 tm_pt_12=-1.537 tm_pt_13=-0.000 tm_pt_14=-6.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=7.000 WordPenalty=-4.777 OOVPenalty=3.000 ||| -333.394
+16 ||| many indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be to upper stage ||| lm_0=-35.464 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-5.000 tm_pt_3=-0.000 tm_pt_4=-5.000 tm_pt_5=-16.518 tm_pt_6=-13.004 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.000 tm_pt_10=-4.474 tm_pt_11=-0.000 tm_pt_12=-2.241 tm_pt_13=-0.000 tm_pt_14=-6.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=8.000 WordPenalty=-4.777 OOVPenalty=3.000 ||| -333.411
+16 ||| many of the indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be from upper stage ||| lm_0=-35.710 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-5.000 tm_pt_3=-0.000 tm_pt_4=-4.000 tm_pt_5=-20.114 tm_pt_6=-12.391 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.368 tm_pt_10=-2.333 tm_pt_11=-0.000 tm_pt_12=-5.855 tm_pt_13=-0.000 tm_pt_14=-8.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=6.000 WordPenalty=-5.646 OOVPenalty=3.000 ||| -333.551
+16 ||| many of indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be from upper stage ||| lm_0=-36.387 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-5.000 tm_pt_3=-0.000 tm_pt_4=-4.000 tm_pt_5=-15.179 tm_pt_6=-12.125 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.050 tm_pt_10=-2.333 tm_pt_11=-0.000 tm_pt_12=-4.534 tm_pt_13=-0.000 tm_pt_14=-7.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=7.000 WordPenalty=-5.212 OOVPenalty=3.000 ||| -333.558
+16 ||| many the indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be from upper stage ||| lm_0=-37.500 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-5.000 tm_pt_3=-0.000 tm_pt_4=-5.000 tm_pt_5=-19.492 tm_pt_6=-12.391 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.000 tm_pt_10=-2.573 tm_pt_11=-0.000 tm_pt_12=-2.281 tm_pt_13=-0.000 tm_pt_14=-7.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=8.000 WordPenalty=-5.212 OOVPenalty=3.000 ||| -333.877
+16 ||| of the indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be from upper stage ||| lm_0=-34.639 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-5.000 tm_pt_3=-0.000 tm_pt_4=-5.000 tm_pt_5=-24.926 tm_pt_6=-14.313 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.050 tm_pt_10=-6.573 tm_pt_11=-0.000 tm_pt_12=-4.846 tm_pt_13=-0.000 tm_pt_14=-7.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=8.000 WordPenalty=-5.212 OOVPenalty=3.000 ||| -334.019
+16 ||| many indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be upper stage from ||| lm_0=-36.278 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-5.000 tm_pt_3=-0.000 tm_pt_4=-4.000 tm_pt_5=-14.556 tm_pt_6=-12.125 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.000 tm_pt_10=-2.830 tm_pt_11=-0.000 tm_pt_12=-1.650 tm_pt_13=-0.000 tm_pt_14=-6.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=7.000 WordPenalty=-4.777 OOVPenalty=3.000 ||| -334.071
+17 ||| britain writers written drama novels stories and recently scripts in \u0986\u09a6\u09c3\u09a4 . ||| lm_0=-40.717 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-11.000 tm_pt_3=-0.000 tm_pt_4=-10.000 tm_pt_5=-20.642 tm_pt_6=-10.927 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-0.874 tm_pt_10=-16.732 tm_pt_11=-0.000 tm_pt_12=-5.024 tm_pt_13=-0.000 tm_pt_14=-11.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=11.000 WordPenalty=-6.080 OOVPenalty=1.000 ||| -140.876
+17 ||| britain writers written drama novels stories and in the recent scripts in \u0986\u09a6\u09c3\u09a4 . ||| lm_0=-40.996 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-11.000 tm_pt_3=-0.000 tm_pt_4=-10.000 tm_pt_5=-33.671 tm_pt_6=-12.314 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-1.738 tm_pt_10=-15.633 tm_pt_11=-0.000 tm_pt_12=-6.122 tm_pt_13=-0.000 tm_pt_14=-13.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=11.000 WordPenalty=-6.949 OOVPenalty=1.000 ||| -141.370
+17 ||| britain writers written drama novel stories and recently scripts in \u0986\u09a6\u09c3\u09a4 . ||| lm_0=-40.536 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-11.000 tm_pt_3=-0.000 tm_pt_4=-10.000 tm_pt_5=-21.690 tm_pt_6=-11.374 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-0.874 tm_pt_10=-17.888 tm_pt_11=-0.000 tm_pt_12=-5.391 tm_pt_13=-0.000 tm_pt_14=-11.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=11.000 WordPenalty=-6.080 OOVPenalty=1.000 ||| -141.436
+17 ||| britain writers written drama novels story and recently scripts in \u0986\u09a6\u09c3\u09a4 . ||| lm_0=-40.669 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-11.000 tm_pt_3=-0.000 tm_pt_4=-10.000 tm_pt_5=-21.556 tm_pt_6=-11.746 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-0.874 tm_pt_10=-17.463 tm_pt_11=-0.000 tm_pt_12=-5.755 tm_pt_13=-0.000 tm_pt_14=-11.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=11.000 WordPenalty=-6.080 OOVPenalty=1.000 ||| -141.720
+17 ||| britain writers written drama novels stories and recently script in \u0986\u09a6\u09c3\u09a4 . ||| lm_0=-40.421 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-11.000 tm_pt_3=-0.000 tm_pt_4=-10.000 tm_pt_5=-21.418 tm_pt_6=-10.442 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-1.824 tm_pt_10=-17.019 tm_pt_11=-0.000 tm_pt_12=-6.410 tm_pt_13=-0.000 tm_pt_14=-11.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=11.000 WordPenalty=-6.080 OOVPenalty=1.000 ||| -141.838
+17 ||| britain writers written drama novel stories and in the recent scripts in \u0986\u09a6\u09c3\u09a4 . ||| lm_0=-40.816 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-11.000 tm_pt_3=-0.000 tm_pt_4=-10.000 tm_pt_5=-34.718 tm_pt_6=-12.760 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-1.739 tm_pt_10=-16.789 tm_pt_11=-0.000 tm_pt_12=-6.490 tm_pt_13=-0.000 tm_pt_14=-13.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=11.000 WordPenalty=-6.949 OOVPenalty=1.000 ||| -141.930
+17 ||| britain writers written drama novels stories and recently scripts in the \u0986\u09a6\u09c3\u09a4 . ||| lm_0=-41.648 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-11.000 tm_pt_3=-0.000 tm_pt_4=-11.000 tm_pt_5=-28.905 tm_pt_6=-10.992 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-1.738 tm_pt_10=-16.679 tm_pt_11=-0.000 tm_pt_12=-6.459 tm_pt_13=-0.000 tm_pt_14=-12.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=12.000 WordPenalty=-6.514 OOVPenalty=1.000 ||| -141.937
+17 ||| britain writers the drama novels stories and recently scripts in \u0986\u09a6\u09c3\u09a4 . ||| lm_0=-39.941 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-11.000 tm_pt_3=-0.000 tm_pt_4=-10.000 tm_pt_5=-25.509 tm_pt_6=-11.095 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-0.959 tm_pt_10=-21.032 tm_pt_11=-0.000 tm_pt_12=-5.311 tm_pt_13=-0.000 tm_pt_14=-11.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=11.000 WordPenalty=-6.080 OOVPenalty=1.000 ||| -142.192
+17 ||| britain writers written drama novels story and in the recent scripts in \u0986\u09a6\u09c3\u09a4 . ||| lm_0=-40.949 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-11.000 tm_pt_3=-0.000 tm_pt_4=-10.000 tm_pt_5=-34.584 tm_pt_6=-13.132 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-1.738 tm_pt_10=-16.364 tm_pt_11=-0.000 tm_pt_12=-6.853 tm_pt_13=-0.000 tm_pt_14=-13.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=11.000 WordPenalty=-6.949 OOVPenalty=1.000 ||| -142.214
+17 ||| britain writers written drama novel story and recently scripts in \u0986\u09a6\u09c3\u09a4 . ||| lm_0=-40.488 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-11.000 tm_pt_3=-0.000 tm_pt_4=-10.000 tm_pt_5=-22.604 tm_pt_6=-12.192 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-0.874 tm_pt_10=-18.619 tm_pt_11=-0.000 tm_pt_12=-6.122 tm_pt_13=-0.000 tm_pt_14=-11.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=11.000 WordPenalty=-6.080 OOVPenalty=1.000 ||| -142.280
+18 ||| 1919 , in the month of may it was published in saogat magazine . ||| lm_0=-24.903 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-8.000 tm_pt_3=-0.000 tm_pt_4=-6.000 tm_pt_5=-32.234 tm_pt_6=-12.237 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-0.745 tm_pt_10=-7.090 tm_pt_11=-0.000 tm_pt_12=-5.917 tm_pt_13=-0.000 tm_pt_14=-14.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=6.000 WordPenalty=-6.949 ||| -19.117
+18 ||| in the month of may , 1919 it was published in saogat magazine . ||| lm_0=-24.586 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-8.000 tm_pt_3=-0.000 tm_pt_4=-5.000 tm_pt_5=-32.234 tm_pt_6=-12.237 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-0.745 tm_pt_10=-1.250 tm_pt_11=-0.000 tm_pt_12=-5.818 tm_pt_13=-0.000 tm_pt_14=-14.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=4.000 WordPenalty=-6.949 ||| -19.140
+18 ||| 1919 , it was published in saogat magazine in the month of may . ||| lm_0=-23.275 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-7.000 tm_pt_3=-0.000 tm_pt_4=-4.000 tm_pt_5=-32.234 tm_pt_6=-12.237 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-1.113 tm_pt_10=-5.992 tm_pt_11=-0.000 tm_pt_12=-7.203 tm_pt_13=-0.000 tm_pt_14=-14.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=3.000 WordPenalty=-6.949 ||| -19.348
+18 ||| 1919 on it was published in saogat magazine in the month of may . ||| lm_0=-25.454 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-7.000 tm_pt_3=-0.000 tm_pt_4=-4.000 tm_pt_5=-29.708 tm_pt_6=-12.226 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-1.113 tm_pt_10=-1.500 tm_pt_11=-0.000 tm_pt_12=-5.763 tm_pt_13=-0.000 tm_pt_14=-14.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=3.000 WordPenalty=-6.949 ||| -19.883
+18 ||| in 1919 it was published in saogat magazine in the month of may . ||| lm_0=-23.553 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-7.000 tm_pt_3=-0.000 tm_pt_4=-3.000 tm_pt_5=-31.785 tm_pt_6=-12.847 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-1.131 tm_pt_10=-4.775 tm_pt_11=-0.000 tm_pt_12=-6.565 tm_pt_13=-0.000 tm_pt_14=-14.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=2.000 WordPenalty=-6.949 ||| -20.253
+18 ||| in 1919 in the month of may it was published in saogat magazine . ||| lm_0=-24.826 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-7.000 tm_pt_3=-0.000 tm_pt_4=-3.000 tm_pt_5=-31.785 tm_pt_6=-12.847 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-0.763 tm_pt_10=-5.505 tm_pt_11=-0.000 tm_pt_12=-5.474 tm_pt_13=-0.000 tm_pt_14=-14.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=3.000 WordPenalty=-6.949 ||| -20.330
+18 ||| 1919 , it was published in saogat magazine during the month of may . ||| lm_0=-24.175 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-7.000 tm_pt_3=-0.000 tm_pt_4=-4.000 tm_pt_5=-31.037 tm_pt_6=-12.589 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-1.246 tm_pt_10=-5.992 tm_pt_11=-0.000 tm_pt_12=-8.050 tm_pt_13=-0.000 tm_pt_14=-14.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=3.000 WordPenalty=-6.949 ||| -20.762
+18 ||| 1919 on it was published in saogat magazine during the month of may . ||| lm_0=-26.354 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-7.000 tm_pt_3=-0.000 tm_pt_4=-4.000 tm_pt_5=-28.511 tm_pt_6=-12.579 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-1.246 tm_pt_10=-1.500 tm_pt_11=-0.000 tm_pt_12=-6.610 tm_pt_13=-0.000 tm_pt_14=-14.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=3.000 WordPenalty=-6.949 ||| -21.296
+18 ||| in 1919 it was published in saogat magazine during the month of may . ||| lm_0=-24.453 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-7.000 tm_pt_3=-0.000 tm_pt_4=-3.000 tm_pt_5=-30.588 tm_pt_6=-13.199 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-1.264 tm_pt_10=-4.775 tm_pt_11=-0.000 tm_pt_12=-7.413 tm_pt_13=-0.000 tm_pt_14=-14.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=2.000 WordPenalty=-6.949 ||| -21.667
+19 ||| 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium was arranged . ||| lm_0=-57.604 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-11.000 tm_pt_3=-0.000 tm_pt_4=-10.000 tm_pt_5=-21.244 tm_pt_6=-8.707 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.249 tm_pt_10=-15.730 tm_pt_11=-0.000 tm_pt_12=-5.148 tm_pt_13=-0.000 tm_pt_14=-12.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=14.000 WordPenalty=-7.817 OOVPenalty=4.000 ||| -452.293
+19 ||| in year 2005 \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium was arranged . ||| lm_0=-58.848 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-11.000 tm_pt_3=-0.000 tm_pt_4=-10.000 tm_pt_5=-21.149 tm_pt_6=-8.568 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-3.114 tm_pt_10=-13.650 tm_pt_11=-0.000 tm_pt_12=-6.247 tm_pt_13=-0.000 tm_pt_14=-13.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=14.000 WordPenalty=-8.252 OOVPenalty=4.000 ||| -452.393
+19 ||| 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium was organized . ||| lm_0=-57.677 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-11.000 tm_pt_3=-0.000 tm_pt_4=-10.000 tm_pt_5=-22.039 tm_pt_6=-8.841 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.249 tm_pt_10=-16.240 tm_pt_11=-0.000 tm_pt_12=-5.148 tm_pt_13=-0.000 tm_pt_14=-12.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=14.000 WordPenalty=-7.817 OOVPenalty=4.000 ||| -452.714
+19 ||| in year 2005 \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium was organized . ||| lm_0=-58.922 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-11.000 tm_pt_3=-0.000 tm_pt_4=-10.000 tm_pt_5=-21.944 tm_pt_6=-8.702 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-3.114 tm_pt_10=-14.161 tm_pt_11=-0.000 tm_pt_12=-6.247 tm_pt_13=-0.000 tm_pt_14=-13.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=14.000 WordPenalty=-8.252 OOVPenalty=4.000 ||| -452.814
+19 ||| 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium was arranged in . ||| lm_0=-59.257 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-11.000 tm_pt_3=-0.000 tm_pt_4=-10.000 tm_pt_5=-28.217 tm_pt_6=-8.589 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.249 tm_pt_10=-15.324 tm_pt_11=-0.000 tm_pt_12=-5.148 tm_pt_13=-0.000 tm_pt_14=-13.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=14.000 WordPenalty=-8.252 OOVPenalty=4.000 ||| -453.595
+19 ||| in year 2005 \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium was arranged in . ||| lm_0=-60.502 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-11.000 tm_pt_3=-0.000 tm_pt_4=-10.000 tm_pt_5=-28.123 tm_pt_6=-8.450 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-3.114 tm_pt_10=-13.245 tm_pt_11=-0.000 tm_pt_12=-6.247 tm_pt_13=-0.000 tm_pt_14=-14.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=14.000 WordPenalty=-8.686 OOVPenalty=4.000 ||| -453.695
+19 ||| 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium arranged in . ||| lm_0=-58.454 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-11.000 tm_pt_3=-0.000 tm_pt_4=-10.000 tm_pt_5=-23.377 tm_pt_6=-9.826 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.249 tm_pt_10=-15.383 tm_pt_11=-0.000 tm_pt_12=-4.247 tm_pt_13=-0.000 tm_pt_14=-12.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=14.000 WordPenalty=-7.817 OOVPenalty=4.000 ||| -453.728
+19 ||| 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association of the tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium was arranged . ||| lm_0=-59.247 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-11.000 tm_pt_3=-0.000 tm_pt_4=-10.000 tm_pt_5=-41.334 tm_pt_6=-9.112 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.249 tm_pt_10=-13.938 tm_pt_11=-0.000 tm_pt_12=-5.148 tm_pt_13=-0.000 tm_pt_14=-14.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=14.000 WordPenalty=-8.686 OOVPenalty=4.000 ||| -453.979
+19 ||| in year 2005 \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association of the tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium was arranged . ||| lm_0=-60.492 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-11.000 tm_pt_3=-0.000 tm_pt_4=-10.000 tm_pt_5=-41.240 tm_pt_6=-8.974 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-3.114 tm_pt_10=-11.858 tm_pt_11=-0.000 tm_pt_12=-6.247 tm_pt_13=-0.000 tm_pt_14=-15.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=14.000 WordPenalty=-9.120 OOVPenalty=4.000 ||| -454.079
+19 ||| 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association of the tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium was organized . ||| lm_0=-59.321 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-11.000 tm_pt_3=-0.000 tm_pt_4=-10.000 tm_pt_5=-42.129 tm_pt_6=-9.246 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.249 tm_pt_10=-14.449 tm_pt_11=-0.000 tm_pt_12=-5.148 tm_pt_13=-0.000 tm_pt_14=-14.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=14.000 WordPenalty=-8.686 OOVPenalty=4.000 ||| -454.400
+20 ||| to prevent this several measures are taken . ||| lm_0=-11.632 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-1.000 tm_pt_3=-0.000 tm_pt_4=-1.000 tm_pt_5=-22.680 tm_pt_6=-30.812 tm_pt_7=-1.000 tm_pt_8=-2.718 tm_pt_9=-1.000 tm_pt_10=0.000 tm_pt_11=-0.000 tm_pt_12=-1.386 tm_pt_13=-0.000 tm_pt_14=-8.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=1.000 WordPenalty=-4.343 ||| -12.088
+20 ||| to avoid this possibility several measures are taken . ||| lm_0=-13.461 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-1.000 tm_pt_3=-0.000 tm_pt_4=-1.000 tm_pt_5=-24.597 tm_pt_6=-31.733 tm_pt_7=-1.000 tm_pt_8=-2.718 tm_pt_9=-1.000 tm_pt_10=0.000 tm_pt_11=-0.000 tm_pt_12=-1.386 tm_pt_13=-0.000 tm_pt_14=-9.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=1.000 WordPenalty=-4.777 ||| -13.100
+20 ||| to prevent this several measures are the . ||| lm_0=-14.066 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-3.000 tm_pt_3=-0.000 tm_pt_4=-3.000 tm_pt_5=-24.227 tm_pt_6=-27.251 tm_pt_7=-3.000 tm_pt_8=-8.154 tm_pt_9=-1.000 tm_pt_10=-3.426 tm_pt_11=-0.000 tm_pt_12=-2.285 tm_pt_13=-0.000 tm_pt_14=-8.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=3.000 WordPenalty=-4.343 ||| -15.767
+20 ||| to prevent this several measures are in the . ||| lm_0=-15.062 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-3.000 tm_pt_3=-0.000 tm_pt_4=-3.000 tm_pt_5=-27.397 tm_pt_6=-27.297 tm_pt_7=-3.000 tm_pt_8=-8.154 tm_pt_9=-1.000 tm_pt_10=-2.477 tm_pt_11=-0.000 tm_pt_12=-4.347 tm_pt_13=-0.000 tm_pt_14=-9.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=3.000 WordPenalty=-4.777 ||| -16.171
+20 ||| to prevent this several measures are in . ||| lm_0=-14.649 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-3.000 tm_pt_3=-0.000 tm_pt_4=-3.000 tm_pt_5=-23.388 tm_pt_6=-27.344 tm_pt_7=-3.000 tm_pt_8=-8.154 tm_pt_9=-1.000 tm_pt_10=-2.771 tm_pt_11=-0.000 tm_pt_12=-2.699 tm_pt_13=-0.000 tm_pt_14=-8.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=3.000 WordPenalty=-4.343 ||| -16.336
+21 ||| on 5th february , \u09e7\u09ef\u09ec\u09ec \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 of \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 a national was held in . ||| lm_0=-40.122 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-7.000 tm_pt_3=-0.000 tm_pt_4=-6.000 tm_pt_5=-30.321 tm_pt_6=-13.470 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-1.386 tm_pt_10=-6.440 tm_pt_11=-0.000 tm_pt_12=-3.037 tm_pt_13=-0.000 tm_pt_14=-11.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=8.000 WordPenalty=-6.949 OOVPenalty=3.000 ||| -334.940
+21 ||| on 5th february , \u09e7\u09ef\u09ec\u09ec \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 against a national was held in . ||| lm_0=-40.847 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-7.000 tm_pt_3=-0.000 tm_pt_4=-5.000 tm_pt_5=-25.571 tm_pt_6=-13.159 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-1.503 tm_pt_10=-3.423 tm_pt_11=-0.000 tm_pt_12=-3.325 tm_pt_13=-0.000 tm_pt_14=-11.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=7.000 WordPenalty=-6.949 OOVPenalty=3.000 ||| -335.202
+21 ||| on 5th february , \u09e7\u09ef\u09ec\u09ec \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 against \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 a national was held in . ||| lm_0=-41.407 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-7.000 tm_pt_3=-0.000 tm_pt_4=-6.000 tm_pt_5=-25.571 tm_pt_6=-13.159 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-1.503 tm_pt_10=-4.246 tm_pt_11=-0.000 tm_pt_12=-3.548 tm_pt_13=-0.000 tm_pt_14=-11.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=8.000 WordPenalty=-6.949 OOVPenalty=3.000 ||| -335.223
+21 ||| on 5th february , \u09e7\u09ef\u09ec\u09ec \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 opposition \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 a national was held in . ||| lm_0=-42.291 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-7.000 tm_pt_3=-0.000 tm_pt_4=-6.000 tm_pt_5=-24.118 tm_pt_6=-13.565 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-1.503 tm_pt_10=-1.442 tm_pt_11=-0.000 tm_pt_12=-3.548 tm_pt_13=-0.000 tm_pt_14=-11.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=8.000 WordPenalty=-6.949 OOVPenalty=3.000 ||| -335.450
+22 ||| bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in in the . ||| lm_0=-45.104 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-11.000 tm_pt_3=-0.000 tm_pt_4=-11.000 tm_pt_5=-50.512 tm_pt_6=-9.888 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.050 tm_pt_10=-23.145 tm_pt_11=-0.000 tm_pt_12=-2.086 tm_pt_13=-0.000 tm_pt_14=-14.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=14.000 WordPenalty=-8.252 OOVPenalty=3.000 ||| -341.281
+22 ||| bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in took the . ||| lm_0=-46.563 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-11.000 tm_pt_3=-0.000 tm_pt_4=-11.000 tm_pt_5=-46.603 tm_pt_6=-9.693 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.368 tm_pt_10=-19.978 tm_pt_11=-0.000 tm_pt_12=-2.779 tm_pt_13=-0.000 tm_pt_14=-14.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=14.000 WordPenalty=-8.252 OOVPenalty=3.000 ||| -341.957
+22 ||| bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank took secured its place in the . ||| lm_0=-45.737 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-11.000 tm_pt_3=-0.000 tm_pt_4=-10.000 tm_pt_5=-46.603 tm_pt_6=-9.693 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.368 tm_pt_10=-19.856 tm_pt_11=-0.000 tm_pt_12=-3.210 tm_pt_13=-0.000 tm_pt_14=-14.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=13.000 WordPenalty=-8.252 OOVPenalty=3.000 ||| -342.020
+22 ||| bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f took bank secured its place in the . ||| lm_0=-45.737 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-11.000 tm_pt_3=-0.000 tm_pt_4=-9.000 tm_pt_5=-46.603 tm_pt_6=-9.693 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.368 tm_pt_10=-19.856 tm_pt_11=-0.000 tm_pt_12=-3.210 tm_pt_13=-0.000 tm_pt_14=-14.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=12.000 WordPenalty=-8.252 OOVPenalty=3.000 ||| -342.985
+22 ||| bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in adopted the . ||| lm_0=-47.359 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-11.000 tm_pt_3=-0.000 tm_pt_4=-11.000 tm_pt_5=-45.590 tm_pt_6=-10.329 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-3.000 tm_pt_10=-18.106 tm_pt_11=-0.000 tm_pt_12=-3.472 tm_pt_13=-0.000 tm_pt_14=-14.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=14.000 WordPenalty=-8.252 OOVPenalty=3.000 ||| -343.243
+22 ||| bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in in the goods . ||| lm_0=-47.396 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-10.000 tm_pt_3=-0.000 tm_pt_4=-9.000 tm_pt_5=-52.623 tm_pt_6=-10.570 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-2.185 tm_pt_10=-20.454 tm_pt_11=-0.000 tm_pt_12=-3.811 tm_pt_13=-0.000 tm_pt_14=-15.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=12.000 WordPenalty=-8.686 OOVPenalty=3.000 ||| -343.618
+22 ||| bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and the islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in in the . ||| lm_0=-46.084 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-11.000 tm_pt_3=-0.000 tm_pt_4=-10.000 tm_pt_5=-52.980 tm_pt_6=-9.888 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.050 tm_pt_10=-23.894 tm_pt_11=-0.000 tm_pt_12=-6.709 tm_pt_13=-0.000 tm_pt_14=-15.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=13.000 WordPenalty=-8.686 OOVPenalty=3.000 ||| -344.062
+22 ||| bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and the islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in took the . ||| lm_0=-47.544 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-11.000 tm_pt_3=-0.000 tm_pt_4=-10.000 tm_pt_5=-49.070 tm_pt_6=-9.693 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.368 tm_pt_10=-20.727 tm_pt_11=-0.000 tm_pt_12=-7.402 tm_pt_13=-0.000 tm_pt_14=-15.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=13.000 WordPenalty=-8.686 OOVPenalty=3.000 ||| -344.738
+22 ||| bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and had \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in in the . ||| lm_0=-44.838 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-11.000 tm_pt_3=-0.000 tm_pt_4=-11.000 tm_pt_5=-55.306 tm_pt_6=-12.832 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-3.050 tm_pt_10=-25.587 tm_pt_11=-0.000 tm_pt_12=-4.389 tm_pt_13=-0.000 tm_pt_14=-14.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=14.000 WordPenalty=-8.252 OOVPenalty=3.000 ||| -345.143
+22 ||| bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and had \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in took the . ||| lm_0=-46.297 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-11.000 tm_pt_3=-0.000 tm_pt_4=-11.000 tm_pt_5=-51.396 tm_pt_6=-12.638 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-3.368 tm_pt_10=-22.421 tm_pt_11=-0.000 tm_pt_12=-5.082 tm_pt_13=-0.000 tm_pt_14=-14.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=14.000 WordPenalty=-8.252 OOVPenalty=3.000 ||| -345.819
+23 ||| subject category : encyclopedia ||| lm_0=-5.763 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-2.000 tm_pt_3=-0.000 tm_pt_4=-2.000 tm_pt_5=-1.727 tm_pt_6=-1.749 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.135 tm_pt_10=0.000 tm_pt_11=-0.000 tm_pt_12=-1.881 tm_pt_13=-0.000 tm_pt_14=-4.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=2.000 WordPenalty=-2.606 ||| 1.726
+23 ||| subject class : encyclopedia ||| lm_0=-6.150 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-2.000 tm_pt_3=-0.000 tm_pt_4=-2.000 tm_pt_5=-2.591 tm_pt_6=-2.071 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.135 tm_pt_10=0.000 tm_pt_11=-0.000 tm_pt_12=-2.866 tm_pt_13=-0.000 tm_pt_14=-4.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=2.000 WordPenalty=-2.606 ||| 0.624
+23 ||| subject matter : encyclopedia ||| lm_0=-6.126 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-2.000 tm_pt_3=-0.000 tm_pt_4=-2.000 tm_pt_5=-3.165 tm_pt_6=-2.139 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.135 tm_pt_10=-0.032 tm_pt_11=-0.000 tm_pt_12=-2.958 tm_pt_13=-0.000 tm_pt_14=-4.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=2.000 WordPenalty=-2.606 ||| 0.479
+23 ||| subject : encyclopedia ||| lm_0=-5.528 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-2.000 tm_pt_3=-0.000 tm_pt_4=-2.000 tm_pt_5=-1.657 tm_pt_6=-1.542 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.000 tm_pt_10=-0.420 tm_pt_11=-0.000 tm_pt_12=-1.500 tm_pt_13=-0.000 tm_pt_14=-3.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=2.000 WordPenalty=-2.171 ||| 0.305
+23 ||| category : encyclopedia ||| lm_0=-5.707 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-2.000 tm_pt_3=-0.000 tm_pt_4=-2.000 tm_pt_5=-1.425 tm_pt_6=-2.012 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.135 tm_pt_10=-0.020 tm_pt_11=-0.000 tm_pt_12=-1.817 tm_pt_13=-0.000 tm_pt_14=-3.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=2.000 WordPenalty=-2.171 ||| -0.153
+24 ||| russia france and israel the main \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 state of india and defense sub country . ||| lm_0=-40.772 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-9.000 tm_pt_3=-0.000 tm_pt_4=-7.000 tm_pt_5=-20.239 tm_pt_6=-9.156 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-1.474 tm_pt_10=-5.167 tm_pt_11=-0.000 tm_pt_12=-4.495 tm_pt_13=-0.000 tm_pt_14=-14.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=8.000 WordPenalty=-7.383 OOVPenalty=1.000 ||| -132.586
+24 ||| russia france and israel is the main \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 country and defense sub country . ||| lm_0=-39.017 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-10.000 tm_pt_3=-0.000 tm_pt_4=-9.000 tm_pt_5=-23.891 tm_pt_6=-8.955 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-1.106 tm_pt_10=-7.345 tm_pt_11=-0.000 tm_pt_12=-6.401 tm_pt_13=-0.000 tm_pt_14=-13.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=10.000 WordPenalty=-6.949 OOVPenalty=1.000 ||| -132.906
+24 ||| russia france and israel the main \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 country of india and defense sub country . ||| lm_0=-41.872 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-9.000 tm_pt_3=-0.000 tm_pt_4=-7.000 tm_pt_5=-20.301 tm_pt_6=-8.306 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-1.474 tm_pt_10=-5.060 tm_pt_11=-0.000 tm_pt_12=-3.413 tm_pt_13=-0.000 tm_pt_14=-14.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=8.000 WordPenalty=-7.383 OOVPenalty=1.000 ||| -133.257
+24 ||| russia france and israel is the main \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 state and defense sub country . ||| lm_0=-38.975 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-9.000 tm_pt_3=-0.000 tm_pt_4=-8.000 tm_pt_5=-23.828 tm_pt_6=-9.805 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-1.474 tm_pt_10=-7.409 tm_pt_11=-0.000 tm_pt_12=-6.492 tm_pt_13=-0.000 tm_pt_14=-13.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=9.000 WordPenalty=-6.949 OOVPenalty=1.000 ||| -133.294
+24 ||| russia france and israel the main \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 state of india and defence sub country . ||| lm_0=-40.463 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-9.000 tm_pt_3=-0.000 tm_pt_4=-7.000 tm_pt_5=-20.651 tm_pt_6=-10.543 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-2.106 tm_pt_10=-4.762 tm_pt_11=-0.000 tm_pt_12=-5.188 tm_pt_13=-0.000 tm_pt_14=-14.000 tm_pt_15=-0.000 tm_pt_16=-1.000 tm_glue_0=8.000 WordPenalty=-7.383 OOVPenalty=1.000 ||| -133.416
+24 ||| russia france and israel is the main \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 country and defence sub country . ||| lm_0=-38.708 tm_pt_0=-0.000 tm_pt_1=-0.000 tm_pt_2=-10.000 tm_pt_3=-0.000 tm_pt_4=-9.000 tm_pt_5=-24.304 tm_pt_6=-10.341 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-1.738 tm_pt_10=-6.939 tm_pt_11=-0.000 tm_pt_12=-7.094 tm_pt_13=-0.000 tm_pt_14=-13.000 tm_pt_15=-0.000 tm_pt_16=-0.000 tm_glue_0=10.000 WordPenalty=-6.949 OOVPenalty=1.000 ||| -133.7
<TRUNCATED>
[05/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/packed/output.gold
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/packed/output.gold b/joshua-core/src/test/resources/bn-en/packed/output.gold
new file mode 100644
index 0000000..444006b
--- /dev/null
+++ b/joshua-core/src/test/resources/bn-en/packed/output.gold
@@ -0,0 +1,862 @@
+0 ||| rabindranath was born in kolkata is a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| lm_0=-26.747 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-36.862 tm_pt_6=-15.484 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-2.002 tm_pt_10=-8.295 tm_pt_11=0.000 tm_pt_12=-9.671 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 WordPenalty=-6.080 OOVPenalty=-200.000 ||| -230.669
+0 ||| rabindranath was born in a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| lm_0=-23.712 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-30.409 tm_pt_6=-15.712 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.138 tm_pt_10=-14.388 tm_pt_11=0.000 tm_pt_12=-8.572 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 WordPenalty=-5.212 OOVPenalty=-200.000 ||| -230.837
+0 ||| rabindranath was born in kolkata is one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| lm_0=-27.693 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-36.092 tm_pt_6=-16.482 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-2.002 tm_pt_10=-6.775 tm_pt_11=0.000 tm_pt_12=-10.337 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 WordPenalty=-6.080 OOVPenalty=-200.000 ||| -231.593
+0 ||| rabindranath was born in kolkata in a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| lm_0=-28.078 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-36.789 tm_pt_6=-15.727 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.005 tm_pt_10=-9.137 tm_pt_11=0.000 tm_pt_12=-10.310 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 WordPenalty=-6.080 OOVPenalty=-200.000 ||| -231.977
+0 ||| rabindranath was born in kolkata a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| lm_0=-27.184 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-31.718 tm_pt_6=-15.214 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.005 tm_pt_10=-10.410 tm_pt_11=0.000 tm_pt_12=-7.725 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 WordPenalty=-5.646 OOVPenalty=-200.000 ||| -232.054
+0 ||| rabindranath was born in the a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| lm_0=-26.715 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-35.425 tm_pt_6=-15.276 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.021 tm_pt_10=-12.028 tm_pt_11=0.000 tm_pt_12=-8.061 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 WordPenalty=-5.646 OOVPenalty=-200.000 ||| -232.473
+1 ||| in recent times of india with united states relationship between improved . ||| lm_0=-23.083 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-32.440 tm_pt_6=-14.884 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-2.503 tm_pt_10=-8.346 tm_pt_11=0.000 tm_pt_12=-7.700 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=7.000 WordPenalty=-6.080 ||| -26.695
+1 ||| in recent times india with united states relationship between improved . ||| lm_0=-23.087 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-31.818 tm_pt_6=-14.884 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-2.503 tm_pt_10=-8.857 tm_pt_11=0.000 tm_pt_12=-6.407 tm_pt_13=0.000 tm_pt_14=-11.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 WordPenalty=-5.646 ||| -27.174
+1 ||| in recent times of india with the united states relationship between improved . ||| lm_0=-23.140 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-40.251 tm_pt_6=-15.213 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-3.368 tm_pt_10=-8.765 tm_pt_11=0.000 tm_pt_12=-8.799 tm_pt_13=0.000 tm_pt_14=-13.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=7.000 WordPenalty=-6.514 ||| -27.329
+1 ||| in recent times of india with united states relation between improved . ||| lm_0=-23.024 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-33.334 tm_pt_6=-15.326 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-2.503 tm_pt_10=-9.262 tm_pt_11=0.000 tm_pt_12=-7.700 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=7.000 WordPenalty=-6.080 ||| -27.367
+1 ||| in recent times in india with united states relationship between improved . ||| lm_0=-23.598 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-37.182 tm_pt_6=-15.449 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-2.553 tm_pt_10=-9.541 tm_pt_11=0.000 tm_pt_12=-9.147 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 WordPenalty=-6.080 ||| -27.695
+1 ||| in recent times india with the united states relationship between improved . ||| lm_0=-23.144 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-39.629 tm_pt_6=-15.213 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-3.368 tm_pt_10=-9.276 tm_pt_11=0.000 tm_pt_12=-7.505 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 WordPenalty=-6.080 ||| -27.807
+1 ||| in recent times india with united states relation between improved . ||| lm_0=-23.028 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-32.712 tm_pt_6=-15.326 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-2.503 tm_pt_10=-9.773 tm_pt_11=0.000 tm_pt_12=-6.407 tm_pt_13=0.000 tm_pt_14=-11.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 WordPenalty=-5.646 ||| -27.845
+1 ||| in recent times of india with the united states relation between improved . ||| lm_0=-23.081 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-41.145 tm_pt_6=-15.655 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-3.368 tm_pt_10=-9.681 tm_pt_11=0.000 tm_pt_12=-8.799 tm_pt_13=0.000 tm_pt_14=-13.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=7.000 WordPenalty=-6.514 ||| -28.000
+1 ||| in recent times of india with united states relation improved . ||| lm_0=-22.396 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-28.996 tm_pt_6=-15.443 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-2.503 tm_pt_10=-10.486 tm_pt_11=0.000 tm_pt_12=-7.700 tm_pt_13=0.000 tm_pt_14=-11.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=7.000 WordPenalty=-5.646 ||| -28.037
+1 ||| in recent times in india with united states relation between improved . ||| lm_0=-23.539 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-38.076 tm_pt_6=-15.891 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-2.553 tm_pt_10=-10.458 tm_pt_11=0.000 tm_pt_12=-9.147 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 WordPenalty=-6.080 ||| -28.367
+2 ||| mathematics is science language . ||| lm_0=-12.890 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-10.375 tm_pt_6=-3.926 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-0.052 tm_pt_10=-8.828 tm_pt_11=0.000 tm_pt_12=-3.463 tm_pt_13=0.000 tm_pt_14=-5.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 WordPenalty=-3.040 ||| -12.270
+2 ||| mathematics so science language . ||| lm_0=-15.141 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-6.483 tm_pt_6=-3.387 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-0.002 tm_pt_10=-4.024 tm_pt_11=0.000 tm_pt_12=-1.959 tm_pt_13=0.000 tm_pt_14=-5.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 WordPenalty=-3.040 ||| -12.290
+2 ||| mathematics hence science language . ||| lm_0=-16.246 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-4.000 tm_pt_5=-5.632 tm_pt_6=-4.485 tm_pt_7=-4.000 tm_pt_8=-10.872 tm_pt_9=-0.005 tm_pt_10=-3.149 tm_pt_11=0.000 tm_pt_12=-2.947 tm_pt_13=0.000 tm_pt_14=-5.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=4.000 WordPenalty=-3.040 ||| -14.648
+2 ||| mathematics that is why science language . ||| lm_0=-16.376 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-4.000 tm_pt_5=-18.343 tm_pt_6=-4.240 tm_pt_7=-4.000 tm_pt_8=-10.872 tm_pt_9=-1.002 tm_pt_10=-4.114 tm_pt_11=0.000 tm_pt_12=-4.893 tm_pt_13=0.000 tm_pt_14=-7.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=4.000 WordPenalty=-3.909 ||| -14.684
+2 ||| mathematics that science language . ||| lm_0=-14.001 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-4.000 tm_pt_5=-9.625 tm_pt_6=-3.926 tm_pt_7=-4.000 tm_pt_8=-10.872 tm_pt_9=-0.370 tm_pt_10=-8.539 tm_pt_11=0.000 tm_pt_12=-4.200 tm_pt_13=0.000 tm_pt_14=-5.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=4.000 WordPenalty=-3.040 ||| -14.686
+2 ||| mathematics is science languages . ||| lm_0=-13.230 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-4.000 tm_pt_5=-11.269 tm_pt_6=-6.157 tm_pt_7=-4.000 tm_pt_8=-10.872 tm_pt_9=-0.420 tm_pt_10=-8.412 tm_pt_11=0.000 tm_pt_12=-5.704 tm_pt_13=0.000 tm_pt_14=-5.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=4.000 WordPenalty=-3.040 ||| -14.843
+2 ||| mathematics so science languages . ||| lm_0=-15.480 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-4.000 tm_pt_5=-7.377 tm_pt_6=-5.618 tm_pt_7=-4.000 tm_pt_8=-10.872 tm_pt_9=-0.370 tm_pt_10=-3.608 tm_pt_11=0.000 tm_pt_12=-4.200 tm_pt_13=0.000 tm_pt_14=-5.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=4.000 WordPenalty=-3.040 ||| -14.864
+2 ||| mathematics is reasons language . ||| lm_0=-13.155 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-4.000 tm_pt_5=-11.525 tm_pt_6=-6.565 tm_pt_7=-4.000 tm_pt_8=-10.872 tm_pt_9=-1.050 tm_pt_10=-7.078 tm_pt_11=0.000 tm_pt_12=-5.452 tm_pt_13=0.000 tm_pt_14=-5.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=4.000 WordPenalty=-3.040 ||| -14.908
+3 ||| from this can easily it can be understood that this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf will be \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| lm_0=-36.632 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-25.797 tm_pt_6=-8.970 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-2.368 tm_pt_10=-3.079 tm_pt_11=0.000 tm_pt_12=-5.769 tm_pt_13=0.000 tm_pt_14=-13.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 WordPenalty=-7.817 OOVPenalty=-300.000 ||| -329.615
+3 ||| from this easily it can be understood that this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf will be \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| lm_0=-36.251 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-21.761 tm_pt_6=-8.823 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-1.503 tm_pt_10=-4.689 tm_pt_11=0.000 tm_pt_12=-4.671 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 WordPenalty=-7.383 OOVPenalty=-300.000 ||| -329.869
+3 ||| from this can easily it can be understood that this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf would be \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| lm_0=-36.991 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-27.059 tm_pt_6=-9.491 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-2.418 tm_pt_10=-3.384 tm_pt_11=0.000 tm_pt_12=-8.072 tm_pt_13=0.000 tm_pt_14=-13.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 WordPenalty=-7.817 OOVPenalty=-300.000 ||| -330.539
+3 ||| from this can easily it can be understood that this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf will \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| lm_0=-36.159 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-23.362 tm_pt_6=-9.017 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-2.386 tm_pt_10=-4.608 tm_pt_11=0.000 tm_pt_12=-7.849 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 WordPenalty=-7.383 OOVPenalty=-300.000 ||| -330.631
+3 ||| from this easily it can be understood that this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf would be \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| lm_0=-36.610 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-23.024 tm_pt_6=-9.344 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-1.553 tm_pt_10=-4.993 tm_pt_11=0.000 tm_pt_12=-6.973 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 WordPenalty=-7.383 OOVPenalty=-300.000 ||| -330.793
+3 ||| from this can easily it can be understood that in this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf will be \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| lm_0=-38.026 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-29.453 tm_pt_6=-9.512 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-2.368 tm_pt_10=-3.171 tm_pt_11=0.000 tm_pt_12=-7.992 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 WordPenalty=-8.252 OOVPenalty=-300.000 ||| -330.878
+3 ||| from this easily it can be understood that this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf will \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| lm_0=-35.778 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-19.326 tm_pt_6=-8.870 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-1.522 tm_pt_10=-6.217 tm_pt_11=0.000 tm_pt_12=-6.750 tm_pt_13=0.000 tm_pt_14=-11.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 WordPenalty=-6.949 OOVPenalty=-300.000 ||| -330.886
+3 ||| from this can easily it can be understood that these \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf will be \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| lm_0=-36.910 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-26.256 tm_pt_6=-10.992 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-2.368 tm_pt_10=-3.339 tm_pt_11=0.000 tm_pt_12=-8.031 tm_pt_13=0.000 tm_pt_14=-13.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 WordPenalty=-7.817 OOVPenalty=-300.000 ||| -331.119
+3 ||| from this easily it can be understood that in this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf will be \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| lm_0=-37.645 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-25.417 tm_pt_6=-9.365 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-1.503 tm_pt_10=-4.781 tm_pt_11=0.000 tm_pt_12=-6.894 tm_pt_13=0.000 tm_pt_14=-13.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 WordPenalty=-7.817 OOVPenalty=-300.000 ||| -331.132
+3 ||| from this can easily it can be understood that the \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf will be \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| lm_0=-36.132 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-29.132 tm_pt_6=-10.262 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-2.368 tm_pt_10=-5.918 tm_pt_11=0.000 tm_pt_12=-7.688 tm_pt_13=0.000 tm_pt_14=-13.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 WordPenalty=-7.817 OOVPenalty=-300.000 ||| -331.188
+4 ||| in the same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match from this novel . ||| lm_0=-39.886 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-27.312 tm_pt_6=-10.335 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-3.066 tm_pt_10=-14.509 tm_pt_11=0.000 tm_pt_12=-8.808 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 WordPenalty=-6.949 OOVPenalty=-200.000 ||| -244.715
+4 ||| in the same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match this novel from . ||| lm_0=-42.072 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-27.312 tm_pt_6=-10.335 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-3.016 tm_pt_10=-14.659 tm_pt_11=0.000 tm_pt_12=-9.596 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=12.000 WordPenalty=-6.949 OOVPenalty=-200.000 ||| -244.892
+4 ||| the same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match this novel from . ||| lm_0=-41.855 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-21.437 tm_pt_6=-10.032 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-3.009 tm_pt_10=-14.340 tm_pt_11=0.000 tm_pt_12=-8.092 tm_pt_13=0.000 tm_pt_14=-11.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=12.000 WordPenalty=-6.514 OOVPenalty=-200.000 ||| -245.063
+4 ||| on the same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match this novel from . ||| lm_0=-42.228 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-27.311 tm_pt_6=-10.413 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-3.145 tm_pt_10=-14.340 tm_pt_11=0.000 tm_pt_12=-10.289 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=12.000 WordPenalty=-6.949 OOVPenalty=-200.000 ||| -245.074
+4 ||| in the same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 presage match this novel from . ||| lm_0=-42.347 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-26.214 tm_pt_6=-10.335 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-3.016 tm_pt_10=-14.659 tm_pt_11=0.000 tm_pt_12=-9.596 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=12.000 WordPenalty=-6.949 OOVPenalty=-200.000 ||| -245.208
+4 ||| in the same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match in this novel from . ||| lm_0=-42.902 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-30.968 tm_pt_6=-10.877 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-3.016 tm_pt_10=-14.751 tm_pt_11=0.000 tm_pt_12=-11.819 tm_pt_13=0.000 tm_pt_14=-13.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=12.000 WordPenalty=-7.383 OOVPenalty=-200.000 ||| -245.411
+4 ||| in the same time with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match from this novel . ||| lm_0=-40.764 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-31.896 tm_pt_6=-10.591 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-4.059 tm_pt_10=-13.903 tm_pt_11=0.000 tm_pt_12=-10.600 tm_pt_13=0.000 tm_pt_14=-13.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 WordPenalty=-7.383 OOVPenalty=-200.000 ||| -245.823
+4 ||| in the same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match this novel to . ||| lm_0=-41.812 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-29.274 tm_pt_6=-11.214 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-3.016 tm_pt_10=-16.472 tm_pt_11=0.000 tm_pt_12=-10.925 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=12.000 WordPenalty=-6.949 OOVPenalty=-200.000 ||| -245.993
+4 ||| in the same time with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match this novel from . ||| lm_0=-42.950 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-31.896 tm_pt_6=-10.591 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-4.009 tm_pt_10=-14.053 tm_pt_11=0.000 tm_pt_12=-11.388 tm_pt_13=0.000 tm_pt_14=-13.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=12.000 WordPenalty=-7.383 OOVPenalty=-200.000 ||| -246.000
+4 ||| in the same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match this novel from the . ||| lm_0=-42.855 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-29.780 tm_pt_6=-10.335 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-3.016 tm_pt_10=-15.286 tm_pt_11=0.000 tm_pt_12=-12.852 tm_pt_13=0.000 tm_pt_14=-13.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=12.000 WordPenalty=-7.383 OOVPenalty=-200.000 ||| -246.051
+5 ||| \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 majority in the . ||| lm_0=-25.263 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-16.310 tm_pt_6=-6.695 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-0.154 tm_pt_10=-12.239 tm_pt_11=0.000 tm_pt_12=-3.873 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 WordPenalty=-5.212 OOVPenalty=-200.000 ||| -222.050
+5 ||| \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 majority in . ||| lm_0=-25.124 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-13.102 tm_pt_6=-8.482 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-0.154 tm_pt_10=-10.478 tm_pt_11=-1.000 tm_pt_12=-3.480 tm_pt_13=0.000 tm_pt_14=-7.000 tm_pt_15=-1.000 tm_pt_16=0.000 tm_glue_0=9.000 WordPenalty=-4.777 OOVPenalty=-200.000 ||| -223.836
+5 ||| \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 majority , in the . ||| lm_0=-27.633 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-25.259 tm_pt_6=-7.101 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-1.135 tm_pt_10=-9.702 tm_pt_11=0.000 tm_pt_12=-5.483 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 WordPenalty=-5.646 OOVPenalty=-200.000 ||| -224.380
+5 ||| \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 majority in the goods . ||| lm_0=-27.555 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-18.420 tm_pt_6=-7.378 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-1.154 tm_pt_10=-9.471 tm_pt_11=0.000 tm_pt_12=-7.759 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=9.000 WordPenalty=-5.646 OOVPenalty=-200.000 ||| -224.789
+5 ||| \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 seikh mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 majority in the . ||| lm_0=-25.455 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-20.559 tm_pt_6=-7.381 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-1.154 tm_pt_10=-15.043 tm_pt_11=0.000 tm_pt_12=-8.511 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=9.000 WordPenalty=-5.646 OOVPenalty=-200.000 ||| -224.989
+5 ||| \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 majority in the . . ||| lm_0=-28.222 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-16.376 tm_pt_6=-6.695 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-0.522 tm_pt_10=-12.220 tm_pt_11=0.000 tm_pt_12=-12.911 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 WordPenalty=-5.646 OOVPenalty=-200.000 ||| -224.991
+5 ||| \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 majority in the of . ||| lm_0=-27.442 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-16.932 tm_pt_6=-6.695 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-1.154 tm_pt_10=-12.220 tm_pt_11=0.000 tm_pt_12=-13.604 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=10.000 WordPenalty=-5.646 OOVPenalty=-200.000 ||| -225.383
+5 ||| \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 majority in the , . ||| lm_0=-27.311 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-21.279 tm_pt_6=-6.695 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-1.154 tm_pt_10=-12.220 tm_pt_11=0.000 tm_pt_12=-13.604 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=10.000 WordPenalty=-5.646 OOVPenalty=-200.000 ||| -225.396
+6 ||| \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his work with to that . ||| lm_0=-33.425 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-6.000 tm_pt_5=-22.509 tm_pt_6=-11.163 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.370 tm_pt_10=-19.369 tm_pt_11=0.000 tm_pt_12=-3.590 tm_pt_13=0.000 tm_pt_14=-6.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 WordPenalty=-5.212 OOVPenalty=-400.000 ||| -439.395
+6 ||| \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his work with can and pajama . ||| lm_0=-38.085 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-6.000 tm_pt_5=-21.424 tm_pt_6=-9.427 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.503 tm_pt_10=-11.370 tm_pt_11=0.000 tm_pt_12=-4.438 tm_pt_13=0.000 tm_pt_14=-7.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 WordPenalty=-5.646 OOVPenalty=-400.000 ||| -439.526
+6 ||| \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his work with to and pajama . ||| lm_0=-37.294 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-6.000 tm_pt_5=-24.691 tm_pt_6=-10.758 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.370 tm_pt_10=-13.152 tm_pt_11=0.000 tm_pt_12=-3.590 tm_pt_13=0.000 tm_pt_14=-7.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 WordPenalty=-5.646 OOVPenalty=-400.000 ||| -440.155
+6 ||| \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his work with can that . ||| lm_0=-35.292 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-6.000 tm_pt_5=-19.242 tm_pt_6=-9.832 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.503 tm_pt_10=-17.586 tm_pt_11=0.000 tm_pt_12=-4.438 tm_pt_13=0.000 tm_pt_14=-6.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 WordPenalty=-5.212 OOVPenalty=-400.000 ||| -440.188
+6 ||| \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 of his work with to that . ||| lm_0=-34.173 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-6.000 tm_pt_5=-23.132 tm_pt_6=-11.163 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.370 tm_pt_10=-19.621 tm_pt_11=0.000 tm_pt_12=-6.145 tm_pt_13=0.000 tm_pt_14=-7.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=10.000 WordPenalty=-5.646 OOVPenalty=-400.000 ||| -440.260
+6 ||| \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his works with to that . ||| lm_0=-33.694 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-6.000 tm_pt_5=-22.839 tm_pt_6=-12.090 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.371 tm_pt_10=-19.532 tm_pt_11=0.000 tm_pt_12=-5.154 tm_pt_13=0.000 tm_pt_14=-6.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 WordPenalty=-5.212 OOVPenalty=-400.000 ||| -440.280
+6 ||| \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 in his work with to that . ||| lm_0=-34.398 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-6.000 tm_pt_5=-28.289 tm_pt_6=-11.756 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.420 tm_pt_10=-19.730 tm_pt_11=0.000 tm_pt_12=-7.467 tm_pt_13=0.000 tm_pt_14=-7.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 WordPenalty=-5.646 OOVPenalty=-400.000 ||| -440.347
+6 ||| \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 of his work with can and pajama . ||| lm_0=-38.833 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-6.000 tm_pt_5=-22.046 tm_pt_6=-9.427 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.503 tm_pt_10=-11.622 tm_pt_11=0.000 tm_pt_12=-6.992 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=10.000 WordPenalty=-6.080 OOVPenalty=-400.000 ||| -440.391
+6 ||| \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his works with can and pajama . ||| lm_0=-38.354 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-6.000 tm_pt_5=-21.754 tm_pt_6=-10.353 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.504 tm_pt_10=-11.532 tm_pt_11=0.000 tm_pt_12=-6.002 tm_pt_13=0.000 tm_pt_14=-7.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 WordPenalty=-5.646 OOVPenalty=-400.000 ||| -440.410
+6 ||| \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 in his work with can and pajama . ||| lm_0=-39.058 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-6.000 tm_pt_5=-27.204 tm_pt_6=-10.020 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.553 tm_pt_10=-11.730 tm_pt_11=0.000 tm_pt_12=-8.314 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 WordPenalty=-6.080 OOVPenalty=-400.000 ||| -440.477
+7 ||| task apart from \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character is , but these are very is not . ||| lm_0=-50.059 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-14.000 tm_pt_5=-44.149 tm_pt_6=-18.359 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-1.657 tm_pt_10=-18.913 tm_pt_11=0.000 tm_pt_12=-11.895 tm_pt_13=0.000 tm_pt_14=-17.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=16.000 WordPenalty=-9.120 OOVPenalty=-200.000 ||| -252.878
+7 ||| task apart from \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character there are , but these are very is not . ||| lm_0=-52.396 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-14.000 tm_pt_5=-44.608 tm_pt_6=-18.011 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-1.657 tm_pt_10=-15.346 tm_pt_11=0.000 tm_pt_12=-10.691 tm_pt_13=0.000 tm_pt_14=-18.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=16.000 WordPenalty=-9.554 OOVPenalty=-200.000 ||| -252.999
+7 ||| task other than \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character is , but these are very is not . ||| lm_0=-50.379 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-14.000 tm_pt_5=-44.596 tm_pt_6=-18.428 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-1.540 tm_pt_10=-18.653 tm_pt_11=0.000 tm_pt_12=-11.384 tm_pt_13=0.000 tm_pt_14=-17.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=16.000 WordPenalty=-9.120 OOVPenalty=-200.000 ||| -253.123
+7 ||| task apart from \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character are , but these are very is not . ||| lm_0=-50.940 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-14.000 tm_pt_5=-43.071 tm_pt_6=-18.105 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-1.657 tm_pt_10=-17.633 tm_pt_11=0.000 tm_pt_12=-11.512 tm_pt_13=0.000 tm_pt_14=-17.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=16.000 WordPenalty=-9.120 OOVPenalty=-200.000 ||| -253.207
+7 ||| task other than \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character there are , but these are very is not . ||| lm_0=-52.715 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-14.000 tm_pt_5=-45.055 tm_pt_6=-18.080 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-1.540 tm_pt_10=-15.087 tm_pt_11=0.000 tm_pt_12=-10.180 tm_pt_13=0.000 tm_pt_14=-18.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=16.000 WordPenalty=-9.554 OOVPenalty=-200.000 ||| -253.244
+7 ||| task apart from \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character there is , but these are very is not . ||| lm_0=-52.594 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-14.000 tm_pt_5=-45.686 tm_pt_6=-18.119 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-1.657 tm_pt_10=-15.360 tm_pt_11=0.000 tm_pt_12=-11.202 tm_pt_13=0.000 tm_pt_14=-18.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=16.000 WordPenalty=-9.554 OOVPenalty=-200.000 ||| -253.351
+7 ||| task apart from \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some of linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character is , but these are very is not . ||| lm_0=-50.739 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-14.000 tm_pt_5=-44.772 tm_pt_6=-18.359 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-1.657 tm_pt_10=-18.877 tm_pt_11=0.000 tm_pt_12=-13.825 tm_pt_13=0.000 tm_pt_14=-18.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=16.000 WordPenalty=-9.554 OOVPenalty=-200.000 ||| -253.526
+7 ||| task apart from \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some of linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character there is , but these are very is not . ||| lm_0=-53.273 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-14.000 tm_pt_5=-46.308 tm_pt_6=-18.119 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-1.657 tm_pt_10=-15.324 tm_pt_11=0.000 tm_pt_12=-13.132 tm_pt_13=0.000 tm_pt_14=-19.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=16.000 WordPenalty=-9.989 OOVPenalty=-200.000 ||| -253.999
+7 ||| task apart from \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character is , but these are very common not . ||| lm_0=-51.322 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-15.000 tm_pt_5=-41.437 tm_pt_6=-19.786 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-0.707 tm_pt_10=-21.351 tm_pt_11=0.000 tm_pt_12=-13.570 tm_pt_13=0.000 tm_pt_14=-17.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=17.000 WordPenalty=-9.120 OOVPenalty=-200.000 ||| -254.604
+7 ||| task apart from \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character there are , but these are very common not . ||| lm_0=-53.658 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-15.000 tm_pt_5=-41.895 tm_pt_6=-19.438 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-0.707 tm_pt_10=-17.785 tm_pt_11=0.000 tm_pt_12=-12.366 tm_pt_13=0.000 tm_pt_14=-18.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=17.000 WordPenalty=-9.554 OOVPenalty=-200.000 ||| -254.725
+8 ||| it social situation in \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the vagina . ||| lm_0=-44.455 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-51.640 tm_pt_6=-26.630 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-4.308 tm_pt_10=-35.488 tm_pt_11=0.000 tm_pt_12=-13.247 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=15.000 WordPenalty=-8.252 OOVPenalty=-200.000 ||| -264.205
+8 ||| it social situation in \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with the \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the vagina . ||| lm_0=-45.310 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-59.209 tm_pt_6=-26.448 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-5.173 tm_pt_10=-33.652 tm_pt_11=0.000 tm_pt_12=-14.345 tm_pt_13=0.000 tm_pt_14=-16.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=15.000 WordPenalty=-8.686 OOVPenalty=-200.000 ||| -264.435
+8 ||| it social situation in view \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the vagina . ||| lm_0=-47.886 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-55.479 tm_pt_6=-27.218 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-4.308 tm_pt_10=-28.297 tm_pt_11=0.000 tm_pt_12=-13.247 tm_pt_13=0.000 tm_pt_14=-16.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=15.000 WordPenalty=-8.686 OOVPenalty=-200.000 ||| -264.473
+8 ||| it social situation in view of \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the vagina . ||| lm_0=-48.077 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-56.102 tm_pt_6=-27.218 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-4.308 tm_pt_10=-28.297 tm_pt_11=0.000 tm_pt_12=-13.247 tm_pt_13=0.000 tm_pt_14=-17.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=15.000 WordPenalty=-9.120 OOVPenalty=-200.000 ||| -264.567
+8 ||| it social situation in view \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with the \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the vagina . ||| lm_0=-48.741 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-63.049 tm_pt_6=-27.036 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-5.173 tm_pt_10=-26.461 tm_pt_11=0.000 tm_pt_12=-14.345 tm_pt_13=0.000 tm_pt_14=-17.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=15.000 WordPenalty=-9.120 OOVPenalty=-200.000 ||| -264.703
+8 ||| this social situation in \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the vagina . ||| lm_0=-43.694 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-53.233 tm_pt_6=-27.577 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-4.308 tm_pt_10=-37.424 tm_pt_11=0.000 tm_pt_12=-13.879 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=15.000 WordPenalty=-8.252 OOVPenalty=-200.000 ||| -264.755
+8 ||| it social situation in view of \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with the \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the vagina . ||| lm_0=-48.932 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-63.671 tm_pt_6=-27.036 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-5.173 tm_pt_10=-26.461 tm_pt_11=0.000 tm_pt_12=-14.345 tm_pt_13=0.000 tm_pt_14=-18.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=15.000 WordPenalty=-9.554 OOVPenalty=-200.000 ||| -264.796
+8 ||| it social situation in \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the in the . ||| lm_0=-46.908 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-14.000 tm_pt_5=-58.491 tm_pt_6=-22.780 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-3.444 tm_pt_10=-39.058 tm_pt_11=0.000 tm_pt_12=-13.356 tm_pt_13=0.000 tm_pt_14=-16.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=16.000 WordPenalty=-8.686 OOVPenalty=-200.000 ||| -264.875
+8 ||| this social situation in \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with the \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rule of the vagina . ||| lm_0=-44.549 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-60.803 tm_pt_6=-27.394 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-5.173 tm_pt_10=-35.588 tm_pt_11=0.000 tm_pt_12=-14.978 tm_pt_13=0.000 tm_pt_14=-16.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=15.000 WordPenalty=-8.686 OOVPenalty=-200.000 ||| -264.985
+8 ||| it social situation in \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 his oppositions with \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making the rules of the vagina . ||| lm_0=-45.017 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-51.041 tm_pt_6=-27.218 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-4.425 tm_pt_10=-34.836 tm_pt_11=0.000 tm_pt_12=-13.757 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=15.000 WordPenalty=-8.252 OOVPenalty=-200.000 ||| -264.988
+9 ||| \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 annual \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf meter ||| lm_0=-26.316 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-3.885 tm_pt_6=-2.821 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.368 tm_pt_10=-2.140 tm_pt_11=0.000 tm_pt_12=-1.337 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 WordPenalty=-3.040 OOVPenalty=-300.000 ||| -324.551
+9 ||| \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 arrange \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf meter ||| lm_0=-26.509 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-3.430 tm_pt_6=-3.514 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-1.000 tm_pt_10=0.000 tm_pt_11=0.000 tm_pt_12=-2.030 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 WordPenalty=-3.040 OOVPenalty=-300.000 ||| -324.635
+9 ||| \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 annual \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf meters ||| lm_0=-26.340 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-3.372 tm_pt_6=-3.055 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.368 tm_pt_10=-2.214 tm_pt_11=0.000 tm_pt_12=-1.337 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 WordPenalty=-3.040 OOVPenalty=-300.000 ||| -324.722
+9 ||| \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 arrange \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf meters ||| lm_0=-26.532 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-2.916 tm_pt_6=-3.748 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-1.000 tm_pt_10=-0.074 tm_pt_11=0.000 tm_pt_12=-2.030 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 WordPenalty=-3.040 OOVPenalty=-300.000 ||| -324.806
+9 ||| \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 annual \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf metres ||| lm_0=-26.771 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-3.608 tm_pt_6=-4.389 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.503 tm_pt_10=-2.140 tm_pt_11=0.000 tm_pt_12=-2.803 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 WordPenalty=-3.040 OOVPenalty=-300.000 ||| -326.046
+9 ||| \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 arrange \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf metres ||| lm_0=-26.963 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-3.153 tm_pt_6=-5.083 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-1.135 tm_pt_10=0.000 tm_pt_11=0.000 tm_pt_12=-3.497 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 WordPenalty=-3.040 OOVPenalty=-300.000 ||| -326.131
+9 ||| \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 annual \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf above ||| lm_0=-26.244 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-6.156 tm_pt_6=-5.306 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-1.368 tm_pt_10=-2.833 tm_pt_11=0.000 tm_pt_12=-3.902 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 WordPenalty=-3.040 OOVPenalty=-300.000 ||| -327.121
+9 ||| \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 arrange \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf above ||| lm_0=-26.437 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-5.700 tm_pt_6=-5.999 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-2.000 tm_pt_10=-0.693 tm_pt_11=0.000 tm_pt_12=-4.595 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 WordPenalty=-3.040 OOVPenalty=-300.000 ||| -327.206
+10 ||| in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in the city in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f as the national conference was he main speech -lrb- keynote speech -rrb- on the . ||| lm_0=-71.410 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-14.000 tm_pt_5=-59.495 tm_pt_6=-17.371 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-3.418 tm_pt_10=-23.386 tm_pt_11=0.000 tm_pt_12=-12.427 tm_pt_13=0.000 tm_pt_14=-19.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=20.000 WordPenalty=-11.726 OOVPenalty=-600.000 ||| -675.013
+10 ||| in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in the city in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f as the national conference was he main speech -lrb- keynote speech -rrb- to the . ||| lm_0=-71.132 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-14.000 tm_pt_5=-60.573 tm_pt_6=-17.525 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-3.387 tm_pt_10=-24.365 tm_pt_11=0.000 tm_pt_12=-12.204 tm_pt_13=0.000 tm_pt_14=-19.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=20.000 WordPenalty=-11.726 OOVPenalty=-600.000 ||| -675.262
+10 ||| in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in the city in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f the national conference was he main speech -lrb- keynote speech -rrb- on the . ||| lm_0=-70.322 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-14.000 tm_pt_5=-53.887 tm_pt_6=-16.887 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-2.553 tm_pt_10=-27.280 tm_pt_11=0.000 tm_pt_12=-11.329 tm_pt_13=0.000 tm_pt_14=-18.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=20.000 WordPenalty=-11.292 OOVPenalty=-600.000 ||| -675.282
+10 ||| in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in the city in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f the national conference was he main speech -lrb- keynote speech -rrb- to the . ||| lm_0=-70.045 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-14.000 tm_pt_5=-54.965 tm_pt_6=-17.041 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-2.522 tm_pt_10=-28.259 tm_pt_11=0.000 tm_pt_12=-11.105 tm_pt_13=0.000 tm_pt_14=-18.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=20.000 WordPenalty=-11.292 OOVPenalty=-600.000 ||| -675.531
+10 ||| in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in the city in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f as the national conference was he the speech -lrb- keynote speech -rrb- on the . ||| lm_0=-69.753 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-14.000 tm_pt_5=-63.941 tm_pt_6=-18.119 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-3.418 tm_pt_10=-28.046 tm_pt_11=0.000 tm_pt_12=-14.412 tm_pt_13=0.000 tm_pt_14=-19.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=20.000 WordPenalty=-11.726 OOVPenalty=-600.000 ||| -675.766
+10 ||| in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in the city in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f as the national conference was he main speech -lrb- keynote speech -rrb- , the . ||| lm_0=-70.279 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-14.000 tm_pt_5=-61.705 tm_pt_6=-17.066 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-3.736 tm_pt_10=-27.330 tm_pt_11=0.000 tm_pt_12=-13.120 tm_pt_13=0.000 tm_pt_14=-19.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=20.000 WordPenalty=-11.726 OOVPenalty=-600.000 ||| -675.800
+10 ||| in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in the city in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f party national conference was he main speech -lrb- keynote speech -rrb- on the . ||| lm_0=-72.890 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-14.000 tm_pt_5=-49.226 tm_pt_6=-17.122 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-2.418 tm_pt_10=-22.222 tm_pt_11=0.000 tm_pt_12=-10.230 tm_pt_13=0.000 tm_pt_14=-18.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=20.000 WordPenalty=-11.292 OOVPenalty=-600.000 ||| -675.864
+10 ||| in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in the city in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f as the national conference was he the main speech -lrb- keynote speech -rrb- on the . ||| lm_0=-72.269 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-14.000 tm_pt_5=-65.118 tm_pt_6=-17.677 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-3.418 tm_pt_10=-24.223 tm_pt_11=0.000 tm_pt_12=-14.044 tm_pt_13=0.000 tm_pt_14=-20.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=20.000 WordPenalty=-12.160 OOVPenalty=-600.000 ||| -675.940
+10 ||| in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in the city in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f as the national conference was he the speech -lrb- keynote speech -rrb- to the . ||| lm_0=-69.475 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-14.000 tm_pt_5=-65.019 tm_pt_6=-18.273 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-3.387 tm_pt_10=-29.024 tm_pt_11=0.000 tm_pt_12=-14.189 tm_pt_13=0.000 tm_pt_14=-19.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=20.000 WordPenalty=-11.726 OOVPenalty=-600.000 ||| -676.015
+10 ||| in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in the city in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f the national conference was he the speech -lrb- keynote speech -rrb- on the . ||| lm_0=-68.665 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-14.000 tm_pt_5=-58.333 tm_pt_6=-17.635 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-2.553 tm_pt_10=-31.939 tm_pt_11=0.000 tm_pt_12=-13.313 tm_pt_13=0.000 tm_pt_14=-18.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=20.000 WordPenalty=-11.292 OOVPenalty=-600.000 ||| -676.035
+11 ||| population on the power distribution of east pakistan where due to the west pakistan " one unit theory , " is a \u0985\u09ad\u09bf\u09a8\u09ac they started to the , where the whole of west pakistan is a province considered as the . ||| lm_0=-81.864 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-29.000 tm_pt_5=-100.677 tm_pt_6=-42.051 tm_pt_7=-29.000 tm_pt_8=-78.822 tm_pt_9=-6.557 tm_pt_10=-50.659 tm_pt_11=0.000 tm_pt_12=-36.155 tm_pt_13=0.000 tm_pt_14=-40.000 tm_pt_15=0.000 tm_pt_16=-3.000 tm_glue_0=30.000 WordPenalty=-18.675 OOVPenalty=-100.000 ||| -199.719
+11 ||| population on the power distribution of east pakistan where due to the west pakistan " one unit theory , " is a \u0985\u09ad\u09bf\u09a8\u09ac they started to the , where the whole of west pakistan a province considered as the . ||| lm_0=-81.398 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-29.000 tm_pt_5=-96.978 tm_pt_6=-41.482 tm_pt_7=-29.000 tm_pt_8=-78.822 tm_pt_9=-6.557 tm_pt_10=-50.672 tm_pt_11=0.000 tm_pt_12=-33.294 tm_pt_13=0.000 tm_pt_14=-39.000 tm_pt_15=0.000 tm_pt_16=-3.000 tm_glue_0=30.000 WordPenalty=-18.240 OOVPenalty=-100.000 ||| -199.743
+11 ||| population on the power distribution of east pakistan where due to the west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac they started to the , where the whole of west pakistan is a province considered as the . ||| lm_0=-80.992 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-29.000 tm_pt_5=-99.267 tm_pt_6=-42.695 tm_pt_7=-29.000 tm_pt_8=-78.822 tm_pt_9=-6.556 tm_pt_10=-50.674 tm_pt_11=0.000 tm_pt_12=-32.501 tm_pt_13=0.000 tm_pt_14=-39.000 tm_pt_15=0.000 tm_pt_16=-3.000 tm_glue_0=30.000 WordPenalty=-18.240 OOVPenalty=-100.000 ||| -199.975
+11 ||| population on the power distribution of east pakistan where due to the west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac they started to the , where the whole of west pakistan a province considered as the . ||| lm_0=-80.526 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-29.000 tm_pt_5=-95.568 tm_pt_6=-42.126 tm_pt_7=-29.000 tm_pt_8=-78.822 tm_pt_9=-6.556 tm_pt_10=-50.687 tm_pt_11=0.000 tm_pt_12=-29.640 tm_pt_13=0.000 tm_pt_14=-38.000 tm_pt_15=0.000 tm_pt_16=-3.000 tm_glue_0=30.000 WordPenalty=-17.806 OOVPenalty=-100.000 ||| -199.999
+11 ||| population on the power distribution of east pakistan where due to the west pakistan " one unit theory , " is a \u0985\u09ad\u09bf\u09a8\u09ac they started to the , where the west pakistan is a province considered as the . ||| lm_0=-79.067 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-29.000 tm_pt_5=-98.330 tm_pt_6=-41.804 tm_pt_7=-29.000 tm_pt_8=-78.822 tm_pt_9=-6.440 tm_pt_10=-57.013 tm_pt_11=0.000 tm_pt_12=-35.644 tm_pt_13=0.000 tm_pt_14=-38.000 tm_pt_15=0.000 tm_pt_16=-2.000 tm_glue_0=30.000 WordPenalty=-17.806 OOVPenalty=-100.000 ||| -200.182
+11 ||| population on the power distribution of east pakistan where due to the west pakistan " one unit theory , " is a \u0985\u09ad\u09bf\u09a8\u09ac they started to the , where the west pakistan a province considered as the . ||| lm_0=-78.601 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-29.000 tm_pt_5=-94.631 tm_pt_6=-41.234 tm_pt_7=-29.000 tm_pt_8=-78.822 tm_pt_9=-6.440 tm_pt_10=-57.025 tm_pt_11=0.000 tm_pt_12=-32.783 tm_pt_13=0.000 tm_pt_14=-37.000 tm_pt_15=0.000 tm_pt_16=-2.000 tm_glue_0=30.000 WordPenalty=-17.372 OOVPenalty=-100.000 ||| -200.206
+11 ||| population based on this distribution of east pakistan where due to the west pakistan " one unit theory , " is a \u0985\u09ad\u09bf\u09a8\u09ac they started to the , where the west pakistan is a province considered as the . ||| lm_0=-78.658 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-29.000 tm_pt_5=-98.009 tm_pt_6=-43.423 tm_pt_7=-29.000 tm_pt_8=-78.822 tm_pt_9=-5.575 tm_pt_10=-58.147 tm_pt_11=0.000 tm_pt_12=-34.545 tm_pt_13=0.000 tm_pt_14=-38.000 tm_pt_15=0.000 tm_pt_16=-2.000 tm_glue_0=30.000 WordPenalty=-17.806 OOVPenalty=-100.000 ||| -200.259
+11 ||| population on the power distribution of east pakistan where due to the west pakistan , " one unit theory , " is a \u0985\u09ad\u09bf\u09a8\u09ac they started to the , where the whole of west pakistan is a province considered as the . ||| lm_0=-83.355 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-29.000 tm_pt_5=-102.086 tm_pt_6=-41.407 tm_pt_7=-29.000 tm_pt_8=-78.822 tm_pt_9=-6.557 tm_pt_10=-50.645 tm_pt_11=0.000 tm_pt_12=-39.809 tm_pt_13=0.000 tm_pt_14=-41.000 tm_pt_15=0.000 tm_pt_16=-3.000 tm_glue_0=30.000 WordPenalty=-19.109 OOVPenalty=-100.000 ||| -200.280
+11 ||| population based on this distribution of east pakistan where due to the west pakistan " one unit theory , " is a \u0985\u09ad\u09bf\u09a8\u09ac they started to the , where the west pakistan a province considered as the . ||| lm_0=-78.192 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-29.000 tm_pt_5=-94.310 tm_pt_6=-42.853 tm_pt_7=-29.000 tm_pt_8=-78.822 tm_pt_9=-5.575 tm_pt_10=-58.159 tm_pt_11=0.000 tm_pt_12=-31.684 tm_pt_13=0.000 tm_pt_14=-37.000 tm_pt_15=0.000 tm_pt_16=-2.000 tm_glue_0=30.000 WordPenalty=-17.372 OOVPenalty=-100.000 ||| -200.283
+11 ||| population on the power distribution of east pakistan where due to the west pakistan , " one unit theory , " is a \u0985\u09ad\u09bf\u09a8\u09ac they started to the , where the whole of west pakistan a province considered as the . ||| lm_0=-82.889 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-29.000 tm_pt_5=-98.387 tm_pt_6=-40.838 tm_pt_7=-29.000 tm_pt_8=-78.822 tm_pt_9=-6.557 tm_pt_10=-50.657 tm_pt_11=0.000 tm_pt_12=-36.948 tm_pt_13=0.000 tm_pt_14=-40.000 tm_pt_15=0.000 tm_pt_16=-3.000 tm_glue_0=30.000 WordPenalty=-18.675 OOVPenalty=-100.000 ||| -200.304
+12 ||| the \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 theory ||| lm_0=-11.112 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-9.309 tm_pt_6=-3.988 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.368 tm_pt_10=-7.989 tm_pt_11=0.000 tm_pt_12=-1.463 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=3.000 WordPenalty=-2.171 OOVPenalty=-100.000 ||| -113.022
+12 ||| mathematical \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 theory ||| lm_0=-12.665 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-2.869 tm_pt_6=-2.890 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-1.000 tm_pt_10=-5.219 tm_pt_11=0.000 tm_pt_12=-2.156 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=3.000 WordPenalty=-2.171 OOVPenalty=-100.000 ||| -113.373
+12 ||| \u2022 \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 theory ||| lm_0=-14.217 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-1.046 tm_pt_6=-5.241 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.368 tm_pt_10=-0.432 tm_pt_11=0.000 tm_pt_12=-1.463 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=3.000 WordPenalty=-2.171 OOVPenalty=-100.000 ||| -113.486
+12 ||| the \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 theory of ||| lm_0=-12.879 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-9.931 tm_pt_6=-3.988 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.368 tm_pt_10=-8.250 tm_pt_11=0.000 tm_pt_12=-3.247 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=3.000 WordPenalty=-2.606 OOVPenalty=-100.000 ||| -115.266
+12 ||| the theory of \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 ||| lm_0=-10.781 tm_pt_0=0.000 tm_pt_1=-1.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-9.931 tm_pt_6=-3.988 tm_pt_7=-3.000 tm_pt_8=-8.154 tm_pt_9=-1.368 tm_pt_10=-7.989 tm_pt_11=0.000 tm_pt_12=-3.409 tm_pt_13=-1.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=2.000 WordPenalty=-2.606 OOVPenalty=-100.000 ||| -115.559
+12 ||| mathematical \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 theory of ||| lm_0=-14.432 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-3.491 tm_pt_6=-2.890 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-1.000 tm_pt_10=-5.481 tm_pt_11=0.000 tm_pt_12=-3.940 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=3.000 WordPenalty=-2.606 OOVPenalty=-100.000 ||| -115.616
+12 ||| \u2022 \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 theory of ||| lm_0=-15.984 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-1.668 tm_pt_6=-5.241 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.368 tm_pt_10=-0.693 tm_pt_11=0.000 tm_pt_12=-3.247 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=3.000 WordPenalty=-2.606 OOVPenalty=-100.000 ||| -115.729
+12 ||| the \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 is ||| lm_0=-10.867 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-15.175 tm_pt_6=-7.086 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.386 tm_pt_10=-12.885 tm_pt_11=0.000 tm_pt_12=-4.773 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=3.000 WordPenalty=-2.171 OOVPenalty=-100.000 ||| -117.032
+13 ||| external links of ||| lm_0=-6.986 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-7.390 tm_pt_6=-2.729 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.018 tm_pt_10=-6.022 tm_pt_11=0.000 tm_pt_12=-2.314 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 WordPenalty=-2.171 ||| -6.764
+13 ||| relation with outside of ||| lm_0=-6.991 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-15.716 tm_pt_6=-4.559 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.018 tm_pt_10=-6.012 tm_pt_11=0.000 tm_pt_12=-3.982 tm_pt_13=0.000 tm_pt_14=-4.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 WordPenalty=-2.606 ||| -7.040
+13 ||| external link of ||| lm_0=-7.533 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-8.091 tm_pt_6=-2.871 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.018 tm_pt_10=-6.012 tm_pt_11=0.000 tm_pt_12=-3.451 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 WordPenalty=-2.171 ||| -7.543
+13 ||| external communication of ||| lm_0=-7.692 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-8.265 tm_pt_6=-2.886 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.018 tm_pt_10=-6.012 tm_pt_11=0.000 tm_pt_12=-3.240 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 WordPenalty=-2.171 ||| -7.776
+13 ||| external connections of ||| lm_0=-7.931 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-7.934 tm_pt_6=-3.100 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.037 tm_pt_10=-5.907 tm_pt_11=0.000 tm_pt_12=-4.675 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 WordPenalty=-2.171 ||| -8.099
+13 ||| external connection of ||| lm_0=-8.109 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-8.551 tm_pt_6=-3.029 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.037 tm_pt_10=-6.012 tm_pt_11=0.000 tm_pt_12=-4.675 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 WordPenalty=-2.171 ||| -8.379
+13 ||| external link by ||| lm_0=-7.855 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-7.019 tm_pt_6=-4.024 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.368 tm_pt_10=-5.545 tm_pt_11=0.000 tm_pt_12=-4.367 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 WordPenalty=-2.171 ||| -8.600
+13 ||| external communication by ||| lm_0=-8.086 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-7.193 tm_pt_6=-4.039 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.368 tm_pt_10=-5.545 tm_pt_11=0.000 tm_pt_12=-4.156 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 WordPenalty=-2.171 ||| -8.928
+13 ||| external links by ||| lm_0=-8.277 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-6.318 tm_pt_6=-3.881 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.368 tm_pt_10=-5.555 tm_pt_11=0.000 tm_pt_12=-3.230 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 WordPenalty=-2.171 ||| -9.099
+13 ||| external connections by ||| lm_0=-8.348 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-6.862 tm_pt_6=-4.252 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.386 tm_pt_10=-5.440 tm_pt_11=0.000 tm_pt_12=-5.591 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 WordPenalty=-2.171 ||| -9.281
+14 ||| tata communication foreign this \u09a8\u09bf\u0997\u09ae limited building , it is the telecommunication system is a is one of the main providers ||| lm_0=-51.724 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-15.000 tm_pt_5=-58.994 tm_pt_6=-16.921 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-3.506 tm_pt_10=-27.630 tm_pt_11=0.000 tm_pt_12=-16.984 tm_pt_13=0.000 tm_pt_14=-20.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=16.000 WordPenalty=-9.989 OOVPenalty=-100.000 ||| -159.669
+14 ||| tata communicationer foreign this \u09a8\u09bf\u0997\u09ae limited building , it is the telecommunication system is a is one of the main providers ||| lm_0=-52.978 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-15.000 tm_pt_5=-54.440 tm_pt_6=-16.921 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-3.506 tm_pt_10=-25.065 tm_pt_11=0.000 tm_pt_12=-16.984 tm_pt_13=0.000 tm_pt_14=-20.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=16.000 WordPenalty=-9.989 OOVPenalty=-100.000 ||| -159.792
+14 ||| tata communication foreign this \u09a8\u09bf\u0997\u09ae limited building , it is the city telecommunication system is a is one of the main providers ||| lm_0=-54.930 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-15.000 tm_pt_5=-60.344 tm_pt_6=-17.068 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-3.506 tm_pt_10=-22.909 tm_pt_11=0.000 tm_pt_12=-17.294 tm_pt_13=0.000 tm_pt_14=-21.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=16.000 WordPenalty=-10.423 OOVPenalty=-100.000 ||| -160.578
+14 ||| tata communication foreign this \u09a8\u09bf\u0997\u09ae limited building , it is in the telecommunication system is a is one of the main providers ||| lm_0=-53.234 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-15.000 tm_pt_5=-64.715 tm_pt_6=-17.464 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-3.642 tm_pt_10=-26.331 tm_pt_11=0.000 tm_pt_12=-18.593 tm_pt_13=0.000 tm_pt_14=-21.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=16.000 WordPenalty=-10.423 OOVPenalty=-100.000 ||| -160.606
+14 ||| tata communicationer foreign this \u09a8\u09bf\u0997\u09ae limited building , it is the city telecommunication system is a is one of the main providers ||| lm_0=-56.184 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-15.000 tm_pt_5=-55.791 tm_pt_6=-17.068 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-3.506 tm_pt_10=-20.344 tm_pt_11=0.000 tm_pt_12=-17.294 tm_pt_13=0.000 tm_pt_14=-21.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=16.000 WordPenalty=-10.423 OOVPenalty=-100.000 ||| -160.701
+14 ||| tata communication foreign this \u09a8\u09bf\u0997\u09ae limited building , it is the telecommunication system is a one of the main providers ||| lm_0=-52.146 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-15.000 tm_pt_5=-52.908 tm_pt_6=-16.672 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-3.371 tm_pt_10=-28.190 tm_pt_11=0.000 tm_pt_12=-15.598 tm_pt_13=0.000 tm_pt_14=-19.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=16.000 WordPenalty=-9.554 OOVPenalty=-100.000 ||| -161.029
+14 ||| tata communication foreign this \u09a8\u09bf\u0997\u09ae limited building , it is in the city telecommunication system is a is one of the main providers ||| lm_0=-55.705 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-15.000 tm_pt_5=-66.065 tm_pt_6=-17.383 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-3.556 tm_pt_10=-22.899 tm_pt_11=0.000 tm_pt_12=-18.306 tm_pt_13=0.000 tm_pt_14=-22.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=16.000 WordPenalty=-10.857 OOVPenalty=-100.000 ||| -161.034
+14 ||| tata communication foreign this \u09a8\u09bf\u0997\u09ae limited building , it is the telecommunication system of a is one of the main providers ||| lm_0=-52.471 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-15.000 tm_pt_5=-55.917 tm_pt_6=-16.351 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-3.506 tm_pt_10=-27.709 tm_pt_11=0.000 tm_pt_12=-17.370 tm_pt_13=0.000 tm_pt_14=-20.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=16.000 WordPenalty=-9.989 OOVPenalty=-100.000 ||| -161.049
+14 ||| tata communicationer foreign this \u09a8\u09bf\u0997\u09ae limited building , it is the telecommunication system is a one of the main providers ||| lm_0=-53.400 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-15.000 tm_pt_5=-48.355 tm_pt_6=-16.672 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-3.371 tm_pt_10=-25.625 tm_pt_11=0.000 tm_pt_12=-15.598 tm_pt_13=0.000 tm_pt_14=-19.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=16.000 WordPenalty=-9.554 OOVPenalty=-100.000 ||| -161.152
+14 ||| tata communicationer foreign this \u09a8\u09bf\u0997\u09ae limited building , it is in the city telecommunication system is a is one of the main providers ||| lm_0=-56.959 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-15.000 tm_pt_5=-61.512 tm_pt_6=-17.383 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-3.556 tm_pt_10=-20.334 tm_pt_11=0.000 tm_pt_12=-18.306 tm_pt_13=0.000 tm_pt_14=-22.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=16.000 WordPenalty=-10.857 OOVPenalty=-100.000 ||| -161.157
+15 ||| he that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national election \u099c\u09af\u09bc\u09c0 in and was elected as the 44th president of the united states . ||| lm_0=-45.486 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-36.305 tm_pt_6=-14.269 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-1.743 tm_pt_10=-8.328 tm_pt_11=0.000 tm_pt_12=-6.254 tm_pt_13=0.000 tm_pt_14=-18.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 WordPenalty=-9.989 OOVPenalty=-300.000 ||| -342.060
+15 ||| he that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national election \u099c\u09af\u09bc\u09c0 in and the 44th president of the united states was elected as . ||| lm_0=-47.286 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-36.305 tm_pt_6=-14.269 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-0.879 tm_pt_10=-8.146 tm_pt_11=0.000 tm_pt_12=-5.849 tm_pt_13=0.000 tm_pt_14=-18.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=12.000 WordPenalty=-9.989 OOVPenalty=-300.000 ||| -342.499
+15 ||| he that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national election \u099c\u09af\u09bc\u09c0 in and the 44th president of the united states elected as . ||| lm_0=-46.823 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-31.599 tm_pt_6=-14.229 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-0.792 tm_pt_10=-8.660 tm_pt_11=0.000 tm_pt_12=-4.441 tm_pt_13=0.000 tm_pt_14=-17.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=12.000 WordPenalty=-9.554 OOVPenalty=-300.000 ||| -342.884
+15 ||| he that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national election \u099c\u09af\u09bc\u09c0 in the 44th president of the united states and was elected as . ||| lm_0=-45.901 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-36.305 tm_pt_6=-14.269 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-1.511 tm_pt_10=-7.057 tm_pt_11=0.000 tm_pt_12=-6.743 tm_pt_13=0.000 tm_pt_14=-18.000 tm_pt_15=0.000 tm_pt_16=-2.000 tm_glue_0=11.000 WordPenalty=-9.989 OOVPenalty=-300.000 ||| -343.137
+15 ||| he that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national election \u099c\u09af\u09bc\u09c0 was and the 44th president of the united states was elected as . ||| lm_0=-47.546 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-35.686 tm_pt_6=-14.665 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-1.007 tm_pt_10=-8.461 tm_pt_11=0.000 tm_pt_12=-6.542 tm_pt_13=0.000 tm_pt_14=-18.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=12.000 WordPenalty=-9.989 OOVPenalty=-300.000 ||| -343.290
+15 ||| he in that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national election \u099c\u09af\u09bc\u09c0 in and the 44th president of the united states was elected as . ||| lm_0=-48.150 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-42.873 tm_pt_6=-14.763 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-0.879 tm_pt_10=-10.279 tm_pt_11=0.000 tm_pt_12=-8.774 tm_pt_13=0.000 tm_pt_14=-19.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=13.000 WordPenalty=-10.423 OOVPenalty=-300.000 ||| -343.354
+15 ||| he that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national election \u099c\u09af\u09bc\u09c0 to and the 44th president of the united states was elected as . ||| lm_0=-47.312 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-37.016 tm_pt_6=-15.512 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-1.007 tm_pt_10=-8.673 tm_pt_11=0.000 tm_pt_12=-6.542 tm_pt_13=0.000 tm_pt_14=-18.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=12.000 WordPenalty=-9.989 OOVPenalty=-300.000 ||| -343.597
+15 ||| he that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national election \u099c\u09af\u09bc\u09c0 was and was elected as the 44th president of the united states . ||| lm_0=-45.745 tm_pt_0=0.000 tm_pt_1=-1.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-35.686 tm_pt_6=-14.665 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-1.007 tm_pt_10=-8.456 tm_pt_11=0.000 tm_pt_12=-6.538 tm_pt_13=0.000 tm_pt_14=-18.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 WordPenalty=-9.989 OOVPenalty=-300.000 ||| -343.600
+15 ||| he that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national election \u099c\u09af\u09bc\u09c0 the and the 44th president of the united states was elected as . ||| lm_0=-47.238 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-38.098 tm_pt_6=-15.129 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-0.890 tm_pt_10=-9.388 tm_pt_11=0.000 tm_pt_12=-6.031 tm_pt_13=0.000 tm_pt_14=-18.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=12.000 WordPenalty=-9.989 OOVPenalty=-300.000 ||| -343.619
+15 ||| he that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 of national election \u099c\u09af\u09bc\u09c0 in and was elected as the 44th president of the united states . ||| lm_0=-46.768 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-36.928 tm_pt_6=-14.269 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-1.746 tm_pt_10=-8.627 tm_pt_11=0.000 tm_pt_12=-9.715 tm_pt_13=0.000 tm_pt_14=-19.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=11.000 WordPenalty=-10.423 OOVPenalty=-300.000 ||| -343.622
+16 ||| many of the indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be from the upper stage ||| lm_0=-35.488 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-22.582 tm_pt_6=-12.391 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.135 tm_pt_10=-4.941 tm_pt_11=0.000 tm_pt_12=-10.172 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=-2.000 tm_glue_0=8.000 WordPenalty=-6.080 OOVPenalty=-300.000 ||| -337.568
+16 ||| many of the indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be from upper stage ||| lm_0=-35.710 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-20.114 tm_pt_6=-12.391 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.135 tm_pt_10=-4.314 tm_pt_11=0.000 tm_pt_12=-6.915 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=8.000 WordPenalty=-5.646 OOVPenalty=-300.000 ||| -337.736
+16 ||| many indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be from the upper stage ||| lm_0=-35.506 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-17.024 tm_pt_6=-12.125 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.000 tm_pt_10=-5.058 tm_pt_11=0.000 tm_pt_12=-6.233 tm_pt_13=0.000 tm_pt_14=-7.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=8.000 WordPenalty=-5.212 OOVPenalty=-300.000 ||| -338.457
+16 ||| many of the \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be from the upper stage ||| lm_0=-33.758 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-22.052 tm_pt_6=-12.755 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.135 tm_pt_10=-8.964 tm_pt_11=0.000 tm_pt_12=-10.151 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=-2.000 tm_glue_0=8.000 WordPenalty=-5.646 OOVPenalty=-300.000 ||| -338.537
+16 ||| many indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be from upper stage ||| lm_0=-35.728 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-14.556 tm_pt_6=-12.125 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.000 tm_pt_10=-4.431 tm_pt_11=0.000 tm_pt_12=-2.976 tm_pt_13=0.000 tm_pt_14=-6.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 WordPenalty=-4.777 OOVPenalty=-300.000 ||| -338.625
+16 ||| many of the \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be from upper stage ||| lm_0=-33.980 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-19.584 tm_pt_6=-12.755 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.135 tm_pt_10=-8.336 tm_pt_11=0.000 tm_pt_12=-6.895 tm_pt_13=0.000 tm_pt_14=-7.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=8.000 WordPenalty=-5.212 OOVPenalty=-300.000 ||| -338.705
+16 ||| many of the indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be to upper stage ||| lm_0=-35.447 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-22.076 tm_pt_6=-13.270 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.135 tm_pt_10=-6.127 tm_pt_11=0.000 tm_pt_12=-8.245 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=8.000 WordPenalty=-5.646 OOVPenalty=-300.000 ||| -338.832
+16 ||| many of indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be from upper stage ||| lm_0=-36.387 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-15.179 tm_pt_6=-12.125 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.000 tm_pt_10=-3.971 tm_pt_11=0.000 tm_pt_12=-4.698 tm_pt_13=0.000 tm_pt_14=-7.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=8.000 WordPenalty=-5.212 OOVPenalty=-300.000 ||| -339.032
+16 ||| many of the \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be to upper stage ||| lm_0=-33.717 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-21.546 tm_pt_6=-13.634 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.135 tm_pt_10=-10.149 tm_pt_11=0.000 tm_pt_12=-8.224 tm_pt_13=0.000 tm_pt_14=-7.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=8.000 WordPenalty=-5.212 OOVPenalty=-300.000 ||| -339.800
+17 ||| britain writers written drama , novels , stories and recently scripts in the \u0986\u09a6\u09c3\u09a4 . ||| lm_0=-40.685 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-31.594 tm_pt_6=-15.592 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-1.738 tm_pt_10=-18.364 tm_pt_11=0.000 tm_pt_12=-8.191 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=14.000 WordPenalty=-7.383 OOVPenalty=-100.000 ||| -144.560
+17 ||| britain writers written drama , novels , stories and recently scripts in \u0986\u09a6\u09c3\u09a4 . ||| lm_0=-39.754 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-23.331 tm_pt_6=-15.528 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-1.738 tm_pt_10=-20.213 tm_pt_11=0.000 tm_pt_12=-8.191 tm_pt_13=0.000 tm_pt_14=-13.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=14.000 WordPenalty=-6.949 OOVPenalty=-100.000 ||| -144.892
+17 ||| britain writers written drama , novels , stories and recently script in the \u0986\u09a6\u09c3\u09a4 . ||| lm_0=-40.389 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-32.369 tm_pt_6=-15.107 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-2.688 tm_pt_10=-18.652 tm_pt_11=0.000 tm_pt_12=-9.577 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=14.000 WordPenalty=-7.383 OOVPenalty=-100.000 ||| -144.986
+17 ||| britain writers written drama , novels , stories and in the recent scripts in the \u0986\u09a6\u09c3\u09a4 . ||| lm_0=-40.964 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-44.622 tm_pt_6=-16.979 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-2.603 tm_pt_10=-17.266 tm_pt_11=0.000 tm_pt_12=-9.290 tm_pt_13=0.000 tm_pt_14=-16.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=14.000 WordPenalty=-8.252 OOVPenalty=-100.000 ||| -145.290
+17 ||| britain writers of written drama , novels , stories and recently scripts in the \u0986\u09a6\u09c3\u09a4 . ||| lm_0=-41.852 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-32.216 tm_pt_6=-15.592 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-1.971 tm_pt_10=-16.755 tm_pt_11=0.000 tm_pt_12=-8.596 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=14.000 WordPenalty=-7.817 OOVPenalty=-100.000 ||| -145.319
+17 ||| britain writers written drama , novels , stories and recently script in \u0986\u09a6\u09c3\u09a4 . ||| lm_0=-39.458 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-24.106 tm_pt_6=-15.042 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-2.688 tm_pt_10=-20.500 tm_pt_11=0.000 tm_pt_12=-9.577 tm_pt_13=0.000 tm_pt_14=-13.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=14.000 WordPenalty=-6.949 OOVPenalty=-100.000 ||| -145.319
+17 ||| britain writers of the drama , novels , stories and recently scripts in the \u0986\u09a6\u09c3\u09a4 . ||| lm_0=-39.928 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-37.083 tm_pt_6=-15.759 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-2.056 tm_pt_10=-21.073 tm_pt_11=0.000 tm_pt_12=-8.884 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=14.000 WordPenalty=-7.817 OOVPenalty=-100.000 ||| -145.402
+17 ||| britain writers written drama , novels , stories and in the recent scripts in \u0986\u09a6\u09c3\u09a4 . ||| lm_0=-40.033 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-36.359 tm_pt_6=-16.914 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-2.603 tm_pt_10=-19.114 tm_pt_11=0.000 tm_pt_12=-9.290 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=14.000 WordPenalty=-7.817 OOVPenalty=-100.000 ||| -145.623
+17 ||| britain writers of written drama , novels , stories and recently scripts in \u0986\u09a6\u09c3\u09a4 . ||| lm_0=-40.921 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-23.953 tm_pt_6=-15.528 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-1.971 tm_pt_10=-18.603 tm_pt_11=0.000 tm_pt_12=-8.596 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=14.000 WordPenalty=-7.383 OOVPenalty=-100.000 ||| -145.651
+17 ||| britain writers written drama , novels , stories and in the recent script in the \u0986\u09a6\u09c3\u09a4 . ||| lm_0=-40.669 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-45.397 tm_pt_6=-16.493 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-3.553 tm_pt_10=-17.553 tm_pt_11=0.000 tm_pt_12=-10.676 tm_pt_13=0.000 tm_pt_14=-16.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=14.000 WordPenalty=-8.252 OOVPenalty=-100.000 ||| -145.717
+18 ||| 1919 , on may month it saogat magazine was published in the . ||| lm_0=-29.940 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-26.450 tm_pt_6=-12.221 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-1.419 tm_pt_10=-4.138 tm_pt_11=0.000 tm_pt_12=-10.969 tm_pt_13=0.000 tm_pt_14=-13.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 WordPenalty=-6.514 ||| -29.082
+18 ||| 1919 on may month it saogat magazine was published in the . ||| lm_0=-28.964 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-21.330 tm_pt_6=-12.215 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-1.284 tm_pt_10=-5.343 tm_pt_11=0.000 tm_pt_12=-7.737 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 WordPenalty=-6.080 ||| -29.174
+18 ||| 1919 , on may month it saogat magazine was published . ||| lm_0=-27.704 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-19.272 tm_pt_6=-13.877 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-1.302 tm_pt_10=-3.608 tm_pt_11=0.000 tm_pt_12=-8.929 tm_pt_13=0.000 tm_pt_14=-11.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 WordPenalty=-5.646 ||| -29.196
+18 ||| 1919 on may month it saogat magazine was published . ||| lm_0=-26.729 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-14.152 tm_pt_6=-13.871 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-1.167 tm_pt_10=-4.813 tm_pt_11=0.000 tm_pt_12=-5.696 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 WordPenalty=-5.212 ||| -29.287
+18 ||| 1919 , on may month it is saogat magazine was published in the . ||| lm_0=-30.979 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-30.304 tm_pt_6=-12.487 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-1.419 tm_pt_10=-3.581 tm_pt_11=0.000 tm_pt_12=-11.685 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 WordPenalty=-6.949 ||| -29.459
+18 ||| 1919 on may month it is saogat magazine was published in the . ||| lm_0=-30.004 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-25.184 tm_pt_6=-12.482 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-1.284 tm_pt_10=-4.786 tm_pt_11=0.000 tm_pt_12=-8.453 tm_pt_13=0.000 tm_pt_14=-13.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 WordPenalty=-6.514 ||| -29.550
+18 ||| 1919 , on may month it is saogat magazine was published . ||| lm_0=-28.743 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-23.126 tm_pt_6=-14.143 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-1.302 tm_pt_10=-3.051 tm_pt_11=0.000 tm_pt_12=-9.644 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 WordPenalty=-6.080 ||| -29.572
+18 ||| 1919 on may month it is saogat magazine was published . ||| lm_0=-27.768 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-18.006 tm_pt_6=-14.138 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-1.167 tm_pt_10=-4.256 tm_pt_11=0.000 tm_pt_12=-6.412 tm_pt_13=0.000 tm_pt_14=-11.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 WordPenalty=-5.646 ||| -29.663
+18 ||| 1919 , on may month it saogat magazine was published in . ||| lm_0=-29.717 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-22.442 tm_pt_6=-12.489 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-1.334 tm_pt_10=-4.544 tm_pt_11=0.000 tm_pt_12=-10.682 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 WordPenalty=-6.080 ||| -29.880
+18 ||| 1919 on may month it saogat magazine was published in . ||| lm_0=-28.741 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-17.322 tm_pt_6=-12.483 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-1.199 tm_pt_10=-5.748 tm_pt_11=0.000 tm_pt_12=-7.450 tm_pt_13=0.000 tm_pt_14=-11.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 WordPenalty=-5.646 ||| -29.971
+19 ||| in 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium was arranged . ||| lm_0=-58.555 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-24.197 tm_pt_6=-8.326 tm_pt_7=-12.000 tm_pt_8=-32.616 tm_pt_9=-2.263 tm_pt_10=-12.612 tm_pt_11=0.000 tm_pt_12=-8.419 tm_pt_13=0.000 tm_pt_14=-13.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=14.000 WordPenalty=-8.252 OOVPenalty=-400.000 ||| -460.093
+19 ||| 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium was arranged . ||| lm_0=-57.604 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-21.244 tm_pt_6=-8.707 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.380 tm_pt_10=-14.899 tm_pt_11=0.000 tm_pt_12=-6.917 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=14.000 WordPenalty=-7.817 OOVPenalty=-400.000 ||| -460.244
+19 ||| in year 2005 \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium was arranged . ||| lm_0=-58.848 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-21.149 tm_pt_6=-8.568 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-3.245 tm_pt_10=-12.819 tm_pt_11=0.000 tm_pt_12=-8.016 tm_pt_13=0.000 tm_pt_14=-13.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=14.000 WordPenalty=-8.252 OOVPenalty=-400.000 ||| -460.557
+19 ||| in 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association of the tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium was arranged . ||| lm_0=-60.199 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-44.288 tm_pt_6=-8.732 tm_pt_7=-12.000 tm_pt_8=-32.616 tm_pt_9=-2.263 tm_pt_10=-11.359 tm_pt_11=0.000 tm_pt_12=-8.419 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=14.000 WordPenalty=-9.120 OOVPenalty=-400.000 ||| -460.712
+19 ||| 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association of the tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium was arranged . ||| lm_0=-59.247 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-41.334 tm_pt_6=-9.112 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.380 tm_pt_10=-13.646 tm_pt_11=0.000 tm_pt_12=-6.917 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=14.000 WordPenalty=-8.686 OOVPenalty=-400.000 ||| -460.863
+19 ||| in 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium is the . ||| lm_0=-56.691 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-11.000 tm_pt_5=-32.027 tm_pt_6=-8.816 tm_pt_7=-12.000 tm_pt_8=-32.616 tm_pt_9=-2.260 tm_pt_10=-21.790 tm_pt_11=0.000 tm_pt_12=-9.322 tm_pt_13=0.000 tm_pt_14=-13.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=15.000 WordPenalty=-8.252 OOVPenalty=-400.000 ||| -461.115
+19 ||| 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium is the . ||| lm_0=-55.740 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-11.000 tm_pt_5=-29.073 tm_pt_6=-9.197 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.377 tm_pt_10=-24.077 tm_pt_11=0.000 tm_pt_12=-7.821 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=15.000 WordPenalty=-7.817 OOVPenalty=-400.000 ||| -461.266
+19 ||| in 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association of the tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium is the . ||| lm_0=-58.335 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-11.000 tm_pt_5=-52.118 tm_pt_6=-9.222 tm_pt_7=-12.000 tm_pt_8=-32.616 tm_pt_9=-2.260 tm_pt_10=-20.538 tm_pt_11=0.000 tm_pt_12=-9.322 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=15.000 WordPenalty=-9.120 OOVPenalty=-400.000 ||| -461.734
+19 ||| in 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium arranged in the . ||| lm_0=-60.337 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-11.000 tm_pt_5=-33.927 tm_pt_6=-9.017 tm_pt_7=-12.000 tm_pt_8=-32.616 tm_pt_9=-2.260 tm_pt_10=-15.546 tm_pt_11=0.000 tm_pt_12=-9.322 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=15.000 WordPenalty=-8.686 OOVPenalty=-400.000 ||| -461.872
+19 ||| 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association of the tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium is the . ||| lm_0=-57.383 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-11.000 tm_pt_5=-49.164 tm_pt_6=-9.602 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.377 tm_pt_10=-22.825 tm_pt_11=0.000 tm_pt_12=-7.821 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=15.000 WordPenalty=-8.686 OOVPenalty=-400.000 ||| -461.885
+20 ||| to prevent this several measures are taken . ||| lm_0=-11.632 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-22.680 tm_pt_6=-30.812 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-1.000 tm_pt_10=-0.019 tm_pt_11=0.000 tm_pt_12=-1.390 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 WordPenalty=-4.343 ||| -21.333
+20 ||| to prevent this several measures are in the . ||| lm_0=-15.062 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-3.000 tm_pt_5=-27.397 tm_pt_6=-27.297 tm_pt_7=-3.000 tm_pt_8=-8.154 tm_pt_9=-1.000 tm_pt_10=-2.653 tm_pt_11=0.000 tm_pt_12=-4.379 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=3.000 WordPenalty=-4.777 ||| -23.640
+20 ||| to prevent this several measures are the . ||| lm_0=-14.066 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-3.000 tm_pt_5=-24.227 tm_pt_6=-27.251 tm_pt_7=-3.000 tm_pt_8=-8.154 tm_pt_9=-1.000 tm_pt_10=-3.499 tm_pt_11=0.000 tm_pt_12=-2.316 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=3.000 WordPenalty=-4.343 ||| -23.669
+20 ||| to prevent this several measures are . ||| lm_0=-12.686 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-20.219 tm_pt_6=-29.189 tm_pt_7=-3.000 tm_pt_8=-8.154 tm_pt_9=-1.000 tm_pt_10=-1.427 tm_pt_11=-1.000 tm_pt_12=-1.717 tm_pt_13=0.000 tm_pt_14=-7.000 tm_pt_15=-1.000 tm_pt_16=0.000 tm_glue_0=2.000 WordPenalty=-3.909 ||| -23.707
+20 ||| to prevent this several measures are in . ||| lm_0=-14.649 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-3.000 tm_pt_5=-23.388 tm_pt_6=-27.344 tm_pt_7=-3.000 tm_pt_8=-8.154 tm_pt_9=-1.000 tm_pt_10=-2.853 tm_pt_11=0.000 tm_pt_12=-2.730 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=3.000 WordPenalty=-4.343 ||| -24.099
+20 ||| to avoid this possibility several measures are taken . ||| lm_0=-13.461 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-1.000 tm_pt_5=-24.597 tm_pt_6=-31.733 tm_pt_7=-1.000 tm_pt_8=-2.718 tm_pt_9=-1.000 tm_pt_10=-0.693 tm_pt_11=0.000 tm_pt_12=-2.079 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=1.000 WordPenalty=-4.777 ||| -24.504
+21 ||| \u09e7\u09ef\u09ec\u09ec on 5 february \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 of \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 a national was held in . ||| lm_0=-42.147 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-24.651 tm_pt_6=-11.840 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-1.018 tm_pt_10=-9.070 tm_pt_11=0.000 tm_pt_12=-4.189 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 WordPenalty=-6.514 OOVPenalty=-300.000 ||| -342.312
+21 ||| \u09e7\u09ef\u09ec\u09ec on 5 february \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 opposition \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 a national was held in . ||| lm_0=-44.316 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-18.447 tm_pt_6=-11.935 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-1.135 tm_pt_10=-4.038 tm_pt_11=0.000 tm_pt_12=-4.700 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 WordPenalty=-6.514 OOVPenalty=-300.000 ||| -342.432
+21 ||| \u09e7\u09ef\u09ec\u09ec on february 5 \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 of \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 a national was held in . ||| lm_0=-40.697 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-24.651 tm_pt_6=-11.840 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-1.154 tm_pt_10=-9.411 tm_pt_11=0.000 tm_pt_12=-5.955 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 WordPenalty=-6.514 OOVPenalty=-300.000 ||| -342.506
+21 ||| \u09e7\u09ef\u09ec\u09ec on 5 february \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 against \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 a national was held in . ||| lm_0=-43.433 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-19.901 tm_pt_6=-11.529 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-1.135 tm_pt_10=-6.871 tm_pt_11=0.000 tm_pt_12=-4.700 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 WordPenalty=-6.514 OOVPenalty=-300.000 ||| -342.592
+21 ||| \u09e7\u09ef\u09ec\u09ec on february 5 \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 opposition \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 a national was held in . ||| lm_0=-42.866 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-18.447 tm_pt_6=-11.935 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-1.271 tm_pt_10=-4.380 tm_pt_11=0.000 tm_pt_12=-6.466 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 WordPenalty=-6.514 OOVPenalty=-300.000 ||| -342.627
+21 ||| \u09e7\u09ef\u09ec\u09ec on 5th february \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 of \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 a national was held in . ||| lm_0=-40.753 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-24.978 tm_pt_6=-13.083 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-2.018 tm_pt_10=-10.356 tm_pt_11=0.000 tm_pt_12=-7.827 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 WordPenalty=-6.514 OOVPenalty=-300.000 ||| -342.668
+21 ||| \u09e7\u09ef\u09ec\u09ec on february 5 \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 against \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 a national was held in . ||| lm_0=-41.983 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-19.901 tm_pt_6=-11.529 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-1.271 tm_pt_10=-7.213 tm_pt_11=0.000 tm_pt_12=-6.466 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 WordPenalty=-6.514 OOVPenalty=-300.000 ||| -342.787
+21 ||| \u09e7\u09ef\u09ec\u09ec on 5th february \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 against \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 a national was held in . ||| lm_0=-42.039 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-20.228 tm_pt_6=-12.773 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-2.135 tm_pt_10=-8.158 tm_pt_11=0.000 tm_pt_12=-8.337 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 WordPenalty=-6.514 OOVPenalty=-300.000 ||| -342.948
+21 ||| \u09e7\u09ef\u09ec\u09ec , on 5 february \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 of \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 a national was held in . ||| lm_0=-43.750 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-29.771 tm_pt_6=-11.845 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-1.154 tm_pt_10=-7.865 tm_pt_11=0.000 tm_pt_12=-7.421 tm_pt_13=0.000 tm_pt_14=-11.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 WordPenalty=-6.949 OOVPenalty=-300.000 ||| -343.049
+21 ||| \u09e7\u09ef\u09ec\u09ec on 5 february \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 of \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 is a national was held in . ||| lm_0=-43.250 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-28.350 tm_pt_6=-12.409 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-1.018 tm_pt_10=-9.057 tm_pt_11=0.000 tm_pt_12=-7.050 tm_pt_13=0.000 tm_pt_14=-11.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 WordPenalty=-6.949 OOVPenalty=-300.000 ||| -343.128
+22 ||| bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in in the . ||| lm_0=-45.104 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-11.000 tm_pt_5=-50.512 tm_pt_6=-9.888 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.050 tm_pt_10=-24.429 tm_pt_11=0.000 tm_pt_12=-3.380 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=14.000 WordPenalty=-8.252 OOVPenalty=-300.000 ||| -348.887
+22 ||| bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in took the . ||| lm_0=-46.563 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-11.000 tm_pt_5=-46.602 tm_pt_6=-9.693 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.369 tm_pt_10=-21.189 tm_pt_11=0.000 tm_pt_12=-4.073 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=14.000 WordPenalty=-8.252 OOVPenalty=-300.000 ||| -349.144
+22 ||| bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in adopted . ||| lm_0=-45.321 tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-42.382 tm_pt
<TRUNCATED>
[36/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/packed/PackedGrammar.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/packed/PackedGrammar.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/packed/PackedGrammar.java
new file mode 100644
index 0000000..b48685d
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/packed/PackedGrammar.java
@@ -0,0 +1,1066 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.tm.packed;
+
+/***
+ * This package implements Joshua's packed grammar structure, which enables the efficient loading
+ * and accessing of grammars. It is described in the paper:
+ *
+ * @article{ganitkevitch2012joshua,
+ * Author = {Ganitkevitch, J. and Cao, Y. and Weese, J. and Post, M. and Callison-Burch, C.},
+ * Journal = {Proceedings of WMT12},
+ * Title = {Joshua 4.0: Packing, PRO, and paraphrases},
+ * Year = {2012}}
+ *
+ * The packed grammar works by compiling out the grammar tries into a compact format that is loaded
+ * and parsed directly from Java arrays. A fundamental problem is that Java arrays are indexed
+ * by ints and not longs, meaning the maximum size of the packed grammar is about 2 GB. This forces
+ * the use of packed grammar slices, which together constitute the grammar. The figure in the
+ * paper above shows what each slice looks like.
+ *
+ * The division across slices is done in a depth-first manner. Consider the entire grammar organized
+ * into a single source-side trie. The splits across tries are done by grouping the root-level
+ * outgoing trie arcs --- and the entire trie beneath them --- across slices.
+ *
+ * This presents a problem: if the subtree rooted beneath a single top-level arc is too big for a
+ * slice, the grammar can't be packed. This happens with very large Hiero grammars, for example,
+ * where there are a *lot* of rules that start with [X].
+ *
+ * A solution being worked on is to split that symbol and pack them into separate grammars with a
+ * shared vocabulary, and then rely on Joshua's ability to query multiple grammars for rules to
+ * solve this problem. This is not currently implemented but could be done directly in the
+ * Grammar Packer.
+ *
+ * *UPDATE 10/2015*
+ * The introduction of a SliceAggregatingTrie together with sorting the grammar by the full source string
+ * (not just by the first source word) allows distributing rules with the same first source word
+ * across multiple slices.
+ * @author fhieber
+ */
+
+import static java.util.Collections.sort;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.BufferUnderflowException;
+import java.nio.ByteBuffer;
+import java.nio.IntBuffer;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileChannel.MapMode;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.security.DigestInputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.ff.FeatureFunction;
+import org.apache.joshua.decoder.ff.FeatureVector;
+import org.apache.joshua.decoder.ff.tm.AbstractGrammar;
+import org.apache.joshua.decoder.ff.tm.BasicRuleCollection;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.ff.tm.RuleCollection;
+import org.apache.joshua.decoder.ff.tm.Trie;
+import org.apache.joshua.decoder.ff.tm.hash_based.ExtensionIterator;
+import org.apache.joshua.util.FormatUtils;
+import org.apache.joshua.util.encoding.EncoderConfiguration;
+import org.apache.joshua.util.encoding.FloatEncoder;
+import org.apache.joshua.util.io.LineReader;
+
+import com.google.common.base.Supplier;
+import com.google.common.base.Suppliers;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class PackedGrammar extends AbstractGrammar {
+
+ private static final Logger LOG = LoggerFactory.getLogger(PackedGrammar.class);
+ public static final String VOCABULARY_FILENAME = "vocabulary";
+
+ private EncoderConfiguration encoding;
+ private PackedRoot root;
+ private ArrayList<PackedSlice> slices;
+
+ private final File vocabFile; // store path to vocabulary file
+
+ // The version number of the earliest supported grammar packer
+ public static final int SUPPORTED_VERSION = 3;
+
+ // A rule cache for commonly used tries to avoid excess object allocations
+ // Testing shows there's up to ~95% hit rate when cache size is 5000 Trie nodes.
+ private final Cache<Trie, List<Rule>> cached_rules;
+
+ private String grammarDir;
+
+ public PackedGrammar(String grammar_dir, int span_limit, String owner, String type,
+ JoshuaConfiguration joshuaConfiguration) throws IOException {
+ super(joshuaConfiguration);
+
+ this.grammarDir = grammar_dir;
+ this.spanLimit = span_limit;
+
+ // Read the vocabulary.
+ vocabFile = new File(grammar_dir + File.separator + VOCABULARY_FILENAME);
+ LOG.info("Reading vocabulary: {}", vocabFile);
+ if (!Vocabulary.read(vocabFile)) {
+ throw new RuntimeException("mismatches or collisions while reading on-disk vocabulary");
+ }
+
+ // Read the config
+ String configFile = grammar_dir + File.separator + "config";
+ if (new File(configFile).exists()) {
+ LOG.info("Reading packed config: {}", configFile);
+ readConfig(configFile);
+ }
+
+ // Read the quantizer setup.
+ LOG.info("Reading encoder configuration: {}{}encoding", grammar_dir, File.separator);
+ encoding = new EncoderConfiguration();
+ encoding.load(grammar_dir + File.separator + "encoding");
+
+ // Set phrase owner.
+ this.owner = Vocabulary.id(owner);
+
+ final List<String> listing = Arrays.asList(new File(grammar_dir).list());
+ sort(listing); // File.list() has arbitrary sort order
+ slices = new ArrayList<PackedSlice>();
+ for (String prefix : listing) {
+ if (prefix.startsWith("slice_") && prefix.endsWith(".source"))
+ slices.add(new PackedSlice(grammar_dir + File.separator + prefix.substring(0, 11)));
+ }
+
+ long count = 0;
+ for (PackedSlice s : slices)
+ count += s.estimated.length;
+ root = new PackedRoot(slices);
+ cached_rules = CacheBuilder.newBuilder().maximumSize(joshuaConfiguration.cachedRuleSize).build();
+
+ LOG.info("Loaded {} rules", count);
+ }
+
+ @Override
+ public Trie getTrieRoot() {
+ return root;
+ }
+
+ @Override
+ public boolean hasRuleForSpan(int startIndex, int endIndex, int pathLength) {
+ return (spanLimit == -1 || pathLength <= spanLimit);
+ }
+
+ @Override
+ public int getNumRules() {
+ int num_rules = 0;
+ for (PackedSlice ps : slices)
+ num_rules += ps.featureSize;
+ return num_rules;
+ }
+
+ @Override
+ public int getNumDenseFeatures() {
+ return encoding.getNumDenseFeatures();
+ }
+
+ /**
+ * Computes the MD5 checksum of the vocabulary file.
+ * Can be used for comparing vocabularies across multiple packedGrammars.
+ * @return the computed checksum
+ */
+ public String computeVocabularyChecksum() {
+ MessageDigest md;
+ try {
+ md = MessageDigest.getInstance("MD5");
+ } catch (NoSuchAlgorithmException e) {
+ throw new RuntimeException("Unknown checksum algorithm");
+ }
+ byte[] buffer = new byte[1024];
+ try (final InputStream is = Files.newInputStream(Paths.get(vocabFile.toString()));
+ DigestInputStream dis = new DigestInputStream(is, md)) {
+ while (dis.read(buffer) != -1) {}
+ } catch (IOException e) {
+ throw new RuntimeException("Can not find vocabulary file. This should not happen.");
+ }
+ byte[] digest = md.digest();
+ // convert the byte to hex format
+ StringBuffer sb = new StringBuffer("");
+ for (int i = 0; i < digest.length; i++) {
+ sb.append(Integer.toString((digest[i] & 0xff) + 0x100, 16).substring(1));
+ }
+ return sb.toString();
+ }
+
+ /**
+ * PackedRoot represents the root of the packed grammar trie.
+ * Tries for different source-side firstwords are organized in
+ * packedSlices on disk. A packedSlice can contain multiple trie
+ * roots (i.e. multiple source-side firstwords).
+ * The PackedRoot builds a lookup table, mapping from
+ * source-side firstwords to the addresses in the packedSlices
+ * that represent the subtrie for a particular firstword.
+ * If the GrammarPacker has to distribute rules for a
+ * source-side firstword across multiple slices, a
+ * SliceAggregatingTrie node is created that aggregates those
+ * tries to hide
+ * this additional complexity from the grammar interface
+ * This feature allows packing of grammars where the list of rules
+ * for a single source-side firstword would exceed the maximum array
+ * size of Java (2gb).
+ */
+ public final class PackedRoot implements Trie {
+
+ private final HashMap<Integer, Trie> lookup;
+
+ public PackedRoot(final List<PackedSlice> slices) {
+ final Map<Integer, List<Trie>> childTries = collectChildTries(slices);
+ lookup = buildLookupTable(childTries);
+ }
+
+ /**
+ * Determines whether trie nodes for source first-words are spread over
+ * multiple packedSlices by counting their occurrences.
+ * @param slices
+ * @return A mapping from first word ids to a list of trie nodes.
+ */
+ private Map<Integer, List<Trie>> collectChildTries(final List<PackedSlice> slices) {
+ final Map<Integer, List<Trie>> childTries = new HashMap<>();
+ for (PackedSlice packedSlice : slices) {
+
+ // number of tries stored in this packedSlice
+ final int num_children = packedSlice.source[0];
+ for (int i = 0; i < num_children; i++) {
+ final int id = packedSlice.source[2 * i + 1];
+
+ /* aggregate tries with same root id
+ * obtain a Trie node, already at the correct address in the packedSlice.
+ * In other words, the lookup index already points to the correct trie node in the packedSlice.
+ * packedRoot.match() thus can directly return the result of lookup.get(id);
+ */
+ if (!childTries.containsKey(id)) {
+ childTries.put(id, new ArrayList<Trie>(1));
+ }
+ final Trie trie = packedSlice.root().match(id);
+ childTries.get(id).add(trie);
+ }
+ }
+ return childTries;
+ }
+
+ /**
+ * Build a lookup table for children tries.
+ * If the list contains only a single child node, a regular trie node
+ * is inserted into the table; otherwise a SliceAggregatingTrie node is
+ * created that hides this partitioning into multiple packedSlices
+ * upstream.
+ */
+ private HashMap<Integer,Trie> buildLookupTable(final Map<Integer, List<Trie>> childTries) {
+ HashMap<Integer,Trie> lookup = new HashMap<>(childTries.size());
+ for (int id : childTries.keySet()) {
+ final List<Trie> tries = childTries.get(id);
+ if (tries.size() == 1) {
+ lookup.put(id, tries.get(0));
+ } else {
+ lookup.put(id, new SliceAggregatingTrie(tries));
+ }
+ }
+ return lookup;
+ }
+
+ @Override
+ public Trie match(int word_id) {
+ return lookup.get(word_id);
+ }
+
+ @Override
+ public boolean hasExtensions() {
+ return !lookup.isEmpty();
+ }
+
+ @Override
+ public HashMap<Integer, ? extends Trie> getChildren() {
+ return lookup;
+ }
+
+ @Override
+ public ArrayList<? extends Trie> getExtensions() {
+ return new ArrayList<>(lookup.values());
+ }
+
+ @Override
+ public boolean hasRules() {
+ return false;
+ }
+
+ @Override
+ public RuleCollection getRuleCollection() {
+ return new BasicRuleCollection(0, new int[0]);
+ }
+
+ @Override
+ public Iterator<Integer> getTerminalExtensionIterator() {
+ return new ExtensionIterator(lookup, true);
+ }
+
+ @Override
+ public Iterator<Integer> getNonterminalExtensionIterator() {
+ return new ExtensionIterator(lookup, false);
+ }
+ }
+
+ public final class PackedSlice {
+ private final String name;
+
+ private final int[] source;
+ private final IntBuffer target;
+ private final ByteBuffer features;
+ private final ByteBuffer alignments;
+
+ private final int[] targetLookup;
+ private int featureSize;
+ private float[] estimated;
+ private float[] precomputable;
+
+ private final static int BUFFER_HEADER_POSITION = 8;
+
+ /**
+ * Provides a cache of packedTrie nodes to be used in getTrie.
+ */
+ private HashMap<Integer, PackedTrie> tries;
+
+ public PackedSlice(String prefix) throws IOException {
+ name = prefix;
+
+ File source_file = new File(prefix + ".source");
+ File target_file = new File(prefix + ".target");
+ File target_lookup_file = new File(prefix + ".target.lookup");
+ File feature_file = new File(prefix + ".features");
+ File alignment_file = new File(prefix + ".alignments");
+
+ source = fullyLoadFileToArray(source_file);
+ // First int specifies the size of this file, load from 1st int on
+ targetLookup = fullyLoadFileToArray(target_lookup_file, 1);
+
+ target = associateMemoryMappedFile(target_file).asIntBuffer();
+ features = associateMemoryMappedFile(feature_file);
+ initializeFeatureStructures();
+
+ if (alignment_file.exists()) {
+ alignments = associateMemoryMappedFile(alignment_file);
+ } else {
+ alignments = null;
+ }
+
+ tries = new HashMap<Integer, PackedTrie>();
+ }
+
+ /**
+ * Helper function to help create all the structures which describe features
+ * in the Slice. Only called during object construction.
+ */
+ private void initializeFeatureStructures() {
+ int num_blocks = features.getInt(0);
+ estimated = new float[num_blocks];
+ precomputable = new float[num_blocks];
+ Arrays.fill(estimated, Float.NEGATIVE_INFINITY);
+ Arrays.fill(precomputable, Float.NEGATIVE_INFINITY);
+ featureSize = features.getInt(4);
+ }
+
+ private int getIntFromByteBuffer(int position, ByteBuffer buffer) {
+ return buffer.getInt(BUFFER_HEADER_POSITION + (4 * position));
+ }
+
+ private int[] fullyLoadFileToArray(File file) throws IOException {
+ return fullyLoadFileToArray(file, 0);
+ }
+
+ /**
+ * This function will use a bulk loading method to fully populate a target
+ * array from file.
+ *
+ * @param file
+ * File that will be read from disk.
+ * @param startIndex
+ * an offset into the read file.
+ * @return an int array of size length(file) - offset containing ints in the
+ * file.
+ * @throws IOException
+ */
+ private int[] fullyLoadFileToArray(File file, int startIndex) throws IOException {
+ IntBuffer buffer = associateMemoryMappedFile(file).asIntBuffer();
+ int size = (int) (file.length() - (4 * startIndex))/4;
+ int[] result = new int[size];
+ buffer.position(startIndex);
+ buffer.get(result, 0, size);
+ return result;
+ }
+
+ private ByteBuffer associateMemoryMappedFile(File file) throws IOException {
+ try(FileInputStream fileInputStream = new FileInputStream(file)) {
+ FileChannel fileChannel = fileInputStream.getChannel();
+ int size = (int) fileChannel.size();
+ MappedByteBuffer result = fileChannel.map(MapMode.READ_ONLY, 0, size);
+ return result;
+ }
+ }
+
+ private final int[] getTarget(int pointer) {
+ // Figure out level.
+ int tgt_length = 1;
+ while (tgt_length < (targetLookup.length + 1) && targetLookup[tgt_length] <= pointer)
+ tgt_length++;
+ int[] tgt = new int[tgt_length];
+ int index = 0;
+ int parent;
+ do {
+ parent = target.get(pointer);
+ if (parent != -1)
+ tgt[index++] = target.get(pointer + 1);
+ pointer = parent;
+ } while (pointer != -1);
+ return tgt;
+ }
+
+ private synchronized PackedTrie getTrie(final int node_address) {
+ PackedTrie t = tries.get(node_address);
+ if (t == null) {
+ t = new PackedTrie(node_address);
+ tries.put(node_address, t);
+ }
+ return t;
+ }
+
+ private synchronized PackedTrie getTrie(int node_address, int[] parent_src, int parent_arity,
+ int symbol) {
+ PackedTrie t = tries.get(node_address);
+ if (t == null) {
+ t = new PackedTrie(node_address, parent_src, parent_arity, symbol);
+ tries.put(node_address, t);
+ }
+ return t;
+ }
+
+ /**
+ * Returns the FeatureVector associated with a rule (represented as a block ID).
+ * These features are in the form "feature1=value feature2=value...". By default, unlabeled
+ * features are named using the pattern.
+ * @param block_id
+ * @return feature vector
+ */
+
+ private final FeatureVector loadFeatureVector(int block_id) {
+ int featurePosition = getIntFromByteBuffer(block_id, features);
+ final int numFeatures = encoding.readId(features, featurePosition);
+
+ featurePosition += EncoderConfiguration.ID_SIZE;
+ final FeatureVector featureVector = new FeatureVector();
+ FloatEncoder encoder;
+ String featureName;
+
+ for (int i = 0; i < numFeatures; i++) {
+ final int innerId = encoding.readId(features, featurePosition);
+ final int outerId = encoding.outerId(innerId);
+ encoder = encoding.encoder(innerId);
+ // TODO (fhieber): why on earth are dense feature ids (ints) encoded in the vocabulary?
+ featureName = Vocabulary.word(outerId);
+ final float value = encoder.read(features, featurePosition);
+ try {
+ int index = Integer.parseInt(featureName);
+ featureVector.increment(index, -value);
+ } catch (NumberFormatException e) {
+ featureVector.increment(featureName, value);
+ }
+ featurePosition += EncoderConfiguration.ID_SIZE + encoder.size();
+ }
+
+ return featureVector;
+ }
+
+ /**
+ * We need to synchronize this method as there is a many to one ratio between
+ * PackedRule/PhrasePair and this class (PackedSlice). This means during concurrent first
+ * getAlignments calls to PackedRule objects they could alter each other's positions within the
+ * buffer before calling read on the buffer.
+ */
+ private synchronized final byte[] getAlignmentArray(int block_id) {
+ if (alignments == null)
+ throw new RuntimeException("No alignments available.");
+ int alignment_position = getIntFromByteBuffer(block_id, alignments);
+ int num_points = (int) alignments.get(alignment_position);
+ byte[] alignment = new byte[num_points * 2];
+
+ alignments.position(alignment_position + 1);
+ try {
+ alignments.get(alignment, 0, num_points * 2);
+ } catch (BufferUnderflowException bue) {
+ LOG.warn("Had an exception when accessing alignment mapped byte buffer");
+ LOG.warn("Attempting to access alignments at position: {}", alignment_position + 1);
+ LOG.warn("And to read this many bytes: {}", num_points * 2);
+ LOG.warn("Buffer capacity is : {}", alignments.capacity());
+ LOG.warn("Buffer position is : {}", alignments.position());
+ LOG.warn("Buffer limit is : {}", alignments.limit());
+ throw bue;
+ }
+ return alignment;
+ }
+
+ private final PackedTrie root() {
+ return getTrie(0);
+ }
+
+ public String toString() {
+ return name;
+ }
+
+ /**
+ * A trie node within the grammar slice. Identified by its position within the source array,
+ * and, as a supplement, the source string leading from the trie root to the node.
+ *
+ * @author jg
+ *
+ */
+ public class PackedTrie implements Trie, RuleCollection {
+
+ private final int position;
+
+ private boolean sorted = false;
+
+ private int[] src;
+ private int arity;
+
+ private PackedTrie(int position) {
+ this.position = position;
+ src = new int[0];
+ arity = 0;
+ }
+
+ private PackedTrie(int position, int[] parent_src, int parent_arity, int symbol) {
+ this.position = position;
+ src = new int[parent_src.length + 1];
+ System.arraycopy(parent_src, 0, src, 0, parent_src.length);
+ src[src.length - 1] = symbol;
+ arity = parent_arity;
+ if (FormatUtils.isNonterminal(symbol))
+ arity++;
+ }
+
+ @Override
+ public final Trie match(int token_id) {
+ int num_children = source[position];
+ if (num_children == 0)
+ return null;
+ if (num_children == 1 && token_id == source[position + 1])
+ return getTrie(source[position + 2], src, arity, token_id);
+ int top = 0;
+ int bottom = num_children - 1;
+ while (true) {
+ int candidate = (top + bottom) / 2;
+ int candidate_position = position + 1 + 2 * candidate;
+ int read_token = source[candidate_position];
+ if (read_token == token_id) {
+ return getTrie(source[candidate_position + 1], src, arity, token_id);
+ } else if (top == bottom) {
+ return null;
+ } else if (read_token > token_id) {
+ top = candidate + 1;
+ } else {
+ bottom = candidate - 1;
+ }
+ if (bottom < top)
+ return null;
+ }
+ }
+
+ @Override
+ public HashMap<Integer, ? extends Trie> getChildren() {
+ HashMap<Integer, Trie> children = new HashMap<Integer, Trie>();
+ int num_children = source[position];
+ for (int i = 0; i < num_children; i++) {
+ int symbol = source[position + 1 + 2 * i];
+ int address = source[position + 2 + 2 * i];
+ children.put(symbol, getTrie(address, src, arity, symbol));
+ }
+ return children;
+ }
+
+ @Override
+ public boolean hasExtensions() {
+ return (source[position] != 0);
+ }
+
+ @Override
+ public ArrayList<? extends Trie> getExtensions() {
+ int num_children = source[position];
+ ArrayList<PackedTrie> tries = new ArrayList<PackedTrie>(num_children);
+
+ for (int i = 0; i < num_children; i++) {
+ int symbol = source[position + 1 + 2 * i];
+ int address = source[position + 2 + 2 * i];
+ tries.add(getTrie(address, src, arity, symbol));
+ }
+
+ return tries;
+ }
+
+ @Override
+ public boolean hasRules() {
+ int num_children = source[position];
+ return (source[position + 1 + 2 * num_children] != 0);
+ }
+
+ @Override
+ public RuleCollection getRuleCollection() {
+ return this;
+ }
+
+ @Override
+ public List<Rule> getRules() {
+ List<Rule> rules = cached_rules.getIfPresent(this);
+ if (rules != null) {
+ return rules;
+ }
+
+ int num_children = source[position];
+ int rule_position = position + 2 * (num_children + 1);
+ int num_rules = source[rule_position - 1];
+
+ rules = new ArrayList<Rule>(num_rules);
+ for (int i = 0; i < num_rules; i++) {
+ rules.add(new PackedRule(rule_position + 3 * i));
+ }
+
+ cached_rules.put(this, rules);
+ return rules;
+ }
+
+ /**
+ * We determine if the Trie is sorted by checking if the estimated cost of the first rule in
+ * the trie has been set.
+ */
+ @Override
+ public boolean isSorted() {
+ return sorted;
+ }
+
+ private synchronized void sortRules(List<FeatureFunction> models) {
+ int num_children = source[position];
+ int rule_position = position + 2 * (num_children + 1);
+ int num_rules = source[rule_position - 1];
+ if (num_rules == 0) {
+ this.sorted = true;
+ return;
+ }
+ Integer[] rules = new Integer[num_rules];
+
+ int target_address;
+ int block_id;
+ for (int i = 0; i < num_rules; ++i) {
+ target_address = source[rule_position + 1 + 3 * i];
+ rules[i] = rule_position + 2 + 3 * i;
+ block_id = source[rules[i]];
+
+ Rule rule = new Rule(source[rule_position + 3 * i], src,
+ getTarget(target_address), loadFeatureVector(block_id), arity, owner);
+ estimated[block_id] = rule.estimateRuleCost(models);
+ precomputable[block_id] = rule.getPrecomputableCost();
+ }
+
+ Arrays.sort(rules, new Comparator<Integer>() {
+ public int compare(Integer a, Integer b) {
+ float a_cost = estimated[source[a]];
+ float b_cost = estimated[source[b]];
+ if (a_cost == b_cost)
+ return 0;
+ return (a_cost > b_cost ? -1 : 1);
+ }
+ });
+
+ int[] sorted = new int[3 * num_rules];
+ int j = 0;
+ for (int i = 0; i < rules.length; i++) {
+ int address = rules[i];
+ sorted[j++] = source[address - 2];
+ sorted[j++] = source[address - 1];
+ sorted[j++] = source[address];
+ }
+ for (int i = 0; i < sorted.length; i++)
+ source[rule_position + i] = sorted[i];
+
+ // Replace rules in cache with their sorted values on next getRules()
+ cached_rules.invalidate(this);
+ this.sorted = true;
+ }
+
+ @Override
+ public List<Rule> getSortedRules(List<FeatureFunction> featureFunctions) {
+ if (!isSorted())
+ sortRules(featureFunctions);
+ return getRules();
+ }
+
+ @Override
+ public int[] getSourceSide() {
+ return src;
+ }
+
+ @Override
+ public int getArity() {
+ return arity;
+ }
+
+ @Override
+ public Iterator<Integer> getTerminalExtensionIterator() {
+ return new PackedChildIterator(position, true);
+ }
+
+ @Override
+ public Iterator<Integer> getNonterminalExtensionIterator() {
+ return new PackedChildIterator(position, false);
+ }
+
+ public final class PackedChildIterator implements Iterator<Integer> {
+
+ private int current;
+ private boolean terminal;
+ private boolean done;
+ private int last;
+
+ PackedChildIterator(int position, boolean terminal) {
+ this.terminal = terminal;
+ int num_children = source[position];
+ done = (num_children == 0);
+ if (!done) {
+ current = (terminal ? position + 1 : position - 1 + 2 * num_children);
+ last = (terminal ? position - 1 + 2 * num_children : position + 1);
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (done)
+ return false;
+ int next = (terminal ? current + 2 : current - 2);
+ if (next == last)
+ return false;
+ return (terminal ? source[next] > 0 : source[next] < 0);
+ }
+
+ @Override
+ public Integer next() {
+ if (done)
+ throw new RuntimeException("No more symbols!");
+ int symbol = source[current];
+ if (current == last)
+ done = true;
+ if (!done) {
+ current = (terminal ? current + 2 : current - 2);
+ done = (terminal ? source[current] < 0 : source[current] > 0);
+ }
+ return symbol;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ /**
+ * A packed phrase pair represents a rule of the form of a phrase pair, packed with the
+ * grammar-packer.pl script, which simply adds a nonterminal [X] to the left-hand side of
+ * all phrase pairs (and converts the Moses features). The packer then packs these. We have
+ * to then put a nonterminal on the source and target sides to treat the phrase pairs like
+ * left-branching rules, which is how Joshua deals with phrase decoding.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ *
+ */
+ public final class PackedPhrasePair extends PackedRule {
+
+ private final Supplier<int[]> englishSupplier;
+ private final Supplier<byte[]> alignmentSupplier;
+
+ public PackedPhrasePair(int address) {
+ super(address);
+ englishSupplier = initializeEnglishSupplier();
+ alignmentSupplier = initializeAlignmentSupplier();
+ }
+
+ @Override
+ public int getArity() {
+ return PackedTrie.this.getArity() + 1;
+ }
+
+ /**
+ * Initialize a number of suppliers which get evaluated when their respective getters
+ * are called.
+ * Inner lambda functions are guaranteed to only be called once, because of this underlying
+ * structures are accessed in a threadsafe way.
+ * Guava's implementation makes sure only one read of a volatile variable occurs per get.
+ * This means this implementation should be as thread-safe and performant as possible.
+ */
+
+ private Supplier<int[]> initializeEnglishSupplier(){
+ Supplier<int[]> result = Suppliers.memoize(() ->{
+ int[] phrase = getTarget(source[address + 1]);
+ int[] tgt = new int[phrase.length + 1];
+ tgt[0] = -1;
+ for (int i = 0; i < phrase.length; i++)
+ tgt[i+1] = phrase[i];
+ return tgt;
+ });
+ return result;
+ }
+
+ private Supplier<byte[]> initializeAlignmentSupplier(){
+ Supplier<byte[]> result = Suppliers.memoize(() ->{
+ byte[] raw_alignment = getAlignmentArray(source[address + 2]);
+ byte[] points = new byte[raw_alignment.length + 2];
+ points[0] = points[1] = 0;
+ for (int i = 0; i < raw_alignment.length; i++)
+ points[i + 2] = (byte) (raw_alignment[i] + 1);
+ return points;
+ });
+ return result;
+ }
+
+ /**
+ * Take the English phrase of the underlying rule and prepend an [X].
+ *
+ * @return the augmented phrase
+ */
+ @Override
+ public int[] getEnglish() {
+ return this.englishSupplier.get();
+ }
+
+ /**
+ * Take the French phrase of the underlying rule and prepend an [X].
+ *
+ * @return the augmented French phrase
+ */
+ @Override
+ public int[] getFrench() {
+ int phrase[] = new int[src.length + 1];
+ int ntid = Vocabulary.id(PackedGrammar.this.joshuaConfiguration.default_non_terminal);
+ phrase[0] = ntid;
+ System.arraycopy(src, 0, phrase, 1, src.length);
+ return phrase;
+ }
+
+ /**
+ * Similarly the alignment array needs to be shifted over by one.
+ *
+ * @return the byte[] alignment
+ */
+ @Override
+ public byte[] getAlignment() {
+ // if no alignments in grammar do not fail
+ if (alignments == null) {
+ return null;
+ }
+
+ return this.alignmentSupplier.get();
+ }
+ }
+
+ public class PackedRule extends Rule {
+ protected final int address;
+ private final Supplier<int[]> englishSupplier;
+ private final Supplier<FeatureVector> featureVectorSupplier;
+ private final Supplier<byte[]> alignmentsSupplier;
+
+ public PackedRule(int address) {
+ this.address = address;
+ this.englishSupplier = intializeEnglishSupplier();
+ this.featureVectorSupplier = initializeFeatureVectorSupplier();
+ this.alignmentsSupplier = initializeAlignmentsSupplier();
+ }
+
+ private Supplier<int[]> intializeEnglishSupplier(){
+ Supplier<int[]> result = Suppliers.memoize(() ->{
+ return getTarget(source[address + 1]);
+ });
+ return result;
+ }
+
+ private Supplier<FeatureVector> initializeFeatureVectorSupplier(){
+ Supplier<FeatureVector> result = Suppliers.memoize(() ->{
+ return loadFeatureVector(source[address + 2]);
+ });
+ return result;
+ }
+
+ private Supplier<byte[]> initializeAlignmentsSupplier(){
+ Supplier<byte[]> result = Suppliers.memoize(()->{
+ // if no alignments in grammar do not fail
+ if (alignments == null){
+ return null;
+ }
+ return getAlignmentArray(source[address + 2]);
+ });
+ return result;
+ }
+
+ @Override
+ public void setArity(int arity) {
+ }
+
+ @Override
+ public int getArity() {
+ return PackedTrie.this.getArity();
+ }
+
+ @Override
+ public void setOwner(int ow) {
+ }
+
+ @Override
+ public int getOwner() {
+ return owner;
+ }
+
+ @Override
+ public void setLHS(int lhs) {
+ }
+
+ @Override
+ public int getLHS() {
+ return source[address];
+ }
+
+ @Override
+ public void setEnglish(int[] eng) {
+ }
+
+ @Override
+ public int[] getEnglish() {
+ return this.englishSupplier.get();
+ }
+
+ @Override
+ public void setFrench(int[] french) {
+ }
+
+ @Override
+ public int[] getFrench() {
+ return src;
+ }
+
+ @Override
+ public FeatureVector getFeatureVector() {
+ return this.featureVectorSupplier.get();
+ }
+
+ @Override
+ public byte[] getAlignment() {
+ return this.alignmentsSupplier.get();
+ }
+
+ @Override
+ public String getAlignmentString() {
+ throw new RuntimeException("AlignmentString not implemented for PackedRule!");
+ }
+
+ @Override
+ public float getEstimatedCost() {
+ return estimated[source[address + 2]];
+ }
+
+// @Override
+// public void setPrecomputableCost(float cost) {
+// precomputable[source[address + 2]] = cost;
+// }
+
+ @Override
+ public float getPrecomputableCost() {
+ return precomputable[source[address + 2]];
+ }
+
+ @Override
+ public float estimateRuleCost(List<FeatureFunction> models) {
+ return estimated[source[address + 2]];
+ }
+
+ @Override
+ public String toString() {
+ StringBuffer sb = new StringBuffer();
+ sb.append(Vocabulary.word(this.getLHS()));
+ sb.append(" ||| ");
+ sb.append(getFrenchWords());
+ sb.append(" ||| ");
+ sb.append(getEnglishWords());
+ sb.append(" |||");
+ sb.append(" " + getFeatureVector());
+ sb.append(String.format(" ||| %.3f", getEstimatedCost()));
+ return sb.toString();
+ }
+ }
+ }
+ }
+
+ @Override
+ public void addOOVRules(int word, List<FeatureFunction> featureFunctions) {
+ throw new RuntimeException("PackedGrammar.addOOVRules(): I can't add OOV rules");
+ }
+
+ @Override
+ public void addRule(Rule rule) {
+ throw new RuntimeException("PackedGrammar.addRule(): I can't add rules");
+ }
+
+ /**
+ * Read the config file
+ *
+ * TODO: this should be rewritten using typeconfig.
+ *
+ * @param config
+ * @throws IOException
+ */
+ private void readConfig(String config) throws IOException {
+ int version = 0;
+
+ for (String line: new LineReader(config)) {
+ String[] tokens = line.split(" = ");
+ if (tokens[0].equals("max-source-len"))
+ this.maxSourcePhraseLength = Integer.parseInt(tokens[1]);
+ else if (tokens[0].equals("version")) {
+ version = Integer.parseInt(tokens[1]);
+ }
+ }
+
+ if (version != 3) {
+ String message = String.format("The grammar at %s was packed with packer version %d, but the earliest supported version is %d",
+ this.grammarDir, version, SUPPORTED_VERSION);
+ throw new RuntimeException(message);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/packed/SliceAggregatingTrie.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/packed/SliceAggregatingTrie.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/packed/SliceAggregatingTrie.java
new file mode 100644
index 0000000..c6d03a6
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/packed/SliceAggregatingTrie.java
@@ -0,0 +1,236 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.tm.packed;
+
+import static java.util.Collections.emptyList;
+import static java.util.Collections.unmodifiableList;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.joshua.decoder.ff.FeatureFunction;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.ff.tm.RuleCollection;
+import org.apache.joshua.decoder.ff.tm.Trie;
+import org.apache.joshua.decoder.ff.tm.hash_based.ExtensionIterator;
+
+/**
+ * <p>SliceAggregatingTrie collapses multiple tries
+ * with the same source root (i.e. tries from multiple packed slices).</p>
+ *
+ * <p>Consider the example below.
+ * Without SliceAggregatingTries, the following grammar rules could have only
+ * partitioned by splitting rule lists when the first word of SOURCE changes. (">" markers).</p>
+ *
+ * <p>Using a SliceAggregatingTrie allows splitting at changes of second SOURCE words (">>" marker).</p>
+ *
+ * <pre>
+ * EXAMPLE: (LHS ||| SOURCE ||| TARGET)
+ * [X] ||| - ||| -
+ * >
+ * [X] ||| [X] ||| [X]
+ * >>
+ * [X] ||| [X] a ||| [X] A
+ * [X] ||| [X] a ||| [X] A
+ * >>
+ * [X] ||| [X] b ||| [X] B
+ * >
+ * [X] ||| u ||| u
+ * </pre>
+ * <p>A SliceAggregatingTrie node behaves just like a regular Trie node but subsumes a list of extensions/children.
+ * This class hides the complexity of having multiple tries with the same root
+ * from nodes one level up.
+ * Similar to PackedRoot, it maintains a lookup table of children's
+ * source-side words to know
+ * in which subtrie (i.e. packedSlice) it needs to traverse into when
+ * match() is called.
+ * A SliceAggregatingTrie never holds any rules associated with it, thus
+ * rules with the source-side represented by the SliceAggregatingTrie node
+ * must be found in exactly one of the subtries.
+ * (!) This assumption relies on the sort order of the packed grammar.
+ * If the grammar was incorrectly sorted and then packed, construction
+ * of SliceAggregatingTrie nodes fails.</p>
+ *
+ * @author fhieber
+ */
+public class SliceAggregatingTrie implements Trie, RuleCollection {
+
+ /**
+ * A multitude of packedTries with the same source-side
+ * firstword. The order is induced by the
+ * sorting order of the text grammar that was input to the GrammarPacker.
+ * This implies that rules for the node represented by this SliceAggregatingTrie
+ * instance must be found in ONE of the sub tries.
+ * This is checked below in the constructor.
+ */
+ private final List<Trie> tries;
+ /** reference to the only subtrie that can contain rules. Set by buildLookupTable() */
+ private Trie trieWithRules = null;
+
+ /** Maintains an index of all children of all sub tries */
+ private final HashMap<Integer, Trie> lookup = new HashMap<>();
+
+ public SliceAggregatingTrie(final List<Trie> tries) {
+ if (tries == null || tries.isEmpty()) {
+ throw new RuntimeException(
+ "SliceAggregatingTrie node requires at least one packedTrie");
+ }
+ this.tries = unmodifiableList(tries);
+ buildLookupTable();
+ }
+
+ /**
+ * Fills the lookup table for child nodes.
+ * Also performs various checks to ensure correctness of the
+ * PackedTrie aggregation.
+ */
+ private void buildLookupTable() {
+ final Set<Integer> seen_child_ids = new HashSet<>();
+ Trie previous_trie = null;
+ boolean first = true;
+ for (final Trie trie : this.tries) {
+ /*
+ * perform some checks to make sure tries are correctly split.
+ */
+ if (!first) {
+ if (!haveSameSourceSide(previous_trie, trie) || !haveSameArity(previous_trie, trie)) {
+ throw new RuntimeException("SliceAggregatingTrie's subtries differ in sourceSide or arity. Was the text grammar sorted insufficiently?");
+ }
+ } else {
+ first = false;
+ }
+ previous_trie = trie;
+
+ if (trie.hasRules()) {
+ if (trieWithRules != null) {
+ throw new RuntimeException("SliceAggregatingTrie can only have one subtrie with rules. Was the text grammar sorted insufficiently?");
+ }
+ trieWithRules = trie;
+ }
+
+ final HashMap<Integer, ? extends Trie> children = trie.getChildren();
+ for (int id : children.keySet()) {
+ if (seen_child_ids.contains(id)) {
+ throw new RuntimeException("SliceAggregatingTrie's subtries contain non-disjoint child words. Was the text grammar sorted insufficiently?");
+ }
+ seen_child_ids.add(id);
+ lookup.put(id, children.get(id));
+ }
+ }
+ }
+
+ private boolean haveSameSourceSide(final Trie t1, final Trie t2) {
+ return Arrays.equals(
+ t1.getRuleCollection().getSourceSide(),
+ t2.getRuleCollection().getSourceSide());
+ }
+
+ private boolean haveSameArity(final Trie t1, final Trie t2) {
+ return t1.getRuleCollection().getArity() == t2.getRuleCollection().getArity();
+ }
+
+ @Override
+ public Trie match(int wordId) {
+ return lookup.get(wordId);
+ }
+
+ @Override
+ public boolean hasExtensions() {
+ return !lookup.isEmpty();
+ }
+
+ @Override
+ public Collection<? extends Trie> getExtensions() {
+ return new ArrayList<>(lookup.values());
+ }
+
+ @Override
+ public HashMap<Integer, ? extends Trie> getChildren() {
+ return lookup;
+ }
+
+ @Override
+ public Iterator<Integer> getTerminalExtensionIterator() {
+ return new ExtensionIterator(lookup, true);
+ }
+
+ @Override
+ public Iterator<Integer> getNonterminalExtensionIterator() {
+ return new ExtensionIterator(lookup, true);
+ }
+
+ @Override
+ public RuleCollection getRuleCollection() {
+ return this;
+ }
+
+ /*
+ * The following method's return values depend on whether there is
+ * a single subtrie encoding rules (trieWithRules).
+ * All other subtries can only contain rules some levels deeper.
+ */
+
+ @Override
+ public boolean hasRules() {
+ return trieWithRules == null ? false : trieWithRules.hasRules();
+ }
+
+ @Override
+ public List<Rule> getRules() {
+ if (!hasRules()) {
+ return emptyList();
+ }
+ return trieWithRules.getRuleCollection().getRules();
+ }
+
+ @Override
+ public List<Rule> getSortedRules(List<FeatureFunction> models) {
+ if (!hasRules()) {
+ return emptyList();
+ }
+ return trieWithRules.getRuleCollection().getSortedRules(models);
+ }
+
+ @Override
+ public boolean isSorted() {
+ return !hasRules() ? false : trieWithRules.getRuleCollection().isSorted();
+ }
+
+ /*
+ * The constructor checked that all sub tries have the same arity and sourceSide.
+ * We can thus simply return the value from the first in list.
+ */
+
+ @Override
+ public int[] getSourceSide() {
+ return tries.get(0).getRuleCollection().getSourceSide();
+ }
+
+ @Override
+ public int getArity() {
+ return tries.get(0).getRuleCollection().getArity();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/AlignedSourceTokens.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/AlignedSourceTokens.java b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/AlignedSourceTokens.java
new file mode 100644
index 0000000..864b383
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/AlignedSourceTokens.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.hypergraph;
+
+import java.util.LinkedList;
+import java.util.ListIterator;
+
+/**
+ * Class that represents a one to (possibly) many alignment from target to
+ * source. Extends from a LinkedList. Instances of this class are updated by the
+ * WordAlignmentExtractor.substitute() method.
+ * The {@link org.apache.joshua.decoder.hypergraph.AlignedSourceTokens#shiftBy(int, int)}
+ * method shifts the
+ * elements in the list by a scalar to reflect substitutions of non terminals in
+ * the rule. if indexes are final, i.e. the point instance has been substituted
+ * into a parent WordAlignmentState once,
+ * {@link org.apache.joshua.decoder.hypergraph.AlignedSourceTokens#isFinal} is set to true.
+ * This is
+ * necessary since the final source index of a point is known once we have
+ * substituted in a complete WordAlignmentState into its parent. If the index in
+ * the list is a non terminal, {@link org.apache.joshua.decoder.hypergraph.AlignedSourceTokens#isNonTerminal} = true
+ */
+class AlignedSourceTokens extends LinkedList<Integer> {
+
+ private static final long serialVersionUID = 1L;
+ /** whether this Point refers to a non terminal in source&target */
+ private boolean isNonTerminal = false;
+ /** whether this instance does not need to be updated anymore */
+ private boolean isFinal = false;
+ /** whether the word this Point corresponds to has no alignment in source */
+ private boolean isNull = false;
+
+ AlignedSourceTokens() {}
+
+ void setFinal() {
+ isFinal = true;
+ }
+
+ void setNonTerminal() {
+ isNonTerminal = true;
+ }
+
+ void setNull() {
+ isNull = true;
+ }
+
+ @Override
+ /**
+ * returns true if element was added.
+ */
+ public boolean add(Integer x) {
+ return isNull ? false : super.add(x);
+ }
+
+ public boolean isNonTerminal() {
+ return isNonTerminal;
+ }
+
+ public boolean isFinal() {
+ return isFinal;
+ }
+
+ public boolean isNull() {
+ return isNull;
+ }
+
+ /**
+ * shifts each item in the LinkedList by <shift>.
+ * Only applies to items larger than <start>
+ */
+ void shiftBy(int start, int shift) {
+ if (!isFinal && !isNull) {
+ final ListIterator<Integer> it = this.listIterator();
+ while (it.hasNext()) {
+ final int x = it.next();
+ if (x > start) {
+ it.set(x + shift);
+ }
+ }
+ }
+ }
+
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ if (isFinal)
+ sb.append("f");
+ if (isNull) {
+ sb.append("[NULL]");
+ } else {
+ sb.append(super.toString());
+ }
+ if (isNonTerminal)
+ sb.append("^");
+ return sb.toString();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/AllSpansWalker.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/AllSpansWalker.java b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/AllSpansWalker.java
new file mode 100644
index 0000000..1aad06f
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/AllSpansWalker.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.hypergraph;
+
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.joshua.corpus.Span;
+
+/***
+ * Uses {@link ForestWalker} to visit one {@link HGNode} per span of the chart. No guarantees are
+ * provided as to which HGNode will be visited in each span.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ *
+ */
+
+public class AllSpansWalker {
+ private Set<Span> visitedSpans;
+
+ public AllSpansWalker() {
+ visitedSpans = new HashSet<Span>();
+ }
+
+ /**
+ * This function wraps a {@link ForestWalker}, preventing calls to its walker function for all but
+ * the first node reached for each span.
+ *
+ * @param node the {@link org.apache.joshua.decoder.hypergraph.HGNode} we wish to walk
+ * @param walker the {@link org.apache.joshua.decoder.hypergraph.WalkerFunction}
+ * implementation to do the walking
+ */
+ public void walk(HGNode node, final WalkerFunction walker) {
+ new ForestWalker().walk(node, new org.apache.joshua.decoder.hypergraph.WalkerFunction() {
+ @Override
+ public void apply(HGNode node, int index) {
+ if (node != null) {
+ Span span = new Span(node.i, node.j);
+ if (!visitedSpans.contains(span)) {
+ walker.apply(node, 0);
+ visitedSpans.add(span);
+ }
+ }
+ }
+ });
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/DefaultInsideOutside.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/DefaultInsideOutside.java b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/DefaultInsideOutside.java
new file mode 100644
index 0000000..c6dae77
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/DefaultInsideOutside.java
@@ -0,0 +1,407 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.hypergraph;
+
+import java.util.HashMap;
+
+
+/**
+ * to use the functions here, one need to extend the class to provide a way to calculate the
+ * transitionLogP based on feature set
+ *
+ * @author Zhifei Li, zhifei.work@gmail.com
+ * @version $LastChangedDate$
+ */
+
+// TODO: currently assume log semiring, need to generalize to other semiring
+// already implement both max-product and sum-product algortithms for log-semiring
+// Note: this class requires the correctness of transitionLogP of each hyperedge, which itself may
+// require the correctness of bestDerivationLogP at each item
+
+public abstract class DefaultInsideOutside {
+ /**
+ * Two operations: add and multi add: different hyperedges lead to a specific item multi: prob of
+ * a derivation is a multi of all constituents
+ */
+ int ADD_MODE = 0; // 0: sum; 1: viterbi-min, 2: viterbi-max
+ int LOG_SEMIRING = 1;
+ int SEMIRING = LOG_SEMIRING; // default is in log; or real, or logic
+ double ZERO_IN_SEMIRING = Double.NEGATIVE_INFINITY;// log-domain
+ double ONE_IN_SEMIRING = 0;// log-domain
+ double scaling_factor; // try to scale the original distribution: smooth or winner-take-all
+
+ private HashMap<HGNode, Double> tbl_inside_prob = new HashMap<HGNode, Double>();// remember inside
+ // prob of each
+ // item:
+ private HashMap<HGNode, Double> tbl_outside_prob = new HashMap<HGNode, Double>();// remember
+ // outside prob
+ // of each item
+ double normalizationConstant = ONE_IN_SEMIRING;
+
+ /**
+ * for each item, remember how many deductions pointering to me, this is needed for outside
+ * estimation during outside estimation, an item will recursive call its deductions to do
+ * outside-estimation only after it itself is done with outside estimation, this is necessary
+ * because the outside estimation of the items under its deductions require the item's outside
+ * value
+ */
+ private HashMap<HGNode, Integer> tbl_num_parent_deductions = new HashMap<HGNode, Integer>();
+
+ private HashMap<HGNode, Integer> tbl_for_sanity_check = null;
+
+ // get feature-set specific **log probability** for each hyperedge
+ protected abstract double getHyperedgeLogProb(HyperEdge dt, HGNode parent_it);
+
+ protected double getHyperedgeLogProb(HyperEdge dt, HGNode parent_it, double scaling_factor) {
+ return getHyperedgeLogProb(dt, parent_it) * scaling_factor;
+ }
+
+ // the results are stored in tbl_inside_prob and tbl_outside_prob
+ public void runInsideOutside(HyperGraph hg, int add_mode, int semiring, double scaling_factor_) {// add_mode|||
+ // 0:
+ // sum;
+ // 1:
+ // viterbi-min,
+ // 2:
+ // viterbi-max
+
+ setup_semiring(semiring, add_mode);
+ scaling_factor = scaling_factor_;
+
+ // System.out.println("outside estimation");
+ inside_estimation_hg(hg);
+ // System.out.println("inside estimation");
+ outside_estimation_hg(hg);
+ normalizationConstant = tbl_inside_prob.get(hg.goalNode);
+ System.out.println("normalization constant is " + normalizationConstant);
+ tbl_num_parent_deductions.clear();
+ sanityCheckHG(hg);
+ }
+
+ // to save memory, external class should call this method
+ public void clearState() {
+ tbl_num_parent_deductions.clear();
+ tbl_inside_prob.clear();
+ tbl_outside_prob.clear();
+ }
+
+ // ######### use of inside-outside probs ##########################
+ // this is the logZ where Z is the sum[ exp( log prob ) ]
+ public double getLogNormalizationConstant() {
+ return normalizationConstant;
+ }
+
+ // this is the log of expected/posterior prob (i.e., LogP, where P is the posterior probability),
+ // without normalization
+ public double getEdgeUnormalizedPosteriorLogProb(HyperEdge dt, HGNode parent) {
+ // ### outside of parent
+ double outside = (Double) tbl_outside_prob.get(parent);
+
+ // ### get inside prob of all my ant-items
+ double inside = ONE_IN_SEMIRING;
+ if (dt.getTailNodes() != null) {
+ for (HGNode ant_it : dt.getTailNodes())
+ inside = multi_in_semiring(inside, (Double) tbl_inside_prob.get(ant_it));
+ }
+
+ // ### add deduction/rule specific prob
+ double merit = multi_in_semiring(inside, outside);
+ merit = multi_in_semiring(merit, getHyperedgeLogProb(dt, parent, this.scaling_factor));
+
+ return merit;
+ }
+
+ // normalized probabily in [0,1]
+ public double getEdgePosteriorProb(HyperEdge dt, HGNode parent) {
+ if (SEMIRING == LOG_SEMIRING) {
+ double res =
+ Math.exp((getEdgeUnormalizedPosteriorLogProb(dt, parent) - getLogNormalizationConstant()));
+ if (res < 0.0 - 1e-2 || res > 1.0 + 1e-2) {
+ throw new RuntimeException("res is not within [0,1], must be wrong value: " + res);
+ }
+ return res;
+ } else {
+ throw new RuntimeException("not implemented");
+ }
+ }
+
+ // this is the log of expected/posterior prob (i.e., LogP, where P is the posterior probability),
+ // without normalization
+ public double getNodeUnnormalizedPosteriorLogProb(HGNode node) {
+ // ### outside of parent
+ double inside = (Double) tbl_inside_prob.get(node);
+ double outside = (Double) tbl_outside_prob.get(node);
+ return multi_in_semiring(inside, outside);
+ }
+
+
+ // normalized probabily in [0,1]
+ public double getNodePosteriorProb(HGNode node) {
+ if (SEMIRING == LOG_SEMIRING) {
+ double res =
+ Math.exp((getNodeUnnormalizedPosteriorLogProb(node) - getLogNormalizationConstant()));
+ if (res < 0.0 - 1e-2 || res > 1.0 + 1e-2) {
+ throw new RuntimeException("res is not within [0,1], must be wrong value: " + res);
+ }
+ return res;
+ } else {
+ throw new RuntimeException("not implemented");
+ }
+ }
+
+ /*
+ * Originally, to see if the sum of the posterior probabilities of all the hyperedges sum to one
+ * However, this won't work! The sum should be greater than 1.
+ */
+ public void sanityCheckHG(HyperGraph hg) {
+ tbl_for_sanity_check = new HashMap<HGNode, Integer>();
+ // System.out.println("num_dts: " + hg.goal_item.l_deductions.size());
+ sanity_check_item(hg.goalNode);
+ System.out.println("survied sanity check!!!!");
+ }
+
+ private void sanity_check_item(HGNode it) {
+ if (tbl_for_sanity_check.containsKey(it)) return;
+ tbl_for_sanity_check.put(it, 1);
+ double prob_sum = 0;
+ // ### recursive call on each deduction
+ for (HyperEdge dt : it.hyperedges) {
+ prob_sum += getEdgePosteriorProb(dt, it);
+ sanity_check_deduction(dt);// deduction-specifc operation
+ }
+ double supposed_sum = getNodePosteriorProb(it);
+ if (Math.abs(prob_sum - supposed_sum) > 1e-3) {
+ throw new RuntimeException("prob_sum=" + prob_sum + "; supposed_sum=" + supposed_sum
+ + "; sanity check fail!!!!");
+ }
+ // ### item-specific operation
+ }
+
+ private void sanity_check_deduction(HyperEdge dt) {
+ // ### recursive call on each ant item
+ if (null != dt.getTailNodes()) {
+ for (HGNode ant_it : dt.getTailNodes()) {
+ sanity_check_item(ant_it);
+ }
+ }
+
+ // ### deduction-specific operation
+
+ }
+
+ // ################## end use of inside-outside probs
+
+
+
+ // ############ bottomn-up insdide estimation ##########################
+ private void inside_estimation_hg(HyperGraph hg) {
+ tbl_inside_prob.clear();
+ tbl_num_parent_deductions.clear();
+ inside_estimation_item(hg.goalNode);
+ }
+
+ private double inside_estimation_item(HGNode it) {
+ // ### get number of deductions that point to me
+ Integer num_called = (Integer) tbl_num_parent_deductions.get(it);
+ if (null == num_called) {
+ tbl_num_parent_deductions.put(it, 1);
+ } else {
+ tbl_num_parent_deductions.put(it, num_called + 1);
+ }
+
+ if (tbl_inside_prob.containsKey(it)) {
+ return (Double) tbl_inside_prob.get(it);
+ }
+ double inside_prob = ZERO_IN_SEMIRING;
+
+ // ### recursive call on each deduction
+ for (HyperEdge dt : it.hyperedges) {
+ double v_dt = inside_estimation_deduction(dt, it);// deduction-specifc operation
+ inside_prob = add_in_semiring(inside_prob, v_dt);
+ }
+ // ### item-specific operation, but all the prob should be factored into each deduction
+
+ tbl_inside_prob.put(it, inside_prob);
+ return inside_prob;
+ }
+
+ private double inside_estimation_deduction(HyperEdge dt, HGNode parent_item) {
+ double inside_prob = ONE_IN_SEMIRING;
+ // ### recursive call on each ant item
+ if (dt.getTailNodes() != null) for (HGNode ant_it : dt.getTailNodes()) {
+ double v_item = inside_estimation_item(ant_it);
+ inside_prob = multi_in_semiring(inside_prob, v_item);
+ }
+
+ // ### deduction operation
+ double deduct_prob = getHyperedgeLogProb(dt, parent_item, this.scaling_factor);// feature-set
+ // specific
+ inside_prob = multi_in_semiring(inside_prob, deduct_prob);
+ return inside_prob;
+ }
+
+ // ########### end inside estimation
+
+ // ############ top-downn outside estimation ##########################
+
+ private void outside_estimation_hg(HyperGraph hg) {
+ tbl_outside_prob.clear();
+ tbl_outside_prob.put(hg.goalNode, ONE_IN_SEMIRING);// initialize
+ for (HyperEdge dt : hg.goalNode.hyperedges)
+ outside_estimation_deduction(dt, hg.goalNode);
+ }
+
+ private void outside_estimation_item(HGNode cur_it, HGNode upper_item, HyperEdge parent_dt,
+ double parent_deduct_prob) {
+ Integer num_called = (Integer) tbl_num_parent_deductions.get(cur_it);
+ if (null == num_called || 0 == num_called) {
+ throw new RuntimeException("un-expected call, must be wrong");
+ }
+ tbl_num_parent_deductions.put(cur_it, num_called - 1);
+
+ double old_outside_prob = ZERO_IN_SEMIRING;
+ if (tbl_outside_prob.containsKey(cur_it)) {
+ old_outside_prob = (Double) tbl_outside_prob.get(cur_it);
+ }
+
+ double additional_outside_prob = ONE_IN_SEMIRING;
+
+ // ### add parent deduction prob
+ additional_outside_prob = multi_in_semiring(additional_outside_prob, parent_deduct_prob);
+
+ // ### sibing specifc
+ if (parent_dt.getTailNodes() != null && parent_dt.getTailNodes().size() > 1)
+ for (HGNode ant_it : parent_dt.getTailNodes()) {
+ if (ant_it != cur_it) {
+ double inside_prob_item = (Double) tbl_inside_prob.get(ant_it);// inside prob
+ additional_outside_prob = multi_in_semiring(additional_outside_prob, inside_prob_item);
+ }
+ }
+
+ // ### upper item
+ double outside_prob_item = (Double) tbl_outside_prob.get(upper_item);// outside prob
+ additional_outside_prob = multi_in_semiring(additional_outside_prob, outside_prob_item);
+
+ // #### add to old prob
+ additional_outside_prob = add_in_semiring(additional_outside_prob, old_outside_prob);
+
+ tbl_outside_prob.put(cur_it, additional_outside_prob);
+
+ // ### recursive call on each deduction
+ if (num_called - 1 <= 0) {// i am done
+ for (HyperEdge dt : cur_it.hyperedges) {
+ // TODO: potentially, we can collect the feature expection in each hyperedge here, to avoid
+ // another pass of the hypergraph to get the counts
+ outside_estimation_deduction(dt, cur_it);
+ }
+ }
+ }
+
+
+ private void outside_estimation_deduction(HyperEdge dt, HGNode parent_item) {
+ // we do not need to outside prob if no ant items
+ if (dt.getTailNodes() != null) {
+ // ### deduction specific prob
+ double deduction_prob = getHyperedgeLogProb(dt, parent_item, this.scaling_factor);// feature-set
+ // specific
+
+ // ### recursive call on each ant item
+ for (HGNode ant_it : dt.getTailNodes()) {
+ outside_estimation_item(ant_it, parent_item, dt, deduction_prob);
+ }
+ }
+ }
+
+ // ########### end outside estimation
+
+
+
+ // ############ common ##########################
+ // BUG: replace integer pseudo-enum with a real Java enum
+ // BUG: use a Semiring class instead of all this?
+ private void setup_semiring(int semiring, int add_mode) {
+ ADD_MODE = add_mode;
+ SEMIRING = semiring;
+ if (SEMIRING == LOG_SEMIRING) {
+ if (ADD_MODE == 0) { // sum
+ ZERO_IN_SEMIRING = Double.NEGATIVE_INFINITY;
+ ONE_IN_SEMIRING = 0;
+ } else if (ADD_MODE == 1) { // viter-min
+ ZERO_IN_SEMIRING = Double.POSITIVE_INFINITY;
+ ONE_IN_SEMIRING = 0;
+ } else if (ADD_MODE == 2) { // viter-max
+ ZERO_IN_SEMIRING = Double.NEGATIVE_INFINITY;
+ ONE_IN_SEMIRING = 0;
+ } else {
+ throw new RuntimeException("invalid add mode");
+ }
+ } else {
+ throw new RuntimeException("un-supported semiring");
+ }
+ }
+
+ private double multi_in_semiring(double x, double y) {
+ if (SEMIRING == LOG_SEMIRING) {
+ return multi_in_log_semiring(x, y);
+ } else {
+ throw new RuntimeException("un-supported semiring");
+ }
+ }
+
+ private double add_in_semiring(double x, double y) {
+ if (SEMIRING == LOG_SEMIRING) {
+ return add_in_log_semiring(x, y);
+ } else {
+ throw new RuntimeException("un-supported semiring");
+ }
+ }
+
+ // AND
+ private double multi_in_log_semiring(double x, double y) { // value is Log prob
+ return x + y;
+ }
+
+
+ // OR: return Math.log(Math.exp(x) + Math.exp(y));
+ // BUG: Replace ADD_MODE pseudo-enum with a real Java enum
+ private double add_in_log_semiring(double x, double y) { // prevent under-flow
+ if (ADD_MODE == 0) { // sum
+ if (x == Double.NEGATIVE_INFINITY) { // if y is also n-infinity, then return n-infinity
+ return y;
+ }
+ if (y == Double.NEGATIVE_INFINITY) {
+ return x;
+ }
+
+ if (y <= x) {
+ return x + Math.log(1 + Math.exp(y - x));
+ } else {
+ return y + Math.log(1 + Math.exp(x - y));
+ }
+ } else if (ADD_MODE == 1) { // viter-min
+ return (x <= y ? x : y);
+ } else if (ADD_MODE == 2) { // viter-max
+ return (x >= y ? x : y);
+ } else {
+ throw new RuntimeException("invalid add mode");
+ }
+ }
+ // ############ end common #####################
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/FeatureVectorExtractor.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/FeatureVectorExtractor.java b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/FeatureVectorExtractor.java
new file mode 100644
index 0000000..a8525be
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/FeatureVectorExtractor.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.hypergraph;
+
+import static org.apache.joshua.decoder.chart_parser.ComputeNodeResult.computeTransitionFeatures;
+
+import java.util.List;
+
+import org.apache.joshua.decoder.ff.FeatureFunction;
+import org.apache.joshua.decoder.ff.FeatureVector;
+import org.apache.joshua.decoder.hypergraph.KBestExtractor.DerivationState;
+import org.apache.joshua.decoder.hypergraph.KBestExtractor.DerivationVisitor;
+import org.apache.joshua.decoder.segment_file.Sentence;
+
+/**
+ * During decoding, individual features values are not stored, only the model score on each edge.
+ * This saves space. If you want to print the actual feature values, they have to be assembled
+ * from the edges of the derivation, which means replaying the feature functions. This visitor
+ * does just that, using the generic derivation visitor.
+ */
+public class FeatureVectorExtractor implements WalkerFunction, DerivationVisitor {
+
+ private final FeatureVector features;
+ private final List<FeatureFunction> featureFunctions;
+ private final Sentence sourceSentence;
+
+ public FeatureVectorExtractor(
+ final List<FeatureFunction> featureFunctions,
+ final Sentence sourceSentence) {
+ this.features = new FeatureVector();
+ this.featureFunctions = featureFunctions;
+ this.sourceSentence = sourceSentence;
+ }
+
+ /** Accumulate edge features from Viterbi path */
+ @Override
+ public void apply(HGNode node, int nodeIndex) {
+ features.add(
+ computeTransitionFeatures(
+ featureFunctions,
+ node.bestHyperedge,
+ node.i, node.j,
+ sourceSentence));
+ }
+
+ /** Accumulate edge features for that DerivationState */
+ @Override
+ public void before(DerivationState state, int level, int tailNodeIndex) {
+ features.add(
+ computeTransitionFeatures(
+ featureFunctions,
+ state.edge,
+ state.parentNode.i, state.parentNode.j,
+ sourceSentence));
+ }
+
+ /** Nothing to do */
+ @Override
+ public void after(DerivationState state, int level, int tailNodeIndex) {}
+
+ public FeatureVector getFeatures() {
+ return features;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/ForestWalker.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/ForestWalker.java b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/ForestWalker.java
new file mode 100644
index 0000000..e58670a
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/ForestWalker.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.hypergraph;
+
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * @author Matt Post <po...@cs.jhu.edu>
+ */
+
+/**
+ * This class visits every node in a forest using a depth-first, preorder traversal, applying the
+ * WalkerFunction to each node. It would be easy to add other traversals if the demand arose.
+ */
+public class ForestWalker {
+
+ public static enum TRAVERSAL {
+ PREORDER, POSTORDER
+ };
+
+ private Set<HGNode> visitedNodes;
+ private TRAVERSAL traversalType = TRAVERSAL.PREORDER;
+
+ public ForestWalker() {
+ visitedNodes = new HashSet<HGNode>();
+ }
+
+ public ForestWalker(TRAVERSAL traversal) {
+ this.traversalType = traversal;
+ visitedNodes = new HashSet<HGNode>();
+ }
+
+ public void walk(HGNode node, WalkerFunction walker) {
+ walk(node, walker, 0);
+ }
+
+ private void walk(HGNode node, WalkerFunction walker, int nodeIndex) {
+ // short circuit
+ if (visitedNodes.contains(node))
+ return;
+
+ visitedNodes.add(node);
+
+ if (this.traversalType == TRAVERSAL.PREORDER)
+ walker.apply(node, 0);
+
+ if (node.getHyperEdges() != null) {
+ for (HyperEdge edge : node.getHyperEdges()) {
+ if (edge.getTailNodes() != null) {
+ int tailNodeIndex = 0;
+ for (HGNode tailNode : edge.getTailNodes()) {
+ walk(tailNode, walker, tailNodeIndex);
+ tailNodeIndex++;
+ }
+ }
+ }
+ }
+
+ if (this.traversalType == TRAVERSAL.POSTORDER)
+ walker.apply(node, nodeIndex);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/GrammarBuilderWalkerFunction.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/GrammarBuilderWalkerFunction.java b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/GrammarBuilderWalkerFunction.java
new file mode 100644
index 0000000..a6edddd
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/GrammarBuilderWalkerFunction.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.hypergraph;
+
+import java.io.PrintStream;
+import java.util.HashSet;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.ff.tm.Grammar;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.ff.tm.format.HieroFormatReader;
+import org.apache.joshua.decoder.ff.tm.hash_based.MemoryBasedBatchGrammar;
+import org.apache.joshua.util.FormatUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This walker function builds up a new context-free grammar by visiting each node in a hypergraph.
+ * For a quick overview, see Chris Dyer's 2010 NAACL paper
+ * "Two monlingual parses are better than one (synchronous parse)".
+ * <p>
+ * From a functional-programming point of view, this walker really wants to calculate a fold over
+ * the entire hypergraph: the initial value is an empty grammar, and as we visit each node, we add
+ * more rules to the grammar. After we have traversed the whole hypergraph, the resulting grammar
+ * will contain all rules needed for synchronous parsing.
+ * <p>
+ * These rules look just like the rules already present in the hypergraph, except that each
+ * non-terminal symbol is annotated with the span of its node.
+ */
+public class GrammarBuilderWalkerFunction implements WalkerFunction {
+
+ private static final Logger LOG = LoggerFactory.getLogger(GrammarBuilderWalkerFunction.class);
+
+ private MemoryBasedBatchGrammar grammar;
+ private static HieroFormatReader reader = new HieroFormatReader();
+ private PrintStream outStream;
+ private int goalSymbol;
+ private HashSet<Rule> rules;
+
+ public GrammarBuilderWalkerFunction(String goal,JoshuaConfiguration joshuaConfiguration) {
+ grammar = new MemoryBasedBatchGrammar(reader,joshuaConfiguration);
+ grammar.setSpanLimit(1000);
+ outStream = null;
+ goalSymbol = Vocabulary.id(goal);
+ rules = new HashSet<Rule>();
+ }
+
+ public GrammarBuilderWalkerFunction(String goal, PrintStream out,JoshuaConfiguration joshuaConfiguration) {
+ this(goal,joshuaConfiguration);
+ outStream = out;
+ }
+
+ public void apply(HGNode node, int index) {
+ // System.err.printf("VISITING NODE: %s\n", getLabelWithSpan(node));
+ for (HyperEdge e : node.hyperedges) {
+ Rule r = getRuleWithSpans(e, node);
+ if (r != null && !rules.contains(r)) {
+ if (outStream != null) outStream.println(r);
+ grammar.addRule(r);
+ rules.add(r);
+ }
+ }
+ }
+
+ private static int getLabelWithSpan(HGNode node) {
+ return Vocabulary.id(getLabelWithSpanAsString(node));
+ }
+
+ private static String getLabelWithSpanAsString(HGNode node) {
+ String label = Vocabulary.word(node.lhs);
+ String unBracketedCleanLabel = label.substring(1, label.length() - 1);
+ return String.format("[%d-%s-%d]", node.i, unBracketedCleanLabel, node.j);
+ }
+
+ private boolean nodeHasGoalSymbol(HGNode node) {
+ return node.lhs == goalSymbol;
+ }
+
+ private Rule getRuleWithSpans(HyperEdge edge, HGNode head) {
+ Rule edgeRule = edge.getRule();
+ int headLabel = getLabelWithSpan(head);
+ // System.err.printf("Head label: %s\n", headLabel);
+ // if (edge.getAntNodes() != null) {
+ // for (HGNode n : edge.getAntNodes())
+ // System.err.printf("> %s\n", getLabelWithSpan(n));
+ // }
+ int[] source = getNewSource(nodeHasGoalSymbol(head), edge);
+ // if this would be unary abstract, getNewSource will be null
+ if (source == null) return null;
+ int[] target = getNewTargetFromSource(source);
+ Rule result =
+ new Rule(headLabel, source, target, edgeRule.getFeatureString(), edgeRule.getArity());
+ // System.err.printf("new rule is %s\n", result);
+ return result;
+ }
+
+ private static int[] getNewSource(boolean isGlue, HyperEdge edge) {
+ Rule rule = edge.getRule();
+ int[] english = rule.getEnglish();
+ // if this is a unary abstract rule, just return null
+ // TODO: except glue rules!
+ if (english.length == 1 && english[0] < 0 && !isGlue) return null;
+ int[] result = new int[english.length];
+ for (int i = 0; i < english.length; i++) {
+ int curr = english[i];
+ if (! FormatUtils.isNonterminal(curr)) {
+ // If it's a terminal symbol, we just copy it into the new rule.
+ result[i] = curr;
+ } else {
+ // If it's a nonterminal, its value is -N, where N is the index
+ // of the nonterminal on the source side.
+ //
+ // That is, if we would call a nonterminal "[X,2]", the value of
+ // curr at this point is -2. And the tail node that it points at
+ // is #1 (since getTailNodes() is 0-indexed).
+ int index = -curr - 1;
+ result[i] = getLabelWithSpan(edge.getTailNodes().get(index));
+ }
+ }
+ // System.err.printf("source: %s\n", result);
+ return result;
+ }
+
+ private static int[] getNewTargetFromSource(int[] source) {
+ int[] result = new int[source.length];
+ int currNT = -1; // value to stick into NT slots
+ for (int i = 0; i < source.length; i++) {
+ result[i] = source[i];
+ if (FormatUtils.isNonterminal(result[i])) {
+ result[i] = currNT;
+ currNT--;
+ }
+ }
+ // System.err.printf("target: %s\n", result);
+ return result;
+ }
+
+ private static HGNode getGoalSymbolNode(HGNode root) {
+ if (root.hyperedges == null || root.hyperedges.size() == 0) {
+ LOG.error("getGoalSymbolNode: root node has no hyperedges");
+ return null;
+ }
+ return root.hyperedges.get(0).getTailNodes().get(0);
+ }
+
+
+ public static int goalSymbol(HyperGraph hg) {
+ if (hg.goalNode == null) {
+ LOG.error("goalSymbol: goalNode of hypergraph is null");
+ return -1;
+ }
+ HGNode symbolNode = getGoalSymbolNode(hg.goalNode);
+ if (symbolNode == null) return -1;
+ // System.err.printf("goalSymbol: %s\n", result);
+ // System.err.printf("symbol node LHS is %d\n", symbolNode.lhs);
+ // System.err.printf("i = %d, j = %d\n", symbolNode.i, symbolNode.j);
+ return getLabelWithSpan(symbolNode);
+ }
+
+ public Grammar getGrammar() {
+ return grammar;
+ }
+}
[40/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/TargetBigram.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/TargetBigram.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/TargetBigram.java
new file mode 100644
index 0000000..e7de1f8
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/TargetBigram.java
@@ -0,0 +1,216 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.chart_parser.SourcePath;
+import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+import org.apache.joshua.decoder.ff.state_maintenance.NgramDPState;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.hypergraph.HGNode;
+import org.apache.joshua.decoder.segment_file.Sentence;
+import org.apache.joshua.util.FormatUtils;
+import org.apache.joshua.util.io.LineReader;
+
+/***
+ * The RuleBigram feature is an indicator feature that counts target word bigrams that are created when
+ * a rule is applied. It accepts three parameters:
+ *
+ * -vocab /path/to/vocab
+ *
+ * The path to a vocabulary, where each line is of the format ID WORD COUNT.
+ *
+ * -threshold N
+ *
+ * Mask to UNK all words whose COUNT is less than N.
+ *
+ * -top-n N
+ *
+ * Only use the top N words.
+ */
+
+public class TargetBigram extends StatefulFF {
+
+ private HashSet<String> vocab = null;
+ private int maxTerms = 1000000;
+ private int threshold = 0;
+
+ public TargetBigram(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, "TargetBigram", args, config);
+
+ if (parsedArgs.containsKey("threshold"))
+ threshold = Integer.parseInt(parsedArgs.get("threshold"));
+
+ if (parsedArgs.containsKey("top-n"))
+ maxTerms = Integer.parseInt(parsedArgs.get("top-n"));
+
+ if (parsedArgs.containsKey("vocab")) {
+ loadVocab(parsedArgs.get("vocab"));
+ }
+ }
+
+ /**
+ * Load vocabulary items passing the 'threshold' and 'top-n' filters.
+ *
+ * @param filename
+ */
+ private void loadVocab(String filename) {
+ this.vocab = new HashSet<String>();
+ this.vocab.add("<s>");
+ this.vocab.add("</s>");
+ try {
+ LineReader lineReader = new LineReader(filename);
+ for (String line: lineReader) {
+ if (lineReader.lineno() > maxTerms)
+ break;
+
+ String[] tokens = line.split("\\s+");
+ String word = tokens[1];
+ int count = Integer.parseInt(tokens[2]);
+
+ if (count >= threshold)
+ vocab.add(word);
+ }
+
+ } catch (IOException e) {
+ throw new RuntimeException(String.format(
+ "* FATAL: couldn't load TargetBigram vocabulary '%s'", filename), e);
+ }
+ }
+
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int spanStart, int spanEnd,
+ SourcePath sourcePath, Sentence sentence, Accumulator acc) {
+
+ int[] enWords = rule.getEnglish();
+
+ int left = -1;
+ int right = -1;
+
+ List<String> currentNgram = new LinkedList<String>();
+ for (int c = 0; c < enWords.length; c++) {
+ int curID = enWords[c];
+
+ if (FormatUtils.isNonterminal(curID)) {
+ int index = -(curID + 1);
+ NgramDPState state = (NgramDPState) tailNodes.get(index).getDPState(stateIndex);
+ int[] leftContext = state.getLeftLMStateWords();
+ int[] rightContext = state.getRightLMStateWords();
+
+ // Left context.
+ for (int token : leftContext) {
+ currentNgram.add(getWord(token));
+ if (left == -1)
+ left = token;
+ right = token;
+ if (currentNgram.size() == 2) {
+ String ngram = join(currentNgram);
+ acc.add(String.format("%s_%s", name, ngram), 1);
+ // System.err.println(String.format("ADDING %s_%s", name, ngram));
+ currentNgram.remove(0);
+ }
+ }
+ // Replace right context.
+ int tSize = currentNgram.size();
+ for (int i = 0; i < rightContext.length; i++)
+ currentNgram.set(tSize - rightContext.length + i, getWord(rightContext[i]));
+
+ } else { // terminal words
+ currentNgram.add(getWord(curID));
+ if (left == -1)
+ left = curID;
+ right = curID;
+ if (currentNgram.size() == 2) {
+ String ngram = join(currentNgram);
+ acc.add(String.format("%s_%s", name, ngram), 1);
+ // System.err.println(String.format("ADDING %s_%s", name, ngram));
+ currentNgram.remove(0);
+ }
+ }
+ }
+
+ NgramDPState state = new NgramDPState(new int[] { left }, new int[] { right });
+ // System.err.println(String.format("RULE %s -> state %s", rule.getRuleString(), state));
+ return state;
+ }
+
+ /**
+ * Returns the word after comparing against the private vocabulary (if set).
+ *
+ * @param curID
+ * @return the word
+ */
+ private String getWord(int curID) {
+ String word = Vocabulary.word(curID);
+
+ if (vocab != null && ! vocab.contains(word)) {
+ return "UNK";
+ }
+
+ return word;
+ }
+
+ /**
+ * We don't compute a future cost.
+ */
+ @Override
+ public float estimateFutureCost(Rule rule, DPState state, Sentence sentence) {
+ return 0.0f;
+ }
+
+ /**
+ * There is nothing to be done here, since <s> and </s> are included in rules that are part
+ * of the grammar. We simply return the DP state of the tail node.
+ */
+ @Override
+ public DPState computeFinal(HGNode tailNode, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+
+ return tailNode.getDPState(stateIndex);
+ }
+
+ /**
+ * TargetBigram features are only computed across hyperedges, so there is nothing to be done here.
+ */
+ @Override
+ public float estimateCost(Rule rule, Sentence sentence) {
+ return 0.0f;
+ }
+
+ /**
+ * Join a list with the _ character. I am sure this is in a library somewhere.
+ *
+ * @param list a list of strings
+ * @return the joined String
+ */
+ private String join(List<String> list) {
+ StringBuilder sb = new StringBuilder();
+ for (String item : list) {
+ sb.append(item.toString() + "_");
+ }
+
+ return sb.substring(0, sb.length() - 1);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/WordPenalty.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/WordPenalty.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/WordPenalty.java
new file mode 100644
index 0000000..da315ec
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/WordPenalty.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.chart_parser.SourcePath;
+import org.apache.joshua.decoder.hypergraph.HGNode;
+import org.apache.joshua.decoder.phrase.Hypothesis;
+import org.apache.joshua.decoder.segment_file.Sentence;
+
+/**
+ *
+ * @author Zhifei Li zhifei.work@gmail.com
+ * @author Matt Post post@cs.jhu.edu
+ */
+public final class WordPenalty extends StatelessFF {
+
+ private float OMEGA = -(float) Math.log10(Math.E); // -0.435
+ private final boolean isCky;
+
+ public WordPenalty(final FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, "WordPenalty", args, config);
+
+ if (parsedArgs.containsKey("value"))
+ OMEGA = Float.parseFloat(parsedArgs.get("value"));
+
+ isCky = config.search_algorithm.equals("cky");
+ }
+
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+
+ if (rule != null) {
+ // TODO: this is an inefficient way to do this. Find a better way to not apply this rule
+ // to start and stop glue rules when phrase-based decoding.
+ if (isCky || (rule != Hypothesis.BEGIN_RULE && rule != Hypothesis.END_RULE)) {
+ acc.add(denseFeatureIndex, OMEGA * (rule.getEnglish().length - rule.getArity()));
+ }
+ }
+
+ return null;
+ }
+
+ @Override
+ public ArrayList<String> reportDenseFeatures(int index) {
+ denseFeatureIndex = index;
+ ArrayList<String> names = new ArrayList<>(1);
+ names.add(name);
+ return names;
+ }
+
+ @Override
+ public float estimateCost(Rule rule, Sentence sentence) {
+ if (rule != null)
+ return weights.getDense(denseFeatureIndex) * OMEGA * (rule.getEnglish().length - rule.getArity());
+ return 0.0f;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/fragmentlm/ConcatenationIterator.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/fragmentlm/ConcatenationIterator.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/fragmentlm/ConcatenationIterator.java
new file mode 100644
index 0000000..f75dffa
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/fragmentlm/ConcatenationIterator.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.fragmentlm;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+/**
+ * Concatenates an iterator over iterators into one long iterator.
+ *
+ * @author Dan Klein
+ */
+public class ConcatenationIterator<E> implements Iterator<E> {
+
+ Iterator<Iterator<E>> sourceIterators;
+ Iterator<E> currentIterator;
+ Iterator<E> lastIteratorToReturn;
+
+ public boolean hasNext() {
+ if (currentIterator.hasNext())
+ return true;
+ return false;
+ }
+
+ public E next() {
+ if (currentIterator.hasNext()) {
+ E e = currentIterator.next();
+ lastIteratorToReturn = currentIterator;
+ advance();
+ return e;
+ }
+ throw new NoSuchElementException();
+ }
+
+ private void advance() {
+ while (! currentIterator.hasNext() && sourceIterators.hasNext()) {
+ currentIterator = sourceIterators.next();
+ }
+ }
+
+ public void remove() {
+ if (lastIteratorToReturn == null)
+ throw new IllegalStateException();
+ currentIterator.remove();
+ }
+
+ public ConcatenationIterator(Iterator<Iterator<E>> sourceIterators) {
+ this.sourceIterators = sourceIterators;
+ this.currentIterator = (new ArrayList<E>()).iterator();
+ this.lastIteratorToReturn = null;
+ advance();
+ }
+
+ public ConcatenationIterator(Collection<Iterator<E>> iteratorCollection) {
+ this(iteratorCollection.iterator());
+ }
+
+ public static void main(String[] args) {
+ List<String> list0 = Collections.emptyList();
+ List<String> list1 = Arrays.asList("a b c d".split(" "));
+ List<String> list2 = Arrays.asList("e f".split(" "));
+ List<Iterator<String>> iterators = new ArrayList<Iterator<String>>();
+ iterators.add(list1.iterator());
+ iterators.add(list0.iterator());
+ iterators.add(list2.iterator());
+ iterators.add(list0.iterator());
+ Iterator<String> iterator = new ConcatenationIterator<String>(iterators);
+ while (iterator.hasNext()) {
+ System.out.println(iterator.next());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/fragmentlm/FragmentLMFF.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/fragmentlm/FragmentLMFF.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/fragmentlm/FragmentLMFF.java
new file mode 100644
index 0000000..fa4c4af
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/fragmentlm/FragmentLMFF.java
@@ -0,0 +1,365 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.fragmentlm;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Stack;
+
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.chart_parser.SourcePath;
+import org.apache.joshua.decoder.ff.FeatureVector;
+import org.apache.joshua.decoder.ff.StatefulFF;
+import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.ff.tm.format.HieroFormatReader;
+import org.apache.joshua.decoder.hypergraph.HGNode;
+import org.apache.joshua.decoder.hypergraph.HyperEdge;
+import org.apache.joshua.decoder.segment_file.Sentence;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * <p>Feature function that reads in a list of language model fragments and matches them against the
+ * hypergraph. This allows for language model fragment "glue" features, which fire when LM fragments
+ * (supplied as input) are assembled. These LM fragments are presumably useful in ensuring
+ * grammaticality and can be independent of the translation model fragments.</p>
+ *
+ * <p>Usage: in the Joshua Configuration file, put</p>
+ *
+ * <code>feature-function = FragmentLM -lm LM_FRAGMENTS_FILE -map RULE_FRAGMENTS_MAP_FILE</code>
+ *
+ * <p>LM_FRAGMENTS_FILE is a pointer to a file containing a list of fragments that it should look for.
+ * The format of the file is one fragment per line in PTB format, e.g.:</p>
+ *
+ * <code>(S NP (VP (VBD said) SBAR) (. .))</code>
+ *
+ * <p>RULE_FRAGMENTS_MAP_FILE points to a file that maps fragments to the flattened SCFG rule format
+ * that Joshua uses. This mapping is necessary because Joshua's rules have been flattened, meaning
+ * that their internal structure has been removed, yet this structure is needed for matching LM
+ * fragments. The format of the file is</p>
+ *
+ * <code>FRAGMENT ||| RULE-TARGET-SIDE</code>
+ *
+ * <p>for example,</p>
+ *
+ * <code>(S (NP (DT the) (NN man)) VP .) ||| the man [VP,1] [.,2] (SBAR (IN that) (S (NP (PRP he)) (VP
+ * (VBD was) (VB done)))) ||| that he was done (VP (VBD said) SBAR) ||| said SBAR</code>
+ *
+ * @author Matt Post post@cs.jhu.edu
+ */
+public class FragmentLMFF extends StatefulFF {
+
+ private static final Logger LOG = LoggerFactory.getLogger(FragmentLMFF.class);
+
+ /*
+ * When building a fragment from a rule rooted in the hypergraph, this parameter determines how
+ * deep we'll go. Smaller values mean less hypergraph traversal but may also limit the LM
+ * fragments that can be fired.
+ */
+ private int BUILD_DEPTH = 1;
+
+ /*
+ * The maximum depth of a fragment, defined as the longest path from the fragment root to any of
+ * its leaves.
+ */
+ private int MAX_DEPTH = 0;
+
+ /*
+ * This is the minimum depth for lexicalized LM fragments. This allows you to easily exclude small
+ * depth-one fragments that may be overfit to the training data. A depth of 1 (the default) does
+ * not exclude any fragments.
+ */
+ private int MIN_LEX_DEPTH = 1;
+
+ /*
+ * Set to true to activate meta-features.
+ */
+ private boolean OPTS_DEPTH = false;
+
+ /*
+ * This contains a list of the language model fragments, indexed by LHS.
+ */
+ private HashMap<String, ArrayList<Tree>> lmFragments = null;
+
+ private int numFragments = 0;
+
+ /* The location of the file containing the language model fragments */
+ private String fragmentLMFile = "";
+
+ /**
+ * @param weights a {@link org.apache.joshua.decoder.ff.FeatureVector} with weights
+ * @param args arguments passed to the feature function
+ * @param config the {@link org.apache.joshua.decoder.JoshuaConfiguration}
+ */
+ public FragmentLMFF(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, "FragmentLMFF", args, config);
+
+ lmFragments = new HashMap<String, ArrayList<Tree>>();
+
+ fragmentLMFile = parsedArgs.get("lm");
+ BUILD_DEPTH = Integer.parseInt(parsedArgs.get("build-depth"));
+ MAX_DEPTH = Integer.parseInt(parsedArgs.get("max-depth"));
+ MIN_LEX_DEPTH = Integer.parseInt(parsedArgs.get("min-lex-depth"));
+
+ /* Read in the language model fragments */
+ try {
+ Collection<Tree> trees = PennTreebankReader.readTrees(fragmentLMFile);
+ for (Tree fragment : trees) {
+ addLMFragment(fragment);
+
+ // System.err.println(String.format("Read fragment: %s",
+ // lmFragments.get(lmFragments.size()-1)));
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(String.format("* WARNING: couldn't read fragment LM file '%s'",
+ fragmentLMFile), e);
+ }
+ LOG.info("FragmentLMFF: Read {} LM fragments from '{}'", numFragments, fragmentLMFile);
+ }
+
+ /**
+ * Add the provided fragment to the language model, subject to some filtering.
+ *
+ * @param fragment a {@link org.apache.joshua.decoder.ff.fragmentlm.Tree} fragment
+ */
+ public void addLMFragment(Tree fragment) {
+ if (lmFragments == null)
+ return;
+
+ int fragmentDepth = fragment.getDepth();
+
+ if (MAX_DEPTH != 0 && fragmentDepth > MAX_DEPTH) {
+ LOG.warn("Skipping fragment {} (depth {} > {})", fragment, fragmentDepth, MAX_DEPTH);
+ return;
+ }
+
+ if (MIN_LEX_DEPTH > 1 && fragment.isLexicalized() && fragmentDepth < MIN_LEX_DEPTH) {
+ LOG.warn("Skipping fragment {} (lex depth {} < {})", fragment, fragmentDepth, MIN_LEX_DEPTH);
+ return;
+ }
+
+ if (lmFragments.get(fragment.getRule()) == null) {
+ lmFragments.put(fragment.getRule(), new ArrayList<Tree>());
+ }
+ lmFragments.get(fragment.getRule()).add(fragment);
+ numFragments++;
+ }
+
+ /**
+ * This function computes the features that fire when the current rule is applied. The features
+ * that fire are any LM fragments that match the fragment associated with the current rule. LM
+ * fragments may recurse over the tail nodes, following 1-best backpointers until the fragment
+ * either matches or fails.
+ *
+ * @param rule {@link org.apache.joshua.decoder.ff.tm.Rule} to be utilized within computation
+ * @param tailNodes {@link java.util.List} of {@link org.apache.joshua.decoder.hypergraph.HGNode} tail nodes
+ * @param i todo
+ * @param j todo
+ * @param sourcePath information about a path taken through the source {@link org.apache.joshua.lattice.Lattice}
+ * @param sentence {@link org.apache.joshua.lattice.Lattice} input
+ * @param acc {@link org.apache.joshua.decoder.ff.FeatureFunction.Accumulator} object permitting generalization of feature computation
+ * @return the new dynamic programming state (null for stateless features)
+ */
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+
+ /*
+ * Get the fragment associated with the target side of this rule.
+ *
+ * This could be done more efficiently. For example, just build the tree fragment once and then
+ * pattern match against it. This would circumvent having to build the tree possibly once every
+ * time you try to apply a rule.
+ */
+ Tree baseTree = Tree.buildTree(rule, tailNodes, BUILD_DEPTH);
+
+ Stack<Tree> nodeStack = new Stack<Tree>();
+ nodeStack.add(baseTree);
+ while (!nodeStack.empty()) {
+ Tree tree = nodeStack.pop();
+ if (tree == null)
+ continue;
+
+ if (lmFragments.get(tree.getRule()) != null) {
+ for (Tree fragment : lmFragments.get(tree.getRule())) {
+// System.err.println(String.format("Does\n %s match\n %s??\n -> %s", fragment, tree,
+// match(fragment, tree)));
+
+ if (fragment.getLabel() == tree.getLabel() && match(fragment, tree)) {
+// System.err.println(String.format(" FIRING: matched %s against %s", fragment, tree));
+ acc.add(fragment.escapedString(), 1);
+ if (OPTS_DEPTH)
+ if (fragment.isLexicalized())
+ acc.add(String.format("FragmentFF_lexdepth%d", fragment.getDepth()), 1);
+ else
+ acc.add(String.format("FragmentFF_depth%d", fragment.getDepth()), 1);
+ }
+ }
+ }
+
+ // We also need to try matching rules against internal nodes of the fragment corresponding to
+ // this
+ // rule
+ if (tree.getChildren() != null)
+ for (Tree childNode : tree.getChildren()) {
+ if (!childNode.isBoundary())
+ nodeStack.add(childNode);
+ }
+ }
+
+ return new FragmentState(baseTree);
+ }
+
+ /**
+ * Matches the fragment against the (possibly partially-built) tree. Assumption
+ *
+ * @param fragment the language model fragment
+ * @param tree the tree to match against (expanded from the hypergraph)
+ * @return
+ */
+ private boolean match(Tree fragment, Tree tree) {
+ // System.err.println(String.format("MATCH(%s,%s)", fragment, tree));
+
+ /* Make sure the root labels match. */
+ if (fragment.getLabel() != tree.getLabel()) {
+ return false;
+ }
+
+ /* Same number of kids? */
+ List<Tree> fkids = fragment.getChildren();
+ if (fkids.size() > 0) {
+ List<Tree> tkids = tree.getChildren();
+ if (fkids.size() != tkids.size()) {
+ return false;
+ }
+
+ /* Do the kids match on all labels? */
+ for (int i = 0; i < fkids.size(); i++)
+ if (fkids.get(i).getLabel() != tkids.get(i).getLabel())
+ return false;
+
+ /* Recursive match. */
+ for (int i = 0; i < fkids.size(); i++) {
+ if (!match(fkids.get(i), tkids.get(i)))
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ @Override
+ public DPState computeFinal(HGNode tailNodes, int i, int j, SourcePath sourcePath, Sentence sentence,
+ Accumulator acc) {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ public float estimateFutureCost(Rule rule, DPState state, Sentence sentence) {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+
+ @Override
+ public float estimateCost(Rule rule, Sentence sentence) {
+ // TODO Auto-generated method stub
+ return 0;
+ }
+
+ public static void main(String[] args) {
+ /* Add an LM fragment, then create a dummy multi-level hypergraph to match the fragment against. */
+ // FragmentLMFF fragmentLMFF = new FragmentLMFF(new FeatureVector(), (StateComputer) null, "");
+ FragmentLMFF fragmentLMFF = new FragmentLMFF(new FeatureVector(),
+ new String[] {"-lm", "test/fragments.txt", "-map", "test/mapping.txt"}, null);
+
+ Tree fragment = Tree.fromString("(S NP (VP (VBD \"said\") SBAR) (. \".\"))");
+
+ Rule ruleS = new HieroFormatReader()
+ .parseLine("[S] ||| the man [VP,1] [.,2] ||| the man [VP,1] [.,2] ||| 0");
+ Rule ruleVP = new HieroFormatReader()
+ .parseLine("[VP] ||| said [SBAR,1] ||| said [SBAR,1] ||| 0");
+ Rule ruleSBAR = new HieroFormatReader()
+ .parseLine("[SBAR] ||| that he was done ||| that he was done ||| 0");
+ Rule rulePERIOD = new HieroFormatReader().parseLine("[.] ||| . ||| . ||| 0");
+
+ ruleS.setOwner(0);
+ ruleVP.setOwner(0);
+ ruleSBAR.setOwner(0);
+ rulePERIOD.setOwner(0);
+
+ HyperEdge edgeSBAR = new HyperEdge(ruleSBAR, 0.0f, 0.0f, null, (SourcePath) null);
+
+ HGNode nodeSBAR = new HGNode(3, 7, ruleSBAR.getLHS(), null, edgeSBAR, 0.0f);
+ ArrayList<HGNode> tailNodesVP = new ArrayList<HGNode>();
+ Collections.addAll(tailNodesVP, nodeSBAR);
+ HyperEdge edgeVP = new HyperEdge(ruleVP, 0.0f, 0.0f, tailNodesVP, (SourcePath) null);
+ HGNode nodeVP = new HGNode(2, 7, ruleVP.getLHS(), null, edgeVP, 0.0f);
+
+ HyperEdge edgePERIOD = new HyperEdge(rulePERIOD, 0.0f, 0.0f, null, (SourcePath) null);
+ HGNode nodePERIOD = new HGNode(7, 8, rulePERIOD.getLHS(), null, edgePERIOD, 0.0f);
+
+ ArrayList<HGNode> tailNodes = new ArrayList<HGNode>();
+ Collections.addAll(tailNodes, nodeVP, nodePERIOD);
+
+ Tree tree = Tree.buildTree(ruleS, tailNodes, 1);
+ boolean matched = fragmentLMFF.match(fragment, tree);
+ LOG.info("Does\n {} match\n {}??\n -> {}", fragment, tree, matched);
+ }
+
+ /**
+ * Maintains a state pointer used by KenLM to implement left-state minimization.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ * @author Juri Ganitkevitch juri@cs.jhu.edu
+ */
+ public class FragmentState extends DPState {
+
+ private Tree tree = null;
+
+ public FragmentState(Tree tree) {
+ this.tree = tree;
+ }
+
+ /**
+ * Every tree is unique.
+ *
+ * Some savings could be had here if we grouped together items with the same string.
+ */
+ @Override
+ public int hashCode() {
+ return tree.hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ return (other instanceof FragmentState && this == other);
+ }
+
+ @Override
+ public String toString() {
+ return String.format("[FragmentState %s]", tree);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/fragmentlm/PennTreebankReader.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/fragmentlm/PennTreebankReader.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/fragmentlm/PennTreebankReader.java
new file mode 100644
index 0000000..1637b5f
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/fragmentlm/PennTreebankReader.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.fragmentlm;
+
+import java.util.*;
+import java.io.*;
+import java.nio.charset.Charset;
+import java.nio.charset.UnsupportedCharsetException;
+
+/**
+ * @author Dan Klein
+ */
+public class PennTreebankReader {
+
+ static class TreeCollection extends AbstractCollection<Tree> {
+
+ List<File> files;
+ Charset charset;
+
+ static class TreeIteratorIterator implements Iterator<Iterator<Tree>> {
+ Iterator<File> fileIterator;
+ Iterator<Tree> nextTreeIterator;
+ Charset charset;
+
+ public boolean hasNext() {
+ return nextTreeIterator != null;
+ }
+
+ public Iterator<Tree> next() {
+ Iterator<Tree> currentTreeIterator = nextTreeIterator;
+ advance();
+ return currentTreeIterator;
+ }
+
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ private void advance() {
+ nextTreeIterator = null;
+ while (nextTreeIterator == null && fileIterator.hasNext()) {
+ File file = fileIterator.next();
+ // System.out.println(file);
+ try {
+ nextTreeIterator = new Trees.PennTreeReader(new BufferedReader(new InputStreamReader(
+ new FileInputStream(file), this.charset)));
+ } catch (FileNotFoundException e) {
+ } catch (UnsupportedCharsetException e) {
+ throw new Error("Unsupported charset in file " + file.getPath());
+ }
+ }
+ }
+
+ TreeIteratorIterator(List<File> files, Charset charset) {
+ this.fileIterator = files.iterator();
+ this.charset = charset;
+ advance();
+ }
+ }
+
+ public Iterator<Tree> iterator() {
+ return new ConcatenationIterator<Tree>(new TreeIteratorIterator(files, this.charset));
+ }
+
+ public int size() {
+ int size = 0;
+ Iterator<Tree> i = iterator();
+ while (i.hasNext()) {
+ size++;
+ i.next();
+ }
+ return size;
+ }
+
+ @SuppressWarnings("unused")
+ private List<File> getFilesUnder(String path, FileFilter fileFilter) {
+ File root = new File(path);
+ List<File> files = new ArrayList<File>();
+ addFilesUnder(root, files, fileFilter);
+ return files;
+ }
+
+ private void addFilesUnder(File root, List<File> files, FileFilter fileFilter) {
+ if (!fileFilter.accept(root))
+ return;
+ if (root.isFile()) {
+ files.add(root);
+ return;
+ }
+ if (root.isDirectory()) {
+ File[] children = root.listFiles();
+ for (int i = 0; i < children.length; i++) {
+ File child = children[i];
+ addFilesUnder(child, files, fileFilter);
+ }
+ }
+ }
+
+ public TreeCollection(String file) throws FileNotFoundException, IOException {
+ this.files = new ArrayList<File>();
+ this.files.add(new File(file));
+ this.charset = Charset.defaultCharset();
+ }
+ }
+
+ public static Collection<Tree> readTrees(String path) throws FileNotFoundException, IOException {
+ return new TreeCollection(path);
+ }
+
+ public static void main(String[] args) {
+/* Collection<Tree> trees = readTrees(args[0], Charset.defaultCharset());
+ for (Tree tree : trees) {
+ tree = (new Trees.StandardTreeNormalizer()).transformTree(tree);
+ System.out.println(Trees.PennTreeRenderer.render(tree));
+ }
+ */
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/fragmentlm/Tree.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/fragmentlm/Tree.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/fragmentlm/Tree.java
new file mode 100644
index 0000000..07c7ecd
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/fragmentlm/Tree.java
@@ -0,0 +1,779 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.fragmentlm;
+
+import java.io.IOException;
+import java.io.Serializable;
+import java.io.StringReader;
+import java.util.*;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.ff.fragmentlm.Trees.PennTreeReader;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.hypergraph.HGNode;
+import org.apache.joshua.decoder.hypergraph.HyperEdge;
+import org.apache.joshua.decoder.hypergraph.KBestExtractor.DerivationState;
+import org.apache.joshua.util.io.LineReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Represent phrase-structure trees, with each node consisting of a label and a list of children.
+ * Borrowed from the Berkeley Parser, and extended to allow the representation of tree fragments in
+ * addition to complete trees (the BP requires terminals to be immediately governed by a
+ * preterminal). To distinguish terminals from nonterminals in fragments, the former must be
+ * enclosed in double-quotes when read in.
+ *
+ * @author Dan Klein
+ * @author Matt Post post@cs.jhu.edu
+ */
+public class Tree implements Serializable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Tree.class);
+ private static final long serialVersionUID = 1L;
+
+ protected int label;
+
+ /* Marks a frontier node as a terminal (as opposed to a nonterminal). */
+ boolean isTerminal = false;
+
+ /*
+ * Marks the root and frontier nodes of a fragment. Useful for denoting fragment derivations in
+ * larger trees.
+ */
+ boolean isBoundary = false;
+
+ /* A list of the node's children. */
+ List<Tree> children;
+
+ /* The maximum distance from the root to any of the frontier nodes. */
+ int depth = -1;
+
+ /* The number of lexicalized items among the tree's frontier. */
+ private int numLexicalItems = -1;
+
+ /*
+ * This maps the flat right-hand sides of Joshua rules to the tree fragments they were derived
+ * from. It is used to lookup the fragment that language model fragments should be match against.
+ * For example, if the target (English) side of your rule is
+ *
+ * [NP,1] said [SBAR,2]
+ *
+ * we will retrieve the unflattened fragment
+ *
+ * (S NP (VP (VBD said) SBAR))
+ *
+ * which presumably was the fronter fragment used to derive the translation rule. With this in
+ * hand, we can iterate through our store of language model fragments to match them against this,
+ * following tail nodes if necessary.
+ */
+ public static HashMap<String, String> rulesToFragmentStrings = new HashMap<String, String>();
+
+ public Tree(String label, List<Tree> children) {
+ setLabel(label);
+ this.children = children;
+ }
+
+ public Tree(String label) {
+ setLabel(label);
+ this.children = Collections.emptyList();
+ }
+
+ public Tree(int label2, ArrayList<Tree> newChildren) {
+ this.label = label2;
+ this.children = newChildren;
+ }
+
+ public void setChildren(List<Tree> c) {
+ this.children = c;
+ }
+
+ public List<Tree> getChildren() {
+ return children;
+ }
+
+ public int getLabel() {
+ return label;
+ }
+
+ /**
+ * Computes the depth-one rule rooted at this node. If the node has no children, null is returned.
+ *
+ * @return string representation of the rule
+ */
+ public String getRule() {
+ if (isLeaf()) {
+ return null;
+ }
+ StringBuilder ruleString = new StringBuilder("(" + Vocabulary.word(getLabel()));
+ for (Tree child : getChildren()) {
+ ruleString.append(" ").append(Vocabulary.word(child.getLabel()));
+ }
+ return ruleString.toString();
+ }
+
+ /*
+ * Boundary nodes are used externally to mark merge points between different fragments. This is
+ * separate from the internal ( (substitution point) denotation.
+ */
+ public boolean isBoundary() {
+ return isBoundary;
+ }
+
+ public void setBoundary(boolean b) {
+ this.isBoundary = b;
+ }
+
+ public boolean isTerminal() {
+ return isTerminal;
+ }
+
+ public boolean isLeaf() {
+ return getChildren().isEmpty();
+ }
+
+ public boolean isPreTerminal() {
+ return getChildren().size() == 1 && getChildren().get(0).isLeaf();
+ }
+
+ public List<Tree> getNonterminalYield() {
+ List<Tree> yield = new ArrayList<Tree>();
+ appendNonterminalYield(this, yield);
+ return yield;
+ }
+
+ public List<Tree> getYield() {
+ List<Tree> yield = new ArrayList<Tree>();
+ appendYield(this, yield);
+ return yield;
+ }
+
+ public List<Tree> getTerminals() {
+ List<Tree> yield = new ArrayList<Tree>();
+ appendTerminals(this, yield);
+ return yield;
+ }
+
+ private static void appendTerminals(Tree tree, List<Tree> yield) {
+ if (tree.isLeaf()) {
+ yield.add(tree);
+ return;
+ }
+ for (Tree child : tree.getChildren()) {
+ appendTerminals(child, yield);
+ }
+ }
+
+ /**
+ * Clone the structure of the tree.
+ *
+ * @return a cloned tree
+ */
+ public Tree shallowClone() {
+ ArrayList<Tree> newChildren = new ArrayList<Tree>(children.size());
+ for (Tree child : children) {
+ newChildren.add(child.shallowClone());
+ }
+
+ Tree newTree = new Tree(label, newChildren);
+ newTree.setIsTerminal(isTerminal());
+ newTree.setBoundary(isBoundary());
+ return newTree;
+ }
+
+ private void setIsTerminal(boolean terminal) {
+ isTerminal = terminal;
+ }
+
+ private static void appendNonterminalYield(Tree tree, List<Tree> yield) {
+ if (tree.isLeaf() && !tree.isTerminal()) {
+ yield.add(tree);
+ return;
+ }
+ for (Tree child : tree.getChildren()) {
+ appendNonterminalYield(child, yield);
+ }
+ }
+
+ private static void appendYield(Tree tree, List<Tree> yield) {
+ if (tree.isLeaf()) {
+ yield.add(tree);
+ return;
+ }
+ for (Tree child : tree.getChildren()) {
+ appendYield(child, yield);
+ }
+ }
+
+ public List<Tree> getPreTerminalYield() {
+ List<Tree> yield = new ArrayList<Tree>();
+ appendPreTerminalYield(this, yield);
+ return yield;
+ }
+
+ private static void appendPreTerminalYield(Tree tree, List<Tree> yield) {
+ if (tree.isPreTerminal()) {
+ yield.add(tree);
+ return;
+ }
+ for (Tree child : tree.getChildren()) {
+ appendPreTerminalYield(child, yield);
+ }
+ }
+
+ /**
+ * A tree is lexicalized if it has terminal nodes among the leaves of its frontier. For normal
+ * trees this is always true since they bottom out in terminals, but for fragments, this may or
+ * may not be true.
+ *
+ * @return true if the tree is lexicalized
+ */
+ public boolean isLexicalized() {
+ if (this.numLexicalItems < 0) {
+ if (isTerminal())
+ this.numLexicalItems = 1;
+ else {
+ this.numLexicalItems = 0;
+ for (Tree child : children)
+ if (child.isLexicalized())
+ this.numLexicalItems += 1;
+ }
+ }
+
+ return (this.numLexicalItems > 0);
+ }
+
+ /**
+ * The depth of a tree is the maximum distance from the root to any of the frontier nodes.
+ *
+ * @return the tree depth
+ */
+ public int getDepth() {
+ if (this.depth >= 0)
+ return this.depth;
+
+ if (isLeaf()) {
+ this.depth = 0;
+ } else {
+ int maxDepth = 0;
+ for (Tree child : children) {
+ int depth = child.getDepth();
+ if (depth > maxDepth)
+ maxDepth = depth;
+ }
+ this.depth = maxDepth + 1;
+ }
+ return this.depth;
+ }
+
+ public List<Tree> getAtDepth(int depth) {
+ List<Tree> yield = new ArrayList<Tree>();
+ appendAtDepth(depth, this, yield);
+ return yield;
+ }
+
+ private static void appendAtDepth(int depth, Tree tree, List<Tree> yield) {
+ if (depth < 0)
+ return;
+ if (depth == 0) {
+ yield.add(tree);
+ return;
+ }
+ for (Tree child : tree.getChildren()) {
+ appendAtDepth(depth - 1, child, yield);
+ }
+ }
+
+ public void setLabel(String label) {
+ if (label.length() >= 3 && label.startsWith("\"") && label.endsWith("\"")) {
+ this.isTerminal = true;
+ label = label.substring(1, label.length() - 1);
+ }
+
+ this.label = Vocabulary.id(label);
+ }
+
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ toStringBuilder(sb);
+ return sb.toString();
+ }
+
+ /**
+ * Removes the quotes around terminals. Note that the resulting tree could not be read back
+ * in by this class, since unquoted leaves are interpreted as nonterminals.
+ *
+ * @return unquoted string
+ */
+ public String unquotedString() {
+ return toString().replaceAll("\"", "");
+ }
+
+ public String escapedString() {
+ return toString().replaceAll(" ", "_");
+ }
+
+ public void toStringBuilder(StringBuilder sb) {
+ if (!isLeaf())
+ sb.append('(');
+
+ if (isTerminal())
+ sb.append(String.format("\"%s\"", Vocabulary.word(getLabel())));
+ else
+ sb.append(Vocabulary.word(getLabel()));
+
+ if (!isLeaf()) {
+ for (Tree child : getChildren()) {
+ sb.append(' ');
+ child.toStringBuilder(sb);
+ }
+ sb.append(')');
+ }
+ }
+
+ /**
+ * Get the set of all subtrees inside the tree by returning a tree rooted at each node. These are
+ * <i>not</i> copies, but all share structure. The tree is regarded as a subtree of itself.
+ *
+ * @return the <code>Set</code> of all subtrees in the tree.
+ */
+ public Set<Tree> subTrees() {
+ return (Set<Tree>) subTrees(new HashSet<Tree>());
+ }
+
+ /**
+ * Get the list of all subtrees inside the tree by returning a tree rooted at each node. These are
+ * <i>not</i> copies, but all share structure. The tree is regarded as a subtree of itself.
+ *
+ * @return the <code>List</code> of all subtrees in the tree.
+ */
+ public List<Tree> subTreeList() {
+ return (List<Tree>) subTrees(new ArrayList<Tree>());
+ }
+
+ /**
+ * Add the set of all subtrees inside a tree (including the tree itself) to the given
+ * <code>Collection</code>.
+ *
+ * @param n A collection of nodes to which the subtrees will be added
+ * @return The collection parameter with the subtrees added
+ */
+ public Collection<Tree> subTrees(Collection<Tree> n) {
+ n.add(this);
+ List<Tree> kids = getChildren();
+ for (Tree kid : kids) {
+ kid.subTrees(n);
+ }
+ return n;
+ }
+
+ /**
+ * Returns an iterator over the nodes of the tree. This method implements the
+ * <code>iterator()</code> method required by the <code>Collections</code> interface. It does a
+ * preorder (children after node) traversal of the tree. (A possible extension to the class at
+ * some point would be to allow different traversal orderings via variant iterators.)
+ *
+ * @return An interator over the nodes of the tree
+ */
+ public TreeIterator iterator() {
+ return new TreeIterator();
+ }
+
+ private class TreeIterator implements Iterator<Tree> {
+
+ private List<Tree> treeStack;
+
+ private TreeIterator() {
+ treeStack = new ArrayList<Tree>();
+ treeStack.add(Tree.this);
+ }
+
+ public boolean hasNext() {
+ return (!treeStack.isEmpty());
+ }
+
+ public Tree next() {
+ int lastIndex = treeStack.size() - 1;
+ Tree tr = treeStack.remove(lastIndex);
+ List<Tree> kids = tr.getChildren();
+ // so that we can efficiently use one List, we reverse them
+ for (int i = kids.size() - 1; i >= 0; i--) {
+ treeStack.add(kids.get(i));
+ }
+ return tr;
+ }
+
+ /**
+ * Not supported
+ */
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ }
+
+ public boolean hasUnaryChain() {
+ return hasUnaryChainHelper(this, false);
+ }
+
+ private boolean hasUnaryChainHelper(Tree tree, boolean unaryAbove) {
+ boolean result = false;
+ if (tree.getChildren().size() == 1) {
+ if (unaryAbove)
+ return true;
+ else if (tree.getChildren().get(0).isPreTerminal())
+ return false;
+ else
+ return hasUnaryChainHelper(tree.getChildren().get(0), true);
+ } else {
+ for (Tree child : tree.getChildren()) {
+ if (!child.isPreTerminal())
+ result = result || hasUnaryChainHelper(child, false);
+ }
+ }
+ return result;
+ }
+
+ /**
+ * Inserts the SOS (and EOS) symbols into a parse tree, attaching them as a left (right) sibling
+ * to the leftmost (rightmost) pre-terminal in the tree. This facilitates using trees as language
+ * models. The arguments have to be passed in to preserve Java generics, even though this is only
+ * ever used with String versions.
+ *
+ * @param sos presumably "<s>"
+ * @param eos presumably "</s>"
+ */
+ public void insertSentenceMarkers(String sos, String eos) {
+ insertSentenceMarker(sos, 0);
+ insertSentenceMarker(eos, -1);
+ }
+
+ public void insertSentenceMarkers() {
+ insertSentenceMarker("<s>", 0);
+ insertSentenceMarker("</s>", -1);
+ }
+
+ /**
+ *
+ * @param symbol the marker to insert
+ * @param pos the position at which to insert
+ */
+ private void insertSentenceMarker(String symbol, int pos) {
+
+ if (isLeaf() || isPreTerminal())
+ return;
+
+ List<Tree> children = getChildren();
+ int index = (pos == -1) ? children.size() - 1 : pos;
+ if (children.get(index).isPreTerminal()) {
+ if (pos == -1)
+ children.add(new Tree(symbol));
+ else
+ children.add(pos, new Tree(symbol));
+ } else {
+ children.get(index).insertSentenceMarker(symbol, pos);
+ }
+ }
+
+ /**
+ * This is a convenience function for producing a fragment from its string representation.
+ *
+ * @param ptbStr input string from which to produce a fragment
+ * @return the fragment
+ */
+ public static Tree fromString(String ptbStr) {
+ PennTreeReader reader = new PennTreeReader(new StringReader(ptbStr));
+ Tree fragment = reader.next();
+ return fragment;
+ }
+
+ public static Tree getFragmentFromYield(String yield) {
+ String fragmentString = rulesToFragmentStrings.get(yield);
+ if (fragmentString != null)
+ return fromString(fragmentString);
+
+ return null;
+ }
+
+ public static void readMapping(String fragmentMappingFile) {
+ /* Read in the rule / fragments mapping */
+ try {
+ LineReader reader = new LineReader(fragmentMappingFile);
+ for (String line : reader) {
+ String[] fields = line.split("\\s+\\|{3}\\s+");
+ if (fields.length != 2 || !fields[0].startsWith("(")) {
+ LOG.warn("malformed line {}: {}", reader.lineno(), line);
+ continue;
+ }
+
+ rulesToFragmentStrings.put(fields[1].trim(), fields[0].trim()); // buildFragment(fields[0]));
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(String.format("* WARNING: couldn't read fragment mapping file '%s'",
+ fragmentMappingFile), e);
+ }
+ LOG.info("FragmentLMFF: Read {} mappings from '{}'", rulesToFragmentStrings.size(),
+ fragmentMappingFile);
+ }
+
+ /**
+ * Builds a tree from the kth-best derivation state. This is done by initializing the tree with
+ * the internal fragment corresponding to the rule; this will be the top of the tree. We then
+ * recursively visit the derivation state objects, following the route through the hypergraph
+ * defined by them.
+ *
+ * This function is like Tree#buildTree(DerivationState, int),
+ * but that one simply follows the best incoming hyperedge for each node.
+ *
+ * @param rule for which corresponding internal fragment can be used to initialize the tree
+ * @param derivationStates array of state objects
+ * @param maxDepth of route through the hypergraph
+ * @return the Tree
+ */
+ public static Tree buildTree(Rule rule, DerivationState[] derivationStates, int maxDepth) {
+ Tree tree = getFragmentFromYield(rule.getEnglishWords());
+
+ if (tree == null) {
+ return null;
+ }
+
+ tree = tree.shallowClone();
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("buildTree({})", tree);
+ for (int i = 0; i < derivationStates.length; i++) {
+ LOG.debug(" -> {}: {}", i, derivationStates[i]);
+ }
+ }
+
+ List<Tree> frontier = tree.getNonterminalYield();
+
+ /* The English side of a rule is a sequence of integers. Nonnegative integers are word
+ * indices in the Vocabulary, while negative indices are used to nonterminals. These negative
+ * indices are a *permutation* of the source side nonterminals, which contain the actual
+ * nonterminal Vocabulary indices for the nonterminal names. Here, we convert this permutation
+ * to a nonnegative 0-based permutation and store it in tailIndices. This is used to index
+ * the incoming DerivationState items, which are ordered by the source side.
+ */
+ ArrayList<Integer> tailIndices = new ArrayList<Integer>();
+ int[] englishInts = rule.getEnglish();
+ for (int i = 0; i < englishInts.length; i++)
+ if (englishInts[i] < 0)
+ tailIndices.add(-(englishInts[i] + 1));
+
+ /*
+ * We now have the tree's yield. The substitution points on the yield should match the
+ * nonterminals of the heads of the derivation states. Since we don't know which of the tree's
+ * frontier items are terminals and which are nonterminals, we walk through the tail nodes,
+ * and then match the label of each against the frontier node labels until we have a match.
+ */
+ // System.err.println(String.format("WORDS: %s\nTREE: %s", rule.getEnglishWords(), tree));
+ for (int i = 0; i < derivationStates.length; i++) {
+
+ Tree frontierTree = frontier.get(tailIndices.get(i));
+ frontierTree.setBoundary(true);
+
+ HyperEdge nextEdge = derivationStates[i].edge;
+ if (nextEdge != null) {
+ DerivationState[] nextStates = null;
+ if (nextEdge.getTailNodes() != null && nextEdge.getTailNodes().size() > 0) {
+ nextStates = new DerivationState[nextEdge.getTailNodes().size()];
+ for (int j = 0; j < nextStates.length; j++)
+ nextStates[j] = derivationStates[i].getChildDerivationState(nextEdge, j);
+ }
+ Tree childTree = buildTree(nextEdge.getRule(), nextStates, maxDepth - 1);
+
+ /* This can be null if there is no entry for the rule in the map */
+ if (childTree != null)
+ frontierTree.children = childTree.children;
+ } else {
+ frontierTree.children = tree.children;
+ }
+ }
+
+ return tree;
+ }
+
+ /**
+ * <p>Builds a tree from the kth-best derivation state. This is done by initializing the tree with
+ * the internal fragment corresponding to the rule; this will be the top of the tree. We then
+ * recursively visit the derivation state objects, following the route through the hypergraph
+ * defined by them.</p>
+ *
+ * @param derivationState array of state objects
+ * @param maxDepth of route through the hypergraph
+ * @return the Tree
+ */
+ public static Tree buildTree(DerivationState derivationState, int maxDepth) {
+ Rule rule = derivationState.edge.getRule();
+
+ Tree tree = getFragmentFromYield(rule.getEnglishWords());
+
+ if (tree == null) {
+ return null;
+ }
+
+ tree = tree.shallowClone();
+
+ LOG.debug("buildTree({})", tree);
+
+ if (rule.getArity() > 0 && maxDepth > 0) {
+ List<Tree> frontier = tree.getNonterminalYield();
+
+ /* The English side of a rule is a sequence of integers. Nonnegative integers are word
+ * indices in the Vocabulary, while negative indices are used to nonterminals. These negative
+ * indices are a *permutation* of the source side nonterminals, which contain the actual
+ * nonterminal Vocabulary indices for the nonterminal names. Here, we convert this permutation
+ * to a nonnegative 0-based permutation and store it in tailIndices. This is used to index
+ * the incoming DerivationState items, which are ordered by the source side.
+ */
+ ArrayList<Integer> tailIndices = new ArrayList<Integer>();
+ int[] englishInts = rule.getEnglish();
+ for (int i = 0; i < englishInts.length; i++)
+ if (englishInts[i] < 0)
+ tailIndices.add(-(englishInts[i] + 1));
+
+ /*
+ * We now have the tree's yield. The substitution points on the yield should match the
+ * nonterminals of the heads of the derivation states. Since we don't know which of the tree's
+ * frontier items are terminals and which are nonterminals, we walk through the tail nodes,
+ * and then match the label of each against the frontier node labels until we have a match.
+ */
+ // System.err.println(String.format("WORDS: %s\nTREE: %s", rule.getEnglishWords(), tree));
+ for (int i = 0; i < rule.getArity(); i++) {
+
+ Tree frontierTree = frontier.get(tailIndices.get(i));
+ frontierTree.setBoundary(true);
+
+ DerivationState childState = derivationState.getChildDerivationState(derivationState.edge, i);
+ Tree childTree = buildTree(childState, maxDepth - 1);
+
+ /* This can be null if there is no entry for the rule in the map */
+ if (childTree != null)
+ frontierTree.children = childTree.children;
+ }
+ }
+
+ return tree;
+ }
+
+ /**
+ * Takes a rule and its tail pointers and recursively constructs a tree (up to maxDepth).
+ *
+ * This could be implemented by using the other buildTree() function and using the 1-best
+ * DerivationState.
+ *
+ * @param rule {@link org.apache.joshua.decoder.ff.tm.Rule} to be used whilst building the tree
+ * @param tailNodes {@link java.util.List} of {@link org.apache.joshua.decoder.hypergraph.HGNode}'s
+ * @param maxDepth to go in the tree
+ * @return shallow clone of the Tree object
+ */
+ public static Tree buildTree(Rule rule, List<HGNode> tailNodes, int maxDepth) {
+ Tree tree = getFragmentFromYield(rule.getEnglishWords());
+
+ if (tree == null) {
+ tree = new Tree(String.format("(%s %s)", Vocabulary.word(rule.getLHS()), rule.getEnglishWords()));
+ // System.err.println("COULDN'T FIND " + rule.getEnglishWords());
+ // System.err.println("RULE " + rule);
+ // for (Entry<String, Tree> pair: rulesToFragments.entrySet())
+ // System.err.println(" FOUND " + pair.getKey());
+
+// return null;
+ } else {
+ tree = tree.shallowClone();
+ }
+
+ if (tree != null && tailNodes != null && tailNodes.size() > 0 && maxDepth > 0) {
+ List<Tree> frontier = tree.getNonterminalYield();
+
+ ArrayList<Integer> tailIndices = new ArrayList<Integer>();
+ int[] englishInts = rule.getEnglish();
+ for (int i = 0; i < englishInts.length; i++)
+ if (englishInts[i] < 0)
+ tailIndices.add(-1 * englishInts[i] - 1);
+
+ /*
+ * We now have the tree's yield. The substitution points on the yield should match the
+ * nonterminals of the tail nodes. Since we don't know which of the tree's frontier items are
+ * terminals and which are nonterminals, we walk through the tail nodes, and then match the
+ * label of each against the frontier node labels until we have a match.
+ */
+ // System.err.println(String.format("WORDS: %s\nTREE: %s", rule.getEnglishWords(), tree));
+ for (int i = 0; i < tailNodes.size(); i++) {
+
+ // String lhs = tailNodes.get(i).getLHS().replaceAll("[\\[\\]]", "");
+ // System.err.println(String.format(" %d: %s", i, lhs));
+ try {
+ Tree frontierTree = frontier.get(tailIndices.get(i).intValue());
+ frontierTree.setBoundary(true);
+
+ HyperEdge edge = tailNodes.get(i).bestHyperedge;
+ if (edge != null) {
+ Tree childTree = buildTree(edge.getRule(), edge.getTailNodes(), maxDepth - 1);
+ /* This can be null if there is no entry for the rule in the map */
+ if (childTree != null)
+ frontierTree.children = childTree.children;
+ } else {
+ frontierTree.children = tree.children;
+ }
+ } catch (IndexOutOfBoundsException e) {
+ LOG.error("ERROR at index {}", i);
+ LOG.error("RULE: {} TREE: {}", rule.getEnglishWords(), tree);
+ LOG.error(" FRONTIER:");
+ for (Tree kid : frontier) {
+ LOG.error(" {}", kid);
+ }
+ throw new RuntimeException(String.format("ERROR at index %d", i), e);
+ }
+ }
+ }
+
+ return tree;
+ }
+
+ public static void main(String[] args) {
+ LineReader reader = new LineReader(System.in);
+
+ for (String line : reader) {
+ try {
+ Tree tree = Tree.fromString(line);
+ tree.insertSentenceMarkers();
+ System.out.println(tree);
+ } catch (Exception e) {
+ System.out.println("");
+ }
+ }
+
+ /*
+ * Tree fragment = Tree
+ * .fromString("(TOP (S (NP (DT the) (NN boy)) (VP (VBD ate) (NP (DT the) (NN food)))))");
+ * fragment.insertSentenceMarkers("<s>", "</s>");
+ *
+ * System.out.println(fragment);
+ *
+ * ArrayList<Tree> trees = new ArrayList<Tree>(); trees.add(Tree.fromString("(NN \"mat\")"));
+ * trees.add(Tree.fromString("(S (NP DT NN) VP)"));
+ * trees.add(Tree.fromString("(S (NP (DT \"the\") NN) VP)"));
+ * trees.add(Tree.fromString("(S (NP (DT the) NN) VP)"));
+ *
+ * for (Tree tree : trees) { System.out.println(String.format("TREE %s DEPTH %d LEX? %s", tree,
+ * tree.getDepth(), tree.isLexicalized())); }
+ */
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/fragmentlm/Trees.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/fragmentlm/Trees.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/fragmentlm/Trees.java
new file mode 100644
index 0000000..d06388c
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/fragmentlm/Trees.java
@@ -0,0 +1,270 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.fragmentlm;
+
+import java.io.IOException;
+import java.io.PushbackReader;
+import java.io.Reader;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+import org.apache.joshua.corpus.Vocabulary;
+/**
+ * Tools for displaying, reading, and modifying trees. Borrowed from the Berkeley Parser.
+ *
+ * @author Dan Klein
+ */
+public class Trees {
+
+ public static class PennTreeReader implements Iterator<Tree> {
+ public static String ROOT_LABEL = "ROOT";
+
+ PushbackReader in;
+ Tree nextTree;
+
+ public boolean hasNext() {
+ return (nextTree != null);
+ }
+
+ public Tree next() {
+ if (!hasNext())
+ throw new NoSuchElementException();
+ Tree tree = nextTree;
+ nextTree = readRootTree();
+ // System.out.println(nextTree);
+ return tree;
+ }
+
+ private Tree readRootTree() {
+ try {
+ readWhiteSpace();
+ if (!isLeftParen(peek()))
+ return null;
+ return readTree(true);
+ } catch (IOException e) {
+ throw new RuntimeException("Error reading tree.");
+ }
+ }
+
+ private Tree readTree(boolean isRoot) throws IOException {
+ if (!isLeftParen(peek())) {
+ return readLeaf();
+ } else {
+ readLeftParen();
+ String label = readLabel();
+ if (label.length() == 0 && isRoot)
+ label = ROOT_LABEL;
+ List<Tree> children = readChildren();
+ readRightParen();
+ return new Tree(label, children);
+ }
+ }
+
+ private String readLabel() throws IOException {
+ readWhiteSpace();
+ return readText();
+ }
+
+ private String readText() throws IOException {
+ StringBuilder sb = new StringBuilder();
+ int ch = in.read();
+ while (!isWhiteSpace(ch) && !isLeftParen(ch) && !isRightParen(ch)) {
+ sb.append((char) ch);
+ ch = in.read();
+ }
+ in.unread(ch);
+ // System.out.println("Read text: ["+sb+"]");
+ return sb.toString().intern();
+ }
+
+ private List<Tree> readChildren() throws IOException {
+ readWhiteSpace();
+ // if (!isLeftParen(peek()))
+ // return Collections.singletonList(readLeaf());
+ return readChildList();
+ }
+
+ private int peek() throws IOException {
+ int ch = in.read();
+ in.unread(ch);
+ return ch;
+ }
+
+ private Tree readLeaf() throws IOException {
+ String label = readText();
+ return new Tree(label);
+ }
+
+ private List<Tree> readChildList() throws IOException {
+ List<Tree> children = new ArrayList<Tree>();
+ readWhiteSpace();
+ while (!isRightParen(peek())) {
+ children.add(readTree(false));
+ readWhiteSpace();
+ }
+ return children;
+ }
+
+ private void readLeftParen() throws IOException {
+ // System.out.println("Read left.");
+ readWhiteSpace();
+ int ch = in.read();
+ if (!isLeftParen(ch))
+ throw new RuntimeException("Format error reading tree. (leftParen)");
+ }
+
+ private void readRightParen() throws IOException {
+ // System.out.println("Read right.");
+ readWhiteSpace();
+ int ch = in.read();
+
+ if (!isRightParen(ch)) {
+ System.out.println((char) ch);
+ throw new RuntimeException("Format error reading tree. (rightParen)");
+ }
+ }
+
+ private void readWhiteSpace() throws IOException {
+ int ch = in.read();
+ while (isWhiteSpace(ch)) {
+ ch = in.read();
+ }
+ in.unread(ch);
+ }
+
+ private boolean isWhiteSpace(int ch) {
+ return (ch == ' ' || ch == '\t' || ch == '\f' || ch == '\r' || ch == '\n');
+ }
+
+ private boolean isLeftParen(int ch) {
+ return ch == '(';
+ }
+
+ private boolean isRightParen(int ch) {
+ return ch == ')';
+ }
+
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ public PennTreeReader(Reader in) {
+ this.in = new PushbackReader((java.io.Reader) in);
+ nextTree = readRootTree();
+ // System.out.println(nextTree);
+ }
+ }
+
+ /**
+ * Renderer for pretty-printing trees according to the Penn Treebank indenting guidelines
+ * (mutliline). Adapted from code originally written by Dan Klein and modified by Chris Manning.
+ */
+ public static class PennTreeRenderer {
+
+ /**
+ * Print the tree as done in Penn Treebank merged files. The formatting should be exactly the
+ * same, but we don't print the trailing whitespace found in Penn Treebank trees. The basic
+ * deviation from a bracketed indented tree is to in general collapse the printing of adjacent
+ * preterminals onto one line of tags and words. Additional complexities are that conjunctions
+ * (tag CC) are not collapsed in this way, and that the unlabeled outer brackets are collapsed
+ * onto the same line as the next bracket down.
+ *
+ * @param tree you wish to render and print
+ * @return a rendered String representation of the tree
+ */
+ public static String render(Tree tree) {
+ StringBuilder sb = new StringBuilder();
+ renderTree(tree, 0, false, false, false, true, sb);
+ sb.append('\n');
+ return sb.toString();
+ }
+
+ /**
+ * Display a node, implementing Penn Treebank style layout
+ */
+ private static void renderTree(Tree tree, int indent, boolean parentLabelNull,
+ boolean firstSibling, boolean leftSiblingPreTerminal, boolean topLevel, StringBuilder sb) {
+ // the condition for staying on the same line in Penn Treebank
+ boolean suppressIndent = (parentLabelNull || (firstSibling && tree.isPreTerminal()) || (leftSiblingPreTerminal
+ && tree.isPreTerminal()));
+ if (suppressIndent) {
+ sb.append(' ');
+ } else {
+ if (!topLevel) {
+ sb.append('\n');
+ }
+ for (int i = 0; i < indent; i++) {
+ sb.append(" ");
+ }
+ }
+ if (tree.isLeaf() || tree.isPreTerminal()) {
+ renderFlat(tree, sb);
+ return;
+ }
+ sb.append('(');
+ sb.append(tree.getLabel());
+ renderChildren(tree.getChildren(), indent + 1, false, sb);
+ sb.append(')');
+ }
+
+ private static void renderFlat(Tree tree, StringBuilder sb) {
+ if (tree.isLeaf()) {
+ sb.append(Vocabulary.word(tree.getLabel()));
+ return;
+ }
+ sb.append('(');
+ sb.append(Vocabulary.word(tree.getLabel()));
+ sb.append(' ');
+ sb.append(Vocabulary.word(tree.getChildren().get(0).getLabel()));
+ sb.append(')');
+ }
+
+ private static void renderChildren(List<Tree> children, int indent,
+ boolean parentLabelNull, StringBuilder sb) {
+ boolean firstSibling = true;
+ boolean leftSibIsPreTerm = true; // counts as true at beginning
+ for (Tree child : children) {
+ renderTree(child, indent, parentLabelNull, firstSibling, leftSibIsPreTerm, false, sb);
+ leftSibIsPreTerm = child.isPreTerminal();
+ firstSibling = false;
+ }
+ }
+ }
+
+ public static void main(String[] args) {
+ String ptbTreeString = "((S (NP (DT the) (JJ quick) (JJ brown) (NN fox)) (VP (VBD jumped) (PP (IN over) (NP (DT the) (JJ lazy) (NN dog)))) (. .)))";
+
+ if (args.length > 0) {
+ String tree = "";
+ for (String str : args) {
+ tree += " " + str;
+ }
+ ptbTreeString = tree.substring(1);
+ }
+
+ PennTreeReader reader = new PennTreeReader(new StringReader(ptbTreeString));
+
+ Tree tree = reader.next();
+ System.out.println(PennTreeRenderer.render(tree));
+ System.out.println(tree);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/AbstractLM.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/AbstractLM.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/AbstractLM.java
new file mode 100644
index 0000000..e8225dc
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/AbstractLM.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.lm;
+
+import org.apache.joshua.decoder.Support;
+import java.util.List;
+
+/**
+ * This class implements NGramLanguageModel by creating wrappers
+ * around the necessary functions to capture common errors. Most
+ * methods are declared final, in an attempt to limit what subclasses
+ * may be defined.
+ *
+ * @author Zhifei Li, zhifei.work@gmail.com
+ * @version $LastChangedDate: 2009-12-30 10:10:38 -0600 (Wed, 30 Dec 2009) $
+ */
+public abstract class AbstractLM extends DefaultNGramLanguageModel {
+
+ public AbstractLM(int symbolTable, int order) {
+ super(symbolTable, order);
+ }
+
+ @SuppressWarnings("null")
+ public final double sentenceLogProbability(
+ List<Integer> sentence, int order, int startIndex
+ ) {
+ //return super.sentenceLogProbability(sentence.stream().toArray(int[]::new) , order, startIndex);
+ return (Double) null;
+ }
+
+ public final float ngramLogProbability(int[] ngram) {
+ return super.ngramLogProbability(ngram);
+ }
+
+ public final float ngramLogProbability(int[] ngram, int order) {
+ if (ngram.length > order) {
+ throw new RuntimeException("ngram length is greather than the max order");
+ }
+ // if (ngram.length==1 && "we".equals(symbolTable.getWord(ngram[0]))) {
+ // System.err.println("Something weird is about to happen");
+ // }
+
+ int historySize = ngram.length - 1;
+ if (historySize >= order || historySize < 0) {
+ // BUG: use logger or exception. Don't zero default
+ throw new RuntimeException("Error: history size is " + historySize);
+ // return 0;
+ }
+ double probability = ngramLogProbability_helper(ngram, order);
+// if (probability < -JoshuaConfiguration.lm_ceiling_cost) {
+// probability = -JoshuaConfiguration.lm_ceiling_cost;
+// }
+ return (float) probability;
+ }
+
+ protected abstract float ngramLogProbability_helper(int[] ngram, int order);
+
+ @Deprecated
+ public final double logProbOfBackoffState(List<Integer> ngram, int order, int qtyAdditionalBackoffWeight) {
+ return logProbabilityOfBackoffState(
+ Support.subIntArray(ngram, 0, ngram.size()),
+ order, qtyAdditionalBackoffWeight);
+ }
+
+
+ public final double logProbabilityOfBackoffState(int[] ngram, int order, int qtyAdditionalBackoffWeight) {
+ if (ngram.length > order) {
+ throw new RuntimeException("ngram length is greather than the max order");
+ }
+ if (ngram[ngram.length-1] != LanguageModelFF.LM_INDEX) {
+ throw new RuntimeException("last wrd is not <bow>");
+ }
+ if (qtyAdditionalBackoffWeight > 0) {
+ return logProbabilityOfBackoffState_helper(
+ ngram, order, qtyAdditionalBackoffWeight);
+ } else {
+ return 0.0;
+ }
+ }
+
+
+ protected abstract double logProbabilityOfBackoffState_helper(
+ int[] ngram, int order, int qtyAdditionalBackoffWeight);
+
+
+ // BUG: We should have different classes based on the configuration in use
+ public int[] leftEquivalentState(int[] originalState, int order,
+ double[] cost
+ ) {
+// if (JoshuaConfiguration.use_left_equivalent_state)
+// throw new UnsupportedOperationException("getLeftEquivalentState is not overwritten by a concrete class");
+
+ return originalState;
+ }
+
+
+ // BUG: We should have different classes based on the configuration in use
+ public int[] rightEquivalentState(int[] originalState, int order) {
+// if ( !JoshuaConfiguration.use_right_equivalent_state
+// || originalState.length != this.ngramOrder-1) {
+ return originalState;
+// } else {
+// throw new UnsupportedOperationException("getRightEquivalentState is not overwritten by a concrete class");
+// }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/ArpaFile.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/ArpaFile.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/ArpaFile.java
new file mode 100644
index 0000000..01d2f39
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/ArpaFile.java
@@ -0,0 +1,328 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.lm;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+import java.util.Scanner;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.zip.GZIPInputStream;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.util.Regex;
+import org.apache.joshua.util.io.LineReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Utility class for reading ARPA language model files.
+ *
+ * @author Lane Schwartz
+ */
+public class ArpaFile implements Iterable<ArpaNgram> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ArpaFile.class);
+
+ /** Regular expression representing a blank line. */
+ public static final Regex BLANK_LINE = new Regex("^\\s*$");
+
+ /**
+ * Regular expression representing a line
+ * starting a new section of n-grams in an ARPA language model file.
+ */
+ public static final Regex NGRAM_HEADER = new Regex("^\\\\\\d-grams:\\s*$");
+
+ /**
+ * Regular expression representing a line
+ * ending an ARPA language model file.
+ */
+ public static final Regex NGRAM_END = new Regex("^\\\\end\\\\s*$");
+
+ /** ARPA file for this object. */
+ private final File arpaFile;
+
+ /** The vocabulary associated with this object. */
+ private final Vocabulary vocab;
+
+ /**
+ * Constructs an object that represents an ARPA language model file.
+ *
+ * @param arpaFileName File name of an ARPA language model file
+ * @param vocab Symbol table to be used by this object
+ */
+ public ArpaFile(String arpaFileName, Vocabulary vocab) {
+ this.arpaFile = new File(arpaFileName);
+ this.vocab = vocab;
+ }
+
+ public ArpaFile(String arpaFileName) throws IOException {
+ this.arpaFile = new File(arpaFileName);
+ this.vocab = new Vocabulary();
+
+ // final Scanner scanner = new Scanner(arpaFile);
+
+ // // Eat initial header lines
+ // while (scanner.hasNextLine()) {
+ // String line = scanner.nextLine();
+ // logger.finest("Discarding line: " + line);
+ // if (NGRAM_HEADER.matches(line)) {
+ // break;
+ // }
+ // }
+
+ // int ngramOrder = 1;
+
+ LineReader grammarReader = new LineReader(arpaFileName);
+
+ try {
+ for (String line : grammarReader) {
+
+
+ // while (scanner.hasNext()) {
+ //
+ // String line = scanner.nextLine();
+
+ String[] parts = Regex.spaces.split(line);
+ if (parts.length > 1) {
+ String[] words = Regex.spaces.split(parts[1]);
+
+ for (String word : words) {
+ LOG.debug("Adding to vocab: {}", word);
+ Vocabulary.addAll(word);
+ }
+ } else {
+ LOG.info(line);
+ }
+
+ }
+ } finally {
+ grammarReader.close();
+ }
+
+ //
+ // boolean lineIsHeader = NGRAM_HEADER.matches(line);
+ //
+ // while (lineIsHeader || BLANK_LINE.matches(line)) {
+ //
+ // if (lineIsHeader) {
+ // ngramOrder++;
+ // }
+ //
+ // if (scanner.hasNext()) {
+ // line = scanner.nextLine().trim();
+ // lineIsHeader = NGRAM_HEADER.matches(line);
+ // } else {
+ // logger.severe("Ran out of lines!");
+ // return;
+ // }
+ // }
+
+
+ //
+ // // Add word to vocab
+ // if (logger.isLoggable(Level.FINE)) logger.fine("Adding word to vocab: " + parts[ngramOrder]);
+ // vocab.addTerminal(parts[ngramOrder]);
+ //
+ // // Add context words to vocab
+ // for (int i=1; i<ngramOrder; i++) {
+ // if (logger.isLoggable(Level.FINE)) logger.fine("Adding context word to vocab: " + parts[i]);
+ // vocab.addTerminal(parts[i]);
+ // }
+
+ // }
+
+ LOG.info("Done constructing ArpaFile");
+
+ }
+
+ /**
+ * Gets the {@link org.apache.joshua.corpus.Vocabulary}
+ * associated with this object.
+ *
+ * @return the symbol table associated with this object
+ */
+ public Vocabulary getVocab() {
+ return vocab;
+ }
+
+ /**
+ * Gets the total number of n-grams
+ * in this ARPA language model file.
+ *
+ * @return total number of n-grams
+ * in this ARPA language model file
+ */
+ @SuppressWarnings("unused")
+ public int size() {
+
+ LOG.debug("Counting n-grams in ARPA file");
+ int count=0;
+
+ for (ArpaNgram ngram : this) {
+ count++;
+ }
+ LOG.debug("Done counting n-grams in ARPA file");
+
+ return count;
+ }
+
+ public int getOrder() throws FileNotFoundException {
+
+ Pattern pattern = Pattern.compile("^ngram (\\d+)=\\d+$");
+ LOG.debug("Pattern is {}", pattern);
+ @SuppressWarnings("resource")
+ final Scanner scanner = new Scanner(arpaFile);
+
+ int order = 0;
+
+ // Eat initial header lines
+ while (scanner.hasNextLine()) {
+ String line = scanner.nextLine();
+
+ if (NGRAM_HEADER.matches(line)) {
+ break;
+ } else {
+ Matcher matcher = pattern.matcher(line);
+ if (matcher.matches()) {
+ LOG.debug("DOES match: '{}'", line);
+ order = Integer.valueOf(matcher.group(1));
+ } else {
+ LOG.debug("Doesn't match: '{}'", line);
+ }
+ }
+ }
+
+ return order;
+ }
+
+ /**
+ * Gets an iterator capable of iterating
+ * over all n-grams in the ARPA file.
+ *
+ * @return an iterator capable of iterating
+ * over all n-grams in the ARPA file
+ */
+ @SuppressWarnings("resource")
+ public Iterator<ArpaNgram> iterator() {
+
+ try {
+ final Scanner scanner;
+
+ if (arpaFile.getName().endsWith("gz")) {
+ InputStream in = new GZIPInputStream(
+ new FileInputStream(arpaFile));
+ scanner = new Scanner(in);
+ } else {
+ scanner = new Scanner(arpaFile);
+ }
+
+ // Eat initial header lines
+ while (scanner.hasNextLine()) {
+ String line = scanner.nextLine();
+ LOG.debug("Discarding line: {}", line);
+ if (NGRAM_HEADER.matches(line)) {
+ break;
+ }
+ }
+
+ return new Iterator<ArpaNgram>() {
+
+ String nextLine = null;
+ int ngramOrder = 1;
+ // int id = 0;
+
+ public boolean hasNext() {
+
+ if (scanner.hasNext()) {
+
+ String line = scanner.nextLine();
+
+ boolean lineIsHeader = NGRAM_HEADER.matches(line) || NGRAM_END.matches(line);
+
+ while (lineIsHeader || BLANK_LINE.matches(line)) {
+
+ if (lineIsHeader) {
+ ngramOrder++;
+ }
+
+ if (scanner.hasNext()) {
+ line = scanner.nextLine().trim();
+ lineIsHeader = NGRAM_HEADER.matches(line) || NGRAM_END.matches(line);
+ } else {
+ nextLine = null;
+ return false;
+ }
+ }
+
+ nextLine = line;
+ return true;
+
+ } else {
+ nextLine = null;
+ return false;
+ }
+
+ }
+
+ public ArpaNgram next() {
+ if (nextLine!=null) {
+
+ String[] parts = Regex.spaces.split(nextLine);
+
+ float value = Float.valueOf(parts[0]);
+
+ int word = Vocabulary.id(parts[ngramOrder]);
+
+ int[] context = new int[ngramOrder-1];
+ for (int i=1; i<ngramOrder; i++) {
+ context[i-1] = Vocabulary.id(parts[i]);
+ }
+
+ float backoff;
+ if (parts.length > ngramOrder+1) {
+ backoff = Float.valueOf(parts[parts.length-1]);
+ } else {
+ backoff = ArpaNgram.DEFAULT_BACKOFF;
+ }
+
+ nextLine = null;
+ return new ArpaNgram(word, context, value, backoff);
+
+ } else {
+ throw new NoSuchElementException();
+ }
+ }
+
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ };
+ } catch (IOException e) {
+ LOG.error(e.getMessage(), e);
+ return null;
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/ArpaNgram.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/ArpaNgram.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/ArpaNgram.java
new file mode 100644
index 0000000..d0077d1
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/ArpaNgram.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.lm;
+
+/**
+ * Represents a single n-gram line
+ * from an ARPA language model file.
+ *
+ * @author Lane Schwartz
+ */
+public class ArpaNgram {
+
+
+ /** Indicates an invalid probability value. */
+ public static final float INVALID_VALUE = Float.NaN;
+
+ /** Default backoff value. */
+ public static final float DEFAULT_BACKOFF = 0.0f;
+
+ private final int word;
+ private final int[] context;
+ private final float value;
+ private final float backoff;
+ // private final int id;
+
+ public ArpaNgram(int word, int[] context, float value, float backoff) {
+ this.word = word;
+ this.context = context;
+ this.value = value;
+ this.backoff = backoff;
+ // this.id = id;
+ }
+
+ // public int getID() {
+ // return id;
+ // }
+
+ public int order() {
+ return context.length + 1;
+ }
+
+ public int getWord() {
+ return word;
+ }
+
+ public int[] getContext() {
+ return context;
+ }
+
+ public float getValue() {
+ return value;
+ }
+
+ public float getBackoff() {
+ return backoff;
+ }
+}
\ No newline at end of file
[13/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/hiero/class.map
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/hiero/class.map b/joshua-core/src/test/resources/bn-en/hiero/class.map
new file mode 100644
index 0000000..e90c7c2
--- /dev/null
+++ b/joshua-core/src/test/resources/bn-en/hiero/class.map
@@ -0,0 +1,5140 @@
+1985 0
+1990 0
+386 0
+390 0
+add-on 0
+adobe 0
+associated 0
+available 0
+boot 0
+care 0
+carnel 0
+carrying 0
+company 0
+compatible 0
+configuration 0
+console 0
+constituents 0
+convenience 0
+credit 0
+database 0
+desktop 0
+develop 0
+developers 0
+distributions 0
+elephant 0
+embedded 0
+enterprise 0
+environments 0
+equivalent 0
+exist 0
+experts 0
+free 0
+freely 0
+gaming 0
+gnome 0
+gnu 0
+graphical 0
+helsinki 0
+ibm 0
+image 0
+intel 0
+karnel 0
+kde 0
+kernel 0
+lamp 0
+lemke 0
+lemmke 0
+libraries 0
+license 0
+linus 0
+linux 0
+logo 0
+mac 0
+manager 0
+mark 0
+microsoft 0
+minix 0
+mobile 0
+naming 0
+needed 0
+nt 0
+operated 0
+operating 0
+os 0
+pearl 0
+phone 0
+polluted 0
+previously 0
+processor 0
+publishing 0
+purpose 0
+rail 0
+running 0
+separately 0
+shell 0
+so-called 0
+software 0
+softwares 0
+subsidiary 0
+systems 0
+task 0
+torvalds 0
+tux 0
+ubuntu 0
+unix 0
+unix-like 0
+usefulness 0
+vista 0
+window 0
+windows 0
+wine 0
+working 0
+x 0
+xp 0
+100 1
+1 1
+18 1
+250 1
+2.5 1
+30 1
+34 1
+35 1
+42 1
+44 1
+45 1
+46 1
+50,000 1
+5 1
+60 1
+6 1
+70 1
+74 1
+75 1
+80 1
+80.5 1
+97 1
+98 1
+ago 1
+almost 1
+americans 1
+approximate 1
+approximately 1
+around 1
+camps 1
+continents 1
+core 1
+crore 1
+crores 1
+crossed 1
+denoted 1
+devastating 1
+died 1
+distance 1
+double 1
+every 1
+everyday 1
+hundreds 1
+indians 1
+just 1
+lakh 1
+lakhs 1
+little 1
+live 1
+lives 1
+living 1
+migration 1
+million 1
+millions 1
+multiple 1
+native 1
+nearly 1
+number 1
+people 1
+peoples 1
+persons 1
+population 1
+refugees 1
+reserved 1
+reside 1
+route 1
+sign 1
+square 1
+stays 1
+survey 1
+thirty 1
+thousand 1
+thousands 1
+tongue 1
+years 1
+50 2
+acquired 2
+additionally 2
+african 2
+airways 2
+amendment 2
+america 2
+american 2
+arm 2
+bilingual 2
+british 2
+canada 2
+captured 2
+cartier 2
+chartered 2
+christopher 2
+citizens 2
+collectively 2
+colonialism 2
+colonies 2
+colony 2
+columbus 2
+consisting 2
+continental 2
+countries 2
+currently 2
+defeating 2
+dutch 2
+england 2
+establish 2
+establishment 2
+european 2
+europeans 2
+exists 2
+explorer 2
+federal 2
+fifty 2
+governed 2
+hawaii 2
+hemisphere 2
+indigenous 2
+ireland 2
+isles 2
+israel 2
+italy 2
+kingdom 2
+legally 2
+lmi 2
+map 2
+massachusetts 2
+mexico 2
+multicultural 2
+name 2
+nation 2
+netherlands 2
+new 2
+permanent 2
+portugal 2
+presently 2
+probably 2
+puerto 2
+reached 2
+refer 2
+renamed 2
+resident 2
+rico 2
+</s> 2
+settlers 2
+seventh 2
+signing 2
+spain 2
+st. 2
+states 2
+step 2
+submarines 2
+telegram 2
+texas 2
+third 2
+thirteen 2
+thirteenth 2
+trademark 2
+uk 2
+un 2
+united 2
+u.s. 2
+usa 2
+washington 2
+york 2
+zealand 2
+1946 3
+abacus 3
+analog 3
+analogue 3
+architecture 3
+attempt 3
+bit 3
+calculating 3
+calculator 3
+calculus 3
+capable 3
+chemistry 3
+chip 3
+circuit 3
+circuits 3
+commercial 3
+computer 3
+computers 3
+computing 3
+construction 3
+contributions 3
+design 3
+desired 3
+developed 3
+device 3
+digital 3
+economically 3
+electro 3
+electro-mechanical 3
+electromechanical 3
+expected 3
+flight 3
+formal 3
+gottfried 3
+groups 3
+hence 3
+household 3
+hybrid 3
+improved 3
+invented 3
+inventing 3
+invention 3
+laboratories 3
+launching 3
+leibniz 3
+level 3
+machine 3
+machineries 3
+machinery 3
+machines 3
+magnetic 3
+magnetron 3
+mainframe 3
+manufactured 3
+mathematician 3
+mathematicians 3
+mechanical 3
+micro 3
+microelectronic 3
+microprocessor 3
+mini 3
+modern 3
+modulation 3
+motor 3
+pascal 3
+paul 3
+pc 3
+performance 3
+personal 3
+primarily 3
+print 3
+quantum 3
+ram 3
+recently 3
+reckoning 3
+research 3
+researches 3
+science 3
+scientific 3
+shaped 3
+shared 3
+shown 3
+skills 3
+speed 3
+starting 3
+such 3
+super 3
+techniques 3
+technology 3
+telecommunication 3
+testing 3
+tools 3
+transistors 3
+ultra 3
+unique 3
+usage 3
+von 3
+wave 3
+well 3
+well-known 3
+1000 4
+1916 4
+2,00,000 4
+360 4
+4.4 4
+71 4
+8.5 4
+95 4
+abscess 4
+animals 4
+anus 4
+arousal 4
+artificial 4
+aryabhatta 4
+atleast 4
+biological 4
+both 4
+breadth 4
+breast 4
+canal 4
+cervix 4
+cleaning 4
+clitoris 4
+cm 4
+covers 4
+custom 4
+degrees 4
+diameter 4
+differs 4
+dividing 4
+e. 4
+each 4
+effective 4
+eighteen 4
+either 4
+ejaculates 4
+enjoy 4
+entry 4
+excited 4
+female 4
+females 4
+finger 4
+fingers 4
+flexible 4
+fluids 4
+folds 4
+girls 4
+glands 4
+hand 4
+hands 4
+handsex 4
+inner 4
+inserted 4
+inserting 4
+inside 4
+j. 4
+kissing 4
+lady 4
+least 4
+length 4
+licking 4
+liquid 4
+lubricating 4
+lubrication 4
+lumps 4
+male 4
+mammals 4
+masterbation 4
+masturbate 4
+masturbated 4
+masturbating 4
+masturbation 4
+mating 4
+membrane 4
+men 4
+moving 4
+mucus 4
+mutual 4
+normal 4
+opening 4
+organ 4
+organs 4
+ovum 4
+own 4
+partner 4
+passes 4
+penetrating 4
+penetration 4
+penis 4
+performed 4
+pregnant 4
+produces 4
+prostate 4
+proved 4
+regulated 4
+reproductive 4
+reptiles 4
+rubbing 4
+seconds 4
+secretion 4
+secrets 4
+semen 4
+sex 4
+sexually 4
+shaking 4
+shows 4
+sometime 4
+sperm 4
+stands 4
+stimulated 4
+stimulation 4
+straight 4
+their 4
+thin 4
+tissue 4
+touch 4
+touching 4
+train 4
+two-thirds 4
+upper 4
+urge 4
+usually 4
+uterus 4
+vagina 4
+vaginal 4
+vibrator 4
+vulva 4
+wall 4
+walls 4
+wash 4
+weeks 4
+wise 4
+woman 4
+women 4
+, 5
+\u2018 5
+\u2019 5
+achievements 5
+agra 5
+among 5
+amongst 5
+andhra 5
+architectural 5
+architectures 5
+attained 5
+automobiles 5
+ballal 5
+bbc 5
+birth> 5
+bright 5
+broadcasting 5
+btv 5
+buddha 5
+cars 5
+channels 5
+chapter 5
+chess 5
+christmas 5
+coal 5
+color 5
+comprehensive 5
+cooking 5
+dance 5
+dances 5
+dancing 5
+descriptive 5
+desh 5
+dialogue 5
+diamond 5
+discovery 5
+diversity 5
+dramas 5
+empty 5
+epics 5
+etc 5
+faraday 5
+forms 5
+ganesh 5
+gujarat 5
+holds 5
+hugely 5
+immense 5
+include 5
+including 5
+induction 5
+jayanti 5
+jointly 5
+kannada 5
+karnataka 5
+kerala 5
+lion 5
+manipuri 5
+marathi 5
+maxwell 5
+mention 5
+mentionable 5
+mentioning 5
+music 5
+mythological 5
+notable 5
+ore 5
+orissa 5
+popular 5
+pradesh 5
+ramna 5
+regional 5
+remarkable 5
+robot 5
+rockefeller 5
+s 5
+sangbad 5
+sangeet 5
+selling 5
+speciality 5
+specialty 5
+styles 5
+successfully 5
+sun 5
+tamil 5
+telegu 5
+television 5
+terrestrial 5
+traditional 5
+trophy 5
+tv 5
+unicef 5
+variety 5
+volume 5
+voted 5
+worth 5
+1950s 6
+1991 6
+20th 6
+54 6
+achieved 6
+advanced 6
+arena 6
+been 6
+centuries 6
+comparatively 6
+decade 6
+decades 6
+expansion 6
+fastest 6
+fifth 6
+growing 6
+half 6
+has 6
+in 6
+increased 6
+india 6
+its 6
+latter 6
+nuclear 6
+past 6
+popularity 6
+power 6
+powerful 6
+progress 6
+prosperity 6
+rapid 6
+rapidly 6
+recent 6
+reconstructed 6
+reduced 6
+reforms 6
+richest 6
+second 6
+smiling 6
+technologically 6
+terms 6
+twentieth 6
+twenty 6
+underground 6
+us 6
+world 6
+worldwide 6
+absence 7
+achieve 7
+activity 7
+adopt 7
+adverse 7
+an 7
+attracted 7
+automatically 7
+bartholin 7
+becomes 7
+biting 7
+bodily 7
+body 7
+case 7
+climax 7
+come 7
+comes 7
+conflict 7
+copulation 7
+create 7
+creatures 7
+criminal 7
+desire 7
+ejaculation 7
+enjoyment 7
+excitement 7
+existence 7
+expands 7
+extreme 7
+false 7
+foreplay 7
+friction 7
+fungus 7
+gets 7
+giving 7
+gland 7
+goes 7
+g-spot 7
+habit 7
+happens 7
+happiness 7
+holding 7
+hymen 7
+illegal 7
+intercourse 7
+kinds 7
+lila 7
+looks 7
+lust 7
+males 7
+man 7
+meets 7
+mouth 7
+needs 7
+normally 7
+objective 7
+opened 7
+opposite 7
+organisms 7
+orgasm 7
+orthodox 7
+paying 7
+perform 7
+perversion 7
+physical 7
+pleasure 7
+possibility 7
+pregnancy 7
+procreate 7
+reach 7
+reasons 7
+reduce 7
+reduces 7
+releasing 7
+remains 7
+reproduce 7
+reproduction 7
+researchers 7
+result 7
+ruptured 7
+sexual 7
+sexuality 7
+shape 7
+skin 7
+sky 7
+sperms 7
+starts 7
+takes 7
+target 7
+transmitted 7
+tube 7
+ultimate 7
+usual 7
+way 7
+ways 7
+whether 7
+zero 7
+13th 8
+1789 8
+1800 8
+18th 8
+384 8
+500 8
+6th 8
+700 8
+a.d 8
+a.d. 8
+ad 8
+age 8
+ages 8
+ajanta 8
+ancient 8
+appeared 8
+architect 8
+aristotle 8
+basin 8
+b.c 8
+b.c. 8
+bc 8
+bce 8
+cave 8
+ce 8
+century 8
+chinese 8
+civilisation 8
+civilization 8
+civilizations 8
+clay 8
+commenced 8
+discovered 8
+egypt 8
+egyptian 8
+egyptians 8
+entering 8
+era 8
+evidence 8
+evidences 8
+farasee 8
+flourished 8
+from 8
+golden 8
+greek 8
+grown 8
+heritage 8
+history 8
+indus 8
+khaldun 8
+lakes 8
+magadhi 8
+maharashtra 8
+maurya 8
+mid 8
+middle 8
+nineteenth 8
+old 8
+painting 8
+paintings 8
+periods 8
+persian 8
+pillar 8
+prehistoric 8
+preserved 8
+primitive 8
+prokousal 8
+rock 8
+scattered 8
+seventeenth 8
+tales 8
+till 8
+traders 8
+until 8
+vedic 8
+11 9
+13 9
+1964 9
+1968 9
+1975 9
+1979 9
+1980 9
+1982 9
+1986 9
+1987 9
+1992 9
+1995 9
+2006 9
+29 9
+3 9
+\u2013 9
+album 9
+albums 9
+april 9
+award 9
+awarded 9
+awards 9
+black 9
+blue 9
+businessman 9
+children 9
+civilian 9
+columbia 9
+conquer 9
+dangerous 9
+date 9
+day 9
+death 9
+destruction 9
+document 9
+dominion 9
+expanding 9
+expired 9
+fame 9
+first 9
+former 9
+happened 9
+holiday 9
+hour 9
+incident 9
+independance 9
+jackson 9
+joseph 9
+julius 9
+la 9
+landed 9
+maria 9
+martin 9
+mexican 9
+michael 9
+michel 9
+minute 9
+moon 9
+mourning 9
+mtv 9
+oscar 9
+peter 9
+pop 9
+prestigious 9
+race 9
+ratna 9
+received 9
+records 9
+rest 9
+retired 9
+scenario 9
+shortly 9
+showing 9
+singer 9
+singing 9
+solo 9
+swedish 9
+three 9
+venice 9
+won 9
+1913 10
+1993 10
+academic 10
+academy 10
+art 10
+arts 10
+azad 10
+basu 10
+basudev 10
+bazle 10
+bose 10
+chandra 10
+choto 10
+classical 10
+commemorate 10
+compose 10
+composing 10
+conferred 10
+cultivation 10
+dasgupta 10
+dedicated 10
+depicted 10
+doctor 10
+doctorate 10
+dr. 10
+drama 10
+earned 10
+edward 10
+eminent 10
+equally 10
+exile 10
+expert 10
+familiar 10
+famous 10
+festival 10
+fine 10
+folk 10
+fr 10
+gatherings 10
+ghatak 10
+globe 10
+hafiz 10
+hall 10
+hind 10
+historic 10
+honor 10
+honorary 10
+honour 10
+idol 10
+inception 10
+introduction 10
+jagannath 10
+jerry 10
+karim 10
+katra 10
+lahiri 10
+lato 10
+learned 10
+learning 10
+learnt 10
+leto 10
+literacy 10
+literary 10
+literature 10
+litt�rateur 10
+litterateurs 10
+lover 10
+maintained 10
+maqtab 10
+mathrun 10
+mazar 10
+minds 10
+mosque 10
+motion 10
+nagendranath 10
+nath 10
+oxford 10
+personality 10
+philosophy 10
+platonism 10
+poet 10
+poetry 10
+poets 10
+practicing 10
+prasad 10
+publisher 10
+puja 10
+punjabi 10
+rabindra 10
+ranjan 10
+rebellious 10
+relative 10
+renowned 10
+respected 10
+revolutionary 10
+shantiniketan 10
+songs 10
+studied 10
+subhas 10
+supervision 10
+targets 10
+teacher 10
+teachers 10
+theater 10
+thinking 10
+visiting 10
+who 10
+wish 10
+.. 11
+1959 11
+37 11
+acknowledged 11
+acted 11
+actor 11
+actors 11
+actress 11
+affirmative 11
+akira 11
+alien 11
+amal 11
+anderson 11
+andrew 11
+answer 11
+aparajito 11
+aparna 11
+apparently 11
+appreciated 11
+apu 11
+apur 11
+aranyer 11
+artist 11
+artists 11
+attention 11
+audience 11
+bagha 11
+bandopadhyay 11
+banerjee 11
+bangshi 11
+bansi 11
+best 11
+bibhuti 11
+biciclette 11
+bijoya 11
+binod 11
+bollywood 11
+brando 11
+brother 11
+camera 11
+career 11
+cast 11
+casting 11
+chalachitra 11
+challenge 11
+chandragupta 11
+chaplin 11
+character 11
+characters 11
+charlie 11
+charu 11
+charulata 11
+chattopadhyay 11
+cinema 11
+cinemas 11
+cinematic 11
+claimed 11
+classic 11
+colour 11
+comedy 11
+commented 11
+companion 11
+compare 11
+complained 11
+completed 11
+completion 11
+conjecture 11
+copies 11
+cousin 11
+creative 11
+critic 11
+critics 11
+das 11
+debi 11
+deeply 11
+designed 11
+designer 11
+details 11
+detective 11
+di 11
+din 11
+directed 11
+direction 11
+directions 11
+director 11
+directors 11
+diwali 11
+documentary 11
+don 11
+draw 11
+drew 11
+dulal 11
+duration 11
+durga 11
+dutta 11
+editing 11
+enthusiastic 11
+episode 11
+evil 11
+expensive 11
+experiences 11
+eye 11
+eyes 11
+faced 11
+favorite 11
+favourite 11
+felt 11
+feluda 11
+fiction 11
+film 11
+filming 11
+filmmaker 11
+films 11
+finished 11
+flawless 11
+ghosh 11
+godard 11
+goddess 11
+goopy 11
+goutam 11
+grandfather 11
+graphic 11
+gupi 11
+helped 11
+hero 11
+highly 11
+hit 11
+humanism 11
+humanity 11
+impression 11
+indoor 11
+indranath 11
+inspired 11
+instance 11
+integral 11
+italian 11
+james 11
+jean 11
+jean-luc 11
+joy 11
+judge 11
+kali 11
+kanya 11
+kella 11
+khiladi 11
+kumar 11
+kurosawa 11
+ladri 11
+legend 11
+light 11
+lights 11
+looked 11
+lot 11
+lyrics 11
+made 11
+madhabi 11
+mahapurush 11
+maker 11
+makers 11
+marlon 11
+mitra 11
+mood 11
+movie 11
+movies 11
+mrinal 11
+mukherjee 11
+mukhopadhyay 11
+musical 11
+musicians 11
+non-bengali 11
+pachali 11
+paid 11
+painter 11
+panchali 11
+pather 11
+personalities 11
+phase 11
+photograph 11
+picture 11
+pioneering 11
+play 11
+praise 11
+praised 11
+pratidwandi 11
+premchand 11
+produced 11
+rabindranath 11
+rare 11
+ratan 11
+ratri 11
+ray 11
+realised 11
+really 11
+regard 11
+remarked 11
+remembering 11
+renoir 11
+robin 11
+robinson 11
+roy 11
+'s 11
+s. 11
+sacred 11
+sadgati 11
+salman 11
+sansar 11
+satyajit 11
+saying 11
+scene 11
+scenes 11
+screen 11
+screenplay 11
+script 11
+scripts 11
+sen 11
+series 11
+serious 11
+shades 11
+shakespeare 11
+sharmila 11
+shooting 11
+shot 11
+sight 11
+sir 11
+slow 11
+soft 11
+sonar 11
+stamp 11
+star 11
+stars 11
+story 11
+subrata 11
+subroto 11
+success 11
+successful 11
+sukumar 11
+t 11
+tagore 11
+talent 11
+technicians 11
+teen 11
+tension 11
+thakur 11
+trilogy 11
+truffaut 11
+upendrakishore 11
+uttam 11
+viewers 11
+vision 11
+watch 11
+watching 11
+wood 11
++ 12
+a 12
+adams 12
+advantage 12
+another 12
+assembled 12
+base 12
+basis 12
+c 12
+co 12
+code 12
+codes 12
+collect 12
+complete 12
+controllers 12
+converted 12
+counter 12
+cover 12
+directly 12
+documentation 12
+drawing 12
+effort 12
+emulator 12
+endeavor 12
+ending 12
+execution 12
+feature 12
+finally 12
+followed 12
+foundation 12
+function 12
+gather 12
+generic 12
+gonu 12
+gpl 12
+hard 12
+hurd 12
+ignored 12
+improvement 12
+incredible 12
+independently 12
+ingredient 12
+initial 12
+initially 12
+inspiration 12
+instructions 12
+java 12
+job 12
+latest 12
+library 12
+likely 12
+limit 12
+magnus 12
+making 12
+managed 12
+mit 12
+net 12
+niazi 12
+o 12
+object 12
+oriented 12
+platform 12
+portability 12
+preparation 12
+prepared 12
+preparing 12
+preserve 12
+press 12
+program 12
+programing 12
+programmers 12
+programming 12
+programs 12
+project 12
+removed 12
+request 12
+richard 12
+run 12
+sellers 12
+separate 12
+short 12
+simultaneously 12
+single 12
+stallman 12
+steps 12
+subsequently 12
+supporting 12
+syntax 12
+taken 12
+technologists 12
+terminal 12
+threads 12
+together 12
+transformed 12
+verdict 12
+versions 12
+wanted 12
+work 12
+write 12
+1936 13
+80s 13
+bachelor 13
+basically 13
+benefit 13
+biomedical 13
+california 13
+cambridge 13
+certificate 13
+college 13
+curriculum 13
+degree 13
+designs 13
+division 13
+education 13
+educational 13
+electric 13
+electrical 13
+electricity 13
+engineer 13
+engineering 13
+engineers 13
+excellent 13
+graduate 13
+graduation 13
+high 13
+higher 13
+hypothesis 13
+ic 13
+institute 13
+institutes 13
+institution 13
+kg 13
+mainly 13
+medical 13
+medium 13
+physics 13
+primary 13
+pritilata 13
+private 13
+professional 13
+professionalism 13
+public 13
+radar 13
+santa 13
+school 13
+schools 13
+sculptor 13
+spencer 13
+stage 13
+study 13
+supervise 13
+syllabus 13
+thomas 13
+universities 13
+university 13
+whose 13
+works 13
+1966 14
+1994 14
+accident 14
+achievement 14
+acquire 14
+affair 14
+afraid 14
+afternoon 14
+alexander 14
+anyway 14
+arrange 14
+atarneus 14
+athens 14
+attain 14
+baire 14
+becoming 14
+biswas 14
+bride 14
+causing 14
+chance 14
+childhood 14
+chobi 14
+clerk 14
+coming 14
+completing 14
+concentration 14
+concern 14
+corruption 14
+couple 14
+crime 14
+darjeeling 14
+dear 14
+detail 14
+devi 14
+divorced 14
+during 14
+earn 14
+exactly 14
+exposed 14
+family 14
+final 14
+finishing 14
+firstly 14
+fled 14
+forget 14
+gain 14
+gathered 14
+gave 14
+generations 14
+ghare 14
+girl 14
+got 14
+he 14
+her 14
+him 14
+his 14
+homage 14
+home 14
+hotel 14
+husband 14
+ill 14
+illness 14
+improve 14
+inability 14
+indifferent 14
+informed 14
+intense 14
+internationally 14
+involved 14
+last 14
+later 14
+lead 14
+leave 14
+lecture 14
+life 14
+lifetime 14
+lived 14
+loose 14
+lost 14
+love 14
+maktab 14
+mansion 14
+marriage 14
+married 14
+mental 14
+mentally 14
+minutes 14
+naxalite 14
+next 14
+pay 14
+philip 14
+plato 14
+poor 14
+promila 14
+question 14
+questions 14
+reaching 14
+recovered 14
+resort 14
+revealed 14
+russel 14
+sandip 14
+santiniketan 14
+savings 14
+saw 14
+secret 14
+senior 14
+she 14
+shoot 14
+situations 14
+son 14
+sons 14
+span 14
+spend 14
+spending 14
+spent 14
+stay 14
+stranger 14
+suddenly 14
+teaching 14
+travel 14
+uncle 14
+visit 14
+watched 14
+week 14
+whom 14
+wife 14
+william 14
+wishes 14
+young 14
+1907 15
+1937 15
+1940 15
+1943 15
+1989 15
+a. 15
+abdul 15
+abu 15
+acting 15
+adda 15
+advertising 15
+ahamed 15
+ahmad 15
+ahmed 15
+a.k. 15
+akbar 15
+ali 15
+asansol 15
+bangiya 15
+basundhara 15
+begum 15
+bharati 15
+bharatiya 15
+bhasani 15
+bhavan 15
+bidhan 15
+block 15
+bongobondhu 15
+brothers 15
+ceremony 15
+charge 15
+chosen 15
+close 15
+companions 15
+d. 15
+daughter 15
+dhak 15
+d.j. 15
+dr 15
+editor 15
+elder 15
+extraordinary 15
+fazlul 15
+flag 15
+friend 15
+gandhi 15
+grave 15
+guard 15
+gupta 15
+hamid 15
+haq 15
+haque 15
+hirok 15
+honorable 15
+hossain 15
+house 15
+hussain 15
+indira 15
+invitation 15
+invited 15
+jawaharlal 15
+joining 15
+journalism 15
+journalist 15
+junior 15
+k 15
+k. 15
+kaji 15
+kazi 15
+kill 15
+leaves 15
+leaving 15
+left 15
+lt. 15
+mahatma 15
+master 15
+met 15
+mohammad 15
+motahar 15
+muezzin 15
+mujaffar 15
+najrul 15
+nargis 15
+nehru 15
+nurul 15
+occasion 15
+office 15
+oxygen 15
+participated 15
+participating 15
+politician 15
+prime-minister 15
+prisoner 15
+publicity 15
+raja 15
+red 15
+refused 15
+requested 15
+requesting 15
+responsibilities 15
+sahid 15
+sahitya 15
+samiti 15
+sang 15
+sayed 15
+served 15
+shaheed 15
+shahidullah 15
+shake 15
+signet 15
+silence 15
+sing 15
+sister 15
+sisters 15
+sub-division 15
+syed 15
+tajuddin 15
+vasani 15
+wooden 15
+worked 15
+younger 15
+youngest 15
+* 16
+& 16
+208.77.188.166 16
+abc 16
+access 16
+accounting 16
+actually 16
+adding 16
+address 16
+addresses 16
+agent 16
+application 16
+arpanet 16
+assigning 16
+attributes 16
+b 16
+baby 16
+bind 16
+board 16
+byte 16
+can 16
+capacity 16
+card 16
+cde.com 16
+centralized 16
+client 16
+cluster 16
+combining 16
+commonly 16
+compilation 16
+components 16
+compulsory 16
+connect 16
+connected 16
+control 16
+controlling 16
+controls 16
+cpu 16
+dependent 16
+depending 16
+describe 16
+designing 16
+devices 16
+directory 16
+disk 16
+dns 16
+domain 16
+door 16
+dynamically 16
+easier 16
+electronic 16
+e-mail 16
+email 16
+enter 16
+equipments 16
+fiber 16
+file 16
+files 16
+ftp 16
+grid 16
+host 16
+hosts.txt 16
+humanware 16
+input 16
+instrument 16
+integrated 16
+interface 16
+ip 16
+jobs 16
+keeps 16
+layer 16
+lists 16
+mail 16
+maintenance 16
+managing 16
+mapping 16
+memory 16
+mixing 16
+mode 16
+monitor 16
+moreover 16
+mouse 16
+multimedia 16
+multiprogramming 16
+names 16
+need 16
+network 16
+networking 16
+networks 16
+or 16
+output 16
+papers 16
+parces 16
+printer 16
+processing 16
+programmes 16
+protection 16
+protocol 16
+provide 16
+provider 16
+receiver 16
+redirection 16
+registration 16
+reliable 16
+remember 16
+runs 16
+safe 16
+scanner 16
+scheduling 16
+sender 16
+sending 16
+serial 16
+serve 16
+signal 16
+signals 16
+silicon 16
+soap 16
+specially 16
+storage 16
+store 16
+suit 16
+suitable 16
+suite 16
+system 16
+tasks 16
+tcp 16
+therefore 16
+timer 16
+transfer 16
+transmission 16
+transmitter 16
+unit 16
+url 16
+useful 16
+user 16
+users 16
+uses 16
+valid 16
+vehicle 16
+vehicles 16
+visible 16
+wait 16
+; 17
+1867 17
+52 17
+600 17
+administrative 17
+appears 17
+assamese 17
+belgium 17
+beyond 17
+celtic 17
+charles 17
+cletis 17
+colonial 17
+communities 17
+consonantal 17
+de 17
+dialect 17
+dialects 17
+dignity 17
+diversified 17
+divisions 17
+earlier 17
+empire 17
+empires 17
+english 17
+entire 17
+evolved 17
+farsi 17
+france 17
+french 17
+gall 17
+galls 17
+gaul 17
+gauls 17
+geographically 17
+german 17
+germans 17
+germany 17
+gol 17
+gols 17
+grammar 17
+greeks 17
+hindi 17
+historically 17
+indo-aryan 17
+indo-european 17
+invader 17
+isolated 17
+language 17
+languages 17
+latin 17
+local 17
+-lsb- 17
+luxembourg 17
+medieval 17
+newly 17
+norman 17
+normans 17
+northern 17
+occupied 17
+official 17
+oldest 17
+originated 17
+oriya 17
+pali 17
+paris 17
+period 17
+practiced 17
+prakrit 17
+prakrits 17
+prevalent 17
+pronunciation 17
+roman 17
+romans 17
+-rsb- 17
+sanskrit 17
+shift 17
+similarities 17
+soil 17
+sound 17
+spanish 17
+speak 17
+speaking 17
+spoken 17
+spread 17
+stable 17
+style 17
+sub 17
+switzerland 17
+taught 17
+tribal 17
+tribe 17
+tribes 17
+unorganized 17
+urdu 17
+verbs 17
+versatile 17
+vocabulary 17
+widely 17
+win 17
+witness 17
+? 18
+ability 18
+adequate 18
+akin 18
+allowed 18
+alms 18
+alone 18
+always 18
+am 18
+angel 18
+any 18
+anything 18
+attitude 18
+avoid 18
+be 18
+burnt 18
+ca 18
+cease 18
+cerebral 18
+clean 18
+complain 18
+condition 18
+consider 18
+continue 18
+could 18
+cure 18
+did 18
+difficult 18
+do 18
+does 18
+done 18
+else 18
+enemy 18
+enough 18
+entirely 18
+everything 18
+fear 18
+feel 18
+feelings 18
+fees 18
+finding 18
+finds 18
+further 18
+future 18
+get 18
+give 18
+go 18
+happen 18
+happy 18
+heart 18
+however 18
+i 18
+if 18
+impossible 18
+indicate 18
+inform 18
+instead 18
+it 18
+judgment 18
+kindly 18
+know 18
+le 18
+learn 18
+let 18
+look 18
+looking 18
+loved 18
+mackisk 18
+make 18
+manner 18
+me 18
+might 18
+mind 18
+mistakes 18
+must 18
+never 18
+no 18
+not 18
+nothing 18
+n't 18
+offer 18
+permitted 18
+possible 18
+properly 18
+prove 18
+rather 18
+read 18
+reading 18
+regarding 18
+remained 18
+remove 18
+replied 18
+represent 18
+representing 18
+right 18
+said 18
+sake 18
+save 18
+say 18
+says 18
+send 18
+should 18
+similarly 18
+so 18
+someone 18
+sort 18
+stand 18
+sure 18
+surely 18
+surgery 18
+take 18
+tasawuf 18
+tasawwuf 18
+tell 18
+that 18
+themselves 18
+then 18
+they 18
+thing 18
+think 18
+thought 18
+thrown 18
+to 18
+told 18
+totally 18
+treatment 18
+turkish 18
+turn 18
+ulma 18
+unable 18
+want 18
+we 18
+what 18
+whatever 18
+whenever 18
+wherever 18
+why 18
+will 18
+willing 18
+without 18
+would 18
+wrong 18
+you 18
+your 18
+. 19
+1919 19
+acquainted 19
+active 19
+adolf 19
+aim 19
+became 19
+begins 19
+broken 19
+clear 19
+closer 19
+communist 19
+consciousness 19
+discrimination 19
+drive 19
+group 19
+had 19
+idealism 19
+imperialism 19
+influenced 19
+influential 19
+involvement 19
+islami 19
+islamic 19
+joined 19
+kamal 19
+khilafat 19
+leader 19
+meetings 19
+movement 19
+movements 19
+muslim 19
+muzaffar 19
+nazrul 19
+non 19
+non-cooperation 19
+opportunity 19
+participation 19
+pasa 19
+pasha 19
+pioneer 19
+prominent 19
+rebel 19
+rebellion 19
+revolt 19
+revolution 19
+role 19
+samaj 19
+socialist 19
+socialistic 19
+strongly 19
+succeeded 19
+supporter 19
+torture 19
+turkey 19
+turks 19
+1776 20
+1870 20
+1910 20
+= 20
+about 20
+acceptable 20
+accepted 20
+adam 20
+agreed 20
+alfred 20
+applicable 20
+article 20
+ascribed 20
+assumptions 20
+background 20
+basic 20
+bazaar 20
+broader 20
+capitalism 20
+classification 20
+concept 20
+concise 20
+considered 20
+contemporary 20
+context 20
+creation 20
+critical 20
+criticism 20
+criticisms 20
+criticized 20
+david 20
+debate 20
+definition 20
+depression 20
+described 20
+detailed 20
+differences 20
+discussed 20
+discussion 20
+doctrine 20
+economics 20
+economist 20
+economists 20
+economy 20
+elementary 20
+employment 20
+essay 20
+examples 20
+explained 20
+explanations 20
+expression 20
+fair 20
+frame 20
+generator 20
+harshly 20
+heterodox 20
+idea 20
+idealistic 20
+identity 20
+ideology 20
+institutional 20
+involving 20
+john 20
+karl 20
+keeping 20
+keynesian 20
+macro 20
+mainstream 20
+malthus 20
+marx 20
+marxian 20
+marxist 20
+matters 20
+mentality 20
+mill 20
+mixture 20
+neo 20
+neoclassical 20
+opinion 20
+opinions 20
+original 20
+overcome 20
+preliminary 20
+presentation 20
+principle 20
+principles 20
+realistic 20
+referred 20
+reflected 20
+regarded 20
+robert 20
+samuelson 20
+sciences 20
+self 20
+smith 20
+social 20
+society 20
+sometimes 20
+stated 20
+stating 20
+stuart 20
+summary 20
+sympathy 20
+synonym 20
+termed 20
+theme 20
+thoughts 20
+topic 20
+treated 20
+truth 20
+unlimited 20
+unrealistic 20
+unselfish 20
+utilization 20
+valuable 20
+views 20
+wealth 20
+welfare 20
+1917 21
+610 21
+advent 21
+axis 21
+barbican 21
+biggest 21
+bolsheviks 21
+catholic 21
+changed 21
+christian 21
+christianity 21
+church 21
+closed 21
+collapsed 21
+conquered 21
+contact 21
+continuously 21
+country 21
+culture 21
+cultures 21
+damaged 21
+decline 21
+down 21
+dynasty 21
+elizabeth 21
+emerged 21
+ended 21
+ends 21
+entered 21
+enters 21
+europe 21
+fall 21
+fallen 21
+fought 21
+fourth 21
+friendly 21
+furious 21
+grand 21
+greatly 21
+henry 21
+iberian 21
+ii 21
+ipa 21
+jack 21
+largest 21
+millennium 21
+monarchy 21
+moscow 21
+occurred 21
+of 21
+petersburg 21
+physically 21
+position 21
+queen 21
+reign 21
+revolutionaries 21
+rome 21
+russia 21
+russian 21
+safety 21
+saint 21
+severely 21
+sixth 21
+societies 21
+soviet 21
+strictly 21
+subsequent 21
+sydney 21
+the 21
+transferred 21
+warfare 21
+welcomed 21
+worlds 21
+yongle 21
+3000 22
+40 22
+all 22
+also 22
+apart 22
+are 22
+bars 22
+besides 22
+bus 22
+compared 22
+contains 22
+customs 22
+different 22
+dresses 22
+educated 22
+engaged 22
+enthusiasm 22
+especially 22
+etc. 22
+festivals 22
+functions 22
+have 22
+here 22
+heritages 22
+hundred 22
+impressed 22
+like 22
+liked 22
+lots 22
+many 22
+more 22
+multinational 22
+numerous 22
+ones 22
+opera 22
+organize 22
+other 22
+others 22
+pattern 22
+places 22
+plants 22
+professionals 22
+races 22
+scholars 22
+several 22
+shops 22
+some 22
+stages 22
+streams 22
+symbol 22
+than 22
+them 22
+there 22
+these 22
+those 22
+too 22
+tourism 22
+traditions 22
+understand 22
+various 22
+were 22
+acquiring 23
+addition 23
+advertisement 23
+algebra 23
+algebraic 23
+alive 23
+analytical 23
+angle 23
+arithmetic 23
+arithmetics 23
+aspect 23
+aspects 23
+astrology 23
+astronomy 23
+automation 23
+babylonian 23
+babylonians 23
+beautiful 23
+better 23
+calculate 23
+calculation 23
+calculations 23
+calender 23
+certain 23
+circle 23
+cited 23
+cloth 23
+communicate 23
+complex 23
+complexity 23
+compound 23
+concerned 23
+conclusions 23
+contain 23
+containing 23
+corn 23
+correct 23
+counting 23
+creating 23
+crop 23
+cube 23
+cuneiform 23
+definitely 23
+differently 23
+discrete 23
+drawn 23
+easily 23
+easy 23
+equations 23
+everywhere 23
+exact 23
+experience 23
+experienced 23
+experiment 23
+experimentation 23
+expertise 23
+express 23
+expressing 23
+extra 23
+extremely 23
+find 23
+following 23
+forecast 23
+formula 23
+foundations 23
+founders 23
+generally 23
+geometrical 23
+geometry 23
+identify 23
+infact 23
+instruments 23
+knew 23
+knowledge 23
+lighting 23
+linear 23
+marginality 23
+material 23
+math 23
+mathematical 23
+mathematics 23
+mccloskey 23
+methods 23
+models 23
+motors 23
+much 23
+multiplication 23
+negative 23
+numbers 23
+numerical 23
+objects 23
+often 23
+ordinary 23
+otherwise 23
+outcome 23
+performs 23
+pictures 23
+planets 23
+positions 23
+practices 23
+predict 23
+prevailing 23
+problem 23
+problems 23
+procedure 23
+proof 23
+proofs 23
+properties 23
+property 23
+pursuit 23
+quadratic 23
+quality 23
+quick 23
+quite 23
+rational 23
+real 23
+receive 23
+receiving 23
+refers 23
+required 23
+results 23
+root 23
+roots 23
+satisfy 23
+scientists 23
+selection 23
+signs 23
+simple 23
+simplicity 23
+simplified 23
+solve 23
+solved 23
+solving 23
+something 23
+specific 23
+subjects 23
+subtraction 23
+sufficient 23
+symbols 23
+tablets 23
+taslima 23
+technical 23
+technique 23
+technologies 23
+ten 23
+tense 23
+tenses 23
+tesla 23
+theorem 23
+theorems 23
+theoretical 23
+tiny 23
+track 23
+transformation 23
+true 23
+universal 23
+universe 23
+unlike 23
+unnecessary 23
+use 23
+yet 23
+1.0 24
+10 24
+1983 24
+2000 24
+2001 24
+2002 24
+2003 24
+2004 24
+2005 24
+2007 24
+20 24
+22 24
+2 24
+26 24
+300 24
+39 24
+90 24
+99 24
+amsterdam 24
+arrival 24
+before 24
+cancer 24
+download 24
+downloaded 24
+early 24
+efforts 24
+engine 24
+essays 24
+florida 24
+homepage 24
+jimmy 24
+larry 24
+launched 24
+lee 24
+mailing 24
+nupedia 24
+publish 24
+sanger 24
+set 24
+settlements 24
+version 24
+volumes 24
+wales 24
+wikimedia 24
+wikipedia 24
+within 24
+" 25
+\u201c 25
+\u201d 25
+@ 25
+adjective 25
+adjectives 25
+are- 25
+as 25
+blanche 25
+blog 25
+blogger 25
+cafe 25
+called 25
+car 25
+comparison 25
+compute 25
+conversation 25
+definite 25
+derived 25
+diode 25
+dit 25
+donated 25
+etre 25
+evan 25
+example 25
+feminine 25
+form 25
+gender 25
+hawaiian 25
+indicates 25
+intelligence 25
+is 25
+je 25
+known 25
+labs 25
+lawrence 25
+marked 25
+meaning 25
+means 25
+napier 25
+ne 25
+negation 25
+news 25
+nikola 25
+noun 25
+nouns 25
+origin 25
+particularly 25
+pas 25
+passive 25
+phrase 25
+plural 25
+posts 25
+prithvi 25
+pronoun 25
+pronouns 25
+pyra 25
+sais 25
+see 25
+singular 25
+slogan 25
+spelling 25
+talk 25
+term 25
+today 25
+une 25
+used 25
+vaginae 25
+values 25
+verb 25
+verbal 25
+voice 25
+voiture 25
+walking 25
+weblog 25
+white 25
+williams 25
+word 25
+yourself 25
+1889 26
+1899 26
+1976 26
+- 26
+: 26
+\u2014 26
+322 26
+amartya 26
+amateur 26
+author 26
+bangladeshi 26
+bengali 26
+birth 26
+births 26
+born 26
+category 26
+citizenship 26
+class 26
+composer 26
+conditional 26
+content 26
+contents 26
+copyright 26
+ctegory 26
+d 26
+dec 26
+ethnicity 26
+feminist 26
+follows 26
+four 26
+given 26
+greatest 26
+honored 26
+imperative 26
+indian 26
+introduced 26
+iron 26
+laureate 26
+legion 26
+lyricist 26
+mater 26
+matter 26
+musician 26
+nobel 26
+novelist 26
+oceania 26
+peace 26
+phalke 26
+philosopher 26
+prize 26
+professor 26
+roles 26
+scientist 26
+secondary 26
+sentence 26
+special 26
+subject 26
+subject-class 26
+subjectclass 26
+subject-line 26
+sweden 26
+topics 26
+winner 26
+writer 26
+writers 26
+1990s 27
+2.0 27
+< 27
+> 27
+882 27
+883 27
+add 27
+added 27
+alternate 27
+anandabazar 27
+anandbazar 27
+antigenic 27
+anybody 27
+anyone 27
+apache 27
+apple 27
+arrangement 27
+articles 27
+asynchronous 27
+attribute 27
+audio 27
+based 27
+berkeley 27
+bloggers 27
+blogging 27
+blogs 27
+browser 27
+browsers 27
+browsing 27
+burning 27
+cache 27
+calendar 27
+cell 27
+clients 27
+collection 27
+combination 27
+comment 27
+comments 27
+consist 27
+contained 27
+contribute 27
+current 27
+daniel 27
+diary 27
+display 27
+edition 27
+everybody 27
+evolution 27
+extension 27
+facilities 27
+facility 27
+features 27
+firefox 27
+forum 27
+google 27
+gregorian 27
+headings 27
+hosting 27
+html 27
+http 27
+hyperlink 27
+hypertext 27
+id 27
+identical 27
+implementation 27
+incorporated 27
+incremental 27
+inspector 27
+internet 27
+kind 27
+launch 27
+linked 27
+list 27
+lucene 27
+media 27
+mediawiki 27
+mostly 27
+mozilla 27
+multilingual 27
+mysql 27
+nero 27
+node 27
+observing 27
+online 27
+outlook 27
+page 27
+pages 27
+particular 27
+php 27
+player 27
+portal 27
+posted 27
+posting 27
+processors 27
+programme 27
+provided 27
+provides 27
+providing 27
+proxy 27
+reader 27
+readers 27
+regular 27
+relevant 27
+requests 27
+rfc 27
+rom 27
+routing 27
+safari 27
+scripting 27
+search 27
+searching 27
+server 27
+servers 27
+service 27
+sharing 27
+site 27
+specification 27
+squid 27
+suggested 27
+tabbed 27
+tags 27
+thread 27
+thunderbird 27
+u 27
+unicode 27
+update 27
+updated 27
+using 27
+video 27
+videos 27
+viewing 27
+web 27
+website 27
+websites 27
+welcome 27
+wide 27
+wiki 27
+wikkipedia 27
+yahoo 27
+youtube 27
+zine 27
+1908 28
+1945 28
+abide 28
+alois 28
+ann 28
+barack 28
+biography 28
+boy 28
+chancellor 28
+child 28
+court 28
+cross 28
+daughters 28
+dunham 28
+employees 28
+father 28
+hitler 28
+hussein 28
+irish 28
+lolo 28
+macedonia 28
+macedonian 28
+mother 28
+obama 28
+parents 28
+profession 28
+salam 28
+selected 28
+seven 28
+twice 28
+when 28
+youth 28
+1950 29
+1956 29
+1973 29
+21 29
+accordance 29
+act 29
+addressed 29
+administrators 29
+alliance 29
+appeal 29
+ban 29
+bicameral 29
+bjp 29
+bush 29
+cabinet 29
+central 29
+charged 29
+chief 29
+citizen 29
+coalition 29
+commons 29
+commonwealth 29
+comprising 29
+consists 29
+constitution 29
+constitutional 29
+council 29
+courts 29
+democratic 29
+destructive 29
+direct 29
+dispute 29
+draft 29
+elected 29
+election 29
+executive 29
+five 29
+formed 29
+founder 29
+general 29
+george 29
+having 29
+headed 29
+indirectly 29
+interim 29
+judicial 29
+jurisdiction 29
+justice 29
+law 29
+laws 29
+legal 29
+legislative 29
+legislature 29
+linguistic 29
+lok 29
+maximum 29
+member 29
+members 29
+membership 29
+menstrual 29
+minister 29
+ministers 29
+namely 29
+nations 29
+owner 29
+p 29
+parliament 29
+parliamentary 29
+president 29
+presidential 29
+prime 29
+proposed 29
+provisional 29
+rajya 29
+ranked 29
+representatives 29
+republic 29
+responsibility 29
+responsible 29
+rights 29
+sabha 29
+scope 29
+seats 29
+secretary 29
+secured 29
+security 29
+senator 29
+singh 29
+state 29
+subordinate 29
+supporters 29
+supreme 29
+territories 29
+tests 29
+trying 29
+under 29
+union 29
+uno 29
+upa 29
+veto 29
+virtue 29
+vote 29
+voting 29
+ward 29
+above 30
+acts 30
+between 30
+c-in-c 30
+communication 30
+completely 30
+connection 30
+connections 30
+depend 30
+description 30
+external 30
+hardware 30
+help 30
+helps 30
+internal 30
+link 30
+links 30
+malik 30
+ohm 30
+outer 30
+out-links 30
+outside 30
+relation 30
+require 30
+space 30
+traveling 30
+virtual 30
+/ 31
+64 31
+</address> 31
+<address> 31
+ajax 31
+amd 31
+carried 31
+carrier 31
+categories 31
+chain 31
+classes 31
+classified 31
+collected 31
+compiled 31
+count 31
+data 31
+dates 31
+diagnosis 31
+divided 31
+documented 31
+earth 31
+encyclopedia 31
+error 31
+events 31
+exchanges 31
+extensive 31
+fact 31
+facts 31
+format 31
+formatting 31
+frequency 31
+generation 31
+hyper 31
+ieee 31
+incidents 31
+information 31
+informations 31
+informatory 31
+into 31
+javascript 31
+kamasutra 31
+lab 31
+markup 31
+mentioned 31
+modification 31
+nomenclature 31
+numeric 31
+one 31
+open 31
+planet 31
+raw 31
+reference 31
+references 31
+satellite 31
+smallest 31
+source 31
+sources 31
+stored 31
+text 31
+texts 31
+through 31
+transmit 31
+type 31
+types 31
+undecided 31
+unsolved 31
+which 31
+xml 31
+zone 31
+2009 32
+ache 32
+affected 32
+animal 32
+assumed 32
+attached 32
+avian 32
+aware 32
+banglapedia 32
+beings 32
+bird 32
+birds 32
+blood 32
+bodies 32
+brain 32
+breathing 32
+britannica 32
+calling 32
+careful 32
+cause 32
+caused 32
+cold 32
+community 32
+conscious 32
+cough 32
+coughing 32
+detected 32
+diagnosed 32
+disease 32
+diseases 32
+doctors 32
+earliest 32
+edgar 32
+effected 32
+encouraged 32
+endemic 32
+epidemic 32
+estimated 32
+expand 32
+extinction 32
+far 32
+fashion 32
+fate 32
+feeling 32
+fever 32
+flu 32
+flue 32
+fluid 32
+found 32
+frontal 32
+generated 32
+gutiarez 32
+gutierrez 32
+h1n1 32
+handkerchief 32
+harmful 32
+headache 32
+health 32
+healthy 32
+helpful 32
+homo 32
+human 32
+humans 32
+identified 32
+immune 32
+immunity 32
+infect 32
+infected 32
+influenza 32
+keep 32
+kitab 32
+lobe 32
+loss 32
+naturally 32
+noticed 32
+observed 32
+obtained 32
+oneself 32
+originates 32
+pain 32
+pandemic 32
+patient 32
+patients 32
+pcr 32
+person 32
+phlegm 32
+pig 32
+pigs 32
+presence 32
+presumed 32
+prevent 32
+preventing 32
+prevention 32
+recover 32
+reported 32
+resistance 32
+respiratory 32
+reverse 32
+saliva 32
+sample 32
+sapiens 32
+seasonal 32
+seen 32
+severe 32
+similar 32
+sneezing 32
+spreads 32
+strain 32
+suffer 32
+suffered 32
+suffering 32
+survived 32
+suspected 32
+swain 32
+swine 32
+symptoms 32
+thinks 32
+threat 32
+throat 32
+tract 32
+transcription 32
+virus 32
+viruses 32
+weight 32
+wild 32
+192 33
+200 33
+� 33
+33 33
+43 33
+4,671 33
+7,517 33
+across 33
+affiliated 33
+africa 33
+aggression 33
+alaska 33
+alps 33
+andaman 33
+andorra 33
+archipelago 33
+arctic 33
+asia 33
+asian 33
+atlantic 33
+atmosphere 33
+australia 33
+banks 33
+barrier 33
+bases 33
+bay 33
+beaches 33
+boarder 33
+border 33
+borders 33
+boundary 33
+brown 33
+caribbean 33
+channel 33
+china 33
+climate 33
+coast 33
+coastal 33
+coasts 33
+comprised 33
+comprises 33
+concentrated 33
+constituted 33
+continent 33
+contrary 33
+coral 33
+covered 33
+crossing 33
+deccan 33
+deep 33
+delta 33
+desert 33
+districts 33
+dominated 33
+easter 33
+eastern 33
+emerging 33
+eurasian 33
+except 33
+expanded 33
+extend 33
+extended 33
+extending 33
+extends 33
+extent 33
+foot 33
+forest 33
+forests 33
+ganga 33
+ganges 33
+gangetic 33
+garonne 33
+geographical 33
+ghat 33
+ghats 33
+gondwana 33
+grass 33
+greater 33
+green 33
+harmony 33
+hills 33
+hilly 33
+himalaya 33
+himalayan 33
+himalayas 33
+hong 33
+ice 33
+indonesia 33
+inhabitants 33
+inhabited 33
+island 33
+islands 33
+jharkhand 33
+kilometer 33
+kilometers 33
+kilometres 33
+kingdoms 33
+km 33
+kong 33
+korea 33
+korean 33
+lake 33
+lakshadweep 33
+land 33
+lands 33
+lanka 33
+latitude 33
+lies 33
+location 33
+locations 33
+longitude 33
+lying 33
+mahal 33
+mainland 33
+maintains 33
+malaysia 33
+maldives 33
+marshy 33
+mediterranean 33
+mere 33
+mile 33
+miles 33
+monsoon 33
+mountain 33
+mountainous 33
+mountains 33
+myanmar 33
+naval 33
+nearby 33
+nearest 33
+neighboring 33
+nice 33
+nicobar 33
+nippon 33
+north 33
+north-east 33
+north-eastern 33
+north-west 33
+ocean 33
+oceans 33
+off 33
+pacific 33
+parallel 33
+parks 33
+part 33
+parts 33
+peninsula 33
+peninsular 33
+plain 33
+plains 33
+plane 33
+plate 33
+plateau 33
+plays 33
+politically 33
+portion 33
+protected 33
+provinces 33
+queensland 33
+range 33
+ranges 33
+rank 33
+reef 33
+region 33
+regions 33
+representation 33
+rhone 33
+rice 33
+rising 33
+river 33
+rivers 33
+rocky 33
+scotland 33
+sea 33
+separated 33
+settled 33
+settlement 33
+shelter 33
+side 33
+sides 33
+situated 33
+smaller 33
+south 33
+south-east 33
+south-eastern 33
+southern 33
+south-west 33
+south-western 33
+sri 33
+strait 33
+stretches 33
+subcontinent 33
+surrounded 33
+taiwan 33
+tasmania 33
+tectonic 33
+territory 33
+thar 33
+touches 33
+trafalgar 33
+trees 33
+tropical 33
+tunnel 33
+ural 33
+uss 33
+valley 33
+vast 33
+victoria 33
+volcanic 33
+western 33
+westminister 33
+widespread 33
+wind 33
+winds 33
+abraham 34
+al 34
+allah 34
+'an 34
+arab 34
+arabian 34
+arabic 34
+arose 34
+ayat 34
+belief 34
+believe 34
+believed 34
+believes 34
+belong 34
+bihari 34
+books 34
+buddhism 34
+characteristic 34
+christians 34
+consent 34
+conservation 34
+corrected 34
+correctly 34
+creator 34
+culturally 34
+diverse 34
+drain 34
+faith 34
+followers 34
+god 34
+hadis 34
+hadith 34
+hindu 34
+holy 34
+islam 34
+jain 34
+jews 34
+koran 34
+kuraan 34
+kuran 34
+largely 34
+leaf 34
+messages 34
+messenger 34
+mistake 34
+mix 34
+monotheistic 34
+muhammad 34
+muslims 34
+nabi 34
+namaz 34
+noted 34
+only 34
+originally 34
+perfect 34
+pillars 34
+plant 34
+prayer 34
+preached 34
+preaching 34
+prophet 34
+prophets 34
+pure 34
+qu 34
+qur 34
+quran 34
+'ran 34
+religion 34
+religions 34
+religious 34
+sermons 34
+shah 34
+similarity 34
+sm 34
+speeches 34
+sura 34
+surah 34
+talks 34
+terminology 34
+torah 34
+tradition 34
+translated 34
+tree 34
+two 34
+words 34
+worship 34
+yehudi 34
+10th 35
+11th 35
+12th 35
+14 35
+144 35
+14th 35
+15 35
+15th 35
+16 35
+16th 35
+17 35
+17th 35
+1918 35
+1923 35
+19 35
+1949 35
+1953 35
+1954 35
+1955 35
+1957 35
+1958 35
+1961 35
+1962 35
+1972 35
+1999 35
+1st 35
+2008 35
+21st 35
+23 35
+23rd 35
+24 35
+24th 35
+25 35
+26th 35
+27 35
+28th 35
+29th 35
+2nd 35
+30th 35
+4 35
+44th 35
+4th 35
+5th 35
+7 35
+7th 35
+8 35
+8th 35
+9th 35
+agreement 35
+aid 35
+anniversary 35
+august 35
+austrian 35
+autumn 35
+broke 35
+cancelled 35
+convention 35
+dead 35
+deaths 35
+december 35
+detained 35
+dropped 35
+february 35
+forth 35
+friendship 35
+g 35
+granted 35
+hours 35
+illinois 35
+january 35
+journey 35
+july 35
+june 35
+late 35
+march 35
+may 35
+ministry 35
+mirza 35
+monday 35
+month 35
+morning 35
+narayanganj 35
+november 35
+oath 35
+october 35
+officially 35
+on 35
+placed 35
+protect 35
+rainy 35
+released 35
+resigned 35
+season 35
+senate 35
+sentenced 35
+september 35
+session 35
+signed 35
+summer 35
+th 35
+tigers 35
+tour 35
+trial 35
+unknown 35
+winter 35
+year 35
+1948 36
+1969 36
+1977 36
+25th 36
+abdur 36
+absolute 36
+abstained 36
+accept 36
+acceptance 36
+accused 36
+action 36
+agartala 36
+agenda 36
+aggrieved 36
+agitation 36
+ahamad 36
+allegations 36
+angry 36
+announced 36
+arranged 36
+arrangements 36
+arrest 36
+assembly 36
+autonomy 36
+awami 36
+ayub 36
+bangabandhu 36
+banned 36
+behalf 36
+bhashani 36
+bhutto 36
+call 36
+candidate 36
+captain 36
+ceasefire 36
+charter 36
+claim 36
+committed 36
+committee 36
+communal 36
+conduct 36
+conducted 36
+conference 36
+conspiracy 36
+cooperation 36
+countrywide 36
+cyclone 36
+decided 36
+decision 36
+declare 36
+declared 36
+declaring 36
+delivered 36
+demanded 36
+demanding 36
+demands 36
+denied 36
+elections 36
+eleven 36
+emergency 36
+farman 36
+favor 36
+forming 36
+freed 36
+front 36
+governance 36
+handed 36
+hasina 36
+himself 36
+historians 36
+immediate 36
+immediately 36
+imposed 36
+initiative 36
+insurgency 36
+issued 36
+jatiyo 36
+jinnah 36
+journalists 36
+kept 36
+khaja 36
+khan 36
+killings 36
+leaded 36
+leaders 36
+leadership 36
+league 36
+leather 36
+lectures 36
+led 36
+maidan 36
+majority 36
+marshal 36
+martial 36
+maulana 36
+meet 36
+meeting 36
+midnight 36
+mujib 36
+mujibar 36
+mujibur 36
+murder 36
+nationalism 36
+nazimuddin 36
+observe 36
+officers 36
+officials 36
+.on 36
+opponent 36
+opposed 36
+opposition 36
+order 36
+ordered 36
+orders 36
+ordinance 36
+our 36
+paltan 36
+parishad 36
+participate 36
+parties 36
+party 36
+plan 36
+planning 36
+points 36
+presented 36
+prior 36
+proceedings 36
+procession 36
+proclaimed 36
+prohibited 36
+proposal 36
+proposals 36
+protested 36
+provincial 36
+publicly 36
+put 36
+rahaman 36
+rahimuddin 36
+rahman 36
+rally 36
+rao 36
+rashid 36
+rear 36
+recognize 36
+rehana 36
+rehman 36
+rejected 36
+release 36
+relief 36
+representative 36
+resolution 36
+response 36
+round 36
+saikh 36
+sangram 36
+seat 36
+secretariat 36
+securities 36
+seikh 36
+sekh 36
+separatist 36
+shaikh 36
+sheikh 36
+shekh 36
+situation 36
+six 36
+sk 36
+sohrawardi 36
+speech 36
+statement 36
+stopped 36
+strategy 36
+strike 36
+student 36
+students 36
+suhrawardy 36
+support 36
+supported 36
+survival 36
+title 36
+understanding 36
+unless 36
+unrest 36
+uprising 36
+vice 36
+votes 36
+withdrawal 36
+workers 36
+yahia 36
+yahya 36
+zia 36
+ziaur 36
+zulfikar 36
+1911 37
+1922 37
+1928 37
+1984 37
+1997 37
+22nd 37
+27th 37
+! 37
+' 37
+alo 37
+anthology 37
+atmakahini 37
+autobiography 37
+badal 37
+bajao 37
+bangla 37
+bazar 37
+bharat 37
+bharatkosh 37
+bidrohi 37
+book 37
+collections 37
+comet 37
+composed 37
+composition 37
+creations 37
+da 37
+daily 37
+dainik 37
+dan 37
+dhumketu 37
+dictionary 37
+e 37
+edited 37
+editions 37
+em 37
+epic 37
+ever 37
+expressed 37
+gaan 37
+gen. 37
+ghore 37
+ghumer 37
+grammer 37
+hena 37
+i.e. 37
+imprisoned 37
+isbn 37
+journal 37
+kalidas 37
+kheya 37
+kheya-parer 37
+ki 37
+kon 37
+letter 37
+-lrb- 37
+magazine 37
+magazines 37
+mahabharata 37
+meher 37
+message 37
+moslem 37
+my 37
+named 37
+negar 37
+newspaper 37
+newspapers 37
+nonsense 37
+novel 37
+novels 37
+oaths 37
+paper 37
+patrika 37
+photo 37
+poem 37
+poems 37
+prater 37
+prose 37
+prothom 37
+publication 37
+published 37
+re 37
+repeatedly 37
+report 37
+review 37
+riemann 37
+-rrb- 37
+sandesh 37
+seeing 37
+sharab 37
+sitting 37
+song 37
+stories 37
+tale 37
+tarani 37
+tome 37
+translation 37
+twelve 37
+where 37
+writing 37
+writings 37
+written 37
+wrote 37
+1914 38
+1915 38
+1920 38
+1921 38
+1927 38
+1929 38
+1934 38
+1939 38
+1942 38
+1952 38
+1963 38
+1974 38
+49 38
+9 38
+admission 38
+admitted 38
+after 38
+afterwards 38
+again 38
+arrested 38
+at 38
+austria 38
+away 38
+back 38
+barrack 38
+barracks 38
+battalion 38
+bavaria 38
+breaking 38
+brought 38
+calcutta 38
+came 38
+cantonment 38
+charges 38
+circumstances 38
+comilla 38
+continued 38
+decorated 38
+deposition 38
+eight 38
+end 38
+entrance 38
+evening 38
+exam 38
+examination 38
+examined 38
+fell 38
+fort 38
+gate 38
+gathering 38
+gopalganj 38
+grew 38
+hans 38
+hoff 38
+hospital 38
+hunger 38
+imprisonment 38
+improving 38
+injured 38
+islamia 38
+jail 38
+join 38
+joins 38
+karachi 38
+kolkata 38
+kumilla 38
+lahore 38
+mission 38
+missionary 38
+months 38
+munich 38
+mymensingh 38
+nejrul 38
+nine 38
+once 38
+onwards 38
+passed 38
+passing 38
+permission 38
+presidency 38
+prison 38
+punished 38
+putting 38
+quarter 38
+rafizullah 38
+railway 38
+raj 38
+ran 38
+ranchi 38
+raniganj 38
+regained 38
+replaced 38
+residing 38
+return 38
+returned 38
+returning 38
+riot 38
+same 38
+saved 38
+sent 38
+shop 38
+sick 38
+simon 38
+soldier 38
+started 38
+station 38
+stayed 38
+staying 38
+streets 38
+studying 38
+thereafter 38
+.this 38
+took 38
+travelled 38
+tried 38
+turned 38
+up 38
+uterine 38
+vienna 38
+visited 38
+was 38
+went 38
+while 38
+12.6 39
+$ 39
+% 39
+according 39
+account 39
+amphibians 39
+annual 39
+average 39
+billion 39
+capita 39
+cash 39
+causes 39
+changes 39
+considering 39
+controlled 39
+decide 39
+decreased 39
+designated 39
+diagram 39
+divorce 39
+dollar 39
+export 39
+extinct 39
+factors 39
+flood 39
+floods 39
+flow 39
+food 39
+gdp 39
+global 39
+growth 39
+highest 39
+holidays 39
+humidity 39
+implemented 39
+import 39
+increase 39
+increases 39
+increment 39
+investment 39
+line 39
+low 39
+lowest 39
+malnutrition 39
+mammalians 39
+minimum 39
+nile 39
+parity 39
+per 39
+percent 39
+percentage 39
+<police 39
+ppp 39
+purchasing 39
+rain 39
+rate 39
+rates 39
+record 39
+relating 39
+respectively 39
+size 39
+species 39
+station> 39
+statistics 39
+</student> 39
+surface 39
+technological 39
+total 39
+trillion 39
+weather 39
+whereas 39
+yearly 39
+1947 40
+1965 40
+1970 40
+administration 40
+adopted 40
+against 40
+aggravated 40
+alan 40
+allies 40
+anti 40
+asked 40
+assistance 40
+assuming 40
+assured 40
+atom 40
+autocracy 40
+began 40
+beginning 40
+berlin 40
+bound 40
+capturing 40
+civil 40
+colonization 40
+combined 40
+confrontation 40
+congress 40
+consequence 40
+consequently 40
+continues 40
+cordial 40
+coup 40
+declaration 40
+defeated 40
+defense 40
+democracy 40
+despite 40
+deteriorated 40
+diplomatic 40
+discipline 40
+dissatisfaction 40
+disturbance 40
+divide 40
+dominance 40
+empowered 40
+equality 40
+exploitation 40
+face 40
+famine 40
+feared 40
+fight 40
+gaulle 40
+getting 40
+government 40
+governments 40
+ideals 40
+independence 40
+independent 40
+inequality 40
+initiated 40
+instability 40
+intervention 40
+intimate 40
+issue 40
+kashmir 40
+keyes 40
+leading 40
+liberal 40
+losses 40
+margin 40
+multi 40
+murders 40
+nagra 40
+negligence 40
+nevertheless 40
+planned 40
+poland 40
+political 40
+politics 40
+post 40
+powers 40
+preamble 40
+principal 40
+progressive 40
+protest 40
+racial 40
+republican 40
+resulted 40
+rise 40
+rival 40
+rule 40
+ruler 40
+ruling 40
+scale 40
+secular 40
+secularism 40
+sino-indian 40
+slavery 40
+socialism 40
+sovereign 40
+spark 40
+spite 40
+strict 40
+strong 40
+struggle 40
+sub-continent 40
+thus 40
+towards 40
+ultimately 40
+unstability 40
+unsuccessful 40
+victory 40
+violence 40
+war 40
+ware 40
+wars 40
+weak 40
+12 41
+1608 41
+1610 41
+1650 41
+1856 41
+1905 41
+32 41
+3rd 41
+800 41
+acres 41
+adjacent 41
+airport 41
+appointed 41
+assam 41
+banga 41
+bangalore 41
+bank 41
+bankside 41
+baro 41
+basilica 41
+bengal 41
+beside 41
+bihar 41
+bridge 41
+build 41
+building 41
+built 41
+bulbul 41
+burial 41
+buried 41
+buriganga 41
+capital 41
+capture 41
+cathedral 41
+celebration 41
+center 41
+centre 41
+chattagram 41
+chisti 41
+churulia 41
+city 41
+click 41
+colchester 41
+constructed 41
+continuing 41
+convent 41
+corner 41
+corporation 41
+course 41
+dacca 41
+delhi 41
+dhaka 41
+district 41
+dramatic 41
+emperor 41
+entertainment 41
+established 41
+faridpur 41
+flights 41
+forty 41
+founded 41
+frontier 41
+gaalib 41
+garden 41
+gardens 41
+governor 41
+halls 41
+hampstead 41
+head 41
+headquarter 41
+headquarters 41
+hill 41
+hub 41
+jahangir 41
+jahangirnagar 41
+king 41
+koshi 41
+lane 41
+located 41
+london 41
+lower 41
+manhattan 41
+mega 41
+metro 41
+moved 41
+mughal 41
+mumbai 41
+museum 41
+near 41
+netaji 41
+offices 41
+palace 41
+park 41
+partition 41
+piccadilly 41
+place 41
+port 41
+province 41
+quebec 41
+rajmahal 41
+recognized 41
+repeated 41
+residence 41
+residential 41
+residents 41
+road 41
+ron 41
+royal 41
+ruled 41
+sculpture 41
+separation 41
+shifted 41
+soho 41
+stations 41
+stock 41
+street 41
+subdivision 41
+sube 41
+subedar 41
+suja 41
+surrounding 41
+telegraph 41
+temple 41
+thames 41
+theaters 41
+theatre 41
+town 41
+village 41
+westminster 41
+x-ray 41
+1971 42
+agitated 42
+air 42
+ally 42
+along 42
+already 42
+ammunition 42
+announcement 42
+apabhramsa 42
+armed 42
+armies 42
+arms 42
+army 42
+attack 42
+attacked 42
+attacking 42
+attacks 42
+authorities 42
+bagura 42
+bahini 42
+bangladesh 42
+battle 42
+begun 42
+bengalis 42
+bonn 42
+brutal 42
+by 42
+camp 42
+chittagong 42
+clash 42
+colonel 42
+command 42
+commander 42
+commander-in-chief 42
+confined 42
+conflicts 42
+curfew 42
+dark 42
+deprived 42
+deputy 42
+destroyed 42
+east 42
+fighter 42
+fighters 42
+fighting 42
+filled 42
+fire 42
+firing 42
+force 42
+forces 42
+formation 42
+freedom 42
+gap 42
+genocide 42
+guerrilla 42
+hindus 42
+homes 42
+horrible 42
+indo-pak 42
+innocent 42
+intellectuals 42
+joint 42
+killed 42
+killing 42
+kushtia 42
+liberation 42
+major 42
+mass 42
+massacre 42
+military 42
+mukti 42
+muktibahini 42
+murdered 42
+mutiny 42
+navy 42
+neglected 42
+night 42
+nikson 42
+nixon 42
+officer 42
+operation 42
+operations 42
+out 42
+pakistan 42
+pakistani 42
+pakistanis 42
+passengers 42
+planes 42
+police 42
+politicians 42
+punjab 42
+racecourse 42
+regiment 42
+reply 42
+revolted 42
+rifles 42
+rulers 42
+salvation 42
+searchlight 42
+secretly 42
+section 42
+ship 42
+soldiers 42
+staff 42
+stagnant 42
+stop 42
+storm 42
+surrender 42
+surrendered 42
+suspension 42
+tactics 42
+temporary 42
+training 42
+traveled 42
+treaty 42
+tripura 42
+troop 42
+troops 42
+west 42
+with 42
+able 43
+although 43
+avoided 43
+because 43
+become 43
+being 43
+but 43
+created 43
+crisis 43
+days 43
+disaster 43
+due 43
+even 43
+failed 43
+few 43
+for 43
+going 43
+gradually 43
+historical 43
+hold 43
+hydrogen 43
+increasing 43
+interested 43
+itself 43
+lack 43
+long 43
+now 43
+over 43
+present 43
+pressure 43
+quickly 43
+reason 43
+regularly 43
+since 43
+slowly 43
+soon 43
+start 43
+still 43
+taking 43
+this 43
+though 43
+throughout 43
+time 43
+times 43
+trained 43
+whole 43
+0 44
+accordingly 44
+aggregates 44
+amar 44
+amount 44
+balance 44
+budget 44
+buy 44
+buyer 44
+buyers 44
+celsius 44
+cheap 44
+chemical 44
+choose 44
+closing 44
+cloud 44
+commodity 44
+connecting 44
+considers 44
+consumer 44
+consumers 44
+cost 44
+curve 44
+decrease 44
+demand 44
+depended 44
+determine 44
+determined 44
+determines 44
+distributed 44
+duty 44
+edison 44
+equilibrium 44
+estimate 44
+eventually 44
+existing 44
+factor 44
+falls 44
+feedback 44
+fixed 44
+flowing 44
+gas 44
+gaseous 44
+gold 44
+good 44
+goods 44
+graph 44
+hidden 44
+indicated 44
+intention 44
+kagoj 44
+labor 44
+laborer 44
+laborers 44
+labour 44
+less 44
+lines 44
+marks 44
+meant 44
+measures 44
+melting 44
+occurs 44
+point 44
+price 44
+produce 44
+producer 44
+producers 44
+product 44
+profitable 44
+purbanchal 44
+purchase 44
+quantities 44
+quantity 44
+radio 44
+reaches 44
+ready 44
+reflection 44
+remaining 44
+represented 44
+salary 44
+seller 44
+shortage 44
+silver 44
+sold 44
+solid 44
+spectacular 44
+statue 44
+steam 44
+stores 44
+strength 44
+supplied 44
+supplies 44
+supply 44
+table 44
+tag 44
+taxes 44
+temperature 44
+upon 44
+utility 44
+value 44
+vapor 44
+warship 44
+water 44
+wireless 44
+anime 45
+area 45
+areas 45
+beauty 45
+behind 45
+big 45
+bridges 45
+britain 45
+centrally 45
+cities 45
+considerably 45
+defeat 45
+densely 45
+emperors 45
+establishments 45
+families 45
+fast 45
+gained 45
+geography 45
+great 45
+heavy 45
+houses 45
+huge 45
+important 45
+japan 45
+japanese 45
+kings 45
+large 45
+larger 45
+mean 45
+metropolitan 45
+minority 45
+most 45
+nihon 45
+peaceful 45
+populated 45
+rich 45
+routes 45
+significant 45
+small 45
+spreading 45
+terrorism 45
+tokyo 45
+tower 45
+urban 45
+very 45
+villages 45
+weapons 45
+witnessed 45
+1930 46
+1960 46
+actual 46
+affect 46
+affecting 46
+affects 46
+aggregate 46
+alcohol 46
+analyze 46
+analyzed 46
+analyzing 46
+argued 46
+argument 46
+attempts 46
+bad 46
+behavior 46
+below 46
+bring 46
+broad 46
+buying 46
+capitalist 46
+cases 46
+change 46
+changing 46
+choice 46
+collective 46
+commodities 46
+competition 46
+competitive 46
+conditions 46
+consideration 46
+constant 46
+constraints 46
+consumption 46
+corollary 46
+costly 46
+costs 46
+criticizing 46
+deciding 46
+decisions 46
+decreases 46
+define 46
+defining 46
+dependence 46
+depends 46
+determination 46
+difference 46
+discussions 46
+earning 46
+effect 46
+effects 46
+equal 46
+examining 46
+excess 46
+expenditure 46
+expenses 46
+explain 46
+explains 46
+exporting 46
+farmers 46
+focused 46
+forward 46
+framework 46
+full 46
+fully 46
+functional 46
+humane 46
+hume 46
+hypothetical 46
+imaginary 46
+importance 46
+imposing 46
+.in 46
+includes 46
+income 46
+index 46
+individual 46
+inflation 46
+influence 46
+influences 46
+inter 46
+interest 46
+interrelationship 46
+invisible 46
+.it 46
+items 46
+keynes 46
+lasting 46
+lifestyle 46
+limited 46
+logically 46
+longer 46
+macroeconomics 46
+makes 46
+marginal 46
+market 46
+markets 46
+maximization 46
+maynard 46
+measure 46
+measurement 46
+microeconomics 46
+mixed 46
+money 46
+monopoly 46
+move 46
+moves 46
+nature 46
+objectives 46
+observational 46
+observations 46
+obstacles 46
+owners 46
+partial 46
+perspective 46
+positive 46
+poverty 46
+practical 46
+practically 46
+practice 46
+preference 46
+previous 46
+prices 46
+producing 46
+production 46
+profit 46
+profits 46
+ratio 46
+reaction 46
+reality 46
+reflect 46
+reflects 46
+relationship 46
+relatively 46
+remain 46
+respect 46
+sale 46
+satisfaction 46
+scarcity 46
+sense 46
+show 46
+showed 46
+sided 46
+solution 46
+stability 46
+standard 46
+structural 46
+structures 46
+studies 46
+tax 46
+tendency 46
+theoretically 46
+theories 46
+theory 46
+things 46
+tries 46
+try 46
+unchanged 46
+understood 46
+unemployment 46
+unhappy 46
+viewpoint 46
+wages 46
+worker 46
+writes 46
+19th 47
+28 47
+abroad 47
+activities 47
+adviser 47
+affairs 47
+agencies 47
+agricultural 47
+agriculture 47
+and 47
+assigned 47
+atomic 47
+australian 47
+automobile 47
+bigger 47
+bullet 47
+business 47
+bussiness 47
+centers 47
+chemicals 47
+commerce 47
+commission 47
+companies 47
+contribution 47
+cultural 47
+currency 47
+cycle 47
+deal 47
+defence 47
+department 47
+developmental 47
+domestic 47
+driven 47
+driving 47
+economical 47
+electronics 47
+element 47
+emergence 47
+energy 47
+equipment 47
+euro 47
+field 47
+foods 47
+foreign 47
+fund 47
+globalization 47
+governmental 47
+humanitarian 47
+industrial 47
+industries 47
+industry 47
+infosys 47
+infrastructure 47
+investments 47
+jute 47
+main 47
+manufacturing 47
+mine 47
+mines 47
+organisation 47
+organization 47
+organizations 47
+petroleum 47
+products 47
+purposes 47
+registered 47
+regulation 47
+reserve 47
+rickshaw 47
+rickshaws 47
+roads 47
+rural 47
+saarc 47
+sector 47
+segment 47
+share 47
+shoguns 47
+significantly 47
+sphere 47
+strategic 47
+<student 47
+tea 47
+.the 47
+trade 47
+trades 47
+traffic 47
+transport 47
+transportation 47
+wheat 47
+1944 48
+1951 48
+1996 48
+1998 48
+agency 48
+architects 48
+association 48
+authority 48
+boxing 48
+celebrated 48
+club 48
+co-operation 48
+cricket 48
+cup 48
+dhanmondi 48
+directing 48
+event 48
+exercise 48
+federation 48
+federations 48
+figure 48
+football 48
+game 48
+games 48
+ground 48
+handball 48
+held 48
+hockey 48
+intellectual 48
+international 48
+janata 48
+jatiya 48
+kabadi 48
+kho 48
+levels 48
+m 48
+martyrs 48
+masters 48
+medal 48
+memorial 48
+minar 48
+mirpur 48
+monument 48
+national 48
+olympic 48
+organised 48
+organized 48
+played 48
+players 48
+playing 48
+pool 48
+pre 48
+rayer 48
+recognition 48
+shahid 48
+sher-e-bangla 48
+sports 48
+stadium 48
+status 48
+swimming 48
+team 48
+tennis 48
+test 48
+tiger 48
+top 48
+tournament 48
+tournaments 48
+venue 48
+winning 48
+abstract 49
+advancement 49
+advised 49
+alternative 49
+analyses 49
+analysis 49
+analyzes 49
+applications 49
+applied 49
+applying 49
+arguments 49
+asymmetry 49
+automatic 49
+awareness 49
+behaviors 49
+branch 49
+branches 49
+capability 49
+carry 49
+characteristics 49
+classifications 49
+collecting 49
+common 49
+comparative 49
+concepts 49
+conducting 49
+continuous 49
+contract 49
+controversy 49
+creates 49
+crops 49
+deals 49
+defined 49
+departments 49
+dependable 49
+describes 49
+determining 49
+developing 49
+development 49
+developments 49
+discuss 49
+discusses 49
+distribute 49
+distribution 49
+doing 49
+econometrics 49
+economic 49
+efficiency 49
+efficient 49
+elements 49
+employed 49
+ensure 49
+enterprises 49
+environment 49
+environmental 49
+essential 49
+establishing 49
+evaluation 49
+excessive 49
+exchange 49
+exchanging 49
+experimental 49
+experiments 49
+explanation 49
+failure 49
+farm 49
+fields 49
+finance 49
+financial 49
+firm 49
+firms 49
+focus 49
+follow 49
+fulfill 49
+fundamental 49
+gives 49
+helping 49
+how 49
+ideal 49
+ideas 49
+impact 49
+imperfect 49
+importing 49
+included 49
+incomplete 49
+individuals 49
+ingredients 49
+institutions 49
+invaluable 49
+issues 49
+laboratory 49
+layers 49
+macroeconomic 49
+maintain 49
+maintaining 49
+management 49
+managerial 49
+manufacture 49
+materials 49
+mechanics 49
+mechanism 49
+method 49
+microelectronics 49
+model 49
+monetary 49
+moral 49
+natural 49
+necessary 49
+necessity 49
+observation 49
+office> 49
+oil 49
+organizational 49
+ownership 49
+parameters 49
+phases 49
+policies 49
+policy 49
+pollution 49
+<post 49
+priority 49
+process 49
+processes 49
+productions 49
+projects 49
+proper 49
+publishes 49
+quantitative 49
+reactions 49
+regulations 49
+related 49
+relations 49
+resource 49
+resources 49
+revenue 49
+risk 49
+rules 49
+satisfies 49
+sectors 49
+services 49
+skill 49
+solutions 49
+statistical 49
+stream 49
+structure 49
+subset 49
+uncertainty 49
+units 49
+unity 49
+usable 49
+variables 49
+verification 49
+view 49
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/hiero/class_lm_2gram.gz
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/hiero/class_lm_2gram.gz b/joshua-core/src/test/resources/bn-en/hiero/class_lm_2gram.gz
new file mode 100644
index 0000000..27b6d1a
Binary files /dev/null and b/joshua-core/src/test/resources/bn-en/hiero/class_lm_2gram.gz differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/hiero/class_lm_9gram.gz
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/hiero/class_lm_9gram.gz b/joshua-core/src/test/resources/bn-en/hiero/class_lm_9gram.gz
new file mode 100644
index 0000000..6a48db6
Binary files /dev/null and b/joshua-core/src/test/resources/bn-en/hiero/class_lm_9gram.gz differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/hiero/glue-grammar
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/hiero/glue-grammar b/joshua-core/src/test/resources/bn-en/hiero/glue-grammar
new file mode 100644
index 0000000..6a1162f
--- /dev/null
+++ b/joshua-core/src/test/resources/bn-en/hiero/glue-grammar
@@ -0,0 +1,3 @@
+[GOAL] ||| <s> ||| <s> ||| 0
+[GOAL] ||| [GOAL,1] [X,2] ||| [GOAL,1] [X,2] ||| -1
+[GOAL] ||| [GOAL,1] </s> ||| [GOAL,1] </s> ||| 0
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/hiero/grammar.gz
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/hiero/grammar.gz b/joshua-core/src/test/resources/bn-en/hiero/grammar.gz
new file mode 100644
index 0000000..47e8b1e
Binary files /dev/null and b/joshua-core/src/test/resources/bn-en/hiero/grammar.gz differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/hiero/input.bn
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/hiero/input.bn b/joshua-core/src/test/resources/bn-en/hiero/input.bn
new file mode 100644
index 0000000..be6a92b
--- /dev/null
+++ b/joshua-core/src/test/resources/bn-en/hiero/input.bn
@@ -0,0 +1,100 @@
+\u09b0\u09ac\u09c0\u09a8\u09cd\u09a6\u09cd\u09b0\u09a8\u09be\u09a5\u09c7\u09b0 \u099c\u09a8\u09cd\u09ae \u09b9\u09af\u09bc\u09c7\u099b\u09bf\u09b2 \u0995\u09b2\u0995\u09be\u09a4\u09be\u09b0 \u098f\u0995 \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 \u09aa\u09b0\u09bf\u09ac\u09be\u09b0\u09c7 \u0964
+\u09b8\u09be\u09ae\u09cd\u09aa\u09cd\u09b0\u09a4\u09bf\u0995\u0995\u09be\u09b2\u09c7 \u09ad\u09be\u09b0\u09a4\u09c7\u09b0 \u09b8\u0999\u09cd\u0997\u09c7 \u09af\u09c1\u0995\u09cd\u09a4\u09b0\u09be\u09b7\u09cd\u099f\u09cd\u09b0\u09c7\u09b0 \u09b8\u09ae\u09cd\u09aa\u09b0\u09cd\u0995\u09c7\u09b0 \u0989\u09a8\u09cd\u09a8\u09a4\u09bf \u09b9\u09af\u09bc\u09c7\u099b\u09c7 \u0964
+\u0997\u09a3\u09bf\u09a4 \u09a4\u09be\u0987 \u09ac\u09bf\u099c\u09cd\u099e\u09be\u09a8\u09c7\u09b0 \u09ad\u09be\u09b7\u09be \u0964
+\u098f \u09a5\u09c7\u0995\u09c7 \u09b8\u09b9\u099c\u09c7\u0987 \u09ac\u09cb\u099d\u09be \u09af\u09be\u09af\u09bc \u09af\u09c7 \u098f\u0987 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf \u09b9\u09ac\u09c7 \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 \u0964
+\u098f\u0995\u0987 \u09b8\u0999\u09cd\u0997\u09c7 \u09ac\u09be\u0982\u09b2\u09be\u09b0 \u09ad\u09c2\u09ae\u09bf\u0995\u09c7\u09a8\u09cd\u09a6\u09cd\u09b0\u09bf\u0995 \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 \u09aa\u09c2\u09b0\u09cd\u09ac\u09be\u09ad\u09be\u09b7\u0993 \u09ae\u09c7\u09b2\u09c7 \u098f\u0987 \u0989\u09aa\u09a8\u09cd\u09af\u09be\u09b8 \u09a5\u09c7\u0995\u09c7 \u0964
+\u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 \u09ae\u09c1\u099c\u09bf\u09ac \u0993 \u09a4\u09be\u0981\u09b0 \u09a6\u09b2 \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 \u09b8\u0982\u0996\u09cd\u09af\u09be\u0997\u09b0\u09bf\u09b7\u09cd\u09a0\u09a4\u09be \u0985\u09b0\u09cd\u099c\u09a8 \u0995\u09b0\u09c7 \u0964
+\u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 \u09a4\u09be\u09b0 \u0995\u09be\u099c \u099a\u09be\u09b2\u09bf\u09af\u09bc\u09c7 \u09af\u09c7\u09a4\u09c7 \u09a5\u09be\u0995\u09c7\u09a8 \u0964
+\u099f\u09be\u0995\u09cd\u09b8 \u099b\u09be\u09a1\u09bc\u09be\u0993 \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 \u0993 \u0986\u09b0\u0993 \u0995\u09bf\u099b\u09c1 \u09b2\u09bf\u09a8\u09be\u0995\u09cd\u09b8 \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 \u099a\u09b0\u09bf\u09a4\u09cd\u09b0 \u09b0\u09af\u09bc\u09c7\u099b\u09c7 \u09a4\u09ac\u09c7 \u098f\u0997\u09c1\u09b2\u09cb \u0996\u09c1\u09ac \u09aa\u09cd\u09b0\u099a\u09b2\u09bf\u09a4 \u09a8\u09af\u09bc \u0964
+\u0987\u09b9\u09be \u09b8\u09be\u09ae\u09be\u099c\u09bf\u0995 \u0985\u09ac\u09b8\u09cd\u09a5\u09be\u09b0 \u09aa\u09cd\u09b0\u09c7\u0995\u09cd\u09b7\u09bf\u09a4\u09c7 \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 \u09a4\u09be\u09b0 \u09aa\u09cd\u09b0\u09a4\u09bf\u09aa\u0995\u09cd\u09b7\u09c7\u09b0 \u09b8\u09b9\u09bf\u09a4 \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4 \u0997\u09cd\u09b0\u09b9\u09a8\u09c7 \u09aa\u09cd\u09b0\u099a\u09b2\u09bf\u09a4 \u09a8\u09bf\u09af\u09bc\u09ae \u09ac\u09cd\u09af\u09be\u0996\u09cd\u09af\u09be \u0995\u09b0\u09c7 \u09a5\u09be\u0995\u09c7 \u0964
+\u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 \u09ac\u09be\u09ce\u09b8\u09b0\u09bf\u0995 \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf \u09ae\u09bf\u099f\u09be\u09b0
+\u09e8\u09e6\u09e6\u09ea \u09b8\u09be\u09b2\u09c7 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 \u09b6\u09b9\u09b0\u09c7 \u0985\u09a8\u09c1\u09b7\u09cd\u09a0\u09bf\u09a4 \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f \u09a6\u09b2\u09c7\u09b0 \u099c\u09be\u09a4\u09c0\u09af\u09bc \u09b8\u09ae\u09cd\u09ae\u09c7\u09b2\u09a8 \u09a4\u09bf\u09a8\u09bf \u09ae\u09c2\u09b2 \u09ac\u0995\u09cd\u09a4\u09c3\u09a4\u09be -lrb- keynote speech -rrb- \u09aa\u09cd\u09b0\u09a6\u09be\u09a8 \u0995\u09b0\u09c7\u09a8 \u0964
+\u099c\u09a8\u09b8\u0982\u0996\u09cd\u09af\u09be\u09b0 \u09ad\u09bf\u09a4\u09cd\u09a4\u09bf\u09a4\u09c7 \u0995\u09cd\u09b7\u09ae\u09a4\u09be\u09b0 \u09ac\u09a8\u09cd\u099f\u09a8 \u09aa\u09c2\u09b0\u09cd\u09ac \u09aa\u09be\u0995\u09bf\u09b8\u09cd\u09a4\u09be\u09a8\u09c7\u09b0 \u0985\u09a8\u09c1\u0995\u09c2\u09b2 \u09b9\u0993\u09af\u09bc\u09be\u09af\u09bc \u09aa\u09b6\u09cd\u099a\u09bf\u09ae \u09aa\u09be\u0995\u09bf\u09b8\u09cd\u09a4\u09be\u09a8 & quot ; \u098f\u0995 \u0987\u0989\u09a8\u09bf\u099f \u09a4\u09a4\u09cd\u09a4\u09cd\u09ac & quot ; \u09a8\u09be\u09ae\u09c7 \u098f\u0995 \u0985\u09ad\u09bf\u09a8\u09ac \u09a7\u09be\u09b0\u09a3\u09be\u09b0 \u09b8\u09c2\u09a4\u09cd\u09b0\u09aa\u09be\u09a4 \u0995\u09b0\u09c7 \u09af\u09c7\u0996\u09be\u09a8\u09c7 \u09b8\u09ae\u0997\u09cd\u09b0 \u09aa\u09b6\u09cd\u099a\u09bf\u09ae \u09aa\u09be\u0995\u09bf\u09b8\u09cd\u09a4\u09be\u09a8 \u098f\u0995\u099f\u09bf \u09aa\u09cd\u09b0\u09a6\u09c7\u09b6 \u09b9\u09bf\u09b8\u09c7\u09ac\u09c7 \u09ac\u09bf\u09ac\u09c7\u099a\u09bf\u09a4 \u09b9\u09af\u09bc \u0964
+\u2022 \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 \u09a4\u09a4\u09cd\u09a4\u09cd\u09ac
+\u09ac\u09b9\u09bf\u0983\u09b8\u0982\u09af\u09cb\u0997 \u09b8\u09ae\u09c2\u09b9
+\u099f\u09be\u099f\u09be \u0995\u09ae\u09bf\u0989\u09a8\u09bf\u0995\u09c7\u09b6\u09a8\u09b8\u09c7\u09b0 \u09ac\u09bf\u09a6\u09c7\u09b6 \u09b8\u099e\u09cd\u099a\u09be\u09b0 \u09a8\u09bf\u0997\u09ae \u09b2\u09bf\u09ae\u09bf\u099f\u09c7\u09a1 \u09ad\u09ac\u09a8 \u098f\u099f\u09bf \u09b6\u09b9\u09b0\u09c7\u09b0 \u099f\u09c7\u09b2\u09bf\u09af\u09cb\u0997\u09be\u09af\u09cb\u0997 \u09ac\u09cd\u09af\u09ac\u09b8\u09cd\u09a5\u09be\u09b0 \u098f\u0995\u099f\u09bf \u0985\u09a8\u09cd\u09af\u09a4\u09ae \u09aa\u09cd\u09b0\u09a7\u09be\u09a8 \u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0
+\u09a4\u09bf\u09a8\u09bf \u09b8\u09c7\u0987 \u09ac\u099b\u09b0\u09c7\u09b0 \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 \u099c\u09be\u09a4\u09c0\u09af\u09bc \u09a8\u09bf\u09b0\u09cd\u09ac\u09be\u099a\u09a8\u09c7 \u099c\u09af\u09bc\u09c0 \u09b9\u09a8 \u098f\u09ac\u0982 \u09ae\u09be\u09b0\u09cd\u0995\u09bf\u09a8 \u09af\u09c1\u0995\u09cd\u09a4\u09b0\u09be\u09b7\u09cd\u099f\u09cd\u09b0\u09c7\u09b0 \u09ea\u09ea\u09a4\u09ae \u09b0\u09be\u09b7\u09cd\u099f\u09cd\u09b0\u09aa\u09a4\u09bf \u09b9\u09bf\u09b8\u09c7\u09ac\u09c7 \u09a8\u09bf\u09b0\u09cd\u09ac\u09be\u099a\u09bf\u09a4 \u09b9\u09a8 \u0964
+\u09ac\u09b9\u09c1 \u09ad\u09be\u09b0\u09a4\u09c0\u09af\u09bc \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf \u0997\u09a8\u09cd\u09a1\u09cb\u09af\u09bc\u09be\u09a8\u09be\u09af\u09bc \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be \u09a5\u09c7\u0995\u09c7 \u0989\u09a6\u09cd\u09ad\u09c1\u09a4 \u0964
+\u09ac\u09cd\u09b0\u09bf\u099f\u09c7\u09a8\u09c7\u09b0 \u09b2\u09c7\u0996\u0995\u09a6\u09c7\u09b0 \u09b0\u099a\u09bf\u09a4 \u09a8\u09be\u099f\u0995 \u0989\u09aa\u09a8\u09cd\u09af\u09be\u09b8 \u0997\u09b2\u09cd\u09aa \u098f\u09ac\u0982 \u09b8\u09ae\u09cd\u09aa\u09cd\u09b0\u09a4\u09bf \u099a\u09bf\u09a4\u09cd\u09b0\u09a8\u09be\u099f\u09cd\u09af \u09ac\u09bf\u09b6\u09cd\u09ac\u09ac\u09cd\u09af\u09be\u09aa\u09c0 \u0986\u09a6\u09c3\u09a4 \u0964
+\u09e7\u09ef\u09e7\u09ef \u09b8\u09be\u09b2\u09c7\u09b0 \u09ae\u09c7 \u09ae\u09be\u09b8\u09c7 \u098f\u099f\u09bf \u09b8\u0993\u0997\u09be\u09a4 \u09aa\u09a4\u09cd\u09b0\u09bf\u0995\u09be\u09af\u09bc \u09aa\u09cd\u09b0\u0995\u09be\u09b6\u09bf\u09a4 \u09b9\u09af\u09bc \u0964
+\u09e8\u09e6\u09e6\u09eb \u09b8\u09be\u09b2\u09c7 \u0989\u0987\u09ae\u09c7\u09a8\u09b8 \u099f\u09c7\u09a8\u09bf\u09b8 \u0985\u09cd\u09af\u09be\u09b8\u09cb\u09b8\u09bf\u09af\u09bc\u09c7\u09b6\u09a8 \u099f\u09cd\u09af\u09c1\u09b0\u09c7\u09b0 \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f \u0993\u09aa\u09c7\u09a8 \u09a8\u09c7\u09a4\u09be\u099c\u09bf \u0987\u09a8\u09cd\u09a1\u09cb\u09b0 \u09b8\u09cd\u099f\u09c7\u09a1\u09bf\u09af\u09bc\u09be\u09ae\u09c7 \u0986\u09af\u09bc\u09cb\u099c\u09bf\u09a4 \u09b9\u09af\u09bc\u09c7\u099b\u09bf\u09b2 \u0964
+\u098f\u0987 \u09b8\u09ae\u09cd\u09ad\u09be\u09ac\u09a8\u09be \u09a6\u09c2\u09b0\u09c0\u0995\u09b0\u09a3\u09c7\u09b0 \u099c\u09a8\u09cd\u09af \u09a8\u09be\u09a8\u09be\u09ac\u09bf\u09a7 \u09ac\u09cd\u09af\u09ac\u09b8\u09cd\u09a5\u09be \u0997\u09cd\u09b0\u09b9\u09a3 \u0995\u09b0\u09be \u09b9\u09af\u09bc \u0964
+\u09e7\u09ef\u09ec\u09ec \u09b8\u09be\u09b2\u09c7\u09b0 \u09eb \u09ab\u09c7\u09ac\u09cd\u09b0\u09c1\u09af\u09bc\u09be\u09b0\u09bf \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 \u09ac\u09bf\u09b0\u09cb\u09a7\u09c0 \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 \u098f\u0995\u099f\u09bf \u099c\u09be\u09a4\u09c0\u09af\u09bc \u09b8\u09ae\u09cd\u09ae\u09c7\u09b2\u09a8 \u0985\u09a8\u09c1\u09b7\u09cd\u09a0\u09bf\u09a4 \u09b9\u09af\u09bc \u0964
+\u09ac\u09be\u0982\u09b2\u09be\u09a6\u09c7\u09b6 \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 \u0985\u09ab \u09a6\u09bf \u0987\u09b8\u09b2\u09be\u09ae\u09bf\u0995 \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 \u0993 \u0987\u09b8\u09b2\u09be\u09ae\u09bf\u0995 \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f \u09ac\u09cd\u09af\u09be\u0982\u0995\u09c7\u09b0 \u09b8\u09a6\u09b8\u09cd\u09af\u09aa\u09a6 \u0997\u09cd\u09b0\u09b9\u09a3 \u0995\u09b0\u09c7 \u0964
+\u09ac\u09bf\u09b7\u09af\u09bc\u09b6\u09cd\u09b0\u09c7\u09a3\u09c0 : \u09ac\u09bf\u09b6\u09cd\u09ac\u0995\u09cb\u09b7
+\u09b0\u09be\u09b6\u09bf\u09af\u09bc\u09be \u09ab\u09cd\u09b0\u09be\u09a8\u09cd\u09b8 \u0993 \u0987\u09b8\u09b0\u09be\u09af\u09bc\u09c7\u09b2 \u09ad\u09be\u09b0\u09a4\u09c7\u09b0 \u09aa\u09cd\u09b0\u09a7\u09be\u09a8 \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 \u09b0\u09be\u09b7\u09cd\u099f\u09cd\u09b0 \u0993 \u09aa\u09cd\u09b0\u09a4\u09bf\u09b0\u0995\u09cd\u09b7\u09be \u09b8\u09b9\u0995\u09be\u09b0\u09c0 \u09a6\u09c7\u09b6 \u0964
+\u098f\u0987 \u09b9\u09b2 \u0986\u09ae\u09be\u09a6\u09c7\u09b0 \u09aa\u09b0\u09bf\u099a\u09bf\u09a4 \u0995\u09be\u09b2\u09cd\u09aa\u09a8\u09bf\u0995 \u098f\u0995\u0995 \u09af\u09be\u09b0 \u09b8\u09be\u09b9\u09be\u09af\u09cd\u09af\u09c7 \u0997\u09a3\u09bf\u09a4\u09c7\u09b0 \u09a5\u09bf\u0993\u09b0\u09c0 \u09b8\u09ae\u09c1\u09b9 \u09ac\u09be\u09b8\u09cd\u09a4\u09ac \u09b8\u0982\u0996\u09cd\u09af\u09be\u09b0 \u09b8\u09c7\u099f \u09a5\u09c7\u0995\u09c7 \u099c\u099f\u09bf\u09b2 \u09b8\u0982\u0996\u09cd\u09af\u09be\u09b0 \u09b8\u09c7\u099f\u09c7 \u0989\u09a8\u09cd\u09a8\u09c0\u09a4 \u09b9\u09af\u09bc \u0964
+& lt ; \u09a0\u09bf\u0995\u09be\u09a8\u09be & gt ;
+\u09b8\u09c7\u09aa\u09cd\u099f\u09c7\u09ae\u09cd\u09ac\u09b0
+\u098f\u0987 \u09a4\u09a4\u09cd\u09a4\u09cd\u09ac \u09a5\u09c7\u0995\u09c7 \u0985\u09ac\u09b6\u09cd\u09af \u09ae\u09b9\u09be \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 \u09ac\u09bf\u09b0\u09cb\u09a7\u09bf\u09a4\u09be \u0995\u09b0\u09be \u09af\u09be\u09af\u09bc \u09a8\u09be \u09ac\u09b0\u0982 \u09a4\u09be \u09b8\u09ae\u09b0\u09cd\u09a5\u09a8 \u0995\u09b0\u09be \u09af\u09c7\u09a4\u09c7 \u09aa\u09be\u09b0\u09c7 \u0964
+\u0995\u09c3\u09b7\u09bf \u0989\u09ce\u09aa\u09be\u09a6\u09a8\u09c7 \u09ab\u09cd\u09b0\u09be\u09a8\u09cd\u09b8 \u0987\u0989\u09b0\u09cb\u09aa\u09c7\u09b0 \u09b8\u09ac\u099a\u09c7\u09af\u09bc\u09c7 \u0997\u09c1\u09b0\u09c1\u09a4\u09cd\u09ac\u09aa\u09c2\u09b0\u09cd\u09a3 \u09a6\u09c7\u09b6 ; \u098f\u099f\u09bf \u09ae\u09c2\u09b2\u09a4 \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af \u0993\u09af\u09bc\u09be\u0987\u09a8 \u09aa\u09a8\u09bf\u09b0 \u0993 \u0985\u09a8\u09cd\u09af\u09be\u09a8\u09cd\u09af \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af \u0987\u0989\u09b0\u09cb\u09aa \u0993 \u09b8\u09be\u09b0\u09be \u09ac\u09bf\u09b6\u09cd\u09ac\u09c7 \u09b0\u09aa\u09cd\u09a4\u09be\u09a8\u09bf \u0995\u09b0\u09c7 \u0964
+\u09a4\u09be\u09a6\u09c7\u09b0 \u0997\u09a3\u09bf\u09a4\u09c7 \u09aa\u09be\u099f\u09c0\u0997\u09a3\u09bf\u09a4\u09c7\u09b0 \u09aa\u09cd\u09b0\u09be\u09a7\u09be\u09a8\u09cd\u09af \u099b\u09bf\u09b2 \u0964
+\u09a6\u09c7\u09b6\u0997\u09c1\u09b2\u09cb \u09b9\u09b2\u09cb\u0983 \u09ab\u09cd\u09b0\u09be\u09a8\u09cd\u09b8 \u09b9\u0982\u0995\u0982 \u099a\u09c0\u09a8 \u09ac\u09c7\u09b2\u099c\u09bf\u09af\u09bc\u09be\u09ae \u09b8\u09c1\u0987\u099c\u09be\u09b0\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 \u099c\u09be\u09b0\u09cd\u09ae\u09be\u09a8\u09c0 \u09a1\u09c7\u09a8\u09ae\u09be\u09b0\u09cd\u0995 \u09b8\u09c1\u0987\u09a1\u09c7\u09a8 \u0985\u09b8\u09cd\u099f\u09cd\u09b0\u09bf\u09af\u09bc\u09be \u099a\u09c7\u0995\u09cb\u09b6\u09cd\u09b2\u09cb\u09ad\u09be\u0995\u09bf\u09af\u09bc\u09be \u0986\u09b0\u09cd\u099c\u09c7\u09a8\u09cd\u099f\u09bf\u09a8\u09be \u0987\u09a4\u09be\u09b2\u09bf \u09a8\u09b0\u0993\u09af\u09bc\u09c7 \u09b9\u09be\u0999\u09cd\u0997\u09c7\u09b0\u09c0 \u09af\u09c1\u0997\u09cb\u09b6\u09cd\u09b2\u09be\u09ad\u09bf\u09af\u09bc\u09be \u09ac\u09c1\u09b2\u0997\u09c7\u09b0\u09bf\u09af\u09bc\u09be \u09b0\u09c1\u09ae\u09be\u09a8\u09bf\u09af\u09bc\u09be \u0997\u09cd\u09b0\u09c0\u09b8 \u09ae\u09bf\u09b6\u09b0 \u09b8\u09bf\u0999\u09cd\u0997\u09be\u09aa\u09c1\u09b0 \u0987\u09a8\u09cd\u09a6\u09cb\u09a8\u09c7\u09b6\u09bf\u09af\u09bc\u09be \u09a5\u09be\u0987\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 \u099c\u09be\u09aa\u09be\u09a8 \u09ac\u09be\u09b0\u09cd\u09ae\u09be \u09b9\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 \u09b8\u09cb\u09ad\u09bf\u09af\u09bc\u09c7\u099f \u09b0\u09be\u09b6\u09bf\u09af\u09bc\u09be \u0987\u09b0\u09be\u09a8 \u0987\u09b0\u09be\u0995 \u0993 \u09b6\u09cd\u09b0\u09c0\u09b2\u0982\u0995\u09be \u0964
+\u098f\u0987 \u09ac\u09cd\u09af\u09be\u09b8\u09bf\u09b2\u09bf\u0995\u09be\u09b0 \u09b8\u09cd\u09a5\u09be\u09a8\u09c7 \u098f\u0996\u09a8 \u09ac\u09cd\u09af\u09be\u0982\u0995 \u0985\u09ab \u0987\u0982\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 \u0985\u09ac\u09b8\u09cd\u09a5\u09bf\u09a4 \u0964
+\u09a6\u09c7\u09b6\u099f\u09bf\u09b0 \u0989\u09a4\u09cd\u09a4\u09b0\u09c7 \u09ac\u09bf\u09b8\u09cd\u0995\u09be\u0987 \u0989\u09aa\u09b8\u09be\u0997\u09b0 \u09a6\u0995\u09cd\u09b7\u09bf\u09a3 \u09a6\u09bf\u0995\u09c7 \u099c\u09bf\u09ac\u09cd\u09b0\u09be\u09b2\u09cd\u099f\u09be\u09b0 \u09aa\u09cd\u09b0\u09a3\u09be\u09b2\u09c0 \u09aa\u09cd\u09b0\u09a3\u09be\u09b2\u09c0\u09b0 \u09a6\u0995\u09cd\u09b7\u09bf\u09a3\u09c7 \u09ae\u09b0\u0995\u09cd\u0995\u09cb \u09aa\u09b6\u09cd\u099a\u09bf\u09ae \u0993 \u09a6\u0995\u09cd\u09b7\u09bf\u09a3-\u09aa\u09b6\u09cd\u099a\u09bf\u09ae \u09a6\u09bf\u0995\u09c7 \u0986\u099f\u09b2\u09be\u09a8\u09cd\u099f\u09bf\u0995 \u09ae\u09b9\u09be\u09b8\u09be\u0997\u09b0 \u0964
+\u09a4\u09be\u099b\u09be\u09a1\u09bc\u09be \u098f \u09aa\u09b0\u09bf\u09b8\u09cd\u09a5\u09bf\u09a4\u09c7 \u09ac\u09cb\u099d\u09be \u09af\u09be\u09af\u09bc \u099c\u09b0\u09c1\u09b0\u09bf \u09ae\u09c1\u09b9\u09c2\u09b0\u09cd\u09a4\u09c7 \u099c\u09be\u09a4\u09bf\u09b8\u0982\u0998\u09c7\u09b0 \u09a6\u09cd\u09b0\u09c1\u09a4 \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4 \u09a8\u09c7\u09ac\u09be\u09b0 \u0985\u0995\u09cd\u09b7\u09ae\u09a4\u09be \u0964
+\u0995\u09be\u09b0\u09cd\u09b2 \u09ae\u09be\u09b0\u09cd\u0995\u09cd\u09b8\u09c7\u09b0 \u0995\u09be\u099c\u09c7\u09b0 \u09ae\u09be\u09a7\u09cd\u09af\u09ae\u09c7\u0987 \u0987\u09b9\u09be \u09aa\u09b0\u09bf\u099a\u09be\u09b2\u09bf\u09a4 \u09b9\u09af\u09bc \u0964
+\u098f\u09b0 \u09ac\u09bf\u09b7\u09af\u09bc\u09ac\u09b8\u09cd\u09a4\u09c1 \u0995\u0996\u09a8\u0993 \u09aa\u09c1\u09b0\u09be\u09a3 \u09a5\u09c7\u0995\u09c7 \u0995\u0996\u09a8\u0993 \u09ae\u09a7\u09cd\u09af\u09af\u09c1\u0997\u09c0\u09af\u09bc \u09aa\u09cd\u09b0\u09c7\u09ae\u0995\u09be\u09b9\u09bf\u09a8\u09bf\u0997\u09c1\u09b2\u09bf \u09a5\u09c7\u0995\u09c7 \u0995\u0996\u09a8\u0993 \u0986\u09ac\u09be\u09b0 \u098f\u0995\u09be\u09b2\u09c7\u09b0 \u09b8\u09be\u09ae\u09be\u099c\u09bf\u0995 \u0993 \u09b0\u09be\u099c\u09a8\u09c8\u09a4\u09bf\u0995 \u0998\u099f\u09a8\u09be\u09ac\u09b2\u09bf \u09a5\u09c7\u0995\u09c7 \u0997\u09c3\u09b9\u09c0\u09a4 \u0964
+\u09a4\u09bf\u09a8\u099f\u09bf \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09c7\u09b0 \u0989\u09aa\u09b0 \u09ad\u09bf\u09a4\u09cd\u09a4\u09bf \u0995\u09b0\u09c7 \u09ae\u09b9\u09be\u09ac\u09bf\u09b6\u09cd\u09ac\u09c7\u09b0 \u09af\u09c7 \u09ac\u09af\u09bc\u09b8 \u09aa\u09be\u0993\u09af\u09bc\u09be \u0997\u09c7\u099b\u09c7 \u09a4\u09be \u09b9\u09b2 \u09aa\u09cd\u09b0\u09be\u09af\u09bc \u09e7\u09e9.\u09ed � \u09e6.\u09e8 \u09ac\u09bf\u09b2\u09bf\u09af\u09bc\u09a8 \u09ac\u099b\u09b0 \u0964
+\u0995\u09be\u099b\u09c7\u0987 \u09b0\u09af\u09bc\u09c7\u099b\u09c7 \u09aa\u09c2\u09b0\u09cd\u09ac \u09b0\u09be\u09b6\u09bf\u09af\u09bc\u09be \u09af\u09be \u0993\u0996\u099f\u09b8\u09cd\u0995 \u09b8\u09be\u0997\u09b0 \u0993 \u099c\u09be\u09aa\u09be\u09a8 \u09b8\u09be\u0997\u09b0\u09c7\u09b0 \u0985\u09aa\u09b0 \u09aa\u09be\u09b0\u09c7 \u0985\u09ac\u09b8\u09cd\u09a5\u09bf\u09a4 \u0964
+\u0995\u09b2\u0995\u09be\u09a4\u09be\u09af\u09bc \u0985\u09ac\u09b8\u09cd\u09a5\u09bf\u09a4 \u09ad\u09be\u09b0\u09a4\u09c7\u09b0 \u099c\u09be\u09a4\u09c0\u09af\u09bc \u0997\u09cd\u09b0\u09a8\u09cd\u09a5\u09be\u0997\u09be\u09b0 \u09a6\u09c7\u09b6\u09c7\u09b0 \u0985\u0997\u09cd\u09b0\u09a3\u09c0 \u09aa\u09be\u09ac\u09b2\u09bf\u0995 \u09b2\u09be\u0987\u09ac\u09cd\u09b0\u09c7\u09b0\u09bf \u0964
+\u099b\u09a4\u09cd\u09b0\u09be\u0995\u09ac\u09bf\u09a6\u09cd\u09af\u09be
+\u09b0\u09be\u09b7\u09cd\u099f\u09cd\u09b0\u09b8\u0999\u09cd\u0998\u09c7\u09b0 \u09ae\u09b9\u09be\u09b8\u099a\u09bf\u09ac \u09ac\u09be\u09a8 \u0995\u09bf \u09ae\u09c1\u09a8
+\u09ae\u09bf\u09a8\u09bf\u0995\u09cd\u09b8\u09c7\u09b0 \u09b0\u099a\u09af\u09bc\u09bf\u09a4\u09be \u099b\u09bf\u09b2\u09c7\u09a8 \u098f\u09a8\u09cd\u09a1\u09cd\u09b0\u09c1 \u099f\u09be\u09a8\u09c7\u09a8\u09ac\u09ae \u098f\u0995 \u09aa\u09cd\u09b0\u0996\u09cd\u09af\u09be\u09a4 \u0985\u09aa\u09be\u09b0\u09c7\u099f\u09bf\u0982 \u09b8\u09bf\u09b8\u09cd\u099f\u09c7\u09ae \u09a1\u09bf\u099c\u09be\u0987\u09a8 \u09aa\u09cd\u09b0\u09b6\u09bf\u0995\u09cd\u09b7\u0995 \u0964
+\u09a6\u09cd\u09af \u099f\u09be\u0987\u09ae\u09cd \u200c \u09b8 \u0985\u09ab \u0987\u09a8\u09cd\u09a1\u09bf\u09af\u09bc\u09be-\u09a4\u09c7 \u09b2\u09c7\u0996\u09be \u09b9\u09af\u09bc \u09af\u09c7 & quot ; it is absurd to compare it with any other indian cinema ... pather panchali is pure cinema & quot ; -lrb- \u201c \u098f\u0995\u09c7 \u0985\u09a8\u09cd\u09af \u09af\u09c7\u0995\u09cb\u09a8\u0993 \u09ad\u09be\u09b0\u09a4\u09c0\u09af\u09bc \u099a\u09b2\u099a\u09cd\u099a\u09bf\u09a4\u09cd\u09b0\u09c7\u09b0 \u09b8\u09be\u09a5\u09c7 \u09a4\u09c1\u09b2\u09a8\u09be \u0995\u09b0\u09be \u0985\u09ac\u09be\u09b8\u09cd\u09a4\u09ac ... \u09aa\u09a5\u09c7\u09b0 \u09aa\u09be\u0981\u099a\u09be\u09b2\u09c0 \u09b9\u09b2 \u09ac\u09bf\u09b6\u09c1\u09a6\u09cd\u09a7 \u099a\u09b2\u099a\u09cd\u099a\u09bf\u09a4\u09cd\u09b0 \u201d -rrb- \u0964
+\u098f\u09b0\u09aa\u09b0 \u09e7\u09ef\u09eb\u09e9 \u0996\u09cd\u09b0\u09bf\u09b8\u09cd\u099f\u09be\u09ac\u09cd\u09a6\u09c7\u09b0 \u09ae\u09c7 \u09ae\u09be\u09b8\u09c7 \u09a8\u099c\u09b0\u09c1\u09b2 \u0993 \u09aa\u09cd\u09b0\u09ae\u09c0\u09b2\u09be \u09a6\u09c7\u09ac\u09c0\u0995\u09c7 \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u09be\u09b0 \u099c\u09a8\u09cd\u09af \u09b2\u09a8\u09cd\u09a1\u09a8 \u09aa\u09be\u09a0\u09be\u09a8\u09cb \u09b9\u09af\u09bc \u0964
+\u098f\u09b0 \u09a6\u0995\u09cd\u09b7\u09bf\u09a3 \u0993 \u09aa\u09c2\u09b0\u09cd\u09ac\u09c7 \u0986\u099b\u09c7 \u09ac\u09bf\u09b8\u09cd\u09a4\u09c0\u09b0\u09cd\u09a3 \u09b8\u09ae\u09ad\u09c2\u09ae\u09bf ; \u09aa\u09b6\u09cd\u099a\u09bf\u09ae \u0993 \u0989\u09a4\u09cd\u09a4\u09b0\u09c7 \u0986\u099b\u09c7 \u09b0\u09c1\u0995\u09cd\u09b7 \u09aa\u09be\u09b9\u09be\u09a1\u09bc \u0993 \u09aa\u09b0\u09cd\u09ac\u09a4 \u0964
+\u099f\u09cd\u09b0\u09c7\u09a1\u09ae\u09be\u09b0\u09cd\u0995
+\u09b2\u09b0\u09cd\u09a1 \u0993\u09af\u09bc\u09c7\u09b2\u09c7\u09b8\u09b2\u09bf\u09b0 -lrb- \u0997\u09ad\u09b0\u09cd\u09a8\u09b0-\u099c\u09c7\u09a8\u09be\u09b0\u09c7\u09b2 \u09e7\u09ed\u09ef\u09ed-\u09e7\u09ee\u09e6\u09eb -rrb- \u09b6\u09be\u09b8\u09a8\u0995\u09be\u09b2\u09c7 \u09b6\u09b9\u09b0\u09c7\u09b0 \u0989\u09b2\u09cd\u09b2\u09c7\u0996\u09af\u09cb\u0997\u09cd\u09af \u09ac\u09c3\u09a6\u09cd\u09a7\u09bf \u0998\u099f\u09c7\u099b\u09bf\u09b2 \u0964
+\u0985\u09a8\u09c7\u0995 \u0997\u09c1\u09b0\u09c1\u09a4\u09cd\u09ac\u09aa\u09c2\u09b0\u09cd\u09a8 \u0993 \u09ac\u09be\u09b8\u09cd\u09a4\u09ac \u09b8\u09ae\u09b8\u09cd\u09af\u09be \u09b8\u09ae\u09be\u09a7\u09be\u09a8\u09c7\u09b0 \u099c\u09a8\u09cd\u09af \u099c\u099f\u09bf\u09b2 \u09b8\u0982\u0996\u09cd\u09af\u09be \u0985\u09aa\u09b0\u09bf\u09b9\u09be\u09b0\u09cd\u09af
+\u09ae\u09b9\u09be \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09c7\u09b0 \u098f\u0995\u099f\u09bf \u0989\u09b2\u09cd\u09b2\u09c7\u0996\u09af\u09cb\u0997\u09cd\u09af \u09ab\u09b2\u09be\u09ab\u09b2 \u09b9\u09b2 \u09ac\u09b0\u09cd\u09a4\u09ae\u09be\u09a8\u0995\u09be\u09b2\u09c7 \u09ae\u09b9\u09be\u09ac\u09bf\u09b6\u09cd\u09ac\u09c7\u09b0 \u0985\u09ac\u09b8\u09cd\u09a5\u09be \u0985\u09a4\u09c0\u09a4 \u098f\u09ac\u0982 \u09ad\u09ac\u09bf\u09b7\u09cd\u09af\u09a4\u09c7\u09b0 \u0985\u09ac\u09b8\u09cd\u09a5\u09be \u09a5\u09c7\u0995\u09c7 \u09b8\u09ae\u09cd\u09aa\u09c2\u09b0\u09cd\u09a3 \u09aa\u09c3\u09a5\u0995 \u0964
+\u0989\u0987\u09a8\u09cd\u09a1\u09cb\u099c \u09ae\u09bf\u09b2\u09bf\u09a8\u09bf\u09af\u09bc\u09be\u09ae
+\u09b0\u09ac\u09c0\u09a8\u09cd\u09a6\u09cd\u09b0\u09a8\u09be\u09a5 \u0985\u09ac\u09b6\u09cd\u09af \u098f\u0995\u09be\u09a7\u09bf\u0995 \u09b6\u09c8\u09b2\u09c0 \u0986\u09af\u09bc\u09a4\u09cd\u09a4\u09cd\u09ac \u0995\u09b0\u09c7\u099b\u09bf\u09b2\u09c7\u09a8 \u0964
+\u09b6\u09cd\u09b0\u09ae \u0985\u09b0\u09cd\u09a5\u09a8\u09c0\u09a4\u09bf
+\u09ac\u09cd\u09b0\u09bf\u099f\u09c7\u09a8 \u098f\u0995\u09b8\u09ae\u09af\u09bc \u09ac\u09bf\u09b6\u09cd\u09ac\u09c7\u09b0 \u09aa\u09cd\u09b0\u09a7\u09be\u09a8 \u0993 \u0985\u0997\u09cd\u09b0\u0997\u09be\u09ae\u09c0 \u0985\u09b0\u09cd\u09a5\u09a8\u09c8\u09a4\u09bf\u0995 \u09b6\u0995\u09cd\u09a4\u09bf \u099b\u09bf\u09b2 \u0964
+\u09aa\u09be\u0995\u09bf\u09b8\u09cd\u09a4\u09be\u09a8\u09c7\u09b0 \u09b8\u09be\u09ae\u09b0\u09bf\u0995 \u09b6\u09be\u09b8\u09a8\u09c7\u09b0 \u09ac\u09bf\u09b0\u09c1\u09a6\u09cd\u09a7\u09c7 \u0986\u09a8\u09cd\u09a6\u09cb\u09b2\u09a8 \u098f\u09ac\u0982 \u09aa\u09be\u0995\u09bf\u09b8\u09cd\u09a4\u09be\u09a8\u09c7\u09b0 \u0997\u09cb\u09b7\u09cd\u09a0\u09c0\u0997\u09a4 \u09ac\u09c8\u09b7\u09ae\u09cd\u09af\u09c7\u09b0 \u09ac\u09bf\u09b0\u09c1\u09a6\u09cd\u09a7\u09c7 \u09aa\u09cd\u09b0\u09a4\u09bf\u09ac\u09be\u09a6 \u0993 \u09ac\u09be\u0999\u09be\u09b2\u09bf\u09a6\u09c7\u09b0 \u0986\u09a8\u09cd\u09a6\u09cb\u09b2\u09a8\u0995\u09c7 \u09b8\u09cd\u09ac\u09be\u09a7\u09c0\u09a8\u09a4\u09be\u09b0 \u09aa\u09a5\u09c7 \u09a7\u09be\u09ac\u09bf\u09a4 \u0995\u09b0\u09be\u09b0 \u099c\u09a8\u09cd\u09af \u09a4\u09bf\u09a8\u09bf \u09ac\u09cd\u09af\u09be\u09aa\u0995\u09ad\u09be\u09ac\u09c7 \u09aa\u09cd\u09b0\u09b6\u0982\u09b8\u09bf\u09a4 \u0964
+\u098f\u0996\u09be\u09a8\u09c7 \u0989\u09b2\u09cd\u09b2\u09c7\u0996 \u0995\u09b0\u09be \u09aa\u09cd\u09b0\u09af\u09bc\u09cb\u099c\u09a8 \u09af\u09c7 \u0985\u09a8\u09c7\u0995\u09c7 \u0987\u09a8\u09cd\u099f\u09be\u09b0\u09a8\u09c7\u099f \u098f\u09ac\u0982 \u0993\u09af\u09bc\u09be\u09b0\u09cd\u09b2\u09cd\u09a1 \u0993\u09af\u09bc\u09be\u0987\u09a1 \u0993\u09af\u09bc\u09c7\u09ac\u0995\u09c7 \u09b8\u09ae\u09be\u09b0\u09cd\u09a5\u0995 \u09b6\u09ac\u09cd\u09a6 \u09b9\u09bf\u09b8\u09c7\u09ac\u09c7 \u0997\u09a3\u09cd\u09af \u0995\u09b0\u09b2\u09c7\u0993 \u09aa\u09cd\u09b0\u0995\u09c3\u09a4\u09aa\u0995\u09cd\u09b7\u09c7 \u09b6\u09ac\u09cd\u09a6\u09a6\u09cd\u09ac\u09af\u09bc \u09ad\u09bf\u09a8\u09cd\u09a8 \u09ac\u09bf\u09b7\u09af\u09bc \u09a8\u09bf\u09b0\u09cd\u09a6\u09c7\u09b6 \u0995\u09b0\u09c7 \u0964
+. z \u098f\u09b0 \u0986\u09a8\u09c1\u09b8\u09be\u0999\u09cd\u0997\u09c0\u0995 \u09aa\u09cb\u09b2\u09be\u09b0 \u0995\u09cb-\u0985\u09b0\u09cd\u09a1\u09bf\u09a8\u09c7\u099f \u09a6\u09c1\u099f\u09bf \u09b9\u099a\u09cd\u099b\u09c7 r = -pipe-
+\u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0
+\u09e7\u09ef\u09ed\u09e8 \u0996\u09cd\u09b0\u09c0\u09b8\u09cd\u099f\u09be\u09ac\u09cd\u09a6\u09c7 \u09a4\u09a6\u09be\u09a8\u09bf\u09a8\u09cd\u09a4\u09a8 \u0986\u09b0\u09aa\u09be\u09a8\u09c7\u099f\u09c7 \u09b8\u09b0\u09cd\u09ac\u09aa\u09cd\u09b0\u09a5\u09ae \u0987\u09b2\u09c7\u0995\u09cd\u099f\u09cd\u09b0\u09a8\u09bf\u0995 \u09ae\u09c7\u0987\u09b2 \u09aa\u09cd\u09b0\u09c7\u09b0\u09a3 \u0995\u09b0\u09be \u09b9\u09af\u09bc \u0964
+\u099c\u09c0\u09ac \u09ac\u09bf\u099c\u09cd\u099e\u09be\u09a8\u09c7\u09b0 \u09af\u09c7 \u09b6\u09be\u0996\u09be\u09af\u09bc \u099b\u09a4\u09cd\u09b0\u09be\u0995 \u0993 \u098f\u09b0 \u09ac\u09cd\u09af\u09ac\u09b9\u09be\u09b0\u09bf\u0995 \u09ac\u09bf\u09b7\u09af\u09bc \u09a8\u09bf\u09af\u09bc\u09c7 \u0986\u09b2\u09cb\u099a\u09a8\u09be \u0995\u09b0\u09be \u09b9\u09af\u09bc \u09a4\u09be\u0995\u09c7 \u099b\u09a4\u09cd\u09b0\u09be\u0995\u09ac\u09bf\u09a6\u09cd\u09af\u09be -lrb- mycology -rrb-
+\u09aa\u09be\u09a8\u09bf \u09a8\u09a6\u09c0 \u09a5\u09c7\u0995\u09c7 \u0989\u09a0\u09be\u09a8\u09cb \u09b9\u09a4\u09cb \u0995\u09af\u09bc\u09c7\u0995\u099f\u09bf \u09aa\u09c1\u09b0 \u09a6\u09bf\u09af\u09bc\u09c7- \u09a6\u09a1\u09bc\u09bf \u0993 \u09ac\u09be\u09b2\u099f\u09bf\u09b0 \u09ae\u09be\u09a7\u09cd\u09af\u09ae\u09c7 \u09aa\u09be\u09a8\u09bf \u09aa\u09b6\u09c1 \u09a6\u09cd\u09ac\u09be\u09b0\u09be \u099f\u09c7\u09a8\u09c7 \u09a4\u09cb\u09b2\u09be\u09b0 \u098f\u0995 \u09aa\u09a6\u09cd\u09a7\u09a4\u09bf \u0964
+\u098f\u09b0 \u09ae\u09a7\u09cd\u09af\u09c7 \u09b0\u09af\u09bc\u09c7\u099b\u09c7 \u0989\u09aa\u099c\u09be\u09a4\u09c0\u09af\u09bc \u09a8\u09c3\u09a4\u09cd\u09af \u09b2\u09cb\u0995\u099c \u09a8\u09c3\u09a4\u09cd\u09af \u09b6\u09be\u09b8\u09cd\u09a4\u09cd\u09b0\u09c0\u09af\u09bc \u09a8\u09c3\u09a4\u09cd\u09af \u0987\u09a4\u09cd\u09af\u09be\u09a6\u09bf \u0964
+\u09ad\u09be\u09b0\u09a4\u09c7\u09b0 \u09aa\u09cd\u09b0\u09be\u099a\u09c0\u09a8\u09a4\u09ae \u09b8\u09be\u09b9\u09bf\u09a4\u09cd\u09af \u09aa\u09cd\u09b0\u09a5\u09ae\u09c7 \u09ae\u09cc\u0996\u09bf\u0995\u09ad\u09be\u09ac\u09c7 \u0993 \u09aa\u09b0\u09c7 \u09b2\u09bf\u0996\u09bf\u09a4 \u0986\u0995\u09be\u09b0\u09c7 \u09aa\u09cd\u09b0\u099a\u09b2\u09bf\u09a4 \u09b9\u09af\u09bc \u0964
+\u09e7\u09ef\u09ee\u09ef \u09b8\u09be\u09b2\u09c7 \u09a8\u09bf\u09b0\u09cd\u09ae\u09bf\u09a4 \u0997\u09a3\u09b6\u09a4\u09cd\u09b0\u09c1 \u099b\u09ac\u09bf\u099f\u09bf\u09a4\u09c7 \u09a4\u09be\u0981\u09b0 \u09aa\u09b0\u09bf\u099a\u09be\u09b2\u09a8\u09be \u09a4\u09c1\u09b2\u09a8\u09be\u09ae\u09c2\u09b2\u0995\u09ad\u09be\u09ac\u09c7 \u09a6\u09c1\u09b0\u09cd\u09ac\u09b2 \u098f\u09ac\u0982 \u098f\u099f\u09bf\u0995\u09c7 \u09a6\u09c0\u09b0\u09cd\u0998\u09a6\u09bf\u09a8\u09c7\u09b0 \u0985\u09b8\u09c1\u09b8\u09cd\u09a5\u09a4\u09be\u09b6\u09c7\u09b7\u09c7 \u09ab\u09bf\u09b0\u09c7 \u0986\u09b8\u09be\u09b0 \u09aa\u09b0 \u09b8\u09a4\u09cd\u09af\u099c\u09bf\u09a4\u09c7\u09b0 \u099a\u09b2\u099a\u09cd\u099a\u09bf\u09a4\u09cd\u09b0 \u09a8\u09bf\u09b0\u09cd\u09ae\u09be\u09a3\u09c7\u09b0 \u09aa\u09c1\u09a8\u09b0\u09cd\u09aa\u09cd\u09b0\u099a\u09c7\u09b7\u09cd\u099f\u09be \u09b9\u09bf\u09b8\u09c7\u09ac\u09c7\u0987 \u0997\u09a3\u09cd\u09af \u0995\u09b0\u09be \u09b9\u09af\u09bc \u0964
+\u2022 \u09ac\u09b2\u09ac\u09bf\u09a6\u09cd\u09af\u09be
+\u0985\u09a8\u09cd\u09af\u09be\u09a8\u09cd\u09af \u09b8\u09cd\u09ac\u09a4\u09cd\u09a4\u09cd\u09ac-\u09b8\u0982\u09b0\u0995\u09cd\u09b7\u09bf\u09a4 \u0985\u09aa\u09be\u09b0\u09c7\u099f\u09bf\u0982 \u09b8\u09bf\u09b8\u09cd\u099f\u09c7\u09ae \u09af\u09c7\u09ae\u09a8 \u0989\u0987\u09a8\u09cd\u09a1\u09cb\u099c \u098f\u09ac\u0982 \u09ae\u09cd\u09af\u09be\u0995 \u0993\u098f\u09b8 \u09b9\u09a4\u09c7 \u09b2\u09bf\u09a8\u09be\u0995\u09cd\u09b8 \u09ac\u09bf\u09ad\u09bf\u09a8\u09cd\u09a8\u09ad\u09be\u09ac\u09c7 \u0986\u09b2\u09be\u09a6\u09be \u0964
+\u098f\u09b6\u09bf\u09af\u09bc\u09be \u099f\u09be\u0987\u09ae\u09b8\u09c7\u09b0 \u09ad\u09be\u09b7\u09cd\u09af \u0985\u09a8\u09c1\u09af\u09be\u09af\u09bc\u09c0 & # 44 ;
+\u09ae\u09c1\u0995\u09cd\u09a4 \u09b8\u09cb\u09b0\u09cd\u09b8 \u09ac\u09be \u0993\u09aa\u09c7\u09a8 \u09b8\u09cb\u09b0\u09cd\u09b8 -lrb- open source -rrb- \u098f\u09b0 \u0985\u09b0\u09cd\u09a5 \u09b9\u09b2\u09cb \u0995\u09ae\u09cd\u09aa\u09bf\u0989\u099f\u09be\u09b0 \u09b8\u09ab\u099f\u0993\u09af\u09bc\u09cd\u09af\u09be\u09b0 \u098f\u09b0 \u09b8\u09cb\u09b0\u09cd\u09b8 \u0995\u09cb\u09a1 \u09ac\u09be \u09ae\u09c2\u09b2 \u09b8\u09be\u0982\u0995\u09c7\u09a4\u09bf\u0995 \u09ad\u09be\u09b7\u09be\u0995\u09c7 \u09ae\u09c1\u0995\u09cd\u09a4 \u09ad\u09be\u09ac\u09c7 \u09ac\u09bf\u09a4\u09b0\u09a3 \u0995\u09b0\u09be \u0964
+\u09ac\u09be\u0982\u09b2\u09be\u09a6\u09c7\u09b6 \u0985\u09a8\u09b2\u09be\u0987\u09a8\u09c7 \u09a2\u09be\u0995\u09be
+\u09aa\u09cd\u09b0\u09a5\u09ae \u09ac\u09bf\u09b6\u09cd\u09ac\u09af\u09c1\u09a6\u09cd\u09a7\u09c7 \u099c\u09be\u09b0\u09cd\u09ae\u09be\u09a8\u09bf \u09b9\u09c7\u09b0\u09c7 \u09af\u09be\u09af\u09bc \u0964
+\u09a4\u09ac\u09c7 \u098f \u09ac\u09bf\u09b7\u09af\u09bc\u099f\u09bf \u09ac\u09cb\u099d\u09be\u09b0 \u099c\u09a8\u09cd\u09af \u09ac\u09b0\u09cd\u09a4\u09ae\u09be\u09a8\u09c7\u0993 \u0997\u09ac\u09c7\u09b7\u09a3\u09be \u098f\u0997\u09bf\u09af\u09bc\u09c7 \u099a\u09b2\u099b\u09c7 \u0964
+\u09b8\u09c1\u09aa\u09be\u09b0\u098f\u0987\u099a
+\u09a4\u09be\u0995\u09c7 \u09b8\u09be\u09ae\u09b0\u09bf\u0995 \u09ac\u09be\u09b9\u09bf\u09a8\u09c0\u09b0 \u099c\u09a8\u09cd\u09af \u0986\u09a8\u09ab\u09bf\u099f \u0998\u09cb\u09b7\u09a3\u09be \u0995\u09b0\u09be \u09b9\u09af\u09bc\u09c7\u099b\u09bf\u09b2 \u0964
+\u09ad\u09c1\u099f\u09cd\u099f\u09cb \u098f\u09cd\u09af\u09be\u09b8\u09c7\u09ae\u09cd\u09ac\u09b2\u09bf \u09ac\u09af\u09bc\u0995\u099f \u0995\u09b0\u09be\u09b0 \u09b9\u09c1\u09ae\u0995\u09bf \u09a6\u09bf\u09af\u09bc\u09c7 \u0998\u09cb\u09b7\u09a3\u09be \u09a6\u09c7\u09a8 \u09af\u09c7 \u0987\u09af\u09bc\u09be\u09b9\u09bf\u09af\u09bc\u09be \u0996\u09be\u09a8 \u09ae\u09c1\u099c\u09bf\u09ac\u0995\u09c7 \u09b8\u09b0\u0995\u09be\u09b0 \u0997\u09a0\u09a8\u09c7\u09b0 \u099c\u09a8\u09cd\u09af \u0986\u09b9\u09cd\u09ac\u09be\u09a8 \u099c\u09be\u09a8\u09be\u09b2\u09c7 \u09a4\u09bf\u09a8\u09bf \u09b8\u09c7 \u09b8\u09b0\u0995\u09be\u09b0\u0995\u09c7 \u09ae\u09c7\u09a8\u09c7 \u09a8\u09c7\u09ac\u09c7\u09a8 \u09a8\u09be \u0964
+\u0986\u09b0 computer \u09b6\u09ac\u09cd\u09a6\u09c7\u09b0 \u0985\u09b0\u09cd\u09a5 \u0997\u09a3\u09a8\u09be\u0995\u09be\u09b0\u09c0 \u09af\u09a8\u09cd\u09a4\u09cd\u09b0 \u0964
+\u09e7\u09ed\u09ed\u09ec \u09b8\u09be\u09b2\u09c7\u09b0 \u09ea \u099c\u09c1\u09b2\u09be\u0987 \u098f\u0987 \u0989\u09aa\u09a8\u09bf\u09ac\u09c7\u09b6\u0997\u09c1\u09b2\u09bf \u098f\u0995\u099f\u09bf \u09b8\u09cd\u09ac\u09be\u09a7\u09c0\u09a8\u09a4\u09be\u09b0 \u0998\u09cb\u09b7\u09a3\u09be\u09aa\u09a4\u09cd\u09b0 \u099c\u09be\u09b0\u09bf \u0995\u09b0\u09c7 \u0964
+\u099c\u09be\u09b0\u09cd\u09ae\u09be\u09a8\u09bf -lrb- \u099c\u09be\u09b0\u09cd\u09ae\u09be\u09a8 \u09ad\u09be\u09b7\u09be\u09af\u09bc : deutschland \u09a1\u09af\u09bc\u099a\u09cd \u200c \u09b2\u09be\u09a8\u09cd\u099f\u09cd \u200c \u0986-\u09a7\u09cd\u09ac-\u09ac : [ d\u0254\u028ft\u0283lant ] -rrb- \u09ae\u09a7\u09cd\u09af \u0987\u0989\u09b0\u09cb\u09aa\u09c7\u09b0 \u098f\u0995\u099f\u09bf \u09b0\u09be\u09b7\u09cd\u099f\u09cd\u09b0 \u0964
+\u0996\u09cd\u09b0\u09bf\u09b8\u09cd\u099f\u09a7\u09b0\u09cd\u09ae \u09b0\u09be\u09b6\u09bf\u09af\u09bc\u09be\u09b0 \u09aa\u09cd\u09b0\u09a7\u09be\u09a8 \u09a7\u09b0\u09cd\u09ae \u0964
+\u0995\u09bf\u09a8\u09cd\u09a4\u09c1 \u0997\u09b2\u09a6\u09c7\u09b0 \u09b6\u09bf\u0995\u09cd\u09b7\u09be\u09b0 \u09b0\u09cb\u09ae\u09be\u09a8\u09c0\u0995\u09b0\u09a3\u09c7\u09b0 \u0997\u09a4\u09bf \u099b\u09bf\u09b2 \u09a7\u09c0\u09b0 \u0964
+\u09ac\u09bf\u09b7\u09af\u09bc\u09b6\u09cd\u09b0\u09c7\u09a3\u09c0 : \u0997\u09a8\u09c1 \u09ab\u09be\u0989\u09a8\u09cd\u09a1\u09c7\u09b6\u09a8
+\u0986\u09b0\u09cd\u09a5\u09bf\u0995 \u09a8\u09c0\u09a4\u09bf \u0993 \u09b0\u09be\u099c\u09b8\u09cd\u09ac \u09a8\u09c0\u09a4\u09bf\u0995\u09c7\u0993 \u0987\u09b9\u09be \u0985\u09a7\u09cd\u09af\u09af\u09bc\u09a8 \u0995\u09b0\u09c7 \u0964
+\u098f\u09b0 \u09ae\u09a7\u09cd\u09af\u09c7 \u09b0\u09af\u09bc\u09c7\u099b\u09c7 : \u09b9\u09af\u09bc\u09a4\u09cb \u09a4\u09cb\u09ae\u09be\u09b0 \u09aa\u09be\u09ac \u09a6\u09c7\u0996\u09be \u0993\u09b0\u09c7 \u098f \u0995\u09cb\u09a8 \u09b8\u09cd\u09a8\u09c7\u09b9-\u09b8\u09c1\u09b0\u09a7\u09c1\u09a8\u09c0 \u0964
+\u09e7\u09ef\u09ef\u09e8 \u09b8\u09be\u09b2\u09c7\u09b0 \u09e8\u09e9 \u098f\u09aa\u09cd\u09b0\u09bf\u09b2 \u09b8\u09a4\u09cd\u09af\u099c\u09bf\u09ce \u09ae\u09c3\u09a4\u09cd\u09af\u09c1\u09ac\u09b0\u09a3 \u0995\u09b0\u09c7\u09a8 \u0964
+\u098f\u0987 \u09b8\u09ae\u09af\u09bc \u09a8\u099c\u09b0\u09c1\u09b2\u09c7\u09b0 \u09ae\u09c7\u09a1\u09bf\u0995\u09c7\u09b2 \u09b0\u09bf\u09aa\u09cb\u09b0\u09cd\u099f \u09ad\u09bf\u09af\u09bc\u09c7\u09a8\u09be\u09b0 \u09ac\u09bf\u0996\u09cd\u09af\u09be\u09a4 \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u0995\u09a6\u09c7\u09b0 \u0995\u09be\u099b\u09c7 \u09aa\u09be\u09a0\u09be\u09a8\u09cb \u09b9\u09af\u09bc \u0964
+\u0985\u09ad\u09bf\u09a8\u09af\u09bc \u099b\u09be\u09a1\u09bc\u09be\u0993 \u09ac\u09bf\u09ad\u09bf\u09a8\u09cd\u09a8 \u09b8\u09ae\u09af\u09bc\u09c7 \u09b0\u09be\u09a8\u09c0 \u09ae\u09c1\u0996\u09be\u09b0\u09cd\u099c\u09c0 \u09ac\u09bf\u09ad\u09bf\u09a8\u09cd\u09a8 \u09a6\u09be\u09a4\u09ac\u09cd\u09af \u09b8\u0982\u09b8\u09cd\u09a5\u09be\u09b0 \u09b8\u09be\u09a5\u09c7 \u09af\u09c1\u0995\u09cd\u09a4 \u09a5\u09c7\u0995\u09c7\u099b\u09c7\u09a8 \u0964
+\u09ac\u09be\u0982\u09b2\u09be \u09b8\u09be\u09b9\u09bf\u09a4\u09cd\u09af \u098f\u09ac\u0982 \u09b8\u0982\u09b8\u09cd\u0995\u09c3\u09a4\u09bf\u09a4\u09c7 \u09a4\u09be\u09b0 \u09ac\u09bf\u09b6\u09c7\u09b7 \u0985\u09ac\u09a6\u09be\u09a8\u09c7\u09b0 \u09b8\u09cd\u09ac\u09c0\u0995\u09c3\u09a4\u09bf\u09b8\u09cd\u09ac\u09b0\u09c1\u09aa \u09e7\u09ef\u09ed\u09ea \u0996\u09cd\u09b0\u09bf\u09b8\u09cd\u099f\u09be\u09ac\u09cd\u09a6\u09c7\u09b0 \u09ef \u09a1\u09bf\u09b8\u09c7\u09ae\u09cd\u09ac\u09b0 \u09a4\u09be\u09b0\u09bf\u0996\u09c7 \u09a2\u09be\u0995\u09be \u09ac\u09bf\u09b6\u09cd\u09ac\u09ac\u09bf\u09a6\u09cd\u09af\u09be\u09b2\u09af\u09bc \u09a4\u09be\u0995\u09c7 \u09b8\u09ae\u09cd\u09ae\u09be\u09a8\u09b8\u09c2\u099a\u0995 \u09a1\u09bf.\u09b2\u09bf\u099f \u0989\u09aa\u09be\u09a7\u09bf\u09a4\u09c7 \u09ad\u09c2\u09b7\u09bf\u09a4 \u0995\u09b0\u09c7 \u0964
+\u0995\u09b2\u0995\u09be\u09a4\u09be\u09b0 \u09a6\u09c1\u09b0\u09cd\u0997\u09be\u09aa\u09c2\u099c\u09be \u09b6\u09b9\u09b0\u09c7\u09b0 \u0985\u09a8\u09cd\u09af\u09a4\u09ae \u09aa\u09b0\u09cd\u09af\u099f\u09a8 \u0986\u0995\u09b0\u09cd\u09b7\u09a3\u0993 \u09ac\u099f\u09c7 \u0964
+\u0995\u09bf\u09a8\u09cd\u09a4\u09c1 \u09ac\u09b9\u09c1 \u09b2\u0995\u09cd\u09b7 \u09aa\u09c2\u09b0\u09cd\u09ac \u099c\u09be\u09b0\u09cd\u09ae\u09be\u09a8 \u0985\u09b0\u09cd\u09a5\u09a8\u09c8\u09a4\u09bf\u0995\u09ad\u09be\u09ac\u09c7 \u09b8\u09ae\u09c3\u09a6\u09cd\u09a7 \u0993 \u0997\u09a3\u09a4\u09be\u09a8\u09cd\u09a4\u09cd\u09b0\u09bf\u0995 \u09aa\u09b6\u09cd\u099a\u09bf\u09ae \u099c\u09be\u09b0\u09cd\u09ae\u09be\u09a8\u09bf\u09a4\u09c7 \u0985\u09ad\u09bf\u09ac\u09be\u09b8\u09c0 \u09b9\u0993\u09af\u09bc\u09be \u09b6\u09c1\u09b0\u09c1 \u0995\u09b0\u09b2\u09c7 \u09e7\u09ef\u09ec\u09e7 \u09b8\u09be\u09b2\u09c7 \u09aa\u09c2\u09b0\u09cd\u09ac \u099c\u09be\u09b0\u09cd\u09ae\u09be\u09a8\u09bf \u09b8\u09b0\u0995\u09be\u09b0 \u09ac\u09be\u09b0\u09cd\u09b2\u09bf\u09a8\u09c7 \u098f\u0995\u099f\u09bf \u09aa\u09cd\u09b0\u09be\u099a\u09c0\u09b0 \u09a4\u09c1\u09b2\u09c7 \u09a6\u09c7\u09af\u09bc \u098f\u09ac\u0982 \u09a6\u09c7\u09b6\u09c7\u09b0 \u09b8\u09c0\u09ae\u09be\u09a8\u09cd\u09a4 \u099c\u09cb\u09b0\u09a6\u09be\u09b0 \u0995\u09b0\u09c7 \u0964
+\u09aa\u09cd\u09b0\u09a5\u09ae\u099f\u09bf \u09b9\u099a\u09cd\u099b\u09c7 \u099b\u09ac\u09bf\u099f\u09bf\u09b0 \u09a8\u09bf\u09b0\u09cd\u09ac\u09be\u0995 \u09aa\u09cd\u09b0\u09a5\u09ae \u09b8\u09be\u09a4 \u09ae\u09bf\u09a8\u09bf\u099f \u09af\u09be \u099a\u09be\u09b0\u09c1\u09b0 \u098f\u0995\u0998\u09c7\u09af\u09bc\u09c7\u09ae\u09bf \u099c\u09c0\u09ac\u09a8 \u09ab\u09c1\u099f\u09bf\u09af\u09bc\u09c7 \u09a4\u09cb\u09b2\u09c7 \u098f\u09ac\u0982 \u09a6\u09cd\u09ac\u09bf\u09a4\u09c0\u09af\u09bc\u099f\u09bf \u09b9\u09b2 & quot ; \u09ac\u09be\u0997\u09be\u09a8\u09c7\u09b0 \u09a6\u09cb\u09b2\u09a8\u09be\u09b0 \u09a6\u09c3\u09b6\u09cd\u09af & quot ; \u09af\u09c7\u0996\u09be\u09a8\u09c7 \u099a\u09be\u09b0\u09c1 \u0985\u09ae\u09b2\u09c7\u09b0 \u099c\u09a8\u09cd\u09af \u09a4\u09be\u09b0 \u09ad\u09be\u09b2\u09ac\u09be\u09b8\u09be\u09b0 \u09ae\u09c1\u0996\u09cb\u09ae\u09c1\u0996\u09bf \u09b9\u09af\u09bc \u0964
+\u09e7\u09ee \u09b6\u09a4\u0995\u09c7\u09b0 \u098f\u0995\u09a6\u09b2 \u099a\u09bf\u09a8\u09cd\u09a4\u09be\u09ac\u09bf\u09a6 \u0993 \u09b2\u09c7\u0996\u0995 \u0986\u09af\u09bc \u0993 \u0989\u09ce\u09aa\u09be\u09a6\u09a8\u09c7\u09b0 \u099a\u0995\u09cd\u09b0\u09be\u0995\u09be\u09b0 \u09aa\u09cd\u09b0\u09ac\u09be\u09b9\u09c7\u09b0 \u09ae\u09be\u09a7\u09cd\u09af\u09ae\u09c7 \u0985\u09b0\u09cd\u09a5\u09a8\u09c8\u09a4\u09bf\u0995 \u099a\u09bf\u09a8\u09cd\u09a4\u09be\u09a7\u09be\u09b0\u09be\u09b0 \u0989\u09a8\u09cd\u09a8\u09af\u09bc\u09a8 \u0998\u099f\u09be\u09a8 \u0964
+\u09af\u09cb\u09a8\u09c0\u09a4\u09c7 \u09b2\u09bf\u0999\u09cd\u0997 \u09aa\u09cd\u09b0\u09ac\u09bf\u09b7\u09cd\u099f\u0995\u09b0\u09a3\u09c7\u09b0 \u09aa\u09c2\u09b0\u09cd\u09ac\u09c7 \u0995\u09be\u09ae\u09cb\u09a6\u09cd\u09a6\u09c0\u09aa\u0995 \u0995\u09be\u09b0\u09cd\u09af\u0995\u09b2\u09be\u09aa\u0995\u09c7 \u09ac\u09b2\u09be \u09b9\u09af\u09bc \u09b6\u09c3\u0999\u09cd\u0997\u09be\u09b0 \u0964
+\u098f\u099f\u09bf \u09ae\u09c2\u09b2\u09a4 \u09b6\u09c2\u0995\u09b0\u09c7\u09b0 \u09ae\u09be\u099d\u09c7\u0987 \u09aa\u09be\u0993\u09af\u09bc\u09be \u09af\u09c7\u09a4 \u09af\u09be \u0995\u09bf\u09a8\u09be \u09b6\u09c2\u0995\u09b0\u0995\u09c7 \u0987\u09a8\u09ab\u09cd\u09b2\u09c1\u09af\u09bc\u09c7\u099e\u09cd\u099c\u09be\u09a4\u09c7 \u0986\u0995\u09cd\u09b0\u09be\u09a8\u09cd\u09a4 \u0995\u09b0\u09a4 \u0964
+\u098f\u0997\u09c1\u09b2\u09bf \u098f\u0995\u098f \u09b9\u09af\u09bc\u09c7 mycelium \u0997\u09a0\u09a8 \u0995\u09b0\u09c7 \u0964
+\u09b0\u09be\u09b6\u09bf\u09af\u09bc\u09be \u09ac\u09b0\u09cd\u09a4\u09ae\u09be\u09a8\u09c7 \u098f\u0995\u099f\u09bf \u0997\u09a3\u09a4\u09be\u09a8\u09cd\u09a4\u09cd\u09b0\u09bf\u0995 \u09b0\u09be\u09b7\u09cd\u099f\u09cd\u09b0 \u0964
+\u09b2\u09bf\u0999\u09cd\u0997
+\u098f\u0987 \u0985\u09ac\u09b8\u09cd\u09a5\u09be\u0995\u09c7 \u09ac\u09b2\u09be \u09b9\u09af\u09bc \u09b0\u09be\u0997\u09ae\u09cb\u099a\u09a8 -lrb- \u0985\u09a8\u0997\u09cd\u09af\u09be\u099c\u09ae -rrb- \u0964
+\u0987\u09a4\u09bf\u09b9\u09be\u09b8\u09c7\u09b0 \u09ac\u09bf\u09ad\u09bf\u09a8\u09cd\u09a8 \u09aa\u09b0\u09cd\u09ac\u09c7 \u098f\u0996\u09be\u09a8\u09c7\u0987 \u09b8\u09cd\u09a5\u09be\u09aa\u09bf\u09a4 \u09b9\u09af\u09bc\u09c7\u099b\u09bf\u09b2 \u09ac\u09bf\u09b6\u09be\u09b2\u09be\u0995\u09be\u09b0 \u098f\u0995\u09be\u09a7\u09bf\u0995 \u09b8\u09be\u09ae\u09cd\u09b0\u09be\u099c\u09cd\u09af \u0964
+\u09ac\u09cd\u09af\u09be\u09b8\u09cd\u099f\u09bf\u0995 \u0985\u09b0\u09cd\u09a5\u09a8\u09c0\u09a4\u09bf
+\u09ac\u09cd\u09af\u09ac\u09b9\u09be\u09b0\u0995\u09be\u09b0\u09c0\u09b0 \u0995\u09be\u099b\u09c7 \u0985\u09aa\u09be\u09b0\u09c7\u099f\u09bf\u0982 \u09b8\u09bf\u09b8\u09cd\u099f\u09c7\u09ae\u09c7\u09b0 \u09b8\u09ac\u099a\u09c7\u09af\u09bc\u09c7 \u09a6\u09c3\u09b6\u09cd\u09af\u09ae\u09be\u09a8 \u09b0\u09c2\u09aa \u09b9\u09b2 \u0995\u09ae\u09cd\u09aa\u09bf\u0989\u099f\u09be\u09b0\u09c7\u09b0 \u0987\u09a8\u09cd\u099f\u09be\u09b0\u09ab\u09c7\u09b8 \u0964
+\u09ac\u09bf\u09b8\u09cd\u09a4\u09be\u09b0\u09bf\u09a4\u0983 \u09e7\u09ef\u09ed\u09e7 \u09b8\u09be\u09b2\u09c7\u09b0 \u0985\u09b8\u09cd\u09a5\u09be\u09af\u09bc\u09c0 \u09ac\u09be\u0982\u09b2\u09be\u09a6\u09c7\u09b6 \u09b8\u09b0\u0995\u09be\u09b0
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/hiero/joshua-berkeleylm.config
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/hiero/joshua-berkeleylm.config b/joshua-core/src/test/resources/bn-en/hiero/joshua-berkeleylm.config
new file mode 100644
index 0000000..e1bf2f6
--- /dev/null
+++ b/joshua-core/src/test/resources/bn-en/hiero/joshua-berkeleylm.config
@@ -0,0 +1,46 @@
+feature-function = LanguageModel -lm_type berkeleylm -lm_order 5 -lm_file lm.gz
+
+tm = thrax pt 12 grammar.gz
+tm = thrax glue -1 glue-grammar
+
+mark-oovs = false
+
+#tm config
+default_non_terminal=X
+goalSymbol=GOAL
+
+#pruning config
+pop-limit = 100
+
+#nbest config
+use_unique_nbest = true
+top_n = 1
+
+feature-function = OOVPenalty
+feature-function = WordPenalty
+
+###### model weights
+#lm order weight
+lm_0 1.2373676802179452
+
+#phrasemodel owner column(0-indexed) weight
+tm_pt_0 -2.4497429277910214
+tm_pt_1 0.7224581556224123
+tm_pt_2 -0.31689069155153504
+tm_pt_3 0.33861043967238036
+tm_pt_4 0.03553113401320236
+tm_pt_5 0.19138972284064748
+tm_pt_6 0.3417994095521415
+tm_pt_7 -0.9936312455671283
+tm_pt_8 0.9070737587091975
+tm_pt_9 0.8202511858619419
+tm_pt_10 0.2593091306160006
+tm_pt_11 0.25597137004462134
+tm_pt_12 0.3538894647790496
+tm_pt_13 -0.36212061186692646
+tm_pt_14 -0.32923261148678096
+tm_pt_15 0.5524863522177359
+tm_pt_16 0.23451595442127693
+tm_glue_0 1
+WordPenalty -3.6942747832593694
+OOVPenalty 1.0
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/hiero/joshua-classlm.config
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/hiero/joshua-classlm.config b/joshua-core/src/test/resources/bn-en/hiero/joshua-classlm.config
new file mode 100644
index 0000000..970b9b7
--- /dev/null
+++ b/joshua-core/src/test/resources/bn-en/hiero/joshua-classlm.config
@@ -0,0 +1,51 @@
+feature-function = LanguageModel -lm_type kenlm -lm_order 5 -minimizing false -lm_file lm.gz
+
+# Class LM feature
+feature-function = LanguageModel -lm_type kenlm -lm_order 9 -minimizing false -lm_file class_lm_9gram.gz -lm_class -class_map class.map
+
+###### Old format for lms
+# lm = kenlm 5 false false 100 lm.gz
+
+# tm = TYPE OWNER MAX_SPAN PATH
+tm = thrax pt 12 grammar.gz
+tm = thrax glue -1 glue-grammar
+
+mark_oovs=false
+
+#tm config
+default_non_terminal=X
+goalSymbol=GOAL
+
+#pruning config
+pop-limit = 10
+
+#nbest config
+use_unique_nbest=true
+top_n = 10
+
+feature-function = WordPenalty
+feature-function = OOVPenalty
+
+###### model weights
+lm_0 1.2373676802179452
+lm_1 1.2373676802179452
+tm_pt_0 -2.4497429277910214
+tm_pt_1 0.7224581556224123
+tm_pt_2 -0.31689069155153504
+tm_pt_3 0.33861043967238036
+tm_pt_4 0.03553113401320236
+tm_pt_5 0.19138972284064748
+tm_pt_6 0.3417994095521415
+tm_pt_7 -0.9936312455671283
+tm_pt_8 0.9070737587091975
+tm_pt_9 0.8202511858619419
+tm_pt_10 0.2593091306160006
+tm_pt_11 0.25597137004462134
+tm_pt_12 0.3538894647790496
+tm_pt_13 -0.36212061186692646
+tm_pt_14 -0.32923261148678096
+tm_pt_15 0.5524863522177359
+tm_pt_16 0.23451595442127693
+tm_glue_0 1
+WordPenalty -3.6942747832593694
+OOVPenalty 1.0
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/hiero/joshua.config
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/hiero/joshua.config b/joshua-core/src/test/resources/bn-en/hiero/joshua.config
new file mode 100644
index 0000000..5a51698
--- /dev/null
+++ b/joshua-core/src/test/resources/bn-en/hiero/joshua.config
@@ -0,0 +1,50 @@
+feature-function = LanguageModel -lm_type kenlm -lm_order 5 -minimizing false -lm_file lm.gz
+
+###### Old format for lms
+# lm = kenlm 5 false false 100 lm.gz
+
+# tm = TYPE OWNER MAX_SPAN PATH
+#tm = thrax pt 12 grammar.gz
+#tm = thrax glue -1 glue-grammar
+tm = thrax -owner pt -maxspan 12 -path grammar.gz
+tm = thrax -owner glue -maxspan -1 -path glue-grammar
+
+mark_oovs=false
+
+#tm config
+default_non_terminal=X
+goalSymbol=GOAL
+
+#pruning config
+pop-limit = 10
+
+#nbest config
+use_unique_nbest=true
+top_n = 10
+
+feature-function = OOVPenalty
+feature-function = WordPenalty
+
+###### model weights
+lm_0 1.2373676802179452
+lm_1 1.2373676802179452
+tm_pt_0 -2.4497429277910214
+tm_pt_1 0.7224581556224123
+tm_pt_2 -0.31689069155153504
+tm_pt_3 0.33861043967238036
+tm_pt_4 0.03553113401320236
+tm_pt_5 0.19138972284064748
+tm_pt_6 0.3417994095521415
+tm_pt_7 -0.9936312455671283
+tm_pt_8 0.9070737587091975
+tm_pt_9 0.8202511858619419
+tm_pt_10 0.2593091306160006
+tm_pt_11 0.25597137004462134
+tm_pt_12 0.3538894647790496
+tm_pt_13 -0.36212061186692646
+tm_pt_14 -0.32923261148678096
+tm_pt_15 0.5524863522177359
+tm_pt_16 0.23451595442127693
+tm_glue_0 1
+WordPenalty -3.6942747832593694
+OOVPenalty 1.0
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/hiero/lm.gz
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/hiero/lm.gz b/joshua-core/src/test/resources/bn-en/hiero/lm.gz
new file mode 100644
index 0000000..a26335e
Binary files /dev/null and b/joshua-core/src/test/resources/bn-en/hiero/lm.gz differ
[45/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/adagrad/Optimizer.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/adagrad/Optimizer.java b/joshua-core/src/main/java/org/apache/joshua/adagrad/Optimizer.java
new file mode 100755
index 0000000..722c593
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/adagrad/Optimizer.java
@@ -0,0 +1,728 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.adagrad;
+
+import java.util.Collections;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.Vector;
+import java.lang.Math;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.metrics.EvaluationMetric;
+
+// this class implements the AdaGrad algorithm
+public class Optimizer {
+ public Optimizer(Vector<String>_output, boolean[] _isOptimizable, double[] _initialLambda,
+ HashMap<String, String>[] _feat_hash, HashMap<String, String>[] _stats_hash) {
+ output = _output; // (not used for now)
+ isOptimizable = _isOptimizable;
+ initialLambda = _initialLambda; // initial weights array
+ paramDim = initialLambda.length - 1;
+ initialLambda = _initialLambda;
+ feat_hash = _feat_hash; // feature hash table
+ stats_hash = _stats_hash; // suff. stats hash table
+ finalLambda = new double[initialLambda.length];
+ for(int i = 0; i < finalLambda.length; i++)
+ finalLambda[i] = initialLambda[i];
+ }
+
+ //run AdaGrad for one epoch
+ public double[] runOptimizer() {
+ List<Integer> sents = new ArrayList<Integer>();
+ for( int i = 0; i < sentNum; ++i )
+ sents.add(i);
+ double[] avgLambda = new double[initialLambda.length]; //only needed if averaging is required
+ for( int i = 0; i < initialLambda.length; ++i )
+ avgLambda[i] = 0;
+ for ( int iter = 0; iter < adagradIter; ++iter ) {
+ System.arraycopy(finalLambda, 1, initialLambda, 1, paramDim);
+ if(needShuffle)
+ Collections.shuffle(sents);
+
+ double oraMetric, oraScore, predMetric, predScore;
+ double[] oraPredScore = new double[4];
+ double loss = 0;
+ double diff = 0;
+ double sumMetricScore = 0;
+ double sumModelScore = 0;
+ String oraFeat = "";
+ String predFeat = "";
+ String[] oraPredFeat = new String[2];
+ String[] vecOraFeat;
+ String[] vecPredFeat;
+ String[] featInfo;
+ int thisBatchSize = 0;
+ int numBatch = 0;
+ int numUpdate = 0;
+ Iterator it;
+ Integer diffFeatId;
+
+ //update weights
+ Integer s;
+ int sentCount = 0;
+ double prevLambda = 0;
+ double diffFeatVal = 0;
+ double oldVal = 0;
+ double gdStep = 0;
+ double Hii = 0;
+ double gradiiSquare = 0;
+ int lastUpdateTime = 0;
+ HashMap<Integer, Integer> lastUpdate = new HashMap<Integer, Integer>();
+ HashMap<Integer, Double> lastVal = new HashMap<Integer, Double>();
+ HashMap<Integer, Double> H = new HashMap<Integer, Double>();
+ while( sentCount < sentNum ) {
+ loss = 0;
+ thisBatchSize = batchSize;
+ ++numBatch;
+ HashMap<Integer, Double> featDiff = new HashMap<Integer, Double>();
+ for(int b = 0; b < batchSize; ++b ) {
+ //find out oracle and prediction
+ s = sents.get(sentCount);
+ findOraPred(s, oraPredScore, oraPredFeat, finalLambda, featScale);
+
+ //the model scores here are already scaled in findOraPred
+ oraMetric = oraPredScore[0];
+ oraScore = oraPredScore[1];
+ predMetric = oraPredScore[2];
+ predScore = oraPredScore[3];
+ oraFeat = oraPredFeat[0];
+ predFeat = oraPredFeat[1];
+
+ //update the scale
+ if(needScale) { //otherwise featscale remains 1.0
+ sumMetricScore += Math.abs(oraMetric + predMetric);
+ //restore the original model score
+ sumModelScore += Math.abs(oraScore + predScore) / featScale;
+
+ if(sumModelScore/sumMetricScore > scoreRatio)
+ featScale = sumMetricScore/sumModelScore;
+ }
+ // processedSent++;
+
+ vecOraFeat = oraFeat.split("\\s+");
+ vecPredFeat = predFeat.split("\\s+");
+
+ //accumulate difference feature vector
+ if ( b == 0 ) {
+ for (int i = 0; i < vecOraFeat.length; i++) {
+ featInfo = vecOraFeat[i].split("=");
+ diffFeatId = Integer.parseInt(featInfo[0]);
+ featDiff.put(diffFeatId, Double.parseDouble(featInfo[1]));
+ }
+ for (int i = 0; i < vecPredFeat.length; i++) {
+ featInfo = vecPredFeat[i].split("=");
+ diffFeatId = Integer.parseInt(featInfo[0]);
+ if (featDiff.containsKey(diffFeatId)) { //overlapping features
+ diff = featDiff.get(diffFeatId)-Double.parseDouble(featInfo[1]);
+ if ( Math.abs(diff) > 1e-20 )
+ featDiff.put(diffFeatId, diff);
+ else
+ featDiff.remove(diffFeatId);
+ }
+ else //features only firing in the 2nd feature vector
+ featDiff.put(diffFeatId, -1.0*Double.parseDouble(featInfo[1]));
+ }
+ } else {
+ for (int i = 0; i < vecOraFeat.length; i++) {
+ featInfo = vecOraFeat[i].split("=");
+ diffFeatId = Integer.parseInt(featInfo[0]);
+ if (featDiff.containsKey(diffFeatId)) { //overlapping features
+ diff = featDiff.get(diffFeatId)+Double.parseDouble(featInfo[1]);
+ if ( Math.abs(diff) > 1e-20 )
+ featDiff.put(diffFeatId, diff);
+ else
+ featDiff.remove(diffFeatId);
+ }
+ else //features only firing in the new oracle feature vector
+ featDiff.put(diffFeatId, Double.parseDouble(featInfo[1]));
+ }
+ for (int i = 0; i < vecPredFeat.length; i++) {
+ featInfo = vecPredFeat[i].split("=");
+ diffFeatId = Integer.parseInt(featInfo[0]);
+ if (featDiff.containsKey(diffFeatId)) { //overlapping features
+ diff = featDiff.get(diffFeatId)-Double.parseDouble(featInfo[1]);
+ if ( Math.abs(diff) > 1e-20 )
+ featDiff.put(diffFeatId, diff);
+ else
+ featDiff.remove(diffFeatId);
+ }
+ else //features only firing in the new prediction feature vector
+ featDiff.put(diffFeatId, -1.0*Double.parseDouble(featInfo[1]));
+ }
+ }
+
+ //remember the model scores here are already scaled
+ double singleLoss = evalMetric.getToBeMinimized() ?
+ (predMetric-oraMetric) - (oraScore-predScore)/featScale:
+ (oraMetric-predMetric) - (oraScore-predScore)/featScale;
+ if(singleLoss > 0)
+ loss += singleLoss;
+ ++sentCount;
+ if( sentCount >= sentNum ) {
+ thisBatchSize = b + 1;
+ break;
+ }
+ } //for(int b : batchSize)
+
+ //System.out.println("\n\n"+sentCount+":");
+
+ if( loss > 0 ) {
+ //if(true) {
+ ++numUpdate;
+ //update weights (see Duchi'11, Eq.23. For l1-reg, use lazy update)
+ Set<Integer> diffFeatSet = featDiff.keySet();
+ it = diffFeatSet.iterator();
+ while(it.hasNext()) { //note these are all non-zero gradients!
+ diffFeatId = (Integer)it.next();
+ diffFeatVal = -1.0 * featDiff.get(diffFeatId); //gradient
+ if( regularization > 0 ) {
+ lastUpdateTime =
+ lastUpdate.get(diffFeatId) == null ? 0 : lastUpdate.get(diffFeatId);
+ if( lastUpdateTime < numUpdate - 1 ) {
+ //haven't been updated (gradient=0) for at least 2 steps
+ //lazy compute prevLambda now
+ oldVal =
+ lastVal.get(diffFeatId) == null ? initialLambda[diffFeatId] : lastVal.get(diffFeatId);
+ Hii =
+ H.get(diffFeatId) == null ? 0 : H.get(diffFeatId);
+ if(Math.abs(Hii) > 1e-20) {
+ if( regularization == 1 )
+ prevLambda =
+ Math.signum(oldVal) * clip( Math.abs(oldVal) - lam * eta * (numBatch - 1 - lastUpdateTime) / Hii );
+ else if( regularization == 2 ) {
+ prevLambda =
+ Math.pow( Hii/(lam+Hii), (numUpdate - 1 - lastUpdateTime) ) * oldVal;
+ if(needAvg) { //fill the gap due to lazy update
+ double prevLambdaCopy = prevLambda;
+ double scale = Hii/(lam+Hii);
+ for( int t = 0; t < numUpdate - 1 - lastUpdateTime; ++t ) {
+ avgLambda[diffFeatId] += prevLambdaCopy;
+ prevLambdaCopy /= scale;
+ }
+ }
+ }
+ } else {
+ if( regularization == 1 )
+ prevLambda = 0;
+ else if( regularization == 2 )
+ prevLambda = oldVal;
+ }
+ } else //just updated at last time step or just started
+ prevLambda = finalLambda[diffFeatId];
+ if(H.get(diffFeatId) != null) {
+ gradiiSquare = H.get(diffFeatId);
+ gradiiSquare *= gradiiSquare;
+ gradiiSquare += diffFeatVal * diffFeatVal;
+ Hii = Math.sqrt(gradiiSquare);
+ } else
+ Hii = Math.abs(diffFeatVal);
+ H.put(diffFeatId, Hii);
+ //update the weight
+ if( regularization == 1 ) {
+ gdStep = prevLambda - eta * diffFeatVal / Hii;
+ finalLambda[diffFeatId] = Math.signum(gdStep) * clip( Math.abs(gdStep) - lam * eta / Hii );
+ } else if(regularization == 2 ) {
+ finalLambda[diffFeatId] = (Hii * prevLambda - eta * diffFeatVal) / (lam + Hii);
+ if(needAvg)
+ avgLambda[diffFeatId] += finalLambda[diffFeatId];
+ }
+ lastUpdate.put(diffFeatId, numUpdate);
+ lastVal.put(diffFeatId, finalLambda[diffFeatId]);
+ } else { //if no regularization
+ if(H.get(diffFeatId) != null) {
+ gradiiSquare = H.get(diffFeatId);
+ gradiiSquare *= gradiiSquare;
+ gradiiSquare += diffFeatVal * diffFeatVal;
+ Hii = Math.sqrt(gradiiSquare);
+ } else
+ Hii = Math.abs(diffFeatVal);
+ H.put(diffFeatId, Hii);
+ finalLambda[diffFeatId] = finalLambda[diffFeatId] - eta * diffFeatVal / Hii;
+ if(needAvg)
+ avgLambda[diffFeatId] += finalLambda[diffFeatId];
+ }
+ } //while(it.hasNext())
+ } //if(loss > 0)
+ else { //no loss, therefore the weight update is skipped
+ //however, the avg weights still need to be accumulated
+ if( regularization == 0 ) {
+ for( int i = 1; i < finalLambda.length; ++i )
+ avgLambda[i] += finalLambda[i];
+ } else if( regularization == 2 ) {
+ if(needAvg) {
+ //due to lazy update, we need to figure out the actual
+ //weight vector at this point first...
+ for( int i = 1; i < finalLambda.length; ++i ) {
+ if( lastUpdate.get(i) != null ) {
+ if( lastUpdate.get(i) < numUpdate ) {
+ oldVal = lastVal.get(i);
+ Hii = H.get(i);
+ //lazy compute
+ avgLambda[i] +=
+ Math.pow( Hii/(lam+Hii), (numUpdate - lastUpdate.get(i)) ) * oldVal;
+ } else
+ avgLambda[i] += finalLambda[i];
+ }
+ avgLambda[i] += finalLambda[i];
+ }
+ }
+ }
+ }
+ } //while( sentCount < sentNum )
+ if( regularization > 0 ) {
+ for( int i = 1; i < finalLambda.length; ++i ) {
+ //now lazy compute those weights that haven't been taken care of
+ if( lastUpdate.get(i) == null )
+ finalLambda[i] = 0;
+ else if( lastUpdate.get(i) < numUpdate ) {
+ oldVal = lastVal.get(i);
+ Hii = H.get(i);
+ if( regularization == 1 )
+ finalLambda[i] =
+ Math.signum(oldVal) * clip( Math.abs(oldVal) - lam * eta * (numUpdate - lastUpdate.get(i)) / Hii );
+ else if( regularization == 2 ) {
+ finalLambda[i] =
+ Math.pow( Hii/(lam+Hii), (numUpdate - lastUpdate.get(i)) ) * oldVal;
+ if(needAvg) { //fill the gap due to lazy update
+ double prevLambdaCopy = finalLambda[i];
+ double scale = Hii/(lam+Hii);
+ for( int t = 0; t < numUpdate - lastUpdate.get(i); ++t ) {
+ avgLambda[i] += prevLambdaCopy;
+ prevLambdaCopy /= scale;
+ }
+ }
+ }
+ }
+ if( regularization == 2 && needAvg ) {
+ if( iter == adagradIter - 1 )
+ finalLambda[i] = avgLambda[i] / ( numBatch * adagradIter );
+ }
+ }
+ } else { //if no regularization
+ if( iter == adagradIter - 1 && needAvg ) {
+ for( int i = 1; i < finalLambda.length; ++i )
+ finalLambda[i] = avgLambda[i] / ( numBatch * adagradIter );
+ }
+ }
+
+ double initMetricScore;
+ if (iter == 0) {
+ initMetricScore = computeCorpusMetricScore(initialLambda);
+ finalMetricScore = computeCorpusMetricScore(finalLambda);
+ } else {
+ initMetricScore = finalMetricScore;
+ finalMetricScore = computeCorpusMetricScore(finalLambda);
+ }
+ // prepare the printing info
+ String result = " Initial "
+ + evalMetric.get_metricName() + "=" + String.format("%.4f", initMetricScore) + " Final "
+ + evalMetric.get_metricName() + "=" + String.format("%.4f", finalMetricScore);
+ //print lambda info
+ // int numParamToPrint = 0;
+ // numParamToPrint = paramDim > 10 ? 10 : paramDim; // how many parameters
+ // // to print
+ // result = paramDim > 10 ? "Final lambda (first 10): {" : "Final lambda: {";
+
+ // for (int i = 1; i <= numParamToPrint; ++i)
+ // result += String.format("%.4f", finalLambda[i]) + " ";
+
+ output.add(result);
+ } //for ( int iter = 0; iter < adagradIter; ++iter ) {
+
+ //non-optimizable weights should remain unchanged
+ ArrayList<Double> cpFixWt = new ArrayList<Double>();
+ for ( int i = 1; i < isOptimizable.length; ++i ) {
+ if ( ! isOptimizable[i] )
+ cpFixWt.add(finalLambda[i]);
+ }
+ normalizeLambda(finalLambda);
+ int countNonOpt = 0;
+ for ( int i = 1; i < isOptimizable.length; ++i ) {
+ if ( ! isOptimizable[i] ) {
+ finalLambda[i] = cpFixWt.get(countNonOpt);
+ ++countNonOpt;
+ }
+ }
+ return finalLambda;
+ }
+
+ private double clip(double x) {
+ return x > 0 ? x : 0;
+ }
+
+ public double computeCorpusMetricScore(double[] finalLambda) {
+ int suffStatsCount = evalMetric.get_suffStatsCount();
+ double modelScore;
+ double maxModelScore;
+ Set<String> candSet;
+ String candStr;
+ String[] feat_str;
+ String[] tmpStatsVal = new String[suffStatsCount];
+ int[] corpusStatsVal = new int[suffStatsCount];
+ for (int i = 0; i < suffStatsCount; i++)
+ corpusStatsVal[i] = 0;
+
+ for (int i = 0; i < sentNum; i++) {
+ candSet = feat_hash[i].keySet();
+
+ // find out the 1-best candidate for each sentence
+ // this depends on the training mode
+ maxModelScore = NegInf;
+ for (Iterator it = candSet.iterator(); it.hasNext();) {
+ modelScore = 0.0;
+ candStr = it.next().toString();
+
+ feat_str = feat_hash[i].get(candStr).split("\\s+");
+
+ String[] feat_info;
+
+ for (int f = 0; f < feat_str.length; f++) {
+ feat_info = feat_str[f].split("=");
+ modelScore +=
+ Double.parseDouble(feat_info[1]) * finalLambda[Vocabulary.id(feat_info[0])];
+ }
+
+ if (maxModelScore < modelScore) {
+ maxModelScore = modelScore;
+ tmpStatsVal = stats_hash[i].get(candStr).split("\\s+"); // save the
+ // suff stats
+ }
+ }
+
+ for (int j = 0; j < suffStatsCount; j++)
+ corpusStatsVal[j] += Integer.parseInt(tmpStatsVal[j]); // accumulate
+ // corpus-leve
+ // suff stats
+ } // for( int i=0; i<sentNum; i++ )
+
+ return evalMetric.score(corpusStatsVal);
+ }
+
+ private void findOraPred(int sentId, double[] oraPredScore, String[] oraPredFeat, double[] lambda, double featScale)
+ {
+ double oraMetric=0, oraScore=0, predMetric=0, predScore=0;
+ String oraFeat="", predFeat="";
+ double candMetric = 0, candScore = 0; //metric and model scores for each cand
+ Set<String> candSet = stats_hash[sentId].keySet();
+ String cand = "";
+ String feats = "";
+ String oraCand = ""; //only used when BLEU/TER-BLEU is used as metric
+ String[] featStr;
+ String[] featInfo;
+
+ int actualFeatId;
+ double bestOraScore;
+ double worstPredScore;
+
+ if(oraSelectMode==1)
+ bestOraScore = NegInf; //larger score will be selected
+ else {
+ if(evalMetric.getToBeMinimized())
+ bestOraScore = PosInf; //smaller score will be selected
+ else
+ bestOraScore = NegInf;
+ }
+
+ if(predSelectMode==1 || predSelectMode==2)
+ worstPredScore = NegInf; //larger score will be selected
+ else {
+ if(evalMetric.getToBeMinimized())
+ worstPredScore = NegInf; //larger score will be selected
+ else
+ worstPredScore = PosInf;
+ }
+
+ for (Iterator it = candSet.iterator(); it.hasNext();) {
+ cand = it.next().toString();
+ candMetric = computeSentMetric(sentId, cand); //compute metric score
+
+ //start to compute model score
+ candScore = 0;
+ featStr = feat_hash[sentId].get(cand).split("\\s+");
+ feats = "";
+
+ for (int i = 0; i < featStr.length; i++) {
+ featInfo = featStr[i].split("=");
+ actualFeatId = Vocabulary.id(featInfo[0]);
+ candScore += Double.parseDouble(featInfo[1]) * lambda[actualFeatId];
+ if ( (actualFeatId < isOptimizable.length && isOptimizable[actualFeatId]) ||
+ actualFeatId >= isOptimizable.length )
+ feats += actualFeatId + "=" + Double.parseDouble(featInfo[1]) + " ";
+ }
+
+ candScore *= featScale; //scale the model score
+
+ //is this cand oracle?
+ if(oraSelectMode == 1) {//"hope", b=1, r=1
+ if(evalMetric.getToBeMinimized()) {//if the smaller the metric score, the better
+ if( bestOraScore<=(candScore-candMetric) ) {
+ bestOraScore = candScore-candMetric;
+ oraMetric = candMetric;
+ oraScore = candScore;
+ oraFeat = feats;
+ oraCand = cand;
+ }
+ }
+ else {
+ if( bestOraScore<=(candScore+candMetric) ) {
+ bestOraScore = candScore+candMetric;
+ oraMetric = candMetric;
+ oraScore = candScore;
+ oraFeat = feats;
+ oraCand = cand;
+ }
+ }
+ }
+ else {//best metric score(ex: max BLEU), b=1, r=0
+ if(evalMetric.getToBeMinimized()) {//if the smaller the metric score, the better
+ if( bestOraScore>=candMetric ) {
+ bestOraScore = candMetric;
+ oraMetric = candMetric;
+ oraScore = candScore;
+ oraFeat = feats;
+ oraCand = cand;
+ }
+ }
+ else {
+ if( bestOraScore<=candMetric ) {
+ bestOraScore = candMetric;
+ oraMetric = candMetric;
+ oraScore = candScore;
+ oraFeat = feats;
+ oraCand = cand;
+ }
+ }
+ }
+
+ //is this cand prediction?
+ if(predSelectMode == 1) {//"fear"
+ if(evalMetric.getToBeMinimized()) {//if the smaller the metric score, the better
+ if( worstPredScore<=(candScore+candMetric) ) {
+ worstPredScore = candScore+candMetric;
+ predMetric = candMetric;
+ predScore = candScore;
+ predFeat = feats;
+ }
+ }
+ else {
+ if( worstPredScore<=(candScore-candMetric) ) {
+ worstPredScore = candScore-candMetric;
+ predMetric = candMetric;
+ predScore = candScore;
+ predFeat = feats;
+ }
+ }
+ }
+ else if(predSelectMode == 2) {//model prediction(max model score)
+ if( worstPredScore<=candScore ) {
+ worstPredScore = candScore;
+ predMetric = candMetric;
+ predScore = candScore;
+ predFeat = feats;
+ }
+ }
+ else {//worst metric score(ex: min BLEU)
+ if(evalMetric.getToBeMinimized()) {//if the smaller the metric score, the better
+ if( worstPredScore<=candMetric ) {
+ worstPredScore = candMetric;
+ predMetric = candMetric;
+ predScore = candScore;
+ predFeat = feats;
+ }
+ }
+ else {
+ if( worstPredScore>=candMetric ) {
+ worstPredScore = candMetric;
+ predMetric = candMetric;
+ predScore = candScore;
+ predFeat = feats;
+ }
+ }
+ }
+ }
+
+ oraPredScore[0] = oraMetric;
+ oraPredScore[1] = oraScore;
+ oraPredScore[2] = predMetric;
+ oraPredScore[3] = predScore;
+ oraPredFeat[0] = oraFeat;
+ oraPredFeat[1] = predFeat;
+
+ //update the BLEU metric statistics if pseudo corpus is used to compute BLEU/TER-BLEU
+ if(evalMetric.get_metricName().equals("BLEU") && usePseudoBleu ) {
+ String statString;
+ String[] statVal_str;
+ statString = stats_hash[sentId].get(oraCand);
+ statVal_str = statString.split("\\s+");
+
+ for (int j = 0; j < evalMetric.get_suffStatsCount(); j++)
+ bleuHistory[sentId][j] = R*bleuHistory[sentId][j]+Integer.parseInt(statVal_str[j]);
+ }
+
+ if(evalMetric.get_metricName().equals("TER-BLEU") && usePseudoBleu ) {
+ String statString;
+ String[] statVal_str;
+ statString = stats_hash[sentId].get(oraCand);
+ statVal_str = statString.split("\\s+");
+
+ for (int j = 0; j < evalMetric.get_suffStatsCount()-2; j++)
+ bleuHistory[sentId][j] = R*bleuHistory[sentId][j]+Integer.parseInt(statVal_str[j+2]); //the first 2 stats are TER stats
+ }
+ }
+
+ // compute *sentence-level* metric score for cand
+ private double computeSentMetric(int sentId, String cand) {
+ String statString;
+ String[] statVal_str;
+ int[] statVal = new int[evalMetric.get_suffStatsCount()];
+
+ statString = stats_hash[sentId].get(cand);
+ statVal_str = statString.split("\\s+");
+
+ if(evalMetric.get_metricName().equals("BLEU") && usePseudoBleu) {
+ for (int j = 0; j < evalMetric.get_suffStatsCount(); j++)
+ statVal[j] = (int) (Integer.parseInt(statVal_str[j]) + bleuHistory[sentId][j]);
+ } else if(evalMetric.get_metricName().equals("TER-BLEU") && usePseudoBleu) {
+ for (int j = 0; j < evalMetric.get_suffStatsCount()-2; j++)
+ statVal[j+2] = (int)(Integer.parseInt(statVal_str[j+2]) + bleuHistory[sentId][j]); //only modify the BLEU stats part(TER has 2 stats)
+ } else { //in all other situations, use normal stats
+ for (int j = 0; j < evalMetric.get_suffStatsCount(); j++)
+ statVal[j] = Integer.parseInt(statVal_str[j]);
+ }
+
+ return evalMetric.score(statVal);
+ }
+
+ // from ZMERT
+ private void normalizeLambda(double[] origLambda) {
+ // private String[] normalizationOptions;
+ // How should a lambda[] vector be normalized (before decoding)?
+ // nO[0] = 0: no normalization
+ // nO[0] = 1: scale so that parameter nO[2] has absolute value nO[1]
+ // nO[0] = 2: scale so that the maximum absolute value is nO[1]
+ // nO[0] = 3: scale so that the minimum absolute value is nO[1]
+ // nO[0] = 4: scale so that the L-nO[1] norm equals nO[2]
+
+ int normalizationMethod = (int) normalizationOptions[0];
+ double scalingFactor = 1.0;
+ if (normalizationMethod == 0) {
+ scalingFactor = 1.0;
+ } else if (normalizationMethod == 1) {
+ int c = (int) normalizationOptions[2];
+ scalingFactor = normalizationOptions[1] / Math.abs(origLambda[c]);
+ } else if (normalizationMethod == 2) {
+ double maxAbsVal = -1;
+ int maxAbsVal_c = 0;
+ for (int c = 1; c <= paramDim; ++c) {
+ if (Math.abs(origLambda[c]) > maxAbsVal) {
+ maxAbsVal = Math.abs(origLambda[c]);
+ maxAbsVal_c = c;
+ }
+ }
+ scalingFactor = normalizationOptions[1] / Math.abs(origLambda[maxAbsVal_c]);
+
+ } else if (normalizationMethod == 3) {
+ double minAbsVal = PosInf;
+ int minAbsVal_c = 0;
+
+ for (int c = 1; c <= paramDim; ++c) {
+ if (Math.abs(origLambda[c]) < minAbsVal) {
+ minAbsVal = Math.abs(origLambda[c]);
+ minAbsVal_c = c;
+ }
+ }
+ scalingFactor = normalizationOptions[1] / Math.abs(origLambda[minAbsVal_c]);
+
+ } else if (normalizationMethod == 4) {
+ double pow = normalizationOptions[1];
+ double norm = L_norm(origLambda, pow);
+ scalingFactor = normalizationOptions[2] / norm;
+ }
+
+ for (int c = 1; c <= paramDim; ++c) {
+ origLambda[c] *= scalingFactor;
+ }
+ }
+
+ // from ZMERT
+ private double L_norm(double[] A, double pow) {
+ // calculates the L-pow norm of A[]
+ // NOTE: this calculation ignores A[0]
+ double sum = 0.0;
+ for (int i = 1; i < A.length; ++i)
+ sum += Math.pow(Math.abs(A[i]), pow);
+
+ return Math.pow(sum, 1 / pow);
+ }
+
+ public static double getScale()
+ {
+ return featScale;
+ }
+
+ public static void initBleuHistory(int sentNum, int statCount)
+ {
+ bleuHistory = new double[sentNum][statCount];
+ for(int i=0; i<sentNum; i++) {
+ for(int j=0; j<statCount; j++) {
+ bleuHistory[i][j] = 0.0;
+ }
+ }
+ }
+
+ public double getMetricScore()
+ {
+ return finalMetricScore;
+ }
+
+ private Vector<String> output;
+ private double[] initialLambda;
+ private double[] finalLambda;
+ private double finalMetricScore;
+ private HashMap<String, String>[] feat_hash;
+ private HashMap<String, String>[] stats_hash;
+ private int paramDim;
+ private boolean[] isOptimizable;
+ public static int sentNum;
+ public static int adagradIter; //AdaGrad internal iterations
+ public static int oraSelectMode;
+ public static int predSelectMode;
+ public static int batchSize;
+ public static int regularization;
+ public static boolean needShuffle;
+ public static boolean needScale;
+ public static double scoreRatio;
+ public static boolean needAvg;
+ public static boolean usePseudoBleu;
+ public static double featScale = 1.0; //scale the features in order to make the model score comparable with metric score
+ //updates in each epoch if necessary
+ public static double eta;
+ public static double lam;
+ public static double R; //corpus decay(used only when pseudo corpus is used to compute BLEU)
+ public static EvaluationMetric evalMetric;
+ public static double[] normalizationOptions;
+ public static double[][] bleuHistory;
+
+ private final static double NegInf = (-1.0 / 0.0);
+ private final static double PosInf = (+1.0 / 0.0);
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/corpus/AbstractPhrase.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/corpus/AbstractPhrase.java b/joshua-core/src/main/java/org/apache/joshua/corpus/AbstractPhrase.java
new file mode 100644
index 0000000..b4637d4
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/corpus/AbstractPhrase.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.corpus;
+
+/**
+ * This class provides a skeletal implementation of the base methods likely to be common to most or
+ * all implementations of the <code>Phrase</code> interface.
+ *
+ * @author Lane Schwartz
+ * @author Chris Callison-Burch
+ */
+public abstract class AbstractPhrase implements Phrase {
+
+ // ===============================================================
+ // Constants
+ // ===============================================================
+
+ /** seed used in hash code generation */
+ public static final int HASH_SEED = 17;
+
+ /** offset used in has code generation */
+ public static final int HASH_OFFSET = 37;
+
+ /**
+ * Splits a sentence (on white space), then looks up the integer representations of each word
+ * using the supplied symbol table.
+ *
+ * @param sentence White-space separated String of words.
+ *
+ * @return Array of integers corresponding to the words in the sentence.
+ */
+ protected int[] splitSentence(String sentence) {
+ String[] w = sentence.split("\\s+");
+ int[] words = new int[w.length];
+ for (int i = 0; i < w.length; i++)
+ words[i] = Vocabulary.id(w[i]);
+ return words;
+ }
+
+ /**
+ * Uses the standard java approach of calculating hashCode. Start with a seed, add in every value
+ * multiplying the exsiting hash times an offset.
+ *
+ * @return int hashCode for the list
+ */
+ public int hashCode() {
+ int result = HASH_SEED;
+ for (int i = 0; i < size(); i++) {
+ result = HASH_OFFSET * result + getWordID(i);
+ }
+ return result;
+ }
+
+
+ /**
+ * Two phrases are their word IDs are the same. Note that this could give a false positive if
+ * their Vocabularies were different but their IDs were somehow the same.
+ */
+ public boolean equals(Object o) {
+
+ if (o instanceof Phrase) {
+ Phrase other = (Phrase) o;
+
+ if (this.size() != other.size()) return false;
+ for (int i = 0; i < size(); i++) {
+ if (this.getWordID(i) != other.getWordID(i)) return false;
+ }
+ return true;
+ } else {
+ return false;
+ }
+
+ }
+
+
+ /**
+ * Compares the two strings based on the lexicographic order of words defined in the Vocabulary.
+ *
+ * @param other the object to compare to
+ * @return -1 if this object is less than the parameter, 0 if equals, 1 if greater
+ * @exception ClassCastException if the passed object is not of type Phrase
+ */
+ public int compareTo(Phrase other) {
+ int length = size();
+ int otherLength = other.size();
+ for (int i = 0; i < length; i++) {
+ if (i < otherLength) {
+ int difference = getWordID(i) - other.getWordID(i);
+ if (difference != 0) return difference;
+ } else {
+ // same but other is shorter, so we are after
+ return 1;
+ }
+ }
+ if (length < otherLength) {
+ return -1;
+ } else {
+ return 0;
+ }
+ }
+
+ /**
+ * Returns a string representation of the phrase.
+ *
+ * @return a space-delimited string of the words in the phrase.
+ */
+ public String toString() {
+ StringBuffer buf = new StringBuffer();
+ for (int i = 0; i < size(); i++) {
+ String word = Vocabulary.word(getWordID(i));
+ if (i != 0) buf.append(' ');
+ buf.append(word);
+ }
+ return buf.toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/corpus/BasicPhrase.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/corpus/BasicPhrase.java b/joshua-core/src/main/java/org/apache/joshua/corpus/BasicPhrase.java
new file mode 100644
index 0000000..6c50458
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/corpus/BasicPhrase.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.corpus;
+
+import java.util.ArrayList;
+
+/**
+ * The simplest concrete implementation of Phrase.
+ *
+ * @author wren ng thornton wren@users.sourceforge.net
+ * @version $LastChangedDate$
+ */
+public class BasicPhrase extends AbstractPhrase {
+ private byte language;
+ private int[] words;
+
+
+ public BasicPhrase(byte language, String sentence) {
+ this.language = language;
+ this.words = splitSentence(sentence);
+ }
+
+ private BasicPhrase() {}
+
+ public int[] getWordIDs() {
+ return words;
+ }
+
+ /* See Javadoc for Phrase interface. */
+ public BasicPhrase subPhrase(int start, int end) {
+ BasicPhrase that = new BasicPhrase();
+ that.language = this.language;
+ that.words = new int[end - start + 1];
+ System.arraycopy(this.words, start, that.words, 0, end - start + 1);
+ return that;
+ }
+
+ /* See Javadoc for Phrase interface. */
+ public ArrayList<Phrase> getSubPhrases() {
+ return this.getSubPhrases(this.size());
+ }
+
+ /* See Javadoc for Phrase interface. */
+ public ArrayList<Phrase> getSubPhrases(int maxLength) {
+ ArrayList<Phrase> phrases = new ArrayList<Phrase>();
+ int len = this.size();
+ for (int n = 1; n <= maxLength; n++)
+ for (int i = 0; i <= len - n; i++)
+ phrases.add(this.subPhrase(i, i + n - 1));
+ return phrases;
+ }
+
+ /* See Javadoc for Phrase interface. */
+ public int size() {
+ return (words == null ? 0 : words.length);
+ }
+
+ /* See Javadoc for Phrase interface. */
+ public int getWordID(int position) {
+ return words[position];
+ }
+
+ /**
+ * Returns a human-readable String representation of the phrase.
+ * <p>
+ * The implementation of this method is slightly more efficient than that inherited from
+ * <code>AbstractPhrase</code>.
+ *
+ * @return a human-readable String representation of the phrase.
+ */
+ public String toString() {
+ StringBuffer sb = new StringBuffer();
+ if (words != null) {
+ for (int i = 0; i < words.length; ++i) {
+ if (i != 0) sb.append(' ');
+ sb.append(Vocabulary.word(words[i]));
+ }
+ }
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/corpus/ContiguousPhrase.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/corpus/ContiguousPhrase.java b/joshua-core/src/main/java/org/apache/joshua/corpus/ContiguousPhrase.java
new file mode 100644
index 0000000..af669b7
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/corpus/ContiguousPhrase.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.corpus;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * ContiguousPhrase implements the Phrase interface by linking into indices within a corpus. This is
+ * intended to be a very low-memory implementation of the class.
+ *
+ * @author Chris Callison-Burch
+ * @since 29 May 2008
+ * @version $LastChangedDate:2008-09-18 12:47:23 -0500 (Thu, 18 Sep 2008) $
+ */
+public class ContiguousPhrase extends AbstractPhrase {
+
+ protected int startIndex;
+ protected int endIndex;
+ protected Corpus corpusArray;
+
+ public ContiguousPhrase(int startIndex, int endIndex, Corpus corpusArray) {
+ this.startIndex = startIndex;
+ this.endIndex = endIndex;
+ this.corpusArray = corpusArray;
+ }
+
+ /**
+ * This method copies the phrase into an array of ints. This method should be avoided if possible.
+ *
+ * @return an int[] corresponding to the ID of each word in the phrase
+ */
+ public int[] getWordIDs() {
+ int[] words = new int[endIndex - startIndex];
+ for (int i = startIndex; i < endIndex; i++) {
+ words[i - startIndex] = corpusArray.getWordID(i); // corpusArray.corpus[i];
+ }
+ return words;
+ }
+
+ public int getWordID(int position) {
+ return corpusArray.getWordID(startIndex + position);
+ // return corpusArray.corpus[startIndex+position];
+ }
+
+ public int size() {
+ return endIndex - startIndex;
+ }
+
+ /**
+ * Gets all possible subphrases of this phrase, up to and including the phrase itself. For
+ * example, the phrase "I like cheese ." would return the following:
+ * <ul>
+ * <li>I
+ * <li>like
+ * <li>cheese
+ * <li>.
+ * <li>I like
+ * <li>like cheese
+ * <li>cheese .
+ * <li>I like cheese
+ * <li>like cheese .
+ * <li>I like cheese .
+ * </ul>
+ *
+ * @return ArrayList of all possible subphrases.
+ */
+ public List<Phrase> getSubPhrases() {
+ return getSubPhrases(size());
+ }
+
+ /**
+ * Returns a list of subphrases only of length <code>maxLength</code> or smaller.
+ *
+ * @param maxLength the maximum length phrase to return.
+ * @return ArrayList of all possible subphrases of length maxLength or less
+ * @see #getSubPhrases()
+ */
+ public List<Phrase> getSubPhrases(int maxLength) {
+ if (maxLength > size()) return getSubPhrases(size());
+ List<Phrase> phrases = new ArrayList<Phrase>();
+ for (int i = 0; i < size(); i++) {
+ for (int j = i + 1; (j <= size()) && (j - i <= maxLength); j++) {
+ Phrase subPhrase = subPhrase(i, j);
+ phrases.add(subPhrase);
+ }
+ }
+ return phrases;
+ }
+
+ /**
+ * creates a new phrase object from the indexes provided.
+ * <P>
+ * NOTE: subList merely creates a "view" of the existing Phrase object. Memory taken up by other
+ * Words in the Phrase is not freed since the underlying subList object still points to the
+ * complete Phrase List.
+ *
+ * @see ArrayList#subList(int, int)
+ */
+ public Phrase subPhrase(int start, int end) {
+ return new ContiguousPhrase(startIndex + start, startIndex + end, corpusArray);
+ }
+
+ /**
+ * Main contains test code
+ * @param args String array of arguments used to run this class.
+ */
+ public static void main(String[] args) {
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/corpus/Corpus.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/corpus/Corpus.java b/joshua-core/src/main/java/org/apache/joshua/corpus/Corpus.java
new file mode 100755
index 0000000..1a7d1b0
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/corpus/Corpus.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.corpus;
+
+/**
+ * Corpus is an interface that contains methods for accessing the information within a monolingual
+ * corpus.
+ *
+ * @author Chris Callison-Burch
+ * @since 7 February 2005
+ * @version $LastChangedDate:2008-07-30 17:15:52 -0400 (Wed, 30 Jul 2008) $
+ */
+
+public interface Corpus { // extends Externalizable {
+
+ // ===============================================================
+ // Attribute definitions
+ // ===============================================================
+
+ /**
+ * @param position the position at which we want to obtain a word ID
+ * @return the integer representation of the Word at the specified position in the corpus.
+ */
+ int getWordID(int position);
+
+
+ /**
+ * Gets the sentence index associated with the specified position in the corpus.
+ *
+ * @param position Index into the corpus
+ * @return the sentence index associated with the specified position in the corpus.
+ */
+ int getSentenceIndex(int position);
+
+
+ /**
+ * Gets the sentence index of each specified position.
+ *
+ * @param positions Index into the corpus
+ * @return array of the sentence indices associated with the specified positions in the corpus.
+ */
+ int[] getSentenceIndices(int[] positions);
+
+ /**
+ * Gets the position in the corpus of the first word of the specified sentence. If the sentenceID
+ * is outside of the bounds of the sentences, then it returns the last position in the corpus + 1.
+ *
+ * @param sentenceID a specific sentence to obtain a position for
+ * @return the position in the corpus of the first word of the specified sentence. If the
+ * sentenceID is outside of the bounds of the sentences, then it returns the last position
+ * in the corpus + 1.
+ */
+ int getSentencePosition(int sentenceID);
+
+ /**
+ * Gets the exclusive end position of a sentence in the corpus.
+ *
+ * @param sentenceID a specific sentence to obtain an end position for
+ * @return the position in the corpus one past the last word of the specified sentence. If the
+ * sentenceID is outside of the bounds of the sentences, then it returns one past the last
+ * position in the corpus.
+ */
+ int getSentenceEndPosition(int sentenceID);
+
+ /**
+ * Gets the specified sentence as a phrase.
+ *
+ * @param sentenceIndex Zero-based sentence index
+ * @return the sentence, or null if the specified sentence number doesn't exist
+ */
+ Phrase getSentence(int sentenceIndex);
+
+
+ /**
+ * Gets the number of words in the corpus.
+ *
+ * @return the number of words in the corpus.
+ */
+ int size();
+
+
+ /**
+ * Gets the number of sentences in the corpus.
+ *
+ * @return the number of sentences in the corpus.
+ */
+ int getNumSentences();
+
+
+ // ===========================================================
+ // Methods
+ // ===========================================================
+
+
+ /**
+ * Compares the phrase that starts at position start with the subphrase indicated by the start and
+ * end points of the phrase.
+ *
+ * @param corpusStart the point in the corpus where the comparison begins
+ * @param phrase the superphrase that the comparsion phrase is drawn from
+ * @param phraseStart the point in the phrase where the comparison begins (inclusive)
+ * @param phraseEnd the point in the phrase where the comparison ends (exclusive)
+ * @return an int that follows the conventions of {@link java.util.Comparator#compare(Object, Object)}
+ */
+ int comparePhrase(int corpusStart, Phrase phrase, int phraseStart, int phraseEnd);
+
+
+ /**
+ * Compares the phrase that starts at position start with the phrase passed in. Compares the
+ * entire phrase.
+ *
+ * @param corpusStart position start
+ * @param phrase {@link org.apache.joshua.corpus.Phrase} to compare against
+ * @return an int that follows the conventions of {@link java.util.Comparator#compare(Object, Object)}
+ */
+ int comparePhrase(int corpusStart, Phrase phrase);
+
+ /**
+ * Compares the suffixes starting a positions index1 and index2.
+ *
+ * @param position1 the position in the corpus where the first suffix begins
+ * @param position2 the position in the corpus where the second suffix begins
+ * @param maxComparisonLength a cutoff point to stop the comparison
+ * @return an int that follows the conventions of {@link java.util.Comparator#compare(Object, Object)}
+ */
+ int compareSuffixes(int position1, int position2, int maxComparisonLength);
+
+ /**
+ *
+ * @param startPosition start position for phrase
+ * @param endPosition end position for phrase
+ * @return the {@link org.apache.joshua.corpus.ContiguousPhrase}
+ */
+ ContiguousPhrase getPhrase(int startPosition, int endPosition);
+
+ /**
+ * Gets an object capable of iterating over all positions in the corpus, in order.
+ *
+ * @return An object capable of iterating over all positions in the corpus, in order.
+ */
+ Iterable<Integer> corpusPositions();
+
+ // void write(String corpusFilename, String vocabFilename, String charset) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/corpus/Phrase.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/corpus/Phrase.java b/joshua-core/src/main/java/org/apache/joshua/corpus/Phrase.java
new file mode 100644
index 0000000..5a06a8b
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/corpus/Phrase.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.corpus;
+
+import java.util.ArrayList;
+import java.util.List;
+
+
+/**
+ * Representation of a sequence of tokens.
+ *
+ * @version $LastChangedDate:2008-09-18 10:31:54 -0500 (Thu, 18 Sep 2008) $
+ */
+public interface Phrase extends Comparable<Phrase> {
+
+ /**
+ * This method gets the integer IDs of the phrase as an array of ints.
+ *
+ * @return an int[] corresponding to the ID of each word in the phrase
+ */
+ public int[] getWordIDs();
+
+ /**
+ * Returns the integer word id of the word at the specified position.
+ *
+ * @param position Index of a word in this phrase.
+ * @return the integer word id of the word at the specified position.
+ */
+ int getWordID(int position);
+
+
+ /**
+ * Returns the number of words in this phrase.
+ *
+ * @return the number of words in this phrase.
+ */
+ int size();
+
+
+
+ /**
+ * Gets all possible subphrases of this phrase, up to and including the phrase itself. For
+ * example, the phrase "I like cheese ." would return the following:
+ * <ul>
+ * <li>I
+ * <li>like
+ * <li>cheese
+ * <li>.
+ * <li>I like
+ * <li>like cheese
+ * <li>cheese .
+ * <li>I like cheese
+ * <li>like cheese .
+ * <li>I like cheese .
+ * </ul>
+ *
+ * @return List of all possible subphrases.
+ */
+ List<Phrase> getSubPhrases();
+
+
+ /**
+ * Returns a list of subphrases only of length <code>maxLength</code> or smaller.
+ *
+ * @param maxLength the maximum length phrase to return.
+ * @return List of all possible subphrases of length maxLength or less
+ * @see #getSubPhrases()
+ */
+ List<Phrase> getSubPhrases(int maxLength);
+
+
+ /**
+ * creates a new phrase object from the indexes provided.
+ * <P>
+ * NOTE: subList merely creates a "view" of the existing Phrase object. Memory taken up by other
+ * Words in the Phrase is not freed since the underlying subList object still points to the
+ * complete Phrase List.
+ *
+ * @see ArrayList#subList(int, int)
+ * @param start start position to begin new phrase
+ * @param end end position to end new phrase
+ * @return a new {@link org.apache.joshua.corpus.Phrase} object from the indexes provided.
+ */
+ Phrase subPhrase(int start, int end);
+
+
+ /**
+ * Compares the two strings based on the lexicographic order of words defined in the Vocabulary.
+ *
+ * @param other the object to compare to
+ * @return -1 if this object is less than the parameter, 0 if equals, 1 if greater
+ */
+ int compareTo(Phrase other);
+
+ /**
+ * Returns a human-readable String representation of the phrase.
+ *
+ * @return a human-readable String representation of the phrase.
+ */
+ String toString();
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/corpus/Span.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/corpus/Span.java b/joshua-core/src/main/java/org/apache/joshua/corpus/Span.java
new file mode 100644
index 0000000..414fe95
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/corpus/Span.java
@@ -0,0 +1,175 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.corpus;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.NoSuchElementException;
+
+/**
+ * Represents a span with an inclusive starting index and an exclusive ending index.
+ *
+ * @author Lane Schwartz
+ */
+public class Span implements Iterable<Integer>, Comparable<Span> {
+
+ /** Inclusive starting index of this span. */
+ public int start;
+
+ /** Exclusive ending index of this span. */
+ public int end;
+
+
+ /**
+ * Constructs a new span with the given inclusive starting and exclusive ending indices.
+ *
+ * @param start Inclusive starting index of this span.
+ * @param end Exclusive ending index of this span.
+ */
+ public Span(int start, int end) {
+ this.start = start;
+ this.end = end;
+ }
+
+
+ /**
+ * Returns the length of the span.
+ *
+ * @return the length of the span; this is equivalent to <code>span.end - span.start</code>.
+ */
+ public int size() {
+ return end - start;
+ }
+
+ /**
+ * Returns all subspans of the given Span.
+ *
+ * @return a list of all subspans.
+ */
+ public List<Span> getSubSpans() {
+ return getSubSpans(size());
+ }
+
+ /**
+ * Returns all subspans of the given Span, up to a specified Span size.
+ *
+ * @param max the maximum Span size to return
+ * @return a list all subspans up to the given size
+ */
+ public List<Span> getSubSpans(int max) {
+ int spanSize = size();
+ ArrayList<Span> result = new ArrayList<Span>(max * spanSize);
+ for (int len = max; len > 0; len--) {
+ for (int i = start; i < end - len + 1; i++) {
+ result.add(new Span(i, i + len));
+ }
+ }
+ return result;
+ }
+
+ public boolean strictlyContainedIn(Span o) {
+ return (start >= o.start) && (end <= o.end) && !(start == o.start && end == o.end);
+ }
+
+ /**
+ * Returns true if the other span does not intersect with this one.
+ * @param o new {@link org.apache.joshua.corpus.Span} to check for intersection
+ * @return true if the other span does not intersect with this one
+ */
+ public boolean disjointFrom(Span o) {
+ if (start < o.start) {
+ return end <= o.start;
+ }
+ if (end > o.end) {
+ return start >= o.end;
+ }
+ return false;
+ }
+
+ public String toString() {
+ return "[" + start + "-" + end + ")";
+ }
+
+
+ public Iterator<Integer> iterator() {
+ return new Iterator<Integer>() {
+
+ int next = start;
+
+ public boolean hasNext() {
+ return next < end;
+ }
+
+ public Integer next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException();
+ }
+ return next++;
+ }
+
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ };
+ }
+
+
+ public int compareTo(Span o) {
+
+ if (o == null) {
+ throw new NullPointerException();
+ } else {
+
+ if (start < o.start) {
+ return -1;
+ } else if (start > o.start) {
+ return 1;
+ } else {
+ if (end < o.end) {
+ return -1;
+ } else if (end > o.end) {
+ return 1;
+ } else {
+ return 0;
+ }
+ }
+ }
+
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ } else if (o instanceof Span) {
+ Span other = (Span) o;
+ return (start == other.start && end == other.end);
+
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ public int hashCode() {
+ return start * 31 + end * 773;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/corpus/SymbolTable.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/corpus/SymbolTable.java b/joshua-core/src/main/java/org/apache/joshua/corpus/SymbolTable.java
new file mode 100644
index 0000000..274e8b9
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/corpus/SymbolTable.java
@@ -0,0 +1,327 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.corpus;
+
+import java.util.Collection;
+
+/**
+ * Represents a symbol table capable of mapping between strings and
+ * symbols.
+ *
+ * @author Lane Schwartz
+ * @author Zhifei Li
+ * @version $LastChangedDate: 2009-11-24 23:07:43 -0600 (Tue, 24 Nov 2009) $
+ */
+public interface SymbolTable {
+
+ //TODO Remove all hard-coded references to nonterminals
+
+ /**
+ * The unknown word's ID will be the size of the vocabulary,
+ * ensuring that it is outside of the vocabulary. Note that
+ * for vocabularies which have not been fixed yet, this
+ * means the actual value is volatile and therefore a word
+ * ID can only be compared against UNKNOWN_WORD at the time
+ * the word ID is generated (otherwise unknown words can
+ * become "known" if new words are added to the vocabulary
+ * before testing).
+ * <p>
+ * Negative IDs are reserved for non-terminals.
+ *
+ * Zero is reserved as the UNKNOWN_WORD.
+ */
+ int UNKNOWN_WORD = 1;
+
+ /** String representation for out-of-vocabulary words. */
+ String UNKNOWN_WORD_STRING = "<unk>";
+
+ /**
+ * Integer representation of the bare (non-indexed) nonterminal X,
+ * which represents a wild-card gap in a phrase.
+ * <p>
+ * All nonterminals are guaranteed to be represented by negative integers.
+ */
+ int X = -1;
+
+ /**
+ * String representation of the bare (non-indexed) nonterminal X,
+ * which represents a wild-card gap in a phrase.
+ */
+ String X_STRING = "[X]";
+
+
+
+ /**
+ * String representation of the nonterminal X with index 1,
+ * which represents a wild-card gap in a phrase.
+ */
+ String X1_STRING = "[X,1]";
+
+
+
+ /**
+ * String representation of the nonterminal X with index 2,
+ * which represents a wild-card gap in a phrase.
+ */
+ String X2_STRING = "[X,2]";
+
+ /**
+ * Integer representation of the nonterminal S.
+ * <p>
+ * All nonterminals are guaranteed to be represented by negative integers.
+ */
+ int S = -4;
+
+ /**
+ * String representation of the nonterminal S..
+ */
+ String S_STRING = "[S]";
+
+ /**
+ * Integer representation of the nonterminal X with index 1,
+ * which represents a wild-card gap in a phrase.
+ * <p>
+ * All nonterminals are guaranteed to be represented by negative integers.
+ */
+ int S1 = -5;
+
+ /**
+ * String representation of the nonterminal X with index 2,
+ * which represents a wild-card gap in a phrase.
+ */
+ String S1_STRING = "[S,1]";
+
+ /**
+ * Gets a unique integer identifier for the nonterminal.
+ * <p>
+ * The integer returned is guaranteed to be a negative number.
+ *
+ * If the nonterminal is {@link #X_STRING},
+ * then the value returned must be {@link #X}.
+ *
+ * Otherwise, the value returned must be a negative number
+ * whose value is less than {@link X}.
+ *
+ * @param nonterminal Nonterminal symbol
+ * @return a unique integer identifier for the nonterminal
+ */
+ int addNonterminal(String nonterminal);
+
+ /**
+ * Gets a unique integer identifier for the terminal.
+ *
+ * @param terminal Terminal symbol
+ * @return a unique integer identifier for the terminal
+ */
+ int addTerminal(String terminal);
+
+ /**
+ * Gets the unique integer identifiers for the words.
+ *
+ * @param words Array of symbols
+ * @return the unique integer identifiers for the words
+ */
+ int[] addTerminals(String[] words);
+
+ /**
+ * Gets the unique integer identifiers for the words
+ * in the sentence.
+ *
+ * @param sentence Space-delimited string of symbols
+ * @return the unique integer identifiers for the words
+ * in the sentence
+ */
+ int[] addTerminals(String sentence);
+
+ /**
+ * Gets an integer identifier for the word.
+ * <p>
+ * If the word is in the vocabulary, the integer returned
+ * will uniquely identify that word.
+ * <p>
+ * If the word is not in the vocabulary, the integer returned
+ * by <code>getUnknownWordID</code> may be returned.
+ *
+ * Alternatively, implementations may, if they choose, add
+ * unknown words and assign them a symbol ID instead of
+ * returning <code>getUnknownWordID</code>.
+ *
+ * @see #getUnknownWordID
+ * @return the unique integer identifier for wordString,
+ * or the result of <code>getUnknownWordID</code>
+ * if wordString is not in the vocabulary
+ * @param wordString the word to retrieve the integer identifier
+ */
+ int getID(String wordString);
+
+ /**
+ * Gets the integer identifiers for all words in the provided
+ * sentence.
+ * <p>
+ * The sentence will be split (on spaces) into words, then
+ * the integer identifier for each word will be retrieved
+ * using <code>getID</code>.
+ *
+ * @see #getID(String)
+ * @param sentence String of words, separated by spaces.
+ * @return Array of integer identifiers for each word in
+ * the sentence
+ */
+ int[] getIDs(String sentence);
+
+ /**
+ * Gets the String that corresponds to the specified integer
+ * identifier.
+ * <p>
+ * If the identifier is in the symbol vocabulary, the String
+ * returned will correspond to that identifier.
+ *
+ * Otherwise, the String returned by <code>getUnknownWord</code>
+ * will be returned.
+ *
+ * @param wordID an integer identifier for a specific String
+ * @return the String that corresponds to the specified
+ * integer identifier, or the result of
+ * <code>getUnknownWord</code> if the identifier
+ * does not correspond to a word in the vocabulary
+ */
+ String getTerminal(int wordID);
+
+ /**
+ * Gets the String that corresponds to the specified integer
+ * identifier.
+ * <p>
+ * This method can be called for terminals or nonterminals.
+ *
+ * @param tokenID Integer identifier
+ * @return the String that corresponds to the specified
+ * integer identifier
+ */
+ String getWord(int tokenID);
+
+ /**
+ * Gets the String that corresponds to the sequence of
+ * specified integer identifiers.
+ *
+ * @param ids Sequence of integer identifiers
+ * @return the String that corresponds to the sequence of
+ * specified integer identifiers
+ */
+ String getWords(int[] ids);
+
+ /**
+ *
+ * @param wordIDs an int[] of identifiers for a specific Strings
+ * @return the String that corresponds to the specified
+ * integer identifiers
+ */
+ String getTerminals(int[] wordIDs);
+
+ /**
+ * Gets a collection over all symbol identifiers for the
+ * vocabulary.
+ *
+ * @return a collection over all symbol identifiers for the
+ * vocabulary
+ */
+ Collection<Integer> getAllIDs();
+
+ /**
+ * Gets the list of all words represented by this vocabulary.
+ *
+ * @return the list of all words represented by this
+ * vocabulary
+ */
+ Collection<String> getWords();
+
+ /**
+ * Gets the number of unique words in the vocabulary.
+ *
+ * @return the number of unique words in the vocabulary.
+ */
+ int size();
+
+ /**
+ * Gets the integer symbol representation of the unknown
+ * word.
+ *
+ * @return the integer symbol representation of the unknown
+ * word.
+ */
+ int getUnknownWordID();
+
+ /**
+ * Gets the string representation of the unknown word.
+ *
+ * @return the string representation of the unknown word.
+ */
+ String getUnknownWord();
+
+ /**
+ * Returns <code>true</code> if the symbol id represents a
+ * nonterminal, <code>false</code> otherwise.
+ *
+ * @param id int symbol id
+ * @return <code>true</code> if the symbol id represents a
+ * nonterminal, <code>false</code> otherwise.
+ */
+ boolean isNonterminal(int id);
+
+ /**
+ * Gets the lowest-valued allowable terminal symbol id in
+ * this table.
+ *
+ * @return the lowest-valued allowable terminal symbol id
+ * in this table.
+ */
+ int getLowestID();
+
+
+ /**
+ * Gets the highest-valued allowable terminal symbol id in
+ * this table.
+ * <p>
+ * NOTE: This may or may not return the same value as
+ * <code>size</code>.
+ *
+ * @return the highest-valued allowable terminal symbol id
+ * in this table.
+ */
+ int getHighestID();
+
+ /**
+ * @param id todo
+ * @return todo
+ */
+ int getTargetNonterminalIndex(int id);//first convert id to its String mapping, then call the function below
+
+ /**
+ * @param word todo
+ * @return todo
+ */
+ int getTargetNonterminalIndex(String word);
+
+ /**
+ * @param wordIDs todo
+ * @param ntIndexIncrements todo
+ * @return todo
+ */
+ String getWords(int[] wordIDs, boolean ntIndexIncrements);
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/corpus/TerminalIterator.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/corpus/TerminalIterator.java b/joshua-core/src/main/java/org/apache/joshua/corpus/TerminalIterator.java
new file mode 100644
index 0000000..fcf5c72
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/corpus/TerminalIterator.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.corpus;
+
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+import org.apache.joshua.util.FormatUtils;
+
+/**
+ * Iterator capable of iterating over those word identifiers in a phrase which represent terminals.
+ * <p>
+ * <em>Note</em>: This class is <em>not</em> thread-safe.
+ *
+ * @author Lane Schwartz
+ */
+public class TerminalIterator implements Iterator<Integer> {
+
+ private final int[] words;
+
+ private int nextIndex = -1;
+ private int next = Integer.MIN_VALUE;
+ private boolean dirty = true;
+
+ /**
+ * Constructs an iterator for the terminals in the given list of words.
+ *
+ * @param words array of words
+ */
+ public TerminalIterator(int[] words) {
+ this.words = words;
+ }
+
+ /* See Javadoc for java.util.Iterator#next(). */
+ public boolean hasNext() {
+
+ while (dirty || FormatUtils.isNonterminal(next)) {
+ nextIndex++;
+ if (nextIndex < words.length) {
+ next = words[nextIndex];
+ dirty = false;
+ } else {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ /* See Javadoc for java.util.Iterator#next(). */
+ public Integer next() {
+ if (hasNext()) {
+ dirty = true;
+ return next;
+ } else {
+ throw new NoSuchElementException();
+ }
+ }
+
+ /**
+ * Unsupported operation, guaranteed to throw an UnsupportedOperationException.
+ *
+ * @throws UnsupportedOperationException operation not supported yet!
+ */
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/corpus/Vocabulary.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/corpus/Vocabulary.java b/joshua-core/src/main/java/org/apache/joshua/corpus/Vocabulary.java
new file mode 100644
index 0000000..24644ee
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/corpus/Vocabulary.java
@@ -0,0 +1,301 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.corpus;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.Externalizable;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.locks.StampedLock;
+
+import org.apache.joshua.decoder.ff.lm.NGramLanguageModel;
+import org.apache.joshua.util.FormatUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Static singular vocabulary class.
+ * Supports (de-)serialization into a vocabulary file.
+ *
+ * @author Juri Ganitkevitch
+ */
+
+public class Vocabulary implements Externalizable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Vocabulary.class);
+ private final static ArrayList<NGramLanguageModel> LMs = new ArrayList<>();
+
+ private static List<String> idToString;
+ private static Map<String, Integer> stringToId;
+ private static final StampedLock lock = new StampedLock();
+
+ static final int UNKNOWN_ID = 0;
+ static final String UNKNOWN_WORD = "<unk>";
+
+ public static final String START_SYM = "<s>";
+ public static final String STOP_SYM = "</s>";
+
+ static {
+ clear();
+ }
+
+ public static boolean registerLanguageModel(NGramLanguageModel lm) {
+ long lock_stamp = lock.writeLock();
+ try {
+ // Store the language model.
+ LMs.add(lm);
+ // Notify it of all the existing words.
+ boolean collision = false;
+ for (int i = idToString.size() - 1; i > 0; i--)
+ collision = collision || lm.registerWord(idToString.get(i), i);
+ return collision;
+ } finally {
+ lock.unlockWrite(lock_stamp);
+ }
+ }
+
+ /**
+ * Reads a vocabulary from file. This deletes any additions to the vocabulary made prior to
+ * reading the file.
+ *
+ * @param vocab_file path to a vocabulary file
+ * @return Returns true if vocabulary was read without mismatches or collisions.
+ * @throws IOException of the file cannot be found or read properly
+ */
+ public static boolean read(final File vocab_file) throws IOException {
+ DataInputStream vocab_stream =
+ new DataInputStream(new BufferedInputStream(new FileInputStream(vocab_file)));
+ int size = vocab_stream.readInt();
+ LOG.info("Read {} entries from the vocabulary", size);
+ clear();
+ for (int i = 0; i < size; i++) {
+ int id = vocab_stream.readInt();
+ String token = vocab_stream.readUTF();
+ if (id != Math.abs(id(token))) {
+ vocab_stream.close();
+ return false;
+ }
+ }
+ vocab_stream.close();
+ return (size + 1 == idToString.size());
+ }
+
+ public static void write(String file_name) throws IOException {
+ long lock_stamp =lock.readLock();
+ try {
+ File vocab_file = new File(file_name);
+ DataOutputStream vocab_stream =
+ new DataOutputStream(new BufferedOutputStream(new FileOutputStream(vocab_file)));
+ vocab_stream.writeInt(idToString.size() - 1);
+ LOG.info("Writing vocabulary: {} tokens", idToString.size() - 1);
+ for (int i = 1; i < idToString.size(); i++) {
+ vocab_stream.writeInt(i);
+ vocab_stream.writeUTF(idToString.get(i));
+ }
+ vocab_stream.close();
+ }
+ finally{
+ lock.unlockRead(lock_stamp);
+ }
+ }
+
+ /**
+ * Get the id of the token if it already exists, new id is created otherwise.
+ *
+ * TODO: currently locks for every call. Separate constant (frozen) ids from
+ * changing (e.g. OOV) ids. Constant ids could be immutable -> no locking.
+ * Alternatively: could we use ConcurrentHashMap to not have to lock if
+ * actually contains it and only lock for modifications?
+ *
+ * @param token a token to obtain an id for
+ * @return the token id
+ */
+ public static int id(String token) {
+ // First attempt an optimistic read
+ long attempt_read_lock = lock.tryOptimisticRead();
+ if (stringToId.containsKey(token)) {
+ int resultId = stringToId.get(token);
+ if (lock.validate(attempt_read_lock)) {
+ return resultId;
+ }
+ }
+
+ // The optimistic read failed, try a read with a stamped read lock
+ long read_lock_stamp = lock.readLock();
+ try {
+ if (stringToId.containsKey(token)) {
+ return stringToId.get(token);
+ }
+ } finally {
+ lock.unlockRead(read_lock_stamp);
+ }
+
+ // Looks like the id we want is not there, let's get a write lock and add it
+ long write_lock_stamp = lock.writeLock();
+ try {
+ if (stringToId.containsKey(token)) {
+ return stringToId.get(token);
+ }
+ int id = idToString.size() * (FormatUtils.isNonterminal(token) ? -1 : 1);
+
+ // register this (token,id) mapping with each language
+ // model, so that they can map it to their own private
+ // vocabularies
+ for (NGramLanguageModel lm : LMs)
+ lm.registerWord(token, Math.abs(id));
+
+ idToString.add(token);
+ stringToId.put(token, id);
+ return id;
+ } finally {
+ lock.unlockWrite(write_lock_stamp);
+ }
+ }
+
+ public static boolean hasId(int id) {
+ long lock_stamp = lock.readLock();
+ try {
+ id = Math.abs(id);
+ return (id < idToString.size());
+ }
+ finally{
+ lock.unlockRead(lock_stamp);
+ }
+ }
+
+ public static int[] addAll(String sentence) {
+ return addAll(sentence.split("\\s+"));
+ }
+
+ public static int[] addAll(String[] tokens) {
+ int[] ids = new int[tokens.length];
+ for (int i = 0; i < tokens.length; i++)
+ ids[i] = id(tokens[i]);
+ return ids;
+ }
+
+ public static String word(int id) {
+ long lock_stamp = lock.readLock();
+ try {
+ id = Math.abs(id);
+ return idToString.get(id);
+ }
+ finally{
+ lock.unlockRead(lock_stamp);
+ }
+ }
+
+ public static String getWords(int[] ids) {
+ return getWords(ids, " ");
+ }
+
+ public static String getWords(int[] ids, final String separator) {
+ if (ids.length == 0) {
+ return "";
+ }
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < ids.length - 1; i++) {
+ sb.append(word(ids[i])).append(separator);
+ }
+ return sb.append(word(ids[ids.length - 1])).toString();
+ }
+
+ public static String getWords(final Iterable<Integer> ids) {
+ StringBuilder sb = new StringBuilder();
+ for (int id : ids)
+ sb.append(word(id)).append(" ");
+ return sb.deleteCharAt(sb.length() - 1).toString();
+ }
+
+ public static int getUnknownId() {
+ return UNKNOWN_ID;
+ }
+
+ public static String getUnknownWord() {
+ return UNKNOWN_WORD;
+ }
+
+ public static int size() {
+ long lock_stamp = lock.readLock();
+ try {
+ return idToString.size();
+ } finally {
+ lock.unlockRead(lock_stamp);
+ }
+ }
+
+ public static synchronized int getTargetNonterminalIndex(int id) {
+ return FormatUtils.getNonterminalIndex(word(id));
+ }
+
+ /**
+ * Clears the vocabulary and initializes it with an unknown word. Registered
+ * language models are left unchanged.
+ */
+ public static void clear() {
+ long lock_stamp = lock.writeLock();
+ try {
+ idToString = new ArrayList<String>();
+ stringToId = new HashMap<String, Integer>();
+
+ idToString.add(UNKNOWN_ID, UNKNOWN_WORD);
+ stringToId.put(UNKNOWN_WORD, UNKNOWN_ID);
+ } finally {
+ lock.unlockWrite(lock_stamp);
+ }
+ }
+
+ public static void unregisterLanguageModels() {
+ LMs.clear();
+ }
+
+ @Override
+ public void writeExternal(ObjectOutput out) throws IOException {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public void readExternal(ObjectInput in)
+ throws IOException, ClassNotFoundException {
+ // TODO Auto-generated method stub
+
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if(getClass() == o.getClass()) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+}
[33/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Stack.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Stack.java b/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Stack.java
new file mode 100644
index 0000000..d0ae2da
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Stack.java
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.phrase;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.PriorityQueue;
+import java.util.Set;
+
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.chart_parser.ComputeNodeResult;
+import org.apache.joshua.decoder.ff.FeatureFunction;
+import org.apache.joshua.decoder.segment_file.Sentence;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Organizes all hypotheses containing the same number of source words.
+ *
+ */
+public class Stack extends ArrayList<Hypothesis> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Stack.class);
+
+ private static final long serialVersionUID = 7885252799032416068L;
+
+ private HashMap<Coverage, ArrayList<Hypothesis>> coverages;
+
+ private Sentence sentence;
+ private List<FeatureFunction> featureFunctions;
+ private JoshuaConfiguration config;
+
+ /* The list of states we've already visited. */
+ private HashSet<Candidate> visitedStates;
+
+ /* A list of candidates sorted for consideration for entry to the chart (for cube pruning) */
+ private PriorityQueue<Candidate> candidates;
+
+ /* Short-circuits adding a cube-prune state more than once */
+ private HashMap<Hypothesis, Hypothesis> deduper;
+
+ /**
+ * Create a new stack. Stacks are organized one for each number of source words that are covered.
+ *
+ * @param featureFunctions {@link java.util.List} of {@link org.apache.joshua.decoder.ff.FeatureFunction}'s
+ * @param sentence input for a {@link org.apache.joshua.lattice.Lattice}
+ * @param config populated {@link org.apache.joshua.decoder.JoshuaConfiguration}
+ */
+ public Stack(List<FeatureFunction> featureFunctions, Sentence sentence, JoshuaConfiguration config) {
+ this.featureFunctions = featureFunctions;
+ this.sentence = sentence;
+ this.config = config;
+
+ this.candidates = new PriorityQueue<Candidate>(1, new CandidateComparator());
+ this.coverages = new HashMap<Coverage, ArrayList<Hypothesis>>();
+ this.visitedStates = new HashSet<Candidate>();
+ this.deduper = new HashMap<Hypothesis,Hypothesis>();
+ }
+
+ /**
+ * A Stack is an ArrayList; here, we intercept the add so we can maintain a list of the items
+ * stored under each distinct coverage vector
+ * @param hyp a {@link org.apache.joshua.decoder.phrase.Hypothesis} to add to the {@link org.apache.joshua.decoder.phrase.Stack}
+ * @return true if the {@link org.apache.joshua.decoder.phrase.Hypothesis} is appended to the list
+ */
+ @Override
+ public boolean add(Hypothesis hyp) {
+
+ if (! coverages.containsKey((hyp.getCoverage())))
+ coverages.put(hyp.getCoverage(), new ArrayList<Hypothesis>());
+ coverages.get(hyp.getCoverage()).add(hyp);
+
+ return super.add(hyp);
+ }
+
+ /**
+ * Intercept calls to remove() so that we can reduce the coverage vector
+ */
+ @Override
+ public boolean remove(Object obj) {
+ boolean found = super.remove(obj);
+ if (found) {
+ Hypothesis item = (Hypothesis) obj;
+ Coverage cov = item.getCoverage();
+ assert coverages.get(cov).remove(obj);
+ if (coverages.get(cov).size() == 0)
+ coverages.remove(cov);
+ }
+ return found;
+ }
+
+ /**
+ * Returns the set of coverages contained in this stack. This is used to iterate over them
+ * in the main decoding loop in Stacks.java.
+ * @return a {@link java.util.Set} of {@link org.apache.joshua.decoder.phrase.Coverage}'s
+ */
+ public Set<Coverage> getCoverages() {
+ return coverages.keySet();
+ }
+
+ /**
+ * Get all items with the same coverage vector.
+ *
+ * @param cov the {@link org.apache.joshua.decoder.phrase.Coverage} vector to get
+ * @return an {@link java.util.ArrayList} of {@link org.apache.joshua.decoder.phrase.Hypothesis}'
+ */
+ public ArrayList<Hypothesis> get(Coverage cov) {
+ ArrayList<Hypothesis> list = coverages.get(cov);
+ Collections.sort(list);
+ return list;
+ }
+
+ /**
+ * Receives a partially-initialized translation candidate and places it on the
+ * priority queue after scoring it with all of the feature functions. In this
+ * respect it is like {@link org.apache.joshua.decoder.chart_parser.CubePruneState} (it could make use of that class with
+ * a little generalization of spans / coverage).
+ *
+ * This function is also used to (fairly concisely) implement constrained decoding. Before
+ * adding a candidate, we ensure that the sequence of English words match the sentence. If not,
+ * the code extends the dot in the cube-pruning chart to the next phrase, since that one might
+ * be a match.
+ * @param cand a partially-initialized translation {@link org.apache.joshua.decoder.phrase.Candidate}
+ */
+ public void addCandidate(Candidate cand) {
+ if (visitedStates.contains(cand))
+ return;
+
+ visitedStates.add(cand);
+
+ // Constrained decoding
+ if (sentence.target() != null) {
+ String oldWords = cand.getHypothesis().bestHyperedge.getRule().getEnglishWords().replace("[X,1] ", "");
+ String newWords = cand.getRule().getEnglishWords().replace("[X,1] ", "");
+
+ // If the string is not found in the target sentence, explore the cube neighbors
+ if (sentence.fullTarget().indexOf(oldWords + " " + newWords) == -1) {
+ Candidate next = cand.extendPhrase();
+ if (next != null)
+ addCandidate(next);
+ return;
+ }
+ }
+
+ // TODO: sourcepath
+ ComputeNodeResult result = new ComputeNodeResult(this.featureFunctions, cand.getRule(),
+ cand.getTailNodes(), -1, cand.getSpan().end, null, this.sentence);
+ cand.setResult(result);
+
+ candidates.add(cand);
+ }
+
+ /**
+ * Cube pruning. Repeatedly pop the top candidate, creating a new hyperedge from it, adding it to
+ * the k-best list, and then extending the list of candidates with extensions of the current
+ * candidate.
+ */
+ public void search() {
+ int to_pop = config.pop_limit;
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Stack::search(): pop: {} size: {}", to_pop, candidates.size());
+ for (Candidate c: candidates)
+ LOG.debug("{}", c);
+ }
+ while (to_pop > 0 && !candidates.isEmpty()) {
+ Candidate got = candidates.poll();
+ if (got != null) {
+ addHypothesis(got);
+ --to_pop;
+
+ for (Candidate c : got.extend())
+ if (c != null) {
+ addCandidate(c);
+ }
+ }
+ }
+ }
+
+ /**
+ * Adds a popped candidate to the chart / main stack. This is a candidate we have decided to
+ * keep around.
+ * @param complete a completely-initialized translation {@link org.apache.joshua.decoder.phrase.Candidate}
+ *
+ */
+ public void addHypothesis(Candidate complete) {
+ Hypothesis added = new Hypothesis(complete);
+
+ String taskName;
+ if (deduper.containsKey(added)) {
+ taskName = "recombining hypothesis";
+ Hypothesis existing = deduper.get(added);
+ existing.absorb(added);
+ } else {
+ taskName = "creating new hypothesis";
+ add(added);
+ deduper.put(added, added);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("{} from ( ... {} )", taskName, complete.getHypothesis().getRule().getEnglishWords());
+ LOG.debug(" base score {}", complete.getResult().getBaseCost());
+ LOG.debug(" covering {}-{}", complete.getSpan().start - 1, complete.getSpan().end - 2);
+ LOG.debug(" translated as: {}", complete.getRule().getEnglishWords());
+ LOG.debug(" score {} + future cost {} = {}",
+ complete.getResult().getTransitionCost(), complete.getFutureEstimate(),
+ complete.getResult().getTransitionCost() + complete.getFutureEstimate());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Stacks.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Stacks.java b/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Stacks.java
new file mode 100644
index 0000000..c688b2c
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Stacks.java
@@ -0,0 +1,269 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.phrase;
+
+/***
+ * Entry point for phrase-based decoding, analogous to {@link Chart} for the CKY algorithm. This
+ * class organizes all the stacks used for decoding, and is responsible for building them. Stack
+ * construction is stack-centric: that is, we loop over the number of source words in increasing sizes;
+ * at each step of this iteration, we break the search between smaller stack sizes and source-side
+ * phrase sizes.
+ *
+ * The end result of decoding is a {@link Hypergraph} with the same format as hierarchical decoding.
+ * Phrases are treating as left-branching rules, and the span information (i,j) is overloaded so
+ * that i means nothing and j represents the index of the last-translated source word in each
+ * hypothesis. This means that most hypergraph code can work without modification. The algorithm
+ * ensures that the coverage vector is consistent but the resulting hypergraph may not be projective,
+ * which is different from the CKY algorithm, which does produce projective derivations.
+ *
+ * TODO Lattice decoding is not yet supported (March 2015).
+ */
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.joshua.corpus.Span;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.chart_parser.ComputeNodeResult;
+import org.apache.joshua.decoder.ff.FeatureFunction;
+import org.apache.joshua.decoder.ff.tm.AbstractGrammar;
+import org.apache.joshua.decoder.ff.tm.Grammar;
+import org.apache.joshua.decoder.hypergraph.HGNode;
+import org.apache.joshua.decoder.hypergraph.HyperEdge;
+import org.apache.joshua.decoder.hypergraph.HyperGraph;
+import org.apache.joshua.decoder.segment_file.Sentence;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class Stacks {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Stacks.class);
+
+ // The list of stacks, grouped according to number of source words covered
+ private List<Stack> stacks;
+
+ // The end state
+ private Hypothesis end;
+
+ List<FeatureFunction> featureFunctions;
+
+ private Sentence sentence;
+
+ private JoshuaConfiguration config;
+
+ /* Contains all the phrase tables */
+ private PhraseChart chart;
+
+ /**
+ * Entry point. Initialize everything. Create pass-through (OOV) phrase table and glue phrase
+ * table (with start-of-sentence and end-of-sentence rules).
+ *
+ * @param sentence input to {@link org.apache.joshua.lattice.Lattice}
+ * @param featureFunctions {@link java.util.List} of {@link org.apache.joshua.decoder.ff.FeatureFunction}'s
+ * @param grammars an array of {@link org.apache.joshua.decoder.ff.tm.Grammar}'s
+ * @param config a populated {@link org.apache.joshua.decoder.JoshuaConfiguration}
+ */
+ public Stacks(Sentence sentence, List<FeatureFunction> featureFunctions, Grammar[] grammars,
+ JoshuaConfiguration config) {
+
+ this.sentence = sentence;
+ this.featureFunctions = featureFunctions;
+ this.config = config;
+
+ int num_phrase_tables = 0;
+ for (int i = 0; i < grammars.length; i++)
+ if (grammars[i] instanceof PhraseTable)
+ ++num_phrase_tables;
+
+ PhraseTable[] phraseTables = new PhraseTable[num_phrase_tables + 2];
+ for (int i = 0, j = 0; i < grammars.length; i++)
+ if (grammars[i] instanceof PhraseTable)
+ phraseTables[j++] = (PhraseTable) grammars[i];
+
+ phraseTables[phraseTables.length - 2] = new PhraseTable("null", config);
+ phraseTables[phraseTables.length - 2].addRule(Hypothesis.END_RULE);
+
+ phraseTables[phraseTables.length - 1] = new PhraseTable("oov", config);
+ AbstractGrammar.addOOVRules(phraseTables[phraseTables.length - 1], sentence.getLattice(), featureFunctions, config.true_oovs_only);
+
+ this.chart = new PhraseChart(phraseTables, featureFunctions, sentence, config.num_translation_options);
+ }
+
+
+ /**
+ * The main algorithm. Returns a hypergraph representing the search space.
+ *
+ * @return a {@link org.apache.joshua.decoder.hypergraph.HyperGraph} representing the search space
+ */
+ public HyperGraph search() {
+
+ long startTime = System.currentTimeMillis();
+
+ Future future = new Future(chart);
+ stacks = new ArrayList<Stack>();
+
+ // <s> counts as the first word. Pushing null lets us count from one.
+ stacks.add(null);
+
+ // Initialize root hypothesis with <s> context and future cost for everything.
+ ComputeNodeResult result = new ComputeNodeResult(this.featureFunctions, Hypothesis.BEGIN_RULE,
+ null, -1, 1, null, this.sentence);
+ Stack firstStack = new Stack(featureFunctions, sentence, config);
+ firstStack.add(new Hypothesis(result.getDPStates(), future.Full()));
+ stacks.add(firstStack);
+
+ // Decode with increasing numbers of source words.
+ for (int source_words = 2; source_words <= sentence.length(); ++source_words) {
+ Stack targetStack = new Stack(featureFunctions, sentence, config);
+ stacks.add(targetStack);
+
+ // Iterate over stacks to continue from.
+ for (int phrase_length = 1; phrase_length <= Math.min(source_words - 1, chart.MaxSourcePhraseLength());
+ phrase_length++) {
+ int from_stack = source_words - phrase_length;
+ Stack tailStack = stacks.get(from_stack);
+
+ LOG.debug("WORDS {} MAX {} (STACK {} phrase_length {})", source_words,
+ chart.MaxSourcePhraseLength(), from_stack, phrase_length);
+
+ // Iterate over antecedents in this stack.
+ for (Coverage coverage: tailStack.getCoverages()) {
+ ArrayList<Hypothesis> hypotheses = tailStack.get(coverage);
+
+ // the index of the starting point of the first possible phrase
+ int begin = coverage.firstZero();
+
+ // the absolute position of the ending spot of the last possible phrase
+ int last_end = Math.min(coverage.firstZero() + config.reordering_limit, chart.SentenceLength());
+ int last_begin = (last_end > phrase_length) ? (last_end - phrase_length) : 0;
+
+ for (begin = coverage.firstZero(); begin <= last_begin; begin++) {
+ if (!coverage.compatible(begin, begin + phrase_length) ||
+ ! permissible(coverage, begin, begin + phrase_length)) {
+ continue;
+ }
+
+ Span span = new Span(begin, begin + phrase_length);
+
+ // Don't append </s> until the end
+ if (begin == sentence.length() - 1 && source_words != sentence.length())
+ continue;
+
+ TargetPhrases phrases = chart.getRange(begin, begin + phrase_length);
+ if (phrases == null)
+ continue;
+
+
+ LOG.debug("Applying {} target phrases over [{}, {}]",
+ phrases.size(), begin, begin + phrase_length);
+
+ // TODO: could also compute some number of features here (e.g., non-LM ones)
+ // float score_delta = context.GetScorer().transition(ant, phrases, begin, begin + phrase_length);
+
+ // Future costs: remove span to be filled.
+ float future_delta = future.Change(coverage, begin, begin + phrase_length);
+
+ /* This associates with each span a set of hypotheses that can be extended by
+ * phrases from that span. The hypotheses are wrapped in HypoState objects, which
+ * augment the hypothesis score with a future cost.
+ */
+ Candidate cand = new Candidate(hypotheses, phrases, span, future_delta);
+ targetStack.addCandidate(cand);
+ }
+ }
+ }
+
+ /* At this point, every vertex contains a list of all existing hypotheses that the target
+ * phrases in that vertex could extend. Now we need to create the search object, which
+ * implements cube pruning. There are up to O(n^2) cubes, n the size of the current stack,
+ * one cube each over each span of the input. Each "cube" has two dimensions: one representing
+ * the target phrases over the span, and one representing all of these incoming hypotheses.
+ * We seed the chart with the best item in each cube, and then repeatedly pop and extend.
+ */
+
+// System.err.println(String.format("\nBuilding cube-pruning chart for %d words", source_words));
+
+ targetStack.search();
+ }
+
+ LOG.info("Input {}: Search took {} seconds", sentence.id(),
+ (System.currentTimeMillis() - startTime) / 1000.0f);
+
+ return createGoalNode();
+ }
+
+ /**
+ * Enforces reordering constraints. Our version of Moses' ReorderingConstraint::Check() and
+ * SearchCubePruning::CheckDistortion().
+ *
+ * @param coverage
+ * @param begin
+ * @param i
+ * @return
+ */
+ private boolean permissible(Coverage coverage, int begin, int end) {
+ int firstZero = coverage.firstZero();
+
+ if (config.reordering_limit < 0)
+ return true;
+
+ /* We can always start with the first zero since it doesn't create a reordering gap
+ */
+ if (begin == firstZero)
+ return true;
+
+ /* If a gap is created by applying this phrase, make sure that you can reach the first
+ * zero later on without violating the distortion constraint.
+ */
+ if (end - firstZero > config.reordering_limit) {
+ return false;
+ }
+
+ return true;
+ }
+
+
+ /**
+ * Searches through the goal stack, calling the final transition function on each node, and then returning
+ * the best item. Usually the final transition code doesn't add anything, because all features
+ * have already computed everything they need to. The standard exception is language models that
+ * have not yet computed their prefix probabilities (which is not the case with KenLM, the default).
+ *
+ * @return
+ */
+ private HyperGraph createGoalNode() {
+ Stack lastStack = stacks.get(sentence.length());
+
+ for (Hypothesis hyp: lastStack) {
+ float score = hyp.getScore();
+ List<HGNode> tailNodes = new ArrayList<HGNode>();
+ tailNodes.add(hyp);
+
+ float finalTransitionScore = ComputeNodeResult.computeFinalCost(featureFunctions, tailNodes, 0, sentence.length(), null, sentence);
+
+ if (null == this.end)
+ this.end = new Hypothesis(null, score + finalTransitionScore, hyp, sentence.length(), null);
+
+ HyperEdge edge = new HyperEdge(null, score + finalTransitionScore, finalTransitionScore, tailNodes, null);
+ end.addHyperedgeInNode(edge);
+ }
+
+ return new HyperGraph(end, -1, -1, this.sentence);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/TargetPhrases.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/TargetPhrases.java b/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/TargetPhrases.java
new file mode 100644
index 0000000..05a4b0a
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/TargetPhrases.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.phrase;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.joshua.decoder.ff.FeatureFunction;
+import org.apache.joshua.decoder.ff.FeatureVector;
+import org.apache.joshua.decoder.ff.tm.Rule;
+
+/**
+ * Represents a sorted collection of target-side phrases. Typically, these are phrases
+ * generated from the same source word sequence. The list of options is reduced to the number
+ * of translation options.
+ *
+ * @author Matt Post
+ */
+
+public class TargetPhrases extends ArrayList<Rule> {
+
+ private static final long serialVersionUID = 1L;
+
+ public TargetPhrases() {
+ super();
+ }
+
+ /**
+ * Initialize with a collection of rules.
+ *
+ * @param list a {@link java.util.List} of {@link org.apache.joshua.decoder.ff.tm.Rule}'s
+ */
+ public TargetPhrases(List<Rule> list) {
+ super();
+
+ for (Rule rule: list) {
+ add(rule);
+ }
+ }
+
+ /**
+ * Score the rules and sort them. Scoring is necessary because rules are only scored if they
+ * are used, in an effort to make reading in rules more efficient. This is starting to create
+ * some trouble and should probably be reworked.
+ * @param features a {@link java.util.List} of {@link org.apache.joshua.decoder.ff.FeatureFunction}'s
+ * @param weights a populated {@link org.apache.joshua.decoder.ff.FeatureVector}
+ * @param num_options the number of options
+ */
+ public void finish(List<FeatureFunction> features, FeatureVector weights, int num_options) {
+ for (Rule rule: this) {
+ rule.estimateRuleCost(features);
+// System.err.println("TargetPhrases:finish(): " + rule);
+ }
+ Collections.sort(this, Rule.EstimatedCostComparator);
+
+ if (this.size() > num_options)
+ this.removeRange(num_options, this.size());
+
+// System.err.println("TargetPhrases::finish()");
+// for (Rule rule: this)
+// System.err.println(" " + rule);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/segment_file/ConstraintRule.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/segment_file/ConstraintRule.java b/joshua-core/src/main/java/org/apache/joshua/decoder/segment_file/ConstraintRule.java
new file mode 100644
index 0000000..5146e2c
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/segment_file/ConstraintRule.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.segment_file;
+
+import javax.swing.text.Segment;
+
+/**
+ * This interface is for an individual (partial) item to seed the chart with. All rules should be
+ * flat (no hierarchical nonterminals).
+ * <p>
+ * The {@link Segment}, {@link ConstraintSpan}, and {@link ConstraintRule} interfaces are for
+ * defining an interchange format between a SegmentFileParser and the Chart class. These interfaces
+ * <b>should not</b> be used internally by the Chart. The objects returned by a
+ * SegmentFileParser will not be optimal for use during decoding. The Chart should convert each of
+ * these objects into its own internal representation during construction. That is the contract
+ * described by these interfaces.
+ *
+ * @see org.apache.joshua.decoder.segment_file.ConstraintRule.Type
+ *
+ * @author wren ng thornton wren@users.sourceforge.net
+ * @version $LastChangedDate: 2009-03-26 15:06:57 -0400 (Thu, 26 Mar 2009) $
+ */
+public interface ConstraintRule {
+
+ /**
+ * <p>There are three types of ConstraintRule. The RULE type returns non-null values for all methods.
+ * The LHS type provides a (non-null) value for the lhs method, but returns null for everything
+ * else. And the RHS type provides a (non-null) value for nativeRhs and foreignRhs but returns
+ * null for the lhs and features.</p>
+ * <p>
+ * The interpretation of a RULE is that it adds a new rule to the grammar which only applies to
+ * the associated span. If the associated span is hard, then the set of rules for that span will
+ * override the regular grammar.</p>
+ * <p>
+ * The intepretation of a LHS is that it provides a hard constraint that the associated span be
+ * treated as the nonterminal for that span, thus filtering the regular grammar.</p>
+ * <p>
+ * The interpretation of a RHS is that it provides a hard constraint to filter the regular grammar
+ * such that only rules generating the desired translation can be used.</p>
+ */
+ public enum Type {
+ RULE, LHS, RHS
+ };
+
+ /**
+ * Return the type of this ConstraintRule.
+ * @return the {@link org.apache.joshua.decoder.segment_file.ConstraintRule.Type}
+ */
+ Type type();
+
+
+ /**
+ * Return the left hand side of the constraint rule. If this is null, then this object is
+ * specifying a translation for the span, but that translation may be derived from any
+ * nonterminal. The nonterminal here must be one used by the regular grammar.
+ * @return the left hand side of the constraint rule
+ */
+ String lhs();
+
+
+ /**
+ * Return the native right hand side of the constraint rule. If this is null, then the regular
+ * grammar will be used to fill in the derivation from the lhs.
+ * @return the native right hand side of the constraint rule
+ */
+ String nativeRhs();
+
+
+ /**
+ * Return the foreign right hand side of the constraint rule. This must be consistent with the
+ * sentence for the associated span, and is provided as a convenience method.
+ * @return the foreign right hand side of the constraint rule
+ */
+ String foreignRhs();
+
+
+ /**
+ * Return the grammar feature values for the RULE. The length of this array must be the same as
+ * for the regular grammar. We cannot enforce this requirement, but the
+ * {@link org.apache.joshua.decoder.chart_parser.Chart} must throw an error if there is a mismatch.
+ * @return an array of floating feature values for the RULE
+ */
+ float[] features();
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/segment_file/ConstraintSpan.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/segment_file/ConstraintSpan.java b/joshua-core/src/main/java/org/apache/joshua/decoder/segment_file/ConstraintSpan.java
new file mode 100644
index 0000000..9863fa6
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/segment_file/ConstraintSpan.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.segment_file;
+
+import java.util.List;
+
+import javax.swing.text.Segment;
+
+/**
+ * This interface represents a collection of constraints for a given span in the associated segment.
+ * Intuitively, each constraint corresponds to one or more items in the chart for parsing, except
+ * that we pre-seed the chart with these items before beginning the parsing algorithm. Some
+ * constraints can be "hard", in which case the regular grammar is not consulted for these spans. It
+ * is an error to have hard constraints for overlapping spans.
+ * <p>
+ * Indices for the span boundaries mark the transitions between words. Thus, the 0 index occurs
+ * before the first word, the 1 index occurs between the first and second words, 2 is between the
+ * second and third, etc. Consequently, it is an error for the end index to be equal to or less than
+ * the start index. It is also an error to have negative indices or to have indices larger than the
+ * count of words in the segment. Clients may assume that no <code>ConstraintSpan</code> objects are
+ * constructed which violate these laws.
+ * <p>
+ * The {@link Segment}, {@link ConstraintSpan}, and {@link ConstraintRule} interfaces are for
+ * defining an interchange format between a SegmentFileParser and the Chart class. These interfaces
+ * <b>should not</b> be used internally by the Chart. The objects returned by a
+ * SegmentFileParser will not be optimal for use during decoding. The Chart should convert each of
+ * these objects into its own internal representation during construction. That is the contract
+ * described by these interfaces.
+ *
+ * @author wren ng thornton wren@users.sourceforge.net
+ */
+public interface ConstraintSpan {
+
+ /**
+ * Return the starting index of the span covered by this constraint.
+ * @return the starting index of the span covered by this constraint
+ */
+ int start();
+
+ /**
+ * Return the ending index of the span covered by this constraint. Clients may assume
+ * <code>this.end() >= 1 + this.start()</code>.
+ * @return the ending index of the span covered by this constraint
+ */
+ int end();
+
+ /**
+ * Return whether this is a hard constraint which should override the grammar. This value only
+ * really matters for sets of <code>RULE</code> type constraints.
+ * @return true if a hard constraint exists which should override the grammar
+ */
+ boolean isHard();
+
+ /**
+ * Return a collection of the "rules" for this constraint span.
+ * <p>
+ * This return type is suboptimal for some SegmentFileParsers. It should be an
+ * {@link java.util.Iterator} instead in order to reduce the coupling between this class and
+ * Chart. See the note above about the fact that this interface should not be used internally by
+ * the Chart class because it will not be performant.
+ * @return a {@link java.util.List} of {@link org.apache.joshua.decoder.segment_file.ConstraintRule}'s
+ */
+ List<ConstraintRule> rules();
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/segment_file/ParseTreeInput.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/segment_file/ParseTreeInput.java b/joshua-core/src/main/java/org/apache/joshua/decoder/segment_file/ParseTreeInput.java
new file mode 100644
index 0000000..b9b1896
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/segment_file/ParseTreeInput.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.segment_file;
+
+import org.apache.joshua.decoder.JoshuaConfiguration;
+
+public class ParseTreeInput extends Sentence {
+
+ public ParseTreeInput(String input, int id, JoshuaConfiguration joshuaConfiguration) {
+ super(input, id,joshuaConfiguration);
+ }
+
+ // looks_like_parse_tree = sentence.sentence().matches("^\\(+[A-Z]+ .*");
+
+ // private SyntaxTree syntax_tree;
+
+ // ParseTreeInput() {
+ // SyntaxTree syntax_tree = new ArraySyntaxTree(sentence.sentence(), Vocabulary);
+ // }
+
+ // public int[] int_sentence() {
+ // return syntax_tree.getTerminals();
+ // }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/segment_file/ParsedSentence.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/segment_file/ParsedSentence.java b/joshua-core/src/main/java/org/apache/joshua/decoder/segment_file/ParsedSentence.java
new file mode 100644
index 0000000..a97718e
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/segment_file/ParsedSentence.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.segment_file;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.corpus.syntax.ArraySyntaxTree;
+import org.apache.joshua.corpus.syntax.SyntaxTree;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+
+public class ParsedSentence extends Sentence {
+
+ private SyntaxTree syntaxTree = null;
+
+ public ParsedSentence(String input, int id,JoshuaConfiguration joshuaConfiguration) {
+ super(input, id, joshuaConfiguration);
+ }
+
+ public int[] getWordIDs() {
+ int[] terminals = syntaxTree().getTerminals();
+ int[] annotated = new int[terminals.length + 2];
+ System.arraycopy(terminals, 0, annotated, 1, terminals.length);
+ annotated[0] = Vocabulary.id(Vocabulary.START_SYM);
+ annotated[annotated.length - 1] = Vocabulary.id(Vocabulary.STOP_SYM);
+ return annotated;
+ }
+
+ public SyntaxTree syntaxTree() {
+ if (syntaxTree == null)
+ syntaxTree = new ArraySyntaxTree(this.source());
+ return syntaxTree;
+ }
+
+ public static boolean matches(String input) {
+ return input.matches("^\\(+[A-Z]+ .*");
+ }
+
+ public String fullSource() {
+ return Vocabulary.getWords(this.getWordIDs());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/segment_file/Sentence.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/segment_file/Sentence.java b/joshua-core/src/main/java/org/apache/joshua/decoder/segment_file/Sentence.java
new file mode 100644
index 0000000..e323ef6
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/segment_file/Sentence.java
@@ -0,0 +1,450 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.segment_file;
+
+import static org.apache.joshua.util.FormatUtils.addSentenceMarkers;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.StringTokenizer;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.ff.tm.Grammar;
+import org.apache.joshua.lattice.Arc;
+import org.apache.joshua.lattice.Lattice;
+import org.apache.joshua.lattice.Node;
+import org.apache.joshua.util.ChartSpan;
+import org.apache.joshua.util.Regex;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class represents lattice input. The lattice is contained on a single line and is represented
+ * in PLF (Python Lattice Format), e.g.,
+ *
+ * <pre>
+ * ((('ein',0.1,1),('dieses',0.2,1),('haus',0.4,2),),(('haus',0.8,1),),)
+ * </pre>
+ *
+ * @author Matt Post post@cs.jhu.edu
+ */
+
+public class Sentence {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Sentence.class);
+
+ /* The sentence number. */
+ public int id = -1;
+
+ /*
+ * The source and target sides of the input sentence. Target sides are present when doing
+ * alignment or forced decoding.
+ */
+ protected String source = null;
+ protected String fullSource = null;
+
+ protected String target = null;
+ protected String fullTarget = null;
+ protected String[] references = null;
+
+ /* Lattice representation of the source sentence. */
+ protected Lattice<Token> sourceLattice = null;
+
+ /* List of constraints */
+ private final List<ConstraintSpan> constraints;
+
+ public JoshuaConfiguration config = null;
+
+ /**
+ * Constructor. Receives a string representing the input sentence. This string may be a
+ * string-encoded lattice or a plain text string for decoding.
+ *
+ * @param inputString representing the input sentence
+ * @param id ID to associate with the input string
+ * @param joshuaConfiguration a populated {@link org.apache.joshua.decoder.JoshuaConfiguration}
+ */
+ public Sentence(String inputString, int id, JoshuaConfiguration joshuaConfiguration) {
+
+ inputString = Regex.spaces.replaceAll(inputString, " ").trim();
+
+ config = joshuaConfiguration;
+
+ this.constraints = new LinkedList<ConstraintSpan>();
+
+ // Check if the sentence has SGML markings denoting the
+ // sentence ID; if so, override the id passed in to the
+ // constructor
+ Matcher start = SEG_START.matcher(inputString);
+ if (start.find()) {
+ source = SEG_END.matcher(start.replaceFirst("")).replaceFirst("");
+ String idstr = start.group(1);
+ this.id = Integer.parseInt(idstr);
+
+ } else {
+ if (inputString.indexOf(" ||| ") != -1) {
+ /* Target-side given; used for parsing and forced decoding */
+ String[] pieces = inputString.split("\\s?\\|{3}\\s?");
+ source = pieces[0];
+ target = pieces[1];
+ if (target.equals(""))
+ target = null;
+ if (pieces.length > 2) {
+ references = new String[pieces.length - 2];
+ System.arraycopy(pieces, 2, references, 0, pieces.length - 2);
+ }
+ this.id = id;
+
+ } else {
+ /* Regular ol' input sentence */
+ source = inputString;
+ this.id = id;
+ }
+ }
+
+ // Only trim strings
+ if (! (joshuaConfiguration.lattice_decoding && source.startsWith("(((")))
+ adjustForLength(joshuaConfiguration.maxlen);
+ }
+
+ /**
+ * Indicates whether the underlying lattice is a linear chain, i.e., a sentence.
+ *
+ * @return true if this is a linear chain, false otherwise
+ */
+ public boolean isLinearChain() {
+ return ! this.getLattice().hasMoreThanOnePath();
+ }
+
+ // Matches the opening and closing <seg> tags, e.g.,
+ // <seg id="72">this is a test input sentence</seg>.
+ protected static final Pattern SEG_START = Pattern
+ .compile("^\\s*<seg\\s+id=\"?(\\d+)\"?[^>]*>\\s*");
+ protected static final Pattern SEG_END = Pattern.compile("\\s*</seg\\s*>\\s*$");
+
+ /**
+ * Returns the length of the sentence. For lattices, the length is the shortest path through the
+ * lattice. The length includes the <s> and </s> sentence markers.
+ *
+ * @return number of input tokens + 2 (for start and end of sentence markers)
+ */
+ public int length() {
+ return this.getLattice().getShortestDistance();
+ }
+
+ /**
+ * Returns the annotations for a specific word (specified by an index) in the
+ * sentence
+ *
+ * @param index The location of the word in the sentence
+ * @param key The annotation identity
+ * @return The annotations associated with this word
+ */
+ public String getAnnotation(int index, String key) {
+ return getTokens().get(index).getAnnotation(key);
+ }
+
+ /**
+ * This function computes the intersection of \sigma^+ (where \sigma is the terminal vocabulary)
+ * with all character-level segmentations of each OOV in the input sentence.
+ *
+ * The idea is to break apart noun compounds in languages like German (such as the word "golfloch"
+ * = "golf" (golf) + "loch" (hole)), allowing them to be translated.
+ *
+ * @param grammars a list of grammars to consult to find in- and out-of-vocabulary items
+ */
+ public void segmentOOVs(Grammar[] grammars) {
+ Lattice<Token> oldLattice = this.getLattice();
+
+ /* Build a list of terminals across all grammars */
+ HashSet<Integer> vocabulary = new HashSet<Integer>();
+ for (Grammar grammar : grammars) {
+ Iterator<Integer> iterator = grammar.getTrieRoot().getTerminalExtensionIterator();
+ while (iterator.hasNext())
+ vocabulary.add(iterator.next());
+ }
+
+ List<Node<Token>> oldNodes = oldLattice.getNodes();
+
+ /* Find all the subwords that appear in the vocabulary, and create the lattice */
+ for (int nodeid = oldNodes.size() - 3; nodeid >= 1; nodeid -= 1) {
+ if (oldNodes.get(nodeid).getOutgoingArcs().size() == 1) {
+ Arc<Token> arc = oldNodes.get(nodeid).getOutgoingArcs().get(0);
+ String word = Vocabulary.word(arc.getLabel().getWord());
+ if (!vocabulary.contains(arc.getLabel())) {
+ // System.err.println(String.format("REPL: '%s'", word));
+ List<Arc<Token>> savedArcs = oldNodes.get(nodeid).getOutgoingArcs();
+
+ char[] chars = word.toCharArray();
+ ChartSpan<Boolean> wordChart = new ChartSpan<Boolean>(chars.length + 1, false);
+ ArrayList<Node<Token>> nodes = new ArrayList<Node<Token>>(chars.length + 1);
+ nodes.add(oldNodes.get(nodeid));
+ for (int i = 1; i < chars.length; i++)
+ nodes.add(new Node<Token>(i));
+ nodes.add(oldNodes.get(nodeid + 1));
+ for (int width = 1; width <= chars.length; width++) {
+ for (int i = 0; i <= chars.length - width; i++) {
+ int j = i + width;
+ if (width != chars.length) {
+ Token token = new Token(word.substring(i, j), config);
+ if (vocabulary.contains(id)) {
+ nodes.get(i).addArc(nodes.get(j), 0.0f, token);
+ wordChart.set(i, j, true);
+ // System.err.println(String.format(" FOUND '%s' at (%d,%d)", word.substring(i, j),
+ // i, j));
+ }
+ }
+
+ for (int k = i + 1; k < j; k++) {
+ if (wordChart.get(i, k) && wordChart.get(k, j)) {
+ wordChart.set(i, j, true);
+ // System.err.println(String.format(" PATH FROM %d-%d-%d", i, k, j));
+ }
+ }
+ }
+ }
+
+ /* If there's a path from beginning to end */
+ if (wordChart.get(0, chars.length)) {
+ // Remove nodes not part of a complete path
+ HashSet<Node<Token>> deletedNodes = new HashSet<Node<Token>>();
+ for (int k = 1; k < nodes.size() - 1; k++)
+ if (!(wordChart.get(0, k) && wordChart.get(k, chars.length)))
+ nodes.set(k, null);
+
+ int delIndex = 1;
+ while (delIndex < nodes.size())
+ if (nodes.get(delIndex) == null) {
+ deletedNodes.add(nodes.get(delIndex));
+ nodes.remove(delIndex);
+ } else
+ delIndex++;
+
+ for (Node<Token> node : nodes) {
+ int arcno = 0;
+ while (arcno != node.getOutgoingArcs().size()) {
+ Arc<Token> delArc = node.getOutgoingArcs().get(arcno);
+ if (deletedNodes.contains(delArc.getHead()))
+ node.getOutgoingArcs().remove(arcno);
+ else {
+ arcno++;
+ // System.err.println(" ARC: " + Vocabulary.word(delArc.getLabel()));
+ }
+ }
+ }
+
+ // Insert into the main lattice
+ this.getLattice().insert(nodeid, nodeid + 1, nodes);
+ } else {
+ nodes.get(0).setOutgoingArcs(savedArcs);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * If the input sentence is too long (not counting the <s> and </s> tokens), it is truncated to
+ * the maximum length, specified with the "maxlen" parameter.
+ *
+ * Note that this code assumes the underlying representation is a sentence, and not a lattice. Its
+ * behavior is undefined for lattices.
+ *
+ * @param length int representing the length to truncate the sentence to
+ */
+ protected void adjustForLength(int length) {
+ int size = this.getLattice().size() - 2; // subtract off the start- and end-of-sentence tokens
+
+ if (size > length) {
+ LOG.warn("sentence {} too long {}, truncating to length {}", id(), size, length);
+
+ // Replace the input sentence (and target) -- use the raw string, not source()
+ String[] tokens = source.split("\\s+");
+ source = tokens[0];
+ for (int i = 1; i < length; i++)
+ source += " " + tokens[i];
+ sourceLattice = null;
+ if (target != null) {
+ target = "";
+ }
+ }
+ }
+
+ public boolean isEmpty() {
+ return source.matches("^\\s*$");
+ }
+
+ public int id() {
+ return id;
+ }
+
+ /**
+ * Returns the raw source-side input string.
+ * @return the raw source-side input string
+ */
+ public String rawSource() {
+ return source;
+ }
+
+ /**
+ * Returns the source-side string with annotations --- if any --- stripped off.
+ *
+ * @return the source-side string with annotations --- if any --- stripped off
+ */
+ public String source() {
+ StringBuilder str = new StringBuilder();
+ int[] ids = getWordIDs();
+ for (int i = 1; i < ids.length - 1; i++) {
+ str.append(Vocabulary.word(ids[i])).append(" ");
+ }
+ return str.toString().trim();
+ }
+
+ /**
+ * Returns a sentence with the start and stop symbols added to the
+ * beginning and the end of the sentence respectively
+ *
+ * @return String The input sentence with start and stop symbols
+ */
+ public String fullSource() {
+ if (fullSource == null) {
+ fullSource = addSentenceMarkers(source());
+ }
+ return fullSource;
+ }
+
+ /**
+ * If a target side was supplied with the sentence, this will be non-null. This is used when doing
+ * synchronous parsing or constrained decoding. The input format is:
+ *
+ * Bill quiere ir a casa ||| Bill wants to go home
+ *
+ * If the parameter parse=true is set, parsing will be triggered, otherwise constrained decoding.
+ *
+ * @return target side of sentence translation
+ */
+ public String target() {
+ return target;
+ }
+
+ public String fullTarget() {
+ if (fullTarget == null) {
+ fullTarget = addSentenceMarkers(target());
+ }
+ return fullTarget;
+ }
+
+ public String source(int i, int j) {
+ StringTokenizer st = new StringTokenizer(fullSource());
+ int index = 0;
+ StringBuilder substring = new StringBuilder();
+ while (st.hasMoreTokens()) {
+ String token = st.nextToken();
+ if (index >= j)
+ break;
+ if (index >= i)
+ substring.append(token).append(" ");
+ index++;
+ }
+ return substring.toString().trim();
+ }
+
+ public String[] references() {
+ return references;
+ }
+
+ /**
+ * Returns the sequence of tokens comprising the sentence. This assumes you've done the checking
+ * to makes sure the input string (the source side) isn't a PLF waiting to be parsed.
+ *
+ * @return a {@link java.util.List} of {@link org.apache.joshua.decoder.segment_file.Token}'s comprising the sentence
+ */
+ public List<Token> getTokens() {
+ assert isLinearChain();
+ List<Token> tokens = new ArrayList<Token>();
+ for (Node<Token> node: getLattice().getNodes())
+ if (node != null && node.getOutgoingArcs().size() > 0)
+ tokens.add(node.getOutgoingArcs().get(0).getLabel());
+ return tokens;
+ }
+
+ /**
+ * Returns the sequence of word IDs comprising the input sentence. Assumes this is not a general
+ * lattice, but a linear chain.
+ * @return an int[] comprising all word ID's
+ */
+ public int[] getWordIDs() {
+ List<Token> tokens = getTokens();
+ int[] ids = new int[tokens.size()];
+ for (int i = 0; i < tokens.size(); i++)
+ ids[i] = tokens.get(i).getWord();
+ return ids;
+ }
+
+ /**
+ * Returns the sequence of word ids comprising the sentence. Assumes this is a sentence and
+ * not a lattice.
+ *
+ * @return the sequence of word ids comprising the sentence
+ */
+ public Lattice<String> stringLattice() {
+ assert isLinearChain();
+ return Lattice.createStringLatticeFromString(source(), config);
+ }
+
+ public List<ConstraintSpan> constraints() {
+ return constraints;
+ }
+
+ public Lattice<Token> getLattice() {
+ if (this.sourceLattice == null) {
+ if (config.lattice_decoding && rawSource().startsWith("(((")) {
+ if (config.search_algorithm.equals("stack")) {
+ throw new RuntimeException("* FATAL: lattice decoding currently not supported for stack-based search algorithm.");
+ }
+ this.sourceLattice = Lattice.createTokenLatticeFromPLF(rawSource(), config);
+ } else
+ this.sourceLattice = Lattice.createTokenLatticeFromString(String.format("%s %s %s", Vocabulary.START_SYM,
+ rawSource(), Vocabulary.STOP_SYM), config);
+ }
+ return this.sourceLattice;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder(source());
+ if (target() != null) {
+ sb.append(" ||| " + target());
+ }
+ return sb.toString();
+ }
+
+ public boolean hasPath(int begin, int end) {
+ return getLattice().distance(begin, end) != -1;
+ }
+
+ public Node<Token> getNode(int i) {
+ return getLattice().getNode(i);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/segment_file/Token.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/segment_file/Token.java b/joshua-core/src/main/java/org/apache/joshua/decoder/segment_file/Token.java
new file mode 100644
index 0000000..b84826d
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/segment_file/Token.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.segment_file;
+
+import static org.apache.joshua.util.FormatUtils.escapeSpecialSymbols;
+
+import java.util.HashMap;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.util.FormatUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Stores the identity of a word and its annotations in a sentence.
+
+ * @author "Gaurav Kumar"
+ * @author Matt Post
+ */
+public class Token {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Token.class);
+
+ // The token without the annotations
+ private String token;
+ private int tokenID;
+
+ private HashMap<String,String> annotations = null;
+ private JoshuaConfiguration joshuaConfiguration;
+
+ /**
+ * <p>Constructor : Creates a Token object from a raw word
+ * Extracts and assigns an annotation when available.
+ * Any word can be marked with annotations, which are arbitrary semicolon-delimited
+ * key[=value] pairs (the value is optional) listed in brackets after a word, e.g.,</p>
+ * <pre>
+ * Je[ref=Samuel;PRO] voudrais[FUT;COND]
+ * </pre>
+ *
+ * <p>This will create a dictionary annotation on the word of the following form for "Je"</p>
+ *
+ * <pre>
+ * ref -> Samuel
+ * PRO -> PRO
+ * </pre>
+ *
+ * <p>and the following for "voudrais":</p>
+ *
+ * <pre>
+ * FUT -> FUT
+ * COND -> COND
+ * </pre>
+ *
+ * @param rawWord A word with annotation information (possibly)
+ * @param config a populated {@link org.apache.joshua.decoder.JoshuaConfiguration}
+ *
+ */
+ public Token(String rawWord, JoshuaConfiguration config) {
+
+ this.joshuaConfiguration = config;
+
+ annotations = new HashMap<String,String>();
+
+ // Matches a word with an annotation
+ // Check guidelines in constructor description
+ Pattern pattern = Pattern.compile("(\\S+)\\[(\\S+)\\]");
+ Matcher tag = pattern.matcher(rawWord);
+ if (tag.find()) {
+ // Annotation match found
+ token = tag.group(1);
+ String tagStr = tag.group(2);
+
+ for (String annotation: tagStr.split(";")) {
+ int where = annotation.indexOf("=");
+ if (where != -1) {
+ annotations.put(annotation.substring(0, where), annotation.substring(where + 1));
+ } else {
+ annotations.put(annotation, annotation);
+ }
+ }
+ } else {
+ // No match found, which implies that this token does not have any annotations
+ token = rawWord;
+ }
+
+ // Mask strings that cause problems for the decoder. This has to be done *after* parsing for
+ // annotations.
+ token = escapeSpecialSymbols(token);
+
+ if (joshuaConfiguration != null && joshuaConfiguration.lowercase) {
+ if (FormatUtils.ISALLUPPERCASE(token))
+ annotations.put("lettercase", "all-upper");
+ else if (Character.isUpperCase(token.charAt(0)))
+ annotations.put("lettercase", "upper");
+ else
+ annotations.put("lettercase", "lower");
+
+ LOG.info("TOKEN: {} -> {} ({})", token, token.toLowerCase(), annotations.get("lettercase"));
+ token = token.toLowerCase();
+ }
+
+ tokenID = Vocabulary.id(token);
+ }
+
+ /**
+ * Returns the word ID (vocab ID) for this token
+ *
+ * @return int A word ID
+ */
+ public int getWord() {
+ return tokenID;
+ }
+
+ /**
+ * Returns the string associated with this token
+ * @return String A word
+ */
+ public String getWordIdentity() {
+ return token;
+ }
+
+ public String toString() {
+ return token;
+ }
+
+ /**
+ * Returns the annotationID (vocab ID)
+ * associated with this token
+ * @param key A type ID
+ * @return the annotationID (vocab ID)
+ */
+ public String getAnnotation(String key) {
+ if (annotations.containsKey(key)) {
+ return annotations.get(key);
+ }
+
+ return null;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/segment_file/package-info.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/segment_file/package-info.java b/joshua-core/src/main/java/org/apache/joshua/decoder/segment_file/package-info.java
new file mode 100644
index 0000000..a615030
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/segment_file/package-info.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * Provides common interfaces for parsing segment files
+ * (aka test corpora to be translated). In order to support
+ * constraint annotations, we provide a general API for
+ * use by JoshuaDecoder and Chart.
+ */
+package org.apache.joshua.decoder.segment_file;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/lattice/Arc.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/lattice/Arc.java b/joshua-core/src/main/java/org/apache/joshua/lattice/Arc.java
new file mode 100644
index 0000000..5d056ab
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/lattice/Arc.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.lattice;
+
+/**
+ * An arc in a directed graph.
+ *
+ * @author Lane Schwartz
+ * @since 2008-07-08
+ *
+ * @param Label Type of label associated with an arc.
+ */
+public class Arc<Label> {
+
+ /**
+ * Weight of this arc.
+ */
+ private float cost;
+
+ /**
+ * Node where this arc ends.
+ */
+ private Node<Label> head;
+
+ /**
+ * Node where this arc begins.
+ */
+ private Node<Label> tail;
+
+ /**
+ * Label associated with this arc.
+ */
+ private Label label;
+
+ /**
+ * Creates an arc with the specified head, tail, cost, and label.
+ *
+ * @param head The node where this arc begins.
+ * @param tail The node where this arc ends.
+ * @param cost The cost of this arc.
+ * @param label The label associated with this arc.
+ */
+ public Arc(Node<Label> tail, Node<Label> head, float cost, Label label) {
+ this.tail = tail;
+ this.head = head;
+ this.cost = cost;
+ this.label = label;
+ }
+
+ /**
+ * Gets the cost of this arc.
+ *
+ * @return The cost of this arc.
+ */
+ public float getCost() {
+ return cost;
+ }
+
+ /**
+ * Gets the tail of this arc (the node where this arc begins).
+ *
+ * @return The tail of this arc.
+ */
+ public Node<Label> getTail() {
+ return tail;
+ }
+
+ /**
+ * Gets the head of this arc (the node where this arc ends).
+ *
+ * @return The head of this arc.
+ */
+ public Node<Label> getHead() {
+ return head;
+ }
+
+ /**
+ * Gets the label associated with this arc.
+ *
+ * @return The label associated with this arc.
+ */
+ public Label getLabel() {
+ return label;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder s = new StringBuilder();
+
+ s.append(label.toString());
+ s.append(" : ");
+ s.append(tail.toString());
+ s.append(" ==> ");
+ s.append(head.toString());
+ s.append(" : ");
+ s.append(cost);
+
+ return s.toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/lattice/Lattice.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/lattice/Lattice.java b/joshua-core/src/main/java/org/apache/joshua/lattice/Lattice.java
new file mode 100644
index 0000000..2332159
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/lattice/Lattice.java
@@ -0,0 +1,587 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.lattice;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Stack;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.segment_file.Token;
+import org.apache.joshua.util.ChartSpan;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * A lattice representation of a directed graph.
+ *
+ * @author Lane Schwartz
+ * @author Matt Post post@cs.jhu.edu
+ * @since 2008-07-08
+ *
+ */
+public class Lattice<Value> implements Iterable<Node<Value>> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Lattice.class);
+
+ /**
+ * True if there is more than one path through the lattice.
+ */
+ private boolean latticeHasAmbiguity;
+
+ /**
+ * Costs of the best path between each pair of nodes in the lattice.
+ */
+ private ChartSpan<Integer> distances = null;
+
+ /**
+ * List of all nodes in the lattice. Nodes are assumed to be in topological order.
+ */
+ private List<Node<Value>> nodes;
+
+
+ JoshuaConfiguration config = null;
+
+ /**
+ * Constructs a new lattice from an existing list of (connected) nodes.
+ * <p>
+ * The list of nodes must already be in topological order. If the list is not in topological
+ * order, the behavior of the lattice is not defined.
+ *
+ * @param nodes A list of nodes which must be in topological order.
+ * @param config a populated {@link org.apache.joshua.decoder.JoshuaConfiguration}
+ */
+ public Lattice(List<Node<Value>> nodes, JoshuaConfiguration config) {
+ this.nodes = nodes;
+ // this.distances = calculateAllPairsShortestPath();
+ this.latticeHasAmbiguity = true;
+ }
+
+ public Lattice(List<Node<Value>> nodes, boolean isAmbiguous, JoshuaConfiguration config) {
+ // Node<Value> sink = new Node<Value>(nodes.size());
+ // nodes.add(sink);
+ this.nodes = nodes;
+ // this.distances = calculateAllPairsShortestPath();
+ this.latticeHasAmbiguity = isAmbiguous;
+ }
+
+ /**
+ * Instantiates a lattice from a linear chain of values, i.e., a sentence.
+ *
+ * @param linearChain a sequence of Value objects
+ * @param config a populated {@link org.apache.joshua.decoder.JoshuaConfiguration}
+ */
+ public Lattice(Value[] linearChain, JoshuaConfiguration config) {
+ this.latticeHasAmbiguity = false;
+ this.nodes = new ArrayList<Node<Value>>();
+
+ Node<Value> previous = new Node<Value>(0);
+ nodes.add(previous);
+
+ int i = 1;
+
+ for (Value value : linearChain) {
+
+ Node<Value> current = new Node<Value>(i);
+ float cost = 0.0f;
+ // if (i > 4) cost = (float)i/1.53432f;
+ previous.addArc(current, cost, value);
+
+ nodes.add(current);
+
+ previous = current;
+ i++;
+ }
+
+ // this.distances = calculateAllPairsShortestPath();
+ }
+
+ public final boolean hasMoreThanOnePath() {
+ return latticeHasAmbiguity;
+ }
+
+ /**
+ * Computes the shortest distance between two nodes, which is used (perhaps among other places) in
+ * computing which rules can apply over which spans of the input
+ *
+ * @param arc an {@link org.apache.joshua.lattice.Arc} of values
+ * @return the shortest distance between two nodes
+ */
+ public int distance(Arc<Value> arc) {
+ return this.getShortestPath(arc.getTail().getNumber(), arc.getHead().getNumber());
+ }
+
+ public int distance(int i, int j) {
+ return this.getShortestPath(i, j);
+ }
+
+ /**
+ * Convenience method to get a lattice from a linear sequence of {@link Token} objects.
+ *
+ * @param source input string from which to create a {@link org.apache.joshua.lattice.Lattice}
+ * @param config a populated {@link org.apache.joshua.decoder.JoshuaConfiguration}
+ * @return Lattice representation of the linear chain.
+ */
+ public static Lattice<Token> createTokenLatticeFromString(String source, JoshuaConfiguration config) {
+ String[] tokens = source.split("\\s+");
+ Token[] integerSentence = new Token[tokens.length];
+ for (int i = 0; i < tokens.length; i++) {
+ integerSentence[i] = new Token(tokens[i], config);
+ }
+
+ return new Lattice<Token>(integerSentence, config);
+ }
+
+ public static Lattice<Token> createTokenLatticeFromPLF(String data, JoshuaConfiguration config) {
+ ArrayList<Node<Token>> nodes = new ArrayList<Node<Token>>();
+
+ // This matches a sequence of tuples, which describe arcs leaving this node
+ Pattern nodePattern = Pattern.compile("(.+?)\\(\\s*(\\(.+?\\),\\s*)\\s*\\)(.*)");
+
+ /*
+ * This matches a comma-delimited, parenthesized tuple of a (a) single-quoted word (b) a number,
+ * optionally in scientific notation (c) an offset (how many states to jump ahead)
+ */
+ Pattern arcPattern = Pattern
+ .compile("\\s*\\('(.+?)',\\s*(-?\\d+\\.?\\d*?(?:[eE]-?\\d+)?),\\s*(\\d+)\\),\\s*(.*)");
+
+ Matcher nodeMatcher = nodePattern.matcher(data);
+
+ boolean latticeIsAmbiguous = false;
+
+ int nodeID = 0;
+ Node<Token> startNode = new Node<Token>(nodeID);
+ nodes.add(startNode);
+
+ while (nodeMatcher.matches()) {
+
+ String nodeData = nodeMatcher.group(2);
+ String remainingData = nodeMatcher.group(3);
+
+ nodeID++;
+
+ Node<Token> currentNode = null;
+ if (nodeID < nodes.size() && nodes.get(nodeID) != null) {
+ currentNode = nodes.get(nodeID);
+ } else {
+ currentNode = new Node<Token>(nodeID);
+ while (nodeID > nodes.size())
+ nodes.add(new Node<Token>(nodes.size()));
+ nodes.add(currentNode);
+ }
+
+ Matcher arcMatcher = arcPattern.matcher(nodeData);
+ int numArcs = 0;
+ if (!arcMatcher.matches()) {
+ throw new RuntimeException("Parse error!");
+ }
+ while (arcMatcher.matches()) {
+ numArcs++;
+ String arcLabel = arcMatcher.group(1);
+ float arcWeight = Float.parseFloat(arcMatcher.group(2));
+ int destinationNodeID = nodeID + Integer.parseInt(arcMatcher.group(3));
+
+ Node<Token> destinationNode;
+ if (destinationNodeID < nodes.size() && nodes.get(destinationNodeID) != null) {
+ destinationNode = nodes.get(destinationNodeID);
+ } else {
+ destinationNode = new Node<Token>(destinationNodeID);
+ while (destinationNodeID > nodes.size())
+ nodes.add(new Node<Token>(nodes.size()));
+ nodes.add(destinationNode);
+ }
+
+ String remainingArcs = arcMatcher.group(4);
+
+ Token arcToken = new Token(arcLabel, config);
+ currentNode.addArc(destinationNode, arcWeight, arcToken);
+
+ arcMatcher = arcPattern.matcher(remainingArcs);
+ }
+ if (numArcs > 1)
+ latticeIsAmbiguous = true;
+
+ nodeMatcher = nodePattern.matcher(remainingData);
+ }
+
+ /* Add <s> to the start of the lattice. */
+ if (nodes.size() > 1 && nodes.get(1) != null) {
+ Node<Token> firstNode = nodes.get(1);
+ startNode.addArc(firstNode, 0.0f, new Token(Vocabulary.START_SYM, config));
+ }
+
+ /* Add </s> as a final state, connect it to the previous end-state */
+ nodeID = nodes.get(nodes.size()-1).getNumber() + 1;
+ Node<Token> endNode = new Node<Token>(nodeID);
+ nodes.get(nodes.size()-1).addArc(endNode, 0.0f, new Token(Vocabulary.STOP_SYM, config));
+ nodes.add(endNode);
+
+ return new Lattice<Token>(nodes, latticeIsAmbiguous, config);
+ }
+
+ /**
+ * Constructs a lattice from a given string representation.
+ *
+ * @param data String representation of a lattice.
+ * @param config a populated {@link org.apache.joshua.decoder.JoshuaConfiguration}
+ * @return A lattice that corresponds to the given string.
+ */
+ public static Lattice<String> createStringLatticeFromString(String data, JoshuaConfiguration config) {
+
+ Map<Integer, Node<String>> nodes = new HashMap<Integer, Node<String>>();
+
+ Pattern nodePattern = Pattern.compile("(.+?)\\((\\(.+?\\),)\\)(.*)");
+ Pattern arcPattern = Pattern.compile("\\('(.+?)',(\\d+.\\d+),(\\d+)\\),(.*)");
+
+ Matcher nodeMatcher = nodePattern.matcher(data);
+
+ int nodeID = -1;
+
+ while (nodeMatcher.matches()) {
+
+ String nodeData = nodeMatcher.group(2);
+ String remainingData = nodeMatcher.group(3);
+
+ nodeID++;
+
+ Node<String> currentNode;
+ if (nodes.containsKey(nodeID)) {
+ currentNode = nodes.get(nodeID);
+ } else {
+ currentNode = new Node<String>(nodeID);
+ nodes.put(nodeID, currentNode);
+ }
+
+ LOG.debug("Node : {}", nodeID);
+
+ Matcher arcMatcher = arcPattern.matcher(nodeData);
+
+ while (arcMatcher.matches()) {
+ String arcLabel = arcMatcher.group(1);
+ float arcWeight = Float.valueOf(arcMatcher.group(2));
+ int destinationNodeID = nodeID + Integer.parseInt(arcMatcher.group(3));
+
+ Node<String> destinationNode;
+ if (nodes.containsKey(destinationNodeID)) {
+ destinationNode = nodes.get(destinationNodeID);
+ } else {
+ destinationNode = new Node<String>(destinationNodeID);
+ nodes.put(destinationNodeID, destinationNode);
+ }
+
+ String remainingArcs = arcMatcher.group(4);
+
+ LOG.debug("\t{} {} {}", arcLabel, arcWeight, destinationNodeID);
+
+ currentNode.addArc(destinationNode, arcWeight, arcLabel);
+
+ arcMatcher = arcPattern.matcher(remainingArcs);
+ }
+
+ nodeMatcher = nodePattern.matcher(remainingData);
+ }
+
+ List<Node<String>> nodeList = new ArrayList<Node<String>>(nodes.values());
+ Collections.sort(nodeList, new NodeIdentifierComparator());
+
+ LOG.debug("Nodelist={}", nodeList);
+
+ return new Lattice<String>(nodeList, config);
+ }
+
+ /**
+ * Gets the cost of the shortest path between two nodes.
+ *
+ * @param from ID of the starting node.
+ * @param to ID of the ending node.
+ * @return The cost of the shortest path between the two nodes.
+ */
+ public int getShortestPath(int from, int to) {
+ // System.err.println(String.format("DISTANCE(%d,%d) = %f", from, to, costs[from][to]));
+ if (distances == null)
+ this.distances = calculateAllPairsShortestPath();
+
+ return distances.get(from, to);
+ }
+
+ /**
+ * Gets the shortest distance through the lattice.
+ * @return int representing the shortest distance through the lattice
+ */
+ public int getShortestDistance() {
+ if (distances == null)
+ distances = calculateAllPairsShortestPath();
+ return distances.get(0, nodes.size()-1);
+ }
+
+ /**
+ * Gets the node with a specified integer identifier. If the identifier is negative, we count
+ * backwards from the end of the array, Perl-style (-1 is the last element, -2 the penultimate,
+ * etc).
+ *
+ * @param index Integer identifier for a node.
+ * @return The node with the specified integer identifier
+ */
+ public Node<Value> getNode(int index) {
+ if (index >= 0)
+ return nodes.get(index);
+ else
+ return nodes.get(size() + index);
+ }
+
+ public List<Node<Value>> getNodes() {
+ return nodes;
+ }
+
+ /**
+ * Returns an iterator over the nodes in this lattice.
+ *
+ * @return An iterator over the nodes in this lattice.
+ */
+ public Iterator<Node<Value>> iterator() {
+ return nodes.iterator();
+ }
+
+ /**
+ * Returns the number of nodes in this lattice.
+ *
+ * @return The number of nodes in this lattice.
+ */
+ public int size() {
+ return nodes.size();
+ }
+
+ /**
+ * Calculate the all-pairs shortest path for all pairs of nodes.
+ * <p>
+ * Note: This method assumes no backward arcs. If there are backward arcs, the returned shortest
+ * path costs for that node may not be accurate.
+ *
+ * @param nodes A list of nodes which must be in topological order.
+ * @return The all-pairs shortest path for all pairs of nodes.
+ */
+ private ChartSpan<Integer> calculateAllPairsShortestPath() {
+
+ ChartSpan<Integer> distance = new ChartSpan<Integer>(nodes.size() - 1, Integer.MAX_VALUE);
+ distance.setDiagonal(0);
+
+ /* Mark reachability between immediate neighbors */
+ for (Node<Value> tail : nodes) {
+ for (Arc<Value> arc : tail.getOutgoingArcs()) {
+ Node<Value> head = arc.getHead();
+ distance.set(tail.id(), head.id(), 1);
+ }
+ }
+
+ int size = nodes.size();
+
+ for (int width = 2; width <= size; width++) {
+ for (int i = 0; i < size - width; i++) {
+ int j = i + width;
+ for (int k = i + 1; k < j; k++) {
+ distance.set(i, j, Math.min(distance.get(i, j), distance.get(i, k) + distance.get(k, j)));
+ }
+ }
+ }
+
+ return distance;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder s = new StringBuilder();
+
+ for (Node<Value> start : this) {
+ for (Arc<Value> arc : start.getOutgoingArcs()) {
+ s.append(arc.toString());
+ s.append('\n');
+ }
+ }
+
+ return s.toString();
+ }
+
+ public static void main(String[] args) {
+
+ List<Node<String>> nodes = new ArrayList<Node<String>>();
+ for (int i = 0; i < 4; i++) {
+ nodes.add(new Node<String>(i));
+ }
+
+ nodes.get(0).addArc(nodes.get(1), 1.0f, "x");
+ nodes.get(1).addArc(nodes.get(2), 1.0f, "y");
+ nodes.get(0).addArc(nodes.get(2), 1.5f, "a");
+ nodes.get(2).addArc(nodes.get(3), 3.0f, "b");
+ nodes.get(2).addArc(nodes.get(3), 5.0f, "c");
+
+ Lattice<String> graph = new Lattice<String>(nodes, null);
+
+ System.out.println("Shortest path from 0 to 3: " + graph.getShortestPath(0, 3));
+ }
+
+ /**
+ * Replaced the arc from node i to j with the supplied lattice. This is used to do OOV
+ * segmentation of words in a lattice.
+ *
+ * @param i start node of arc
+ * @param j end node of arc
+ * @param newNodes new nodes used within the replacement operation
+ */
+ public void insert(int i, int j, List<Node<Value>> newNodes) {
+
+ nodes.get(i).setOutgoingArcs(newNodes.get(0).getOutgoingArcs());
+
+ newNodes.remove(0);
+ nodes.remove(j);
+ Collections.reverse(newNodes);
+
+ for (Node<Value> node: newNodes)
+ nodes.add(j, node);
+
+ this.latticeHasAmbiguity = false;
+ for (int x = 0; x < nodes.size(); x++) {
+ nodes.get(x).setID(x);
+ this.latticeHasAmbiguity |= (nodes.get(x).getOutgoingArcs().size() > 1);
+ }
+
+ this.distances = null;
+ }
+
+ /**
+ * Topologically sorts the nodes and reassigns their numbers. Assumes that the first node is the
+ * source, but otherwise assumes nothing about the input.
+ *
+ * Probably correct, but untested.
+ */
+ @SuppressWarnings("unused")
+ private void topologicalSort() {
+ HashMap<Node<Value>, List<Arc<Value>>> outgraph = new HashMap<Node<Value>, List<Arc<Value>>>();
+ HashMap<Node<Value>, List<Arc<Value>>> ingraph = new HashMap<Node<Value>, List<Arc<Value>>>();
+ for (Node<Value> node: nodes) {
+ ArrayList<Arc<Value>> arcs = new ArrayList<Arc<Value>>();
+ for (Arc<Value> arc: node.getOutgoingArcs()) {
+ arcs.add(arc);
+
+ if (! ingraph.containsKey(arc.getHead()))
+ ingraph.put(arc.getHead(), new ArrayList<Arc<Value>>());
+ ingraph.get(arc.getHead()).add(arc);
+
+ outgraph.put(node, arcs);
+ }
+ }
+
+ ArrayList<Node<Value>> sortedNodes = new ArrayList<Node<Value>>();
+ Stack<Node<Value>> stack = new Stack<Node<Value>>();
+ stack.push(nodes.get(0));
+
+ while (! stack.empty()) {
+ Node<Value> node = stack.pop();
+ sortedNodes.add(node);
+ for (Arc<Value> arc: outgraph.get(node)) {
+ outgraph.get(node).remove(arc);
+ ingraph.get(arc.getHead()).remove(arc);
+
+ if (ingraph.get(arc.getHead()).size() == 0)
+ sortedNodes.add(arc.getHead());
+ }
+ }
+
+ int id = 0;
+ for (Node<Value> node : sortedNodes)
+ node.setID(id++);
+
+ this.nodes = sortedNodes;
+ }
+
+ /**
+ * Constructs a lattice from a given string representation.
+ *
+ * @param data String representation of a lattice.
+ * @return A lattice that corresponds to the given string.
+ */
+ public static Lattice<String> createFromString(String data) {
+
+ Map<Integer,Node<String>> nodes = new HashMap<Integer,Node<String>>();
+
+ Pattern nodePattern = Pattern.compile("(.+?)\\((\\(.+?\\),)\\)(.*)");
+ Pattern arcPattern = Pattern.compile("\\('(.+?)',(\\d+.\\d+),(\\d+)\\),(.*)");
+
+ Matcher nodeMatcher = nodePattern.matcher(data);
+
+ int nodeID = -1;
+
+ while (nodeMatcher.matches()) {
+
+ String nodeData = nodeMatcher.group(2);
+ String remainingData = nodeMatcher.group(3);
+
+ nodeID++;
+
+ Node<String> currentNode;
+ if (nodes.containsKey(nodeID)) {
+ currentNode = nodes.get(nodeID);
+ } else {
+ currentNode = new Node<String>(nodeID);
+ nodes.put(nodeID, currentNode);
+ }
+
+ LOG.debug("Node : {}", nodeID);
+
+ Matcher arcMatcher = arcPattern.matcher(nodeData);
+
+ while (arcMatcher.matches()) {
+ String arcLabel = arcMatcher.group(1);
+ double arcWeight = Double.valueOf(arcMatcher.group(2));
+ int destinationNodeID = nodeID + Integer.valueOf(arcMatcher.group(3));
+
+ Node<String> destinationNode;
+ if (nodes.containsKey(destinationNodeID)) {
+ destinationNode = nodes.get(destinationNodeID);
+ } else {
+ destinationNode = new Node<String>(destinationNodeID);
+ nodes.put(destinationNodeID, destinationNode);
+ }
+
+ String remainingArcs = arcMatcher.group(4);
+
+ LOG.debug("\t {} {} {}", arcLabel, arcWeight, destinationNodeID);
+
+ currentNode.addArc(destinationNode, (float) arcWeight, arcLabel);
+
+ arcMatcher = arcPattern.matcher(remainingArcs);
+ }
+
+ nodeMatcher = nodePattern.matcher(remainingData);
+ }
+
+ List<Node<String>> nodeList = new ArrayList<Node<String>>(nodes.values());
+ Collections.sort(nodeList, new NodeIdentifierComparator());
+
+ LOG.debug("Nodelist={}", nodeList);
+
+ return new Lattice<String>(nodeList, new JoshuaConfiguration());
+ }
+}
[48/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/resources/kbest_extraction/output.scores.gold
----------------------------------------------------------------------
diff --git a/joshua-core/resources/kbest_extraction/output.scores.gold b/joshua-core/resources/kbest_extraction/output.scores.gold
new file mode 100644
index 0000000..2f8b814
--- /dev/null
+++ b/joshua-core/resources/kbest_extraction/output.scores.gold
@@ -0,0 +1,3126 @@
+0 ||| A A A A A ||| -195.045
+0 ||| B A A A A ||| -196.045
+0 ||| C A A A A ||| -198.045
+0 ||| A B A A A ||| -199.045
+0 ||| B B A A A ||| -200.045
+0 ||| D A A A A ||| -200.045
+0 ||| A A A A B ||| -201.045
+0 ||| A A A B A ||| -201.045
+0 ||| A A B A A ||| -201.045
+0 ||| A C A A A ||| -201.045
+0 ||| B A A A B ||| -202.045
+0 ||| B A A B A ||| -202.045
+0 ||| B A B A A ||| -202.045
+0 ||| B C A A A ||| -202.045
+0 ||| C B A A A ||| -202.045
+0 ||| A A A C A ||| -203.045
+0 ||| C A A A B ||| -204.045
+0 ||| C A A B A ||| -204.045
+0 ||| B A A C A ||| -204.045
+0 ||| C A B A A ||| -204.045
+0 ||| E A A A A ||| -204.045
+0 ||| C C A A A ||| -204.045
+0 ||| D B A A A ||| -204.045
+0 ||| A A A A C ||| -205.045
+0 ||| A B A B A ||| -205.045
+0 ||| A B A A B ||| -205.045
+0 ||| A A C A A ||| -205.045
+0 ||| A B B A A ||| -205.045
+0 ||| A D A A A ||| -205.045
+0 ||| B A A A C ||| -206.045
+0 ||| B B A A B ||| -206.045
+0 ||| B B A B A ||| -206.045
+0 ||| D A A A B ||| -206.045
+0 ||| B A C A A ||| -206.045
+0 ||| D A A B A ||| -206.045
+0 ||| B B B A A ||| -206.045
+0 ||| C A A C A ||| -206.045
+0 ||| D A B A A ||| -206.045
+0 ||| B D A A A ||| -206.045
+0 ||| D C A A A ||| -206.045
+0 ||| A A B A B ||| -207.045
+0 ||| A A B B A ||| -207.045
+0 ||| A A A B B ||| -207.045
+0 ||| A B A C A ||| -207.045
+0 ||| A C A A B ||| -207.045
+0 ||| A C A B A ||| -207.045
+0 ||| A C B A A ||| -207.045
+0 ||| A A D A A ||| -207.045
+0 ||| C A A A C ||| -208.045
+0 ||| B A A B B ||| -208.045
+0 ||| C B A B A ||| -208.045
+0 ||| B A B A B ||| -208.045
+0 ||| B C A B A ||| -208.045
+0 ||| B C A A B ||| -208.045
+0 ||| B A B B A ||| -208.045
+0 ||| C B A A B ||| -208.045
+0 ||| D A A C A ||| -208.045
+0 ||| C A C A A ||| -208.045
+0 ||| B B A C A ||| -208.045
+0 ||| E B A A A ||| -208.045
+0 ||| B C B A A ||| -208.045
+0 ||| C B B A A ||| -208.045
+0 ||| C D A A A ||| -208.045
+0 ||| B A D A A ||| -208.045
+0 ||| A B A A C ||| -209.045
+0 ||| A A A C B ||| -209.045
+0 ||| A A B C A ||| -209.045
+0 ||| A B C A A ||| -209.045
+0 ||| A C A C A ||| -209.045
+0 ||| A A A D A ||| -209.045
+0 ||| C A A B B ||| -210.045
+0 ||| D A A A C ||| -210.045
+0 ||| C A B A B ||| -210.045
+0 ||| B A A C B ||| -210.045
+0 ||| C C A A B ||| -210.045
+0 ||| D B A A B ||| -210.045
+0 ||| B B A A C ||| -210.045
+0 ||| E A A A B ||| -210.045
+0 ||| D B A B A ||| -210.045
+0 ||| D A C A A ||| -210.045
+0 ||| E A A B A ||| -210.045
+0 ||| B A A D A ||| -210.045
+0 ||| D D A A A ||| -210.045
+0 ||| B C A C A ||| -210.045
+0 ||| C A D A A ||| -210.045
+0 ||| C A B B A ||| -210.045
+0 ||| E A B A A ||| -210.045
+0 ||| C B A C A ||| -210.045
+0 ||| E C A A A ||| -210.045
+0 ||| C C A B A ||| -210.045
+0 ||| B B C A A ||| -210.045
+0 ||| B A B C A ||| -210.045
+0 ||| C C B A A ||| -210.045
+0 ||| D B B A A ||| -210.045
+0 ||| A B A B B ||| -211.045
+0 ||| A B B B A ||| -211.045
+0 ||| A A A B C ||| -211.045
+0 ||| A D A B A ||| -211.045
+0 ||| A C A A C ||| -211.045
+0 ||| A A C B A ||| -211.045
+0 ||| A A C A B ||| -211.045
+0 ||| A C C A A ||| -211.045
+0 ||| A A B A C ||| -211.045
+0 ||| A B D A A ||| -211.045
+0 ||| A A A A D ||| -211.045
+0 ||| A A E A A ||| -211.045
+0 ||| A B B A B ||| -211.045
+0 ||| A E A A A ||| -211.045
+0 ||| A D A A B ||| -211.045
+0 ||| A D B A A ||| -211.045
+0 ||| B D A A B ||| -212.045
+0 ||| C B A A C ||| -212.045
+0 ||| B B B B A ||| -212.045
+0 ||| B A C A B ||| -212.045
+0 ||| D A A B B ||| -212.045
+0 ||| E A A C A ||| -212.045
+0 ||| B B A B B ||| -212.045
+0 ||| C A A D A ||| -212.045
+0 ||| B A A B C ||| -212.045
+0 ||| D C A B A ||| -212.045
+0 ||| B C A A C ||| -212.045
+0 ||| D B A C A ||| -212.045
+0 ||| B A A A D ||| -212.045
+0 ||| B A C B A ||| -212.045
+0 ||| D C A A B ||| -212.045
+0 ||| D A B B A ||| -212.045
+0 ||| B B B A B ||| -212.045
+0 ||| B D A B A ||| -212.045
+0 ||| C A A C B ||| -212.045
+0 ||| B A E A A ||| -212.045
+0 ||| B A B A C ||| -212.045
+0 ||| C C A C A ||| -212.045
+0 ||| D A B A B ||| -212.045
+0 ||| B C C A A ||| -212.045
+0 ||| C A B C A ||| -212.045
+0 ||| B E A A A ||| -212.045
+0 ||| D A D A A ||| -212.045
+0 ||| C B C A A ||| -212.045
+0 ||| B B D A A ||| -212.045
+0 ||| D C B A A ||| -212.045
+0 ||| B D B A A ||| -212.045
+0 ||| A C B A B ||| -213.045
+0 ||| A A D A B ||| -213.045
+0 ||| A A B B B ||| -213.045
+0 ||| A A A C C ||| -213.045
+0 ||| A B A C B ||| -213.045
+0 ||| A C A B B ||| -213.045
+0 ||| A B B C A ||| -213.045
+0 ||| A A A E A ||| -213.045
+0 ||| A C B B A ||| -213.045
+0 ||| A A D B A ||| -213.045
+0 ||| A B A D A ||| -213.045
+0 ||| A D A C A ||| -213.045
+0 ||| A A C C A ||| -213.045
+0 ||| A C D A A ||| -213.045
+0 ||| B C A B B ||| -214.045
+0 ||| B A B B B ||| -214.045
+0 ||| B C B A B ||| -214.045
+0 ||| C B B A B ||| -214.045
+0 ||| E B A A B ||| -214.045
+0 ||| C A A B C ||| -214.045
+0 ||| D A A C B ||| -214.045
+0 ||| C A C A B ||| -214.045
+0 ||| B B A C B ||| -214.045
+0 ||| C A B A C ||| -214.045
+0 ||| C D A A B ||| -214.045
+0 ||| B A D A B ||| -214.045
+0 ||| C B A B B ||| -214.045
+0 ||| E A A A C ||| -214.045
+0 ||| D B A A C ||| -214.045
+0 ||| C A A A D ||| -214.045
+0 ||| D A B C A ||| -214.045
+0 ||| C C A A C ||| -214.045
+0 ||| B A A C C ||| -214.045
+0 ||| B A D B A ||| -214.045
+0 ||| B A C C A ||| -214.045
+0 ||| B B B C A ||| -214.045
+0 ||| D C A C A ||| -214.045
+0 ||| D A A D A ||| -214.045
+0 ||| B B A D A ||| -214.045
+0 ||| C A C B A ||| -214.045
+0 ||| B A A E A ||| -214.045
+0 ||| C C C A A ||| -214.045
+0 ||| B D A C A ||| -214.045
+0 ||| E B A B A ||| -214.045
+0 ||| E B B A A ||| -214.045
+0 ||| B C B B A ||| -214.045
+0 ||| C B B B A ||| -214.045
+0 ||| C A E A A ||| -214.045
+0 ||| C D A B A ||| -214.045
+0 ||| C D B A A ||| -214.045
+0 ||| B C D A A ||| -214.045
+0 ||| E D A A A ||| -214.045
+0 ||| D B C A A ||| -214.045
+0 ||| C E A A A ||| -214.045
+0 ||| E A C A A ||| -214.045
+0 ||| C B D A A ||| -214.045
+0 ||| A B C A B ||| -215.045
+0 ||| A B B A C ||| -215.045
+0 ||| A D A A C ||| -215.045
+0 ||| A A C A C ||| -215.045
+0 ||| A B A B C ||| -215.045
+0 ||| A A B C B ||| -215.045
+0 ||| A C A C B ||| -215.045
+0 ||| A B A A D ||| -215.045
+0 ||| A A A D B ||| -215.045
+0 ||| A B C B A ||| -215.045
+0 ||| A C A D A ||| -215.045
+0 ||| A A D C A ||| -215.045
+0 ||| A C B C A ||| -215.045
+0 ||| A A B D A ||| -215.045
+0 ||| A B E A A ||| -215.045
+0 ||| A D C A A ||| -215.045
+0 ||| B B C A B ||| -216.045
+0 ||| E C A A B ||| -216.045
+0 ||| C C A B B ||| -216.045
+0 ||| B A B C B ||| -216.045
+0 ||| C C B B A ||| -216.045
+0 ||| C C B A B ||| -216.045
+0 ||| D B B A B ||| -216.045
+0 ||| E A A B B ||| -216.045
+0 ||| D A C B A ||| -216.045
+0 ||| B A C A C ||| -216.045
+0 ||| D A C A B ||| -216.045
+0 ||| D D A B A ||| -216.045
+0 ||| D A B A C ||| -216.045
+0 ||| B D A A C ||| -216.045
+0 ||| C C D A A ||| -216.045
+0 ||| D B A B B ||| -216.045
+0 ||| B A B D A ||| -216.045
+0 ||| B A A D B ||| -216.045
+0 ||| C A D B A ||| -216.045
+0 ||| B B A A D ||| -216.045
+0 ||| E A B B A ||| -216.045
+0 ||| D A A A D ||| -216.045
+0 ||| D C C A A ||| -216.045
+0 ||| D A A B C ||| -216.045
+0 ||| E C A B A ||| -216.045
+0 ||| C A D A B ||| -216.045
+0 ||| B B C B A ||| -216.045
+0 ||| B C A C B ||| -216.045
+0 ||| B D C A A ||| -216.045
+0 ||| C A B B B ||| -216.045
+0 ||| C D A C A ||| -216.045
+0 ||| D D A A B ||| -216.045
+0 ||| D B B B A ||| -216.045
+0 ||| C A A C C ||| -216.045
+0 ||| D D B A A ||| -216.045
+0 ||| B B A B C ||| -216.045
+0 ||| B A D C A ||| -216.045
+0 ||| C B A C B ||| -216.045
+0 ||| C A A E A ||| -216.045
+0 ||| E A B A B ||| -216.045
+0 ||| D B D A A ||| -216.045
+0 ||| D C A A C ||| -216.045
+0 ||| B C A D A ||| -216.045
+0 ||| B B B A C ||| -216.045
+0 ||| D E A A A ||| -216.045
+0 ||| C A C C A ||| -216.045
+0 ||| E A D A A ||| -216.045
+0 ||| C B B C A ||| -216.045
+0 ||| E C B A A ||| -216.045
+0 ||| B C B C A ||| -216.045
+0 ||| B B E A A ||| -216.045
+0 ||| C B A D A ||| -216.045
+0 ||| D A E A A ||| -216.045
+0 ||| E B A C A ||| -216.045
+0 ||| A D B A B ||| -217.045
+0 ||| A A C B B ||| -217.045
+0 ||| A C A B C ||| -217.045
+0 ||| A B B B B ||| -217.045
+0 ||| A B A C C ||| -217.045
+0 ||| A B D A B ||| -217.045
+0 ||| A A E A B ||| -217.045
+0 ||| A C C A B ||| -217.045
+0 ||| A E A A B ||| -217.045
+0 ||| A D A B B ||| -217.045
+0 ||| A A B B C ||| -217.045
+0 ||| A B D B A ||| -217.045
+0 ||| A A D A C ||| -217.045
+0 ||| A C B A C ||| -217.045
+0 ||| A B C C A ||| -217.045
+0 ||| A A A B D ||| -217.045
+0 ||| A C C B A ||| -217.045
+0 ||| A A B A D ||| -217.045
+0 ||| A D D A A ||| -217.045
+0 ||| A C A A D ||| -217.045
+0 ||| A D B B A ||| -217.045
+0 ||| A E A B A ||| -217.045
+0 ||| A C E A A ||| -217.045
+0 ||| A A E B A ||| -217.045
+0 ||| A E B A A ||| -217.045
+0 ||| A B A E A ||| -217.045
+0 ||| B B B B B ||| -218.045
+0 ||| E A A C B ||| -218.045
+0 ||| D C A B B ||| -218.045
+0 ||| B C A B C ||| -218.045
+0 ||| B D A B B ||| -218.045
+0 ||| B E A A B ||| -218.045
+0 ||| D A B B B ||| -218.045
+0 ||| B A E A B ||| -218.045
+0 ||| B A C B B ||| -218.045
+0 ||| B C B A C ||| -218.045
+0 ||| C C A C B ||| -218.045
+0 ||| C A B C B ||| -218.045
+0 ||| C A A D B ||| -218.045
+0 ||| D B A C B ||| -218.045
+0 ||| B C C A B ||| -218.045
+0 ||| B B C C A ||| -218.045
+0 ||| B A B B C ||| -218.045
+0 ||| B B D A B ||| -218.045
+0 ||| C B C A B ||| -218.045
+0 ||| C B E A A ||| -218.045
+0 ||| C B A A D ||| -218.045
+0 ||| D A D A B ||| -218.045
+0 ||| D C B A B ||| -218.045
+0 ||| E A B C A ||| -218.045
+0 ||| B D B A B ||| -218.045
+0 ||| E C A C A ||| -218.045
+0 ||| C B A B C ||| -218.045
+0 ||| B A E B A ||| -218.045
+0 ||| D A A C C ||| -218.045
+0 ||| D A C C A ||| -218.045
+0 ||| C B B A C ||| -218.045
+0 ||| C C B C A ||| -218.045
+0 ||| E B A A C ||| -218.045
+0 ||| C A B D A ||| -218.045
+0 ||| B A D A C ||| -218.045
+0 ||| C A D C A ||| -218.045
+0 ||| B B A C C ||| -218.045
+0 ||| C D C A A ||| -218.045
+0 ||| C A C A C ||| -218.045
+0 ||| D D A C A ||| -218.045
+0 ||| C D A A C ||| -218.045
+0 ||| B B D B A ||| -218.045
+0 ||| B A B A D ||| -218.045
+0 ||| E B C A A ||| -218.045
+0 ||| B C A A D ||| -218.045
+0 ||| D A D B A ||| -218.045
+0 ||| B A A B D ||| -218.045
+0 ||| B E A B A ||| -218.045
+0 ||| D C D A A ||| -218.045
+0 ||| D B B C A ||| -218.045
+0 ||| D B A D A ||| -218.045
+0 ||| E A A D A ||| -218.045
+0 ||| B D D A A ||| -218.045
+0 ||| C B C B A ||| -218.045
+0 ||| B C E A A ||| -218.045
+0 ||| B C C B A ||| -218.045
+0 ||| B E B A A ||| -218.045
+0 ||| C C A D A ||| -218.045
+0 ||| B D B B A ||| -218.045
+0 ||| D A A E A ||| -218.045
+0 ||| D C B B A ||| -218.045
+0 ||| B B A E A ||| -218.045
+0 ||| A B B C B ||| -219.045
+0 ||| A A A C D ||| -219.045
+0 ||| A B A D B ||| -219.045
+0 ||| A A A D C ||| -219.045
+0 ||| A C D A B ||| -219.045
+0 ||| A C A C C ||| -219.045
+0 ||| A A A E B ||| -219.045
+0 ||| A C B B B ||| -219.045
+0 ||| A A D B B ||| -219.045
+0 ||| A A C C B ||| -219.045
+0 ||| A D A C B ||| -219.045
+0 ||| A A B E A ||| -219.045
+0 ||| A A A A E ||| -219.045
+0 ||| A B C A C ||| -219.045
+0 ||| A D B C A ||| -219.045
+0 ||| A A B C C ||| -219.045
+0 ||| A E A C A ||| -219.045
+0 ||| A B D C A ||| -219.045
+0 ||| A C C C A ||| -219.045
+0 ||| A C D B A ||| -219.045
+0 ||| A A C D A ||| -219.045
+0 ||| A D A D A ||| -219.045
+0 ||| A B B D A ||| -219.045
+0 ||| A A E C A ||| -219.045
+0 ||| A C A E A ||| -219.045
+0 ||| E A A B C ||| -220.045
+0 ||| D C A C B ||| -220.045
+0 ||| D A C A C ||| -220.045
+0 ||| B B B C B ||| -220.045
+0 ||| B A C C B ||| -220.045
+0 ||| B A D B B ||| -220.045
+0 ||| B A B C C ||| -220.045
+0 ||| C C B A C ||| -220.045
+0 ||| B A A C D ||| -220.045
+0 ||| D A A D B ||| -220.045
+0 ||| E B B A B ||| -220.045
+0 ||| C C A B C ||| -220.045
+0 ||| B C A C C ||| -220.045
+0 ||| C A B A D ||| -220.045
+0 ||| C A A B D ||| -220.045
+0 ||| C C A A D ||| -220.045
+0 ||| D B B A C ||| -220.045
+0 ||| E C A A C ||| -220.045
+0 ||| B B C A C ||| -220.045
+0 ||| B A A D C ||| -220.045
+0 ||| E A A A D ||| -220.045
+0 ||| B B A D B ||| -220.045
+0 ||| C A D A C ||| -220.045
+0 ||| D A B C B ||| -220.045
+0 ||| D C B C A ||| -220.045
+0 ||| D B A B C ||| -220.045
+0 ||| D B A A D ||| -220.045
+0 ||| E B A B B ||| -220.045
+0 ||| C A C B B ||| -220.045
+0 ||| B A C D A ||| -220.045
+0 ||| B D A C B ||| -220.045
+0 ||| C C C A B ||| -220.045
+0 ||| B A A E B ||| -220.045
+0 ||| B A E C A ||| -220.045
+0 ||| D B C A B ||| -220.045
+0 ||| C A E A B ||| -220.045
+0 ||| B A B E A ||| -220.045
+0 ||| C B B B B ||| -220.045
+0 ||| B C B B B ||| -220.045
+0 ||| C D B B A ||| -220.045
+0 ||| D D A A C ||| -220.045
+0 ||| E D A A B ||| -220.045
+0 ||| B B D C A ||| -220.045
+0 ||| C D A B B ||| -220.045
+0 ||| C A B B C ||| -220.045
+0 ||| B D B C A ||| -220.045
+0 ||| B C D A B ||| -220.045
+0 ||| C D B A B ||| -220.045
+0 ||| D A D C A ||| -220.045
+0 ||| E A B A C ||| -220.045
+0 ||| B A A A E ||| -220.045
+0 ||| D A B D A ||| -220.045
+0 ||| C E A A B ||| -220.045
+0 ||| C B A C C ||| -220.045
+0 ||| B E A C A ||| -220.045
+0 ||| C B D A B ||| -220.045
+0 ||| E A C A B ||| -220.045
+0 ||| C B C C A ||| -220.045
+0 ||| E B D A A ||| -220.045
+0 ||| C A E B A ||| -220.045
+0 ||| E D B A A ||| -220.045
+0 ||| C B A E A ||| -220.045
+0 ||| E E A A A ||| -220.045
+0 ||| B C C C A ||| -220.045
+0 ||| D B E A A ||| -220.045
+0 ||| B B B D A ||| -220.045
+0 ||| C D D A A ||| -220.045
+0 ||| D C A D A ||| -220.045
+0 ||| D D C A A ||| -220.045
+0 ||| C C C B A ||| -220.045
+0 ||| C C E A A ||| -220.045
+0 ||| E B B B A ||| -220.045
+0 ||| C E B A A ||| -220.045
+0 ||| B C D B A ||| -220.045
+0 ||| E C C A A ||| -220.045
+0 ||| C E A B A ||| -220.045
+0 ||| E A E A A ||| -220.045
+0 ||| B C A E A ||| -220.045
+0 ||| D B C B A ||| -220.045
+0 ||| E D A B A ||| -220.045
+0 ||| E A C B A ||| -220.045
+0 ||| C B D B A ||| -220.045
+0 ||| B D A D A ||| -220.045
+0 ||| A E A A C ||| -221.045
+0 ||| A B B A D ||| -221.045
+0 ||| A A B D B ||| -221.045
+0 ||| A B B B C ||| -221.045
+0 ||| A A E A C ||| -221.045
+0 ||| A D B A C ||| -221.045
+0 ||| A A C B C ||| -221.045
+0 ||| A C B C B ||| -221.045
+0 ||| A B D A C ||| -221.045
+0 ||| A D A B C ||| -221.045
+0 ||| A C C A C ||| -221.045
+0 ||| A B E A B ||| -221.045
+0 ||| A A D C B ||| -221.045
+0 ||| A C A D B ||| -221.045
+0 ||| A B C B B ||| -221.045
+0 ||| A A C A D ||| -221.045
+0 ||| A D C A B ||| -221.045
+0 ||| A B A B D ||| -221.045
+0 ||| A D C B A ||| -221.045
+0 ||| A D A A D ||| -221.045
+0 ||| A B E B A ||| -221.045
+0 ||| A C D C A ||| -221.045
+0 ||| A E C A A ||| -221.045
+0 ||| A C B D A ||| -221.045
+0 ||| A D E A A ||| -221.045
+0 ||| A A D D A ||| -221.045
+0 ||| D C B A C ||| -222.045
+0 ||| C C D A B ||| -222.045
+0 ||| B A B D B ||| -222.045
+0 ||| B B C B B ||| -222.045
+0 ||| C A D B B ||| -222.045
+0 ||| E A B B B ||| -222.045
+0 ||| E C A B B ||| -222.045
+0 ||| B D A A D ||| -222.045
+0 ||| D C C A B ||| -222.045
+0 ||| D B D A B ||| -222.045
+0 ||| D D B A B ||| -222.045
+0 ||| D B B B B ||| -222.045
+0 ||| E C B A B ||| -222.045
+0 ||| C A A E B ||| -222.045
+0 ||| B A D C B ||| -222.045
+0 ||| C B B C B ||| -222.045
+0 ||| B B B A D ||| -222.045
+0 ||| B C A D B ||| -222.045
+0 ||| E A D A B ||| -222.045
+0 ||| D E A A B ||| -222.045
+0 ||| C A C C B ||| -222.045
+0 ||| C B A D B ||| -222.045
+0 ||| B B E A B ||| -222.045
+0 ||| B C B C B ||| -222.045
+0 ||| D A E A B ||| -222.045
+0 ||| E B A C B ||| -222.045
+0 ||| D C A B C ||| -222.045
+0 ||| C A B C C ||| -222.045
+0 ||| E A A C C ||| -222.045
+0 ||| B B B B C ||| -222.045
+0 ||| B A E A C ||| -222.045
+0 ||| B D A B C ||| -222.045
+0 ||| C C A C C ||| -222.045
+0 ||| D A B B C ||| -222.045
+0 ||| B E A A C ||| -222.045
+0 ||| B A C B C ||| -222.045
+0 ||| D A D A C ||| -222.045
+0 ||| B D B A C ||| -222.045
+0 ||| C A A D C ||| -222.045
+0 ||| B C C A C ||| -222.045
+0 ||| D B A C C ||| -222.045
+0 ||| C B C A C ||| -222.045
+0 ||| B A C A D ||| -222.045
+0 ||| B B D A C ||| -222.045
+0 ||| D A B A D ||| -222.045
+0 ||| D D A B B ||| -222.045
+0 ||| D A C B B ||| -222.045
+0 ||| B D C A B ||| -222.045
+0 ||| D C A A D ||| -222.045
+0 ||| C C B B B ||| -222.045
+0 ||| D A A B D ||| -222.045
+0 ||| B B A B D ||| -222.045
+0 ||| C A A A E ||| -222.045
+0 ||| C A A C D ||| -222.045
+0 ||| C D A C B ||| -222.045
+0 ||| B A D D A ||| -222.045
+0 ||| C C D B A ||| -222.045
+0 ||| D C C B A ||| -222.045
+0 ||| E A D B A ||| -222.045
+0 ||| B D C B A ||| -222.045
+0 ||| D E A B A ||| -222.045
+0 ||| E C B B A ||| -222.045
+0 ||| B B E B A ||| -222.045
+0 ||| E B A D A ||| -222.045
+0 ||| D A E B A ||| -222.045
+0 ||| C A B E A ||| -222.045
+0 ||| E A A E A ||| -222.045
+0 ||| D B D B A ||| -222.045
+0 ||| C A E C A ||| -222.045
+0 ||| C C A E A ||| -222.045
+0 ||| C B D C A ||| -222.045
+0 ||| B C D C A ||| -222.045
+0 ||| E B B C A ||| -222.045
+0 ||| C D A D A ||| -222.045
+0 ||| D D D A A ||| -222.045
+0 ||| C D B C A ||| -222.045
+0 ||| E C D A A ||| -222.045
+0 ||| E A C C A ||| -222.045
+0 ||| B E C A A ||| -222.045
+0 ||| D B A E A ||| -222.045
+0 ||| D E B A A ||| -222.045
+0 ||| D D B B A ||| -222.045
+0 ||| B D E A A ||| -222.045
+0 ||| E D A C A ||| -222.045
+0 ||| D C E A A ||| -222.045
+0 ||| C E A C A ||| -222.045
+0 ||| B C B D A ||| -222.045
+0 ||| D B C C A ||| -222.045
+0 ||| C B B D A ||| -222.045
+0 ||| C A C D A ||| -222.045
+0 ||| C C C C A ||| -222.045
+0 ||| A C A B D ||| -223.045
+0 ||| A D B B B ||| -223.045
+0 ||| A D D A B ||| -223.045
+0 ||| A B C C B ||| -223.045
+0 ||| A E B B A ||| -223.045
+0 ||| A C C B B ||| -223.045
+0 ||| A A B B D ||| -223.045
+0 ||| A D C C A ||| -223.045
+0 ||| A B D B B ||| -223.045
+0 ||| A A E B B ||| -223.045
+0 ||| A C E B A ||| -223.045
+0 ||| A C E A B ||| -223.045
+0 ||| A E B A B ||| -223.045
+0 ||| A D D B A ||| -223.045
+0 ||| A B A E B ||| -223.045
+0 ||| A B E C A ||| -223.045
+0 ||| A A D A D ||| -223.045
+0 ||| A E D A A ||| -223.045
+0 ||| A B A A E ||| -223.045
+0 ||| A A C E A ||| -223.045
+0 ||| A C B B C ||| -223.045
+0 ||| A B B E A ||| -223.045
+0 ||| A C D A C ||| -223.045
+0 ||| A D A E A ||| -223.045
+0 ||| A B A D C ||| -223.045
+0 ||| A B C D A ||| -223.045
+0 ||| A A A E C ||| -223.045
+0 ||| A B A C D ||| -223.045
+0 ||| A E A B B ||| -223.045
+0 ||| A C B A D ||| -223.045
+0 ||| A A D B C ||| -223.045
+0 ||| A B B C C ||| -223.045
+0 ||| A D A C C ||| -223.045
+0 ||| A A C C C ||| -223.045
+0 ||| B B B C C ||| -224.045
+0 ||| D A C C B ||| -224.045
+0 ||| D C B B B ||| -224.045
+0 ||| B A E B B ||| -224.045
+0 ||| E A C A C ||| -224.045
+0 ||| D C D B A ||| -224.045
+0 ||| E C A C B ||| -224.045
+0 ||| D D A C B ||| -224.045
+0 ||| C C D C A ||| -224.045
+0 ||| B B C C B ||| -224.045
+0 ||| D A A E B ||| -224.045
+0 ||| E B C B A ||| -224.045
+0 ||| C D C A B ||| -224.045
+0 ||| B C A B D ||| -224.045
+0 ||| C C B D A ||| -224.045
+0 ||| C A D C B ||| -224.045
+0 ||| B C B A D ||| -224.045
+0 ||| C D C B A ||| -224.045
+0 ||| C B E A B ||| -224.045
+0 ||| C A B D B ||| -224.045
+0 ||| B B E C A ||| -224.045
+0 ||| C C B C B ||| -224.045
+0 ||| D C A C C ||| -224.045
+0 ||| C B E B A ||| -224.045
+0 ||| B D B B B ||| -224.045
+0 ||| E B C A B ||| -224.045
+0 ||| E A D C A ||| -224.045
+0 ||| D B B C B ||| -224.045
+0 ||| B A B B D ||| -224.045
+0 ||| B B C D A ||| -224.045
+0 ||| B B D B B ||| -224.045
+0 ||| D A D B B ||| -224.045
+0 ||| D C C C A ||| -224.045
+0 ||| D C D A B ||| -224.045
+0 ||| B E A B B ||| -224.045
+0 ||| E C B C A ||| -224.045
+0 ||| C C A D B ||| -224.045
+0 ||| B D D A B ||| -224.045
+0 ||| D A C D A ||| -224.045
+0 ||| E A A D B ||| -224.045
+0 ||| D B A D B ||| -224.045
+0 ||| E C A D A ||| -224.045
+0 ||| B E B A B ||| -224.045
+0 ||| C B C B B ||| -224.045
+0 ||| E A B D A ||| -224.045
+0 ||| B C E A B ||| -224.045
+0 ||| D E A C A ||| -224.045
+0 ||| E A B C B ||| -224.045
+0 ||| B D D B A ||| -224.045
+0 ||| B B A E B ||| -224.045
+0 ||| B D C C A ||| -224.045
+0 ||| B B A C D ||| -224.045
+0 ||| B B B E A ||| -224.045
+0 ||| B C C B B ||| -224.045
+0 ||| B D A E A ||| -224.045
+0 ||| C B A B D ||| -224.045
+0 ||| D A E C A ||| -224.045
+0 ||| C A C A D ||| -224.045
+0 ||| D B D C A ||| -224.045
+0 ||| E B B A C ||| -224.045
+0 ||| B C E B A ||| -224.045
+0 ||| D A A C D ||| -224.045
+0 ||| C D E A A ||| -224.045
+0 ||| B A D A D ||| -224.045
+0 ||| D B B D A ||| -224.045
+0 ||| B A C C C ||| -224.045
+0 ||| E D C A A ||| -224.045
+0 ||| D A A D C ||| -224.045
+0 ||| D D A D A ||| -224.045
+0 ||| C B B A D ||| -224.045
+0 ||| C E C A A ||| -224.045
+0 ||| B A D B C ||| -224.045
+0 ||| D A B E A ||| -224.045
+0 ||| E B A A D ||| -224.045
+0 ||| E B E A A ||| -224.045
+0 ||| B B A D C ||| -224.045
+0 ||| D C A E A ||| -224.045
+0 ||| E B A B C ||| -224.045
+0 ||| B E D A A ||| -224.045
+0 ||| B D A C C ||| -224.045
+0 ||| B E B B A ||| -224.045
+0 ||| D A B C C ||| -224.045
+0 ||| B A C E A ||| -224.045
+0 ||| C D A B C ||| -224.045
+0 ||| D D B C A ||| -224.045
+0 ||| C A C B C ||| -224.045
+0 ||| C A D D A ||| -224.045
+0 ||| C E A A C ||| -224.045
+0 ||| D A A A E ||| -224.045
+0 ||| C B B B C ||| -224.045
+0 ||| B A A E C ||| -224.045
+0 ||| C D A A D ||| -224.045
+0 ||| C C C A C ||| -224.045
+0 ||| E D A A C ||| -224.045
+0 ||| C A E A C ||| -224.045
+0 ||| D B C A C ||| -224.045
+0 ||| B C B B C ||| -224.045
+0 ||| B B A A E ||| -224.045
+0 ||| B C D A C ||| -224.045
+0 ||| C D B A C ||| -224.045
+0 ||| C B D A C ||| -224.045
+0 ||| A C D B B ||| -225.045
+0 ||| A A B C D ||| -225.045
+0 ||| A C C C B ||| -225.045
+0 ||| A A C D B ||| -225.045
+0 ||| A D A D B ||| -225.045
+0 ||| A E A C B ||| -225.045
+0 ||| A B D C B ||| -225.045
+0 ||| A B B D B ||| -225.045
+0 ||| A A E D A ||| -225.045
+0 ||| A A E C B ||| -225.045
+0 ||| A C A E B ||| -225.045
+0 ||| A A B A E ||| -225.045
+0 ||| A C E C A ||| -225.045
+0 ||| A A A D D ||| -225.045
+0 ||| A D D C A ||| -225.045
+0 ||| A A B E B ||| -225.045
+0 ||| A A D E A ||| -225.045
+0 ||| A C A C D ||| -225.045
+0 ||| A C C D A ||| -225.045
+0 ||| A B C A D ||| -225.045
+0 ||| A D B D A ||| -225.045
+0 ||| A D B C B ||| -225.045
+0 ||| A E A D A ||| -225.045
+0 ||| A C A A E ||| -225.045
+0 ||| A C B E A ||| -225.045
+0 ||| A D C A C ||| -225.045
+0 ||| A B D D A ||| -225.045
+0 ||| A A A B E ||| -225.045
+0 ||| A E B C A ||| -225.045
+0 ||| A A B D C ||| -225.045
+0 ||| A C A D C ||| -225.045
+0 ||| A C B C C ||| -225.045
+0 ||| A A D C C ||| -225.045
+0 ||| A B E A C ||| -225.045
+0 ||| A B C B C ||| -225.045
+0 ||| C C E A B ||| -226.045
+0 ||| B B C B C ||| -226.045
+0 ||| B A B D C ||| -226.045
+0 ||| C C D A C ||| -226.045
+0 ||| D C C A C ||| -226.045
+0 ||| C B B C C ||| -226.045
+0 ||| E A B B C ||| -226.045
+0 ||| E C A B C ||| -226.045
+0 ||| B C C C B ||| -226.045
+0 ||| B A D C C ||| -226.045
+0 ||| C B A D C ||| -226.045
+0 ||| D B D A C ||| -226.045
+0 ||| D D B A C ||| -226.045
+0 ||| D B B B C ||| -226.045
+0 ||| C A A E C ||| -226.045
+0 ||| E C B A C ||| -226.045
+0 ||| C D A C C ||| -226.045
+0 ||| C A C C C ||| -226.045
+0 ||| B C A D C ||| -226.045
+0 ||| E E A A B ||| -226.045
+0 ||| C B A E B ||| -226.045
+0 ||| D E A A C ||| -226.045
+0 ||| E A D A C ||| -226.045
+0 ||| C B C C B ||| -226.045
+0 ||| E B D A B ||| -226.045
+0 ||| B B E A C ||| -226.045
+0 ||| D B E A B ||| -226.045
+0 ||| C D D A B ||| -226.045
+0 ||| B C B C C ||| -226.045
+0 ||| C A E B B ||| -226.045
+0 ||| B B B D B ||| -226.045
+0 ||| C C C B B ||| -226.045
+0 ||| D A C A D ||| -226.045
+0 ||| D A E A C ||| -226.045
+0 ||| C B A A E ||| -226.045
+0 ||| C A D B C ||| -226.045
+0 ||| E D B A B ||| -226.045
+0 ||| E B A C C ||| -226.045
+0 ||| D C D C A ||| -226.045
+0 ||| D D C A B ||| -226.045
+0 ||| B A A B E ||| -226.045
+0 ||| E A A B D ||| -226.045
+0 ||| E B A E A ||| -226.045
+0 ||| D C A D B ||| -226.045
+0 ||| D A D D A ||| -226.045
+0 ||| C C A B D ||| -226.045
+0 ||| E D B B A ||| -226.045
+0 ||| B A C D B ||| -226.045
+0 ||| B C E C A ||| -226.045
+0 ||| B A B C D ||| -226.045
+0 ||| C D A E A ||| -226.045
+0 ||| C C B A D ||| -226.045
+0 ||| B A D E A ||| -226.045
+0 ||| B B C A D ||| -226.045
+0 ||| C E B B A ||| -226.045
+0 ||| B C A C D ||| -226.045
+0 ||| C A C E A ||| -226.045
+0 ||| E C A A D ||| -226.045
+0 ||| C D C C A ||| -226.045
+0 ||| D B B A D ||| -226.045
+0 ||| E B D B A ||| -226.045
+0 ||| B A E C B ||| -226.045
+0 ||| E B C C A ||| -226.045
+0 ||| C A D A D ||| -226.045
+0 ||| C B E C A ||| -226.045
+0 ||| B A B E B ||| -226.045
+0 ||| E E A B A ||| -226.045
+0 ||| B C D B B ||| -226.045
+0 ||| D B E B A ||| -226.045
+0 ||| B D B C B ||| -226.045
+0 ||| D D C B A ||| -226.045
+0 ||| C D B B B ||| -226.045
+0 ||| C D D B A ||| -226.045
+0 ||| B B D C B ||| -226.045
+0 ||| C C E B A ||| -226.045
+0 ||| B E A C B ||| -226.045
+0 ||| E C C B A ||| -226.045
+0 ||| D A B D B ||| -226.045
+0 ||| B E B C A ||| -226.045
+0 ||| C E B A B ||| -226.045
+0 ||| B D D C A ||| -226.045
+0 ||| D B A B D ||| -226.045
+0 ||| D E C A A ||| -226.045
+0 ||| E B B B B ||| -226.045
+0 ||| B E A D A ||| -226.045
+0 ||| B C A A E ||| -226.045
+0 ||| D D E A A ||| -226.045
+0 ||| E A C B B ||| -226.045
+0 ||| C B B E A ||| -226.045
+0 ||| C E A B B ||| -226.045
+0 ||| E C E A A ||| -226.045
+0 ||| B A B A E ||| -226.045
+0 ||| B B D D A ||| -226.045
+0 ||| E C C A B ||| -226.045
+0 ||| C E D A A ||| -226.045
+0 ||| E D A B B ||| -226.045
+0 ||| C B C D A ||| -226.045
+0 ||| B A A D D ||| -226.045
+0 ||| E E B A A ||| -226.045
+0 ||| E A E A B ||| -226.045
+0 ||| B A E D A ||| -226.045
+0 ||| D B C B B ||| -226.045
+0 ||| E D D A A ||| -226.045
+0 ||| B C A E B ||| -226.045
+0 ||| B C C D A ||| -226.045
+0 ||| B D A D B ||| -226.045
+0 ||| D C B D A ||| -226.045
+0 ||| C B D B B ||| -226.045
+0 ||| B D B D A ||| -226.045
+0 ||| D C B C B ||| -226.045
+0 ||| B C B E A ||| -226.045
+0 ||| D D A A D ||| -226.045
+0 ||| E A E B A ||| -226.045
+0 ||| C A B B D ||| -226.045
+0 ||| E A B A D ||| -226.045
+0 ||| B D C A C ||| -226.045
+0 ||| D A D C B ||| -226.045
+0 ||| D A C B C ||| -226.045
+0 ||| D D A B C ||| -226.045
+0 ||| C C B B C ||| -226.045
+0 ||| C B A C D ||| -226.045
+0 ||| A C E A C ||| -227.045
+0 ||| A B D B C ||| -227.045
+0 ||| A E B A C ||| -227.045
+0 ||| A A E B C ||| -227.045
+0 ||| A D D A C ||| -227.045
+0 ||| A B C C C ||| -227.045
+0 ||| A E C A B ||| -227.045
+0 ||| A A D D B ||| -227.045
+0 ||| A D E A B ||| -227.045
+0 ||| A D B B C ||| -227.045
+0 ||| A C D C B ||| -227.045
+0 ||| A B E B B ||| -227.045
+0 ||| A D E B A ||| -227.045
+0 ||| A D C B B ||| -227.045
+0 ||| A B C E A ||| -227.045
+0 ||| A B A E C ||| -227.045
+0 ||| A E C B A ||| -227.045
+0 ||| A A A C E ||| -227.045
+0 ||| A E E A A ||| -227.045
+0 ||| A A C B D ||| -227.045
+0 ||| A C D D A ||| -227.045
+0 ||| A D B A D ||| -227.045
+0 ||| A E A A D ||| -227.045
+0 ||| A C C B C ||| -227.045
+0 ||| A A E A D ||| -227.045
+0 ||| A B B B D ||| -227.045
+0 ||| A D A B D ||| -227.045
+0 ||| A B D A D ||| -227.045
+0 ||| A C C A D ||| -227.045
+0 ||| A E A B C ||| -227.045
+0 ||| A C B D B ||| -227.045
+0 ||| E C B B B ||| -228.045
+0 ||| E C D A B ||| -228.045
+0 ||| B B E B B ||| -228.045
+0 ||| C A B E B ||| -228.045
+0 ||| E B A D B ||| -228.045
+0 ||| D A E B B ||| -228.045
+0 ||| C B D C B ||| -228.045
+0 ||| E A A E B ||| -228.045
+0 ||| D B D B B ||| -228.045
+0 ||| C D B C B ||| -228.045
+0 ||| C A E C B ||| -228.045
+0 ||| C C A E B ||| -228.045
+0 ||| B C D C B ||| -228.045
+0 ||| D D D A B ||| -228.045
+0 ||| E B B C B ||| -228.045
+0 ||| C D A D B ||| -228.045
+0 ||| B E C A B ||| -228.045
+0 ||| B D E A B ||| -228.045
+0 ||| E A C C B ||| -228.045
+0 ||| D B A E B ||| -228.045
+0 ||| D E B A B ||| -228.045
+0 ||| D D B B B ||| -228.045
+0 ||| B C B D B ||| -228.045
+0 ||| C E A C B ||| -228.045
+0 ||| E D A C B ||| -228.045
+0 ||| D C E A B ||| -228.045
+0 ||| D B C C B ||| -228.045
+0 ||| C A C D B ||| -228.045
+0 ||| D C B A D ||| -228.045
+0 ||| C B B D B ||| -228.045
+0 ||| C C C C B ||| -228.045
+0 ||| D C B B C ||| -228.045
+0 ||| D A C C C ||| -228.045
+0 ||| D C D A C ||| -228.045
+0 ||| C B C B C ||| -228.045
+0 ||| E A A D C ||| -228.045
+0 ||| B A D D B ||| -228.045
+0 ||| D C C B B ||| -228.045
+0 ||| C C D B B ||| -228.045
+0 ||| B D C B B ||| -228.045
+0 ||| B E B A C ||| -228.045
+0 ||| C B C A D ||| -228.045
+0 ||| B D A B D ||| -228.045
+0 ||| E A A C D ||| -228.045
+0 ||| C A B C D ||| -228.045
+0 ||| E C A C C ||| -228.045
+0 ||| D C A B D ||| -228.045
+0 ||| B B C C C ||| -228.045
+0 ||| B A E A D ||| -228.045
+0 ||| D D A C C ||| -228.045
+0 ||| B B B B D ||| -228.045
+0 ||| C D C A C ||| -228.045
+0 ||| D A A E C ||| -228.045
+0 ||| E A D B B ||| -228.045
+0 ||| D B A C D ||| -228.045
+0 ||| B A C B D ||| -228.045
+0 ||| C B E A C ||| -228.045
+0 ||| B D B B C ||| -228.045
+0 ||| D B B C C ||| -228.045
+0 ||| B E A A D ||| -228.045
+0 ||| D A B B D ||| -228.045
+0 ||| D B A D C ||| -228.045
+0 ||| E B C A C ||| -228.045
+0 ||| D E A B B ||| -228.045
+0 ||| B A E B C ||| -228.045
+0 ||| B C C A D ||| -228.045
+0 ||| C C A C D ||| -228.045
+0 ||| C C B C C ||| -228.045
+0 ||| D A D A D ||| -228.045
+0 ||| B B D B C ||| -228.045
+0 ||| E D B C A ||| -228.045
+0 ||| B D D A C ||| -228.045
+0 ||| B E A B C ||| -228.045
+0 ||| C A A D D ||| -228.045
+0 ||| C C A D C ||| -228.045
+0 ||| B D B A D ||| -228.045
+0 ||| B E C B A ||| -228.045
+0 ||| E A B C C ||| -228.045
+0 ||| D D D B A ||| -228.045
+0 ||| B C E A C ||| -228.045
+0 ||| C C E C A ||| -228.045
+0 ||| B B A E C ||| -228.045
+0 ||| D E B B A ||| -228.045
+0 ||| B C C B C ||| -228.045
+0 ||| E A B E A ||| -228.045
+0 ||| B A A C E ||| -228.045
+0 ||| E B B D A ||| -228.045
+0 ||| C A A B E ||| -228.045
+0 ||| D E D A A ||| -228.045
+0 ||| C A B A E ||| -228.045
+0 ||| E C A E A ||| -228.045
+0 ||| C C A A E ||| -228.045
+0 ||| E C D B A ||| -228.045
+0 ||| E A A A E ||| -228.045
+0 ||| B E E A A ||| -228.045
+0 ||| D B A A E ||| -228.045
+0 ||| C C B E A ||| -228.045
+0 ||| C A B D C ||| -228.045
+0 ||| C E B C A ||| -228.045
+0 ||| D A D B C ||| -228.045
+0 ||| D A C E A ||| -228.045
+0 ||| B B D A D ||| -228.045
+0 ||| B B C E A ||| -228.045
+0 ||| C A D C C ||| -228.045
+0 ||| E A E C A ||| -228.045
+0 ||| D B E C A ||| -228.045
+0 ||| B D E B A ||| -228.045
+0 ||| E B D C A ||| -228.045
+0 ||| C A E D A ||| -228.045
+0 ||| C D D C A ||| -228.045
+0 ||| E E A C A ||| -228.045
+0 ||| D D C C A ||| -228.045
+0 ||| D B B E A ||| -228.045
+0 ||| C A D E A ||| -228.045
+0 ||| C B D D A ||| -228.045
+0 ||| E C C C A ||| -228.045
+0 ||| C C C D A ||| -228.045
+0 ||| E A C D A ||| -228.045
+0 ||| C D B D A ||| -228.045
+0 ||| D D A E A ||| -228.045
+0 ||| B C D D A ||| -228.045
+0 ||| D B C D A ||| -228.045
+0 ||| D C E B A ||| -228.045
+0 ||| E D A D A ||| -228.045
+0 ||| C E A D A ||| -228.045
+0 ||| A B E C B ||| -229.045
+0 ||| A A C E B ||| -229.045
+0 ||| A D A E B ||| -229.045
+0 ||| A C B B D ||| -229.045
+0 ||| A C D B C ||| -229.045
+0 ||| A D C C B ||| -229.045
+0 ||| A C C C C ||| -229.045
+0 ||| A A C C D ||| -229.045
+0 ||| A D D B B ||| -229.045
+0 ||| A D A C D ||| -229.045
+0 ||| A B B C D ||| -229.045
+0 ||| A C E B B ||| -229.045
+0 ||| A A C D C ||| -229.045
+0 ||| A B B E B ||| -229.045
+0 ||| A D A D C ||| -229.045
+0 ||| A E A C C ||| -229.045
+0 ||| A B D C C ||| -229.045
+0 ||| A A B E C ||| -229.045
+0 ||| A C D A D ||| -229.045
+0 ||| A A E C C ||| -229.045
+0 ||| A E B B B ||| -229.045
+0 ||| A E D A B ||| -229.045
+0 ||| A B D E A ||| -229.045
+0 ||| A C A E C ||| -229.045
+0 ||| A B A B E ||| -229.045
+0 ||| A D E C A ||| -229.045
+0 ||| A A C A E ||| -229.045
+0 ||| A D B C C ||| -229.045
+0 ||| A C C E A ||| -229.045
+0 ||| A A D B D ||| -229.045
+0 ||| A B B D C ||| -229.045
+0 ||| A A E E A ||| -229.045
+0 ||| A B B A E ||| -229.045
+0 ||| A D A A E ||| -229.045
+0 ||| A D B E A ||| -229.045
+0 ||| A B A D D ||| -229.045
+0 ||| A A A E D ||| -229.045
+0 ||| A B E D A ||| -229.045
+0 ||| A B C D B ||| -229.045
+0 ||| A E C C A ||| -229.045
+0 ||| A E A E A ||| -229.045
+0 ||| A D C D A ||| -229.045
+0 ||| A E D B A ||| -229.045
+0 ||| B A C A E ||| -230.045
+0 ||| E E A A C ||| -230.045
+0 ||| D C D B B ||| -230.045
+0 ||| E B C B B ||| -230.045
+0 ||| C C D C B ||| -230.045
+0 ||| D C C C B ||| -230.045
+0 ||| B B B C D ||| -230.045
+0 ||| C C E A C ||| -230.045
+0 ||| B B C D B ||| -230.045
+0 ||| B C C C C ||| -230.045
+0 ||| C D C B B ||| -230.045
+0 ||| E A D C B ||| -230.045
+0 ||| C B E B B ||| -230.045
+0 ||| B B E C B ||| -230.045
+0 ||| B D C C B ||| -230.045
+0 ||| E C A D B ||| -230.045
+0 ||| C E C A B ||| -230.045
+0 ||| E C B C B ||| -230.045
+0 ||| B D D B B ||| -230.045
+0 ||| D E A C B ||| -230.045
+0 ||| E A B D B ||| -230.045
+0 ||| D B D C B ||| -230.045
+0 ||| B B B E B ||| -230.045
+0 ||| D D A D B ||| -230.045
+0 ||| D A E C B ||| -230.045
+0 ||| B D A E B ||| -230.045
+0 ||| E D C A B ||| -230.045
+0 ||| B C E B B ||| -230.045
+0 ||| D B B D B ||| -230.045
+0 ||| C D E A B ||| -230.045
+0 ||| C A E B C ||| -230.045
+0 ||| D C A E B ||| -230.045
+0 ||| C D B A D ||| -230.045
+0 ||| B B B D C ||| -230.045
+0 ||| D D B C B ||| -230.045
+0 ||| B E D A B ||| -230.045
+0 ||| B A C E B ||| -230.045
+0 ||| D A B E B ||| -230.045
+0 ||| C B D A D ||| -230.045
+0 ||| B E B B B ||| -230.045
+0 ||| C B A E C ||| -230.045
+0 ||| C D D A C ||| -230.045
+0 ||| C B C C C ||| -230.045
+0 ||| D B E A C ||| -230.045
+0 ||| E B D A C ||| -230.045
+0 ||| C C C B C ||| -230.045
+0 ||| D D C A C ||| -230.045
+0 ||| C D B B C ||| -230.045
+0 ||| D C A C D ||| -230.045
+0 ||| B E A C C ||| -230.045
+0 ||| C E B A C ||| -230.045
+0 ||| B B B A E ||| -230.045
+0 ||| C B D B C ||| -230.045
+0 ||| E D B A C ||| -230.045
+0 ||| D A C D B ||| -230.045
+0 ||| C C B D B ||| -230.045
+0 ||| B D A A E ||| -230.045
+0 ||| E B E A B ||| -230.045
+0 ||| C A A C E ||| -230.045
+0 ||| D C A D C ||| -230.045
+0 ||| B A C C D ||| -230.045
+0 ||| B C D A D ||| -230.045
+0 ||| D A A D D ||| -230.045
+0 ||| C A D D B ||| -230.045
+0 ||| E B B A D ||| -230.045
+0 ||| E D E A A ||| -230.045
+0 ||| D A B D C ||| -230.045
+0 ||| B A C D C ||| -230.045
+0 ||| C A C B D ||| -230.045
+0 ||| B E C C A ||| -230.045
+0 ||| B A D B D ||| -230.045
+0 ||| B A E C C ||| -230.045
+0 ||| C D A B D ||| -230.045
+0 ||| C B C E A ||| -230.045
+0 ||| B C D B C ||| -230.045
+0 ||| B A B E C ||| -230.045
+0 ||| B B D C C ||| -230.045
+0 ||| C D E B A ||| -230.045
+0 ||| D A B C D ||| -230.045
+0 ||| E B A B D ||| -230.045
+0 ||| B B A D D ||| -230.045
+0 ||| C E E A A ||| -230.045
+0 ||| B D A C D ||| -230.045
+0 ||| B D B C C ||| -230.045
+0 ||| C E C B A ||| -230.045
+0 ||| E B B B C ||| -230.045
+0 ||| C C C A D ||| -230.045
+0 ||| D A D E A ||| -230.045
+0 ||| C E A A D ||| -230.045
+0 ||| E D A B C ||| -230.045
+0 ||| D E B C A ||| -230.045
+0 ||| B D A D C ||| -230.045
+0 ||| E C C A C ||| -230.045
+0 ||| E E C A A ||| -230.045
+0 ||| E A C B C ||| -230.045
+0 ||| D C A A E ||| -230.045
+0 ||| D D D C A ||| -230.045
+0 ||| C E A B C ||| -230.045
+0 ||| B C A E C ||| -230.045
+0 ||| E D C B A ||| -230.045
+0 ||| C B B B D ||| -230.045
+0 ||| E A C A D ||| -230.045
+0 ||| B D C D A ||| -230.045
+0 ||| D B C B C ||| -230.045
+0 ||| B A A E D ||| -230.045
+0 ||| C C D D A ||| -230.045
+0 ||| E A E A C ||| -230.045
+0 ||| B C B B D ||| -230.045
+0 ||| E C D C A ||| -230.045
+0 ||| D A D C C ||| -230.045
+0 ||| C A E A D ||| -230.045
+0 ||| B B E D A ||| -230.045
+0 ||| E D A A D ||| -230.045
+0 ||| B B A B E ||| -230.045
+0 ||| E C B D A ||| -230.045
+0 ||| D B C A D ||| -230.045
+0 ||| D A B A E ||| -230.045
+0 ||| E B E B A ||| -230.045
+0 ||| D A A B E ||| -230.045
+0 ||| D C B C C ||| -230.045
+0 ||| D B D D A ||| -230.045
+0 ||| D C E C A ||| -230.045
+0 ||| B D E C A ||| -230.045
+0 ||| D E A D A ||| -230.045
+0 ||| D C C D A ||| -230.045
+0 ||| D D B D A ||| -230.045
+0 ||| D A E D A ||| -230.045
+0 ||| B B D E A ||| -230.045
+0 ||| B E A E A ||| -230.045
+0 ||| E A D D A ||| -230.045
+0 ||| B D B E A ||| -230.045
+0 ||| B A E E A ||| -230.045
+0 ||| D C B E A ||| -230.045
+0 ||| B E D B A ||| -230.045
+0 ||| B C C E A ||| -230.045
+0 ||| A D E A C ||| -231.045
+0 ||| A A D D C ||| -231.045
+0 ||| A E C A C ||| -231.045
+0 ||| A A B B E ||| -231.045
+0 ||| A C A B E ||| -231.045
+0 ||| A C B E B ||| -231.045
+0 ||| A B C B D ||| -231.045
+0 ||| A A D A E ||| -231.045
+0 ||| A D C B C ||| -231.045
+0 ||| A A D C D ||| -231.045
+0 ||| A B E B C ||| -231.045
+0 ||| A E A D B ||| -231.045
+0 ||| A C B D C ||| -231.045
+0 ||| A D B D B ||| -231.045
+0 ||| A C D C C ||| -231.045
+0 ||| A D D C B ||| -231.045
+0 ||| A C C D B ||| -231.045
+0 ||| A A D E B ||| -231.045
+0 ||| A C E C B ||| -231.045
+0 ||| A B D D B ||| -231.045
+0 ||| A B E A D ||| -231.045
+0 ||| A E D C A ||| -231.045
+0 ||| A A E D B ||| -231.045
+0 ||| A C B C D ||| -231.045
+0 ||| A C E D A ||| -231.045
+0 ||| A E B C B ||| -231.045
+0 ||| A C A D D ||| -231.045
+0 ||| A E B D A ||| -231.045
+0 ||| A C B A E ||| -231.045
+0 ||| A D C A D ||| -231.045
+0 ||| A C D E A ||| -231.045
+0 ||| A A B D D ||| -231.045
+0 ||| A B A C E ||| -231.045
+0 ||| A D D D A ||| -231.045
+0 ||| B A B B E ||| -232.045
+0 ||| C A C A E ||| -232.045
+0 ||| D E C B A ||| -232.045
+0 ||| D C E A C ||| -232.045
+0 ||| C A C E B ||| -232.045
+0 ||| B C E C B ||| -232.045
+0 ||| C A E C C ||| -232.045
+0 ||| C D B C C ||| -232.045
+0 ||| E C B B C ||| -232.045
+0 ||| E B A D C ||| -232.045
+0 ||| C A B E C ||| -232.045
+0 ||| E C D A C ||| -232.045
+0 ||| B B E B C ||| -232.045
+0 ||| D A E B C ||| -232.045
+0 ||| D B D B C ||| -232.045
+0 ||| E A A E C ||| -232.045
+0 ||| C B D C C ||| -232.045
+0 ||| E D A C C ||| -232.045
+0 ||| C D A D C ||| -232.045
+0 ||| E B B C C ||| -232.045
+0 ||| C C A E C ||| -232.045
+0 ||| E D B B B ||| -232.045
+0 ||| D D D A C ||| -232.045
+0 ||| B C D C C ||| -232.045
+0 ||| C E A C C ||| -232.045
+0 ||| E A C C C ||| -232.045
+0 ||| B D E A C ||| -232.045
+0 ||| D A D D B ||| -232.045
+0 ||| B E C A C ||| -232.045
+0 ||| B C B D C ||| -232.045
+0 ||| E B A E B ||| -232.045
+0 ||| D D B B C ||| -232.045
+0 ||| D B A E C ||| -232.045
+0 ||| D E B A C ||| -232.045
+0 ||| C E B B B ||| -232.045
+0 ||| B A D E B ||| -232.045
+0 ||| C B A D D ||| -232.045
+0 ||| B B C B D ||| -232.045
+0 ||| C D A E B ||| -232.045
+0 ||| D B C C C ||| -232.045
+0 ||| B C A D D ||| -232.045
+0 ||| B C B A E ||| -232.045
+0 ||| B A D C D ||| -232.045
+0 ||| C B B C D ||| -232.045
+0 ||| D C C A D ||| -232.045
+0 ||| B A B D D ||| -232.045
+0 ||| C C D A D ||| -232.045
+0 ||| E A B B D ||| -232.045
+0 ||| E C A B D ||| -232.045
+0 ||| C B B D C ||| -232.045
+0 ||| D B D A D ||| -232.045
+0 ||| D B B B D ||| -232.045
+0 ||| C A C C D ||| -232.045
+0 ||| D D B A D ||| -232.045
+0 ||| C A C D C ||| -232.045
+0 ||| C A A E D ||| -232.045
+0 ||| C D A C D ||| -232.045
+0 ||| E C B A D ||| -232.045
+0 ||| C C C C C ||| -232.045
+0 ||| B A D A E ||| -232.045
+0 ||| E C E A B ||| -232.045
+0 ||| B A D D C ||| -232.045
+0 ||| B C B C D ||| -232.045
+0 ||| D E A A D ||| -232.045
+0 ||| D C D C B ||| -232.045
+0 ||| E A D A D ||| -232.045
+0 ||| B B E A D ||| -232.045
+0 ||| C B B E B ||| -232.045
+0 ||| E B D B B ||| -232.045
+0 ||| D D C B B ||| -232.045
+0 ||| C D C C B ||| -232.045
+0 ||| B D C B C ||| -232.045
+0 ||| D C C B C ||| -232.045
+0 ||| E B C C B ||| -232.045
+0 ||| D B E B B ||| -232.045
+0 ||| C C D B C ||| -232.045
+0 ||| E E A B B ||| -232.045
+0 ||| C B E C B ||| -232.045
+0 ||| D D E A B ||| -232.045
+0 ||| B C A B E ||| -232.045
+0 ||| E C C B B ||| -232.045
+0 ||| C C E B B ||| -232.045
+0 ||| C D D B B ||| -232.045
+0 ||| B E B C B ||| -232.045
+0 ||| B E A D B ||| -232.045
+0 ||| D E C A B ||| -232.045
+0 ||| B D D C B ||| -232.045
+0 ||| B B D D B ||| -232.045
+0 ||| B C C D B ||| -232.045
+0 ||| E D D A B ||| -232.045
+0 ||| C E D A B ||| -232.045
+0 ||| B A E D B ||| -232.045
+0 ||| C B C D B ||| -232.045
+0 ||| E E B A B ||| -232.045
+0 ||| D C B D B ||| -232.045
+0 ||| C C B B D ||| -232.045
+0 ||| B C B E B ||| -232.045
+0 ||| B D B D B ||| -232.045
+0 ||| E A D B C ||| -232.045
+0 ||| D E A B C ||| -232.045
+0 ||| D A A C E ||| -232.045
+0 ||| B B A C E ||| -232.045
+0 ||| E B A A E ||| -232.045
+0 ||| C B A B E ||| -232.045
+0 ||| D D A B D ||| -232.045
+0 ||| D A E A D ||| -232.045
+0 ||| D D E B A ||| -232.045
+0 ||| C B B A E ||| -232.045
+0 ||| C A D B D ||| -232.045
+0 ||| C D C D A ||| -232.045
+0 ||| D A C B D ||| -232.045
+0 ||| C D A A E ||| -232.045
+0 ||| D C D D A ||| -232.045
+0 ||| E B A C D ||| -232.045
+0 ||| E A E B B ||| -232.045
+0 ||| C D E C A ||| -232.045
+0 ||| B D C A D ||| -232.045
+0 ||| C E C C A ||| -232.045
+0 ||| B E D C A ||| -232.045
+0 ||| B C E D A ||| -232.045
+0 ||| E D C C A ||| -232.045
+0 ||| D E E A A ||| -232.045
+0 ||| E E B B A ||| -232.045
+0 ||| E B E C A ||| -232.045
+0 ||| E E D A A ||| -232.045
+0 ||| C A E E A ||| -232.045
+0 ||| C E A E A ||| -232.045
+0 ||| E D A E A ||| -232.045
+0 ||| C B D E A ||| -232.045
+0 ||| B E B D A ||| -232.045
+0 ||| C B E D A ||| -232.045
+0 ||| E C E B A ||| -232.045
+0 ||| C E D B A ||| -232.045
+0 ||| C C C E A ||| -232.045
+0 ||| E B B E A ||| -232.045
+0 ||| B D D D A ||| -232.045
+0 ||| E B C D A ||| -232.045
+0 ||| D B C E A ||| -232.045
+0 ||| E A C E A ||| -232.045
+0 ||| C D B E A ||| -232.045
+0 ||| B C D E A ||| -232.045
+0 ||| E D D B A ||| -232.045
+0 ||| A B C C D ||| -233.045
+0 ||| A A C E C ||| -233.045
+0 ||| A C E B C ||| -233.045
+0 ||| A C A C E ||| -233.045
+0 ||| A B E C C ||| -233.045
+0 ||| A D B B D ||| -233.045
+0 ||| A E A B D ||| -233.045
+0 ||| A D E B B ||| -233.045
+0 ||| A A B C E ||| -233.045
+0 ||| A E E A B ||| -233.045
+0 ||| A E C B B ||| -233.045
+0 ||| A B C E B ||| -233.045
+0 ||| A B C D C ||| -233.045
+0 ||| A E B A D ||| -233.045
+0 ||| A A E B D ||| -233.045
+0 ||| A D D A D ||| -233.045
+0 ||| A B B E C ||| -233.045
+0 ||| A B D B D ||| -233.045
+0 ||| A D D B C ||| -233.045
+0 ||| A D A E C ||| -233.045
+0 ||| A E D A C ||| -233.045
+0 ||| A E B B C ||| -233.045
+0 ||| A D C C C ||| -233.045
+0 ||| A A A D E ||| -233.045
+0 ||| A B A E D ||| -233.045
+0 ||| A B C A E ||| -233.045
+0 ||| A C E A D ||| -233.045
+0 ||| A D C E A ||| -233.045
+0 ||| A C D D B ||| -233.045
+0 ||| A C C B D ||| -233.045
+0 ||| A E E B A ||| -233.045
+0 ||| A B E E A ||| -233.045
+0 ||| C D B D B ||| -234.045
+0 ||| C C E C B ||| -234.045
+0 ||| C C B E B ||| -234.045
+0 ||| E A A D D ||| -234.045
+0 ||| C C A D D ||| -234.045
+0 ||| B E C B B ||| -234.045
+0 ||| E A A B E ||| -234.045
+0 ||| B B C D C ||| -234.045
+0 ||| E C D B B ||| -234.045
+0 ||| E B B D B ||| -234.045
+0 ||| E A B E B ||| -234.045
+0 ||| B E E A B ||| -234.045
+0 ||| C B C B D ||| -234.045
+0 ||| D E D A B ||| -234.045
+0 ||| E C A E B ||| -234.045
+0 ||| B E A B D ||| -234.045
+0 ||| E B C B C ||| -234.045
+0 ||| D C D B C ||| -234.045
+0 ||| D C C C C ||| -234.045
+0 ||| C C D C C ||| -234.045
+0 ||| B D A E C ||| -234.045
+0 ||| B D C C C ||| -234.045
+0 ||| D A E C C ||| -234.045
+0 ||| C D C B C ||| -234.045
+0 ||| D C D A D ||| -234.045
+0 ||| B B E C C ||| -234.045
+0 ||| C B E B C ||| -234.045
+0 ||| E A D C C ||| -234.045
+0 ||| D D A D C ||| -234.045
+0 ||| D A C C D ||| -234.045
+0 ||| B D D B C ||| -234.045
+0 ||| E C B C C ||| -234.045
+0 ||| E C A D C ||| -234.045
+0 ||| C E C A C ||| -234.045
+0 ||| B B B E C ||| -234.045
+0 ||| D E A C C ||| -234.045
+0 ||| D C B B D ||| -234.045
+0 ||| D B D C C ||| -234.045
+0 ||| E A B D C ||| -234.045
+0 ||| E A E C B ||| -234.045
+0 ||| B E B B C ||| -234.045
+0 ||| C A D C D ||| -234.045
+0 ||| D C A E C ||| -234.045
+0 ||| B C E B C ||| -234.045
+0 ||| E A C D B ||| -234.045
+0 ||| D B B D C ||| -234.045
+0 ||| C D E A C ||| -234.045
+0 ||| C E B C B ||| -234.045
+0 ||| D A B E C ||| -234.045
+0 ||| D A C E B ||| -234.045
+0 ||| D D B C C ||| -234.045
+0 ||| B A C E C ||| -234.045
+0 ||| B E D A C ||| -234.045
+0 ||| D A C A E ||| -234.045
+0 ||| B B C C D ||| -234.045
+0 ||| B E B A D ||| -234.045
+0 ||| E C A C D ||| -234.045
+0 ||| C C C D B ||| -234.045
+0 ||| B D B B D ||| -234.045
+0 ||| D D A C D ||| -234.045
+0 ||| C B E A D ||| -234.045
+0 ||| D A A E D ||| -234.045
+0 ||| C D C A D ||| -234.045
+0 ||| C B A C E ||| -234.045
+0 ||| D B B C D ||| -234.045
+0 ||| E C C C B ||| -234.045
+0 ||| B D D A D ||| -234.045
+0 ||| E B C A D ||| -234.045
+0 ||| D B A D D ||| -234.045
+0 ||| B A E B D ||| -234.045
+0 ||| C B D D B ||| -234.045
+0 ||| E B D C B ||| -234.045
+0 ||| E D B C B ||| -234.045
+0 ||| B B D B D ||| -234.045
+0 ||| B D E B B ||| -234.045
+0 ||| D D D B B ||| -234.045
+0 ||| D B E C B ||| -234.045
+0 ||| C A D E B ||| -234.045
+0 ||| E E A C B ||| -234.045
+0 ||| C A E D B ||| -234.045
+0 ||| D B B E B ||| -234.045
+0 ||| C C B C D ||| -234.045
+0 ||| D D C C B ||| -234.045
+0 ||| E D C A C ||| -234.045
+0 ||| D B C D B ||| -234.045
+0 ||| B C D D B ||| -234.045
+0 ||| D D A E B ||| -234.045
+0 ||| B C E A D ||| -234.045
+0 ||| D C E B B ||| -234.045
+0 ||| C E A D B ||| -234.045
+0 ||| C D D C B ||| -234.045
+0 ||| E D A D B ||| -234.045
+0 ||| C C A B E ||| -234.045
+0 ||| E A B C D ||| -234.045
+0 ||| C C B D C ||| -234.045
+0 ||| D A C D C ||| -234.045
+0 ||| E B E A C ||| -234.045
+0 ||| E C A A E ||| -234.045
+0 ||| B A B C E ||| -234.045
+0 ||| B B A E D ||| -234.045
+0 ||| D B B A E ||| -234.045
+0 ||| D E B B B ||| -234.045
+0 ||| C A D D C ||| -234.045
+0 ||| B C C B D ||| -234.045
+0 ||| B B C A E ||| -234.045
+0 ||| C A D A E ||| -234.045
+0 ||| B B C E B ||| -234.045
+0 ||| D B A B E ||| -234.045
+0 ||| D A D B D ||| -234.045
+0 ||| E D D C A ||| -234.045
+0 ||| B A A D E ||| -234.045
+0 ||| C C B A E ||| -234.045
+0 ||| C C D E A ||| -234.045
+0 ||| B C A C E ||| -234.045
+0 ||| C A B D D ||| -234.045
+0 ||| C E B D A ||| -234.045
+0 ||| D D A A E ||| -234.045
+0 ||| E A B A E ||| -234.045
+0 ||| E D B D A ||| -234.045
+0 ||| C A B B E ||| -234.045
+0 ||| E C E C A ||| -234.045
+0 ||| B E E B A ||| -234.045
+0 ||| D D E C A ||| -234.045
+0 ||| E C B E A ||| -234.045
+0 ||| D D C D A ||| -234.045
+0 ||| D B D E A ||| -234.045
+0 ||| B B E E A ||| -234.045
+0 ||| E A E D A ||| -234.045
+0 ||| E E A D A ||| -234.045
+0 ||| E B D D A ||| -234.045
+0 ||| E E B C A ||| -234.045
+0 ||| C D D D A ||| -234.045
+0 ||| D E D B A ||| -234.045
+0 ||| D D B E A ||| -234.045
+0 ||| D E C C A ||| -234.045
+0 ||| C E D C A ||| -234.045
+0 ||| D C C E A ||| -234.045
+0 ||| E C C D A ||| -234.045
+0 ||| D E A E A ||| -234.045
+0 ||| D A E E A ||| -234.045
+0 ||| B D C E A ||| -234.045
+0 ||| E A D E A ||| -234.045
+0 ||| C C E D A ||| -234.045
+0 ||| D B E D A ||| -234.045
+0 ||| A C C A E ||| -235.045
+0 ||| A C B E C ||| -235.045
+0 ||| A B D E B ||| -235.045
+0 ||| A C C D C ||| -235.045
+0 ||| A D A D D ||| -235.045
+0 ||| A B E D B ||| -235.045
+0 ||| A C C C D ||| -235.045
+0 ||| A C D B D ||| -235.045
+0 ||| A A D E C ||| -235.045
+0 ||| A C E C C ||| -235.045
+0 ||| A A E E B ||| -235.045
+0 ||| A D B C D ||| -235.045
+0 ||| A D B A E ||| -235.045
+0 ||| A C A E D ||| -235.045
+0 ||| A E D B B ||| -235.045
+0 ||| A E A C D ||| -235.045
+0 ||| A D D C C ||| -235.045
+0 ||| A D B E B ||| -235.045
+0 ||| A A C D D ||| -235.045
+0 ||| A B D C D ||| -235.045
+0 ||| A D B D C ||| -235.045
+0 ||| A D C D B ||| -235.045
+0 ||| A E C C B ||| -235.045
+0 ||| A E A E B ||| -235.045
+0 ||| A A E C D ||| -235.045
+0 ||| A E A D C ||| -235.045
+0 ||| A D E C B ||| -235.045
+0 ||| A B D D C ||| -235.045
+0 ||| A D E D A ||| -235.045
+0 ||| A B D A E ||| -235.045
+0 ||| A A C B E ||| -235.045
+0 ||| A E B C C ||| -235.045
+0 ||| A E E C A ||| -235.045
+0 ||| A C C E B ||| -235.045
+0 ||| A A E D C ||| -235.045
+0 ||| A E A A E ||| -235.045
+0 ||| A E B E A ||| -235.045
+0 ||| A D A B E ||| -235.045
+0 ||| A A B E D ||| -235.045
+0 ||| A B B D D ||| -235.045
+0 ||| A C E E A ||| -235.045
+0 ||| A B B B E ||| -235.045
+0 ||| A A E A E ||| -235.045
+0 ||| A E C D A ||| -235.045
+0 ||| A D D E A ||| -235.045
+0 ||| B E A E B ||| -236.045
+0 ||| B D E C B ||| -236.045
+0 ||| B D B D C ||| -236.045
+0 ||| B C E C C ||| -236.045
+0 ||| D C E C B ||| -236.045
+0 ||| C A C E C ||| -236.045
+0 ||| C B D B D ||| -236.045
+0 ||| D B D D B ||| -236.045
+0 ||| D C B C D ||| -236.045
+0 ||| B D B A E ||| -236.045
+0 ||| B B D E B ||| -236.045
+0 ||| B C C A E ||| -236.045
+0 ||| D C C D B ||| -236.045
+0 ||| D A D D C ||| -236.045
+0 ||| D A E D B ||| -236.045
+0 ||| E D B B C ||| -236.045
+0 ||| D D B D B ||| -236.045
+0 ||| B A D E C ||| -236.045
+0 ||| E E A A D ||| -236.045
+0 ||| E B A E C ||| -236.045
+0 ||| E A D D B ||| -236.045
+0 ||| C E B B C ||| -236.045
+0 ||| D A D A E ||| -236.045
+0 ||| C C E A D ||| -236.045
+0 ||| C D A E C ||| -236.045
+0 ||| B C C C D ||| -236.045
+0 ||| D C B A E ||| -236.045
+0 ||| C A A D E ||| -236.045
+0 ||| B A E A E ||| -236.045
+0 ||| B C B E C ||| -236.045
+0 ||| C B C C D ||| -236.045
+0 ||| B C C E B ||| -236.045
+0 ||| C A E B D ||| -236.045
+0 ||| C D D B C ||| -236.045
+0 ||| E C E A C ||| -236.045
+0 ||| B B B D D ||| -236.045
+0 ||| B B D A E ||| -236.045
+0 ||| C B A E D ||| -236.045
+0 ||| C C E B C ||| -236.045
+0 ||| C D D A D ||| -236.045
+0 ||| C D C C C ||| -236.045
+0 ||| D C D C C ||| -236.045
+0 ||| D D C B C ||| -236.045
+0 ||| C B B E C ||| -236.045
+0 ||| E B D B C ||| -236.045
+0 ||| B D A B E ||| -236.045
+0 ||| D C A B E ||| -236.045
+0 ||| D B E B C ||| -236.045
+0 ||| C B C A E ||| -236.045
+0 ||| E B C C C ||| -236.045
+0 ||| E C C B C ||| -236.045
+0 ||| E A A C E ||| -236.045
+0 ||| E E A B C ||| -236.045
+0 ||| D D E A C ||| -236.045
+0 ||| C A B C E ||| -236.045
+0 ||| C B E C C ||| -236.045
+0 ||| B B D D C ||| -236.045
+0 ||| E B D A D ||| -236.045
+0 ||| B B B B E ||| -236.045
+0 ||| E E B A C ||| -236.045
+0 ||| C C C B D ||| -236.045
+0 ||| C E B A D ||| -236.045
+0 ||| B D D C C ||| -236.045
+0 ||| D B E A D ||| -236.045
+0 ||| D E C A C ||| -236.045
+0 ||| B E B C C ||| -236.045
+0 ||| B E A D C ||| -236.045
+0 ||| C E D A C ||| -236.045
+0 ||| C B C D C ||| -236.045
+0 ||| E D D A C ||| -236.045
+0 ||| B C C D C ||| -236.045
+0 ||| B A E D C ||| -236.045
+0 ||| B A C B E ||| -236.045
+0 ||| D B A C E ||| -236.045
+0 ||| D C B E B ||| -236.045
+0 ||| D D C A D ||| -236.045
+0 ||| B E A C D ||| -236.045
+0 ||| C D B B D ||| -236.045
+0 ||| D C B D C ||| -236.045
+0 ||| D A B B E ||| -236.045
+0 ||| B E A A E ||| -236.045
+0 ||| C C A C E ||| -236.045
+0 ||| B D B E B ||| -236.045
+0 ||| C C D D B ||| -236.045
+0 ||| E A E B C ||| -236.045
+0 ||| B E C C B ||| -236.045
+0 ||| C B C E B ||| -236.045
+0 ||| B D C D B ||| -236.045
+0 ||| E D C B B ||| -236.045
+0 ||| E D B A D ||| -236.045
+0 ||| C D E B B ||| -236.045
+0 ||| D A D E B ||| -236.045
+0 ||| C E E A B ||| -236.045
+0 ||| C E C B B ||| -236.045
+0 ||| D D D C B ||| -236.045
+0 ||| D C A D D ||| -236.045
+0 ||| E E C A B ||| -236.045
+0 ||| E C B D B ||| -236.045
+0 ||| E B E B B ||| -236.045
+0 ||| B E D B B ||| -236.045
+0 ||| D E A D B ||| -236.045
+0 ||| B A E E B ||| -236.045
+0 ||| D A B D D ||| -236.045
+0 ||| B D A D D ||| -236.045
+0 ||| B A C D D ||| -236.045
+0 ||| B B D C D ||| -236.045
+0 ||| B A B E D ||| -236.045
+0 ||| B C D B D ||| -236.045
+0 ||| E D E A B ||| -236.045
+0 ||| B B E D B ||| -236.045
+0 ||| B A E C D ||| -236.045
+0 ||| B D B C D ||| -236.045
+0 ||| E B B B D ||| -236.045
+0 ||| E D A B D ||| -236.045
+0 ||| E B C E A ||| -236.045
+0 ||| E C C A D ||| -236.045
+0 ||| E A C B D ||| -236.045
+0 ||| D C E D A ||| -236.045
+0 ||| B C A E D ||| -236.045
+0 ||| C E A B D ||| -236.045
+0 ||| B D D E A ||| -236.045
+0 ||| D E B C B ||| -236.045
+0 ||| D B C B D ||| -236.045
+0 ||| C D C E A ||| -236.045
+0 ||| E A E A D ||| -236.045
+0 ||| E C D C B ||| -236.045
+0 ||| D A D C D ||| -236.045
+0 ||| D C D E A ||| -236.045
+0 ||| D D D D A ||| -236.045
+0 ||| C E E B A ||| -236.045
+0 ||| E C D D A ||| -236.045
+0 ||| B E E C A ||| -236.045
+0 ||| B C E E A ||| -236.045
+0 ||| E E C B A ||| -236.045
+0 ||| B E C D A ||| -236.045
+0 ||| D E B D A ||| -236.045
+0 ||| B E B E A ||| -236.045
+0 ||| E E E A A ||| -236.045
+0 ||| D E D C A ||| -236.045
+0 ||| C B E E A ||| -236.045
+0 ||| E D E B A ||| -236.045
+0 ||| B D E D A ||| -236.045
+0 ||| A C D D C ||| -237.045
+0 ||| A B C E C ||| -237.045
+0 ||| A C D A E ||| -237.045
+0 ||| A E C A D ||| -237.045
+0 ||| A E D D A ||| -237.045
+0 ||| A B E B D ||| -237.045
+0 ||| A C D E B ||| -237.045
+0 ||| A C B D D ||| -237.045
+0 ||| A E C B C ||| -237.045
+0 ||| A D A C E ||| -237.045
+0 ||| A E E A C ||| -237.045
+0 ||| A D D D B ||| -237.045
+0 ||| A B A D E ||| -237.045
+0 ||| A D E A D ||| -237.045
+0 ||| A C B B E ||| -237.045
+0 ||| A A D D D ||| -237.045
+0 ||| A A A E E ||| -237.045
+0 ||| A E D C B ||| -237.045
+0 ||| A A C C E ||| -237.045
+0 ||| A B B C E ||| -237.045
+0 ||| A E B D B ||| -237.045
+0 ||| A C E D B ||| -237.045
+0 ||| A A D B E ||| -237.045
+0 ||| A D C B D ||| -237.045
+0 ||| A C D C D ||| -237.045
+0 ||| A D E B C ||| -237.045
+0 ||| C B D E B ||| -238.045
+0 ||| C E C C B ||| -238.045
+0 ||| B C E D B ||| -238.045
+0 ||| B E D C B ||| -238.045
+0 ||| E C D A D ||| -238.045
+0 ||| B C B D D ||| -238.045
+0 ||| C A B E D ||| -238.045
+0 ||| C C B E C ||| -238.045
+0 ||| D E C B B ||| -238.045
+0 ||| D C E A D ||| -238.045
+0 ||| C A E C D ||| -238.045
+0 ||| E B A D D ||| -238.045
+0 ||| E C B B D ||| -238.045
+0 ||| C D B C D ||| -238.045
+0 ||| C E A A E ||| -238.045
+0 ||| D B B E C ||| -238.045
+0 ||| E D A C D ||| -238.045
+0 ||| B E C A D ||| -238.045
+0 ||| C C E C C ||| -238.045
+0 ||| D A E B D ||| -238.045
+0 ||| E B A B E ||| -238.045
+0 ||| B B E B D ||| -238.045
+0 ||| C B D C D ||| -238.045
+0 ||| E A A E D ||| -238.045
+0 ||| D B D B D ||| -238.045
+0 ||| C D B D C ||| -238.045
+0 ||| B A D B E ||| -238.045
+0 ||| C C A E D ||| -238.045
+0 ||| C D A D D ||| -238.045
+0 ||| E B B C D ||| -238.045
+0 ||| B D E A D ||| -238.045
+0 ||| D D D A D ||| -238.045
+0 ||| E A C C D ||| -238.045
+0 ||| B C D C D ||| -238.045
+0 ||| C E A C D ||| -238.045
+0 ||| D D B B D ||| -238.045
+0 ||| C A C D D ||| -238.045
+0 ||| E C A E C ||| -238.045
+0 ||| B E C B C ||| -238.045
+0 ||| D B A E D ||| -238.045
+0 ||| D E B A D ||| -238.045
+0 ||| D B C C D ||| -238.045
+0 ||| D E D A C ||| -238.045
+0 ||| E B B D C ||| -238.045
+0 ||| B B B C E ||| -238.045
+0 ||| E C D B C ||| -238.045
+0 ||| B E E A C ||| -238.045
+0 ||| E A B E C ||| -238.045
+0 ||| C B B D D ||| -238.045
+0 ||| E D C C B ||| -238.045
+0 ||| C A E D C ||| -238.045
+0 ||| B A D D D ||| -238.045
+0 ||| E A E C C ||| -238.045
+0 ||| E D A E B ||| -238.045
+0 ||| C B D A E ||| -238.045
+0 ||| C E B C C ||| -238.045
+0 ||| C D B A E ||| -238.045
+0 ||| E A C D C ||| -238.045
+0 ||| C C C C D ||| -238.045
+0 ||| B D C B D ||| -238.045
+0 ||| D A C E C ||| -238.045
+0 ||| D C C B D ||| -238.045
+0 ||| C C D B D ||| -238.045
+0 ||| E E D A B ||| -238.045
+0 ||| D A B C E ||| -238.045
+0 ||| C E A E B ||| -238.045
+0 ||| C C C D C ||| -238.045
+0 ||| E B E C B ||| -238.045
+0 ||| E E B B B ||| -238.045
+0 ||| D E E A B ||| -238.045
+0 ||| E E A C C ||| -238.045
+0 ||| C A E E B ||| -238.045
+0 ||| C A C B E ||| -238.045
+0 ||| C A D E C ||| -238.045
+0 ||| E C C C C ||| -238.045
+0 ||| D C A C E ||| -238.045
+0 ||| D B E C C ||| -238.045
+0 ||| D D D B C ||| -238.045
+0 ||| C B D D C ||| -238.045
+0 ||| E B D C C ||| -238.045
+0 ||| B D E B C ||| -238.045
+0 ||| E D B C C ||| -238.045
+0 ||| D D E B B ||| -238.045
+0 ||| B C D D C ||| -238.045
+0 ||| C C C E B ||| -238.045
+0 ||| B E B D B ||| -238.045
+0 ||| D B C D C ||| -238.045
+0 ||| D D C C C ||| -238.045
+0 ||| C D C D B ||| -238.045
+0 ||| D D A E C ||| -238.045
+0 ||| D C D D B ||| -238.045
+0 ||| C D E C B ||| -238.045
+0 ||| B C D A E ||| -238.045
+0 ||| C E A D C ||| -238.045
+0 ||| C E D B B ||| -238.045
+0 ||| E D A D C ||| -238.045
+0 ||| C B E D B ||| -238.045
+0 ||| D C E B C ||| -238.045
+0 ||| E A C A E ||| -238.045
+0 ||| C D D C C ||| -238.045
+0 ||| B B A D E ||| -238.045
+0 ||| E C E B B ||| -238.045
+0 ||| B A C C E ||| -238.045
+0 ||| D A A D E ||| -238.045
+0 ||| B C D E B ||| -238.045
+0 ||| E B B A E ||| -238.045
+0 ||| D E B B C ||| -238.045
+0 ||| C D B E B ||| -238.045
+0 ||| E B B E B ||| -238.045
+0 ||| B D D D B ||| -238.045
+0 ||| E A C E B ||| -238.045
+0 ||| D B C E B ||| -238.045
+0 ||| E B C D B ||| -238.045
+0 ||| E A D B D ||| -238.045
+0 ||| D E A B D ||| -238.045
+0 ||| E D D B B ||| -238.045
+0 ||| B D A C E ||| -238.045
+0 ||| D B E E A ||| -238.045
+0 ||| B B C E C ||| -238.045
+0 ||| C B B B E ||| -238.045
+0 ||| E B D E A ||| -238.045
+0 ||| B C B B E ||| -238.045
+0 ||| C C C A E ||| -238.045
+0 ||| E D B E A ||| -238.045
+0 ||| B A A E E ||| -238.045
+0 ||| D B C A E ||| -238.045
+0 ||| E D C D A ||| -238.045
+0 ||| E D A A E ||| -238.045
+0 ||| C A E A E ||| -238.045
+0 ||| E B E D A ||| -238.045
+0 ||| C D A B E ||| -238.045
+0 ||| C C E E A ||| -238.045
+0 ||| C E C D A ||| -238.045
+0 ||| C E B E A ||| -238.045
+0 ||| C D E D A ||| -238.045
+0 ||| E E D B A ||| -238.045
+0 ||| E E A E A ||| -238.045
+0 ||| D D C E A ||| -238.045
+0 ||| C E E C A ||| -238.045
+0 ||| E A E E A ||| -238.045
+0 ||| E E C C A ||| -238.045
+0 ||| E D E C A ||| -238.045
+0 ||| E C C E A ||| -238.045
+0 ||| D E E B A ||| -238.045
+0 ||| B E D D A ||| -238.045
+0 ||| C D D E A ||| -238.045
+0 ||| A A D C E ||| -239.045
+0 ||| A E B B D ||| -239.045
+0 ||| A B E C D ||| -239.045
+0 ||| A A B D E ||| -239.045
+0 ||| A D C E B ||| -239.045
+0 ||| A A E E C ||| -239.045
+0 ||| A B C B E ||| -239.045
+0 ||| A D C C D ||| -239.045
+0 ||| A B D E C ||| -239.045
+0 ||| A C E B D ||| -239.045
+0 ||| A B B E D ||| -239.045
+0 ||| A A C E D ||| -239.045
+0 ||| A E E B B ||| -239.045
+0 ||| A B E E B ||| -239.045
+0 ||| A D A E D ||| -239.045
+0 ||| A B E D C ||| -239.045
+0 ||| A E D A D ||| -239.045
+0 ||| A B C D D ||| -239.045
+0 ||| A D E E A ||| -239.045
+0 ||| A D E C C ||| -239.045
+0 ||| A E D B C ||| -239.045
+0 ||| A D C D C ||| -239.045
+0 ||| A E C C C ||| -239.045
+0 ||| A E C E A ||| -239.045
+0 ||| A E A E C ||| -239.045
+0 ||| A C B C E ||| -239.045
+0 ||| A D D B D ||| -239.045
+0 ||| A D B E C ||| -239.045
+0 ||| A B E A E ||| -239.045
+0 ||| A D C A E ||| -239.045
+0 ||| A C A D E ||| -239.045
+0 ||| A C C E C ||| -239.045
+0 ||| B E E B B ||| -240.045
+0 ||| B D C A E ||| -240.045
+0 ||| D E A C D ||| -240.045
+0 ||| E A D E B ||| -240.045
+0 ||| D D E C B ||| -240.045
+0 ||| D B D E B ||| -240.045
+0 ||| C A A E E ||| -240.045
+0 ||| D C C D C ||| -240.045
+0 ||| B D E C C ||| -240.045
+0 ||| B E A E C ||| -240.045
+0 ||| D C E C C ||| -240.045
+0 ||| E C E C B ||| -240.045
+0 ||| B B D E C ||| -240.045
+0 ||| D B D D C ||| -240.045
+0 ||| D D B D C ||| -240.045
+0 ||| C B B C E ||| -240.045
+0 ||| B B B E D ||| -240.045
+0 ||| D A E D C ||| -240.045
+0 ||| B A D C E ||| -240.045
+0 ||| B B C D D ||| -240.045
+0 ||| C B A D E ||| -240.045
+0 ||| B B C B E ||| -240.045
+0 ||| B C A D E ||| -240.045
+0 ||| E A D D C ||| -240.045
+0 ||| B D C C D ||| -240.045
+0 ||| D D B A E ||| -240.045
+0 ||| D C C A E ||| -240.045
+0 ||| E C A B E ||| -240.045
+0 ||| D C D B D ||| -240.045
+0 ||| E B C B D ||| -240.045
+0 ||| C E C A D ||| -240.045
+0 ||| B D A E D ||| -240.045
+0 ||| E A B B E ||| -240.045
+0 ||| B A B D E ||| -240.045
+0 ||| C C D C D ||| -240.045
+0 ||| C C D A E ||| -240.045
+0 ||| D C C C D ||| -240.045
+0 ||| E C A D D ||| -240.045
+0 ||| C B E B D ||| -240.045
+0 ||| D A E C D ||| -240.045
+0 ||| C A C C E ||| -240.045
+0 ||| B B E C D ||| -240.045
+0 ||| C D C B D ||| -240.045
+0 ||| E C B C D ||| -240.045
+0 ||| D D A D D ||| -240.045
+0 ||| D B B B E ||| -240.045
+0 ||| E A D C D ||| -240.045
+0 ||| B D D B D ||| -240.045
+0 ||| D B D A E ||| -240.045
+0 ||| D D C D B ||| -240.045
+0 ||| C D A C E ||| -240.045
+0 ||| B E B B D ||| -240.045
+0 ||| E C C D B ||| -240.045
+0 ||| B C C E C ||| -240.045
+0 ||| B E D A D ||| -240.045
+0 ||| E A B D D ||| -240.045
+0 ||| D B D C D ||| -240.045
+0 ||| E C B A E ||| -240.045
+0 ||| D B B D D ||| -240.045
+0 ||| D C A E D ||| -240.045
+0 ||| C D E A D ||| -240.045
+0 ||| B C E B D ||| -240.045
+0 ||| D A B E D ||| -240.045
+0 ||| D E A A E ||| -240.045
+0 ||| B C B C E ||| -240.045
+0 ||| B A C E D ||| -240.045
+0 ||| B B E A E ||| -240.045
+0 ||| E A D A E ||| -240.045
+0 ||| D D B C D ||| -240.045
+0 ||| E C B E B ||| -240.045
+0 ||| E E A D B ||| -240.045
+0 ||| C C B B E ||| -240.045
+0 ||| D C B E C ||| -240.045
+0 ||| D A D E C ||| -240.045
+0 ||| E C D C C ||| -240.045
+0 ||| C C E D B ||| -240.045
+0 ||| D A E A E ||| -240.045
+0 ||| E D C A D ||| -240.045
+0 ||| E D D C B ||| -240.045
+0 ||| C C D E B ||| -240.045
+0 ||| C A D D D ||| -240.045
+0 ||| E E B C B ||| -240.045
+0 ||| E A E D B ||| -240.045
+0 ||| D B E D B ||| -240.045
+0 ||| C E B D B ||| -240.045
+0 ||| E D B D B ||| -240.045
+0 ||| C D E B C ||| -240.045
+0 ||| E B D D B ||| -240.045
+0 ||| C C D D C ||| -240.045
+0 ||| B D B E C ||| -240.045
+0 ||| E D C B C ||| -240.045
+0 ||| B E C C C ||| -240.045
+0 ||| B D C D C ||| -240.045
+0 ||| C B C E C ||| -240.045
+0 ||| B B E D C ||| -240.045
+0 ||| D E A D C ||| -240.045
+0 ||| D E D B B ||| -240.045
+0 ||| D D A B E ||| -240.045
+0 ||| C D D D B ||| -240.045
+0 ||| E E C A C ||| -240.045
+0 ||| C E C B C ||| -240.045
+0 ||| C E E A C ||| -240.045
+0 ||| D D D C C ||| -240.045
+0 ||| C C B D D ||| -240.045
+0 ||| D C C E B ||| -240.045
+0 ||| D A C D D ||| -240.045
+0 ||| E B E A D ||| -240.045
+0 ||| E B E B C ||| -240.045
+0 ||| E C B D C ||| -240.045
+0 ||| D E A E B ||| -240.045
+0 ||| B D C E B ||| -240.045
+0 ||| B A E E C ||| -240.045
+0 ||| B E D B C ||| -240.045
+0 ||| D E B C C ||| -240.045
+0 ||| B B E E B ||| -240.045
+0 ||| D D B E B ||| -240.045
+0 ||| B D E E A ||| -240.045
+0 ||| D E C C B ||| -240.045
+0 ||| E D E A C ||| -240.045
+0 ||| E E B D A ||| -240.045
+0 ||| E B A C E ||| -240.045
+0 ||| C E D C B ||| -240.045
+0 ||| D E E C A ||| -240.045
+0 ||| D A C B E ||| -240.045
+0 ||| D A E E B ||| -240.045
+0 ||| D D E D A ||| -240.045
+0 ||| C A D B E ||| -240.045
+0 ||| D C E E A ||| -240.045
+0 ||| E C D E A ||| -240.045
+0 ||| E D D D A ||| -240.045
+0 ||| E E D C A ||| -240.045
+0 ||| D D D E A ||| -240.045
+0 ||| E C E D A ||| -240.045
+0 ||| B E C E A ||| -240.045
+0 ||| D E B E A ||| -240.045
+0 ||| C E D D A ||| -240.045
+0 ||| D E C D A ||| -240.045
+0 ||| A C E A E ||| -241.045
+0 ||| A C E C D ||| -241.045
+0 ||| A E E D A ||| -241.045
+0 ||| A A D E D ||| -241.045
+0 ||| A D E D B ||| -241.045
+0 ||| A C E E B ||| -241.045
+0 ||| A D D E B ||| -241.045
+0 ||| A C C D D ||| -241.045
+0 ||| A D D A E ||| -241.045
+0 ||| A D D C D ||| -241.045
+0 ||| A E D E A ||| -241.045
+0 ||| A E A B E ||| -241.045
+0 ||| A E C D B ||| -241.045
+0 ||| A C C B E ||| -241.045
+0 ||| A D B D D ||| -241.045
+0 ||| A A E B E ||| -241.045
+0 ||| A D B B E ||| -241.045
+0 ||| A B C C E ||| -241.045
+0 ||| A E D C C ||| -241.045
+0 ||| A C D E C ||| -241.045
+0 ||| A B D B E ||| -241.045
+0 ||| A E A D D ||| -241.045
+0 ||| A C B E D ||| -241.045
+0 ||| A E B C D ||| -241.045
+0 ||| A B D D D ||| -241.045
+0 ||| A B A E E ||| -241.045
+0 ||| A C E D C ||| -241.045
+0 ||| A E B E B ||| -241.045
+0 ||| A E B D C ||| -241.045
+0 ||| A E E C B ||| -241.045
+0 ||| A D D D C ||| -241.045
+0 ||| A E B A E ||| -241.045
+0 ||| A A E D D ||| -241.045
+0 ||| B C E C D ||| -242.045
+0 ||| D E C B C ||| -242.045
+0 ||| D D D D B ||| -242.045
+0 ||| B C E D C ||| -242.045
+0 ||| B D B D D ||| -242.045
+0 ||| C E C C C ||| -242.045
+0 ||| C B D E C ||| -242.045
+0 ||| B E D C C ||| -242.045
+0 ||| E D C C C ||| -242.045
+0 ||| C A C E D ||| -242.045
+0 ||| C B E E B ||| -242.045
+0 ||| B D E D B ||| -242.045
+0 ||| D A D D D ||| -242.045
+0 ||| E B A E D ||| -242.045
+0 ||| E D B B D ||| -242.045
+0 ||| E A A D E ||| -242.045
+0 ||| C C A D E ||| -242.045
+0 ||| B A D E D ||| -242.045
+0 ||| C B C B E ||| -242.045
+0 ||| B E A B E ||| -242.045
+0 ||| C E B B D ||| -242.045
+0 ||| D C D A E ||| -242.045
+0 ||| D A C C E ||| -242.045
+0 ||| E D A E C ||| -242.045
+0 ||| E B B E C ||| -242.045
+0 ||| D C D E B ||| -242.045
+0 ||| D C B B E ||| -242.045
+0 ||| C D A E D ||| -242.045
+0 ||| B C B E D ||| -242.045
+0 ||| C A D C E ||| -242.045
+0 ||| E C C B D ||| -242.045
+0 ||| B C D E C ||| -242.045
+0 ||| C C E B D ||| -242.045
+0 ||| C D D B D ||| -242.045
+0 ||| E C E A D ||| -242.045
+0 ||| E B C C D ||| -242.045
+0 ||| D D C B D ||| -242.045
+0 ||| D C D C D ||| -242.045
+0 ||| C D C C D ||| -242.045
+0 ||| E D E B B ||| -242.045
+0 ||| E B D B D ||| -242.045
+0 ||| C B B E D ||| -242.045
+0 ||| D B E B D ||| -242.045
+0 ||| B E C D B ||| -242.045
+0 ||| B D B B E ||| -242.045
+0 ||| C B C D D ||| -242.045
+0 ||| D D E A D ||| -242.045
+0 ||| E E A B D ||| -242.045
+0 ||| C B E C D ||| -242.045
+0 ||| E E B A D ||| -242.045
+0 ||| D C B D D ||| -242.045
+0 ||| B B D D D ||| -242.045
+0 ||| C E D A D ||| -242.045
+0 ||| B E B A E ||| -242.045
+0 ||| B D D C D ||| -242.045
+0 ||| B B C C E ||| -242.045
+0 ||| E E D A C ||| -242.045
+0 ||| B E A D D ||| -242.045
+0 ||| D E C A D ||| -242.045
+0 ||| E C A C E ||| -242.045
+0 ||| B E B C D ||| -242.045
+0 ||| D E B D B ||| -242.045
+0 ||| D D A C E ||| -242.045
+0 ||| B A E D D ||| -242.045
+0 ||| B C C D D ||| -242.045
+0 ||| E D D A D ||| -242.045
+0 ||| C E A E C ||| -242.045
+0 ||| C D C A E ||| -242.045
+0 ||| D A A E E ||| -242.045
+0 ||| C B E A E ||| -242.045
+0 ||| C D C E B ||| -242.045
+0 ||| E E B B C ||| -242.045
+0 ||| E B E C C ||| -242.045
+0 ||| B A E B E ||| -242.045
+0 ||| D B B C E ||| -242.045
+0 ||| D B A D E ||| -242.045
+0 ||| B D D A E ||| -242.045
+0 ||| E B C A E ||| -242.045
+0 ||| C A E E C ||| -242.045
+0 ||| C E E B B ||| -242.045
+0 ||| D E E A C ||| -242.045
+0 ||| B B D B E ||| -242.045
+0 ||| B C E A E ||| -242.045
+0 ||| E B C D C ||| -242.045
+0 ||| E E C B B ||| -242.045
+0 ||| C C B C E ||| -242.045
+0 ||| D D E B C ||| -242.045
+0 ||| D C E D B ||| -242.045
+0 ||| B E B D C ||| -242.045
+0 ||| E B C E B ||| -242.045
+0 ||| C C C E C ||| -242.045
+0 ||| D E D C B ||| -242.045
+0 ||| E C D D B ||| -242.045
+0 ||| C D C D C ||| -242.045
+0 ||| B C E E B ||| -242.045
+0 ||| C D E C C ||| -242.045
+0 ||| D C D D C ||| -242.045
+0 ||| B C C B E ||| -242.045
+0 ||| B E E C B ||| -242.045
+0 ||| C E D B C ||| -242.045
+0 ||| C B E D C ||| -242.045
+0 ||| E A E B D ||| -242.045
+0 ||| E A C E C ||| -242.045
+0 ||| E A B C E ||| -242.045
+0 ||| C D B E C ||| -242.045
+0 ||| B B A E E ||| -242.045
+0 ||| D B C E C ||| -242.045
+0 ||| E C E B C ||| -242.045
+0 ||| B E B E B ||| -242.045
+0 ||| D E D D A ||| -242.045
+0 ||| B D D D C ||| -242.045
+0 ||| E D D B C ||| -242.045
+0 ||| B E D E A ||| -242.045
+0 ||| E E E A B ||| -242.045
+0 ||| C A B D E ||| -242.045
+0 ||| B E E D A ||| -242.045
+0 ||| D A D B E ||| -242.045
+0 ||| B D D E B ||| -242.045
+0 ||| E D C E A ||| -242.045
+0 ||| C E C E A ||| -242.045
+0 ||| C D E E A ||| -242.045
+0 ||| E E E B A ||| -242.045
+0 ||| E B E E A ||| -242.045
+0 ||| A E A C E ||| -243.045
+0 ||| A B D C E ||| -243.045
+0 ||| A C D B E ||| -243.045
+0 ||| A D A D E ||| -243.045
+0 ||| A E C B D ||| -243.045
+0 ||| A C C C E ||| -243.045
+0 ||| A D B C E ||| -243.045
+0 ||| A E D D B ||| -243.045
+0 ||| A C A E E ||| -243.045
+0 ||| A A C D E ||| -243.045
+0 ||| A D E B D ||| -243.045
+0 ||| A B C E D ||| -243.045
+0 ||| A B B D E ||| -243.045
+0 ||| A C D D D ||| -243.045
+0 ||| A D C E C ||| -243.045
+0 ||| A E E B C ||| -243.045
+0 ||| A B E E C ||| -243.045
+0 ||| A A E C E ||| -243.045
+0 ||| A A B E E ||| -243.045
+0 ||| A E E A D ||| -243.045
+0 ||| D E C C C ||| -244.045
+0 ||| C C E A E ||| -244.045
+0 ||| D D E C C ||| -244.045
+0 ||| E B D C D ||| -244.045
+0 ||| C E B E B ||| -244.045
+0 ||| C C E E B ||| -244.045
+0 ||| D A D C E ||| -244.045
+0 ||| E E A D C ||| -244.045
+0 ||| C C B E D ||| -244.045
+0 ||| D E D B C ||| -244.045
+0 ||| E A D E C ||| -244.045
+0 ||| C E E C B ||| -244.045
+0 ||| D B D E C ||| -244.045
+0 ||| E C D B D ||| -244.045
+0 ||| C B D B E ||| -244.045
+0 ||| C C E C D ||| -244.045
+0 ||| D B B E D ||| -244.045
+0 ||| D C B C E ||| -244.045
+0 ||| C D B D D ||| -244.045
+0 ||| E B E D B ||| -244.045
+0 ||| E C E C C ||| -244.045
+0 ||| D E D A D ||| -244.045
+0 ||| E E A A E ||| -244.045
+0 ||| E C A E D ||| -244.045
+0 ||| B E C B D ||| -244.045
+0 ||| E B B D D ||| -244.045
+0 ||| C D D E B ||| -244.045
+0 ||| E E C C B ||| -244.045
+0 ||| E C C E B ||| -244.045
+0 ||| D E E B B ||| -244.045
+0 ||| E A E E B ||| -244.045
+0 ||| E E D B B ||| -244.045
+0 ||| C D E D B ||| -244.045
+0 ||| B B D C E ||| -244.045
+0 ||| D A B D E ||| -244.045
+0 ||| B A E C E ||| -244.045
+0 ||| D D C D C ||| -244.045
+0 ||| E A B E D ||| -244.045
+0 ||| C B D D D ||| -244.045
+0 ||| C E C D B ||| -244.045
+0 ||| E C B E C ||| -244.045
+0 ||| B C C C E ||| -244.045
+0 ||| B E E A D ||| -244.045
+0 ||| C B C C E ||| -244.045
+0 ||| E A E C D ||| -244.045
+0 ||| E D B E B ||| -244.045
+0 ||| C A E D D ||| -244.045
+0 ||| E C C D C ||| -244.045
+0 ||| C A E B E ||| -244.045
+0 ||| C B A E E ||| -244.045
+0 ||| C A D E D ||| -244.045
+0 ||| B B B D E ||| -244.045
+0 ||| C E B C D ||| -244.045
+0 ||| E E A C D ||| -244.045
+0 ||| E A C D D ||| -244.045
+0 ||| C D D A E ||| -244.045
+0 ||| C C C B E ||| -244.045
+0 ||| D D D B D ||| -244.045
+0 ||| C E A D D ||| -244.045
+0 ||| C E B D C ||| -244.045
+0 ||| C D B B E ||| -244.045
+0 ||| E B D A E ||| -244.045
+0 ||| D B E A E ||| -244.045
+0 ||| D A C E D ||| -244.045
+0 ||| C E B A E ||| -244.045
+0 ||| D D A E D ||| -244.045
+0 ||| B E A C E ||| -244.045
+0 ||| D B E C D ||| -244.045
+0 ||| D D C A E ||| -244.045
+0 ||| E D D C C ||| -244.045
+0 ||| E D B A E ||| -244.045
+0 ||| C C C D D ||| -244.045
+0 ||| E D A D D ||| -244.045
+0 ||| D C E B D ||| -244.045
+0 ||| D B E D C ||| -244.045
+0 ||| E A E D C ||| -244.045
+0 ||| E C C C D ||| -244.045
+0 ||| B C D B E ||| -244.045
+0 ||| D B C D D ||| -244.045
+0 ||| E D B C D ||| -244.045
+0 ||| D E B B D ||| -244.045
+0 ||| D D C C D ||| -244.045
+0 ||| E B D E B ||| -244.045
+0 ||| B D E B D ||| -244.045
+0 ||| C D D C D ||| -244.045
+0 ||| B A B E E ||| -244.045
+0 ||| E A C B E ||| -244.045
+0 ||| B A C D E ||| -244.045
+0 ||| D C A D E ||| -244.045
+0 ||| B C D D D ||| -244.045
+0 ||| C C E D C ||| -244.045
+0 ||| D E A E C ||| -244.045
+0 ||| C C D E C ||| -244.045
+0 ||| D B E E B ||| -244.045
+0 ||| D E C E A ||| -244.045
+0 ||| E B D D C ||| -244.045
+0 ||| E D E D A ||| -244.045
+0 ||| E D B D C ||| -244.045
+0 ||| D D E E A ||| -244.045
+0 ||| D A E E C ||| -244.045
+0 ||| E E B E A ||| -244.045
+0 ||| B D A D E ||| -244.045
+0 ||| E D D E A ||| -244.045
+0 ||| E D C D B ||| -244.045
+0 ||| E E C D A ||| -244.045
+0 ||| C D D D C ||| -244.045
+0 ||| C E E D A ||| -244.045
+0 ||| D D C E B ||| -244.045
+0 ||| C E D E A ||| -244.045
+0 ||| E E A E B ||| -244.045
+0 ||| E E E C A ||| -244.045
+0 ||| E E B C C ||| -244.045
+0 ||| E C E E A ||| -244.045
+0 ||| E D E C B ||| -244.045
+0 ||| B E D D B ||| -244.045
+0 ||| D C C E C ||| -244.045
+0 ||| D D B E C ||| -244.045
+0 ||| B E E B C ||| -244.045
+0 ||| B D C E C ||| -244.045
+0 ||| B B E E C ||| -244.045
+0 ||| E A E A E ||| -244.045
+0 ||| E D A B E ||| -244.045
+0 ||| E C C A E ||| -244.045
+0 ||| E B B B E ||| -244.045
+0 ||| B D B C E ||| -244.045
+0 ||| C E D C C ||| -244.045
+0 ||| D B C B E ||| -244.045
+0 ||| B C A E E ||| -244.045
+0 ||| B B C E D ||| -244.045
+0 ||| C E A B E ||| -244.045
+0 ||| A A E E D ||| -245.045
+0 ||| A E C A E ||| -245.045
+0 ||| A D D E C ||| -245.045
+0 ||| A C B D E ||| -245.045
+0 ||| A B D E D ||| -245.045
+0 ||| A A D D E ||| -245.045
+0 ||| A C D C E ||| -245.045
+0 ||| A C C E D ||| -245.045
+0 ||| A B E B E ||| -245.045
+0 ||| A D C B E ||| -245.045
+0 ||| A C E E C ||| -245.045
+0 ||| A D E D C ||| -245.045
+0 ||| A E C D C ||| -245.045
+0 ||| A E E E A ||| -245.045
+0 ||| A E E C C ||| -245.045
+0 ||| A B E D D ||| -245.045
+0 ||| A E C E B ||| -245.045
+0 ||| A E D B D ||| -245.045
+0 ||| A D E A E ||| -245.045
+0 ||| A D E C D ||| -245.045
+0 ||| A D B E D ||| -245.045
+0 ||| A E C C D ||| -245.045
+0 ||| A D C D D ||| -245.045
+0 ||| A E B E C ||| -245.045
+0 ||| A E A E D ||| -245.045
+0 ||| A D E E B ||| -245.045
+0 ||| E D D D B ||| -246.045
+0 ||| B D D E C ||| -246.045
+0 ||| E C D E B ||| -246.045
+0 ||| E E C B C ||| -246.045
+0 ||| D C B E D ||| -246.045
+0 ||| D D D D C ||| -246.045
+0 ||| B E A E D ||| -246.045
+0 ||| C E D D B ||| -246.045
+0 ||| C A B E E ||| -246.045
+0 ||| B C B D E ||| -246.045
+0 ||| E C D A E ||| -246.045
+0 ||| C A E C E ||| -246.045
+0 ||| D C E A E ||| -246.045
+0 ||| E B B C E ||| -246.045
+0 ||| C D B C E ||| -246.045
+0 ||| E B A D E ||| -246.045
+0 ||| E C B B E ||| -246.045
+0 ||| D C E E B ||| -246.045
+0 ||| E D A C E ||| -246.045
+0 ||| B E C A E ||| -246.045
+0 ||| C B D C E ||| -246.045
+0 ||| B D E A E ||| -246.045
+0 ||| D A E B E ||| -246.045
+0 ||| B B E B E ||| -246.045
+0 ||| C D A D E ||| -246.045
+0 ||| E A A E E ||| -246.045
+0 ||| C C A E E ||| -246.045
+0 ||| B D E C D ||| -246.045
+0 ||| D B D B E ||| -246.045
+0 ||| D C C D D ||| -246.045
+0 ||| C B E E C ||| -246.045
+0 ||| D A E D D ||| -246.045
+0 ||| C E A C E ||| -246.045
+0 ||| C E E B C ||| -246.045
+0 ||| D D D A E ||| -246.045
+0 ||| D C E C D ||| -246.045
+0 ||| B C D C E ||| -246.045
+0 ||| E A C C E ||| -246.045
+0 ||| D D B D D ||| -246.045
+0 ||| D B C C E ||| -246.045
+0 ||| D D B B E ||| -246.045
+0 ||| D B D D D ||| -246.045
+0 ||| B B D E D ||| -246.045
+0 ||| B D E D C ||| -246.045
+0 ||| C A C D E ||| -246.045
+0 ||| D B A E E ||| -246.045
+0 ||| D E B A E ||| -246.045
+0 ||| B B E D D ||| -246.045
+0 ||| E A D D D ||| -246.045
+0 ||| C C D D D ||| -246.045
+0 ||| D E A D D ||| -246.045
+0 ||| E C E D B ||| -246.045
+0 ||| D E B E B ||| -246.045
+0 ||| C B B D E ||| -246.045
+0 ||| B A D D E ||| -246.045
+0 ||| B C C E D ||| -246.045
+0 ||| D C D E C ||| -246.045
+0 ||| D D D E B ||| -246.045
+0 ||| C C C C E ||| -246.045
+0 ||| E D C B D ||| -246.045
+0 ||| E E D C B ||| -246.045
+0 ||| D D E D B ||| -246.045
+0 ||| B D C B E ||| -246.045
+0 ||| B E C E B ||| -246.045
+0 ||| D C E D C ||| -246.045
+0 ||| C E E A D ||| -246.045
+0 ||| B E C D C ||| -246.045
+0 ||| E D E B C ||| -246.045
+0 ||| D E D E A ||| -246.045
+0 ||| B E C C D ||| -246.045
+0 ||| C C D B E ||| -246.045
+0 ||| C D E B D ||| -246.045
+0 ||| D E E D A ||| -246.045
+0 ||| C D C E C ||| -246.045
+0 ||| D C C B E ||| -246.045
+0 ||| B E E E A ||| -246.045
+0 ||| D A D E D ||| -246.045
+0 ||| E E D D A ||| -246.045
+0 ||| B D E E B ||| -246.045
+0 ||| B D B E D ||| -246.045
+0 ||| E C D C D ||| -246.045
+0 ||| B E E C C ||| -246.045
+0 ||| E B C E C ||| -246.045
+0 ||| E C D D C ||| -246.045
+0 ||| D E D C C ||| -246.045
+0 ||| B C E E C ||| -246.045
+0 ||| E C B D D ||| -246.045
+0 ||| C B C E D ||| -246.045
+0 ||| E B E B D ||| -246.045
+0 ||| D D D C D ||| -246.045
+0 ||| B A E E D ||| -246.045
+0 ||| B E B E C ||| -246.045
+0 ||| B E D B D ||| -246.045
+0 ||| D E B C D ||| -246.045
+0 ||| E A D B E ||| -246.045
+0 ||| D E A B E ||| -246.045
+0 ||| D E E C B ||| -246.045
+0 ||| E E C A D ||| -246.045
+0 ||| E E E A C ||| -246.045
+0 ||| D E B D C ||| -246.045
+0 ||| B D C D D ||| -246.045
+0 ||| E E B D B ||| -246.045
+0 ||| C E C B D ||| -246.045
+0 ||| E D E A D ||| -246.045
+0 ||| D E C D B ||| -246.045
+0 ||| A C E B E ||| -247.045
+0 ||| A E B B E ||| -247.045
+0 ||| A B E C E ||| -247.045
+0 ||| A D C C E ||| -247.045
+0 ||| A B B E E ||| -247.045
+0 ||| A D D D D ||| -247.045
+0 ||| A E D D C ||| -247.045
+0 ||| A E D E B ||| -247.045
+0 ||| A E E D B ||| -247.045
+0 ||| A D A E E ||| -247.045
+0 ||| A E D A E ||| -247.045
+0 ||| A B C D E ||| -247.045
+0 ||| A A C E E ||| -247.045
+0 ||| A E B D D ||| -247.045
+0 ||| A C D E D ||| -247.045
+0 ||| A E D C D ||| -247.045
+0 ||| A D D B E ||| -247.045
+0 ||| A C E D D ||| -247.045
+0 ||| B C D E D ||| -248.045
+0 ||| B C E D D ||| -248.045
+0 ||| C D E D C ||| -248.045
+0 ||| B E D C D ||| -248.045
+0 ||| C C E E C ||| -248.045
+0 ||| D D E B D ||| -248.045
+0 ||| C E B E C ||| -248.045
+0 ||| D E C B D ||| -248.045
+0 ||| D E A C E ||| -248.045
+0 ||| C B D E D ||| -248.045
+0 ||| B B E C E ||| -248.045
+0 ||| C E C C D ||| -248.045
+0 ||| C E E C C ||| -248.045
+0 ||| E E D B C ||| -248.045
+0 ||| E B E D C ||| -248.045
+0 ||| D D A D E ||| -248.045
+0 ||| E D C C D ||| -248.045
+0 ||| E C A D E ||| -248.045
+0 ||| C D E E B ||| -248.045
+0 ||| D C C C E ||| -248.045
+0 ||| C D E A E ||| -248.045
+0 ||| C C D C E ||| -248.045
+0 ||| C D C B E ||| -248.045
+0 ||| B B B E E ||| -248.045
+0 ||| E B E C D ||| -248.045
+0 ||| E C B C E ||| -248.045
+0 ||| B B C D E ||| -248.045
+0 ||| C E C A E ||| -248.045
+0 ||| C D D E C ||| -248.045
+0 ||| E B C B E ||| -248.045
+0 ||| B D C C E ||| -248.045
+0 ||| D C D B E ||| -248.045
+0 ||| E A E E C ||| -248.045
+0 ||| E E C C C ||| -248.045
+0 ||| B D A E E ||| -248.045
+0 ||| D E E B C ||| -248.045
+0 ||| E C C E C ||| -248.045
+0 ||| C B E B E ||| -248.045
+0 ||| D A E C E ||| -248.045
+0 ||| D E D D B ||| -248.045
+0 ||| E A D C E ||| -248.045
+0 ||| D B D C E ||| -248.045
+0 ||| E D C E B ||| -248.045
+0 ||| B E E D B ||| -248.045
+0 ||| B D D B E ||| -248.045
+0 ||| E E E B B ||| -248.045
+0 ||| E B B E D ||| -248.045
+0 ||| C E C D C ||| -248.045
+0 ||| B E D A E ||| -248.045
+0 ||| B E B B E ||| -248.045
+0 ||| D B B D E ||| -248.045
+0 ||| B C E B E ||| -248.045
+0 ||| D C A E E ||| -248.045
+0 ||| E C E B D ||| -248.045
+0 ||| C E C E B ||| -248.045
+0 ||| E D B E C ||| -248.045
+0 ||| D A B E E ||| -248.045
+0 ||| E D A E D ||| -248.045
+0 ||| E B E E B ||| -248.045
+0 ||| C A D D E ||| -248.045
+0 ||| B A C E E ||| -248.045
+0 ||| D D B C E ||| -248.045
+0 ||| C E A E D ||| -248.045
+0 ||| E D E C C ||| -248.045
+0 ||| B E D D C ||| -248.045
+0 ||| E A B D E ||| -248.045
+0 ||| E B D E C ||| -248.045
+0 ||| D C D D D ||| -248.045
+0 ||| C D E C D ||| -248.045
+0 ||| C D C D D ||| -248.045
+0 ||| D B C E D ||| -248.045
+0 ||| E D C A E ||| -248.045
+0 ||| C C C E D ||| -248.045
+0 ||| C E D B D ||| -248.045
+0 ||| E A C E D ||| -248.045
+0 ||| C B E D D ||| -248.045
+0 ||| D B E E C ||| -248.045
+0 ||| B E B D D ||| -248.045
+0 ||| D E E A D ||| -248.045
+0 ||| C D B E D ||| -248.045
+0 ||| D D C E C ||| -248.045
+0 ||| E D C D C ||| -248.045
+0 ||| E B E A E ||| -248.045
+0 ||| E E D A D ||| -248.045
+0 ||| C C B D E ||| -248.045
+0 ||| D A C D E ||| -248.045
+0 ||| B E D E B ||| -248.045
+0 ||| E B C D D ||| -248.045
+0 ||| C A E E D ||| -248.045
+0 ||| E E A E C ||| -248.045
+0 ||| B D D D D ||| -248.045
+0 ||| E D E E A ||| -248.045
+0 ||| E E B B D ||| -248.045
+0 ||| E E C E A ||| -248.045
+0 ||| E D D B D ||| -248.045
+0 ||| C E E E A ||| -248.045
+0 ||| A D D C E ||| -249.045
+0 ||| A A E D E ||| -249.045
+0 ||| A D E E C ||| -249.045
+0 ||| A C E C E ||| -249.045
+0 ||| A C C D E ||| -249.045
+0 ||| A D B D E ||| -249.045
+0 ||| A B E E D ||| -249.045
+0 ||| A A D E E ||| -249.045
+0 ||| A E E B D ||| -249.045
+0 ||| A E A D E ||| -249.045
+0 ||| A E C E C ||| -249.045
+0 ||| A D C E D ||| -249.045
+0 ||| A C B E E ||| -249.045
+0 ||| A E B C E ||| -249.045
+0 ||| A B D D E ||| -249.045
+0 ||| D C E E C ||| -250.045
+0 ||| C C E B E ||| -250.045
+0 ||| C E E D B ||| -250.045
+0 ||| C E D D C ||| -250.045
+0 ||| E A E D D ||| -250.045
+0 ||| D E C C D ||| -250.045
+0 ||| E E B A E ||| -250.045
+0 ||| E E C D B ||| -250.045
+0 ||| E D E D B ||| -250.045
+0 ||| E D D D C ||| -250.045
+0 ||| D D E C D ||| -250.045
+0 ||| E C D E C ||| -250.045
+0 ||| B C E C E ||| -250.045
+0 ||| E E A D D ||| -250.045
+0 ||| E A D E D ||| -250.045
+0 ||| C B E C E ||| -250.045
+0 ||| D E D B D ||| -250.045
+0 ||| D B E B E ||| -250.045
+0 ||| D E C E B ||| -250.045
+0 ||| E B D B E ||| -250.045
+0 ||| D B D E D ||| -250.045
+0 ||| C E B D D ||| -250.045
+0 ||| B D B D E ||| -250.045
+0 ||| C D D B E ||| -250.045
+0 ||| E C C B E ||| -250.045
+0 ||| E E E D A ||| -250.045
+0 ||| E D D E B ||| -250.045
+0 ||| E B A E E ||| -250.045
+0 ||| C A C E E ||| -250.045
+0 ||| E E D E A ||| -250.045
+0 ||| D E C D C ||| -250.045
+0 ||| D A D D E ||| -250.045
+0 ||| D C D C E ||| -250.045
+0 ||| D E E E A ||| -250.045
+0 ||| B C B E E ||| -250.045
+0 ||| E E B E B ||| -250.045
+0 ||| B A D E E ||| -250.045
+0 ||| E C E C D ||| -250.045
+0 ||| E D B B E ||| -250.045
+0 ||| D D E E B ||| -250.045
+0 ||| D D D E C ||| -250.045
+0 ||| D D C D D ||| -250.045
+0 ||| D B E D D ||| -250.045
+0 ||| C D C C E ||| -250.045
+0 ||| E C E D C ||| -250.045
+0 ||| C D A E E ||| -250.045
+0 ||| D E B E C ||| -250.045
+0 ||| E C B E D ||| -250.045
+0 ||| E B C C E ||| -250.045
+0 ||| E C C D D ||| -250.045
+0 ||| C B C D E ||| -250.045
+0 ||| E E D C C ||| -250.045
+0 ||| C B B E E ||| -250.045
+0 ||| B D D C E ||| -250.045
+0 ||| D D E D C ||| -250.045
+0 ||| B E C E C ||| -250.045
+0 ||| B B D D E ||| -250.045
+0 ||| C E D A E ||| -250.045
+0 ||| B C C D E ||| -250.045
+0 ||| E D D C D ||| -250.045
+0 ||| D D C B E ||| -250.045
+0 ||| C E B B E ||| -250.045
+0 ||| D D E A E ||| -250.045
+0 ||| B E A D E ||| -250.045
+0 ||| C C D E D ||| -250.045
+0 ||| E D D A E ||| -250.045
+0 ||| D E C A E ||| -250.045
+0 ||| D E A E D ||| -250.045
+0 ||| C C E D D ||| -250.045
+0 ||| B D E E C ||| -250.045
+0 ||| E E A B E ||| -250.045
+0 ||| E E B C D ||| -250.045
+0 ||| E A E B E ||| -250.045
+0 ||| E D B D D ||| -250.045
+0 ||| E B D D D ||| -250.045
+0 ||| D A E E D ||| -250.045
+0 ||| C E D E B ||| -250.045
+0 ||| C D D D D ||| -250.045
+0 ||| E E B D C ||| -250.045
+0 ||| E C E E B ||| -250.045
+0 ||| E E E C B ||| -250.045
+0 ||| B A E D E ||| -250.045
+0 ||| B D C E D ||| -250.045
+0 ||| B B E E D ||| -250.045
+0 ||| B E E B D ||| -250.045
+0 ||| D C C E D ||| -250.045
+0 ||| D D B E D ||| -250.045
+0 ||| B E B C E ||| -250.045
+0 ||| D C B D E ||| -250.045
+0 ||| E C E A E ||| -250.045
+0 ||| C E D C D ||| -250.045
+0 ||| D E E C C ||| -250.045
+0 ||| A E C B E ||| -251.045
+0 ||| A E E A E ||| -251.045
+0 ||| A B C E E ||| -251.045
+0 ||| A D D E D ||| -251.045
+0 ||| A E B E D ||| -251.045
+0 ||| A D E B E ||| -251.045
+0 ||| A E D E C ||| -251.045
+0 ||| A D E D D ||| -251.045
+0 ||| A C D D E ||| -251.045
+0 ||| A C E E D ||| -251.045
+0 ||| A E C D D ||| -251.045
+0 ||| A E E E B ||| -251.045
+0 ||| A E E D C ||| -251.045
+0 ||| A E E C D ||| -251.045
+0 ||| E A E C E ||| -252.045
+0 ||| B E E E B ||| -252.045
+0 ||| D D D D D ||| -252.045
+0 ||| D D D B E ||| -252.045
+0 ||| E B D C E ||| -252.045
+0 ||| E E C B D ||| -252.045
+0 ||| B D D E D ||| -252.045
+0 ||| E E A C E ||| -252.045
+0 ||| C C B E E ||| -252.045
+0 ||| C D C E D ||| -252.045
+0 ||| C C E C E ||| -252.045
+0 ||| C D E E C ||| -252.045
+0 ||| C B E E D ||| -252.045
+0 ||| C D B D E ||| -252.045
+0 ||| B D E D D ||| -252.045
+0 ||| D B B E E ||| -252.045
+0 ||| E C A E E ||| -252.045
+0 ||| E B B D E ||| -252.045
+0 ||| B E C B E ||| -252.045
+0 ||| D E D A E ||| -252.045
+0 ||| C E B C E ||| -252.045
+0 ||| C E E B D ||| -252.045
+0 ||| D E E D B ||| -252.045
+0 ||| B E E D C ||| -252.045
+0 ||| D E D D C ||| -252.045
+0 ||| E D C E C ||| -252.045
+0 ||| D C D E D ||| -252.045
+0 ||| E E E B C ||| -252.045
+0 ||| E A B E E ||| -252.045
+0 ||| D C E D D ||| -252.045
+0 ||| C E C E C ||| -252.045
+0 ||| C B D D E ||| -252.045
+0 ||| E D E B D ||| -252.045
+0 ||| B E C D D ||| -252.045
+0 ||| B B C E E ||| -252.045
+0 ||| C C C D E ||| -252.045
+0 ||| D A C E E ||| -252.045
+0 ||| E B E E C ||| -252.045
+0 ||| E C D B E ||| -252.045
+0 ||| D B E C E ||| -252.045
+0 ||| C A E D E ||| -252.045
+0 ||| E C C C E ||| -252.045
+0 ||| D B C D E ||| -252.045
+0 ||| D D C C E ||| -252.045
+0 ||| E D B C E ||| -252.045
+0 ||| D E B B E ||| -252.045
+0 ||| B D E B E ||| -252.045
+0 ||| C D D C E ||| -252.045
+0 ||| E E E A D ||| -252.045
+0 ||| B E E C D ||| -252.045
+0 ||| C E A D E ||| -252.045
+0 ||| D D A E E ||| -252.045
+0 ||| E E D D B ||| -252.045
+0 ||| B C D D E ||| -252.045
+0 ||| E C D D D ||| -252.045
+0 ||| E B C E D ||| -252.045
+0 ||| D E D C D ||| -252.045
+0 ||| B C E E D ||| -252.045
+0 ||| B E B E D ||| -252.045
+0 ||| E A C D E ||| -252.045
+0 ||| B E D E C ||| -252.045
+0 ||| D C E B E ||| -252.045
+0 ||| E D A D E ||| -252.045
+0 ||| B E E A E ||| -252.045
+0 ||| D E B D D ||| -252.045
+0 ||| D E D E B ||| -252.045
+0 ||| C A D E E ||| -252.045
+0 ||| A B D E E ||| -253.045
+0 ||| A E A E E ||| -253.045
+0 ||| A A E E E ||| -253.045
+0 ||| A D B E E ||| -253.045
+0 ||| A C C E E ||| -253.045
+0 ||| A D C D E ||| -253.045
+0 ||| A B E D E ||| -253.045
+0 ||| A E D D D ||| -253.045
+0 ||| A E D B E ||| -253.045
+0 ||| A E C C E ||| -253.045
+0 ||| A D E C E ||| -253.045
+0 ||| D C B E E ||| -254.045
+0 ||| D E A D E ||| -254.045
+0 ||| E E C D C ||| -254.045
+0 ||| C C D D E ||| -254.045
+0 ||| D E E B D ||| -254.045
+0 ||| E E C E B ||| -254.045
+0 ||| E D E D C ||| -254.045
+0 ||| D E C E C ||| -254.045
+0 ||| C E B E D ||| -254.045
+0 ||| B E A E E ||| -254.045
+0 ||| E D E E B ||| -254.045
+0 ||| C E E E B ||| -254.045
+0 ||| B E D B E ||| -254.045
+0 ||| E B E D D ||| -254.045
+0 ||| E D D E C ||| -254.045
+0 ||| E E B E C ||| -254.045
+0 ||| E E C C D ||| -254.045
+0 ||| E E E E A ||| -254.045
+0 ||| C D E D D ||| -254.045
+0 ||| E A E E D ||| -254.045
+0 ||| C D D E D ||| -254.045
+0 ||| B B D E E ||| -254.045
+0 ||| D D E E C ||| -254.045
+0 ||| D C C D E ||| -254.045
+0 ||| E D E A E ||| -254.045
+0 ||| D B D D E ||| -254.045
+0 ||| D D B D E ||| -254.045
+0 ||| E C C E D ||| -254.045
+0 ||| C B C E E ||| -254.045
+0 ||| E E D B D ||| -254.045
+0 ||| B B E D E ||| -254.045
+0 ||| E A D D E ||| -254.045
+0 ||| C E C D D ||| -254.045
+0 ||| D C E C E ||| -254.045
+0 ||| B C C E E ||| -254.045
+0 ||| C E E C D ||| -254.045
+0 ||| E D B E D ||| -254.045
+0 ||| C E E A E ||| -254.045
+0 ||| E D C B E ||| -254.045
+0 ||| B D E C E ||| -254.045
+0 ||| B E C C E ||| -254.045
+0 ||| C E E D C ||| -254.045
+0 ||| D D C E D ||| -254.045
+0 ||| E E A E D ||| -254.045
+0 ||| E E C A E ||| -254.045
+0 ||| E C B D E ||| -254.045
+0 ||| D A D E E ||| -254.045
+0 ||| B E D D D ||| -254.045
+0 ||| E C D C E ||| -254.045
+0 ||| C E C B E ||| -254.045
+0 ||| E D E C D ||| -254.045
+0 ||| E D C D D ||| -254.045
+0 ||| D A E D E ||| -254.045
+0 ||| E B D E D ||| -254.045
+0 ||| D B E E D ||| -254.045
+0 ||| C C E E D ||| -254.045
+0 ||| D E B C E ||| -254.045
+0 ||| E B E B E ||| -254.045
+0 ||| B A E E E ||| -254.045
+0 ||| D D D C E ||| -254.045
+0 ||| E E E C C ||| -254.045
+0 ||| C E D E C ||| -254.045
+0 ||| E C E E C ||| -254.045
+0 ||| B D C D E ||| -254.045
+0 ||| C D E B E ||| -254.045
+0 ||| B D B E E ||| -254.045
+0 ||| A C E D E ||| -255.045
+0 ||| A D D D E ||| -255.045
+0 ||| A E B D E ||| -255.045
+0 ||| A D E E D ||| -255.045
+0 ||| A E E E C ||| -255.045
+0 ||| A E D C E ||| -255.045
+0 ||| A C D E E ||| -255.045
+0 ||| A E C E D ||| -255.045
+0 ||| D E E A E ||| -256.045
+0 ||| C E A E E ||| -256.045
+0 ||| D E E C D ||| -256.045
+0 ||| B E E E C ||| -256.045
+0 ||| E E B D D ||| -256.045
+0 ||| E C D E D ||| -256.045
+0 ||| E D D D D ||| -256.045
+0 ||| E D A E E ||| -256.045
+0 ||| B C D E E ||| -256.045
+0 ||| B C E D E ||| -256.045
+0 ||| E E E D B ||| -256.045
+0 ||| B D D D E ||| -256.045
+0 ||| C B D E E ||| -256.045
+0 ||| D E C D D ||| -256.045
+0 ||| D E E E B ||| -256.045
+0 ||| E E D E B ||| -256.045
+0 ||| E B E C E ||| -256.045
+0 ||| D E C B E ||| -256.045
+0 ||| D D E D D ||| -256.045
+0 ||| D E E D C ||| -256.045
+0 ||| D D D E D ||| -256.045
+0 ||| E B B E E ||| -256.045
+0 ||| C E C C E ||| -256.045
+0 ||| C E D D D ||| -256.045
+0 ||| D E B E D ||| -256.045
+0 ||| E E D C D ||| -256.045
+0 ||| E C E B E ||| -256.045
+0 ||| D E D E C ||| -256.045
+0 ||| D C E E D ||| -256.045
+0 ||| E D C C E ||| -256.045
+0 ||| B E B D E ||| -256.045
+0 ||| B E C E D ||| -256.045
+0 ||| E D D B E ||| -256.045
+0 ||| B E D C E ||| -256.045
+0 ||| C B E D E ||| -256.045
+0 ||| C E D B E ||| -256.045
+0 ||| C C C E E ||| -256.045
+0 ||| D B C E E ||| -256.045
+0 ||| E C E D D ||| -256.045
+0 ||| D C D D E ||| -256.045
+0 ||| C D E C E ||| -256.045
+0 ||| C D C D E ||| -256.045
+0 ||| E E D D C ||| -256.045
+0 ||| B D E E D ||| -256.045
+0 ||| C A E E E ||| -256.045
+0 ||| E A C E E ||| -256.045
+0 ||| C D B E E ||| -256.045
+0 ||| E B C D E ||| -256.045
+0 ||| E E D A E ||| -256.045
+0 ||| E E B B E ||| -256.045
+0 ||| D D E B E ||| -256.045
+0 ||| A E E D D ||| -257.045
+0 ||| A D C E E ||| -257.045
+0 ||| A B E E E ||| -257.045
+0 ||| A E D E D ||| -257.045
+0 ||| A E E B E ||| -257.045
+0 ||| C C E D E ||| -258.045
+0 ||| D E D B E ||| -258.045
+0 |||
<TRUNCATED>
[37/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/RuleCollection.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/RuleCollection.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/RuleCollection.java
new file mode 100644
index 0000000..a45c41b
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/RuleCollection.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.tm;
+
+import java.util.List;
+
+import org.apache.joshua.decoder.ff.FeatureFunction;
+
+/**
+ * A RuleCollection represents a set of rules that share the same source side (and hence the same
+ * arity). These rules are likely stored together in a Trie data structure, although the interface
+ * allows any implementation to be used.
+ *
+ * @author Zhifei Li
+ * @author Lane Schwartz
+ * @author Matt Post post@cs.jhu.edu
+ */
+public interface RuleCollection {
+
+ /**
+ * Returns true if the rules are sorted. This is used to allow rules to be sorted in an amortized
+ * fashion; rather than sorting all trie nodes when the grammar is originally loaded, we sort them
+ * only as the decoder actually needs them.
+ * @return true if rules are sorted
+ */
+ boolean isSorted();
+
+ /**
+ * This returns a list of the rules, sorting them if necessary.
+ *
+ * Implementations of this function should be synchronized.
+ * @param models {@link java.util.List} of {@link org.apache.joshua.decoder.ff.FeatureFunction}'s
+ * @return the {@link java.util.List} of sorted rules
+ */
+ List<Rule> getSortedRules(List<FeatureFunction> models);
+
+ /**
+ * Get the list of rules. There are no guarantees about whether they're sorted or not.
+ * @return the {@link java.util.List} of rules, there is no gurantee they will be sorted
+ */
+ List<Rule> getRules();
+
+ /**
+ * Gets the source side for all rules in this RuleCollection. This source side is the same for all
+ * the rules in the RuleCollection.
+ *
+ * @return the (common) source side for all rules in this RuleCollection
+ */
+ int[] getSourceSide();
+
+ /**
+ * Gets the number of nonterminals in the source side of the rules in this RuleCollection. The
+ * source side is the same for all the rules in the RuleCollection, so the arity will also be the
+ * same for all of these rules.
+ *
+ * @return the (common) number of nonterminals in the source side of the rules in this
+ * RuleCollection
+ */
+ int getArity();
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/SentenceFilteredGrammar.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/SentenceFilteredGrammar.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/SentenceFilteredGrammar.java
new file mode 100644
index 0000000..c952b05
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/SentenceFilteredGrammar.java
@@ -0,0 +1,366 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.tm;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map.Entry;
+
+import org.apache.joshua.decoder.ff.tm.hash_based.ExtensionIterator;
+import org.apache.joshua.decoder.ff.tm.hash_based.MemoryBasedBatchGrammar;
+import org.apache.joshua.decoder.segment_file.Sentence;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class implements dynamic sentence-level filtering. This is accomplished with a parallel
+ * trie, a subset of the original trie, that only contains trie paths that are reachable from
+ * traversals of the current sentence.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ */
+public class SentenceFilteredGrammar extends MemoryBasedBatchGrammar {
+
+ private static final Logger LOG = LoggerFactory.getLogger(SentenceFilteredGrammar.class);
+
+ private AbstractGrammar baseGrammar;
+ private SentenceFilteredTrie filteredTrie;
+ private int[] tokens;
+ private Sentence sentence;
+
+ /**
+ * Construct a new sentence-filtered grammar. The main work is done in the enclosed trie (obtained
+ * from the base grammar, which contains the complete grammar).
+ *
+ * @param baseGrammar a new {@link org.apache.joshua.decoder.ff.tm.AbstractGrammar} to populate
+ * @param sentence {@link org.apache.joshua.lattice.Lattice} input
+ */
+ SentenceFilteredGrammar(AbstractGrammar baseGrammar, Sentence sentence) {
+ super(baseGrammar.joshuaConfiguration);
+ this.baseGrammar = baseGrammar;
+ this.sentence = sentence;
+ this.tokens = sentence.getWordIDs();
+
+ int origCount = getNumRules(baseGrammar.getTrieRoot());
+ long startTime = System.currentTimeMillis();
+
+ /* Filter the rules; returns non-null object */
+ this.filteredTrie = filter(baseGrammar.getTrieRoot());
+ int filteredCount = getNumRules();
+
+ float seconds = (System.currentTimeMillis() - startTime) / 1000.0f;
+
+ LOG.debug("Sentence-level filtering of sentence {} ({} -> {} rules) in {} seconds",
+ sentence.id(), origCount, filteredCount, seconds);
+ }
+
+ @Override
+ public Trie getTrieRoot() {
+ return filteredTrie;
+ }
+
+ /**
+ * This function is poorly named: it doesn't mean whether a rule exists in the grammar for the
+ * current span, but whether the grammar is permitted to apply rules to the current span (a
+ * grammar-level parameter). As such we can just chain to the underlying grammar.
+ */
+ @Override
+ public boolean hasRuleForSpan(int startIndex, int endIndex, int pathLength) {
+ return baseGrammar.hasRuleForSpan(startIndex, endIndex, pathLength);
+ }
+
+ @Override
+ public int getNumRules() {
+ return getNumRules(getTrieRoot());
+ }
+
+ /**
+ * A convenience function that counts the number of rules in a grammar's trie.
+ *
+ * @param node the {@link org.apache.joshua.decoder.ff.tm.Trie} implementation for which to count rules
+ * @return the number of rules
+ */
+ public int getNumRules(Trie node) {
+ int numRules = 0;
+ if (node != null) {
+ if (node.getRuleCollection() != null)
+ numRules += node.getRuleCollection().getRules().size();
+
+ if (node.getExtensions() != null)
+ for (Trie child : node.getExtensions())
+ numRules += getNumRules(child);
+ }
+
+ return numRules;
+ }
+
+ /**
+ * What is the algorithm?
+ *
+ * Take the first word of the sentence, and start at the root of the trie. There are two things to
+ * consider: (a) word matches and (b) nonterminal matches.
+ *
+ * For a word match, simply follow that arc along the trie. We create a parallel arc in our
+ * filtered grammar to represent it. Each arc in the filtered trie knows about its
+ * corresponding/underlying node in the unfiltered grammar trie.
+ *
+ * A nonterminal is always permitted to match. The question then is how much of the input sentence
+ * we imagine it consumed. The answer is that it could have been any amount. So the recursive call
+ * has to be a set of calls, one each to the next trie node with different lengths of the sentence
+ * remaining.
+ *
+ * A problem occurs when we have multiple sequential nonterminals. For scope-3 grammars, there can
+ * be four sequential nonterminals (in the case when they are grounded by terminals on both ends
+ * of the nonterminal chain). We'd like to avoid looking at all possible ways to split up the
+ * subsequence, because with respect to filtering rules, they are all the same.
+ *
+ * We accomplish this with the following restriction: for purposes of grammar filtering, only the
+ * first in a sequence of nonterminal traversals can consume more than one word. Each of the
+ * subsequent ones would have to consume just one word. We then just have to record in the
+ * recursive call whether the last traversal was a nonterminal or not.
+ *
+ * @param unfilteredTrieRoot todo
+ * @return the root of the filtered trie
+ */
+ private SentenceFilteredTrie filter(Trie unfilteredTrieRoot) {
+ SentenceFilteredTrie filteredTrieRoot = new SentenceFilteredTrie(unfilteredTrieRoot);
+
+ // System.err.println(String.format("FILTERING TO SENTENCE\n %s\n",
+ // Vocabulary.getWords(tokens)));
+
+ /*
+ * The root of the trie is where rule applications start, so we simply try all possible
+ * positions in the sentence.
+ */
+ for (int i = 0; i < tokens.length; i++) {
+ filter(i, filteredTrieRoot, false);
+ }
+
+ return filteredTrieRoot;
+ }
+
+ /**
+ * Matches rules against the sentence. Intelligently handles chains of sequential nonterminals.
+ * Marks arcs that are traversable for this sentence.
+ *
+ * @param i the position in the sentence to start matching
+ * @param trie the trie node to match against
+ * @param lastWasNT true if the match that brought us here was against a nonterminal
+ */
+ private void filter(int i, SentenceFilteredTrie trieNode, boolean lastWasNT) {
+ if (i >= tokens.length)
+ return;
+
+ /* Make sure the underlying unfiltered node has children. */
+ Trie unfilteredTrieNode = trieNode.unfilteredTrieNode;
+ if (unfilteredTrieNode.getChildren() == null) {
+ // trieNode.path.retreat();
+ return;
+ }
+
+ /* Match a word */
+ Trie trie = unfilteredTrieNode.match(tokens[i]);
+ if (trie != null) {
+ /*
+ * The current filtered node might already have an arc for this label. If so, retrieve it
+ * (since we still need to follow it); if not, create it.
+ */
+ SentenceFilteredTrie nextFilteredTrie = trieNode.match(tokens[i]);
+ if (nextFilteredTrie == null) {
+ nextFilteredTrie = new SentenceFilteredTrie(trie);
+ trieNode.children.put(tokens[i], nextFilteredTrie);
+ }
+
+ /*
+ * Now continue, trying to match the child node against the next position in the sentence. The
+ * third argument records that this match was not against a nonterminal.
+ */
+ filter(i + 1, nextFilteredTrie, false);
+ }
+
+ /*
+ * Now we attempt to match nonterminals. Any nonterminal is permitted to match any region of the
+ * sentence, up to the maximum span for that grammar. So we enumerate all children of the
+ * current (unfiltered) trie grammar node, looking for nonterminals (items whose label value is
+ * less than 0), then recurse.
+ *
+ * There is one subtlely. Adjacent nonterminals in a grammar rule can match a span (i, j) in (j
+ * - i - 1) ways, but for purposes of determining whether a rule fits, this is all wasted
+ * effort. To handle this, we allow the first nonterminal in a sequence to record 1, 2, 3, ...
+ * terminals (up to the grammar's span limit, or the rest of the sentence, whichever is
+ * shorter). Subsequent adjacent nonterminals are permitted to consume only a single terminal.
+ */
+ HashMap<Integer, ? extends Trie> children = unfilteredTrieNode.getChildren();
+ if (children != null) {
+ for (int label : children.keySet()) {
+ if (label < 0) {
+ SentenceFilteredTrie nextFilteredTrie = trieNode.match(label);
+ if (nextFilteredTrie == null) {
+ nextFilteredTrie = new SentenceFilteredTrie(unfilteredTrieNode.match(label));
+ trieNode.children.put(label, nextFilteredTrie);
+ }
+
+ /*
+ * Recurse. If the last match was a nonterminal, we can only consume one more token.
+ *
+ * TODO: This goes too far by looking at the whole sentence; each grammar has a maximum
+ * span limit which should be consulted. What we should be doing is passing the point
+ * where we started matching the current sentence, so we can apply this span limit, which
+ * is easily accessible (baseGrammar.spanLimit).
+ */
+ int maxJ = lastWasNT ? (i + 1) : tokens.length;
+ for (int j = i + 1; j <= maxJ; j++) {
+ filter(j, nextFilteredTrie, true);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Alternate filter that uses regular expressions, walking the grammar trie and matching the
+ * source side of each rule collection against the input sentence. Failed matches are discarded,
+ * and trie nodes extending from that position need not be explored.
+ *
+ * @param unfilteredTrie todo
+ * @return the root of the filtered trie if any rules were retained, otherwise null
+ */
+ @SuppressWarnings("unused")
+ private SentenceFilteredTrie filter_regexp(Trie unfilteredTrie) {
+ SentenceFilteredTrie trie = null;
+
+ /* Case 1: keep the trie node if it has a rule collection that matches the sentence */
+ if (unfilteredTrie.hasRules())
+ if (matchesSentence(unfilteredTrie))
+ trie = new SentenceFilteredTrie(unfilteredTrie);
+ else
+ return null;
+
+ /* Case 2: keep the trie node if it has children who have valid rule collections */
+ if (unfilteredTrie.hasExtensions())
+ for (Entry<Integer, ? extends Trie> arc : unfilteredTrie.getChildren().entrySet()) {
+ Trie unfilteredChildTrie = arc.getValue();
+ SentenceFilteredTrie nextTrie = filter_regexp(unfilteredChildTrie);
+ if (nextTrie != null) {
+ if (trie == null)
+ trie = new SentenceFilteredTrie(unfilteredTrie);
+ trie.children.put(arc.getKey(), nextTrie);
+ }
+ }
+
+ return trie;
+ }
+
+ private boolean matchesSentence(Trie childTrie) {
+ Rule rule = childTrie.getRuleCollection().getRules().get(0);
+ return rule.matches(sentence);
+ }
+
+ /**
+ * Implements a filtered trie, by sitting on top of a base trie and annotating nodes that match
+ * the given input sentence.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ *
+ */
+ public class SentenceFilteredTrie implements Trie {
+
+ /* The underlying unfiltered trie node. */
+ private Trie unfilteredTrieNode;
+
+ /* The child nodes in the filtered trie. */
+ private HashMap<Integer, SentenceFilteredTrie> children = null;
+
+ /**
+ * Constructor.
+ *
+ * @param unfilteredTrieNode todo
+ */
+ public SentenceFilteredTrie(Trie unfilteredTrieNode) {
+ this.unfilteredTrieNode = unfilteredTrieNode;
+ this.children = new HashMap<Integer, SentenceFilteredTrie>();
+ }
+
+ @Override
+ public SentenceFilteredTrie match(int wordID) {
+ if (children != null)
+ return children.get(wordID);
+ return null;
+ }
+
+ @Override
+ public boolean hasExtensions() {
+ return children != null;
+ }
+
+ @Override
+ public Collection<SentenceFilteredTrie> getExtensions() {
+ if (children != null)
+ return children.values();
+
+ return null;
+ }
+
+ @Override
+ public HashMap<Integer, SentenceFilteredTrie> getChildren() {
+ return children;
+ }
+
+ @Override
+ public boolean hasRules() {
+ // Chain to the underlying unfiltered node.
+ return unfilteredTrieNode.hasRules();
+ }
+
+ @Override
+ public RuleCollection getRuleCollection() {
+ // Chain to the underlying unfiltered node, since the rule collection just varies by target
+ // side.
+ return unfilteredTrieNode.getRuleCollection();
+ }
+
+ /**
+ * Counts the number of rules.
+ *
+ * @return the number of rules rooted at this node.
+ */
+ public int getNumRules() {
+ int numRules = 0;
+ if (getTrieRoot() != null)
+ if (getTrieRoot().getRuleCollection() != null)
+ numRules += getTrieRoot().getRuleCollection().getRules().size();
+
+ for (SentenceFilteredTrie node : getExtensions())
+ numRules += node.getNumRules();
+
+ return numRules;
+ }
+
+ @Override
+ public Iterator<Integer> getTerminalExtensionIterator() {
+ return new ExtensionIterator(children, true);
+ }
+
+ @Override
+ public Iterator<Integer> getNonterminalExtensionIterator() {
+ return new ExtensionIterator(children, false);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/Trie.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/Trie.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/Trie.java
new file mode 100644
index 0000000..51d2dd8
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/Trie.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.tm;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+
+/**
+ * An interface for trie-like data structures.
+ *
+ * @author wren ng thornton wren@users.sourceforge.net
+ * @author Zhifei Li, zhifei.work@gmail.com
+ */
+public interface Trie {
+
+ /**
+ * Traverse one ply further down the trie. If there is no match, the result is null.
+ *
+ * @param wordID input word ID
+ * @return Child node of this trie
+ */
+ Trie match(int wordID);
+
+
+ /**
+ * Returns whether matchOne(Symbol) could succeed for any symbol.
+ *
+ * @return <code>true</code> if {@link #match(int)} could succeed for some symbol,
+ * <code>false</code> otherwise
+ */
+ boolean hasExtensions();
+
+
+ /**
+ * If the trie node has extensions, then return a list of extended trie nodes, otherwise return
+ * null.
+ *
+ * @return A list of extended <code>Trie</code> nodes if this node has extensions,
+ * <code>null</code>
+ * otherwise
+ */
+ Collection<? extends Trie> getExtensions();
+
+
+ /**
+ * If the trie node has extensions, get a {@link java.util.HashMap} of their labels.
+ *
+ * @return a {@link java.util.HashMap} pf node extensions
+ */
+ HashMap<Integer,? extends Trie> getChildren();
+
+ /**
+ * Returns an iterator over the trie node's extensions with terminal labels.
+ *
+ * @return the {@link java.util.Iterator} created over the trie node's extensions with terminal labels
+ */
+ Iterator<Integer> getTerminalExtensionIterator();
+
+ /**
+ * Returns an iterator over the trie node's extensions with nonterminal labels.
+ *
+ * @return the {@link java.util.Iterator} created over the trie node's extensions with terminal labels
+ */
+ Iterator<Integer> getNonterminalExtensionIterator();
+
+
+ /**
+ * Gets whether the current node/state is a "final state" that has matching rules.
+ *
+ * @return <code>true</code> if the current node/state is a "final state" that has matching rules,
+ * <code>false</code> otherwise
+ */
+ boolean hasRules();
+
+
+ /**
+ * Retrieve the rules at the current node/state. The implementation of this method must adhere to
+ * the following laws:
+ *
+ * <ol>
+ * <li>The return value is always non-null. The collection may be empty however.</li>
+ * <li>The collection must be empty if hasRules() is false, and must be non-empty if hasRules() is
+ * true.</li>
+ * <li>The collection must be sorted (at least as used by TMGrammar)</li>
+ * </ol>
+ * @return a {@link org.apache.joshua.decoder.ff.tm.RuleCollection} representing the rules
+ * at the current node/state
+ */
+ RuleCollection getRuleCollection();
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/UnsortedRuleCollectionException.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/UnsortedRuleCollectionException.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/UnsortedRuleCollectionException.java
new file mode 100644
index 0000000..3358775
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/UnsortedRuleCollectionException.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.tm;
+
+/**
+ * Unchecked runtime exception thrown to indicate that a collection of rules has not been properly
+ * sorted according to the feature functions in effect.
+ *
+ * @author Lane Schwartz
+ */
+public class UnsortedRuleCollectionException extends RuntimeException {
+
+ private static final long serialVersionUID = -4819014771607378835L;
+
+ /**
+ * Constructs an <code>UnsortedRuleCollectionException</code> with the specified detail message.
+ *
+ * @param message the detail message
+ */
+ public UnsortedRuleCollectionException(String message) {
+ super(message);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/format/HieroFormatReader.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/format/HieroFormatReader.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/format/HieroFormatReader.java
new file mode 100644
index 0000000..45c8c33
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/format/HieroFormatReader.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.tm.format;
+
+import java.io.IOException;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.ff.tm.GrammarReader;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.util.Constants;
+import org.apache.joshua.util.FormatUtils;
+
+/**
+ * This class implements reading files in the format defined by David Chiang for Hiero.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ */
+
+public class HieroFormatReader extends GrammarReader<Rule> {
+
+ static {
+ description = "Original Hiero format";
+ }
+
+ public HieroFormatReader() {
+ super();
+ }
+
+ public HieroFormatReader(String grammarFile) throws IOException {
+ super(grammarFile);
+ }
+
+ @Override
+ public Rule parseLine(String line) {
+ String[] fields = line.split(Constants.fieldDelimiter);
+ if (fields.length < 3) {
+ throw new RuntimeException(String.format("Rule '%s' does not have four fields", line));
+ }
+
+ int lhs = Vocabulary.id(fields[0]);
+
+ /**
+ * On the foreign side, we map nonterminals to negative IDs, and terminals to positive IDs.
+ */
+ int arity = 0;
+ String[] sourceWords = fields[1].split("\\s+");
+ int[] sourceIDs = new int[sourceWords.length];
+ for (int i = 0; i < sourceWords.length; i++) {
+ /* NOTE: This redundantly creates vocab items for terms like [X,1]. This might actually
+ * be necessary, so don't try to turn this into an if/else.
+ */
+ sourceIDs[i] = Vocabulary.id(sourceWords[i]);
+ if (FormatUtils.isNonterminal(sourceWords[i])) {
+ sourceIDs[i] = Vocabulary.id(FormatUtils.stripNonTerminalIndex(sourceWords[i]));
+ arity++;
+
+ // TODO: the arity here (after incrementing) should match the rule index. Should
+ // check that arity == FormatUtils.getNonterminalIndex(foreignWords[i]), throw runtime
+ // error if not
+ }
+ }
+
+ /**
+ * The English side maps terminal symbols to positive IDs. Nonterminal symbols are linked
+ * to the index of the source-side nonterminal they are linked to. So for a rule
+ *
+ * [X] ||| [X,1] [X,2] [X,3] ||| [X,2] [X,1] [X,3] ||| ...
+ *
+ * the English side nonterminals will be -2, -1, -3. This assumes that the source side of
+ * the rule is always listed monotonically.
+ */
+ String[] targetWords = fields[2].split("\\s+");
+ int[] targetIDs = new int[targetWords.length];
+ for (int i = 0; i < targetWords.length; i++) {
+ targetIDs[i] = Vocabulary.id(targetWords[i]);
+ if (FormatUtils.isNonterminal(targetWords[i])) {
+ targetIDs[i] = -FormatUtils.getNonterminalIndex(targetWords[i]);
+ }
+ }
+
+ String sparse_features = (fields.length > 3 ? fields[3] : "");
+ String alignment = (fields.length > 4) ? fields[4] : null;
+
+ return new Rule(lhs, sourceIDs, targetIDs, sparse_features, arity, alignment);
+ }
+
+ public static boolean isNonTerminal(final String word) {
+ return FormatUtils.isNonterminal(word);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/format/MosesFormatReader.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/format/MosesFormatReader.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/format/MosesFormatReader.java
new file mode 100644
index 0000000..7811b3b
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/format/MosesFormatReader.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.tm.format;
+
+import java.io.IOException;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.util.io.LineReader;
+import org.apache.joshua.util.Constants;
+import org.apache.joshua.util.FormatUtils;
+
+/***
+ * This class reads in the Moses phrase table format, with support for the source and target side,
+ * list of features, and word alignments. It works by
+ *
+ * - casting the phrase-based rules to left-branching hierarchical rules and passing them on \
+ * to its parent class, {@link HieroFormatReader}.
+ * - converting the probabilities to -log probabilities
+ *
+ * There is also a tool to convert the grammars directly, so that they can be suitably packed. Usage:
+ *
+ * <pre>
+ * cat PHRASE_TABLE | java -cp $JOSHUA/target/classes org.apache.joshua.decoder.ff.tm.format.MosesFormatReader
+ * </pre>
+ *
+ * @author Matt Post post@cs.jhu.edu
+ *
+ */
+
+public class MosesFormatReader extends HieroFormatReader {
+
+ public MosesFormatReader(String grammarFile) throws IOException {
+ super(grammarFile);
+ Vocabulary.id(Constants.defaultNT);
+ }
+
+ public MosesFormatReader() {
+ super();
+ Vocabulary.id(Constants.defaultNT);
+ }
+
+ /**
+ * When dealing with Moses format, this munges a Moses-style phrase table into a grammar.
+ *
+ * mots francaises ||| French words ||| 1 2 3 ||| 0-1 1-0
+ *
+ * becomes
+ *
+ * [X] ||| [X,1] mots francaises ||| [X,1] French words ||| 1 2 3 ||| 0-1 1-0
+ *
+ * For thrax-extracted phrasal grammars, it transforms
+ *
+ * [X] ||| mots francaises ||| French words ||| 1 2 3 ||| 0-1 1-0
+ *
+ * into
+ *
+ * [X] ||| [X,1] mots francaises ||| [X,1] French words ||| 1 2 3 ||| 0-1 1-0
+ */
+ @Override
+ public Rule parseLine(String line) {
+ String[] fields = line.split(Constants.fieldDelimiter);
+
+ String nt = FormatUtils.cleanNonTerminal(Constants.defaultNT);
+ StringBuffer hieroLine = new StringBuffer(Constants.defaultNT + " ||| [" + nt + ",1] " + fields[0] + " ||| [" + nt + ",1] " + fields[1] + " |||");
+
+ String mosesFeatureString = fields[2];
+ for (String value: mosesFeatureString.split(" ")) {
+ float f = Float.parseFloat(value);
+ hieroLine.append(String.format(" %f", f <= 0.0 ? -100 : -Math.log(f)));
+ }
+
+ // alignments
+ if (fields.length >= 4)
+ hieroLine.append(" ||| " + fields[3]);
+
+ return super.parseLine(hieroLine.toString());
+ }
+
+ /**
+ * Converts a Moses phrase table to a Joshua grammar.
+ *
+ * @param args the command-line arguments
+ */
+ public static void main(String[] args) {
+ MosesFormatReader reader = new MosesFormatReader();
+ for (String line: new LineReader(System.in)) {
+ Rule rule = reader.parseLine(line);
+ System.out.println(rule.textFormat());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/hash_based/ExtensionIterator.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/hash_based/ExtensionIterator.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/hash_based/ExtensionIterator.java
new file mode 100644
index 0000000..ecb355d
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/hash_based/ExtensionIterator.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.tm.hash_based;
+
+import java.util.HashMap;
+import java.util.Iterator;
+
+public class ExtensionIterator implements Iterator<Integer> {
+
+ private Iterator<Integer> iterator;
+ private boolean terminal;
+ private boolean done;
+ private int next;
+
+ public ExtensionIterator(HashMap<Integer, ?> map, boolean terminal) {
+ this.terminal = terminal;
+ done = false;
+ if (map == null) {
+ done = true;
+ } else {
+ this.iterator = map.keySet().iterator();
+ forward();
+ }
+ }
+
+ private void forward() {
+ if (done)
+ return;
+ while (iterator.hasNext()) {
+ int candidate = iterator.next();
+ if ((terminal && candidate > 0) || (!terminal && candidate < 0)) {
+ next = candidate;
+ return;
+ }
+ }
+ done = true;
+ }
+
+ @Override
+ public boolean hasNext() {
+ return !done;
+ }
+
+ @Override
+ public Integer next() {
+ if (done)
+ throw new RuntimeException();
+ int consumed = next;
+ forward();
+ return consumed;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/hash_based/MemoryBasedBatchGrammar.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/hash_based/MemoryBasedBatchGrammar.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/hash_based/MemoryBasedBatchGrammar.java
new file mode 100644
index 0000000..f346e7a
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/hash_based/MemoryBasedBatchGrammar.java
@@ -0,0 +1,292 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.tm.hash_based;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.JoshuaConfiguration.OOVItem;
+import org.apache.joshua.decoder.ff.FeatureFunction;
+import org.apache.joshua.decoder.ff.tm.AbstractGrammar;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.ff.tm.GrammarReader;
+import org.apache.joshua.decoder.ff.tm.Trie;
+import org.apache.joshua.decoder.ff.tm.format.HieroFormatReader;
+import org.apache.joshua.decoder.ff.tm.format.MosesFormatReader;
+import org.apache.joshua.util.FormatUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class implements a memory-based bilingual BatchGrammar.
+ * <p>
+ * The rules are stored in a trie. Each trie node has: (1) RuleBin: a list of rules matching the
+ * french sides so far (2) A HashMap of next-layer trie nodes, the next french word used as the key
+ * in HashMap
+ *
+ * @author Zhifei Li zhifei.work@gmail.com
+ * @author Matt Post post@cs.jhu.edu
+ */
+public class MemoryBasedBatchGrammar extends AbstractGrammar {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MemoryBasedBatchGrammar.class);
+
+ // ===============================================================
+ // Instance Fields
+ // ===============================================================
+
+ /* The number of rules read. */
+ private int qtyRulesRead = 0;
+
+ /* The number of distinct source sides. */
+ private int qtyRuleBins = 0;
+
+ private int numDenseFeatures = 0;
+
+ /* The trie root. */
+ private MemoryBasedTrie root = null;
+
+ /* The file containing the grammar. */
+ private String grammarFile;
+
+ private GrammarReader<Rule> modelReader;
+
+ // ===============================================================
+ // Static Fields
+ // ===============================================================
+
+ // ===============================================================
+ // Constructors
+ // ===============================================================
+
+ public MemoryBasedBatchGrammar(JoshuaConfiguration joshuaConfiguration) {
+ super(joshuaConfiguration);
+ this.root = new MemoryBasedTrie();
+ this.joshuaConfiguration = joshuaConfiguration;
+ setSpanLimit(20);
+ }
+
+ public MemoryBasedBatchGrammar(String owner, JoshuaConfiguration joshuaConfiguration) {
+ this(joshuaConfiguration);
+ this.owner = Vocabulary.id(owner);
+ }
+
+ public MemoryBasedBatchGrammar(GrammarReader<Rule> gr, JoshuaConfiguration joshuaConfiguration) {
+ // this.defaultOwner = Vocabulary.id(defaultOwner);
+ // this.defaultLHS = Vocabulary.id(defaultLHSSymbol);
+ this(joshuaConfiguration);
+ modelReader = gr;
+ }
+
+ public MemoryBasedBatchGrammar(String formatKeyword, String grammarFile, String owner,
+ String defaultLHSSymbol, int spanLimit, JoshuaConfiguration joshuaConfiguration)
+ throws IOException {
+
+ this(joshuaConfiguration);
+ this.owner = Vocabulary.id(owner);
+ Vocabulary.id(defaultLHSSymbol);
+ this.spanLimit = spanLimit;
+ this.grammarFile = grammarFile;
+
+ // ==== loading grammar
+ this.modelReader = createReader(formatKeyword, grammarFile);
+ if (modelReader != null) {
+ for (Rule rule : modelReader)
+ if (rule != null) {
+ addRule(rule);
+ }
+ } else {
+ LOG.info("Couldn't create a GrammarReader for file {} with format {}",
+ grammarFile, formatKeyword);
+ }
+
+ this.printGrammar();
+ }
+
+ protected GrammarReader<Rule> createReader(String format, String grammarFile) throws IOException {
+
+ if (grammarFile != null) {
+ if ("hiero".equals(format) || "thrax".equals(format)) {
+ return new HieroFormatReader(grammarFile);
+ } else if ("moses".equals(format)) {
+ return new MosesFormatReader(grammarFile);
+ } else {
+ throw new RuntimeException(String.format("* FATAL: unknown grammar format '%s'", format));
+ }
+ }
+ return null;
+ }
+
+ // ===============================================================
+ // Methods
+ // ===============================================================
+
+ public void setSpanLimit(int spanLimit) {
+ this.spanLimit = spanLimit;
+ }
+
+ @Override
+ public int getNumRules() {
+ return this.qtyRulesRead;
+ }
+
+ /**
+ * if the span covered by the chart bin is greater than the limit, then return false
+ */
+ public boolean hasRuleForSpan(int i, int j, int pathLength) {
+ if (this.spanLimit == -1) { // mono-glue grammar
+ return (i == 0);
+ } else {
+ // System.err.println(String.format("%s HASRULEFORSPAN(%d,%d,%d)/%d = %s",
+ // Vocabulary.word(this.owner), i, j, pathLength, spanLimit, pathLength <= this.spanLimit));
+ return (pathLength <= this.spanLimit);
+ }
+ }
+
+ public Trie getTrieRoot() {
+ return this.root;
+ }
+
+ /**
+ * Adds a rule to the grammar.
+ */
+ public void addRule(Rule rule) {
+
+ // TODO: Why two increments?
+ this.qtyRulesRead++;
+
+ // if (owner == -1) {
+ // System.err.println("* FATAL: MemoryBasedBatchGrammar::addRule(): owner not set for grammar");
+ // System.exit(1);
+ // }
+ rule.setOwner(owner);
+
+ if (numDenseFeatures == 0)
+ numDenseFeatures = rule.getFeatureVector().getDenseFeatures().size();
+
+ // === identify the position, and insert the trie nodes as necessary
+ MemoryBasedTrie pos = root;
+ int[] french = rule.getFrench();
+
+ maxSourcePhraseLength = Math.max(maxSourcePhraseLength, french.length);
+
+ for (int k = 0; k < french.length; k++) {
+ int curSymID = french[k];
+
+ /*
+ * Note that the nonTerminal symbol in the french is not cleaned (i.e., will be sth like
+ * [X,1]), but the symbol in the Trie has to be cleaned, so that the match does not care about
+ * the markup (i.e., [X,1] or [X,2] means the same thing, that is X) if
+ * (Vocabulary.nt(french[k])) { curSymID = modelReader.cleanNonTerminal(french[k]); if
+ * (logger.isLoggable(Level.FINEST)) logger.finest("Amended to: " + curSymID); }
+ */
+
+ MemoryBasedTrie nextLayer = (MemoryBasedTrie) pos.match(curSymID);
+ if (null == nextLayer) {
+ nextLayer = new MemoryBasedTrie();
+ if (pos.hasExtensions() == false) {
+ pos.childrenTbl = new HashMap<Integer, MemoryBasedTrie>();
+ }
+ pos.childrenTbl.put(curSymID, nextLayer);
+ }
+ pos = nextLayer;
+ }
+
+ // === add the rule into the trie node
+ if (!pos.hasRules()) {
+ pos.ruleBin = new MemoryBasedRuleBin(rule.getArity(), rule.getFrench());
+ this.qtyRuleBins++;
+ }
+ pos.ruleBin.addRule(rule);
+ }
+
+ protected void printGrammar() {
+ LOG.info("MemoryBasedBatchGrammar: Read {} rules with {} distinct source sides from '{}'",
+ this.qtyRulesRead, this.qtyRuleBins, grammarFile);
+ }
+
+ /***
+ * Takes an input word and creates an OOV rule in the current grammar for that word.
+ *
+ * @param sourceWord integer representation of word
+ * @param featureFunctions {@link java.util.List} of {@link org.apache.joshua.decoder.ff.FeatureFunction}'s
+ */
+ @Override
+ public void addOOVRules(int sourceWord, List<FeatureFunction> featureFunctions) {
+
+ // TODO: _OOV shouldn't be outright added, since the word might not be OOV for the LM (but now
+ // almost
+ // certainly is)
+ final int targetWord = this.joshuaConfiguration.mark_oovs ? Vocabulary.id(Vocabulary
+ .word(sourceWord) + "_OOV") : sourceWord;
+
+ int[] sourceWords = { sourceWord };
+ int[] targetWords = { targetWord };
+ final String oovAlignment = "0-0";
+
+ if (this.joshuaConfiguration.oovList != null && this.joshuaConfiguration.oovList.size() != 0) {
+ for (OOVItem item : this.joshuaConfiguration.oovList) {
+ Rule oovRule = new Rule(Vocabulary.id(item.label), sourceWords, targetWords, "", 0,
+ oovAlignment);
+ addRule(oovRule);
+ oovRule.estimateRuleCost(featureFunctions);
+ }
+ } else {
+ int nt_i = Vocabulary.id(this.joshuaConfiguration.default_non_terminal);
+ Rule oovRule = new Rule(nt_i, sourceWords, targetWords, "", 0, oovAlignment);
+ addRule(oovRule);
+ oovRule.estimateRuleCost(featureFunctions);
+ }
+ }
+
+ /**
+ * Adds a default set of glue rules.
+ *
+ * @param featureFunctions an {@link java.util.ArrayList} of {@link org.apache.joshua.decoder.ff.FeatureFunction}'s
+ */
+ public void addGlueRules(ArrayList<FeatureFunction> featureFunctions) {
+ HieroFormatReader reader = new HieroFormatReader();
+
+ String goalNT = FormatUtils.cleanNonTerminal(joshuaConfiguration.goal_symbol);
+ String defaultNT = FormatUtils.cleanNonTerminal(joshuaConfiguration.default_non_terminal);
+
+ String[] ruleStrings = new String[] {
+ String.format("[%s] ||| %s ||| %s ||| 0", goalNT, Vocabulary.START_SYM,
+ Vocabulary.START_SYM),
+ String.format("[%s] ||| [%s,1] [%s,2] ||| [%s,1] [%s,2] ||| -1", goalNT, goalNT, defaultNT,
+ goalNT, defaultNT),
+ String.format("[%s] ||| [%s,1] %s ||| [%s,1] %s ||| 0", goalNT, goalNT,
+ Vocabulary.STOP_SYM, goalNT, Vocabulary.STOP_SYM) };
+
+ for (String ruleString : ruleStrings) {
+ Rule rule = reader.parseLine(ruleString);
+ addRule(rule);
+ rule.estimateRuleCost(featureFunctions);
+ }
+ }
+
+ @Override
+ public int getNumDenseFeatures() {
+ return numDenseFeatures;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/hash_based/MemoryBasedRuleBin.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/hash_based/MemoryBasedRuleBin.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/hash_based/MemoryBasedRuleBin.java
new file mode 100644
index 0000000..f91df1e
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/hash_based/MemoryBasedRuleBin.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.tm.hash_based;
+
+import org.apache.joshua.decoder.ff.tm.BasicRuleCollection;
+import org.apache.joshua.decoder.ff.tm.Rule;
+
+/**
+ * Stores a collection of all rules with the same french side (and thus same arity).
+ *
+ * @author Zhifei Li, zhifei.work@gmail.com
+ */
+public class MemoryBasedRuleBin extends BasicRuleCollection {
+
+ /**
+ * Constructs an initially empty rule collection.
+ *
+ * @param arity Number of nonterminals in the source pattern
+ * @param sourceTokens Sequence of terminals and nonterminals in the source pattern
+ */
+ public MemoryBasedRuleBin(int arity, int[] sourceTokens) {
+ super(arity, sourceTokens);
+ }
+
+ /**
+ * Adds a rule to this collection.
+ *
+ * @param rule Rule to add to this collection.
+ */
+ public void addRule(Rule rule) {
+ // XXX This if clause seems bogus.
+ if (rules.size() <= 0) { // first time
+ this.arity = rule.getArity();
+ this.sourceTokens = rule.getFrench();
+ }
+ if (rule.getArity() != this.arity) {
+ return;
+ }
+ rules.add(rule);
+ sorted = false;
+ rule.setFrench(this.sourceTokens);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/hash_based/MemoryBasedTrie.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/hash_based/MemoryBasedTrie.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/hash_based/MemoryBasedTrie.java
new file mode 100644
index 0000000..998688a
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/hash_based/MemoryBasedTrie.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.tm.hash_based;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Iterator;
+
+import org.apache.joshua.decoder.ff.tm.RuleCollection;
+import org.apache.joshua.decoder.ff.tm.Trie;
+
+/**
+ * @author Zhifei Li, zhifei.work@gmail.com
+ */
+public class MemoryBasedTrie implements Trie {
+ MemoryBasedRuleBin ruleBin = null;
+ HashMap<Integer, MemoryBasedTrie> childrenTbl = null;
+
+ public MemoryBasedTrie() {
+ }
+
+ @Override
+ public Trie match(int wordID) {
+ if (childrenTbl != null)
+ return childrenTbl.get(wordID);
+ return null;
+ }
+
+ /* See Javadoc for Trie interface. */
+ public boolean hasExtensions() {
+ return (null != this.childrenTbl);
+ }
+
+ public HashMap<Integer, MemoryBasedTrie> getChildren() {
+ return this.childrenTbl;
+ }
+
+ public void setExtensions(HashMap<Integer, MemoryBasedTrie> tbl_children_) {
+ this.childrenTbl = tbl_children_;
+ }
+
+ /* See Javadoc for Trie interface. */
+ public boolean hasRules() {
+ return (null != this.ruleBin);
+ }
+
+ public void setRuleBin(MemoryBasedRuleBin rb) {
+ ruleBin = rb;
+ }
+
+ /* See Javadoc for Trie interface. */
+ public RuleCollection getRuleCollection() {
+ return this.ruleBin;
+ }
+
+ /* See Javadoc for Trie interface. */
+ public Collection<MemoryBasedTrie> getExtensions() {
+ if (this.childrenTbl != null)
+ return this.childrenTbl.values();
+ return null;
+ }
+
+ @Override
+ public Iterator<Integer> getTerminalExtensionIterator() {
+ return new ExtensionIterator(childrenTbl, true);
+ }
+
+ @Override
+ public Iterator<Integer> getNonterminalExtensionIterator() {
+ return new ExtensionIterator(childrenTbl, false);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/hash_based/package-info.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/hash_based/package-info.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/hash_based/package-info.java
new file mode 100644
index 0000000..695a0a4
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/hash_based/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Provides implementations of hierarchical phrase-based translation grammars.
+ */
+package org.apache.joshua.decoder.ff.tm.hash_based;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/package-info.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/package-info.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/package-info.java
new file mode 100644
index 0000000..b804db6
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/package-info.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Defines interfaces and provides infrastructure for hierarchical
+ * phrase-based translation grammars.
+ */
+package org.apache.joshua.decoder.ff.tm;
+
[12/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/hiero/output-classlm.gold
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/hiero/output-classlm.gold b/joshua-core/src/test/resources/bn-en/hiero/output-classlm.gold
new file mode 100644
index 0000000..066773e
--- /dev/null
+++ b/joshua-core/src/test/resources/bn-en/hiero/output-classlm.gold
@@ -0,0 +1,678 @@
+0 ||| rabindranath was born in a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-30.409 tm_pt_6=-15.712 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.135 tm_pt_10=-14.979 tm_pt_11=0.000 tm_pt_12=-7.729 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 lm_0=-23.712 lm_1=-22.418 WordPenalty=-5.212 OOVPenalty=-200.000 ||| -254.042
+0 ||| rabindranath was born in one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-29.639 tm_pt_6=-16.710 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.135 tm_pt_10=-13.438 tm_pt_11=0.000 tm_pt_12=-8.350 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 lm_0=-25.676 lm_1=-23.515 WordPenalty=-5.212 OOVPenalty=-200.000 ||| -257.842
+0 ||| rabindranath born in the a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-5.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-32.737 tm_pt_6=-16.092 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.000 tm_pt_10=-12.188 tm_pt_11=0.000 tm_pt_12=-3.876 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 lm_0=-26.980 lm_1=-24.179 WordPenalty=-5.212 OOVPenalty=-200.000 ||| -258.306
+0 ||| rabindranath born in kolkata a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-5.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-29.029 tm_pt_6=-16.002 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.000 tm_pt_10=-10.090 tm_pt_11=0.000 tm_pt_12=-4.282 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 lm_0=-27.803 lm_1=-24.299 WordPenalty=-5.212 OOVPenalty=-200.000 ||| -258.333
+0 ||| rabindranath born in the one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-5.000 tm_pt_3=0.000 tm_pt_4=-4.000 tm_pt_5=-31.967 tm_pt_6=-17.090 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.000 tm_pt_10=-10.415 tm_pt_11=0.000 tm_pt_12=-4.100 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=6.000 lm_0=-27.320 lm_1=-24.209 WordPenalty=-5.212 OOVPenalty=-200.000 ||| -259.542
+0 ||| rabindranath born in kolkata one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 in the family ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-5.000 tm_pt_3=0.000 tm_pt_4=-4.000 tm_pt_5=-28.259 tm_pt_6=-16.999 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.000 tm_pt_10=-8.317 tm_pt_11=0.000 tm_pt_12=-4.505 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=6.000 lm_0=-28.555 lm_1=-24.206 WordPenalty=-5.212 OOVPenalty=-200.000 ||| -259.925
+0 ||| rabindranath was born in a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 \u09aa\u09b0\u09bf\u09ac\u09be\u09b0\u09c7 . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-12.867 tm_pt_6=-7.153 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-0.135 tm_pt_10=-14.988 tm_pt_11=0.000 tm_pt_12=-7.732 tm_pt_13=0.000 tm_pt_14=-6.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-23.574 lm_1=-22.046 WordPenalty=-4.777 OOVPenalty=-300.000 ||| -347.574
+0 ||| rabindranath was born in one \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 \u09aa\u09b0\u09bf\u09ac\u09be\u09b0\u09c7 . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-12.097 tm_pt_6=-8.150 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-0.135 tm_pt_10=-13.447 tm_pt_11=0.000 tm_pt_12=-8.353 tm_pt_13=0.000 tm_pt_14=-6.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-25.537 lm_1=-23.143 WordPenalty=-4.777 OOVPenalty=-300.000 ||| -351.374
+0 ||| rabindranath born in the a \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 \u09aa\u09b0\u09bf\u09ac\u09be\u09b0\u09c7 . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-5.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-15.195 tm_pt_6=-7.533 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-1.000 tm_pt_10=-12.198 tm_pt_11=0.000 tm_pt_12=-3.880 tm_pt_13=0.000 tm_pt_14=-6.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-26.841 lm_1=-23.807 WordPenalty=-4.777 OOVPenalty=-300.000 ||| -351.838
+1 ||| recently with the united states relation improved . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-18.913 tm_pt_6=-15.946 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.743 tm_pt_10=-14.886 tm_pt_11=0.000 tm_pt_12=-6.633 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-18.873 lm_1=-15.933 WordPenalty=-4.343 OOVPenalty=0.000 ||| -43.205
+1 ||| recently with the united states matters improved . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-20.258 tm_pt_6=-17.332 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.743 tm_pt_10=-14.678 tm_pt_11=0.000 tm_pt_12=-6.633 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-18.768 lm_1=-15.680 WordPenalty=-4.343 OOVPenalty=0.000 ||| -43.440
+1 ||| recently india with united states relation improved . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-6.000 tm_pt_5=-15.661 tm_pt_6=-15.849 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.742 tm_pt_10=-10.885 tm_pt_11=0.000 tm_pt_12=-4.412 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=6.000 lm_0=-21.491 lm_1=-16.859 WordPenalty=-4.343 OOVPenalty=0.000 ||| -44.146
+1 ||| recently with the united states relation between improved . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-23.251 tm_pt_6=-15.828 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.743 tm_pt_10=-13.722 tm_pt_11=0.000 tm_pt_12=-6.633 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-19.500 lm_1=-17.313 WordPenalty=-4.777 OOVPenalty=0.000 ||| -44.244
+1 ||| recently india with united states matters improved . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-6.000 tm_pt_5=-17.007 tm_pt_6=-17.235 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.742 tm_pt_10=-10.677 tm_pt_11=0.000 tm_pt_12=-4.412 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=6.000 lm_0=-21.386 lm_1=-16.606 WordPenalty=-4.343 OOVPenalty=0.000 ||| -44.381
+1 ||| recently with the united states relationship between improved . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-22.357 tm_pt_6=-15.386 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.743 tm_pt_10=-12.806 tm_pt_11=0.000 tm_pt_12=-6.633 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-19.559 lm_1=-17.901 WordPenalty=-4.777 OOVPenalty=0.000 ||| -44.484
+1 ||| recently with united states with the relationship between improved . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-5.000 tm_pt_3=0.000 tm_pt_4=-3.000 tm_pt_5=-25.015 tm_pt_6=-15.386 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.742 tm_pt_10=-3.975 tm_pt_11=0.000 tm_pt_12=-2.234 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=3.000 lm_0=-21.908 lm_1=-18.571 WordPenalty=-5.212 OOVPenalty=0.000 ||| -44.540
+2 ||| mathematics so science language . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-4.000 tm_pt_3=0.000 tm_pt_4=-4.000 tm_pt_5=-6.483 tm_pt_6=-3.387 tm_pt_7=-4.000 tm_pt_8=-10.872 tm_pt_9=-0.002 tm_pt_10=-3.378 tm_pt_11=0.000 tm_pt_12=-1.626 tm_pt_13=0.000 tm_pt_14=-5.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=4.000 lm_0=-15.141 lm_1=-10.511 WordPenalty=-3.040 OOVPenalty=0.000 ||| -23.476
+2 ||| mathematics is science language . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-4.000 tm_pt_3=0.000 tm_pt_4=-4.000 tm_pt_5=-10.375 tm_pt_6=-3.926 tm_pt_7=-4.000 tm_pt_8=-10.872 tm_pt_9=-0.052 tm_pt_10=-8.326 tm_pt_11=0.000 tm_pt_12=-3.330 tm_pt_13=0.000 tm_pt_14=-5.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=4.000 lm_0=-12.890 lm_1=-10.773 WordPenalty=-3.040 OOVPenalty=0.000 ||| -23.873
+2 ||| mathematics that science language . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-4.000 tm_pt_3=0.000 tm_pt_4=-4.000 tm_pt_5=-9.625 tm_pt_6=-3.926 tm_pt_7=-4.000 tm_pt_8=-10.872 tm_pt_9=-0.052 tm_pt_10=-7.607 tm_pt_11=0.000 tm_pt_12=-3.330 tm_pt_13=0.000 tm_pt_14=-5.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=4.000 lm_0=-14.001 lm_1=-10.511 WordPenalty=-3.040 OOVPenalty=0.000 ||| -24.593
+2 ||| science mathematics that language . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-4.000 tm_pt_3=0.000 tm_pt_4=-3.000 tm_pt_5=-9.625 tm_pt_6=-3.926 tm_pt_7=-4.000 tm_pt_8=-10.872 tm_pt_9=-0.138 tm_pt_10=-2.832 tm_pt_11=0.000 tm_pt_12=-1.486 tm_pt_13=0.000 tm_pt_14=-5.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 lm_0=-14.271 lm_1=-10.297 WordPenalty=-3.040 OOVPenalty=0.000 ||| -24.806
+3 ||| from this it will be understood that the \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-3.000 tm_pt_5=-20.366 tm_pt_6=-14.416 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.389 tm_pt_10=-6.943 tm_pt_11=0.000 tm_pt_12=-4.457 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-28.474 lm_1=-26.724 WordPenalty=-6.080 OOVPenalty=-300.000 ||| -358.253
+3 ||| from this it will be understood that this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-7.000 tm_pt_3=0.000 tm_pt_4=-3.000 tm_pt_5=-17.030 tm_pt_6=-13.124 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-1.386 tm_pt_10=-8.291 tm_pt_11=0.000 tm_pt_12=-4.208 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-28.974 lm_1=-26.750 WordPenalty=-6.080 OOVPenalty=-300.000 ||| -359.239
+3 ||| from this it will be it can be understood that the \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-3.000 tm_pt_5=-28.998 tm_pt_6=-11.009 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.389 tm_pt_10=-5.844 tm_pt_11=0.000 tm_pt_12=-4.457 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-31.634 lm_1=-28.993 WordPenalty=-7.383 OOVPenalty=-300.000 ||| -359.373
+3 ||| from this it can be it will be that the \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-7.000 tm_pt_3=0.000 tm_pt_4=-3.000 tm_pt_5=-27.558 tm_pt_6=-10.958 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-1.407 tm_pt_10=-14.576 tm_pt_11=0.000 tm_pt_12=-5.598 tm_pt_13=0.000 tm_pt_14=-11.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=6.000 lm_0=-31.727 lm_1=-26.439 WordPenalty=-6.949 OOVPenalty=-300.000 ||| -360.806
+3 ||| from this it can be it will be that this \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-8.000 tm_pt_3=0.000 tm_pt_4=-3.000 tm_pt_5=-24.223 tm_pt_6=-9.666 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-1.389 tm_pt_10=-14.553 tm_pt_11=0.000 tm_pt_12=-5.139 tm_pt_13=0.000 tm_pt_14=-11.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=6.000 lm_0=-32.227 lm_1=-26.088 WordPenalty=-6.949 OOVPenalty=-300.000 ||| -360.882
+3 ||| from this it will it can be understood that the \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-3.000 tm_pt_5=-26.563 tm_pt_6=-11.056 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.524 tm_pt_10=-5.333 tm_pt_11=0.000 tm_pt_12=-5.479 tm_pt_13=0.000 tm_pt_14=-11.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=4.000 lm_0=-31.565 lm_1=-28.432 WordPenalty=-6.949 OOVPenalty=-300.000 ||| -361.417
+3 ||| from this it can be it that this will \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-8.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-21.787 tm_pt_6=-9.713 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-1.524 tm_pt_10=-14.418 tm_pt_11=0.000 tm_pt_12=-6.372 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 lm_0=-32.769 lm_1=-25.204 WordPenalty=-6.514 OOVPenalty=-300.000 ||| -361.526
+4 ||| same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match from this novel . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-9.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-16.212 tm_pt_6=-10.084 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-3.005 tm_pt_10=-12.338 tm_pt_11=0.000 tm_pt_12=-5.018 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 lm_0=-39.641 lm_1=-31.966 WordPenalty=-6.080 OOVPenalty=-200.000 ||| -277.519
+4 ||| same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 presage match from this novel . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-9.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-15.113 tm_pt_6=-10.084 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-3.005 tm_pt_10=-12.338 tm_pt_11=0.000 tm_pt_12=-5.018 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 lm_0=-39.916 lm_1=-31.966 WordPenalty=-6.080 OOVPenalty=-200.000 ||| -277.649
+4 ||| same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match from this novels . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-9.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-15.164 tm_pt_6=-9.637 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-3.005 tm_pt_10=-11.182 tm_pt_11=0.000 tm_pt_12=-4.651 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 lm_0=-41.798 lm_1=-31.966 WordPenalty=-6.080 OOVPenalty=-200.000 ||| -279.404
+4 ||| same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 presage match from this novels . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-9.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-14.066 tm_pt_6=-9.637 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-3.005 tm_pt_10=-11.182 tm_pt_11=0.000 tm_pt_12=-4.651 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 lm_0=-42.073 lm_1=-31.966 WordPenalty=-6.080 OOVPenalty=-200.000 ||| -279.534
+4 ||| with the same earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match from this novel . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-9.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-16.212 tm_pt_6=-10.084 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-3.373 tm_pt_10=-14.188 tm_pt_11=0.000 tm_pt_12=-7.809 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 lm_0=-37.796 lm_1=-32.110 WordPenalty=-6.080 OOVPenalty=-200.000 ||| -280.076
+4 ||| with the same earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 presage match from this novel . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-9.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-15.113 tm_pt_6=-10.084 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-3.373 tm_pt_10=-14.188 tm_pt_11=0.000 tm_pt_12=-7.809 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 lm_0=-38.071 lm_1=-32.110 WordPenalty=-6.080 OOVPenalty=-200.000 ||| -280.206
+4 ||| the with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match from this novel . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-9.000 tm_pt_3=0.000 tm_pt_4=-6.000 tm_pt_5=-20.640 tm_pt_6=-9.983 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-3.024 tm_pt_10=-18.514 tm_pt_11=0.000 tm_pt_12=-6.668 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-37.546 lm_1=-32.463 WordPenalty=-6.080 OOVPenalty=-200.000 ||| -280.483
+4 ||| the same with the earthcentered \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 indication match from this novel . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-9.000 tm_pt_3=0.000 tm_pt_4=-6.000 tm_pt_5=-21.437 tm_pt_6=-10.032 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-3.005 tm_pt_10=-12.086 tm_pt_11=0.000 tm_pt_12=-4.843 tm_pt_13=0.000 tm_pt_14=-11.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-39.669 lm_1=-33.675 WordPenalty=-6.514 OOVPenalty=-200.000 ||| -280.518
+5 ||| \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 majority in the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-7.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-16.310 tm_pt_6=-6.695 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-0.289 tm_pt_10=-10.344 tm_pt_11=0.000 tm_pt_12=-2.428 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=9.000 lm_0=-25.263 lm_1=-22.356 WordPenalty=-5.212 OOVPenalty=-200.000 ||| -245.557
+5 ||| \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 majority that the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-6.000 tm_pt_5=-16.018 tm_pt_6=-7.571 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.271 tm_pt_10=-5.619 tm_pt_11=0.000 tm_pt_12=-1.161 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-27.702 lm_1=-22.018 WordPenalty=-5.212 OOVPenalty=-200.000 ||| -247.342
+5 ||| \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 majority in . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-13.102 tm_pt_6=-8.482 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-0.289 tm_pt_10=-14.216 tm_pt_11=0.000 tm_pt_12=-2.256 tm_pt_13=0.000 tm_pt_14=-7.000 tm_pt_15=-1.000 tm_pt_16=0.000 tm_glue_0=7.000 lm_0=-25.124 lm_1=-21.005 WordPenalty=-4.777 OOVPenalty=-200.000 ||| -247.913
+5 ||| mujib and his party \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 majority in the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-7.000 tm_pt_3=0.000 tm_pt_4=-4.000 tm_pt_5=-16.310 tm_pt_6=-6.695 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-0.657 tm_pt_10=-10.233 tm_pt_11=0.000 tm_pt_12=-5.719 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-24.288 lm_1=-22.355 WordPenalty=-5.212 OOVPenalty=-200.000 ||| -249.681
+5 ||| \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 . majority in the ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-7.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-16.310 tm_pt_6=-6.695 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-0.370 tm_pt_10=-9.011 tm_pt_11=0.000 tm_pt_12=-4.591 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=6.000 lm_0=-28.212 lm_1=-20.384 WordPenalty=-5.212 OOVPenalty=-200.000 ||| -250.181
+5 ||| \u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 mujib and his party \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 the . majority in ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-16.310 tm_pt_6=-6.695 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.271 tm_pt_10=-2.153 tm_pt_11=0.000 tm_pt_12=-0.468 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=6.000 lm_0=-30.418 lm_1=-22.020 WordPenalty=-5.212 OOVPenalty=-200.000 ||| -251.282
+6 ||| \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his work with to that . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-6.000 tm_pt_5=-22.509 tm_pt_6=-11.163 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.370 tm_pt_10=-18.845 tm_pt_11=0.000 tm_pt_12=-2.681 tm_pt_13=0.000 tm_pt_14=-6.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 lm_0=-33.425 lm_1=-25.130 WordPenalty=-5.212 OOVPenalty=-400.000 ||| -463.452
+6 ||| \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his work with a that . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-24.229 tm_pt_6=-13.109 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.418 tm_pt_10=-18.986 tm_pt_11=0.000 tm_pt_12=-2.612 tm_pt_13=0.000 tm_pt_14=-6.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=9.000 lm_0=-33.108 lm_1=-25.595 WordPenalty=-5.212 OOVPenalty=-400.000 ||| -465.645
+6 ||| \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his work with that can . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-19.242 tm_pt_6=-9.832 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-2.368 tm_pt_10=-17.376 tm_pt_11=0.000 tm_pt_12=-3.305 tm_pt_13=0.000 tm_pt_14=-6.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=9.000 lm_0=-33.973 lm_1=-25.956 WordPenalty=-5.212 OOVPenalty=-400.000 ||| -465.694
+6 ||| \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his work with can that . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-6.000 tm_pt_5=-19.242 tm_pt_6=-9.832 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.503 tm_pt_10=-17.011 tm_pt_11=0.000 tm_pt_12=-3.528 tm_pt_13=0.000 tm_pt_14=-6.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 lm_0=-35.292 lm_1=-26.410 WordPenalty=-5.212 OOVPenalty=-400.000 ||| -466.199
+6 ||| \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 his work could that with . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-19.925 tm_pt_6=-12.193 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-1.736 tm_pt_10=-13.793 tm_pt_11=0.000 tm_pt_12=-1.919 tm_pt_13=0.000 tm_pt_14=-6.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-35.742 lm_1=-24.982 WordPenalty=-5.212 OOVPenalty=-400.000 ||| -466.677
+6 ||| \u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 its work with that can . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-6.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-20.130 tm_pt_6=-11.794 tm_pt_7=-6.000 tm_pt_8=-16.308 tm_pt_9=-2.368 tm_pt_10=-17.845 tm_pt_11=0.000 tm_pt_12=-5.310 tm_pt_13=0.000 tm_pt_14=-6.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=9.000 lm_0=-34.521 lm_1=-25.533 WordPenalty=-5.212 OOVPenalty=-400.000 ||| -467.521
+7 ||| task , \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character is but these a is not . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-40.521 tm_pt_6=-16.440 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-1.889 tm_pt_10=-28.483 tm_pt_11=0.000 tm_pt_12=-9.906 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=15.000 lm_0=-46.205 lm_1=-34.518 WordPenalty=-7.817 OOVPenalty=-200.000 ||| -292.687
+7 ||| task , \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character is but these very is not . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-36.805 tm_pt_6=-15.372 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-1.871 tm_pt_10=-24.380 tm_pt_11=0.000 tm_pt_12=-9.030 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=15.000 lm_0=-47.390 lm_1=-35.336 WordPenalty=-7.817 OOVPenalty=-200.000 ||| -292.701
+7 ||| task , \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character are but these very is not . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-35.727 tm_pt_6=-15.118 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-1.871 tm_pt_10=-23.123 tm_pt_11=0.000 tm_pt_12=-8.647 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=15.000 lm_0=-48.326 lm_1=-35.181 WordPenalty=-7.817 OOVPenalty=-200.000 ||| -292.912
+7 ||| task , \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and some more linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character is but these a is not . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-12.000 tm_pt_5=-38.538 tm_pt_6=-15.147 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-1.892 tm_pt_10=-28.563 tm_pt_11=0.000 tm_pt_12=-10.079 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=14.000 lm_0=-46.230 lm_1=-34.518 WordPenalty=-7.817 OOVPenalty=-200.000 ||| -292.945
+7 ||| task , \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and some more linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character is but these very is not . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-12.000 tm_pt_5=-34.821 tm_pt_6=-14.079 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-1.874 tm_pt_10=-24.460 tm_pt_11=0.000 tm_pt_12=-9.204 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=14.000 lm_0=-47.416 lm_1=-35.336 WordPenalty=-7.817 OOVPenalty=-200.000 ||| -292.960
+7 ||| task , \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and some more linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character are but these very is not . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-12.000 tm_pt_5=-33.743 tm_pt_6=-13.825 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-1.874 tm_pt_10=-23.203 tm_pt_11=0.000 tm_pt_12=-8.821 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=14.000 lm_0=-48.352 lm_1=-35.181 WordPenalty=-7.817 OOVPenalty=-200.000 ||| -293.171
+7 ||| task , \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character are but these is not common . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-12.000 tm_pt_3=0.000 tm_pt_4=-11.000 tm_pt_5=-36.984 tm_pt_6=-17.526 tm_pt_7=-12.000 tm_pt_8=-32.616 tm_pt_9=-2.871 tm_pt_10=-24.262 tm_pt_11=0.000 tm_pt_12=-8.564 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=13.000 lm_0=-47.411 lm_1=-33.956 WordPenalty=-7.817 OOVPenalty=-200.000 ||| -293.188
+7 ||| task , \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and also some linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character there but these very is not . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-34.439 tm_pt_6=-14.939 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-1.871 tm_pt_10=-22.476 tm_pt_11=0.000 tm_pt_12=-9.541 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=15.000 lm_0=-48.680 lm_1=-35.181 WordPenalty=-7.817 OOVPenalty=-200.000 ||| -293.191
+7 ||| task , \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and some more linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character are but these is not common . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-12.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-35.001 tm_pt_6=-16.233 tm_pt_7=-12.000 tm_pt_8=-32.616 tm_pt_9=-2.874 tm_pt_10=-24.342 tm_pt_11=0.000 tm_pt_12=-8.737 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=12.000 lm_0=-47.437 lm_1=-33.956 WordPenalty=-7.817 OOVPenalty=-200.000 ||| -293.447
+7 ||| task , \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 and some more linux \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 character there but these very is not . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-12.000 tm_pt_5=-32.456 tm_pt_6=-13.646 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-1.874 tm_pt_10=-22.557 tm_pt_11=0.000 tm_pt_12=-9.715 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=14.000 lm_0=-48.706 lm_1=-35.181 WordPenalty=-7.817 OOVPenalty=-200.000 ||| -293.450
+8 ||| it social situation in his oppositions \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making with the rules of the the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-14.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-51.949 tm_pt_6=-23.028 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-4.346 tm_pt_10=-40.862 tm_pt_11=0.000 tm_pt_12=-16.582 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 lm_0=-44.904 lm_1=-32.036 WordPenalty=-8.252 OOVPenalty=-200.000 ||| -303.147
+8 ||| it social situation in his oppositions \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making with the rule of the the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-14.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-52.547 tm_pt_6=-22.440 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-4.218 tm_pt_10=-41.230 tm_pt_11=0.000 tm_pt_12=-15.889 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 lm_0=-44.342 lm_1=-33.010 WordPenalty=-8.252 OOVPenalty=-200.000 ||| -303.315
+8 ||| it social situation in his oppositions \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making with the rule of the in the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-14.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-58.491 tm_pt_6=-22.780 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-4.303 tm_pt_10=-38.507 tm_pt_11=0.000 tm_pt_12=-16.177 tm_pt_13=0.000 tm_pt_14=-16.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 lm_0=-45.015 lm_1=-33.550 WordPenalty=-8.686 OOVPenalty=-200.000 ||| -303.603
+8 ||| it social situation in his oppositions \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making with the rule of the fingers . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-48.497 tm_pt_6=-24.838 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-4.536 tm_pt_10=-34.420 tm_pt_11=0.000 tm_pt_12=-15.157 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 lm_0=-45.197 lm_1=-34.200 WordPenalty=-8.252 OOVPenalty=-200.000 ||| -303.936
+8 ||| it social situation in his oppositions \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making with the rules of the is . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-14.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-50.981 tm_pt_6=-23.411 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-4.298 tm_pt_10=-38.862 tm_pt_11=0.000 tm_pt_12=-15.889 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 lm_0=-45.232 lm_1=-33.325 WordPenalty=-8.252 OOVPenalty=-200.000 ||| -304.291
+8 ||| this social situation in his oppositions \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making with the rule of the fingers . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-50.091 tm_pt_6=-25.785 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-4.536 tm_pt_10=-36.353 tm_pt_11=0.000 tm_pt_12=-15.790 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 lm_0=-44.436 lm_1=-34.295 WordPenalty=-8.252 OOVPenalty=-200.000 ||| -304.466
+8 ||| it social situation in his oppositions \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making with the rules of the fingers . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-47.899 tm_pt_6=-25.426 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-4.665 tm_pt_10=-34.052 tm_pt_11=0.000 tm_pt_12=-15.850 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 lm_0=-45.758 lm_1=-33.931 WordPenalty=-8.252 OOVPenalty=-200.000 ||| -304.640
+8 ||| it social situation in his oppositions \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making with the rule of the is . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-14.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-51.579 tm_pt_6=-22.823 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-4.169 tm_pt_10=-39.230 tm_pt_11=0.000 tm_pt_12=-15.196 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 lm_0=-44.670 lm_1=-34.452 WordPenalty=-8.252 OOVPenalty=-200.000 ||| -304.647
+8 ||| this social situation in his oppositions \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making with the rules of the is . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-14.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-52.574 tm_pt_6=-24.358 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-4.298 tm_pt_10=-40.795 tm_pt_11=0.000 tm_pt_12=-16.521 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 lm_0=-44.472 lm_1=-33.420 WordPenalty=-8.252 OOVPenalty=-200.000 ||| -304.820
+8 ||| this social situation in his oppositions \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 decision making with the rules of the fingers . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-49.492 tm_pt_6=-26.373 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-4.665 tm_pt_10=-35.985 tm_pt_11=0.000 tm_pt_12=-16.483 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 lm_0=-44.998 lm_1=-34.026 WordPenalty=-8.252 OOVPenalty=-200.000 ||| -305.170
+9 ||| \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 annual \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf meters ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-3.372 tm_pt_6=-3.054 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.368 tm_pt_10=-2.140 tm_pt_11=0.000 tm_pt_12=-1.263 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-26.340 lm_1=-17.443 WordPenalty=-3.040 OOVPenalty=-300.000 ||| -342.660
+9 ||| \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 annual \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf meter ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-3.885 tm_pt_6=-2.821 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.368 tm_pt_10=-2.140 tm_pt_11=0.000 tm_pt_12=-1.337 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-26.316 lm_1=-17.443 WordPenalty=-3.040 OOVPenalty=-300.000 ||| -342.676
+9 ||| \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 arrange \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf meters ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-2.916 tm_pt_6=-3.748 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-1.000 tm_pt_10=0.000 tm_pt_11=0.000 tm_pt_12=-1.956 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-26.532 lm_1=-17.735 WordPenalty=-3.040 OOVPenalty=-300.000 ||| -343.618
+9 ||| \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 arrange \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf meter ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-3.430 tm_pt_6=-3.514 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-1.000 tm_pt_10=0.000 tm_pt_11=0.000 tm_pt_12=-2.030 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-26.509 lm_1=-17.735 WordPenalty=-3.040 OOVPenalty=-300.000 ||| -343.633
+9 ||| \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 annual \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf is ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-10.780 tm_pt_6=-4.900 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-1.368 tm_pt_10=-9.020 tm_pt_11=0.000 tm_pt_12=-3.902 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-24.572 lm_1=-16.003 WordPenalty=-3.040 OOVPenalty=-300.000 ||| -344.278
+9 ||| \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 annual \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf above ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-6.156 tm_pt_6=-5.306 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-1.368 tm_pt_10=-2.833 tm_pt_11=0.000 tm_pt_12=-3.902 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-26.244 lm_1=-16.235 WordPenalty=-3.040 OOVPenalty=-300.000 ||| -344.284
+9 ||| \u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 annual \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf metres ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-3.608 tm_pt_6=-4.389 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.503 tm_pt_10=-2.140 tm_pt_11=0.000 tm_pt_12=-2.803 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-26.771 lm_1=-17.443 WordPenalty=-3.040 OOVPenalty=-300.000 ||| -344.352
+10 ||| in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f in the national conference was he the speech -lrb- keynote speech -rrb- , the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-15.000 tm_pt_3=0.000 tm_pt_4=-12.000 tm_pt_5=-52.291 tm_pt_6=-17.578 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-1.738 tm_pt_10=-37.639 tm_pt_11=0.000 tm_pt_12=-11.291 tm_pt_13=0.000 tm_pt_14=-16.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=17.000 lm_0=-63.388 lm_1=-50.340 WordPenalty=-10.423 OOVPenalty=-600.000 ||| -728.897
+10 ||| in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 he in national \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f party was the main speech -lrb- keynote speech -rrb- , the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-15.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-41.284 tm_pt_6=-16.735 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-1.525 tm_pt_10=-31.599 tm_pt_11=0.000 tm_pt_12=-10.353 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=13.000 lm_0=-63.061 lm_1=-49.582 WordPenalty=-9.989 OOVPenalty=-600.000 ||| -728.950
+10 ||| in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f in the national conference was he the speech -lrb- keynote speech -rrb- on the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-15.000 tm_pt_3=0.000 tm_pt_4=-12.000 tm_pt_5=-50.081 tm_pt_6=-17.884 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-1.420 tm_pt_10=-33.895 tm_pt_11=0.000 tm_pt_12=-10.598 tm_pt_13=0.000 tm_pt_14=-16.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=17.000 lm_0=-64.518 lm_1=-50.766 WordPenalty=-10.423 OOVPenalty=-600.000 ||| -729.027
+10 ||| in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 he in national \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f party was the main speech -lrb- keynote speech -rrb- on the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-15.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-39.075 tm_pt_6=-17.041 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-1.207 tm_pt_10=-27.855 tm_pt_11=0.000 tm_pt_12=-9.660 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=13.000 lm_0=-64.192 lm_1=-50.008 WordPenalty=-9.989 OOVPenalty=-600.000 ||| -729.081
+10 ||| in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f in the national was he the speech -lrb- keynote speech -rrb- , the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-15.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-49.789 tm_pt_6=-17.578 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-2.056 tm_pt_10=-43.723 tm_pt_11=0.000 tm_pt_12=-10.805 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=18.000 lm_0=-61.947 lm_1=-50.201 WordPenalty=-9.989 OOVPenalty=-600.000 ||| -729.099
+10 ||| in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in national \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f party was the he the speech -lrb- keynote speech -rrb- , the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-15.000 tm_pt_3=0.000 tm_pt_4=-11.000 tm_pt_5=-45.731 tm_pt_6=-17.483 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-1.524 tm_pt_10=-36.236 tm_pt_11=0.000 tm_pt_12=-9.345 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=15.000 lm_0=-64.100 lm_1=-48.774 WordPenalty=-9.989 OOVPenalty=-600.000 ||| -729.223
+10 ||| in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f in the national was he the speech -lrb- keynote speech -rrb- on the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-15.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-47.579 tm_pt_6=-17.884 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-1.738 tm_pt_10=-39.979 tm_pt_11=0.000 tm_pt_12=-10.112 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=18.000 lm_0=-63.078 lm_1=-50.627 WordPenalty=-9.989 OOVPenalty=-600.000 ||| -729.230
+10 ||| in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f in the national conference was he main speech -lrb- keynote speech -rrb- , the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-15.000 tm_pt_3=0.000 tm_pt_4=-12.000 tm_pt_5=-47.844 tm_pt_6=-16.831 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-1.738 tm_pt_10=-33.032 tm_pt_11=0.000 tm_pt_12=-9.306 tm_pt_13=0.000 tm_pt_14=-16.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=17.000 lm_0=-65.045 lm_1=-51.497 WordPenalty=-10.423 OOVPenalty=-600.000 ||| -729.376
+10 ||| in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in national \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f party was the he main speech -lrb- keynote speech -rrb- , the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-15.000 tm_pt_3=0.000 tm_pt_4=-11.000 tm_pt_5=-41.284 tm_pt_6=-16.735 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-1.524 tm_pt_10=-31.629 tm_pt_11=0.000 tm_pt_12=-7.360 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=15.000 lm_0=-65.757 lm_1=-49.826 WordPenalty=-9.989 OOVPenalty=-600.000 ||| -729.571
+10 ||| in 2004 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 in \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f in the national was he main speech -lrb- keynote speech -rrb- , the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-15.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-45.343 tm_pt_6=-16.831 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-2.056 tm_pt_10=-39.115 tm_pt_11=0.000 tm_pt_12=-8.820 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=18.000 lm_0=-63.604 lm_1=-51.358 WordPenalty=-9.989 OOVPenalty=-600.000 ||| -729.578
+11 ||| population , power of east pakistan where to west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac they started the where the west pakistan was considered as a province . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-24.000 tm_pt_3=0.000 tm_pt_4=-21.000 tm_pt_5=-83.737 tm_pt_6=-46.104 tm_pt_7=-24.000 tm_pt_8=-65.232 tm_pt_9=-5.578 tm_pt_10=-65.560 tm_pt_11=0.000 tm_pt_12=-18.490 tm_pt_13=0.000 tm_pt_14=-30.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=21.000 lm_0=-67.857 lm_1=-46.089 WordPenalty=-14.332 OOVPenalty=-100.000 ||| -245.539
+11 ||| population , power of east pakistan where to west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac started with the where the west pakistan was considered as a province . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-24.000 tm_pt_3=0.000 tm_pt_4=-20.000 tm_pt_5=-85.129 tm_pt_6=-46.104 tm_pt_7=-24.000 tm_pt_8=-65.232 tm_pt_9=-5.896 tm_pt_10=-66.642 tm_pt_11=0.000 tm_pt_12=-18.372 tm_pt_13=0.000 tm_pt_14=-30.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=20.000 lm_0=-66.308 lm_1=-46.684 WordPenalty=-14.332 OOVPenalty=-100.000 ||| -246.090
+11 ||| population , power of east pakistan where to west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac given started the where the west pakistan was considered as a province . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-24.000 tm_pt_3=0.000 tm_pt_4=-20.000 tm_pt_5=-83.065 tm_pt_6=-46.104 tm_pt_7=-24.000 tm_pt_8=-65.232 tm_pt_9=-4.713 tm_pt_10=-61.832 tm_pt_11=0.000 tm_pt_12=-17.391 tm_pt_13=0.000 tm_pt_14=-30.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=20.000 lm_0=-69.330 lm_1=-46.228 WordPenalty=-14.332 OOVPenalty=-100.000 ||| -246.304
+11 ||| population , power of east pakistan where to west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac they started the where the regarded as a province west pakistan . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-25.000 tm_pt_3=0.000 tm_pt_4=-21.000 tm_pt_5=-81.357 tm_pt_6=-47.996 tm_pt_7=-25.000 tm_pt_8=-67.950 tm_pt_9=-4.630 tm_pt_10=-63.514 tm_pt_11=0.000 tm_pt_12=-16.959 tm_pt_13=0.000 tm_pt_14=-29.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=21.000 lm_0=-66.590 lm_1=-47.021 WordPenalty=-13.897 OOVPenalty=-100.000 ||| -246.554
+11 ||| population , power of east pakistan where to west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac with started the where the west pakistan was considered as a province . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-24.000 tm_pt_3=0.000 tm_pt_4=-21.000 tm_pt_5=-85.129 tm_pt_6=-46.104 tm_pt_7=-24.000 tm_pt_8=-65.232 tm_pt_9=-5.578 tm_pt_10=-66.463 tm_pt_11=0.000 tm_pt_12=-18.490 tm_pt_13=0.000 tm_pt_14=-30.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=21.000 lm_0=-68.152 lm_1=-46.328 WordPenalty=-14.332 OOVPenalty=-100.000 ||| -246.701
+11 ||| population on power of east pakistan where to west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac started with the where the west pakistan was considered as a province . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-25.000 tm_pt_3=0.000 tm_pt_4=-21.000 tm_pt_5=-81.148 tm_pt_6=-44.637 tm_pt_7=-25.000 tm_pt_8=-67.950 tm_pt_9=-6.528 tm_pt_10=-64.590 tm_pt_11=0.000 tm_pt_12=-19.542 tm_pt_13=0.000 tm_pt_14=-30.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=21.000 lm_0=-67.384 lm_1=-46.965 WordPenalty=-14.332 OOVPenalty=-100.000 ||| -247.096
+11 ||| population , power of east pakistan where to west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac they started the where the considered as a province west pakistan . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-25.000 tm_pt_3=0.000 tm_pt_4=-21.000 tm_pt_5=-81.467 tm_pt_6=-46.438 tm_pt_7=-25.000 tm_pt_8=-67.950 tm_pt_9=-5.262 tm_pt_10=-65.817 tm_pt_11=0.000 tm_pt_12=-17.653 tm_pt_13=0.000 tm_pt_14=-29.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=21.000 lm_0=-66.450 lm_1=-47.021 WordPenalty=-13.897 OOVPenalty=-100.000 ||| -247.231
+11 ||| population , power of east pakistan where to west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac given started the where the regarded as a province west pakistan . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-25.000 tm_pt_3=0.000 tm_pt_4=-20.000 tm_pt_5=-80.686 tm_pt_6=-47.996 tm_pt_7=-25.000 tm_pt_8=-67.950 tm_pt_9=-3.765 tm_pt_10=-59.786 tm_pt_11=0.000 tm_pt_12=-15.861 tm_pt_13=0.000 tm_pt_14=-29.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=20.000 lm_0=-68.063 lm_1=-47.160 WordPenalty=-13.897 OOVPenalty=-100.000 ||| -247.319
+11 ||| population , power of east pakistan where to west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac with started the where the regarded as a province west pakistan . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-25.000 tm_pt_3=0.000 tm_pt_4=-21.000 tm_pt_5=-82.750 tm_pt_6=-47.996 tm_pt_7=-25.000 tm_pt_8=-67.950 tm_pt_9=-4.630 tm_pt_10=-64.417 tm_pt_11=0.000 tm_pt_12=-16.959 tm_pt_13=0.000 tm_pt_14=-29.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=21.000 lm_0=-66.885 lm_1=-47.260 WordPenalty=-13.897 OOVPenalty=-100.000 ||| -247.715
+11 ||| population , power of east pakistan where to west pakistan " one unit theory " is a \u0985\u09ad\u09bf\u09a8\u09ac given started the where the considered as a province west pakistan . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-25.000 tm_pt_3=0.000 tm_pt_4=-20.000 tm_pt_5=-80.795 tm_pt_6=-46.438 tm_pt_7=-25.000 tm_pt_8=-67.950 tm_pt_9=-4.397 tm_pt_10=-62.089 tm_pt_11=0.000 tm_pt_12=-16.554 tm_pt_13=0.000 tm_pt_14=-29.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=20.000 lm_0=-67.923 lm_1=-47.160 WordPenalty=-13.897 OOVPenalty=-100.000 ||| -247.996
+12 ||| mathematical \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 theory ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-2.869 tm_pt_6=-2.890 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-1.000 tm_pt_10=-4.888 tm_pt_11=0.000 tm_pt_12=-2.010 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=3.000 lm_0=-12.665 lm_1=-9.989 WordPenalty=-2.171 OOVPenalty=-100.000 ||| -123.067
+12 ||| the \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 theory ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-9.309 tm_pt_6=-3.988 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.368 tm_pt_10=-7.916 tm_pt_11=0.000 tm_pt_12=-1.316 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=3.000 lm_0=-11.112 lm_1=-10.414 WordPenalty=-2.171 OOVPenalty=-100.000 ||| -123.301
+12 ||| . \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 theory ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-1.000 tm_pt_5=-7.900 tm_pt_6=-2.990 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.018 tm_pt_10=-3.641 tm_pt_11=0.000 tm_pt_12=-1.712 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 lm_0=-12.758 lm_1=-10.015 WordPenalty=-2.171 OOVPenalty=-100.000 ||| -123.942
+12 ||| \u2022 \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 theory ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-1.046 tm_pt_6=-5.241 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.368 tm_pt_10=-0.422 tm_pt_11=0.000 tm_pt_12=-1.316 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=3.000 lm_0=-14.217 lm_1=-10.419 WordPenalty=-2.171 OOVPenalty=-100.000 ||| -124.052
+13 ||| external links of ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-7.390 tm_pt_6=-2.729 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.018 tm_pt_10=-5.285 tm_pt_11=0.000 tm_pt_12=-1.611 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 lm_0=-6.986 lm_1=-5.946 WordPenalty=-2.171 OOVPenalty=0.000 ||| -11.675
+13 ||| external link of ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-8.091 tm_pt_6=-2.871 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.018 tm_pt_10=-5.285 tm_pt_11=0.000 tm_pt_12=-2.767 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 lm_0=-7.533 lm_1=-5.946 WordPenalty=-2.171 OOVPenalty=0.000 ||| -12.944
+13 ||| outer link of ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-8.247 tm_pt_6=-3.617 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.018 tm_pt_10=-5.285 tm_pt_11=0.000 tm_pt_12=-3.202 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 lm_0=-7.249 lm_1=-5.946 WordPenalty=-2.171 OOVPenalty=0.000 ||| -13.031
+13 ||| external communication of ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-8.265 tm_pt_6=-2.886 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.018 tm_pt_10=-5.285 tm_pt_11=0.000 tm_pt_12=-2.555 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 lm_0=-7.692 lm_1=-5.946 WordPenalty=-2.171 OOVPenalty=0.000 ||| -13.104
+13 ||| outer communication of ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-8.420 tm_pt_6=-3.648 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.018 tm_pt_10=-5.285 tm_pt_11=0.000 tm_pt_12=-3.297 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 lm_0=-7.553 lm_1=-5.946 WordPenalty=-2.171 OOVPenalty=0.000 ||| -13.484
+13 ||| external connection of ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-8.551 tm_pt_6=-3.029 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.037 tm_pt_10=-5.285 tm_pt_11=0.000 tm_pt_12=-3.990 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 lm_0=-8.109 lm_1=-5.946 WordPenalty=-2.171 OOVPenalty=0.000 ||| -14.247
+13 ||| out-links of ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-6.938 tm_pt_6=-4.795 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.018 tm_pt_10=-5.285 tm_pt_11=0.000 tm_pt_12=-3.297 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 lm_0=-8.078 lm_1=-4.396 WordPenalty=-1.737 OOVPenalty=0.000 ||| -14.259
+14 ||| tata communication " foreign sanchar nigam limited building is the telecommunication system of a main providers ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-11.000 tm_pt_5=-40.840 tm_pt_6=-15.009 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-3.507 tm_pt_10=-27.395 tm_pt_11=0.000 tm_pt_12=-13.114 tm_pt_13=0.000 tm_pt_14=-16.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 lm_0=-45.756 lm_1=-35.626 WordPenalty=-7.817 OOVPenalty=0.000 ||| -98.525
+14 ||| tata communication " foreign sanchar nigam limited building is the telecommunication system of a the providers ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-11.000 tm_pt_5=-44.626 tm_pt_6=-15.096 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-3.507 tm_pt_10=-30.934 tm_pt_11=0.000 tm_pt_12=-14.030 tm_pt_13=0.000 tm_pt_14=-16.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 lm_0=-45.036 lm_1=-35.036 WordPenalty=-7.817 OOVPenalty=0.000 ||| -98.899
+14 ||| tata communication " foreign sanchar nigam limited building it the telecommunication system of a main providers ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-38.870 tm_pt_6=-14.378 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-2.524 tm_pt_10=-23.054 tm_pt_11=0.000 tm_pt_12=-10.369 tm_pt_13=0.000 tm_pt_14=-16.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 lm_0=-47.786 lm_1=-36.147 WordPenalty=-7.817 OOVPenalty=0.000 ||| -99.150
+14 ||| tata communication " foreign sanchar nigam limited building it the telecommunication system of a the providers ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-11.000 tm_pt_5=-42.656 tm_pt_6=-14.466 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-3.506 tm_pt_10=-27.222 tm_pt_11=0.000 tm_pt_12=-11.190 tm_pt_13=0.000 tm_pt_14=-16.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 lm_0=-47.066 lm_1=-35.556 WordPenalty=-7.817 OOVPenalty=0.000 ||| -99.494
+14 ||| tata communication " foreign sanchar nigam limited building is the telecommunication system is the main providers ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-44.366 tm_pt_6=-18.068 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-4.390 tm_pt_10=-25.931 tm_pt_11=0.000 tm_pt_12=-12.761 tm_pt_13=0.000 tm_pt_14=-16.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 lm_0=-43.128 lm_1=-36.462 WordPenalty=-7.817 OOVPenalty=0.000 ||| -99.795
+14 ||| tata communication " foreign sanchar nigam limited building is the telecommunication system is one of the providers ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-12.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-49.485 tm_pt_6=-17.961 tm_pt_7=-12.000 tm_pt_8=-32.616 tm_pt_9=-4.371 tm_pt_10=-27.543 tm_pt_11=0.000 tm_pt_12=-15.099 tm_pt_13=0.000 tm_pt_14=-17.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 lm_0=-43.606 lm_1=-37.456 WordPenalty=-8.252 OOVPenalty=0.000 ||| -100.118
+14 ||| tata communication " foreign sanchar nigam limited building is the main providers of telecommunication system is ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-12.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-43.603 tm_pt_6=-17.632 tm_pt_7=-12.000 tm_pt_8=-32.616 tm_pt_9=-3.439 tm_pt_10=-24.629 tm_pt_11=0.000 tm_pt_12=-15.084 tm_pt_13=0.000 tm_pt_14=-16.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 lm_0=-42.899 lm_1=-37.040 WordPenalty=-7.817 OOVPenalty=0.000 ||| -100.827
+15 ||| he that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national assembly in the won all and was elected as the 44th president of the united states . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-10.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-67.250 tm_pt_6=-18.027 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-2.383 tm_pt_10=-13.421 tm_pt_11=0.000 tm_pt_12=-6.471 tm_pt_13=0.000 tm_pt_14=-21.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-46.628 lm_1=-35.442 WordPenalty=-10.857 OOVPenalty=-200.000 ||| -285.117
+15 ||| in that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national assembly in the won all and was elected as the 44th president of the united states . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-10.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-72.353 tm_pt_6=-21.308 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-2.383 tm_pt_10=-17.619 tm_pt_11=0.000 tm_pt_12=-10.104 tm_pt_13=0.000 tm_pt_14=-21.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-43.704 lm_1=-35.202 WordPenalty=-10.857 OOVPenalty=-200.000 ||| -285.675
+15 ||| in the year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national assembly in the won all and was elected as the 44th president of the united states . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-74.382 tm_pt_6=-21.237 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.015 tm_pt_10=-23.652 tm_pt_11=0.000 tm_pt_12=-10.678 tm_pt_13=0.000 tm_pt_14=-21.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=9.000 lm_0=-42.047 lm_1=-35.283 WordPenalty=-10.857 OOVPenalty=-200.000 ||| -285.744
+15 ||| he the year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national assembly in the won all and was elected as the 44th president of the united states . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-69.279 tm_pt_6=-17.956 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.015 tm_pt_10=-19.454 tm_pt_11=0.000 tm_pt_12=-7.046 tm_pt_13=0.000 tm_pt_14=-21.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=9.000 lm_0=-46.478 lm_1=-34.757 WordPenalty=-10.857 OOVPenalty=-200.000 ||| -286.105
+15 ||| he that year \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 national assembly in the won all and elected as the 44th president of the united states . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-10.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-62.544 tm_pt_6=-17.987 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-2.382 tm_pt_10=-14.160 tm_pt_11=0.000 tm_pt_12=-5.506 tm_pt_13=0.000 tm_pt_14=-20.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-46.630 lm_1=-35.981 WordPenalty=-10.423 OOVPenalty=-200.000 ||| -286.655
+16 ||| many indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be from upper stage ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-5.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-14.556 tm_pt_6=-12.125 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.000 tm_pt_10=-2.670 tm_pt_11=0.000 tm_pt_12=-0.912 tm_pt_13=0.000 tm_pt_14=-6.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-35.728 lm_1=-25.931 WordPenalty=-4.777 OOVPenalty=-300.000 ||| -364.209
+16 ||| many indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be to upper stage ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-5.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-16.518 tm_pt_6=-13.004 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.000 tm_pt_10=-4.474 tm_pt_11=0.000 tm_pt_12=-2.241 tm_pt_13=0.000 tm_pt_14=-6.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-35.464 lm_1=-25.936 WordPenalty=-4.777 OOVPenalty=-300.000 ||| -365.503
+16 ||| many indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be upper stage from ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-5.000 tm_pt_3=0.000 tm_pt_4=-4.000 tm_pt_5=-14.556 tm_pt_6=-12.125 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.000 tm_pt_10=-2.830 tm_pt_11=0.000 tm_pt_12=-1.650 tm_pt_13=0.000 tm_pt_14=-6.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 lm_0=-36.278 lm_1=-25.431 WordPenalty=-4.777 OOVPenalty=-300.000 ||| -365.538
+16 ||| many indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 from \u099f\u09c7\u0995\u09cd\u09b8\u09be upper stage ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-5.000 tm_pt_3=0.000 tm_pt_4=-4.000 tm_pt_5=-14.556 tm_pt_6=-12.125 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.000 tm_pt_10=-3.002 tm_pt_11=0.000 tm_pt_12=-1.537 tm_pt_13=0.000 tm_pt_14=-6.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 lm_0=-35.728 lm_1=-26.794 WordPenalty=-4.777 OOVPenalty=-300.000 ||| -366.548
+16 ||| of indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be from upper stage ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-5.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-19.990 tm_pt_6=-14.047 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.050 tm_pt_10=-6.670 tm_pt_11=0.000 tm_pt_12=-3.477 tm_pt_13=0.000 tm_pt_14=-6.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-34.630 lm_1=-26.215 WordPenalty=-4.777 OOVPenalty=-300.000 ||| -366.885
+16 ||| many the \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be from upper stage ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-5.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-18.962 tm_pt_6=-12.755 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.000 tm_pt_10=-7.024 tm_pt_11=0.000 tm_pt_12=-2.862 tm_pt_13=0.000 tm_pt_14=-6.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-35.098 lm_1=-26.535 WordPenalty=-4.777 OOVPenalty=-300.000 ||| -367.054
+16 ||| many indian \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf fighting \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be from the upper stage ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-5.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-20.408 tm_pt_6=-12.744 tm_pt_7=-5.000 tm_pt_8=-13.590 tm_pt_9=-2.000 tm_pt_10=-3.443 tm_pt_11=0.000 tm_pt_12=-4.470 tm_pt_13=0.000 tm_pt_14=-7.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-35.506 lm_1=-27.896 WordPenalty=-5.212 OOVPenalty=-300.000 ||| -367.223
+17 ||| britain writers written drama novels and stories recently scripts in \u0986\u09a6\u09c3\u09a4 . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-20.642 tm_pt_6=-10.927 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-0.874 tm_pt_10=-21.259 tm_pt_11=0.000 tm_pt_12=-8.774 tm_pt_13=0.000 tm_pt_14=-11.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=9.000 lm_0=-36.970 lm_1=-24.414 WordPenalty=-6.080 OOVPenalty=-100.000 ||| -170.879
+17 ||| britain writers the drama novels and stories recently scripts in \u0986\u09a6\u09c3\u09a4 . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-25.509 tm_pt_6=-11.095 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-0.959 tm_pt_10=-25.559 tm_pt_11=0.000 tm_pt_12=-9.061 tm_pt_13=0.000 tm_pt_14=-11.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=9.000 lm_0=-36.194 lm_1=-23.463 WordPenalty=-6.080 OOVPenalty=-100.000 ||| -171.018
+17 ||| britain writers written drama novels stories and recently scripts in \u0986\u09a6\u09c3\u09a4 . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-20.642 tm_pt_6=-10.927 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-0.874 tm_pt_10=-16.732 tm_pt_11=0.000 tm_pt_12=-5.024 tm_pt_13=0.000 tm_pt_14=-11.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 lm_0=-40.717 lm_1=-24.768 WordPenalty=-6.080 OOVPenalty=-100.000 ||| -171.523
+17 ||| britain writers written drama novels and stories recently script in \u0986\u09a6\u09c3\u09a4 . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-21.418 tm_pt_6=-10.442 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-1.824 tm_pt_10=-21.547 tm_pt_11=0.000 tm_pt_12=-10.160 tm_pt_13=0.000 tm_pt_14=-11.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=9.000 lm_0=-36.674 lm_1=-24.414 WordPenalty=-6.080 OOVPenalty=-100.000 ||| -171.840
+17 ||| britain writers the drama novels and stories recently script in \u0986\u09a6\u09c3\u09a4 . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-26.285 tm_pt_6=-10.609 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-1.909 tm_pt_10=-25.847 tm_pt_11=0.000 tm_pt_12=-10.448 tm_pt_13=0.000 tm_pt_14=-11.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=9.000 lm_0=-35.899 lm_1=-23.463 WordPenalty=-6.080 OOVPenalty=-100.000 ||| -171.980
+17 ||| britain writers written drama novel stories and recently scripts in \u0986\u09a6\u09c3\u09a4 . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-21.690 tm_pt_6=-11.374 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-0.874 tm_pt_10=-17.888 tm_pt_11=0.000 tm_pt_12=-5.391 tm_pt_13=0.000 tm_pt_14=-11.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 lm_0=-40.536 lm_1=-24.768 WordPenalty=-6.080 OOVPenalty=-100.000 ||| -172.083
+18 ||| 1919 on may month it -lrb- magazine was published . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-8.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-20.792 tm_pt_6=-13.466 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-1.167 tm_pt_10=-12.018 tm_pt_11=0.000 tm_pt_12=-7.168 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-26.824 lm_1=-17.292 WordPenalty=-5.212 OOVPenalty=0.000 ||| -48.758
+18 ||| 1919 on may month it -lrb- paper published . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-8.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-18.147 tm_pt_6=-15.056 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-1.645 tm_pt_10=-12.241 tm_pt_11=0.000 tm_pt_12=-7.456 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 lm_0=-25.565 lm_1=-16.091 WordPenalty=-4.777 OOVPenalty=0.000 ||| -49.202
+18 ||| 1919 on may month it -lrb- in published . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-8.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-21.062 tm_pt_6=-12.823 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-1.327 tm_pt_10=-17.056 tm_pt_11=0.000 tm_pt_12=-8.085 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-25.508 lm_1=-16.231 WordPenalty=-4.777 OOVPenalty=0.000 ||| -49.344
+18 ||| 1919 on may month it -lrb- magazine published . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-8.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-16.574 tm_pt_6=-13.110 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-1.284 tm_pt_10=-12.018 tm_pt_11=0.000 tm_pt_12=-7.679 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-27.629 lm_1=-16.091 WordPenalty=-4.777 OOVPenalty=0.000 ||| -49.550
+18 ||| 1919 on may , it -lrb- paper published . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-7.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-24.019 tm_pt_6=-15.257 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-2.510 tm_pt_10=-12.137 tm_pt_11=0.000 tm_pt_12=-5.525 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-23.603 lm_1=-17.056 WordPenalty=-4.777 OOVPenalty=0.000 ||| -49.933
+18 ||| 1919 on may , it -lrb- in published . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-7.000 tm_pt_3=0.000 tm_pt_4=-6.000 tm_pt_5=-26.935 tm_pt_6=-13.023 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-2.192 tm_pt_10=-16.952 tm_pt_11=0.000 tm_pt_12=-6.153 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=6.000 lm_0=-23.546 lm_1=-17.195 WordPenalty=-4.777 OOVPenalty=0.000 ||| -50.076
+18 ||| 1919 in may , it -lrb- paper published . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-7.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-26.096 tm_pt_6=-15.877 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-1.878 tm_pt_10=-11.444 tm_pt_11=0.000 tm_pt_12=-4.831 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-24.006 lm_1=-17.081 WordPenalty=-4.777 OOVPenalty=0.000 ||| -50.130
+18 ||| 1919 on may , it is -lrb- in published . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-7.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-30.789 tm_pt_6=-13.290 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-2.192 tm_pt_10=-16.360 tm_pt_11=0.000 tm_pt_12=-6.691 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-24.112 lm_1=-16.790 WordPenalty=-5.212 OOVPenalty=0.000 ||| -50.170
+18 ||| 1919 in may , it -lrb- in published . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-7.000 tm_pt_3=0.000 tm_pt_4=-6.000 tm_pt_5=-29.012 tm_pt_6=-13.644 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-1.560 tm_pt_10=-16.259 tm_pt_11=0.000 tm_pt_12=-5.460 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=6.000 lm_0=-23.949 lm_1=-17.221 WordPenalty=-4.777 OOVPenalty=0.000 ||| -50.272
+19 ||| 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium was arranged . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-21.244 tm_pt_6=-8.707 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.249 tm_pt_10=-15.730 tm_pt_11=0.000 tm_pt_12=-5.148 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=14.000 lm_0=-57.604 lm_1=-39.798 WordPenalty=-7.817 OOVPenalty=-400.000 ||| -501.538
+19 ||| 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium was organized . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-22.039 tm_pt_6=-8.841 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.249 tm_pt_10=-16.240 tm_pt_11=0.000 tm_pt_12=-5.148 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=14.000 lm_0=-57.677 lm_1=-39.749 WordPenalty=-7.817 OOVPenalty=-400.000 ||| -501.898
+19 ||| 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium organized in . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-11.000 tm_pt_5=-24.172 tm_pt_6=-9.959 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.249 tm_pt_10=-16.993 tm_pt_11=0.000 tm_pt_12=-6.193 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=15.000 lm_0=-58.336 lm_1=-39.009 WordPenalty=-7.817 OOVPenalty=-400.000 ||| -502.189
+19 ||| 2005 , \u0989\u0987\u09ae\u09c7\u09a8\u09b8 tennis association tour \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f open netaji indoor stadium organized was . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-11.000 tm_pt_5=-22.039 tm_pt_6=-8.841 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.249 tm_pt_10=-15.520 tm_pt_11=0.000 tm_pt_12=-5.062 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=15.000 lm_0=-59.983 lm_1=-39.504 WordPenalty=-7.817 OOVPenalty=-400.000 ||| -503.267
+20 ||| to prevent this several measures are taken . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-1.000 tm_pt_3=0.000 tm_pt_4=-1.000 tm_pt_5=-22.680 tm_pt_6=-30.812 tm_pt_7=-1.000 tm_pt_8=-2.718 tm_pt_9=-1.000 tm_pt_10=0.000 tm_pt_11=0.000 tm_pt_12=-1.386 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=1.000 lm_0=-11.632 lm_1=-10.983 WordPenalty=-4.343 OOVPenalty=0.000 ||| -25.678
+20 ||| to avoid this possibility several measures are taken . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-1.000 tm_pt_3=0.000 tm_pt_4=-1.000 tm_pt_5=-24.597 tm_pt_6=-31.733 tm_pt_7=-1.000 tm_pt_8=-2.718 tm_pt_9=-1.000 tm_pt_10=0.000 tm_pt_11=0.000 tm_pt_12=-1.386 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=1.000 lm_0=-13.461 lm_1=-12.428 WordPenalty=-4.777 OOVPenalty=0.000 ||| -28.478
+20 ||| to prevent this several measures are the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-3.000 tm_pt_3=0.000 tm_pt_4=-3.000 tm_pt_5=-24.227 tm_pt_6=-27.251 tm_pt_7=-3.000 tm_pt_8=-8.154 tm_pt_9=-1.000 tm_pt_10=-3.426 tm_pt_11=0.000 tm_pt_12=-2.285 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=3.000 lm_0=-14.066 lm_1=-12.982 WordPenalty=-4.343 OOVPenalty=0.000 ||| -31.830
+20 ||| to prevent this several measures are . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-1.000 tm_pt_5=-20.219 tm_pt_6=-29.189 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-1.000 tm_pt_10=-6.851 tm_pt_11=0.000 tm_pt_12=-1.946 tm_pt_13=0.000 tm_pt_14=-7.000 tm_pt_15=-1.000 tm_pt_16=0.000 tm_glue_0=1.000 lm_0=-12.686 lm_1=-11.625 WordPenalty=-3.909 OOVPenalty=0.000 ||| -32.367
+20 ||| to prevent this several measures are in . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-3.000 tm_pt_3=0.000 tm_pt_4=-3.000 tm_pt_5=-23.388 tm_pt_6=-27.344 tm_pt_7=-3.000 tm_pt_8=-8.154 tm_pt_9=-1.000 tm_pt_10=-2.771 tm_pt_11=0.000 tm_pt_12=-2.699 tm_pt_13=0.000 tm_pt_14=-8.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=3.000 lm_0=-14.649 lm_1=-12.982 WordPenalty=-4.343 OOVPenalty=0.000 ||| -32.400
+20 ||| to prevent this several measures are taken . . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-22.746 tm_pt_6=-30.812 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-1.368 tm_pt_10=0.000 tm_pt_11=0.000 tm_pt_12=-10.427 tm_pt_13=0.000 tm_pt_14=-9.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 lm_0=-15.278 lm_1=-12.549 WordPenalty=-4.777 OOVPenalty=0.000 ||| -33.898
+21 ||| \u09e7\u09ef\u09ec\u09ec on 5 february \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 against a national was held in . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-8.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-19.901 tm_pt_6=-11.529 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-1.135 tm_pt_10=-5.272 tm_pt_11=0.000 tm_pt_12=-3.806 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 lm_0=-42.873 lm_1=-29.512 WordPenalty=-6.514 OOVPenalty=-300.000 ||| -373.091
+21 ||| \u09e7\u09ef\u09ec\u09ec on 5th february , \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 against a national was held in . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-7.000 tm_pt_3=0.000 tm_pt_4=-5.000 tm_pt_5=-25.571 tm_pt_6=-13.159 tm_pt_7=-7.000 tm_pt_8=-19.026 tm_pt_9=-1.503 tm_pt_10=-3.423 tm_pt_11=0.000 tm_pt_12=-3.325 tm_pt_13=0.000 tm_pt_14=-11.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-42.544 lm_1=-30.355 WordPenalty=-6.949 OOVPenalty=-300.000 ||| -373.863
+21 ||| \u09e7\u09ef\u09ec\u09ec on 5 february \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 against \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 a national was held in . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-8.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-19.901 tm_pt_6=-11.529 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-1.135 tm_pt_10=-6.095 tm_pt_11=0.000 tm_pt_12=-4.029 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 lm_0=-43.433 lm_1=-30.334 WordPenalty=-6.514 OOVPenalty=-300.000 ||| -374.129
+21 ||| \u09e7\u09ef\u09ec\u09ec on 5 february \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 of \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 a national was held in . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-8.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-24.651 tm_pt_6=-11.840 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-1.018 tm_pt_10=-8.289 tm_pt_11=0.000 tm_pt_12=-3.518 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 lm_0=-42.147 lm_1=-30.745 WordPenalty=-6.514 OOVPenalty=-300.000 ||| -374.354
+21 ||| \u09e7\u09ef\u09ec\u09ec on 5 february \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 a national was held in against . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-8.000 tm_pt_3=0.000 tm_pt_4=-3.000 tm_pt_5=-19.901 tm_pt_6=-11.529 tm_pt_7=-8.000 tm_pt_8=-21.744 tm_pt_9=-0.154 tm_pt_10=-5.625 tm_pt_11=0.000 tm_pt_12=-3.764 tm_pt_13=0.000 tm_pt_14=-10.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=6.000 lm_0=-43.384 lm_1=-28.686 WordPenalty=-6.514 OOVPenalty=-300.000 ||| -375.831
+22 ||| bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in in the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-11.000 tm_pt_5=-50.512 tm_pt_6=-9.888 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.050 tm_pt_10=-23.145 tm_pt_11=0.000 tm_pt_12=-2.086 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=14.000 lm_0=-45.104 lm_1=-39.650 WordPenalty=-8.252 OOVPenalty=-300.000 ||| -390.343
+22 ||| bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank took secured its place in the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-46.603 tm_pt_6=-9.693 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.368 tm_pt_10=-19.856 tm_pt_11=0.000 tm_pt_12=-3.210 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=13.000 lm_0=-45.737 lm_1=-39.265 WordPenalty=-8.252 OOVPenalty=-300.000 ||| -390.606
+22 ||| bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in took the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-11.000 tm_pt_5=-46.603 tm_pt_6=-9.693 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.368 tm_pt_10=-19.978 tm_pt_11=0.000 tm_pt_12=-2.779 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=14.000 lm_0=-46.563 lm_1=-39.704 WordPenalty=-8.252 OOVPenalty=-300.000 ||| -391.085
+22 ||| bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in in . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-10.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-47.305 tm_pt_6=-11.674 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-2.050 tm_pt_10=-27.018 tm_pt_11=0.000 tm_pt_12=-1.913 tm_pt_13=0.000 tm_pt_14=-13.000 tm_pt_15=-1.000 tm_pt_16=0.000 tm_glue_0=12.000 lm_0=-44.173 lm_1=-37.800 WordPenalty=-7.817 OOVPenalty=-300.000 ||| -391.101
+22 ||| bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in adopted . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-10.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-42.382 tm_pt_6=-12.116 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-3.000 tm_pt_10=-21.979 tm_pt_11=0.000 tm_pt_12=-3.300 tm_pt_13=0.000 tm_pt_14=-13.000 tm_pt_15=-1.000 tm_pt_16=0.000 tm_glue_0=12.000 lm_0=-45.321 lm_1=-37.892 WordPenalty=-7.817 OOVPenalty=-300.000 ||| -391.808
+22 ||| bangladesh \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in took . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-10.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-43.395 tm_pt_6=-11.480 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-2.368 tm_pt_10=-23.851 tm_pt_11=0.000 tm_pt_12=-2.607 tm_pt_13=0.000 tm_pt_14=-13.000 tm_pt_15=-1.000 tm_pt_16=0.000 tm_glue_0=12.000 lm_0=-45.354 lm_1=-38.351 WordPenalty=-7.817 OOVPenalty=-300.000 ||| -392.115
+22 ||| \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 bangladesh of the islamic \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 and islamic \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f bank secured its place in in the . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-50.512 tm_pt_6=-9.888 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-2.050 tm_pt_10=-25.870 tm_pt_11=0.000 tm_pt_12=-5.409 tm_pt_13=0.000 tm_pt_14=-14.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=13.000 lm_0=-46.143 lm_1=-37.905 WordPenalty=-8.252 OOVPenalty=-300.000 ||| -392.315
+23 ||| subject : encyclopedia ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-1.657 tm_pt_6=-1.542 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.000 tm_pt_10=-0.420 tm_pt_11=0.000 tm_pt_12=-1.500 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 lm_0=-5.528 lm_1=-7.730 WordPenalty=-2.171 OOVPenalty=0.000 ||| -9.260
+23 ||| subject category : encyclopedia ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-1.727 tm_pt_6=-1.749 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.135 tm_pt_10=0.000 tm_pt_11=0.000 tm_pt_12=-1.881 tm_pt_13=0.000 tm_pt_14=-4.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 lm_0=-5.763 lm_1=-9.076 WordPenalty=-2.606 OOVPenalty=0.000 ||| -9.504
+23 ||| category : encyclopedia ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-1.425 tm_pt_6=-2.012 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.135 tm_pt_10=-0.020 tm_pt_11=0.000 tm_pt_12=-1.817 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 lm_0=-5.707 lm_1=-7.730 WordPenalty=-2.171 OOVPenalty=0.000 ||| -9.718
+23 ||| subject matter : encyclopedia ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-3.165 tm_pt_6=-2.139 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.135 tm_pt_10=-0.032 tm_pt_11=0.000 tm_pt_12=-2.958 tm_pt_13=0.000 tm_pt_14=-4.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 lm_0=-6.126 lm_1=-9.076 WordPenalty=-2.606 OOVPenalty=0.000 ||| -10.751
+23 ||| subject-class : encyclopedia ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-1.379 tm_pt_6=-3.561 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.135 tm_pt_10=0.000 tm_pt_11=0.000 tm_pt_12=-2.703 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=2.000 lm_0=-5.989 lm_1=-7.730 WordPenalty=-2.171 OOVPenalty=0.000 ||| -10.896
+24 ||| russia france and israel the main \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 state and defense sub country . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-9.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-18.467 tm_pt_6=-9.206 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-1.474 tm_pt_10=-10.535 tm_pt_11=0.000 tm_pt_12=-5.933 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 lm_0=-38.802 lm_1=-26.775 WordPenalty=-6.514 OOVPenalty=-100.000 ||| -166.562
+24 ||| russia france and israel the main \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 country and defence sub country . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-10.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-18.942 tm_pt_6=-9.742 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-1.738 tm_pt_10=-10.065 tm_pt_11=0.000 tm_pt_12=-6.534 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 lm_0=-38.535 lm_1=-26.552 WordPenalty=-6.514 OOVPenalty=-100.000 ||| -166.728
+24 ||| russia france and israel the main \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 country and defense sub country . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-10.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-18.530 tm_pt_6=-8.356 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-1.106 tm_pt_10=-10.471 tm_pt_11=0.000 tm_pt_12=-5.841 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=11.000 lm_0=-38.844 lm_1=-27.238 WordPenalty=-6.514 OOVPenalty=-100.000 ||| -166.746
+24 ||| russia france and israel the main \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 countries and defense sub country . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-9.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-19.426 tm_pt_6=-10.343 tm_pt_7=-9.000 tm_pt_8=-24.462 tm_pt_9=-1.474 tm_pt_10=-10.130 tm_pt_11=0.000 tm_pt_12=-5.933 tm_pt_13=0.000 tm_pt_14=-12.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=10.000 lm_0=-38.815 lm_1=-27.108 WordPenalty=-6.514 OOVPenalty=-100.000 ||| -167.457
+25 ||| this is the known imaginary formed with the help of which mathematics are real number set from \u09b8\u09c7\u099f\u09c7 par with the complex number . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-60.559 tm_pt_6=-22.004 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-6.553 tm_pt_10=-17.782 tm_pt_11=0.000 tm_pt_12=-5.911 tm_pt_13=0.000 tm_pt_14=-23.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=9.000 lm_0=-50.351 lm_1=-44.022 WordPenalty=-11.292 OOVPenalty=-100.000 ||| -205.244
+25 ||| this is the known as imaginary formed with the help of which mathematics are real number set from \u09b8\u09c7\u099f\u09c7 par with the complex number . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-14.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-63.543 tm_pt_6=-22.038 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-5.553 tm_pt_10=-25.182 tm_pt_11=0.000 tm_pt_12=-8.041 tm_pt_13=0.000 tm_pt_14=-24.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=10.000 lm_0=-50.186 lm_1=-43.889 WordPenalty=-11.726 OOVPenalty=-100.000 ||| -205.568
+25 ||| this is the known imaginary formed with the help of which mathematics are real number set to \u09b8\u09c7\u099f\u09c7 par with the complex number . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-13.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-62.521 tm_pt_6=-22.883 tm_pt_7=-13.000 tm_pt_8=-35.334 tm_pt_9=-6.553 tm_pt_10=-19.587 tm_pt_11=0.000 tm_pt_12=-7.240 tm_pt_13=0.000 tm_pt_14=-23.000 tm_pt_15=0.000 tm_pt_16=-1.000 tm_glue_0=9.000 lm_0=-50.087 lm_1=-44.043 WordPenalty=-11.292 OOVPenalty=-100.000 ||| -206.559
+25 ||| this is the known imaginary formed with the help of which mathematics are set of real numbers from \u09b8\u09c7\u099f\u09c7 par with the complex number . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-14.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-60.947 tm_pt_6=-22.704 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-5.921 tm_pt_10=-18.256 tm_pt_11=0.000 tm_pt_12=-6.322 tm_pt_13=0.000 tm_pt_14=-24.000 tm_pt_15=0.000 tm_pt_16=-2.000 tm_glue_0=9.000 lm_0=-49.282 lm_1=-46.640 WordPenalty=-11.726 OOVPenalty=-100.000 ||| -206.680
+25 ||| this is the known imaginary formed with the help of which mathematics are set of real numbers to \u09b8\u09c7\u099f\u09c7 par with the complex number . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-14.000 tm_pt_3=0.000 tm_pt_4=-9.000 tm_pt_5=-62.909 tm_pt_6=-23.583 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-5.921 tm_pt_10=-20.060 tm_pt_11=0.000 tm_pt_12=-7.652 tm_pt_13=0.000 tm_pt_14=-24.000 tm_pt_15=0.000 tm_pt_16=-2.000 tm_glue_0=9.000 lm_0=-48.838 lm_1=-46.153 WordPenalty=-11.726 OOVPenalty=-100.000 ||| -207.143
+26 ||| <address> ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-1.000 tm_pt_3=0.000 tm_pt_4=-1.000 tm_pt_5=-1.494 tm_pt_6=-38.184 tm_pt_7=-1.000 tm_pt_8=-2.718 tm_pt_9=-0.050 tm_pt_10=0.000 tm_pt_11=0.000 tm_pt_12=0.000 tm_pt_13=0.000 tm_pt_14=-1.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=1.000 lm_0=-4.240 lm_1=-3.850 WordPenalty=-1.303 OOVPenalty=0.000 ||| -18.437
+26 ||| < \u09a0\u09bf\u0995\u09be\u09a8\u09be > ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-2.000 tm_pt_5=-2.518 tm_pt_6=-29.231 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.000 tm_pt_10=-0.118 tm_pt_11=0.000 tm_pt_12=0.000 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=3.000 lm_0=-15.853 lm_1=-9.647 WordPenalty=-2.171 OOVPenalty=-100.000 ||| -132.757
+26 ||| the lt \u09a0\u09bf\u0995\u09be\u09a8\u09be > ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-2.000 tm_pt_3=0.000 tm_pt_4=-1.000 tm_pt_5=-6.362 tm_pt_6=-20.589 tm_pt_7=-2.000 tm_pt_8=-5.436 tm_pt_9=-0.135 tm_pt_10=-2.453 tm_pt_11=0.000 tm_pt_12=0.000 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=-1.000 tm_pt_16=0.000 tm_glue_0=2.000 lm_0=-17.709 lm_1=-13.346 WordPenalty=-2.606 OOVPenalty=-200.000 ||| -238.041
+26 ||| < \u09a0\u09bf\u0995\u09be\u09a8\u09be , gt , ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-3.000 tm_pt_3=0.000 tm_pt_4=-3.000 tm_pt_5=-4.258 tm_pt_6=-15.720 tm_pt_7=-3.000 tm_pt_8=-8.154 tm_pt_9=-0.000 tm_pt_10=-5.328 tm_pt_11=0.000 tm_pt_12=-1.262 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-20.678 lm_1=-15.916 WordPenalty=-3.040 OOVPenalty=-200.000 ||| -239.649
+26 ||| < \u09a0\u09bf\u0995\u09be\u09a8\u09be , gt ; ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-3.000 tm_pt_3=0.000 tm_pt_4=-3.000 tm_pt_5=-3.201 tm_pt_6=-18.449 tm_pt_7=-3.000 tm_pt_8=-8.154 tm_pt_9=-0.000 tm_pt_10=-1.596 tm_pt_11=0.000 tm_pt_12=-1.248 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-21.561 lm_1=-15.792 WordPenalty=-3.040 OOVPenalty=-200.000 ||| -240.345
+26 ||| , lt , \u09a0\u09bf\u0995\u09be\u09a8\u09be > ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-3.000 tm_pt_3=0.000 tm_pt_4=-3.000 tm_pt_5=-4.245 tm_pt_6=-15.998 tm_pt_7=-3.000 tm_pt_8=-8.154 tm_pt_9=-0.000 tm_pt_10=-5.446 tm_pt_11=0.000 tm_pt_12=-1.262 tm_pt_13=0.000 tm_pt_14=-3.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=5.000 lm_0=-20.961 lm_1=-16.373 WordPenalty=-3.040 OOVPenalty=-200.000 ||| -240.688
+27 ||| september ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-1.000 tm_pt_3=0.000 tm_pt_4=-1.000 tm_pt_5=-0.176 tm_pt_6=-0.047 tm_pt_7=-1.000 tm_pt_8=-2.718 tm_pt_9=-0.000 tm_pt_10=-0.013 tm_pt_11=0.000 tm_pt_12=-0.025 tm_pt_13=0.000 tm_pt_14=-1.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=1.000 lm_0=-3.024 lm_1=-3.825 WordPenalty=-1.303 OOVPenalty=0.000 ||| -3.585
+27 ||| september . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-1.000 tm_pt_3=0.000 tm_pt_4=-1.000 tm_pt_5=-9.282 tm_pt_6=-0.716 tm_pt_7=-1.000 tm_pt_8=-2.718 tm_pt_9=-0.368 tm_pt_10=-1.099 tm_pt_11=0.000 tm_pt_12=-3.689 tm_pt_13=0.000 tm_pt_14=-2.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=1.000 lm_0=-4.832 lm_1=-5.754 WordPenalty=-1.737 OOVPenalty=0.000 ||| -10.126
+27 ||| \u09b8\u09c7\u09aa\u09cd\u099f\u09c7\u09ae\u09cd\u09ac\u09b0 ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=0.000 tm_pt_3=0.000 tm_pt_4=0.000 tm_pt_5=0.000 tm_pt_6=0.000 tm_pt_7=0.000 tm_pt_8=0.000 tm_pt_9=0.000 tm_pt_10=0.000 tm_pt_11=0.000 tm_pt_12=0.000 tm_pt_13=0.000 tm_pt_14=0.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=1.000 lm_0=-7.355 lm_1=-4.974 WordPenalty=-1.303 OOVPenalty=-100.000 ||| -109.443
+28 ||| from this theory though big \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 to can not be rather it can be support . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-10.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-29.487 tm_pt_6=-24.381 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-2.525 tm_pt_10=-16.787 tm_pt_11=0.000 tm_pt_12=-5.740 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-36.059 lm_1=-28.266 WordPenalty=-7.817 OOVPenalty=-100.000 ||| -172.006
+28 ||| from this theory though big \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 against can not be rather it can be support . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-10.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-25.087 tm_pt_6=-23.283 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-1.575 tm_pt_10=-12.592 tm_pt_11=0.000 tm_pt_12=-4.353 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-37.068 lm_1=-30.729 WordPenalty=-7.817 OOVPenalty=-100.000 ||| -172.727
+28 ||| from this theory though big \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 opposed can not be rather it can be support . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-10.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-23.358 tm_pt_6=-23.283 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-2.525 tm_pt_10=-10.887 tm_pt_11=0.000 tm_pt_12=-5.740 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-37.843 lm_1=-30.003 WordPenalty=-7.817 OOVPenalty=-100.000 ||| -173.285
+28 ||| from this theory though big \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 to can not be rather it can be supported . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-10.000 tm_pt_3=0.000 tm_pt_4=-6.000 tm_pt_5=-29.070 tm_pt_6=-23.889 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-3.524 tm_pt_10=-17.914 tm_pt_11=0.000 tm_pt_12=-7.821 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 lm_0=-35.106 lm_1=-28.266 WordPenalty=-7.817 OOVPenalty=-100.000 ||| -173.391
+28 ||| from this theory though big \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 against can not be but it can be support . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-10.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-27.960 tm_pt_6=-23.108 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-1.575 tm_pt_10=-14.990 tm_pt_11=0.000 tm_pt_12=-4.353 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=8.000 lm_0=-35.950 lm_1=-31.509 WordPenalty=-7.817 OOVPenalty=-100.000 ||| -173.420
+28 ||| from this theory though big \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 against can not be support rather it can be . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-10.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-25.087 tm_pt_6=-23.283 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-1.942 tm_pt_10=-13.423 tm_pt_11=0.000 tm_pt_12=-5.046 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 lm_0=-37.028 lm_1=-30.243 WordPenalty=-7.817 OOVPenalty=-100.000 ||| -173.838
+28 ||| from this theory though big \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 against can not be support but it can be . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-10.000 tm_pt_3=0.000 tm_pt_4=-7.000 tm_pt_5=-27.960 tm_pt_6=-23.108 tm_pt_7=-10.000 tm_pt_8=-27.180 tm_pt_9=-1.942 tm_pt_10=-15.821 tm_pt_11=0.000 tm_pt_12=-5.046 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=7.000 lm_0=-35.911 lm_1=-30.724 WordPenalty=-7.817 OOVPenalty=-100.000 ||| -174.163
+28 ||| from this theory though big \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 protested can not be rather it can be support . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-11.000 tm_pt_3=0.000 tm_pt_4=-8.000 tm_pt_5=-23.293 tm_pt_6=-24.158 tm_pt_7=-11.000 tm_pt_8=-29.898 tm_pt_9=-1.525 tm_pt_10=-12.568 tm_pt_11=0.000 tm_pt_12=-6.573 tm_pt_13=0.000 tm_pt_14=-15.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=9.000 lm_0=-38.421 lm_1=-30.003 WordPenalty=-7.817 OOVPenalty=-100.000 ||| -174.388
+29 ||| agricultural in production france is the most important country ; it is \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af wine cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and export in the whole world . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-15.000 tm_pt_3=0.000 tm_pt_4=-12.000 tm_pt_5=-61.979 tm_pt_6=-21.379 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-3.566 tm_pt_10=-19.501 tm_pt_11=0.000 tm_pt_12=-6.075 tm_pt_13=0.000 tm_pt_14=-24.000 tm_pt_15=-1.000 tm_pt_16=0.000 tm_glue_0=14.000 lm_0=-60.198 lm_1=-49.635 WordPenalty=-12.160 OOVPenalty=-200.000 ||| -316.684
+29 ||| agricultural in production france is the most important country ; it basically \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af wine cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and export in the whole world . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-15.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-56.780 tm_pt_6=-21.542 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-3.477 tm_pt_10=-13.337 tm_pt_11=0.000 tm_pt_12=-5.633 tm_pt_13=0.000 tm_pt_14=-24.000 tm_pt_15=-1.000 tm_pt_16=0.000 tm_glue_0=15.000 lm_0=-63.240 lm_1=-49.990 WordPenalty=-12.160 OOVPenalty=-200.000 ||| -317.154
+29 ||| agricultural in production france country ; it is the most important mainly \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af wine cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and export in the whole world . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-14.000 tm_pt_3=0.000 tm_pt_4=-11.000 tm_pt_5=-54.055 tm_pt_6=-20.128 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-3.477 tm_pt_10=-11.319 tm_pt_11=0.000 tm_pt_12=-3.338 tm_pt_13=0.000 tm_pt_14=-24.000 tm_pt_15=-1.000 tm_pt_16=0.000 tm_glue_0=12.000 lm_0=-62.879 lm_1=-50.811 WordPenalty=-12.160 OOVPenalty=-200.000 ||| -317.157
+29 ||| agricultural in production france is the most important country ; it mainly \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af wine cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and export in the whole world . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-15.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-57.050 tm_pt_6=-20.546 tm_pt_7=-15.000 tm_pt_8=-40.770 tm_pt_9=-3.474 tm_pt_10=-13.783 tm_pt_11=0.000 tm_pt_12=-4.113 tm_pt_13=0.000 tm_pt_14=-24.000 tm_pt_15=-1.000 tm_pt_16=0.000 tm_glue_0=15.000 lm_0=-63.884 lm_1=-49.990 WordPenalty=-12.160 OOVPenalty=-200.000 ||| -317.238
+29 ||| agriculture in production france is the most important country ; it is \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af wine cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and export in the whole world . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-14.000 tm_pt_3=0.000 tm_pt_4=-10.000 tm_pt_5=-62.243 tm_pt_6=-21.939 tm_pt_7=-14.000 tm_pt_8=-38.052 tm_pt_9=-2.838 tm_pt_10=-19.370 tm_pt_11=0.000 tm_pt_12=-7.203 tm_pt_13=0.000 tm_pt_14=-24.000 tm_pt_15=-1.000 tm_pt_16=0.000 tm_glue_0=12.000 lm_0=-60.690 lm_1=-49.635 WordPenalty=-12.160 OOVPenalty=-200.000 ||| -318.076
+29 ||| agricultural in production france is the most important country ; it is \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af wine cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and export the in the whole world . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-16.000 tm_pt_3=0.000 tm_pt_4=-12.000 tm_pt_5=-65.187 tm_pt_6=-19.593 tm_pt_7=-16.000 tm_pt_8=-43.488 tm_pt_9=-3.560 tm_pt_10=-16.076 tm_pt_11=0.000 tm_pt_12=-5.939 tm_pt_13=0.000 tm_pt_14=-25.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=14.000 lm_0=-62.535 lm_1=-50.774 WordPenalty=-12.595 OOVPenalty=-200.000 ||| -318.715
+29 ||| agricultural in production france is the most important country ; it basically \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af wine cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and export the in the whole world . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-16.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-59.988 tm_pt_6=-19.755 tm_pt_7=-16.000 tm_pt_8=-43.488 tm_pt_9=-3.470 tm_pt_10=-9.911 tm_pt_11=0.000 tm_pt_12=-5.497 tm_pt_13=0.000 tm_pt_14=-25.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=15.000 lm_0=-65.577 lm_1=-51.128 WordPenalty=-12.595 OOVPenalty=-200.000 ||| -319.185
+29 ||| agricultural in production france is the most important country ; it mainly \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af wine cheese and other \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af europe and export the in the whole world . ||| tm_pt_0=0.000 tm_pt_1=0.000 tm_pt_2=-16.000 tm_pt_3=0.000 tm_pt_4=-13.000 tm_pt_5=-60.258 tm_pt_6=-18.760 tm_pt_7=-16.000 tm_pt_8=-43.488 tm_pt_9=-3.468 tm_pt_10=-10.358 tm_pt_11=0.000 tm_pt_12=-3.977 tm_pt_13=0.000 tm_pt_14=-25.000 tm_pt_15=0.000 tm_pt_16=0.000 tm_glue_0=15.000 lm_0=-66.221 lm_1=-51.128 WordPenalty=-12.595 OOVPenalty=-200.000 ||| -319.2
<TRUNCATED>
[38/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/buildin_lm/TrieLM.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/buildin_lm/TrieLM.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/buildin_lm/TrieLM.java
new file mode 100644
index 0000000..ccfff46
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/buildin_lm/TrieLM.java
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.lm.buildin_lm;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.Map;
+import java.util.Scanner;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.ff.lm.AbstractLM;
+import org.apache.joshua.decoder.ff.lm.ArpaFile;
+import org.apache.joshua.decoder.ff.lm.ArpaNgram;
+import org.apache.joshua.util.Bits;
+import org.apache.joshua.util.Regex;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Relatively memory-compact language model
+ * stored as a reversed-word-order trie.
+ * <p>
+ * The trie itself represents language model context.
+ * <p>
+ * Conceptually, each node in the trie stores a map
+ * from conditioning word to log probability.
+ * <p>
+ * Additionally, each node in the trie stores
+ * the backoff weight for that context.
+ *
+ * @author Lane Schwartz
+ * @see <a href="http://www.speech.sri.com/projects/srilm/manpages/ngram-discount.7.html">SRILM ngram-discount documentation</a>
+ */
+public class TrieLM extends AbstractLM { //DefaultNGramLanguageModel {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TrieLM.class);
+
+ /**
+ * Node ID for the root node.
+ */
+ private static final int ROOT_NODE_ID = 0;
+
+
+ /**
+ * Maps from (node id, word id for child) --> node id of child.
+ */
+ private final Map<Long,Integer> children;
+
+ /**
+ * Maps from (node id, word id for lookup word) -->
+ * log prob of lookup word given context
+ *
+ * (the context is defined by where you are in the tree).
+ */
+ private final Map<Long,Float> logProbs;
+
+ /**
+ * Maps from (node id) -->
+ * backoff weight for that context
+ *
+ * (the context is defined by where you are in the tree).
+ */
+ private final Map<Integer,Float> backoffs;
+
+ public TrieLM(Vocabulary vocab, String file) throws FileNotFoundException {
+ this(new ArpaFile(file,vocab));
+ }
+
+ /**
+ * Constructs a language model object from the specified ARPA file.
+ *
+ * @param arpaFile input ARPA file
+ * @throws FileNotFoundException if the input file cannot be located
+ */
+ public TrieLM(ArpaFile arpaFile) throws FileNotFoundException {
+ super(arpaFile.getVocab().size(), arpaFile.getOrder());
+
+ int ngramCounts = arpaFile.size();
+ LOG.debug("ARPA file contains {} n-grams", ngramCounts);
+
+ this.children = new HashMap<Long,Integer>(ngramCounts);
+ this.logProbs = new HashMap<Long,Float>(ngramCounts);
+ this.backoffs = new HashMap<Integer,Float>(ngramCounts);
+
+ int nodeCounter = 0;
+
+ int lineNumber = 0;
+ for (ArpaNgram ngram : arpaFile) {
+ lineNumber += 1;
+ if (lineNumber % 100000 == 0){
+ LOG.info("Line: {}", lineNumber);
+ }
+
+ LOG.debug("{}-gram: ({} | {})", ngram.order(), ngram.getWord(),
+ Arrays.toString(ngram.getContext()));
+ int word = ngram.getWord();
+
+ int[] context = ngram.getContext();
+
+ {
+ // Find where the log prob should be stored
+ int contextNodeID = ROOT_NODE_ID;
+ {
+ for (int i=context.length-1; i>=0; i--) {
+ long key = Bits.encodeAsLong(contextNodeID, context[i]);
+ int childID;
+ if (children.containsKey(key)) {
+ childID = children.get(key);
+ } else {
+ childID = ++nodeCounter;
+ LOG.debug("children.put({}:{}, {})", contextNodeID, context[i], childID);
+ children.put(key, childID);
+ }
+ contextNodeID = childID;
+ }
+ }
+
+ // Store the log prob for this n-gram at this node in the trie
+ {
+ long key = Bits.encodeAsLong(contextNodeID, word);
+ float logProb = ngram.getValue();
+ LOG.debug("logProbs.put({}:{}, {}", contextNodeID, word, logProb);
+ this.logProbs.put(key, logProb);
+ }
+ }
+
+ {
+ // Find where the backoff should be stored
+ int backoffNodeID = ROOT_NODE_ID;
+ {
+ long backoffNodeKey = Bits.encodeAsLong(backoffNodeID, word);
+ int wordChildID;
+ if (children.containsKey(backoffNodeKey)) {
+ wordChildID = children.get(backoffNodeKey);
+ } else {
+ wordChildID = ++nodeCounter;
+ LOG.debug("children.put({}: {}, {})", backoffNodeID, word, wordChildID);
+ children.put(backoffNodeKey, wordChildID);
+ }
+ backoffNodeID = wordChildID;
+
+ for (int i=context.length-1; i>=0; i--) {
+ long key = Bits.encodeAsLong(backoffNodeID, context[i]);
+ int childID;
+ if (children.containsKey(key)) {
+ childID = children.get(key);
+ } else {
+ childID = ++nodeCounter;
+ LOG.debug("children.put({}:{}, {})", backoffNodeID, context[i], childID);
+ children.put(key, childID);
+ }
+ backoffNodeID = childID;
+ }
+ }
+
+ // Store the backoff for this n-gram at this node in the trie
+ {
+ float backoff = ngram.getBackoff();
+ LOG.debug("backoffs.put({}:{}, {})", backoffNodeID, word, backoff);
+ this.backoffs.put(backoffNodeID, backoff);
+ }
+ }
+
+ }
+ }
+
+
+ @Override
+ protected double logProbabilityOfBackoffState_helper(
+ int[] ngram, int order, int qtyAdditionalBackoffWeight
+ ) {
+ throw new UnsupportedOperationException("probabilityOfBackoffState_helper undefined for TrieLM");
+ }
+
+ @Override
+ protected float ngramLogProbability_helper(int[] ngram, int order) {
+
+// float logProb = (float) -JoshuaConfiguration.lm_ceiling_cost;//Float.NEGATIVE_INFINITY; // log(0.0f)
+ float backoff = 0.0f; // log(1.0f)
+
+ int i = ngram.length - 1;
+ int word = ngram[i];
+ i -= 1;
+
+ int nodeID = ROOT_NODE_ID;
+
+ while (true) {
+
+ {
+ long key = Bits.encodeAsLong(nodeID, word);
+ if (logProbs.containsKey(key)) {
+// logProb = logProbs.get(key);
+ backoff = 0.0f; // log(0.0f)
+ }
+ }
+
+ if (i < 0) {
+ break;
+ }
+
+ {
+ long key = Bits.encodeAsLong(nodeID, ngram[i]);
+
+ if (children.containsKey(key)) {
+ nodeID = children.get(key);
+
+ backoff += backoffs.get(nodeID);
+
+ i -= 1;
+
+ } else {
+ break;
+ }
+ }
+
+ }
+
+// double result = logProb + backoff;
+// if (result < -JoshuaConfiguration.lm_ceiling_cost) {
+// result = -JoshuaConfiguration.lm_ceiling_cost;
+// }
+//
+// return result;
+ return (Float) null;
+ }
+
+ public Map<Long,Integer> getChildren() {
+ return this.children;
+ }
+
+ public static void main(String[] args) throws IOException {
+
+ LOG.info("Constructing ARPA file");
+ ArpaFile arpaFile = new ArpaFile(args[0]);
+
+ LOG.info("Getting symbol table");
+ Vocabulary vocab = arpaFile.getVocab();
+
+ LOG.info("Constructing TrieLM");
+ TrieLM lm = new TrieLM(arpaFile);
+
+ int n = Integer.valueOf(args[2]);
+ LOG.info("N-gram order will be {}", n);
+
+ Scanner scanner = new Scanner(new File(args[1]));
+
+ LinkedList<String> wordList = new LinkedList<String>();
+ LinkedList<String> window = new LinkedList<String>();
+
+ LOG.info("Starting to scan {}", args[1]);
+ while (scanner.hasNext()) {
+
+ LOG.info("Getting next line...");
+ String line = scanner.nextLine();
+ LOG.info("Line: {}", line);
+
+ String[] words = Regex.spaces.split(line);
+ wordList.clear();
+
+ wordList.add("<s>");
+ for (String word : words) {
+ wordList.add(word);
+ }
+ wordList.add("</s>");
+
+ ArrayList<Integer> sentence = new ArrayList<Integer>();
+ // int[] ids = new int[wordList.size()];
+ for (int i=0, size=wordList.size(); i<size; i++) {
+ sentence.add(vocab.id(wordList.get(i)));
+ // ids[i] = ;
+ }
+
+
+
+ while (! wordList.isEmpty()) {
+ window.clear();
+
+ {
+ int i=0;
+ for (String word : wordList) {
+ if (i>=n) break;
+ window.add(word);
+ i++;
+ }
+ wordList.remove();
+ }
+
+ {
+ int i=0;
+ int[] wordIDs = new int[window.size()];
+ for (String word : window) {
+ wordIDs[i] = vocab.id(word);
+ i++;
+ }
+
+ LOG.info("logProb {} = {}", window, lm.ngramLogProbability(wordIDs, n));
+ }
+ }
+
+ double logProb = lm.sentenceLogProbability(sentence, n, 2);//.ngramLogProbability(ids, n);
+ double prob = Math.exp(logProb);
+
+ LOG.info("Total logProb = {}", logProb);
+ LOG.info("Total prob = {}", prob);
+ }
+
+ }
+
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/buildin_lm/package-info.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/buildin_lm/package-info.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/buildin_lm/package-info.java
new file mode 100644
index 0000000..6c84703
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/buildin_lm/package-info.java
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.lm.buildin_lm;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/package-info.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/package-info.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/package-info.java
new file mode 100644
index 0000000..22da71e
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/package-info.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * <p>Provides abstraction and support for the language model
+ * feature function typically used in hierarchical phrase-based
+ * decoding for statistical machine translation.</p>
+ * <p>The classes contained within this directory are
+ * responsible for two tasks: implementing the feature function,
+ * and representing the language model itself. The class
+ * `LanguageModelFF` implements the feature function by exending
+ * the class `DefaultStatefulFF`. One of these is instantiated
+ * for each language model present in the decoder.</p>
+ * <p>The language models themselves are implemented as a
+ * combination of an interface (`NGramLanguageModel`), a default
+ * implementation (`DefaultNgramLangaugeModel`), and an abstract
+ * implementation of the default (`AbstractLM`).</p>
+ *
+ * <pre>
+ * DefaultStatefulFF
+ * |- LanguageModelFF
+ *
+ * DefaultNgramLanguageModel implements interface NGramLanguageModel
+ * |- AbstractLM
+ * </pre>
+ */
+package org.apache.joshua.decoder.ff.lm;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/package-info.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/package-info.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/package-info.java
new file mode 100644
index 0000000..b0af73e
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/package-info.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * <p>Provides an implementation of the linear feature functions
+ * typically used in hierarchical phrase-based decoding for
+ * statistical machine translation.</p>
+ * <p>The following is a note from Juri describing some of the
+ * functionality of the feature functions interfaces and default
+ * abstract classes.</p>
+ * <pre>
+ * The equality that I intended for is ff.transitionLogP() =
+ * ff.estimateLogP() + ff.reEstimateTransitionLogP(). The re-estimate
+ * fixes the estimate to be the true transition cost that takes into
+ * account the state. Before decoding the cost of applying a rule is
+ * estimated via estimateLogP() and yields the phrasal feature costs plus
+ * an LM estimate of the cost of the lexical portions of the rule.
+ * transitionLogP() takes rule and state and computes everything from
+ * scratch, whereas reEstimateTransitionLogP() adds in the cost of new
+ * n-grams that result from combining the rule with the LM states and
+ * subtracts out the cost of superfluous less-than-n-grams that were
+ * overridden by the updated cost calculation.
+ *
+ * Hope this helps.
+ * </pre>
+ */
+package org.apache.joshua.decoder.ff;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/phrase/Distortion.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/phrase/Distortion.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/phrase/Distortion.java
new file mode 100644
index 0000000..f9e6a29
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/phrase/Distortion.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.phrase;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.chart_parser.SourcePath;
+import org.apache.joshua.decoder.ff.FeatureVector;
+import org.apache.joshua.decoder.ff.StatelessFF;
+import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.hypergraph.HGNode;
+import org.apache.joshua.decoder.phrase.Hypothesis;
+import org.apache.joshua.decoder.segment_file.Sentence;
+
+public class Distortion extends StatelessFF {
+
+ public Distortion(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, "Distortion", args, config);
+
+ if (! config.search_algorithm.equals("stack")) {
+ String msg = "* FATAL: Distortion feature only application for phrase-based decoding. "
+ + "Use -search phrase or remove this feature";
+ throw new RuntimeException(msg);
+ }
+ }
+
+ @Override
+ public ArrayList<String> reportDenseFeatures(int index) {
+ denseFeatureIndex = index;
+
+ ArrayList<String> names = new ArrayList<String>();
+ names.add(name);
+ return names;
+ }
+
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+
+ if (rule != Hypothesis.BEGIN_RULE && rule != Hypothesis.END_RULE) {
+ int start_point = j - rule.getFrench().length + rule.getArity();
+
+ int jump_size = Math.abs(tailNodes.get(0).j - start_point);
+// acc.add(name, -jump_size);
+ acc.add(denseFeatureIndex, -jump_size);
+ }
+
+// System.err.println(String.format("DISTORTION(%d, %d) from %d = %d", i, j, tailNodes != null ? tailNodes.get(0).j : -1, jump_size));
+
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/similarity/EdgePhraseSimilarityFF.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/similarity/EdgePhraseSimilarityFF.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/similarity/EdgePhraseSimilarityFF.java
new file mode 100644
index 0000000..91af58b
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/similarity/EdgePhraseSimilarityFF.java
@@ -0,0 +1,279 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.similarity;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
+import java.net.Socket;
+import java.net.UnknownHostException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import com.google.common.base.Throwables;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.chart_parser.SourcePath;
+import org.apache.joshua.decoder.ff.FeatureVector;
+import org.apache.joshua.decoder.ff.StatefulFF;
+import org.apache.joshua.decoder.ff.SourceDependentFF;
+import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+import org.apache.joshua.decoder.ff.state_maintenance.NgramDPState;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.hypergraph.HGNode;
+import org.apache.joshua.decoder.segment_file.Sentence;
+import org.apache.joshua.util.Cache;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class EdgePhraseSimilarityFF extends StatefulFF implements SourceDependentFF {
+
+ private static final Logger LOG = LoggerFactory.getLogger(EdgePhraseSimilarityFF.class);
+
+ private static Cache<String, Float> cache = new Cache<String, Float>(100000000);
+
+ private String host;
+ private int port;
+
+ private Socket socket;
+ private PrintWriter serverAsk;
+ private BufferedReader serverReply;
+
+ private int[] source;
+
+ private final int MAX_PHRASE_LENGTH = 4;
+ private final int GAP = 0;
+
+ public EdgePhraseSimilarityFF(FeatureVector weights, String[] args, JoshuaConfiguration config) throws NumberFormatException, UnknownHostException, IOException {
+ super(weights, "EdgePhraseSimilarity", args, config);
+
+ this.host = parsedArgs.get("host");
+ this.port = Integer.parseInt(parsedArgs.get("port"));
+
+ initializeConnection();
+ }
+
+ private void initializeConnection() throws NumberFormatException, IOException {
+ LOG.info("Opening connection.");
+ socket = new Socket(host, port);
+ serverAsk = new PrintWriter(socket.getOutputStream(), true);
+ serverReply = new BufferedReader(new InputStreamReader(socket.getInputStream()));
+ }
+
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+
+ float value = computeScore(rule, tailNodes);
+ acc.add(name, value);
+
+ // TODO 07/2013: EdgePhraseSimilarity needs to know its order rather than inferring it from tail
+ // nodes.
+ return new NgramDPState(new int[1], new int[1]);
+ }
+
+ @Override
+ public DPState computeFinal(HGNode tailNode, int i, int j, SourcePath path, Sentence sentence, Accumulator acc) {
+ return null;
+ }
+
+ public float computeScore(Rule rule, List<HGNode> tailNodes) {
+ if (tailNodes == null || tailNodes.isEmpty())
+ return 0;
+
+ // System.err.println("RULE [" + spanStart + ", " + spanEnd + "]: " + rule.toString());
+
+ int[] target = rule.getEnglish();
+ int lm_state_size = 0;
+ for (HGNode node : tailNodes) {
+ NgramDPState state = (NgramDPState) node.getDPState(stateIndex);
+ lm_state_size += state.getLeftLMStateWords().length + state.getRightLMStateWords().length;
+ }
+
+ ArrayList<int[]> batch = new ArrayList<int[]>();
+
+ // Build joined target string.
+ int[] join = new int[target.length + lm_state_size];
+
+ int idx = 0, num_gaps = 1, num_anchors = 0;
+ int[] anchors = new int[rule.getArity() * 2];
+ int[] indices = new int[rule.getArity() * 2];
+ int[] gaps = new int[rule.getArity() + 2];
+ gaps[0] = 0;
+ for (int t = 0; t < target.length; t++) {
+ if (target[t] < 0) {
+ HGNode node = tailNodes.get(-(target[t] + 1));
+ if (t != 0) {
+ indices[num_anchors] = node.i;
+ anchors[num_anchors++] = idx;
+ }
+ NgramDPState state = (NgramDPState) node.getDPState(stateIndex);
+ // System.err.print("LEFT: ");
+ // for (int w : state.getLeftLMStateWords()) System.err.print(Vocabulary.word(w) + " ");
+ // System.err.println();
+ for (int w : state.getLeftLMStateWords())
+ join[idx++] = w;
+ join[idx++] = GAP;
+ gaps[num_gaps++] = idx;
+ // System.err.print("RIGHT: ");
+ // for (int w : state.getRightLMStateWords()) System.err.print(Vocabulary.word(w) + " ");
+ // System.err.println();
+ for (int w : state.getRightLMStateWords())
+ join[idx++] = w;
+ if (t != target.length - 1) {
+ indices[num_anchors] = node.j;
+ anchors[num_anchors++] = idx;
+ }
+ } else {
+ join[idx++] = target[t];
+ }
+ }
+ gaps[gaps.length - 1] = join.length + 1;
+
+ // int c = 0;
+ // System.err.print("> ");
+ // for (int k = 0; k < join.length; k++) {
+ // if (c < num_anchors && anchors[c] == k) {
+ // c++;
+ // System.err.print("| ");
+ // }
+ // System.err.print(Vocabulary.word(join[k]) + " ");
+ // }
+ // System.err.println("<");
+
+ int g = 0;
+ for (int a = 0; a < num_anchors; a++) {
+ if (a > 0 && anchors[a - 1] == anchors[a])
+ continue;
+ if (anchors[a] > gaps[g + 1])
+ g++;
+ int left = Math.max(gaps[g], anchors[a] - MAX_PHRASE_LENGTH + 1);
+ int right = Math.min(gaps[g + 1] - 1, anchors[a] + MAX_PHRASE_LENGTH - 1);
+
+ int[] target_phrase = new int[right - left];
+ System.arraycopy(join, left, target_phrase, 0, target_phrase.length);
+ int[] source_phrase = getSourcePhrase(indices[a]);
+
+ if (source_phrase != null && target_phrase.length != 0) {
+ // System.err.println("ANCHOR: " + indices[a]);
+ batch.add(source_phrase);
+ batch.add(target_phrase);
+ }
+ }
+ return getSimilarity(batch);
+ }
+
+ @Override
+ public float estimateFutureCost(Rule rule, DPState currentState, Sentence sentence) {
+ return 0.0f;
+ }
+
+ /**
+ * From SourceDependentFF interface.
+ */
+ @Override
+ public void setSource(Sentence sentence) {
+ if (! sentence.isLinearChain())
+ throw new RuntimeException("EdgePhraseSimilarity not defined for lattices");
+ this.source = sentence.getWordIDs();
+ }
+
+ public EdgePhraseSimilarityFF clone() {
+ try {
+ return new EdgePhraseSimilarityFF(this.weights, args, config);
+ } catch (Exception e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ public float estimateCost(Rule rule, Sentence sentence) {
+ return 0.0f;
+ }
+
+ private final int[] getSourcePhrase(int anchor) {
+ int idx;
+ int length = Math.min(anchor, MAX_PHRASE_LENGTH - 1)
+ + Math.min(source.length - anchor, MAX_PHRASE_LENGTH - 1);
+ if (length <= 0)
+ return null;
+ int[] phrase = new int[length];
+ idx = 0;
+ for (int p = Math.max(0, anchor - MAX_PHRASE_LENGTH + 1); p < Math.min(source.length, anchor
+ + MAX_PHRASE_LENGTH - 1); p++)
+ phrase[idx++] = source[p];
+ return phrase;
+ }
+
+ private float getSimilarity(List<int[]> batch) {
+ float similarity = 0.0f;
+ int count = 0;
+ StringBuilder query = new StringBuilder();
+ List<String> to_cache = new ArrayList<String>();
+ query.append("xb");
+ for (int i = 0; i < batch.size(); i += 2) {
+ int[] source = batch.get(i);
+ int[] target = batch.get(i + 1);
+
+ if (Arrays.equals(source, target)) {
+ similarity += 1;
+ count++;
+ } else {
+ String source_string = Vocabulary.getWords(source);
+ String target_string = Vocabulary.getWords(target);
+
+ String both;
+ if (source_string.compareTo(target_string) > 0)
+ both = source_string + " ||| " + target_string;
+ else
+ both = target_string + " ||| " + source_string;
+
+ Float cached = cache.get(both);
+ if (cached != null) {
+ // System.err.println("SIM: " + source_string + " X " + target_string + " = " + cached);
+ similarity += cached;
+ count++;
+ } else {
+ query.append("\t").append(source_string);
+ query.append("\t").append(target_string);
+ to_cache.add(both);
+ }
+ }
+ }
+ if (!to_cache.isEmpty()) {
+ try {
+ serverAsk.println(query.toString());
+ String response = serverReply.readLine();
+ String[] scores = response.split("\\s+");
+ for (int i = 0; i < scores.length; i++) {
+ Float score = Float.parseFloat(scores[i]);
+ cache.put(to_cache.get(i), score);
+ similarity += score;
+ count++;
+ }
+ } catch (Exception e) {
+ return 0;
+ }
+ }
+ return (count == 0 ? 0 : similarity / count);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/state_maintenance/DPState.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/state_maintenance/DPState.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/state_maintenance/DPState.java
new file mode 100644
index 0000000..e117fde
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/state_maintenance/DPState.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.state_maintenance;
+
+/**
+ * Abstract class enforcing explicit implementation of the standard methods.
+ *
+ * @author Zhifei Li, zhifei.work@gmail.com
+ * @author Juri Ganitkevitch, juri@cs.jhu.edu
+ */
+public abstract class DPState {
+
+ public abstract String toString();
+
+ public abstract int hashCode();
+
+ public abstract boolean equals(Object other);
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/state_maintenance/KenLMState.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/state_maintenance/KenLMState.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/state_maintenance/KenLMState.java
new file mode 100644
index 0000000..4fdc631
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/state_maintenance/KenLMState.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.state_maintenance;
+
+/**
+ * Maintains a state pointer used by KenLM to implement left-state minimization.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ * @author Juri Ganitkevitch juri@cs.jhu.edu
+ */
+public class KenLMState extends DPState {
+
+ private long state = 0;
+
+ public KenLMState() {
+ }
+
+ public KenLMState(long stateId) {
+ this.state = stateId;
+ }
+
+ public long getState() {
+ return state;
+ }
+
+ @Override
+ public int hashCode() {
+ return (int) ((getState() >> 32) ^ getState());
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ return (other instanceof KenLMState && this.getState() == ((KenLMState) other).getState());
+ }
+
+ @Override
+ public String toString() {
+ return String.format("[KenLMState %d]", getState());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/state_maintenance/NgramDPState.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/state_maintenance/NgramDPState.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/state_maintenance/NgramDPState.java
new file mode 100644
index 0000000..b269bd9
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/state_maintenance/NgramDPState.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.state_maintenance;
+
+import java.util.Arrays;
+
+import org.apache.joshua.corpus.Vocabulary;
+
+/**
+ * @author Zhifei Li, zhifei.work@gmail.com
+ * @author Juri Ganitkevitch, juri@cs.jhu.edu
+ */
+public class NgramDPState extends DPState {
+
+ private int[] left;
+ private int[] right;
+
+ private int hash = 0;
+
+ public NgramDPState(int[] l, int[] r) {
+ left = l;
+ right = r;
+ assertLengths();
+ }
+
+ public void setLeftLMStateWords(int[] words) {
+ left = words;
+ assertLengths();
+ }
+
+ public int[] getLeftLMStateWords() {
+ return left;
+ }
+
+ public void setRightLMStateWords(int[] words) {
+ right = words;
+ assertLengths();
+ }
+
+ public int[] getRightLMStateWords() {
+ return right;
+ }
+
+ private final void assertLengths() {
+ if (left.length != right.length)
+ throw new RuntimeException("Unequal lengths in left and right state: < "
+ + Vocabulary.getWords(left) + " | " + Vocabulary.getWords(right) + " >");
+ }
+
+ @Override
+ public int hashCode() {
+ if (hash == 0) {
+ hash = 31 + Arrays.hashCode(left);
+ hash = hash * 19 + Arrays.hashCode(right);
+ }
+ return hash;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other instanceof NgramDPState) {
+ NgramDPState that = (NgramDPState) other;
+ if (this.left.length == that.left.length && this.right.length == that.right.length) {
+ for (int i = 0; i < left.length; ++i)
+ if (this.left[i] != that.left[i] || this.right[i] != that.right[i])
+ return false;
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("<");
+ for (int id : left)
+ sb.append(" " + Vocabulary.word(id));
+ sb.append(" |");
+ for (int id : right)
+ sb.append(" " + Vocabulary.word(id));
+ sb.append(" >");
+ return sb.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/AbstractGrammar.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/AbstractGrammar.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/AbstractGrammar.java
new file mode 100644
index 0000000..6c512bd
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/AbstractGrammar.java
@@ -0,0 +1,228 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.tm;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.ff.FeatureFunction;
+import org.apache.joshua.decoder.phrase.PhraseTable;
+import org.apache.joshua.decoder.segment_file.Token;
+import org.apache.joshua.lattice.Arc;
+import org.apache.joshua.lattice.Lattice;
+import org.apache.joshua.lattice.Node;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Partial implementation of the <code>Grammar</code> interface that provides logic for sorting a
+ * grammar.
+ * <p>
+ * <em>Note</em>: New classes implementing the <code>Grammar</code> interface should probably
+ * inherit from this class, unless a specific sorting technique different from that implemented by
+ * this class is required.
+ *
+ * @author Zhifei Li
+ * @author Lane Schwartz
+ * @author Matt Post post@cs.jhu.edu
+ */
+public abstract class AbstractGrammar implements Grammar {
+
+ /** Logger for this class. */
+ private static final Logger LOG = LoggerFactory.getLogger(AbstractGrammar.class);
+ /**
+ * Indicates whether the rules in this grammar have been sorted based on the latest feature
+ * function values.
+ */
+ protected boolean sorted = false;
+
+ /*
+ * The grammar's owner, used to determine which weights are applicable to the dense features found
+ * within.
+ */
+ protected int owner = -1;
+
+ /*
+ * The maximum length of a source-side phrase. Mostly used by the phrase-based decoder.
+ */
+ protected int maxSourcePhraseLength = -1;
+
+ /**
+ * Returns the longest source phrase read.
+ *
+ * @return the longest source phrase read (nonterminal + terminal symbols).
+ */
+ @Override
+ public int getMaxSourcePhraseLength() {
+ return maxSourcePhraseLength;
+ }
+
+ @Override
+ public int getOwner() {
+ return owner;
+ }
+
+ /* The maximum span of the input this rule can be applied to. */
+ protected int spanLimit = 1;
+
+ protected JoshuaConfiguration joshuaConfiguration;
+
+ /**
+ * Constructs an empty, unsorted grammar.
+ *
+ * @see Grammar#isSorted()
+ * @param config a {@link org.apache.joshua.decoder.JoshuaConfiguration} object
+ */
+ public AbstractGrammar(JoshuaConfiguration config) {
+ this.joshuaConfiguration = config;
+ this.sorted = false;
+ }
+
+ public AbstractGrammar(int owner, int spanLimit) {
+ this.sorted = false;
+ this.owner = owner;
+ this.spanLimit = spanLimit;
+ }
+
+ public static final int OOV_RULE_ID = 0;
+
+ /**
+ * Cube-pruning requires that the grammar be sorted based on the latest feature functions. To
+ * avoid synchronization, this method should be called before multiple threads are initialized for
+ * parallel decoding
+ * @param models {@link java.util.List} of {@link org.apache.joshua.decoder.ff.FeatureFunction}'s
+ */
+ public void sortGrammar(List<FeatureFunction> models) {
+ Trie root = getTrieRoot();
+ if (root != null) {
+ sort(root, models);
+ setSorted(true);
+ }
+ }
+
+ /* See Javadoc comments for Grammar interface. */
+ public boolean isSorted() {
+ return sorted;
+ }
+
+ /**
+ * Sets the flag indicating whether this grammar is sorted.
+ * <p>
+ * This method is called by {@link org.apache.joshua.decoder.ff.tm.AbstractGrammar#sortGrammar(List)}
+ * to indicate that the grammar has been sorted.</p>
+ *
+ * <p>Its scope is protected so that child classes that override <code>sortGrammar</code> will also
+ * be able to call this method to indicate that the grammar has been sorted.</p>
+ *
+ * @param sorted set to true if the grammar is sorted
+ */
+ protected void setSorted(boolean sorted) {
+ this.sorted = sorted;
+ LOG.debug("This grammar is now sorted: {}", this);
+ }
+
+ /**
+ * Recursively sorts the grammar using the provided feature functions.
+ * <p>
+ * This method first sorts the rules stored at the provided node, then recursively calls itself on
+ * the child nodes of the provided node.
+ *
+ * @param node Grammar node in the <code>Trie</code> whose rules should be sorted.
+ * @param models Feature function models to use during sorting.
+ */
+ private void sort(Trie node, List<FeatureFunction> models) {
+
+ if (node != null) {
+ if (node.hasRules()) {
+ RuleCollection rules = node.getRuleCollection();
+ LOG.debug("Sorting node {}", Arrays.toString(rules.getSourceSide()));
+
+ /* This causes the rules at this trie node to be sorted */
+ rules.getSortedRules(models);
+
+ if (LOG.isDebugEnabled()) {
+ StringBuilder s = new StringBuilder();
+ for (Rule r : rules.getSortedRules(models)) {
+ s.append("\n\t" + r.getLHS() + " ||| " + Arrays.toString(r.getFrench()) + " ||| "
+ + Arrays.toString(r.getEnglish()) + " ||| " + r.getFeatureVector() + " ||| "
+ + r.getEstimatedCost() + " " + r.getClass().getName() + "@"
+ + Integer.toHexString(System.identityHashCode(r)));
+ }
+ LOG.debug("{}", s);
+ }
+ }
+
+ if (node.hasExtensions()) {
+ for (Trie child : node.getExtensions()) {
+ sort(child, models);
+ }
+ } else {
+ LOG.debug("Node has 0 children to extend: {}", node);
+ }
+ }
+ }
+
+ // write grammar to disk
+ public void writeGrammarOnDisk(String file) {
+ }
+
+ /**
+ * Adds OOV rules for all words in the input lattice to the current grammar. Uses addOOVRule() so that
+ * sub-grammars can define different types of OOV rules if needed (as is used in {@link PhraseTable}).
+ *
+ * @param grammar Grammar in the Trie
+ * @param inputLattice the lattice representing the input sentence
+ * @param featureFunctions a list of feature functions used for scoring
+ * @param onlyTrue determine if word is actual OOV.
+ */
+ public static void addOOVRules(Grammar grammar, Lattice<Token> inputLattice,
+ List<FeatureFunction> featureFunctions, boolean onlyTrue) {
+ /*
+ * Add OOV rules; This should be called after the manual constraints have
+ * been set up.
+ */
+ HashSet<Integer> words = new HashSet<Integer>();
+ for (Node<Token> node : inputLattice) {
+ for (Arc<Token> arc : node.getOutgoingArcs()) {
+ // create a rule, but do not add into the grammar trie
+ // TODO: which grammar should we use to create an OOV rule?
+ int sourceWord = arc.getLabel().getWord();
+ if (sourceWord == Vocabulary.id(Vocabulary.START_SYM)
+ || sourceWord == Vocabulary.id(Vocabulary.STOP_SYM))
+ continue;
+
+ // Determine if word is actual OOV.
+ if (onlyTrue && ! Vocabulary.hasId(sourceWord))
+ continue;
+
+ words.add(sourceWord);
+ }
+ }
+
+ for (int sourceWord: words)
+ grammar.addOOVRules(sourceWord, featureFunctions);
+
+ // Sort all the rules (not much to actually do, this just marks it as sorted)
+ grammar.sortGrammar(featureFunctions);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/BasicRuleCollection.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/BasicRuleCollection.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/BasicRuleCollection.java
new file mode 100644
index 0000000..4cffb2f
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/BasicRuleCollection.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.tm;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.joshua.decoder.ff.FeatureFunction;
+
+/**
+ * Basic collection of translation rules.
+ *
+ * @author Lane Schwartz
+ * @author Zhifei Li
+ */
+public class BasicRuleCollection implements RuleCollection {
+
+ /**
+ * Indicates whether the rules in this collection have been sorted based on the latest feature
+ * function values.
+ */
+ protected boolean sorted;
+
+ /** List of rules stored in this collection. */
+ protected final List<Rule> rules;
+
+ /** Number of nonterminals in the source pattern. */
+ protected int arity;
+
+ /**
+ * Sequence of terminals and nonterminals in the source pattern.
+ */
+ protected int[] sourceTokens;
+
+ /**
+ * Constructs an initially empty rule collection.
+ *
+ * @param arity Number of nonterminals in the source pattern
+ * @param sourceTokens Sequence of terminals and nonterminals in the source pattern
+ */
+ public BasicRuleCollection(int arity, int[] sourceTokens) {
+ this.rules = new ArrayList<Rule>();
+ this.sourceTokens = sourceTokens;
+ this.arity = arity;
+ this.sorted = false;
+ }
+
+ public int getArity() {
+ return this.arity;
+ }
+
+ /**
+ * Returns a list of the rules, without ensuring that they are first sorted.
+ */
+ @Override
+ public List<Rule> getRules() {
+ return this.rules;
+ }
+
+ @Override
+ public boolean isSorted() {
+ return sorted;
+ }
+
+ /**
+ * Return a list of rules sorted according to their estimated model costs.
+ */
+ @Override
+ public synchronized List<Rule> getSortedRules(List<FeatureFunction> models) {
+ if (! isSorted()) {
+ for (Rule rule: getRules())
+ rule.estimateRuleCost(models);
+
+ Collections.sort(rules, Rule.EstimatedCostComparator);
+ this.sorted = true;
+ }
+
+ return this.rules;
+ }
+
+ public int[] getSourceSide() {
+ return this.sourceTokens;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/CreateGlueGrammar.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/CreateGlueGrammar.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/CreateGlueGrammar.java
new file mode 100644
index 0000000..ce1e7d1
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/CreateGlueGrammar.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.tm;
+
+import static org.apache.joshua.decoder.ff.tm.packed.PackedGrammar.VOCABULARY_FILENAME;
+import static org.apache.joshua.util.FormatUtils.cleanNonTerminal;
+import static org.apache.joshua.util.FormatUtils.isNonterminal;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.util.io.LineReader;
+
+import org.kohsuke.args4j.CmdLineException;
+import org.kohsuke.args4j.CmdLineParser;
+import org.kohsuke.args4j.Option;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class CreateGlueGrammar {
+
+
+ private static final Logger LOG = LoggerFactory.getLogger(CreateGlueGrammar.class);
+
+ private final Set<String> nonTerminalSymbols = new HashSet<>();
+
+ @Option(name = "--grammar", aliases = {"-g"}, required = true, usage = "provide grammar to determine list of NonTerminal symbols.")
+ private String grammarPath;
+
+ @Option(name = "--goal", aliases = {"-goal"}, required = false, usage = "specify custom GOAL symbol. Default: 'GOAL'")
+ private String goalSymbol = cleanNonTerminal(new JoshuaConfiguration().goal_symbol);
+
+ /* Rule templates */
+ // [GOAL] ||| <s> ||| <s> ||| 0
+ private static final String R_START = "[%1$s] ||| <s> ||| <s> ||| 0";
+ // [GOAL] ||| [GOAL,1] [X,2] ||| [GOAL,1] [X,2] ||| -1
+ private static final String R_TWO = "[%1$s] ||| [%1$s,1] [%2$s,2] ||| [%1$s,1] [%2$s,2] ||| -1";
+ // [GOAL] ||| [GOAL,1] </s> ||| [GOAL,1] </s> ||| 0
+ private static final String R_END = "[%1$s] ||| [%1$s,1] </s> ||| [%1$s,1] </s> ||| 0";
+ // [GOAL] ||| <s> [X,1] </s> ||| <s> [X,1] </s> ||| 0
+ private static final String R_TOP = "[%1$s] ||| <s> [%2$s,1] </s> ||| <s> [%2$s,1] </s> ||| 0";
+
+ private void run() throws IOException {
+
+ File grammar_file = new File(grammarPath);
+ if (!grammar_file.exists()) {
+ throw new IOException("Grammar file doesn't exist: " + grammarPath);
+ }
+
+ // in case of a packedGrammar, we read the serialized vocabulary,
+ // collecting all cleaned nonTerminal symbols.
+ if (grammar_file.isDirectory()) {
+ Vocabulary.read(new File(grammarPath + File.separator + VOCABULARY_FILENAME));
+ for (int i = 0; i < Vocabulary.size(); ++i) {
+ final String token = Vocabulary.word(i);
+ if (isNonterminal(token)) {
+ nonTerminalSymbols.add(cleanNonTerminal(token));
+ }
+ }
+ // otherwise we collect cleaned left-hand sides from the rules in the text grammar.
+ } else {
+ final LineReader reader = new LineReader(grammarPath);
+ while (reader.hasNext()) {
+ final String line = reader.next();
+ int lhsStart = line.indexOf("[") + 1;
+ int lhsEnd = line.indexOf("]");
+ if (lhsStart < 1 || lhsEnd < 0) {
+ LOG.info("malformed rule: {}\n", line);
+ continue;
+ }
+ final String lhs = line.substring(lhsStart, lhsEnd);
+ nonTerminalSymbols.add(lhs);
+ }
+ }
+
+ LOG.info("{} nonTerminal symbols read: {}", nonTerminalSymbols.size(),
+ nonTerminalSymbols.toString());
+
+ // write glue rules to stdout
+
+ System.out.println(String.format(R_START, goalSymbol));
+
+ for (String nt : nonTerminalSymbols)
+ System.out.println(String.format(R_TWO, goalSymbol, nt));
+
+ System.out.println(String.format(R_END, goalSymbol));
+
+ for (String nt : nonTerminalSymbols)
+ System.out.println(String.format(R_TOP, goalSymbol, nt));
+
+ }
+
+ public static void main(String[] args) throws IOException {
+ final CreateGlueGrammar glueCreator = new CreateGlueGrammar();
+ final CmdLineParser parser = new CmdLineParser(glueCreator);
+
+ try {
+ parser.parseArgument(args);
+ glueCreator.run();
+ } catch (CmdLineException e) {
+ LOG.error(e.getMessage(), e);
+ parser.printUsage(System.err);
+ System.exit(1);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/Grammar.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/Grammar.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/Grammar.java
new file mode 100644
index 0000000..06252a1
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/Grammar.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.tm;
+
+import java.util.List;
+
+import org.apache.joshua.decoder.ff.FeatureFunction;
+
+/**
+ * Grammar is a class for wrapping a trie of TrieGrammar in order to store holistic metadata.
+ *
+ * @author wren ng thornton wren@users.sourceforge.net
+ * @author Zhifei Li, zhifei.work@gmail.com
+ */
+public interface Grammar {
+
+ /**
+ * Gets the root of the <code>Trie</code> backing this grammar.
+ * <p>
+ * <em>Note</em>: This method should run as a small constant-time function.
+ *
+ * @return the root of the <code>Trie</code> backing this grammar
+ */
+ Trie getTrieRoot();
+
+ /**
+ * After calling this method, the rules in this grammar are guaranteed to be sorted based on the
+ * latest feature function values.
+ * <p>
+ * Cube-pruning requires that the grammar be sorted based on the latest feature functions.
+ *
+ * @param models list of {@link org.apache.joshua.decoder.ff.FeatureFunction}'s
+ */
+ void sortGrammar(List<FeatureFunction> models);
+
+ /**
+ * Determines whether the rules in this grammar have been sorted based on the latest feature
+ * function values.
+ * <p>
+ * This method is needed for the cube-pruning algorithm.
+ *
+ * @return <code>true</code> if the rules in this grammar have been sorted based on the latest
+ * feature function values, <code>false</code> otherwise
+ */
+ boolean isSorted();
+
+ /**
+ * Returns whether this grammar has any valid rules for covering a particular span of a sentence.
+ * Hiero's "glue" grammar will only say True if the span is longer than our span limit, and is
+ * anchored at startIndex==0. Hiero's "regular" grammar will only say True if the span is less
+ * than the span limit. Other grammars, e.g. for rule-based systems, may have different behaviors.
+ *
+ * @param startIndex Indicates the starting index of a phrase in a source input phrase, or a
+ * starting node identifier in a source input lattice
+ * @param endIndex Indicates the ending index of a phrase in a source input phrase, or an ending
+ * node identifier in a source input lattice
+ * @param pathLength Length of the input path in a source input lattice. If a source input phrase
+ * is used instead of a lattice, this value will likely be ignored by the underlying
+ * implementation, but would normally be defined as <code>endIndex-startIndex</code>
+ * @return true if there is a rule for this span
+ */
+ boolean hasRuleForSpan(int startIndex, int endIndex, int pathLength);
+
+ /**
+ * Gets the number of rules stored in the grammar.
+ *
+ * @return the number of rules stored in the grammar
+ */
+ int getNumRules();
+
+ /**
+ * Returns the number of dense features.
+ *
+ * @return the number of dense features
+ */
+ int getNumDenseFeatures();
+
+ /**
+ * Return the grammar's owner.
+ * @return grammar owner
+ */
+ int getOwner();
+
+ /**
+ * Return the maximum source phrase length (terminals + nonterminals)
+ * @return the maximum source phrase length
+ */
+ int getMaxSourcePhraseLength();
+
+ /**
+ * Add an OOV rule for the requested word for the grammar.
+ *
+ * @param word input word to add rules to
+ * @param featureFunctions a {@link java.util.List} of {@link org.apache.joshua.decoder.ff.FeatureFunction}'s
+ */
+ void addOOVRules(int word, List<FeatureFunction> featureFunctions);
+
+ /**
+ * Add a rule to the grammar.
+ *
+ * @param rule the {@link org.apache.joshua.decoder.ff.tm.Rule}
+ */
+ void addRule(Rule rule);
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/GrammarReader.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/GrammarReader.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/GrammarReader.java
new file mode 100644
index 0000000..df00255
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/GrammarReader.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.tm;
+
+import java.io.IOException;
+import java.util.Iterator;
+
+import org.apache.joshua.decoder.Decoder;
+import org.apache.joshua.util.io.LineReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This is a base class for simple, ASCII line-based grammars that are stored on disk.
+ *
+ * @author Juri Ganitkevitch
+ *
+ */
+public abstract class GrammarReader<R extends Rule> implements Iterable<R>, Iterator<R> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(GrammarReader.class);
+
+ protected static String fieldDelimiter;
+ protected static String description;
+
+ protected String fileName;
+ protected LineReader reader;
+ protected String lookAhead;
+ protected int numRulesRead;
+
+
+ // dummy constructor for
+ public GrammarReader() {
+ this.fileName = null;
+ }
+
+ public GrammarReader(String fileName) throws IOException {
+ this.fileName = fileName;
+ this.reader = new LineReader(fileName);
+ LOG.info("Reading grammar from file {}...", fileName);
+ numRulesRead = 0;
+ advanceReader();
+ }
+
+ // the reader is the iterator itself
+ public Iterator<R> iterator() {
+ return this;
+ }
+
+ /** Unsupported Iterator method. */
+ public void remove() throws UnsupportedOperationException {
+ throw new UnsupportedOperationException();
+ }
+
+ public void close() {
+ if (null != this.reader) {
+ try {
+ this.reader.close();
+ } catch (IOException e) {
+ LOG.warn(e.getMessage(), e);
+ LOG.error("Error closing grammar file stream: {}", this.fileName);
+ }
+ this.reader = null;
+ }
+ }
+
+ /**
+ * For correct behavior <code>close</code> must be called on every GrammarReader, however this
+ * code attempts to avoid resource leaks.
+ *
+ * @see org.apache.joshua.util.io.LineReader
+ */
+ @Override
+ protected void finalize() throws Throwable {
+ if (this.reader != null) {
+ LOG.error("Grammar file stream was not closed, this indicates a coding error: {}",
+ this.fileName);
+ }
+
+ this.close();
+ super.finalize();
+ }
+
+ @Override
+ public boolean hasNext() {
+ return lookAhead != null;
+ }
+
+ private void advanceReader() {
+ try {
+ lookAhead = reader.readLine();
+ numRulesRead++;
+ } catch (IOException e) {
+ LOG.error("Error reading grammar from file: {}", fileName);
+ LOG.error(e.getMessage(), e);
+ }
+ if (lookAhead == null && reader != null) {
+ this.close();
+ }
+ }
+
+ /**
+ * Read the next line, and print reader progress.
+ */
+ @Override
+ public R next() {
+ String line = lookAhead;
+
+ int oldProgress = reader.progress();
+ advanceReader();
+
+
+ if (Decoder.VERBOSE >= 1) {
+ int newProgress = (reader != null) ? reader.progress() : 100;
+
+ //TODO: review this code. It is better to print progress based on time gap (like for every 1s or 2sec) than %!
+ if (newProgress > oldProgress) {
+ for (int i = oldProgress + 1; i <= newProgress; i++)
+ if (i == 97) {
+ System.err.print("1");
+ } else if (i == 98) {
+ System.err.print("0");
+ } else if (i == 99) {
+ System.err.print("0");
+ } else if (i == 100) {
+ System.err.println("%");
+ } else if (i % 10 == 0) {
+ System.err.print(String.format("%d", i));
+ System.err.flush();
+ } else if ((i - 1) % 10 == 0)
+ ; // skip at 11 since 10, 20, etc take two digits
+ else {
+ System.err.print(".");
+ System.err.flush();
+ }
+ }
+ }
+ return parseLine(line);
+ }
+
+ protected abstract R parseLine(String line);
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/Rule.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/Rule.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/Rule.java
new file mode 100644
index 0000000..0e5a4bc
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/Rule.java
@@ -0,0 +1,633 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.tm;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Pattern;
+
+import com.google.common.base.Supplier;
+import com.google.common.base.Suppliers;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.ff.FeatureFunction;
+import org.apache.joshua.decoder.ff.FeatureVector;
+import org.apache.joshua.decoder.segment_file.Sentence;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class define the interface for Rule.
+ *
+ * All feature scores are interpreted as negative log probabilities, and are therefore negated.
+ * Note that not all features need to be negative log probs, but you should be aware that they
+ * will be negated, so if you want a positive count, it should come in as negative.
+ *
+ * Normally, the feature score in the rule should be *cost* (i.e., -LogP), so that the feature
+ * weight should be positive
+ *
+ * @author Zhifei Li, zhifei.work@gmail.com
+ * @author Matt Post post@cs.jhu.edu
+ */
+public class Rule implements Comparator<Rule>, Comparable<Rule> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Rule.class);
+ private int lhs; // tag of this rule
+ private int[] source; // pointer to the RuleCollection, as all the rules under it share the same
+ // Source side
+ protected int arity;
+
+ // And a string containing the sparse ones
+ //protected final String sparseFeatureString;
+ protected final Supplier<String> sparseFeatureStringSupplier;
+ private final Supplier<FeatureVector> featuresSupplier;
+
+ /*
+ * a feature function will be fired for this rule only if the owner of the rule matches the owner
+ * of the feature function
+ */
+ private int owner = -1;
+
+ /**
+ * This is the cost computed only from the features present with the grammar rule. This cost is
+ * needed to sort the rules in the grammar for cube pruning, but isn't the full cost of applying
+ * the rule (which will include contextual features that can't be computed until the rule is
+ * applied).
+ */
+ private float estimatedCost = Float.NEGATIVE_INFINITY;
+
+ private float precomputableCost = Float.NEGATIVE_INFINITY;
+
+ private int[] target;
+
+ // The alignment string, e.g., 0-0 0-1 1-1 2-1
+ private String alignmentString;
+ private final Supplier<byte[]> alignmentSupplier;
+
+ /**
+ * Constructs a new rule using the provided parameters. Rule id for this rule is
+ * undefined. Note that some of the sparse features may be unlabeled, but they cannot be mapped to
+ * their default names ("tm_OWNER_INDEX") until later, when we know the owner of the rule. This is
+ * not known until the rule is actually added to a grammar in Grammar::addRule().
+ *
+ * Constructor used by other constructors below;
+ *
+ * @param lhs Left-hand side of the rule.
+ * @param source Source language right-hand side of the rule.
+ * @param target Target language right-hand side of the rule.
+ * @param sparseFeatures Feature value scores for the rule.
+ * @param arity Number of nonterminals in the source language right-hand side.
+ * @param owner todo
+ */
+ public Rule(int lhs, int[] source, int[] target, String sparseFeatures, int arity, int owner) {
+ this.lhs = lhs;
+ this.source = source;
+ this.arity = arity;
+ this.owner = owner;
+ this.target = target;
+ this.sparseFeatureStringSupplier = Suppliers.memoize(() -> { return sparseFeatures; });
+ this.featuresSupplier = initializeFeatureSupplierFromString();
+ this.alignmentSupplier = initializeAlignmentSupplier();
+ }
+
+ /**
+ * Constructor used by PackedGrammar's sortRules()
+ * @param lhs todo
+ * @param sourceRhs todo
+ * @param targetRhs todo
+ * @param features todo
+ * @param arity todo
+ * @param owner todo
+ */
+ public Rule(int lhs, int[] sourceRhs, int[] targetRhs, FeatureVector features, int arity, int owner) {
+ this.lhs = lhs;
+ this.source = sourceRhs;
+ this.arity = arity;
+ this.owner = owner;
+ this.target = targetRhs;
+ this.featuresSupplier = Suppliers.memoize(() -> { return features; });
+ this.sparseFeatureStringSupplier = initializeSparseFeaturesStringSupplier();
+ this.alignmentSupplier = initializeAlignmentSupplier();
+ }
+
+ /**
+ * Constructor used for SamtFormatReader and GrammarBuilderWalkerFunction's getRuleWithSpans()
+ * Owner set to -1
+ * @param lhs todo
+ * @param sourceRhs todo
+ * @param targetRhs todo
+ * @param sparseFeatures todo
+ * @param arity todo
+ */
+ public Rule(int lhs, int[] sourceRhs, int[] targetRhs, String sparseFeatures, int arity) {
+ this(lhs, sourceRhs, targetRhs, sparseFeatures, arity, -1);
+ }
+
+ /**
+ * Constructor used for addOOVRules(), HieroFormatReader and PhraseRule.
+ * @param lhs todo
+ * @param sourceRhs todo
+ * @param targetRhs todo
+ * @param sparseFeatures todo
+ * @param arity todo
+ * @param alignment todo
+ */
+ public Rule(int lhs, int[] sourceRhs, int[] targetRhs, String sparseFeatures, int arity, String alignment) {
+ this(lhs, sourceRhs, targetRhs, sparseFeatures, arity);
+ this.alignmentString = alignment;
+ }
+
+ /**
+ * Constructor (implicitly) used by PackedRule
+ */
+ public Rule() {
+ this.lhs = -1;
+ this.sparseFeatureStringSupplier = initializeSparseFeaturesStringSupplier();
+ this.featuresSupplier = initializeFeatureSupplierFromString();
+ this.alignmentSupplier = initializeAlignmentSupplier();
+ }
+
+ // ==========================================================================
+ // Lazy loading Suppliers for alignments, feature vector, and feature strings
+ // ==========================================================================
+
+ private Supplier<byte[]> initializeAlignmentSupplier(){
+ return Suppliers.memoize(() ->{
+ byte[] alignment = null;
+ String alignmentString = getAlignmentString();
+ if (alignmentString != null) {
+ String[] tokens = alignmentString.split("[-\\s]+");
+ alignment = new byte[tokens.length];
+ for (int i = 0; i < tokens.length; i++)
+ alignment[i] = (byte) Short.parseShort(tokens[i]);
+ }
+ return alignment;
+ });
+ }
+
+ /**
+ * If Rule was constructed with sparseFeatures String, we lazily populate the
+ * FeatureSupplier.
+ */
+ private Supplier<FeatureVector> initializeFeatureSupplierFromString(){
+ return Suppliers.memoize(() ->{
+ if (owner != -1) {
+ return new FeatureVector(getFeatureString(), "tm_" + Vocabulary.word(owner) + "_");
+ } else {
+ return new FeatureVector();
+ }
+ });
+ }
+
+ /**
+ * If Rule was constructed with a FeatureVector, we lazily populate the sparseFeaturesStringSupplier.
+ */
+ private Supplier<String> initializeSparseFeaturesStringSupplier() {
+ return Suppliers.memoize(() -> {
+ return getFeatureVector().toString();
+ });
+ }
+
+ // ===============================================================
+ // Attributes
+ // ===============================================================
+
+ public void setEnglish(int[] eng) {
+ this.target = eng;
+ }
+
+ public int[] getEnglish() {
+ return this.target;
+ }
+
+ /**
+ * Two Rules are equal of they have the same LHS, the same source RHS and the same target
+ * RHS.
+ *
+ * @param o the object to check for equality
+ * @return true if o is the same Rule as this rule, false otherwise
+ */
+ public boolean equals(Object o) {
+ if (!(o instanceof Rule)) {
+ return false;
+ }
+ Rule other = (Rule) o;
+ if (getLHS() != other.getLHS()) {
+ return false;
+ }
+ if (!Arrays.equals(getFrench(), other.getFrench())) {
+ return false;
+ }
+ if (!Arrays.equals(target, other.getEnglish())) {
+ return false;
+ }
+ return true;
+ }
+
+ public int hashCode() {
+ // I just made this up. If two rules are equal they'll have the
+ // same hashcode. Maybe someone else can do a better job though?
+ int frHash = Arrays.hashCode(getFrench());
+ int enHash = Arrays.hashCode(target);
+ return frHash ^ enHash ^ getLHS();
+ }
+
+ // ===============================================================
+ // Attributes
+ // ===============================================================
+
+ public void setArity(int arity) {
+ this.arity = arity;
+ }
+
+ public int getArity() {
+ return this.arity;
+ }
+
+ public void setOwner(int owner) {
+ this.owner = owner;
+ }
+
+ public int getOwner() {
+ return this.owner;
+ }
+
+ public void setLHS(int lhs) {
+ this.lhs = lhs;
+ }
+
+ public int getLHS() {
+ return this.lhs;
+ }
+
+ public void setFrench(int[] french) {
+ this.source = french;
+ }
+
+ public int[] getFrench() {
+ return this.source;
+ }
+
+ /**
+ * This function does the work of turning the string version of the sparse features (passed in
+ * when the rule was created) into an actual set of features. This is a bit complicated because we
+ * support intermingled labeled and unlabeled features, where the unlabeled features are mapped to
+ * a default name template of the form "tm_OWNER_INDEX".
+ *
+ * This function returns the dense (phrasal) features discovered when the rule was loaded. Dense
+ * features are the list of unlabeled features that preceded labeled ones. They can also be
+ * specified as labeled features of the form "tm_OWNER_INDEX", but the former format is preferred.
+ *
+ * @return the {@link org.apache.joshua.decoder.ff.FeatureVector} for this rule
+ */
+ public FeatureVector getFeatureVector() {
+ return featuresSupplier.get();
+ }
+
+ /**
+ * This function returns the estimated cost of a rule, which should have been computed when the
+ * grammar was first sorted via a call to Rule::estimateRuleCost(). This function is a getter
+ * only; it will not compute the value if it has not already been set. It is necessary in addition
+ * to estimateRuleCost(models) because sometimes the value needs to be retrieved from contexts
+ * that do not have access to the feature functions.
+ *
+ * This function is called by the rule comparator when sorting the grammar. As such it may be
+ * called many times and any implementation of it should be a cached implementation.
+ *
+ * @return the estimated cost of the rule (a lower bound on the true cost)
+ */
+ public float getEstimatedCost() {
+ return estimatedCost;
+ }
+
+ /**
+ * Precomputable costs is the inner product of the weights found on each grammar rule and the
+ * weight vector. This is slightly different from the estimated rule cost, which can include other
+ * features (such as a language model estimate). This getter and setter should also be cached, and
+ * is basically provided to allow the PhraseModel feature to cache its (expensive) computation for
+ * each rule.
+ *
+ * The weights are passed in as dense weights and sparse weights. This allows the dense weight
+ * precomputation to be even faster (since we don't have to query a hash map.
+ *
+ * @param dense_weights the dense weights from the model
+ * @param weights the sparse weights from the model
+ */
+ public void setPrecomputableCost(float[] dense_weights, FeatureVector weights) {
+ float cost = 0.0f;
+ FeatureVector features = getFeatureVector();
+ for (int i = 0; i < features.getDenseFeatures().size() && i < dense_weights.length; i++) {
+ cost += dense_weights[i] * features.getDense(i);
+ }
+
+ for (String key: features.getSparseFeatures().keySet()) {
+ cost += weights.getSparse(key) * features.getSparse(key);
+ }
+
+ this.precomputableCost = cost;
+ }
+
+ /**
+ * @return the precomputed model cost of each rule
+ */
+ public float getPrecomputableCost() {
+ return precomputableCost;
+ }
+
+ public float getDenseFeature(int k) {
+ return getFeatureVector().getDense(k);
+ }
+
+ /**
+ * This function estimates the cost of a rule, which is used for sorting the rules for cube
+ * pruning. The estimated cost is basically the set of precomputable features (features listed
+ * along with the rule in the grammar file) along with any other estimates that other features
+ * would like to contribute (e.g., a language model estimate). This cost will be a lower bound on
+ * the rule's actual cost.
+ *
+ * The value of this function is used only for sorting the rules. When the rule is later applied
+ * in context to particular hypernodes, the rule's actual cost is computed.
+ *
+ * @param models the list of models available to the decoder
+ * @return estimated cost of the rule
+ */
+ public float estimateRuleCost(List<FeatureFunction> models) {
+ if (null == models)
+ return 0.0f;
+
+ if (this.estimatedCost <= Float.NEGATIVE_INFINITY) {
+ this.estimatedCost = 0.0f; // weights.innerProduct(computeFeatures());
+
+ LOG.debug("estimateCost({} ;; {})", getFrenchWords(), getEnglishWords());
+ for (FeatureFunction ff : models) {
+ float val = ff.estimateCost(this, null);
+ LOG.debug(" FEATURE {} -> {}", ff.getName(), val);
+ this.estimatedCost += val;
+ }
+ }
+
+ return estimatedCost;
+ }
+
+ // ===============================================================
+ // Methods
+ // ===============================================================
+
+ public String toString() {
+ StringBuffer sb = new StringBuffer();
+ sb.append(Vocabulary.word(this.getLHS()));
+ sb.append(" ||| ");
+ sb.append(getFrenchWords());
+ sb.append(" ||| ");
+ sb.append(getEnglishWords());
+ sb.append(" |||");
+ sb.append(" " + getFeatureVector());
+ sb.append(String.format(" ||| est=%.3f", getEstimatedCost()));
+ sb.append(String.format(" pre=%.3f", getPrecomputableCost()));
+ return sb.toString();
+ }
+
+ /**
+ * Returns a version of the rule suitable for reading in from a text file.
+ *
+ * @return string version of the rule
+ */
+ public String textFormat() {
+ StringBuffer sb = new StringBuffer();
+ sb.append(Vocabulary.word(this.getLHS()));
+ sb.append(" |||");
+
+ int nt = 1;
+ for (int i = 0; i < getFrench().length; i++) {
+ if (getFrench()[i] < 0)
+ sb.append(" " + Vocabulary.word(getFrench()[i]).replaceFirst("\\]", String.format(",%d]", nt++)));
+ else
+ sb.append(" " + Vocabulary.word(getFrench()[i]));
+ }
+ sb.append(" |||");
+ nt = 1;
+ for (int i = 0; i < getEnglish().length; i++) {
+ if (getEnglish()[i] < 0)
+ sb.append(" " + Vocabulary.word(getEnglish()[i]).replaceFirst("\\]", String.format(",%d]", nt++)));
+ else
+ sb.append(" " + Vocabulary.word(getEnglish()[i]));
+ }
+ sb.append(" |||");
+ sb.append(" " + getFeatureString());
+ if (getAlignmentString() != null)
+ sb.append(" ||| " + getAlignmentString());
+ return sb.toString();
+ }
+
+ public String getFeatureString() {
+ return sparseFeatureStringSupplier.get();
+ }
+
+ /**
+ * Returns an alignment as a sequence of integers. The integers at positions i and i+1 are paired,
+ * with position i indexing the source and i+1 the target.
+ *
+ * @return a byte[] from the {@link com.google.common.base.Supplier}
+ */
+ public byte[] getAlignment() {
+ return this.alignmentSupplier.get();
+ }
+
+ public String getAlignmentString() {
+ return this.alignmentString;
+ }
+
+ /**
+ * The nonterminals on the English side are pointers to the source side nonterminals (-1 and -2),
+ * rather than being directly encoded. These number indicate the correspondence between the
+ * nonterminals on each side, introducing a level of indirection however when we want to resolve
+ * them. So to get the ID, we need to look up the corresponding source side ID.
+ *
+ * @return The string of English words
+ */
+ public String getEnglishWords() {
+ int[] foreignNTs = getForeignNonTerminals();
+
+ StringBuilder sb = new StringBuilder();
+ for (Integer index : getEnglish()) {
+ if (index >= 0)
+ sb.append(Vocabulary.word(index) + " ");
+ else
+ sb.append(Vocabulary.word(foreignNTs[-index - 1]).replace("]",
+ String.format(",%d] ", Math.abs(index))));
+ }
+
+ return sb.toString().trim();
+ }
+
+ public boolean isTerminal() {
+ for (int i = 0; i < getEnglish().length; i++)
+ if (getEnglish()[i] < 0)
+ return false;
+
+ return true;
+ }
+
+ /**
+ * Return the French (source) nonterminals as list of Strings
+ *
+ * @return a list of strings
+ */
+ public int[] getForeignNonTerminals() {
+ int[] nts = new int[getArity()];
+ int index = 0;
+ for (int id : getFrench())
+ if (id < 0)
+ nts[index++] = -id;
+ return nts;
+ }
+
+ /**
+ * Returns an array of size getArity() containing the source indeces of non terminals.
+ *
+ * @return an array of size getArity() containing the source indeces of non terminals
+ */
+ public int[] getNonTerminalSourcePositions() {
+ int[] nonTerminalPositions = new int[getArity()];
+ int ntPos = 0;
+ for (int sourceIdx = 0; sourceIdx < getFrench().length; sourceIdx++) {
+ if (getFrench()[sourceIdx] < 0)
+ nonTerminalPositions[ntPos++] = sourceIdx;
+ }
+ return nonTerminalPositions;
+ }
+
+ /**
+ * Parses the Alignment byte[] into a Map from target to (possibly a list of) source positions.
+ * Used by the WordAlignmentExtractor.
+ *
+ * @return a {@link java.util.Map} of alignments
+ */
+ public Map<Integer, List<Integer>> getAlignmentMap() {
+ byte[] alignmentArray = getAlignment();
+ Map<Integer, List<Integer>> alignmentMap = new HashMap<Integer, List<Integer>>();
+ if (alignmentArray != null) {
+ for (int alignmentIdx = 0; alignmentIdx < alignmentArray.length; alignmentIdx += 2 ) {
+ int s = alignmentArray[alignmentIdx];
+ int t = alignmentArray[alignmentIdx + 1];
+ List<Integer> values = alignmentMap.get(t);
+ if (values == null)
+ alignmentMap.put(t, values = new ArrayList<Integer>());
+ values.add(s);
+ }
+ }
+ return alignmentMap;
+ }
+
+ /**
+ * Return the English (target) nonterminals as list of Strings
+ *
+ * @return list of strings
+ */
+ public int[] getEnglishNonTerminals() {
+ int[] nts = new int[getArity()];
+ int[] foreignNTs = getForeignNonTerminals();
+ int index = 0;
+
+ for (int i : getEnglish()) {
+ if (i < 0)
+ nts[index++] = foreignNTs[Math.abs(getEnglish()[i]) - 1];
+ }
+
+ return nts;
+ }
+
+ private int[] getNormalizedEnglishNonterminalIndices() {
+ int[] result = new int[getArity()];
+
+ int ntIndex = 0;
+ for (Integer index : getEnglish()) {
+ if (index < 0)
+ result[ntIndex++] = -index - 1;
+ }
+
+ return result;
+ }
+
+ public boolean isInverting() {
+ int[] normalizedEnglishNonTerminalIndices = getNormalizedEnglishNonterminalIndices();
+ if (normalizedEnglishNonTerminalIndices.length == 2) {
+ if (normalizedEnglishNonTerminalIndices[0] == 1) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public String getFrenchWords() {
+ return Vocabulary.getWords(getFrench());
+ }
+
+ public static final String NT_REGEX = "\\[[^\\]]+?\\]";
+
+ private Pattern getPattern() {
+ String source = getFrenchWords();
+ String pattern = Pattern.quote(source);
+ pattern = pattern.replaceAll(NT_REGEX, "\\\\E.+\\\\Q");
+ pattern = pattern.replaceAll("\\\\Q\\\\E", "");
+ pattern = "(?:^|\\s)" + pattern + "(?:$|\\s)";
+ return Pattern.compile(pattern);
+ }
+
+ /**
+ * Matches the string representation of the rule's source side against a sentence
+ *
+ * @param sentence {@link org.apache.joshua.lattice.Lattice} input
+ * @return true if there is a match
+ */
+ public boolean matches(Sentence sentence) {
+ boolean match = getPattern().matcher(sentence.fullSource()).find();
+ // System.err.println(String.format("match(%s,%s) = %s", Pattern.quote(getFrenchWords()),
+ // sentence.annotatedSource(), match));
+ return match;
+ }
+
+ /**
+ * This comparator is used for sorting the rules during cube pruning. An estimate of the cost
+ * of each rule is computed and used to sort.
+ */
+ public static Comparator<Rule> EstimatedCostComparator = new Comparator<Rule>() {
+ public int compare(Rule rule1, Rule rule2) {
+ float cost1 = rule1.getEstimatedCost();
+ float cost2 = rule2.getEstimatedCost();
+ return Float.compare(cost2, cost1);
+ }
+ };
+
+ public int compare(Rule rule1, Rule rule2) {
+ return EstimatedCostComparator.compare(rule1, rule2);
+ }
+
+ public int compareTo(Rule other) {
+ return EstimatedCostComparator.compare(this, other);
+ }
+
+ public String getRuleString() {
+ return String.format("%s -> %s ||| %s", Vocabulary.word(getLHS()), getFrenchWords(), getEnglishWords());
+ }
+}
[28/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/pro/ClassifierSVM.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/pro/ClassifierSVM.java b/joshua-core/src/main/java/org/apache/joshua/pro/ClassifierSVM.java
new file mode 100755
index 0000000..5c1f4e3
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/pro/ClassifierSVM.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.pro;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Vector;
+
+import org.apache.joshua.util.StreamGobbler;
+import org.apache.joshua.util.io.LineReader;
+
+public class ClassifierSVM implements ClassifierInterface {
+ @Override
+ public double[] runClassifier(Vector<String> samples, double[] initialLambda, int featDim) {
+ System.out.println("------- SVM training starts ------");
+
+ double[] lambda = new double[featDim + 1];
+ for (int i = 1; i <= featDim; i++)
+ lambda[i] = 0;
+
+ // String root_dir =
+ // "/media/Data/JHU/Research/MT discriminative LM training/joshua_expbleu/PRO_test/";
+ // String root_dir = "/home/ycao/WS11/nist_zh_en_percep/pro_forward/pro_libsvm/";
+
+ try {
+ // prepare training file for MegaM
+ PrintWriter prt = new PrintWriter(new FileOutputStream(trainingFilePath));
+
+ for (String line : samples) {
+ String[] feat = line.split("\\s+");
+
+ if (feat[feat.length - 1].equals("1"))
+ prt.print("+1 ");
+ else
+ prt.print("-1 ");
+
+ for (int i = 0; i < feat.length - 1; i++)
+ prt.print((i + 1) + ":" + feat[i] + " "); // feat id starts from 1!
+
+ prt.println();
+ }
+ prt.close();
+
+ // start running SVM
+ Runtime rt = Runtime.getRuntime();
+ // String cmd = "/home/yuan/tmp_libsvm_command";
+
+ Process p = rt.exec(commandFilePath); // only linear kernel is used
+
+ StreamGobbler errorGobbler = new StreamGobbler(p.getErrorStream(), 1);
+ StreamGobbler outputGobbler = new StreamGobbler(p.getInputStream(), 1);
+
+ errorGobbler.start();
+ outputGobbler.start();
+
+ int decStatus = p.waitFor();
+ if (decStatus != 0) {
+ throw new RuntimeException("Call to decoder returned " + decStatus + "; was expecting "
+ + 0 + ".");
+ }
+
+ // read the model file
+ boolean sv_start = false;
+ double coef;
+
+ for (String line: new LineReader(modelFilePath)) {
+ if (sv_start) // start reading support vectors and coefs
+ {
+ String[] val = line.split("\\s+");
+ coef = Double.parseDouble(val[0]);
+
+ // System.out.print(coef+" ");
+
+ for (int i = 1; i < val.length; i++) // only valid for linear kernel
+ // W = \sum_{i=1}^{l} y_i alpha_i phi(x_i)
+ // = \sum_{i=1}^{l} coef_i x_i
+ {
+ String[] sv = val[i].split(":"); // feat id
+ lambda[Integer.parseInt(sv[0])] += coef * Double.parseDouble(sv[1]); // index starts
+ // from 1
+ // System.out.print(Integer.parseInt(sv[0])+" "+Double.parseDouble(sv[1])+" ");
+ }
+
+ // System.out.println();
+ }
+
+ if (line.equals("SV")) sv_start = true;
+ }
+
+ File file = new File(trainingFilePath);
+ file.delete();
+ file = new File(modelFilePath);
+ file.delete();
+ } catch (IOException | InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+
+ System.out.println("------- SVM training ends ------");
+
+ return lambda;
+ }
+
+ @Override
+ /*
+ * for LibSVM: param[0] = LibSVM command file path param[1] = LibSVM training data file(generated
+ * on the fly) path param[2] = LibSVM model file(generated after training) path note: the training
+ * file path should be consistent with the one specified in command file
+ */
+ public void setClassifierParam(String[] param) {
+ if (param == null) {
+ throw new RuntimeException("ERROR: must provide parameters for LibSVM classifier!");
+ } else {
+ commandFilePath = param[0];
+ trainingFilePath = param[1];
+ modelFilePath = param[2];
+ }
+ }
+
+ String commandFilePath;
+ String trainingFilePath;
+ String modelFilePath;
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/pro/Optimizer.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/pro/Optimizer.java b/joshua-core/src/main/java/org/apache/joshua/pro/Optimizer.java
new file mode 100755
index 0000000..ad80305
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/pro/Optimizer.java
@@ -0,0 +1,454 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.pro;
+
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.Vector;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.metrics.EvaluationMetric;
+
+// this class implements the PRO tuning method
+public class Optimizer {
+ public Optimizer(long _seed, boolean[] _isOptimizable, Vector<String> _output, double[] _initialLambda,
+ HashMap<String, String>[] _feat_hash, HashMap<String, String>[] _stats_hash,
+ EvaluationMetric _evalMetric, int _Tau, int _Xi, double _metricDiff,
+ double[] _normalizationOptions, String _classifierAlg, String[] _classifierParam) {
+ sentNum = _feat_hash.length; // total number of training sentences
+ output = _output; // (not used for now)
+ initialLambda = _initialLambda;
+ isOptimizable = _isOptimizable;
+ paramDim = initialLambda.length - 1;
+ feat_hash = _feat_hash; // feature hash table
+ stats_hash = _stats_hash; // suff. stats hash table
+ evalMetric = _evalMetric; // evaluation metric
+ Tau = _Tau; // param Tau in PRO
+ Xi = _Xi; // param Xi in PRO
+ metricDiff = _metricDiff; // threshold for sampling acceptance
+ normalizationOptions = _normalizationOptions; // weight normalization option
+ randgen = new Random(_seed); // random number generator
+ classifierAlg = _classifierAlg; // classification algorithm
+ classifierParam = _classifierParam; // params for the specified classifier
+ }
+
+ public double[] run_Optimizer() {
+ // sampling from all candidates
+ Vector<String> allSamples = process_Params();
+
+ try {
+ // create classifier object from the given class name string
+ ClassifierInterface myClassifier =
+ (ClassifierInterface) Class.forName(classifierAlg).newInstance();
+ System.out.println("Total training samples(class +1 & class -1): " + allSamples.size());
+
+ // set classifier parameters
+ myClassifier.setClassifierParam(classifierParam);
+ //run classifier
+ finalLambda = myClassifier.runClassifier(allSamples, initialLambda, paramDim);
+ normalizeLambda(finalLambda);
+ //parameters that are not optimizable are assigned with initial values
+ for ( int i = 1; i < isOptimizable.length; ++i ) {
+ if ( !isOptimizable[i] )
+ finalLambda[i] = initialLambda[i];
+ }
+
+ double initMetricScore = computeCorpusMetricScore(initialLambda); // compute the initial
+ // corpus-level metric score
+ finalMetricScore = computeCorpusMetricScore(finalLambda); // compute the final
+ // corpus-level metric score
+
+ // for( int i=0; i<finalLambda.length; i++ ) System.out.print(finalLambda[i]+" ");
+ // System.out.println(); System.exit(0);
+
+ // prepare the printing info
+ // int numParamToPrint = 0;
+ // String result = "";
+ // numParamToPrint = paramDim > 10 ? 10 : paramDim; // how many parameters to print
+ // result = paramDim > 10 ? "Final lambda (first 10): {" : "Final lambda: {";
+
+ // for (int i = 1; i <= numParamToPrint; i++)
+ // result += String.format("%.4f", finalLambda[i]) + " ";
+
+ output.add("Initial "
+ + evalMetric.get_metricName() + ": " + String.format("%.4f", initMetricScore) + "\nFinal "
+ + evalMetric.get_metricName() + ": " + String.format("%.4f", finalMetricScore));
+
+ // System.out.println(output);
+
+ return finalLambda;
+ } catch (ClassNotFoundException | InstantiationException | IllegalAccessException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public double computeCorpusMetricScore(double[] finalLambda) {
+ int suffStatsCount = evalMetric.get_suffStatsCount();
+ double modelScore;
+ double maxModelScore;
+ Set<String> candSet;
+ String candStr;
+ String[] feat_str;
+ String[] tmpStatsVal = new String[suffStatsCount];
+ int[] corpusStatsVal = new int[suffStatsCount];
+ for (int i = 0; i < suffStatsCount; i++)
+ corpusStatsVal[i] = 0;
+
+ for (int i = 0; i < sentNum; i++) {
+ candSet = feat_hash[i].keySet();
+
+ // find out the 1-best candidate for each sentence
+ maxModelScore = NegInf;
+ for (Iterator<String> it = candSet.iterator(); it.hasNext();) {
+ modelScore = 0.0;
+ candStr = it.next().toString();
+
+ feat_str = feat_hash[i].get(candStr).split("\\s+");
+
+ for (int f = 0; f < feat_str.length; f++) {
+ String[] feat_info = feat_str[f].split("[=]");
+ modelScore +=
+ Double.parseDouble(feat_info[1]) * finalLambda[Vocabulary.id(feat_info[0])];
+ }
+
+ if (maxModelScore < modelScore) {
+ maxModelScore = modelScore;
+ tmpStatsVal = stats_hash[i].get(candStr).split("\\s+"); // save the suff stats
+ }
+ }
+
+ for (int j = 0; j < suffStatsCount; j++)
+ corpusStatsVal[j] += Integer.parseInt(tmpStatsVal[j]); // accumulate corpus-leve suff stats
+ } // for( int i=0; i<sentNum; i++ )
+
+ return evalMetric.score(corpusStatsVal);
+ }
+
+ public Vector<String> process_Params() {
+ Vector<String> allSamples = new Vector<String>(); // to save all sampled pairs
+
+ // sampling
+ Vector<String> sampleVec = new Vector<String>(); // use String to make sparse representation
+ // easy
+ for (int i = 0; i < sentNum; i++) {
+ sampleVec = Sampler(i);
+ allSamples.addAll(sampleVec);
+ }
+
+ return allSamples;
+ }
+
+ private Vector<String> Sampler(int sentId) {
+ int candCount = stats_hash[sentId].size();
+ Vector<String> sampleVec = new Vector<String>();
+ HashMap<String, Double> candScore = new HashMap<String, Double>(); // metric(e.g BLEU) score of
+ // all candidates
+
+ // extract all candidates to a string array to save time in computing BLEU score
+ String[] cands = new String[candCount];
+ Set<String> candSet = stats_hash[sentId].keySet();
+ HashMap<Integer, String> candMap = new HashMap<Integer, String>();
+
+ int candId = 0;
+ for (Iterator<String> it = candSet.iterator(); it.hasNext();) {
+ cands[candId] = it.next().toString();
+ candMap.put(candId, cands[candId]); // map an integer to each candidate
+ candId++;
+ }
+ candScore = compute_Score(sentId, cands); // compute BLEU for each candidate
+
+ // start sampling
+ double scoreDiff;
+ double probAccept;
+ boolean accept;
+ HashMap<String, Double> acceptedPair = new HashMap<String, Double>();
+
+ if (Tau < candCount * (candCount - 1)) // otherwise no need to sample
+ {
+ int j1, j2;
+ for (int i = 0; i < Tau; i++) {
+ // here the case in which the same pair is sampled more than once is allowed
+ // otherwise if Tau is almost the same as candCount^2, it might take a lot of time to find
+ // Tau distinct pairs
+ j1 = randgen.nextInt(candCount);
+ j2 = randgen.nextInt(candCount);
+ while (j1 == j2)
+ j2 = randgen.nextInt(candCount);
+
+ // accept or not?
+ scoreDiff = Math.abs(candScore.get(candMap.get(j1)) - candScore.get(candMap.get(j2)));
+ probAccept = Alpha(scoreDiff);
+
+// System.err.println("Diff: " + scoreDiff + " = " + candScore.get(candMap.get(j1)) + " - "
+// + candScore.get(candMap.get(j2)));
+
+ accept = randgen.nextDouble() <= probAccept ? true : false;
+
+ if (accept) acceptedPair.put(j1 + " " + j2, scoreDiff);
+ }
+ } else {
+ for (int i = 0; i < candCount; i++) {
+ for (int j = 0; j < candCount; j++) {
+ if (j != i) {
+ // accept or not?
+ scoreDiff = Math.abs(candScore.get(candMap.get(i)) - candScore.get(candMap.get(j)));
+ probAccept = Alpha(scoreDiff);
+
+ accept = randgen.nextDouble() <= probAccept ? true : false;
+
+ if (accept) acceptedPair.put(i + " " + j, scoreDiff);
+ }
+ }
+ }
+ }
+
+ //System.out.println("Tau="+Tau+"\nAll possible pair number: "+candCount*(candCount-1));
+ //System.out.println("Number of accepted pairs after random selection: "+acceptedPair.size());
+
+ // sort sampled pairs according to "scoreDiff"
+ ValueComparator comp = new ValueComparator(acceptedPair);
+ TreeMap<String, Double> acceptedPairSort = new TreeMap<String, Double>(comp);
+ acceptedPairSort.putAll(acceptedPair);
+
+ int topCount = 0;
+ int label;
+ String[] pair_str;
+ String[] feat_str_j1, feat_str_j2;
+ String j1Cand, j2Cand;
+ String featDiff, neg_featDiff;
+ HashSet<String> added = new HashSet<String>(); // to avoid symmetric duplicate
+
+ for (String key : acceptedPairSort.keySet()) {
+ if (topCount == Xi) break;
+
+ pair_str = key.split("\\s+");
+ // System.out.println(pair_str[0]+" "+pair_str[1]+" "+acceptedPair.get(key));
+
+ if (!added.contains(key)) {
+ j1Cand = candMap.get(Integer.parseInt(pair_str[0]));
+ j2Cand = candMap.get(Integer.parseInt(pair_str[1]));
+
+ if (evalMetric.getToBeMinimized()) // if smaller metric score is better(like TER)
+ label = (candScore.get(j1Cand) - candScore.get(j2Cand)) < 0 ? 1 : -1;
+ else
+ // like BLEU
+ label = (candScore.get(j1Cand) - candScore.get(j2Cand)) > 0 ? 1 : -1;
+
+ feat_str_j1 = feat_hash[sentId].get(j1Cand).split("\\s+");
+ feat_str_j2 = feat_hash[sentId].get(j2Cand).split("\\s+");
+
+ featDiff = "";
+ neg_featDiff = "";
+
+ HashMap<Integer, String> feat_diff = new HashMap<Integer, String>();
+ String[] feat_info;
+ int feat_id;
+
+ for (int i = 0; i < feat_str_j1.length; i++) {
+ feat_info = feat_str_j1[i].split("[=]");
+ feat_id = Vocabulary.id(feat_info[0]);
+ if ( (feat_id < isOptimizable.length &&
+ isOptimizable[feat_id]) ||
+ feat_id >= isOptimizable.length )
+ feat_diff.put( feat_id, feat_info[1] );
+ }
+ for (int i = 0; i < feat_str_j2.length; i++) {
+ feat_info = feat_str_j2[i].split("[=]");
+ feat_id = Vocabulary.id(feat_info[0]);
+ if ( (feat_id < isOptimizable.length &&
+ isOptimizable[feat_id]) ||
+ feat_id >= isOptimizable.length ) {
+ if (feat_diff.containsKey(feat_id))
+ feat_diff.put( feat_id,
+ Double.toString(Double.parseDouble(feat_diff.get(feat_id))-Double.parseDouble(feat_info[1])) );
+ else //only fired in the cand 2
+ feat_diff.put( feat_id, Double.toString(-1.0*Double.parseDouble(feat_info[1])));
+ }
+ }
+
+ for (Integer id: feat_diff.keySet()) {
+ featDiff += id + ":" + feat_diff.get(id) + " ";
+ neg_featDiff += id + ":" + -1.0*Double.parseDouble(feat_diff.get(id)) + " ";
+ }
+
+ featDiff += label;
+ neg_featDiff += -label;
+
+ // System.out.println(sentId+": "+key);
+ // System.out.println(featDiff + " | " + candScore.get(j1Cand) + " " +
+ // candScore.get(j2Cand));
+ // System.out.println(neg_featDiff);
+ // System.out.println("-------");
+
+ sampleVec.add(featDiff);
+ sampleVec.add(neg_featDiff);
+
+ // both (j1,j2) and (j2,j1) have been added to training set
+ added.add(key);
+ added.add(pair_str[1] + " " + pair_str[0]);
+
+ topCount++;
+ }
+ }
+
+ // System.out.println("Selected top "+topCount+ "pairs for training");
+
+ return sampleVec;
+ }
+
+ private double Alpha(double x) {
+ return x < metricDiff ? 0 : 1; // default implementation of the paper's method
+ // other functions possible
+ }
+
+ // compute *sentence-level* metric score
+ private HashMap<String, Double> compute_Score(int sentId, String[] cands) {
+ HashMap<String, Double> candScore = new HashMap<String, Double>();
+ String statString;
+ String[] statVal_str;
+ int[] statVal = new int[evalMetric.get_suffStatsCount()];
+
+ // for all candidates
+ for (int i = 0; i < cands.length; i++) {
+ statString = stats_hash[sentId].get(cands[i]);
+ statVal_str = statString.split("\\s+");
+
+ for (int j = 0; j < evalMetric.get_suffStatsCount(); j++)
+ statVal[j] = Integer.parseInt(statVal_str[j]);
+
+// System.err.println("Score: " + evalMetric.score(statVal));
+
+ candScore.put(cands[i], evalMetric.score(statVal));
+ }
+
+ return candScore;
+ }
+
+ // from ZMERT
+ private void normalizeLambda(double[] origLambda) {
+ // private String[] normalizationOptions;
+ // How should a lambda[] vector be normalized (before decoding)?
+ // nO[0] = 0: no normalization
+ // nO[0] = 1: scale so that parameter nO[2] has absolute value nO[1]
+ // nO[0] = 2: scale so that the maximum absolute value is nO[1]
+ // nO[0] = 3: scale so that the minimum absolute value is nO[1]
+ // nO[0] = 4: scale so that the L-nO[1] norm equals nO[2]
+
+ int normalizationMethod = (int) normalizationOptions[0];
+ double scalingFactor = 1.0;
+ if (normalizationMethod == 0) {
+ scalingFactor = 1.0;
+ } else if (normalizationMethod == 1) {
+ int c = (int) normalizationOptions[2];
+ scalingFactor = normalizationOptions[1] / Math.abs(origLambda[c]);
+ } else if (normalizationMethod == 2) {
+ double maxAbsVal = -1;
+ int maxAbsVal_c = 0;
+ for (int c = 1; c <= paramDim; ++c) {
+ if (Math.abs(origLambda[c]) > maxAbsVal) {
+ maxAbsVal = Math.abs(origLambda[c]);
+ maxAbsVal_c = c;
+ }
+ }
+ scalingFactor = normalizationOptions[1] / Math.abs(origLambda[maxAbsVal_c]);
+
+ } else if (normalizationMethod == 3) {
+ double minAbsVal = PosInf;
+ int minAbsVal_c = 0;
+
+ for (int c = 1; c <= paramDim; ++c) {
+ if (Math.abs(origLambda[c]) < minAbsVal) {
+ minAbsVal = Math.abs(origLambda[c]);
+ minAbsVal_c = c;
+ }
+ }
+ scalingFactor = normalizationOptions[1] / Math.abs(origLambda[minAbsVal_c]);
+
+ } else if (normalizationMethod == 4) {
+ double pow = normalizationOptions[1];
+ double norm = L_norm(origLambda, pow);
+ scalingFactor = normalizationOptions[2] / norm;
+ }
+
+ for (int c = 1; c <= paramDim; ++c) {
+ origLambda[c] *= scalingFactor;
+ }
+ }
+
+ // from ZMERT
+ private double L_norm(double[] A, double pow) {
+ // calculates the L-pow norm of A[]
+ // NOTE: this calculation ignores A[0]
+ double sum = 0.0;
+ for (int i = 1; i < A.length; ++i)
+ sum += Math.pow(Math.abs(A[i]), pow);
+
+ return Math.pow(sum, 1 / pow);
+ }
+
+ public double getMetricScore() {
+ return finalMetricScore;
+ }
+
+ private EvaluationMetric evalMetric;
+ private Vector<String> output;
+ private boolean[] isOptimizable;
+ private double[] initialLambda;
+ private double[] finalLambda;
+ private double[] normalizationOptions;
+ private double finalMetricScore;
+ private HashMap<String, String>[] feat_hash;
+ private HashMap<String, String>[] stats_hash;
+ private Random randgen;
+ private int paramDim;
+ private int sentNum;
+ private int Tau; // size of sampled candidate set(say 5000)
+ private int Xi; // choose top Xi candidates from sampled set(say 50)
+ private double metricDiff; // metric difference threshold(to select the qualified candidates)
+ private String classifierAlg; // optimization algorithm
+ private String[] classifierParam;
+
+ private final static double NegInf = (-1.0 / 0.0);
+ private final static double PosInf = (+1.0 / 0.0);
+}
+
+
+class ValueComparator implements Comparator<Object> {
+ Map<String,Double> base;
+
+ public ValueComparator(Map<String,Double> base) {
+ this.base = base;
+ }
+
+ @Override
+ public int compare(Object a, Object b) {
+ if ((Double) base.get(a) <= (Double) base.get(b))
+ return 1;
+ else if ((Double) base.get(a) == (Double) base.get(b))
+ return 0;
+ else
+ return -1;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/pro/PRO.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/pro/PRO.java b/joshua-core/src/main/java/org/apache/joshua/pro/PRO.java
new file mode 100755
index 0000000..fd9a7cb
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/pro/PRO.java
@@ -0,0 +1,159 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.pro;
+
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.util.FileUtility;
+import org.apache.joshua.util.StreamGobbler;
+
+public class PRO {
+ public static void main(String[] args) throws Exception {
+ JoshuaConfiguration joshuaConfiguration = new JoshuaConfiguration();
+ boolean external = false; // should each PRO iteration be launched externally?
+
+ if (args.length == 1) {
+ if (args[0].equals("-h")) {
+ printPROUsage(args.length, true);
+ System.exit(2);
+ } else {
+ external = false;
+ }
+ } else if (args.length == 3) {
+ external = true;
+ } else {
+ printPROUsage(args.length, false);
+ System.exit(1);
+ }
+
+ if (!external) {
+ PROCore myPRO = new PROCore(args[0],joshuaConfiguration);
+ myPRO.run_PRO(); // optimize lambda[]!!!
+ myPRO.finish();
+ } else {
+
+ int maxMem = Integer.parseInt(args[1]);
+ String configFileName = args[2];
+ String stateFileName = FileUtility.dirname(configFileName) + "/PRO.temp.state";
+ String cp = System.getProperty("java.class.path");
+ boolean done = false;
+ int iteration = 0;
+
+ while (!done) {
+ ++iteration;
+ Runtime rt = Runtime.getRuntime();
+ Process p =
+ rt.exec("java -Xmx" + maxMem + "m -cp " + cp + " org.apache.joshua.pro.PROCore " + configFileName
+ + " " + stateFileName + " " + iteration);
+ /*
+ * BufferedReader br_i = new BufferedReader(new InputStreamReader(p.getInputStream()));
+ * BufferedReader br_e = new BufferedReader(new InputStreamReader(p.getErrorStream()));
+ * String dummy_line = null; while ((dummy_line = br_i.readLine()) != null) {
+ * System.out.println(dummy_line); } while ((dummy_line = br_e.readLine()) != null) {
+ * System.out.println(dummy_line); }
+ */
+ StreamGobbler errorGobbler = new StreamGobbler(p.getErrorStream(), 1);
+ StreamGobbler outputGobbler = new StreamGobbler(p.getInputStream(), 1);
+
+ errorGobbler.start();
+ outputGobbler.start();
+
+ int status = p.waitFor();
+
+ if (status == 90) {
+ done = true;
+ } else if (status == 91) {
+ done = false;
+ } else {
+ System.out.println("PRO exiting prematurely (PROCore returned " + status + ")...");
+ break;
+ }
+ }
+ }
+
+ System.exit(0);
+
+ } // main(String[] args)
+
+ public static void printPROUsage(int argsLen, boolean detailed) {
+ if (!detailed) {
+ println("Oops, you provided " + argsLen + " args!");
+ println("");
+ println("Usage:");
+ println(" PRO -maxMem maxMemoryInMB PRO_configFile");
+ println("");
+ println("Where -maxMem specifies the maximum amount of memory (in MB) PRO is");
+ println("allowed to use when performing its calculations (no memroy is needed while");
+ println("the decoder is running),");
+ println("and the config file contains any subset of PRO's 20-some parameters,");
+ println("one per line. Run PRO -h for more details on those parameters.");
+ } else {
+ println("Usage:");
+ println(" PRO -maxMem maxMemoryInMB PRO_configFile");
+ println("");
+ println("Where -maxMem specifies the maximum amount of memory (in MB) PRO is");
+ println("allowed to use when performing its calculations (no memroy is needed while");
+ println("the decoder is running),");
+ println("and the config file contains any subset of PRO's 20-some parameters,");
+ println("one per line. Those parameters, and their default values, are:");
+ println("");
+ println("Relevant files:");
+ println(" -dir dirPrefix: working directory\n [[default: null string (i.e. they are in the current directory)]]");
+ println(" -s sourceFile: source sentences (foreign sentences) of the PRO dataset\n [[default: null string (i.e. file name is not needed by PRO)]]");
+ println(" -r refFile: target sentences (reference translations) of the PRO dataset\n [[default: reference.txt]]");
+ println(" -rps refsPerSen: number of reference translations per sentence\n [[default: 1]]");
+ println(" -txtNrm textNormMethod: how should text be normalized?\n (0) don't normalize text,\n or (1) \"NIST-style\", and also rejoin 're, *'s, n't, etc,\n or (2) apply 1 and also rejoin dashes between letters,\n or (3) apply 1 and also drop non-ASCII characters,\n or (4) apply 1+2+3\n [[default: 1]]");
+ println(" -p paramsFile: file containing parameter names, initial values, and ranges\n [[default: params.txt]]");
+ println(" -docInfo documentInfoFile: file informing PRO which document each\n sentence belongs to\n [[default: null string (i.e. all sentences are in one 'document')]]");
+ println(" -fin finalLambda: file name for final lambda[] values\n [[default: null string (i.e. no such file will be created)]]");
+ println("");
+ println("PRO specs:");
+ println(" -m metricName metric options: name of evaluation metric and its options\n [[default: BLEU 4 closest]]");
+ println(" -maxIt maxPROIts: maximum number of PRO iterations\n [[default: 20]]");
+ println(" -prevIt prevPROIts: maximum number of previous PRO iterations to\n construct candidate sets from\n [[default: 20]]");
+ println(" -minIt minPROIts: number of iterations before considering an early exit\n [[default: 5]]");
+ println(" -stopIt stopMinIts: some early stopping criterion must be satisfied in\n stopMinIts *consecutive* iterations before an early exit\n [[default: 3]]");
+ println(" -stopSig sigValue: early PRO exit if no weight changes by more than sigValue\n [[default: -1 (i.e. this criterion is never investigated)]]");
+ println(" -thrCnt threadCount: number of threads to run in parallel when optimizing\n [[default: 1]]");
+ println(" -save saveInter: save intermediate cfg files (1) or decoder outputs (2)\n or both (3) or neither (0)\n [[default: 3]]");
+ println(" -compress compressFiles: should PRO compress the files it produces (1)\n or not (0)\n [[default: 0]]");
+ println(" -opi oncePerIt: modify a parameter only once per iteration (1) or not (0)\n [[default: 0]]");
+ println(" -rand randInit: choose initial point randomly (1) or from paramsFile (0)\n [[default: 0]]");
+ println(" -seed seed: seed used to initialize random number generator\n [[default: time (i.e. value returned by System.currentTimeMillis()]]");
+ // println(" -ud useDisk: reliance on disk (0-2; higher value => more reliance)\n [[default: 2]]");
+ println("");
+ println("Decoder specs:");
+ println(" -cmd commandFile: name of file containing commands to run the decoder\n [[default: null string (i.e. decoder is a JoshuaDecoder object)]]");
+ println(" -passIt passIterationToDecoder: should iteration number be passed\n to command file (1) or not (0)\n [[default: 0]]");
+ println(" -decOut decoderOutFile: name of the output file produced by the decoder\n [[default: output.nbest]]");
+ println(" -decExit validExit: value returned by decoder to indicate success\n [[default: 0]]");
+ println(" -dcfg decConfigFile: name of decoder config file\n [[default: dec_cfg.txt]]");
+ println(" -N N: size of N-best list (per sentence) generated in each PRO iteration\n [[default: 100]]");
+ println("");
+ println("Output specs:");
+ println(" -v verbosity: PRO verbosity level (0-2; higher value => more verbose)\n [[default: 1]]");
+ println(" -decV decVerbosity: should decoder output be printed (1) or ignored (0)\n [[default: 0]]");
+ println("");
+ }
+ }
+
+ private static void println(Object obj) {
+ System.out.println(obj);
+ }
+
+}
[06/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/packed/grammar.packed/encoding
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/packed/grammar.packed/encoding b/joshua-core/src/test/resources/bn-en/packed/grammar.packed/encoding
new file mode 100644
index 0000000..97ab6a6
Binary files /dev/null and b/joshua-core/src/test/resources/bn-en/packed/grammar.packed/encoding differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/packed/grammar.packed/slice_00000.features
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/packed/grammar.packed/slice_00000.features b/joshua-core/src/test/resources/bn-en/packed/grammar.packed/slice_00000.features
new file mode 100644
index 0000000..4bd5e88
Binary files /dev/null and b/joshua-core/src/test/resources/bn-en/packed/grammar.packed/slice_00000.features differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/packed/grammar.packed/slice_00000.source
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/packed/grammar.packed/slice_00000.source b/joshua-core/src/test/resources/bn-en/packed/grammar.packed/slice_00000.source
new file mode 100644
index 0000000..a7312fa
Binary files /dev/null and b/joshua-core/src/test/resources/bn-en/packed/grammar.packed/slice_00000.source differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/packed/grammar.packed/slice_00000.target
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/packed/grammar.packed/slice_00000.target b/joshua-core/src/test/resources/bn-en/packed/grammar.packed/slice_00000.target
new file mode 100644
index 0000000..f9ee8b3
Binary files /dev/null and b/joshua-core/src/test/resources/bn-en/packed/grammar.packed/slice_00000.target differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/packed/grammar.packed/slice_00000.target.lookup
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/packed/grammar.packed/slice_00000.target.lookup b/joshua-core/src/test/resources/bn-en/packed/grammar.packed/slice_00000.target.lookup
new file mode 100644
index 0000000..4d018d6
Binary files /dev/null and b/joshua-core/src/test/resources/bn-en/packed/grammar.packed/slice_00000.target.lookup differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/packed/grammar.packed/vocabulary
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/packed/grammar.packed/vocabulary b/joshua-core/src/test/resources/bn-en/packed/grammar.packed/vocabulary
new file mode 100644
index 0000000..aa94d05
Binary files /dev/null and b/joshua-core/src/test/resources/bn-en/packed/grammar.packed/vocabulary differ
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/packed/input.bn
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/packed/input.bn b/joshua-core/src/test/resources/bn-en/packed/input.bn
new file mode 100644
index 0000000..d4f7913
--- /dev/null
+++ b/joshua-core/src/test/resources/bn-en/packed/input.bn
@@ -0,0 +1,100 @@
+\u09b0\u09ac\u09c0\u09a8\u09cd\u09a6\u09cd\u09b0\u09a8\u09be\u09a5\u09c7\u09b0 \u099c\u09a8\u09cd\u09ae \u09b9\u09af\u09bc\u09c7\u099b\u09bf\u09b2 \u0995\u09b2\u0995\u09be\u09a4\u09be\u09b0 \u098f\u0995 \u09aa\u09bf\u09b0\u09be\u09b2\u09c0 \u09ac\u09cd\u09b0\u09be\u09b9\u09cd\u09ae\u09a3 \u09aa\u09b0\u09bf\u09ac\u09be\u09b0\u09c7 \u0964
+\u09b8\u09be\u09ae\u09cd\u09aa\u09cd\u09b0\u09a4\u09bf\u0995\u0995\u09be\u09b2\u09c7 \u09ad\u09be\u09b0\u09a4\u09c7\u09b0 \u09b8\u0999\u09cd\u0997\u09c7 \u09af\u09c1\u0995\u09cd\u09a4\u09b0\u09be\u09b7\u09cd\u099f\u09cd\u09b0\u09c7\u09b0 \u09b8\u09ae\u09cd\u09aa\u09b0\u09cd\u0995\u09c7\u09b0 \u0989\u09a8\u09cd\u09a8\u09a4\u09bf \u09b9\u09af\u09bc\u09c7\u099b\u09c7 \u0964
+\u0997\u09a3\u09bf\u09a4 \u09a4\u09be\u0987 \u09ac\u09bf\u099c\u09cd\u099e\u09be\u09a8\u09c7\u09b0 \u09ad\u09be\u09b7\u09be \u0964
+\u098f \u09a5\u09c7\u0995\u09c7 \u09b8\u09b9\u099c\u09c7\u0987 \u09ac\u09cb\u099d\u09be \u09af\u09be\u09af\u09bc \u09af\u09c7 \u098f\u0987 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995\u099f\u09bf \u09b9\u09ac\u09c7 \u098f\u09ab\u0986\u09b0\u09a1\u09ac\u09cd\u09b2\u09bf\u0989 \u09ae\u09c7\u099f\u09cd\u09b0\u09bf\u0995 \u0964
+\u098f\u0995\u0987 \u09b8\u0999\u09cd\u0997\u09c7 \u09ac\u09be\u0982\u09b2\u09be\u09b0 \u09ad\u09c2\u09ae\u09bf\u0995\u09c7\u09a8\u09cd\u09a6\u09cd\u09b0\u09bf\u0995 \u09b8\u09be\u09ae\u09a8\u09cd\u09a4\u09a4\u09a8\u09cd\u09a4\u09cd\u09b0\u09c7\u09b0 \u09aa\u09a4\u09a8\u09c7\u09b0 \u09aa\u09c2\u09b0\u09cd\u09ac\u09be\u09ad\u09be\u09b7\u0993 \u09ae\u09c7\u09b2\u09c7 \u098f\u0987 \u0989\u09aa\u09a8\u09cd\u09af\u09be\u09b8 \u09a5\u09c7\u0995\u09c7 \u0964
+\u09a8\u09bf\u09b0\u09cd\u09ac\u099a\u09a8\u09c7 \u09ae\u09c1\u099c\u09bf\u09ac \u0993 \u09a4\u09be\u0981\u09b0 \u09a6\u09b2 \u09a8\u09bf\u09b0\u0999\u09cd\u0995\u09c1\u09b7 \u09b8\u0982\u0996\u09cd\u09af\u09be\u0997\u09b0\u09bf\u09b7\u09cd\u09a0\u09a4\u09be \u0985\u09b0\u09cd\u099c\u09a8 \u0995\u09b0\u09c7 \u0964
+\u09b9\u09bf\u099f\u09b2\u09be\u09b0\u0993 \u09ac\u09be\u09ad\u09be\u09b0\u09bf\u09af\u09bc\u09be\u09b0 \u09ae\u09bf\u0989\u09a8\u09bf\u0996 \u09b6\u09b9\u09b0\u09c7\u0987 \u09a4\u09be\u09b0 \u0995\u09be\u099c \u099a\u09be\u09b2\u09bf\u09af\u09bc\u09c7 \u09af\u09c7\u09a4\u09c7 \u09a5\u09be\u0995\u09c7\u09a8 \u0964
+\u099f\u09be\u0995\u09cd\u09b8 \u099b\u09be\u09a1\u09bc\u09be\u0993 \u0993\u098f\u09b8-\u099f\u09cd\u09af\u09be\u09a8 \u0993 \u0986\u09b0\u0993 \u0995\u09bf\u099b\u09c1 \u09b2\u09bf\u09a8\u09be\u0995\u09cd\u09b8 \u09aa\u09cd\u09b0\u09a4\u09bf\u09a8\u09bf\u09a7\u09bf\u09a4\u09cd\u09ac\u0995\u09be\u09b0\u09c0 \u099a\u09b0\u09bf\u09a4\u09cd\u09b0 \u09b0\u09af\u09bc\u09c7\u099b\u09c7 & # 44 ; \u09a4\u09ac\u09c7 \u098f\u0997\u09c1\u09b2\u09cb \u0996\u09c1\u09ac \u09aa\u09cd\u09b0\u099a\u09b2\u09bf\u09a4 \u09a8\u09af\u09bc \u0964
+\u0987\u09b9\u09be \u09b8\u09be\u09ae\u09be\u099c\u09bf\u0995 \u0985\u09ac\u09b8\u09cd\u09a5\u09be\u09b0 \u09aa\u09cd\u09b0\u09c7\u0995\u09cd\u09b7\u09bf\u09a4\u09c7 \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4\u0997\u09cd\u09b0\u09b9\u09a8\u0995\u09be\u09b0\u09c0 \u09a4\u09be\u09b0 \u09aa\u09cd\u09b0\u09a4\u09bf\u09aa\u0995\u09cd\u09b7\u09c7\u09b0 \u09b8\u09b9\u09bf\u09a4 \u0995\u09cd\u09b0\u09c0\u09a1\u09bc\u09be\u0995\u09cd\u09b7\u09c7\u09a4\u09cd\u09b0\u09c7 \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4 \u0997\u09cd\u09b0\u09b9\u09a8\u09c7 \u09aa\u09cd\u09b0\u099a\u09b2\u09bf\u09a4 \u09a8\u09bf\u09af\u09bc\u09ae \u09ac\u09cd\u09af\u09be\u0996\u09cd\u09af\u09be \u0995\u09b0\u09c7 \u09a5\u09be\u0995\u09c7 \u0964
+\u09ac\u09c3\u09b7\u09cd\u099f\u09bf\u09aa\u09be\u09a4\u0983 \u09ac\u09be\u09ce\u09b8\u09b0\u09bf\u0995 \u09e8\u09eb\u09ea\u09e6 \u09ae\u09bf\u09b2\u09bf \u09ae\u09bf\u099f\u09be\u09b0
+\u09e8\u09e6\u09e6\u09ea \u09b8\u09be\u09b2\u09c7 \u09ae\u09cd\u09af\u09be\u09b8\u09be\u099a\u09c1\u09b8\u09c7\u099f\u09b8 \u0985\u0999\u09cd\u0997\u09b0\u09be\u099c\u09cd\u09af\u09c7\u09b0 \u09ac\u09b8\u09cd\u099f\u09a8 \u09b6\u09b9\u09b0\u09c7 \u0985\u09a8\u09c1\u09b7\u09cd\u09a0\u09bf\u09a4 \u09a1\u09c7\u09ae\u09cb\u0995\u09cd\u09b0\u09cd\u09af\u09be\u099f \u09a6\u09b2\u09c7\u09b0 \u099c\u09be\u09a4\u09c0\u09af\u09bc \u09b8\u09ae\u09cd\u09ae\u09c7\u09b2\u09a8 \u09a4\u09bf\u09a8\u09bf \u09ae\u09c2\u09b2 \u09ac\u0995\u09cd\u09a4\u09c3\u09a4\u09be -lrb- keynote speech -rrb- \u09aa\u09cd\u09b0\u09a6\u09be\u09a8 \u0995\u09b0\u09c7\u09a8 \u0964
+\u099c\u09a8\u09b8\u0982\u0996\u09cd\u09af\u09be\u09b0 \u09ad\u09bf\u09a4\u09cd\u09a4\u09bf\u09a4\u09c7 \u0995\u09cd\u09b7\u09ae\u09a4\u09be\u09b0 \u09ac\u09a8\u09cd\u099f\u09a8 \u09aa\u09c2\u09b0\u09cd\u09ac \u09aa\u09be\u0995\u09bf\u09b8\u09cd\u09a4\u09be\u09a8\u09c7\u09b0 \u0985\u09a8\u09c1\u0995\u09c2\u09b2 \u09b9\u0993\u09af\u09bc\u09be\u09af\u09bc \u09aa\u09b6\u09cd\u099a\u09bf\u09ae \u09aa\u09be\u0995\u09bf\u09b8\u09cd\u09a4\u09be\u09a8 & quot ; \u098f\u0995 \u0987\u0989\u09a8\u09bf\u099f \u09a4\u09a4\u09cd\u09a4\u09cd\u09ac & quot ; \u09a8\u09be\u09ae\u09c7 \u098f\u0995 \u0985\u09ad\u09bf\u09a8\u09ac \u09a7\u09be\u09b0\u09a3\u09be\u09b0 \u09b8\u09c2\u09a4\u09cd\u09b0\u09aa\u09be\u09a4 \u0995\u09b0\u09c7 & # 44 ; \u09af\u09c7\u0996\u09be\u09a8\u09c7 \u09b8\u09ae\u0997\u09cd\u09b0 \u09aa\u09b6\u09cd\u099a\u09bf\u09ae \u09aa\u09be\u0995\u09bf\u09b8\u09cd\u09a4\u09be\u09a8 \u098f\u0995\u099f\u09bf \u09aa\u09cd\u09b0\u09a6\u09c7\u09b6 \u09b9\u09bf\u09b8\u09c7\u09ac\u09c7 \u09ac\u09bf\u09ac\u09c7\u099a\u09bf\u09a4 \u09b9\u09af\u09bc \u0964
+\u2022 \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09a8 \u09a4\u09a4\u09cd\u09a4\u09cd\u09ac
+\u09ac\u09b9\u09bf\u0983\u09b8\u0982\u09af\u09cb\u0997 \u09b8\u09ae\u09c2\u09b9
+\u099f\u09be\u099f\u09be \u0995\u09ae\u09bf\u0989\u09a8\u09bf\u0995\u09c7\u09b6\u09a8\u09b8\u09c7\u09b0 \u09ac\u09bf\u09a6\u09c7\u09b6 \u09b8\u099e\u09cd\u099a\u09be\u09b0 \u09a8\u09bf\u0997\u09ae \u09b2\u09bf\u09ae\u09bf\u099f\u09c7\u09a1 \u09ad\u09ac\u09a8 & # 44 ; \u098f\u099f\u09bf \u09b6\u09b9\u09b0\u09c7\u09b0 \u099f\u09c7\u09b2\u09bf\u09af\u09cb\u0997\u09be\u09af\u09cb\u0997 \u09ac\u09cd\u09af\u09ac\u09b8\u09cd\u09a5\u09be\u09b0 \u098f\u0995\u099f\u09bf \u0985\u09a8\u09cd\u09af\u09a4\u09ae \u09aa\u09cd\u09b0\u09a7\u09be\u09a8 \u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0
+\u09a4\u09bf\u09a8\u09bf \u09b8\u09c7\u0987 \u09ac\u099b\u09b0\u09c7\u09b0 \u09ea\u0987 \u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0\u09c7\u09b0 \u099c\u09be\u09a4\u09c0\u09af\u09bc \u09a8\u09bf\u09b0\u09cd\u09ac\u09be\u099a\u09a8\u09c7 \u099c\u09af\u09bc\u09c0 \u09b9\u09a8 \u098f\u09ac\u0982 \u09ae\u09be\u09b0\u09cd\u0995\u09bf\u09a8 \u09af\u09c1\u0995\u09cd\u09a4\u09b0\u09be\u09b7\u09cd\u099f\u09cd\u09b0\u09c7\u09b0 \u09ea\u09ea\u09a4\u09ae \u09b0\u09be\u09b7\u09cd\u099f\u09cd\u09b0\u09aa\u09a4\u09bf \u09b9\u09bf\u09b8\u09c7\u09ac\u09c7 \u09a8\u09bf\u09b0\u09cd\u09ac\u09be\u099a\u09bf\u09a4 \u09b9\u09a8 \u0964
+\u09ac\u09b9\u09c1 \u09ad\u09be\u09b0\u09a4\u09c0\u09af\u09bc \u09aa\u09cd\u09b0\u099c\u09be\u09a4\u09bf \u0997\u09a8\u09cd\u09a1\u09cb\u09af\u09bc\u09be\u09a8\u09be\u09af\u09bc \u099c\u09be\u09a4 \u099f\u09c7\u0995\u09cd\u09b8\u09be \u09a5\u09c7\u0995\u09c7 \u0989\u09a6\u09cd\u09ad\u09c1\u09a4 \u0964
+\u09ac\u09cd\u09b0\u09bf\u099f\u09c7\u09a8\u09c7\u09b0 \u09b2\u09c7\u0996\u0995\u09a6\u09c7\u09b0 \u09b0\u099a\u09bf\u09a4 \u09a8\u09be\u099f\u0995 & # 44 ; \u0989\u09aa\u09a8\u09cd\u09af\u09be\u09b8 & # 44 ; \u0997\u09b2\u09cd\u09aa \u098f\u09ac\u0982 \u09b8\u09ae\u09cd\u09aa\u09cd\u09b0\u09a4\u09bf \u099a\u09bf\u09a4\u09cd\u09b0\u09a8\u09be\u099f\u09cd\u09af \u09ac\u09bf\u09b6\u09cd\u09ac\u09ac\u09cd\u09af\u09be\u09aa\u09c0 \u0986\u09a6\u09c3\u09a4 \u0964
+\u09e7\u09ef\u09e7\u09ef \u09b8\u09be\u09b2\u09c7\u09b0 \u09ae\u09c7 \u09ae\u09be\u09b8\u09c7 \u098f\u099f\u09bf \u09b8\u0993\u0997\u09be\u09a4 \u09aa\u09a4\u09cd\u09b0\u09bf\u0995\u09be\u09af\u09bc \u09aa\u09cd\u09b0\u0995\u09be\u09b6\u09bf\u09a4 \u09b9\u09af\u09bc \u0964
+\u09e8\u09e6\u09e6\u09eb \u09b8\u09be\u09b2\u09c7 \u0989\u0987\u09ae\u09c7\u09a8\u09b8 \u099f\u09c7\u09a8\u09bf\u09b8 \u0985\u09cd\u09af\u09be\u09b8\u09cb\u09b8\u09bf\u09af\u09bc\u09c7\u09b6\u09a8 \u099f\u09cd\u09af\u09c1\u09b0\u09c7\u09b0 \u099f\u09be\u09af\u09bc\u09be\u09b0-\u09a5\u09cd\u09b0\u09bf \u099f\u09c1\u09b0\u09cd\u09a8\u09be\u09ae\u09c7\u09a8\u09cd\u099f \u09b8\u09be\u09a8\u09ab\u09bf\u09b8\u09cd\u099f \u0993\u09aa\u09c7\u09a8 \u09a8\u09c7\u09a4\u09be\u099c\u09bf \u0987\u09a8\u09cd\u09a1\u09cb\u09b0 \u09b8\u09cd\u099f\u09c7\u09a1\u09bf\u09af\u09bc\u09be\u09ae\u09c7 \u0986\u09af\u09bc\u09cb\u099c\u09bf\u09a4 \u09b9\u09af\u09bc\u09c7\u099b\u09bf\u09b2 \u0964
+\u098f\u0987 \u09b8\u09ae\u09cd\u09ad\u09be\u09ac\u09a8\u09be \u09a6\u09c2\u09b0\u09c0\u0995\u09b0\u09a3\u09c7\u09b0 \u099c\u09a8\u09cd\u09af \u09a8\u09be\u09a8\u09be\u09ac\u09bf\u09a7 \u09ac\u09cd\u09af\u09ac\u09b8\u09cd\u09a5\u09be \u0997\u09cd\u09b0\u09b9\u09a3 \u0995\u09b0\u09be \u09b9\u09af\u09bc \u0964
+\u09e7\u09ef\u09ec\u09ec \u09b8\u09be\u09b2\u09c7\u09b0 \u09eb \u09ab\u09c7\u09ac\u09cd\u09b0\u09c1\u09af\u09bc\u09be\u09b0\u09bf \u09b2\u09be\u09b9\u09cb\u09b0\u09c7 \u09ac\u09bf\u09b0\u09cb\u09a7\u09c0 \u09a6\u09b2\u09b8\u09ae\u09c2\u09b9\u09c7\u09b0 \u098f\u0995\u099f\u09bf \u099c\u09be\u09a4\u09c0\u09af\u09bc \u09b8\u09ae\u09cd\u09ae\u09c7\u09b2\u09a8 \u0985\u09a8\u09c1\u09b7\u09cd\u09a0\u09bf\u09a4 \u09b9\u09af\u09bc \u0964
+\u09ac\u09be\u0982\u09b2\u09be\u09a6\u09c7\u09b6 \u0985\u09b0\u09cd\u0997\u09be\u09a8\u09be\u0987\u099c\u09c7\u09b6\u09a8 \u0985\u09ab \u09a6\u09bf \u0987\u09b8\u09b2\u09be\u09ae\u09bf\u0995 \u0995\u09a8\u09ab\u09be\u09b0\u09c7\u09a8\u09cd\u09b8 \u0993 \u0987\u09b8\u09b2\u09be\u09ae\u09bf\u0995 \u09a1\u09c7\u09ad\u09c7\u09b2\u09aa\u09ae\u09c7\u09a3\u09cd\u099f \u09ac\u09cd\u09af\u09be\u0982\u0995\u09c7\u09b0 \u09b8\u09a6\u09b8\u09cd\u09af\u09aa\u09a6 \u0997\u09cd\u09b0\u09b9\u09a3 \u0995\u09b0\u09c7 \u0964
+\u09ac\u09bf\u09b7\u09af\u09bc\u09b6\u09cd\u09b0\u09c7\u09a3\u09c0 : \u09ac\u09bf\u09b6\u09cd\u09ac\u0995\u09cb\u09b7
+\u09b0\u09be\u09b6\u09bf\u09af\u09bc\u09be & # 44 ; \u09ab\u09cd\u09b0\u09be\u09a8\u09cd\u09b8 \u0993 \u0987\u09b8\u09b0\u09be\u09af\u09bc\u09c7\u09b2 \u09ad\u09be\u09b0\u09a4\u09c7\u09b0 \u09aa\u09cd\u09b0\u09a7\u09be\u09a8 \u0985\u09b8\u09cd\u09a4\u09cd\u09b0\u09b8\u09b0\u09ac\u09b0\u09be\u09b9\u0995\u09be\u09b0\u09c0 \u09b0\u09be\u09b7\u09cd\u099f\u09cd\u09b0 \u0993 \u09aa\u09cd\u09b0\u09a4\u09bf\u09b0\u0995\u09cd\u09b7\u09be \u09b8\u09b9\u0995\u09be\u09b0\u09c0 \u09a6\u09c7\u09b6 \u0964
+\u098f\u0987 \u09b9\u09b2 \u0986\u09ae\u09be\u09a6\u09c7\u09b0 \u09aa\u09b0\u09bf\u099a\u09bf\u09a4 \u0995\u09be\u09b2\u09cd\u09aa\u09a8\u09bf\u0995 \u098f\u0995\u0995 \u09af\u09be\u09b0 \u09b8\u09be\u09b9\u09be\u09af\u09cd\u09af\u09c7 \u0997\u09a3\u09bf\u09a4\u09c7\u09b0 \u09a5\u09bf\u0993\u09b0\u09c0 \u09b8\u09ae\u09c1\u09b9 \u09ac\u09be\u09b8\u09cd\u09a4\u09ac \u09b8\u0982\u0996\u09cd\u09af\u09be\u09b0 \u09b8\u09c7\u099f \u09a5\u09c7\u0995\u09c7 \u099c\u099f\u09bf\u09b2 \u09b8\u0982\u0996\u09cd\u09af\u09be\u09b0 \u09b8\u09c7\u099f\u09c7 \u0989\u09a8\u09cd\u09a8\u09c0\u09a4 \u09b9\u09af\u09bc \u0964
+& lt ; \u09a0\u09bf\u0995\u09be\u09a8\u09be & gt ;
+\u09b8\u09c7\u09aa\u09cd\u099f\u09c7\u09ae\u09cd\u09ac\u09b0
+\u098f\u0987 \u09a4\u09a4\u09cd\u09a4\u09cd\u09ac \u09a5\u09c7\u0995\u09c7 \u0985\u09ac\u09b6\u09cd\u09af \u09ae\u09b9\u09be \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09cb\u09b0 \u09ac\u09bf\u09b0\u09cb\u09a7\u09bf\u09a4\u09be \u0995\u09b0\u09be \u09af\u09be\u09af\u09bc \u09a8\u09be & # 44 ; \u09ac\u09b0\u0982 \u09a4\u09be \u09b8\u09ae\u09b0\u09cd\u09a5\u09a8 \u0995\u09b0\u09be \u09af\u09c7\u09a4\u09c7 \u09aa\u09be\u09b0\u09c7 \u0964
+\u0995\u09c3\u09b7\u09bf \u0989\u09ce\u09aa\u09be\u09a6\u09a8\u09c7 \u09ab\u09cd\u09b0\u09be\u09a8\u09cd\u09b8 \u0987\u0989\u09b0\u09cb\u09aa\u09c7\u09b0 \u09b8\u09ac\u099a\u09c7\u09af\u09bc\u09c7 \u0997\u09c1\u09b0\u09c1\u09a4\u09cd\u09ac\u09aa\u09c2\u09b0\u09cd\u09a3 \u09a6\u09c7\u09b6 ; \u098f\u099f\u09bf \u09ae\u09c2\u09b2\u09a4 \u0996\u09be\u09a6\u09cd\u09af\u09b6\u09b8\u09cd\u09af & # 44 ; \u0993\u09af\u09bc\u09be\u0987\u09a8 & # 44 ; \u09aa\u09a8\u09bf\u09b0 \u0993 \u0985\u09a8\u09cd\u09af\u09be\u09a8\u09cd\u09af \u0995\u09c3\u09b7\u09bf\u09a6\u09cd\u09b0\u09ac\u09cd\u09af \u0987\u0989\u09b0\u09cb\u09aa \u0993 \u09b8\u09be\u09b0\u09be \u09ac\u09bf\u09b6\u09cd\u09ac\u09c7 \u09b0\u09aa\u09cd\u09a4\u09be\u09a8\u09bf \u0995\u09b0\u09c7 \u0964
+\u09a4\u09be\u09a6\u09c7\u09b0 \u0997\u09a3\u09bf\u09a4\u09c7 \u09aa\u09be\u099f\u09c0\u0997\u09a3\u09bf\u09a4\u09c7\u09b0 \u09aa\u09cd\u09b0\u09be\u09a7\u09be\u09a8\u09cd\u09af \u099b\u09bf\u09b2 \u0964
+\u09a6\u09c7\u09b6\u0997\u09c1\u09b2\u09cb \u09b9\u09b2\u09cb\u0983 \u09ab\u09cd\u09b0\u09be\u09a8\u09cd\u09b8 & # 44 ; \u09b9\u0982\u0995\u0982 & # 44 ; \u099a\u09c0\u09a8 & # 44 ; \u09ac\u09c7\u09b2\u099c\u09bf\u09af\u09bc\u09be\u09ae & # 44 ; \u09b8\u09c1\u0987\u099c\u09be\u09b0\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 & # 44 ; \u099c\u09be\u09b0\u09cd\u09ae\u09be\u09a8\u09c0 & # 44 ; \u09a1\u09c7\u09a8\u09ae\u09be\u09b0\u09cd\u0995 & # 44 ; \u09b8\u09c1\u0987\u09a1\u09c7\u09a8 & # 44 ; \u0985\u09b8\u09cd\u099f\u09cd\u09b0\u09bf\u09af\u09bc\u09be & # 44 ; \u099a\u09c7\u0995\u09cb\u09b6\u09cd\u09b2\u09cb\u09ad\u09be\u0995\u09bf\u09af\u09bc\u09be & # 44 ; \u0986\u09b0\u09cd\u099c\u09c7\u09a8\u09cd\u099f\u09bf\u09a8\u09be & # 44 ; \u0987\u09a4\u09be\u09b2\u09bf & # 44 ; \u09a8\u09b0\u0993\u09af\u09bc\u09c7 & # 44 ; \u09b9\u09be\u0999\u09cd\u0997\u09c7\u09b0\u09c0 & # 44 ; \u09af\u09c1\u0997\u09cb\u09b6\u09cd\u09b2\u09be\u09ad\u09bf\u09af\u09bc\u09be & # 44 ; \u09ac\u09c1\u09b2\u0997\u09c7\u09b0\u09bf\u09af\u09bc\u09be & # 44 ; \u09b0\u09c1\u09ae\u09be\u09a8\u09bf\u09af\u09bc\u09be & # 44 ; \u0997\u09cd\u09b0\u09c0\u09b8 & # 44 ; \u09ae\u09bf\u09b6\u09b0 & # 44 ; \u09b8\u09bf\u0999\u09cd\u0997\u09be\u09aa\u09c1\u09b0 & # 44 ; \u0987\u09a8\u09cd\u09a6\u09cb\u09a8\u09c7\u09b6\u09bf\u09af\u09bc\u09be & # 44 ; \u09a5\u09be\u0987\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 & # 44 ; \u099c\u09be\u09aa\u09be\u09a8 & # 44 ; \u09ac\u09be\u09b0\u09cd\u09ae\u09be & # 44 ; \u09b9\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 & # 44 ; \u09b8\u09cb\u09ad\u09bf\u09af\u09bc\u09c7\u099f \u09b0\u09be\u09b6\u09bf\u09af\u09bc\u09be & # 44 ; \u0987\u09b0\u09be\u09a8 & # 44 ; \u0987\u09b0\u09be\u0995 \u0993 \u09b6
\u09b0\u09c0\u09b2\u0982\u0995\u09be \u0964
+\u098f\u0987 \u09ac\u09cd\u09af\u09be\u09b8\u09bf\u09b2\u09bf\u0995\u09be\u09b0 \u09b8\u09cd\u09a5\u09be\u09a8\u09c7 \u098f\u0996\u09a8 \u09ac\u09cd\u09af\u09be\u0982\u0995 \u0985\u09ab \u0987\u0982\u09b2\u09cd\u09af\u09be\u09a8\u09cd\u09a1 \u0985\u09ac\u09b8\u09cd\u09a5\u09bf\u09a4 \u0964
+\u09a6\u09c7\u09b6\u099f\u09bf\u09b0 \u0989\u09a4\u09cd\u09a4\u09b0\u09c7 \u09ac\u09bf\u09b8\u09cd\u0995\u09be\u0987 \u0989\u09aa\u09b8\u09be\u0997\u09b0 & # 44 ; \u09a6\u0995\u09cd\u09b7\u09bf\u09a3 \u09a6\u09bf\u0995\u09c7 \u099c\u09bf\u09ac\u09cd\u09b0\u09be\u09b2\u09cd\u099f\u09be\u09b0 \u09aa\u09cd\u09b0\u09a3\u09be\u09b2\u09c0 & # 44 ; \u09aa\u09cd\u09b0\u09a3\u09be\u09b2\u09c0\u09b0 \u09a6\u0995\u09cd\u09b7\u09bf\u09a3\u09c7 \u09ae\u09b0\u0995\u09cd\u0995\u09cb & # 44 ; \u09aa\u09b6\u09cd\u099a\u09bf\u09ae \u0993 \u09a6\u0995\u09cd\u09b7\u09bf\u09a3-\u09aa\u09b6\u09cd\u099a\u09bf\u09ae \u09a6\u09bf\u0995\u09c7 \u0986\u099f\u09b2\u09be\u09a8\u09cd\u099f\u09bf\u0995 \u09ae\u09b9\u09be\u09b8\u09be\u0997\u09b0 \u0964
+\u09a4\u09be\u099b\u09be\u09a1\u09bc\u09be \u098f \u09aa\u09b0\u09bf\u09b8\u09cd\u09a5\u09bf\u09a4\u09c7 \u09ac\u09cb\u099d\u09be \u09af\u09be\u09af\u09bc \u099c\u09b0\u09c1\u09b0\u09bf \u09ae\u09c1\u09b9\u09c2\u09b0\u09cd\u09a4\u09c7 \u099c\u09be\u09a4\u09bf\u09b8\u0982\u0998\u09c7\u09b0 \u09a6\u09cd\u09b0\u09c1\u09a4 \u09b8\u09bf\u09a6\u09cd\u09a7\u09be\u09a8\u09cd\u09a4 \u09a8\u09c7\u09ac\u09be\u09b0 \u0985\u0995\u09cd\u09b7\u09ae\u09a4\u09be \u0964
+\u0995\u09be\u09b0\u09cd\u09b2 \u09ae\u09be\u09b0\u09cd\u0995\u09cd\u09b8\u09c7\u09b0 \u0995\u09be\u099c\u09c7\u09b0 \u09ae\u09be\u09a7\u09cd\u09af\u09ae\u09c7\u0987 \u0987\u09b9\u09be \u09aa\u09b0\u09bf\u099a\u09be\u09b2\u09bf\u09a4 \u09b9\u09af\u09bc \u0964
+\u098f\u09b0 \u09ac\u09bf\u09b7\u09af\u09bc\u09ac\u09b8\u09cd\u09a4\u09c1 \u0995\u0996\u09a8\u0993 \u09aa\u09c1\u09b0\u09be\u09a3 \u09a5\u09c7\u0995\u09c7 & # 44 ; \u0995\u0996\u09a8\u0993 \u09ae\u09a7\u09cd\u09af\u09af\u09c1\u0997\u09c0\u09af\u09bc \u09aa\u09cd\u09b0\u09c7\u09ae\u0995\u09be\u09b9\u09bf\u09a8\u09bf\u0997\u09c1\u09b2\u09bf \u09a5\u09c7\u0995\u09c7 & # 44 ; \u0995\u0996\u09a8\u0993 \u0986\u09ac\u09be\u09b0 \u098f\u0995\u09be\u09b2\u09c7\u09b0 \u09b8\u09be\u09ae\u09be\u099c\u09bf\u0995 \u0993 \u09b0\u09be\u099c\u09a8\u09c8\u09a4\u09bf\u0995 \u0998\u099f\u09a8\u09be\u09ac\u09b2\u09bf \u09a5\u09c7\u0995\u09c7 \u0997\u09c3\u09b9\u09c0\u09a4 \u0964
+\u09a4\u09bf\u09a8\u099f\u09bf \u09aa\u09b0\u09bf\u09ae\u09be\u09aa\u09c7\u09b0 \u0989\u09aa\u09b0 \u09ad\u09bf\u09a4\u09cd\u09a4\u09bf \u0995\u09b0\u09c7 \u09ae\u09b9\u09be\u09ac\u09bf\u09b6\u09cd\u09ac\u09c7\u09b0 \u09af\u09c7 \u09ac\u09af\u09bc\u09b8 \u09aa\u09be\u0993\u09af\u09bc\u09be \u0997\u09c7\u099b\u09c7 \u09a4\u09be \u09b9\u09b2 \u09aa\u09cd\u09b0\u09be\u09af\u09bc \u09e7\u09e9.\u09ed � \u09e6.\u09e8 \u09ac\u09bf\u09b2\u09bf\u09af\u09bc\u09a8 \u09ac\u099b\u09b0 \u0964
+\u0995\u09be\u099b\u09c7\u0987 \u09b0\u09af\u09bc\u09c7\u099b\u09c7 \u09aa\u09c2\u09b0\u09cd\u09ac \u09b0\u09be\u09b6\u09bf\u09af\u09bc\u09be & # 44 ; \u09af\u09be \u0993\u0996\u099f\u09b8\u09cd\u0995 \u09b8\u09be\u0997\u09b0 \u0993 \u099c\u09be\u09aa\u09be\u09a8 \u09b8\u09be\u0997\u09b0\u09c7\u09b0 \u0985\u09aa\u09b0 \u09aa\u09be\u09b0\u09c7 \u0985\u09ac\u09b8\u09cd\u09a5\u09bf\u09a4 \u0964
+\u0995\u09b2\u0995\u09be\u09a4\u09be\u09af\u09bc \u0985\u09ac\u09b8\u09cd\u09a5\u09bf\u09a4 \u09ad\u09be\u09b0\u09a4\u09c7\u09b0 \u099c\u09be\u09a4\u09c0\u09af\u09bc \u0997\u09cd\u09b0\u09a8\u09cd\u09a5\u09be\u0997\u09be\u09b0 \u09a6\u09c7\u09b6\u09c7\u09b0 \u0985\u0997\u09cd\u09b0\u09a3\u09c0 \u09aa\u09be\u09ac\u09b2\u09bf\u0995 \u09b2\u09be\u0987\u09ac\u09cd\u09b0\u09c7\u09b0\u09bf \u0964
+\u099b\u09a4\u09cd\u09b0\u09be\u0995\u09ac\u09bf\u09a6\u09cd\u09af\u09be
+\u09b0\u09be\u09b7\u09cd\u099f\u09cd\u09b0\u09b8\u0999\u09cd\u0998\u09c7\u09b0 \u09ae\u09b9\u09be\u09b8\u099a\u09bf\u09ac \u09ac\u09be\u09a8 \u0995\u09bf \u09ae\u09c1\u09a8
+\u09ae\u09bf\u09a8\u09bf\u0995\u09cd\u09b8\u09c7\u09b0 \u09b0\u099a\u09af\u09bc\u09bf\u09a4\u09be \u099b\u09bf\u09b2\u09c7\u09a8 \u098f\u09a8\u09cd\u09a1\u09cd\u09b0\u09c1 \u099f\u09be\u09a8\u09c7\u09a8\u09ac\u09ae & # 44 ; \u098f\u0995 \u09aa\u09cd\u09b0\u0996\u09cd\u09af\u09be\u09a4 \u0985\u09aa\u09be\u09b0\u09c7\u099f\u09bf\u0982 \u09b8\u09bf\u09b8\u09cd\u099f\u09c7\u09ae \u09a1\u09bf\u099c\u09be\u0987\u09a8 \u09aa\u09cd\u09b0\u09b6\u09bf\u0995\u09cd\u09b7\u0995 \u0964
+\u09a6\u09cd\u09af \u099f\u09be\u0987\u09ae\u09cd \u200c \u09b8 \u0985\u09ab \u0987\u09a8\u09cd\u09a1\u09bf\u09af\u09bc\u09be-\u09a4\u09c7 \u09b2\u09c7\u0996\u09be \u09b9\u09af\u09bc \u09af\u09c7 & quot ; it is absurd to compare it with any other indian cinema ... pather panchali is pure cinema & quot ; -lrb- \u201c \u098f\u0995\u09c7 \u0985\u09a8\u09cd\u09af \u09af\u09c7\u0995\u09cb\u09a8\u0993 \u09ad\u09be\u09b0\u09a4\u09c0\u09af\u09bc \u099a\u09b2\u099a\u09cd\u099a\u09bf\u09a4\u09cd\u09b0\u09c7\u09b0 \u09b8\u09be\u09a5\u09c7 \u09a4\u09c1\u09b2\u09a8\u09be \u0995\u09b0\u09be \u0985\u09ac\u09be\u09b8\u09cd\u09a4\u09ac ... \u09aa\u09a5\u09c7\u09b0 \u09aa\u09be\u0981\u099a\u09be\u09b2\u09c0 \u09b9\u09b2 \u09ac\u09bf\u09b6\u09c1\u09a6\u09cd\u09a7 \u099a\u09b2\u099a\u09cd\u099a\u09bf\u09a4\u09cd\u09b0 \u201d -rrb- \u0964
+\u098f\u09b0\u09aa\u09b0 \u09e7\u09ef\u09eb\u09e9 \u0996\u09cd\u09b0\u09bf\u09b8\u09cd\u099f\u09be\u09ac\u09cd\u09a6\u09c7\u09b0 \u09ae\u09c7 \u09ae\u09be\u09b8\u09c7 \u09a8\u099c\u09b0\u09c1\u09b2 \u0993 \u09aa\u09cd\u09b0\u09ae\u09c0\u09b2\u09be \u09a6\u09c7\u09ac\u09c0\u0995\u09c7 \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u09be\u09b0 \u099c\u09a8\u09cd\u09af \u09b2\u09a8\u09cd\u09a1\u09a8 \u09aa\u09be\u09a0\u09be\u09a8\u09cb \u09b9\u09af\u09bc \u0964
+\u098f\u09b0 \u09a6\u0995\u09cd\u09b7\u09bf\u09a3 \u0993 \u09aa\u09c2\u09b0\u09cd\u09ac\u09c7 \u0986\u099b\u09c7 \u09ac\u09bf\u09b8\u09cd\u09a4\u09c0\u09b0\u09cd\u09a3 \u09b8\u09ae\u09ad\u09c2\u09ae\u09bf ; \u09aa\u09b6\u09cd\u099a\u09bf\u09ae \u0993 \u0989\u09a4\u09cd\u09a4\u09b0\u09c7 \u0986\u099b\u09c7 \u09b0\u09c1\u0995\u09cd\u09b7 \u09aa\u09be\u09b9\u09be\u09a1\u09bc \u0993 \u09aa\u09b0\u09cd\u09ac\u09a4 \u0964
+\u099f\u09cd\u09b0\u09c7\u09a1\u09ae\u09be\u09b0\u09cd\u0995
+\u09b2\u09b0\u09cd\u09a1 \u0993\u09af\u09bc\u09c7\u09b2\u09c7\u09b8\u09b2\u09bf\u09b0 -lrb- \u0997\u09ad\u09b0\u09cd\u09a8\u09b0-\u099c\u09c7\u09a8\u09be\u09b0\u09c7\u09b2 \u09e7\u09ed\u09ef\u09ed-\u09e7\u09ee\u09e6\u09eb -rrb- \u09b6\u09be\u09b8\u09a8\u0995\u09be\u09b2\u09c7 \u09b6\u09b9\u09b0\u09c7\u09b0 \u0989\u09b2\u09cd\u09b2\u09c7\u0996\u09af\u09cb\u0997\u09cd\u09af \u09ac\u09c3\u09a6\u09cd\u09a7\u09bf \u0998\u099f\u09c7\u099b\u09bf\u09b2 \u0964
+\u0985\u09a8\u09c7\u0995 \u0997\u09c1\u09b0\u09c1\u09a4\u09cd\u09ac\u09aa\u09c2\u09b0\u09cd\u09a8 \u0993 \u09ac\u09be\u09b8\u09cd\u09a4\u09ac \u09b8\u09ae\u09b8\u09cd\u09af\u09be \u09b8\u09ae\u09be\u09a7\u09be\u09a8\u09c7\u09b0 \u099c\u09a8\u09cd\u09af \u099c\u099f\u09bf\u09b2 \u09b8\u0982\u0996\u09cd\u09af\u09be \u0985\u09aa\u09b0\u09bf\u09b9\u09be\u09b0\u09cd\u09af
+\u09ae\u09b9\u09be \u09ac\u09bf\u09b8\u09cd\u09ab\u09cb\u09b0\u09a3\u09c7\u09b0 \u098f\u0995\u099f\u09bf \u0989\u09b2\u09cd\u09b2\u09c7\u0996\u09af\u09cb\u0997\u09cd\u09af \u09ab\u09b2\u09be\u09ab\u09b2 \u09b9\u09b2 & # 44 ; \u09ac\u09b0\u09cd\u09a4\u09ae\u09be\u09a8\u0995\u09be\u09b2\u09c7 \u09ae\u09b9\u09be\u09ac\u09bf\u09b6\u09cd\u09ac\u09c7\u09b0 \u0985\u09ac\u09b8\u09cd\u09a5\u09be \u0985\u09a4\u09c0\u09a4 \u098f\u09ac\u0982 \u09ad\u09ac\u09bf\u09b7\u09cd\u09af\u09a4\u09c7\u09b0 \u0985\u09ac\u09b8\u09cd\u09a5\u09be \u09a5\u09c7\u0995\u09c7 \u09b8\u09ae\u09cd\u09aa\u09c2\u09b0\u09cd\u09a3 \u09aa\u09c3\u09a5\u0995 \u0964
+\u0989\u0987\u09a8\u09cd\u09a1\u09cb\u099c \u09ae\u09bf\u09b2\u09bf\u09a8\u09bf\u09af\u09bc\u09be\u09ae
+\u09b0\u09ac\u09c0\u09a8\u09cd\u09a6\u09cd\u09b0\u09a8\u09be\u09a5 \u0985\u09ac\u09b6\u09cd\u09af \u098f\u0995\u09be\u09a7\u09bf\u0995 \u09b6\u09c8\u09b2\u09c0 \u0986\u09af\u09bc\u09a4\u09cd\u09a4\u09cd\u09ac \u0995\u09b0\u09c7\u099b\u09bf\u09b2\u09c7\u09a8 \u0964
+\u09b6\u09cd\u09b0\u09ae \u0985\u09b0\u09cd\u09a5\u09a8\u09c0\u09a4\u09bf
+\u09ac\u09cd\u09b0\u09bf\u099f\u09c7\u09a8 \u098f\u0995\u09b8\u09ae\u09af\u09bc \u09ac\u09bf\u09b6\u09cd\u09ac\u09c7\u09b0 \u09aa\u09cd\u09b0\u09a7\u09be\u09a8 \u0993 \u0985\u0997\u09cd\u09b0\u0997\u09be\u09ae\u09c0 \u0985\u09b0\u09cd\u09a5\u09a8\u09c8\u09a4\u09bf\u0995 \u09b6\u0995\u09cd\u09a4\u09bf \u099b\u09bf\u09b2 \u0964
+\u09aa\u09be\u0995\u09bf\u09b8\u09cd\u09a4\u09be\u09a8\u09c7\u09b0 \u09b8\u09be\u09ae\u09b0\u09bf\u0995 \u09b6\u09be\u09b8\u09a8\u09c7\u09b0 \u09ac\u09bf\u09b0\u09c1\u09a6\u09cd\u09a7\u09c7 \u0986\u09a8\u09cd\u09a6\u09cb\u09b2\u09a8 \u098f\u09ac\u0982 \u09aa\u09be\u0995\u09bf\u09b8\u09cd\u09a4\u09be\u09a8\u09c7\u09b0 \u0997\u09cb\u09b7\u09cd\u09a0\u09c0\u0997\u09a4 \u09ac\u09c8\u09b7\u09ae\u09cd\u09af\u09c7\u09b0 \u09ac\u09bf\u09b0\u09c1\u09a6\u09cd\u09a7\u09c7 \u09aa\u09cd\u09b0\u09a4\u09bf\u09ac\u09be\u09a6 \u0993 \u09ac\u09be\u0999\u09be\u09b2\u09bf\u09a6\u09c7\u09b0 \u0986\u09a8\u09cd\u09a6\u09cb\u09b2\u09a8\u0995\u09c7 \u09b8\u09cd\u09ac\u09be\u09a7\u09c0\u09a8\u09a4\u09be\u09b0 \u09aa\u09a5\u09c7 \u09a7\u09be\u09ac\u09bf\u09a4 \u0995\u09b0\u09be\u09b0 \u099c\u09a8\u09cd\u09af \u09a4\u09bf\u09a8\u09bf \u09ac\u09cd\u09af\u09be\u09aa\u0995\u09ad\u09be\u09ac\u09c7 \u09aa\u09cd\u09b0\u09b6\u0982\u09b8\u09bf\u09a4 \u0964
+\u098f\u0996\u09be\u09a8\u09c7 \u0989\u09b2\u09cd\u09b2\u09c7\u0996 \u0995\u09b0\u09be \u09aa\u09cd\u09b0\u09af\u09bc\u09cb\u099c\u09a8 \u09af\u09c7 \u0985\u09a8\u09c7\u0995\u09c7 \u0987\u09a8\u09cd\u099f\u09be\u09b0\u09a8\u09c7\u099f \u098f\u09ac\u0982 \u0993\u09af\u09bc\u09be\u09b0\u09cd\u09b2\u09cd\u09a1 \u0993\u09af\u09bc\u09be\u0987\u09a1 \u0993\u09af\u09bc\u09c7\u09ac\u0995\u09c7 \u09b8\u09ae\u09be\u09b0\u09cd\u09a5\u0995 \u09b6\u09ac\u09cd\u09a6 \u09b9\u09bf\u09b8\u09c7\u09ac\u09c7 \u0997\u09a3\u09cd\u09af \u0995\u09b0\u09b2\u09c7\u0993 \u09aa\u09cd\u09b0\u0995\u09c3\u09a4\u09aa\u0995\u09cd\u09b7\u09c7 \u09b6\u09ac\u09cd\u09a6\u09a6\u09cd\u09ac\u09af\u09bc \u09ad\u09bf\u09a8\u09cd\u09a8 \u09ac\u09bf\u09b7\u09af\u09bc \u09a8\u09bf\u09b0\u09cd\u09a6\u09c7\u09b6 \u0995\u09b0\u09c7 \u0964
+. z \u098f\u09b0 \u0986\u09a8\u09c1\u09b8\u09be\u0999\u09cd\u0997\u09c0\u0995 \u09aa\u09cb\u09b2\u09be\u09b0 \u0995\u09cb-\u0985\u09b0\u09cd\u09a1\u09bf\u09a8\u09c7\u099f \u09a6\u09c1\u099f\u09bf \u09b9\u099a\u09cd\u099b\u09c7 r = -pipe-
+\u09a8\u09ad\u09c7\u09ae\u09cd\u09ac\u09b0
+\u09e7\u09ef\u09ed\u09e8 \u0996\u09cd\u09b0\u09c0\u09b8\u09cd\u099f\u09be\u09ac\u09cd\u09a6\u09c7 \u09a4\u09a6\u09be\u09a8\u09bf\u09a8\u09cd\u09a4\u09a8 \u0986\u09b0\u09aa\u09be\u09a8\u09c7\u099f\u09c7 \u09b8\u09b0\u09cd\u09ac\u09aa\u09cd\u09b0\u09a5\u09ae \u0987\u09b2\u09c7\u0995\u09cd\u099f\u09cd\u09b0\u09a8\u09bf\u0995 \u09ae\u09c7\u0987\u09b2 \u09aa\u09cd\u09b0\u09c7\u09b0\u09a3 \u0995\u09b0\u09be \u09b9\u09af\u09bc \u0964
+\u099c\u09c0\u09ac \u09ac\u09bf\u099c\u09cd\u099e\u09be\u09a8\u09c7\u09b0 \u09af\u09c7 \u09b6\u09be\u0996\u09be\u09af\u09bc \u099b\u09a4\u09cd\u09b0\u09be\u0995 \u0993 \u098f\u09b0 \u09ac\u09cd\u09af\u09ac\u09b9\u09be\u09b0\u09bf\u0995 \u09ac\u09bf\u09b7\u09af\u09bc \u09a8\u09bf\u09af\u09bc\u09c7 \u0986\u09b2\u09cb\u099a\u09a8\u09be \u0995\u09b0\u09be \u09b9\u09af\u09bc \u09a4\u09be\u0995\u09c7 \u099b\u09a4\u09cd\u09b0\u09be\u0995\u09ac\u09bf\u09a6\u09cd\u09af\u09be -lrb- mycology -rrb-
+\u09aa\u09be\u09a8\u09bf \u09a8\u09a6\u09c0 \u09a5\u09c7\u0995\u09c7 \u0989\u09a0\u09be\u09a8\u09cb \u09b9\u09a4\u09cb \u0995\u09af\u09bc\u09c7\u0995\u099f\u09bf \u09aa\u09c1\u09b0 \u09a6\u09bf\u09af\u09bc\u09c7- \u09a6\u09a1\u09bc\u09bf \u0993 \u09ac\u09be\u09b2\u099f\u09bf\u09b0 \u09ae\u09be\u09a7\u09cd\u09af\u09ae\u09c7 \u09aa\u09be\u09a8\u09bf \u09aa\u09b6\u09c1 \u09a6\u09cd\u09ac\u09be\u09b0\u09be \u099f\u09c7\u09a8\u09c7 \u09a4\u09cb\u09b2\u09be\u09b0 \u098f\u0995 \u09aa\u09a6\u09cd\u09a7\u09a4\u09bf \u0964
+\u098f\u09b0 \u09ae\u09a7\u09cd\u09af\u09c7 \u09b0\u09af\u09bc\u09c7\u099b\u09c7 \u0989\u09aa\u099c\u09be\u09a4\u09c0\u09af\u09bc \u09a8\u09c3\u09a4\u09cd\u09af & # 44 ; \u09b2\u09cb\u0995\u099c \u09a8\u09c3\u09a4\u09cd\u09af & # 44 ; \u09b6\u09be\u09b8\u09cd\u09a4\u09cd\u09b0\u09c0\u09af\u09bc \u09a8\u09c3\u09a4\u09cd\u09af & # 44 ; \u0987\u09a4\u09cd\u09af\u09be\u09a6\u09bf \u0964
+\u09ad\u09be\u09b0\u09a4\u09c7\u09b0 \u09aa\u09cd\u09b0\u09be\u099a\u09c0\u09a8\u09a4\u09ae \u09b8\u09be\u09b9\u09bf\u09a4\u09cd\u09af \u09aa\u09cd\u09b0\u09a5\u09ae\u09c7 \u09ae\u09cc\u0996\u09bf\u0995\u09ad\u09be\u09ac\u09c7 \u0993 \u09aa\u09b0\u09c7 \u09b2\u09bf\u0996\u09bf\u09a4 \u0986\u0995\u09be\u09b0\u09c7 \u09aa\u09cd\u09b0\u099a\u09b2\u09bf\u09a4 \u09b9\u09af\u09bc \u0964
+\u09e7\u09ef\u09ee\u09ef \u09b8\u09be\u09b2\u09c7 \u09a8\u09bf\u09b0\u09cd\u09ae\u09bf\u09a4 \u0997\u09a3\u09b6\u09a4\u09cd\u09b0\u09c1 \u099b\u09ac\u09bf\u099f\u09bf\u09a4\u09c7 \u09a4\u09be\u0981\u09b0 \u09aa\u09b0\u09bf\u099a\u09be\u09b2\u09a8\u09be \u09a4\u09c1\u09b2\u09a8\u09be\u09ae\u09c2\u09b2\u0995\u09ad\u09be\u09ac\u09c7 \u09a6\u09c1\u09b0\u09cd\u09ac\u09b2 \u098f\u09ac\u0982 \u098f\u099f\u09bf\u0995\u09c7 \u09a6\u09c0\u09b0\u09cd\u0998\u09a6\u09bf\u09a8\u09c7\u09b0 \u0985\u09b8\u09c1\u09b8\u09cd\u09a5\u09a4\u09be\u09b6\u09c7\u09b7\u09c7 \u09ab\u09bf\u09b0\u09c7 \u0986\u09b8\u09be\u09b0 \u09aa\u09b0 \u09b8\u09a4\u09cd\u09af\u099c\u09bf\u09a4\u09c7\u09b0 \u099a\u09b2\u099a\u09cd\u099a\u09bf\u09a4\u09cd\u09b0 \u09a8\u09bf\u09b0\u09cd\u09ae\u09be\u09a3\u09c7\u09b0 \u09aa\u09c1\u09a8\u09b0\u09cd\u09aa\u09cd\u09b0\u099a\u09c7\u09b7\u09cd\u099f\u09be \u09b9\u09bf\u09b8\u09c7\u09ac\u09c7\u0987 \u0997\u09a3\u09cd\u09af \u0995\u09b0\u09be \u09b9\u09af\u09bc \u0964
+\u2022 \u09ac\u09b2\u09ac\u09bf\u09a6\u09cd\u09af\u09be
+\u0985\u09a8\u09cd\u09af\u09be\u09a8\u09cd\u09af \u09b8\u09cd\u09ac\u09a4\u09cd\u09a4\u09cd\u09ac-\u09b8\u0982\u09b0\u0995\u09cd\u09b7\u09bf\u09a4 \u0985\u09aa\u09be\u09b0\u09c7\u099f\u09bf\u0982 \u09b8\u09bf\u09b8\u09cd\u099f\u09c7\u09ae \u09af\u09c7\u09ae\u09a8 \u0989\u0987\u09a8\u09cd\u09a1\u09cb\u099c \u098f\u09ac\u0982 \u09ae\u09cd\u09af\u09be\u0995 \u0993\u098f\u09b8 \u09b9\u09a4\u09c7 \u09b2\u09bf\u09a8\u09be\u0995\u09cd\u09b8 \u09ac\u09bf\u09ad\u09bf\u09a8\u09cd\u09a8\u09ad\u09be\u09ac\u09c7 \u0986\u09b2\u09be\u09a6\u09be \u0964
+\u098f\u09b6\u09bf\u09af\u09bc\u09be \u099f\u09be\u0987\u09ae\u09b8\u09c7\u09b0 \u09ad\u09be\u09b7\u09cd\u09af \u0985\u09a8\u09c1\u09af\u09be\u09af\u09bc\u09c0 & # 44 ;
+\u09ae\u09c1\u0995\u09cd\u09a4 \u09b8\u09cb\u09b0\u09cd\u09b8 \u09ac\u09be \u0993\u09aa\u09c7\u09a8 \u09b8\u09cb\u09b0\u09cd\u09b8 -lrb- open source -rrb- \u098f\u09b0 \u0985\u09b0\u09cd\u09a5 \u09b9\u09b2\u09cb \u0995\u09ae\u09cd\u09aa\u09bf\u0989\u099f\u09be\u09b0 \u09b8\u09ab\u099f\u0993\u09af\u09bc\u09cd\u09af\u09be\u09b0 \u098f\u09b0 \u09b8\u09cb\u09b0\u09cd\u09b8 \u0995\u09cb\u09a1 \u09ac\u09be \u09ae\u09c2\u09b2 \u09b8\u09be\u0982\u0995\u09c7\u09a4\u09bf\u0995 \u09ad\u09be\u09b7\u09be\u0995\u09c7 \u09ae\u09c1\u0995\u09cd\u09a4 \u09ad\u09be\u09ac\u09c7 \u09ac\u09bf\u09a4\u09b0\u09a3 \u0995\u09b0\u09be \u0964
+\u09ac\u09be\u0982\u09b2\u09be\u09a6\u09c7\u09b6 \u0985\u09a8\u09b2\u09be\u0987\u09a8\u09c7 \u09a2\u09be\u0995\u09be
+\u09aa\u09cd\u09b0\u09a5\u09ae \u09ac\u09bf\u09b6\u09cd\u09ac\u09af\u09c1\u09a6\u09cd\u09a7\u09c7 \u099c\u09be\u09b0\u09cd\u09ae\u09be\u09a8\u09bf \u09b9\u09c7\u09b0\u09c7 \u09af\u09be\u09af\u09bc \u0964
+\u09a4\u09ac\u09c7 \u098f \u09ac\u09bf\u09b7\u09af\u09bc\u099f\u09bf \u09ac\u09cb\u099d\u09be\u09b0 \u099c\u09a8\u09cd\u09af \u09ac\u09b0\u09cd\u09a4\u09ae\u09be\u09a8\u09c7\u0993 \u0997\u09ac\u09c7\u09b7\u09a3\u09be \u098f\u0997\u09bf\u09af\u09bc\u09c7 \u099a\u09b2\u099b\u09c7 \u0964
+\u09b8\u09c1\u09aa\u09be\u09b0\u098f\u0987\u099a
+\u09a4\u09be\u0995\u09c7 \u09b8\u09be\u09ae\u09b0\u09bf\u0995 \u09ac\u09be\u09b9\u09bf\u09a8\u09c0\u09b0 \u099c\u09a8\u09cd\u09af \u0986\u09a8\u09ab\u09bf\u099f \u0998\u09cb\u09b7\u09a3\u09be \u0995\u09b0\u09be \u09b9\u09af\u09bc\u09c7\u099b\u09bf\u09b2 \u0964
+\u09ad\u09c1\u099f\u09cd\u099f\u09cb \u098f\u09cd\u09af\u09be\u09b8\u09c7\u09ae\u09cd\u09ac\u09b2\u09bf \u09ac\u09af\u09bc\u0995\u099f \u0995\u09b0\u09be\u09b0 \u09b9\u09c1\u09ae\u0995\u09bf \u09a6\u09bf\u09af\u09bc\u09c7 \u0998\u09cb\u09b7\u09a3\u09be \u09a6\u09c7\u09a8 \u09af\u09c7 & # 44 ; \u0987\u09af\u09bc\u09be\u09b9\u09bf\u09af\u09bc\u09be \u0996\u09be\u09a8 \u09ae\u09c1\u099c\u09bf\u09ac\u0995\u09c7 \u09b8\u09b0\u0995\u09be\u09b0 \u0997\u09a0\u09a8\u09c7\u09b0 \u099c\u09a8\u09cd\u09af \u0986\u09b9\u09cd\u09ac\u09be\u09a8 \u099c\u09be\u09a8\u09be\u09b2\u09c7 \u09a4\u09bf\u09a8\u09bf \u09b8\u09c7 \u09b8\u09b0\u0995\u09be\u09b0\u0995\u09c7 \u09ae\u09c7\u09a8\u09c7 \u09a8\u09c7\u09ac\u09c7\u09a8 \u09a8\u09be \u0964
+\u0986\u09b0 computer \u09b6\u09ac\u09cd\u09a6\u09c7\u09b0 \u0985\u09b0\u09cd\u09a5 \u0997\u09a3\u09a8\u09be\u0995\u09be\u09b0\u09c0 \u09af\u09a8\u09cd\u09a4\u09cd\u09b0 \u0964
+\u09e7\u09ed\u09ed\u09ec \u09b8\u09be\u09b2\u09c7\u09b0 \u09ea \u099c\u09c1\u09b2\u09be\u0987 \u098f\u0987 \u0989\u09aa\u09a8\u09bf\u09ac\u09c7\u09b6\u0997\u09c1\u09b2\u09bf \u098f\u0995\u099f\u09bf \u09b8\u09cd\u09ac\u09be\u09a7\u09c0\u09a8\u09a4\u09be\u09b0 \u0998\u09cb\u09b7\u09a3\u09be\u09aa\u09a4\u09cd\u09b0 \u099c\u09be\u09b0\u09bf \u0995\u09b0\u09c7 \u0964
+\u099c\u09be\u09b0\u09cd\u09ae\u09be\u09a8\u09bf -lrb- \u099c\u09be\u09b0\u09cd\u09ae\u09be\u09a8 \u09ad\u09be\u09b7\u09be\u09af\u09bc : deutschland & # 44 ; \u09a1\u09af\u09bc\u099a\u09cd \u200c \u09b2\u09be\u09a8\u09cd\u099f\u09cd \u200c & # 44 ; \u0986-\u09a7\u09cd\u09ac-\u09ac : -lsb- d\u0254\u028ft\u0283lant -rsb- -rrb- & # 44 ; \u09ae\u09a7\u09cd\u09af \u0987\u0989\u09b0\u09cb\u09aa\u09c7\u09b0 \u098f\u0995\u099f\u09bf \u09b0\u09be\u09b7\u09cd\u099f\u09cd\u09b0 \u0964
+\u0996\u09cd\u09b0\u09bf\u09b8\u09cd\u099f\u09a7\u09b0\u09cd\u09ae \u09b0\u09be\u09b6\u09bf\u09af\u09bc\u09be\u09b0 \u09aa\u09cd\u09b0\u09a7\u09be\u09a8 \u09a7\u09b0\u09cd\u09ae \u0964
+\u0995\u09bf\u09a8\u09cd\u09a4\u09c1 \u0997\u09b2\u09a6\u09c7\u09b0 \u09b6\u09bf\u0995\u09cd\u09b7\u09be\u09b0 \u09b0\u09cb\u09ae\u09be\u09a8\u09c0\u0995\u09b0\u09a3\u09c7\u09b0 \u0997\u09a4\u09bf \u099b\u09bf\u09b2 \u09a7\u09c0\u09b0 \u0964
+\u09ac\u09bf\u09b7\u09af\u09bc\u09b6\u09cd\u09b0\u09c7\u09a3\u09c0 : \u0997\u09a8\u09c1 \u09ab\u09be\u0989\u09a8\u09cd\u09a1\u09c7\u09b6\u09a8
+\u0986\u09b0\u09cd\u09a5\u09bf\u0995 \u09a8\u09c0\u09a4\u09bf \u0993 \u09b0\u09be\u099c\u09b8\u09cd\u09ac \u09a8\u09c0\u09a4\u09bf\u0995\u09c7\u0993 \u0987\u09b9\u09be \u0985\u09a7\u09cd\u09af\u09af\u09bc\u09a8 \u0995\u09b0\u09c7 \u0964
+\u098f\u09b0 \u09ae\u09a7\u09cd\u09af\u09c7 \u09b0\u09af\u09bc\u09c7\u099b\u09c7 : \u09b9\u09af\u09bc\u09a4\u09cb \u09a4\u09cb\u09ae\u09be\u09b0 \u09aa\u09be\u09ac \u09a6\u09c7\u0996\u09be & # 44 ; \u0993\u09b0\u09c7 \u098f \u0995\u09cb\u09a8 \u09b8\u09cd\u09a8\u09c7\u09b9-\u09b8\u09c1\u09b0\u09a7\u09c1\u09a8\u09c0 \u0964
+\u09e7\u09ef\u09ef\u09e8 \u09b8\u09be\u09b2\u09c7\u09b0 \u09e8\u09e9 \u098f\u09aa\u09cd\u09b0\u09bf\u09b2 \u09b8\u09a4\u09cd\u09af\u099c\u09bf\u09ce \u09ae\u09c3\u09a4\u09cd\u09af\u09c1\u09ac\u09b0\u09a3 \u0995\u09b0\u09c7\u09a8 \u0964
+\u098f\u0987 \u09b8\u09ae\u09af\u09bc \u09a8\u099c\u09b0\u09c1\u09b2\u09c7\u09b0 \u09ae\u09c7\u09a1\u09bf\u0995\u09c7\u09b2 \u09b0\u09bf\u09aa\u09cb\u09b0\u09cd\u099f \u09ad\u09bf\u09af\u09bc\u09c7\u09a8\u09be\u09b0 \u09ac\u09bf\u0996\u09cd\u09af\u09be\u09a4 \u099a\u09bf\u0995\u09bf\u09ce\u09b8\u0995\u09a6\u09c7\u09b0 \u0995\u09be\u099b\u09c7 \u09aa\u09be\u09a0\u09be\u09a8\u09cb \u09b9\u09af\u09bc \u0964
+\u0985\u09ad\u09bf\u09a8\u09af\u09bc \u099b\u09be\u09a1\u09bc\u09be\u0993 \u09ac\u09bf\u09ad\u09bf\u09a8\u09cd\u09a8 \u09b8\u09ae\u09af\u09bc\u09c7 \u09b0\u09be\u09a8\u09c0 \u09ae\u09c1\u0996\u09be\u09b0\u09cd\u099c\u09c0 \u09ac\u09bf\u09ad\u09bf\u09a8\u09cd\u09a8 \u09a6\u09be\u09a4\u09ac\u09cd\u09af \u09b8\u0982\u09b8\u09cd\u09a5\u09be\u09b0 \u09b8\u09be\u09a5\u09c7 \u09af\u09c1\u0995\u09cd\u09a4 \u09a5\u09c7\u0995\u09c7\u099b\u09c7\u09a8 \u0964
+\u09ac\u09be\u0982\u09b2\u09be \u09b8\u09be\u09b9\u09bf\u09a4\u09cd\u09af \u098f\u09ac\u0982 \u09b8\u0982\u09b8\u09cd\u0995\u09c3\u09a4\u09bf\u09a4\u09c7 \u09a4\u09be\u09b0 \u09ac\u09bf\u09b6\u09c7\u09b7 \u0985\u09ac\u09a6\u09be\u09a8\u09c7\u09b0 \u09b8\u09cd\u09ac\u09c0\u0995\u09c3\u09a4\u09bf\u09b8\u09cd\u09ac\u09b0\u09c1\u09aa \u09e7\u09ef\u09ed\u09ea \u0996\u09cd\u09b0\u09bf\u09b8\u09cd\u099f\u09be\u09ac\u09cd\u09a6\u09c7\u09b0 \u09ef \u09a1\u09bf\u09b8\u09c7\u09ae\u09cd\u09ac\u09b0 \u09a4\u09be\u09b0\u09bf\u0996\u09c7 \u09a2\u09be\u0995\u09be \u09ac\u09bf\u09b6\u09cd\u09ac\u09ac\u09bf\u09a6\u09cd\u09af\u09be\u09b2\u09af\u09bc \u09a4\u09be\u0995\u09c7 \u09b8\u09ae\u09cd\u09ae\u09be\u09a8\u09b8\u09c2\u099a\u0995 \u09a1\u09bf.\u09b2\u09bf\u099f \u0989\u09aa\u09be\u09a7\u09bf\u09a4\u09c7 \u09ad\u09c2\u09b7\u09bf\u09a4 \u0995\u09b0\u09c7 \u0964
+\u0995\u09b2\u0995\u09be\u09a4\u09be\u09b0 \u09a6\u09c1\u09b0\u09cd\u0997\u09be\u09aa\u09c2\u099c\u09be \u09b6\u09b9\u09b0\u09c7\u09b0 \u0985\u09a8\u09cd\u09af\u09a4\u09ae \u09aa\u09b0\u09cd\u09af\u099f\u09a8 \u0986\u0995\u09b0\u09cd\u09b7\u09a3\u0993 \u09ac\u099f\u09c7 \u0964
+\u0995\u09bf\u09a8\u09cd\u09a4\u09c1 \u09ac\u09b9\u09c1 \u09b2\u0995\u09cd\u09b7 \u09aa\u09c2\u09b0\u09cd\u09ac \u099c\u09be\u09b0\u09cd\u09ae\u09be\u09a8 \u0985\u09b0\u09cd\u09a5\u09a8\u09c8\u09a4\u09bf\u0995\u09ad\u09be\u09ac\u09c7 \u09b8\u09ae\u09c3\u09a6\u09cd\u09a7 \u0993 \u0997\u09a3\u09a4\u09be\u09a8\u09cd\u09a4\u09cd\u09b0\u09bf\u0995 \u09aa\u09b6\u09cd\u099a\u09bf\u09ae \u099c\u09be\u09b0\u09cd\u09ae\u09be\u09a8\u09bf\u09a4\u09c7 \u0985\u09ad\u09bf\u09ac\u09be\u09b8\u09c0 \u09b9\u0993\u09af\u09bc\u09be \u09b6\u09c1\u09b0\u09c1 \u0995\u09b0\u09b2\u09c7 \u09e7\u09ef\u09ec\u09e7 \u09b8\u09be\u09b2\u09c7 \u09aa\u09c2\u09b0\u09cd\u09ac \u099c\u09be\u09b0\u09cd\u09ae\u09be\u09a8\u09bf \u09b8\u09b0\u0995\u09be\u09b0 \u09ac\u09be\u09b0\u09cd\u09b2\u09bf\u09a8\u09c7 \u098f\u0995\u099f\u09bf \u09aa\u09cd\u09b0\u09be\u099a\u09c0\u09b0 \u09a4\u09c1\u09b2\u09c7 \u09a6\u09c7\u09af\u09bc \u098f\u09ac\u0982 \u09a6\u09c7\u09b6\u09c7\u09b0 \u09b8\u09c0\u09ae\u09be\u09a8\u09cd\u09a4 \u099c\u09cb\u09b0\u09a6\u09be\u09b0 \u0995\u09b0\u09c7 \u0964
+\u09aa\u09cd\u09b0\u09a5\u09ae\u099f\u09bf \u09b9\u099a\u09cd\u099b\u09c7 \u099b\u09ac\u09bf\u099f\u09bf\u09b0 \u09a8\u09bf\u09b0\u09cd\u09ac\u09be\u0995 \u09aa\u09cd\u09b0\u09a5\u09ae \u09b8\u09be\u09a4 \u09ae\u09bf\u09a8\u09bf\u099f & # 44 ; \u09af\u09be \u099a\u09be\u09b0\u09c1\u09b0 \u098f\u0995\u0998\u09c7\u09af\u09bc\u09c7\u09ae\u09bf \u099c\u09c0\u09ac\u09a8 \u09ab\u09c1\u099f\u09bf\u09af\u09bc\u09c7 \u09a4\u09cb\u09b2\u09c7 & # 44 ; \u098f\u09ac\u0982 \u09a6\u09cd\u09ac\u09bf\u09a4\u09c0\u09af\u09bc\u099f\u09bf \u09b9\u09b2 & quot ; \u09ac\u09be\u0997\u09be\u09a8\u09c7\u09b0 \u09a6\u09cb\u09b2\u09a8\u09be\u09b0 \u09a6\u09c3\u09b6\u09cd\u09af & quot ; & # 44 ; \u09af\u09c7\u0996\u09be\u09a8\u09c7 \u099a\u09be\u09b0\u09c1 \u0985\u09ae\u09b2\u09c7\u09b0 \u099c\u09a8\u09cd\u09af \u09a4\u09be\u09b0 \u09ad\u09be\u09b2\u09ac\u09be\u09b8\u09be\u09b0 \u09ae\u09c1\u0996\u09cb\u09ae\u09c1\u0996\u09bf \u09b9\u09af\u09bc \u0964
+\u09e7\u09ee \u09b6\u09a4\u0995\u09c7\u09b0 \u098f\u0995\u09a6\u09b2 \u099a\u09bf\u09a8\u09cd\u09a4\u09be\u09ac\u09bf\u09a6 \u0993 \u09b2\u09c7\u0996\u0995 \u0986\u09af\u09bc \u0993 \u0989\u09ce\u09aa\u09be\u09a6\u09a8\u09c7\u09b0 \u099a\u0995\u09cd\u09b0\u09be\u0995\u09be\u09b0 \u09aa\u09cd\u09b0\u09ac\u09be\u09b9\u09c7\u09b0 \u09ae\u09be\u09a7\u09cd\u09af\u09ae\u09c7 \u0985\u09b0\u09cd\u09a5\u09a8\u09c8\u09a4\u09bf\u0995 \u099a\u09bf\u09a8\u09cd\u09a4\u09be\u09a7\u09be\u09b0\u09be\u09b0 \u0989\u09a8\u09cd\u09a8\u09af\u09bc\u09a8 \u0998\u099f\u09be\u09a8 \u0964
+\u09af\u09cb\u09a8\u09c0\u09a4\u09c7 \u09b2\u09bf\u0999\u09cd\u0997 \u09aa\u09cd\u09b0\u09ac\u09bf\u09b7\u09cd\u099f\u0995\u09b0\u09a3\u09c7\u09b0 \u09aa\u09c2\u09b0\u09cd\u09ac\u09c7 \u0995\u09be\u09ae\u09cb\u09a6\u09cd\u09a6\u09c0\u09aa\u0995 \u0995\u09be\u09b0\u09cd\u09af\u0995\u09b2\u09be\u09aa\u0995\u09c7 \u09ac\u09b2\u09be \u09b9\u09af\u09bc \u09b6\u09c3\u0999\u09cd\u0997\u09be\u09b0 \u0964
+\u098f\u099f\u09bf \u09ae\u09c2\u09b2\u09a4 \u09b6\u09c2\u0995\u09b0\u09c7\u09b0 \u09ae\u09be\u099d\u09c7\u0987 \u09aa\u09be\u0993\u09af\u09bc\u09be \u09af\u09c7\u09a4 \u09af\u09be \u0995\u09bf\u09a8\u09be \u09b6\u09c2\u0995\u09b0\u0995\u09c7 \u0987\u09a8\u09ab\u09cd\u09b2\u09c1\u09af\u09bc\u09c7\u099e\u09cd\u099c\u09be\u09a4\u09c7 \u0986\u0995\u09cd\u09b0\u09be\u09a8\u09cd\u09a4 \u0995\u09b0\u09a4 \u0964
+\u098f\u0997\u09c1\u09b2\u09bf \u098f\u0995\u098f \u09b9\u09af\u09bc\u09c7 mycelium \u0997\u09a0\u09a8 \u0995\u09b0\u09c7 \u0964
+\u09b0\u09be\u09b6\u09bf\u09af\u09bc\u09be \u09ac\u09b0\u09cd\u09a4\u09ae\u09be\u09a8\u09c7 \u098f\u0995\u099f\u09bf \u0997\u09a3\u09a4\u09be\u09a8\u09cd\u09a4\u09cd\u09b0\u09bf\u0995 \u09b0\u09be\u09b7\u09cd\u099f\u09cd\u09b0 \u0964
+\u09b2\u09bf\u0999\u09cd\u0997
+\u098f\u0987 \u0985\u09ac\u09b8\u09cd\u09a5\u09be\u0995\u09c7 \u09ac\u09b2\u09be \u09b9\u09af\u09bc \u09b0\u09be\u0997\u09ae\u09cb\u099a\u09a8 -lrb- \u0985\u09a8\u0997\u09cd\u09af\u09be\u099c\u09ae -rrb- \u0964
+\u0987\u09a4\u09bf\u09b9\u09be\u09b8\u09c7\u09b0 \u09ac\u09bf\u09ad\u09bf\u09a8\u09cd\u09a8 \u09aa\u09b0\u09cd\u09ac\u09c7 \u098f\u0996\u09be\u09a8\u09c7\u0987 \u09b8\u09cd\u09a5\u09be\u09aa\u09bf\u09a4 \u09b9\u09af\u09bc\u09c7\u099b\u09bf\u09b2 \u09ac\u09bf\u09b6\u09be\u09b2\u09be\u0995\u09be\u09b0 \u098f\u0995\u09be\u09a7\u09bf\u0995 \u09b8\u09be\u09ae\u09cd\u09b0\u09be\u099c\u09cd\u09af \u0964
+\u09ac\u09cd\u09af\u09be\u09b8\u09cd\u099f\u09bf\u0995 \u0985\u09b0\u09cd\u09a5\u09a8\u09c0\u09a4\u09bf
+\u09ac\u09cd\u09af\u09ac\u09b9\u09be\u09b0\u0995\u09be\u09b0\u09c0\u09b0 \u0995\u09be\u099b\u09c7 \u0985\u09aa\u09be\u09b0\u09c7\u099f\u09bf\u0982 \u09b8\u09bf\u09b8\u09cd\u099f\u09c7\u09ae\u09c7\u09b0 \u09b8\u09ac\u099a\u09c7\u09af\u09bc\u09c7 \u09a6\u09c3\u09b6\u09cd\u09af\u09ae\u09be\u09a8 \u09b0\u09c2\u09aa \u09b9\u09b2 \u0995\u09ae\u09cd\u09aa\u09bf\u0989\u099f\u09be\u09b0\u09c7\u09b0 \u0987\u09a8\u09cd\u099f\u09be\u09b0\u09ab\u09c7\u09b8 \u0964
+\u09ac\u09bf\u09b8\u09cd\u09a4\u09be\u09b0\u09bf\u09a4\u0983 \u09e7\u09ef\u09ed\u09e7 \u09b8\u09be\u09b2\u09c7\u09b0 \u0985\u09b8\u09cd\u09a5\u09be\u09af\u09bc\u09c0 \u09ac\u09be\u0982\u09b2\u09be\u09a6\u09c7\u09b6 \u09b8\u09b0\u0995\u09be\u09b0
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/packed/joshua.config
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/packed/joshua.config b/joshua-core/src/test/resources/bn-en/packed/joshua.config
new file mode 100644
index 0000000..40c612d
--- /dev/null
+++ b/joshua-core/src/test/resources/bn-en/packed/joshua.config
@@ -0,0 +1,47 @@
+lm = kenlm 5 false false 100 lm.gz
+
+tm = thrax pt 12 grammar.packed
+tm = thrax glue -1 grammar.glue
+
+mark_oovs = false
+
+#tm config
+default_non_terminal = OOV
+goalSymbol = GOAL
+
+#pruning config
+num-translation-options = 0
+pop-limit = 10
+
+#nbest config
+use_unique_nbest = true
+top_n = 10
+
+feature-function = OOVPenalty
+feature-function = WordPenalty
+
+###### model weights
+#lm order weight
+lm_0 1.3200621467242506
+
+#phrasemodel owner column(0-indexed) weight
+tm_pt_0 0.4571255198114019
+tm_pt_1 -0.17399038425384106
+tm_pt_2 -0.784547842535801
+tm_pt_3 0.76254324621594
+tm_pt_4 -0.8628695028838571
+tm_pt_5 0.04258438925263152
+tm_pt_6 0.5278815893934184
+tm_pt_7 0.9255662450788644
+tm_pt_8 0.03385066779097645
+tm_pt_9 0.9918446849428446
+tm_pt_10 0.52186013168725
+tm_pt_11 -0.7874679555197446
+tm_pt_12 -0.03770136145251124
+tm_pt_13 0.37085201114442157
+tm_pt_14 0.34054825749510886
+tm_pt_15 0.008348471483412778
+tm_pt_16 0.7984119288127296
+tm_glue_0 1
+WordPenalty -3.0476045270236662
+OOVPenalty 1.0
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/packed/lm.gz
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/packed/lm.gz b/joshua-core/src/test/resources/bn-en/packed/lm.gz
new file mode 100644
index 0000000..21f3092
Binary files /dev/null and b/joshua-core/src/test/resources/bn-en/packed/lm.gz differ
[56/60] incubator-joshua git commit: Merge branch
'maven-multi-module' of https://github.com/logogin/incubator-joshua into
maven-multi-module
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/packed/PackedGrammar.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/packed/PackedGrammar.java
index 0000000,b48685d..37bffb7
mode 000000,100644..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/packed/PackedGrammar.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/packed/PackedGrammar.java
@@@ -1,0 -1,1066 +1,1064 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.decoder.ff.tm.packed;
+
+ /***
- * This package implements Joshua's packed grammar structure, which enables the efficient loading
++ * This package implements Joshua's packed grammar structure, which enables the efficient loading
+ * and accessing of grammars. It is described in the paper:
+ *
+ * @article{ganitkevitch2012joshua,
+ * Author = {Ganitkevitch, J. and Cao, Y. and Weese, J. and Post, M. and Callison-Burch, C.},
+ * Journal = {Proceedings of WMT12},
+ * Title = {Joshua 4.0: Packing, PRO, and paraphrases},
+ * Year = {2012}}
+ *
+ * The packed grammar works by compiling out the grammar tries into a compact format that is loaded
+ * and parsed directly from Java arrays. A fundamental problem is that Java arrays are indexed
+ * by ints and not longs, meaning the maximum size of the packed grammar is about 2 GB. This forces
+ * the use of packed grammar slices, which together constitute the grammar. The figure in the
+ * paper above shows what each slice looks like.
+ *
+ * The division across slices is done in a depth-first manner. Consider the entire grammar organized
+ * into a single source-side trie. The splits across tries are done by grouping the root-level
+ * outgoing trie arcs --- and the entire trie beneath them --- across slices.
+ *
+ * This presents a problem: if the subtree rooted beneath a single top-level arc is too big for a
+ * slice, the grammar can't be packed. This happens with very large Hiero grammars, for example,
+ * where there are a *lot* of rules that start with [X].
+ *
+ * A solution being worked on is to split that symbol and pack them into separate grammars with a
+ * shared vocabulary, and then rely on Joshua's ability to query multiple grammars for rules to
+ * solve this problem. This is not currently implemented but could be done directly in the
+ * Grammar Packer.
+ *
+ * *UPDATE 10/2015*
+ * The introduction of a SliceAggregatingTrie together with sorting the grammar by the full source string
+ * (not just by the first source word) allows distributing rules with the same first source word
+ * across multiple slices.
+ * @author fhieber
+ */
+
+ import static java.util.Collections.sort;
+
+ import java.io.File;
+ import java.io.FileInputStream;
+ import java.io.IOException;
+ import java.io.InputStream;
+ import java.nio.BufferUnderflowException;
+ import java.nio.ByteBuffer;
+ import java.nio.IntBuffer;
+ import java.nio.MappedByteBuffer;
+ import java.nio.channels.FileChannel;
+ import java.nio.channels.FileChannel.MapMode;
+ import java.nio.file.Files;
+ import java.nio.file.Paths;
+ import java.security.DigestInputStream;
+ import java.security.MessageDigest;
+ import java.security.NoSuchAlgorithmException;
+ import java.util.ArrayList;
+ import java.util.Arrays;
+ import java.util.Comparator;
+ import java.util.HashMap;
+ import java.util.Iterator;
+ import java.util.List;
+ import java.util.Map;
+
+ import org.apache.joshua.corpus.Vocabulary;
+ import org.apache.joshua.decoder.JoshuaConfiguration;
+ import org.apache.joshua.decoder.ff.FeatureFunction;
+ import org.apache.joshua.decoder.ff.FeatureVector;
+ import org.apache.joshua.decoder.ff.tm.AbstractGrammar;
+ import org.apache.joshua.decoder.ff.tm.BasicRuleCollection;
++import org.apache.joshua.decoder.ff.tm.OwnerId;
+ import org.apache.joshua.decoder.ff.tm.Rule;
+ import org.apache.joshua.decoder.ff.tm.RuleCollection;
+ import org.apache.joshua.decoder.ff.tm.Trie;
+ import org.apache.joshua.decoder.ff.tm.hash_based.ExtensionIterator;
+ import org.apache.joshua.util.FormatUtils;
+ import org.apache.joshua.util.encoding.EncoderConfiguration;
+ import org.apache.joshua.util.encoding.FloatEncoder;
+ import org.apache.joshua.util.io.LineReader;
+
+ import com.google.common.base.Supplier;
+ import com.google.common.base.Suppliers;
+ import com.google.common.cache.Cache;
+ import com.google.common.cache.CacheBuilder;
++
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
+ public class PackedGrammar extends AbstractGrammar {
+
+ private static final Logger LOG = LoggerFactory.getLogger(PackedGrammar.class);
+ public static final String VOCABULARY_FILENAME = "vocabulary";
+
+ private EncoderConfiguration encoding;
+ private PackedRoot root;
+ private ArrayList<PackedSlice> slices;
+
+ private final File vocabFile; // store path to vocabulary file
+
+ // The version number of the earliest supported grammar packer
+ public static final int SUPPORTED_VERSION = 3;
+
+ // A rule cache for commonly used tries to avoid excess object allocations
+ // Testing shows there's up to ~95% hit rate when cache size is 5000 Trie nodes.
+ private final Cache<Trie, List<Rule>> cached_rules;
+
+ private String grammarDir;
+
+ public PackedGrammar(String grammar_dir, int span_limit, String owner, String type,
+ JoshuaConfiguration joshuaConfiguration) throws IOException {
- super(joshuaConfiguration);
++ super(owner, joshuaConfiguration, span_limit);
+
+ this.grammarDir = grammar_dir;
- this.spanLimit = span_limit;
+
+ // Read the vocabulary.
+ vocabFile = new File(grammar_dir + File.separator + VOCABULARY_FILENAME);
+ LOG.info("Reading vocabulary: {}", vocabFile);
+ if (!Vocabulary.read(vocabFile)) {
+ throw new RuntimeException("mismatches or collisions while reading on-disk vocabulary");
+ }
+
+ // Read the config
+ String configFile = grammar_dir + File.separator + "config";
+ if (new File(configFile).exists()) {
+ LOG.info("Reading packed config: {}", configFile);
+ readConfig(configFile);
+ }
+
+ // Read the quantizer setup.
+ LOG.info("Reading encoder configuration: {}{}encoding", grammar_dir, File.separator);
+ encoding = new EncoderConfiguration();
+ encoding.load(grammar_dir + File.separator + "encoding");
+
- // Set phrase owner.
- this.owner = Vocabulary.id(owner);
-
+ final List<String> listing = Arrays.asList(new File(grammar_dir).list());
+ sort(listing); // File.list() has arbitrary sort order
+ slices = new ArrayList<PackedSlice>();
+ for (String prefix : listing) {
+ if (prefix.startsWith("slice_") && prefix.endsWith(".source"))
+ slices.add(new PackedSlice(grammar_dir + File.separator + prefix.substring(0, 11)));
+ }
+
+ long count = 0;
+ for (PackedSlice s : slices)
+ count += s.estimated.length;
+ root = new PackedRoot(slices);
+ cached_rules = CacheBuilder.newBuilder().maximumSize(joshuaConfiguration.cachedRuleSize).build();
+
+ LOG.info("Loaded {} rules", count);
+ }
+
+ @Override
+ public Trie getTrieRoot() {
+ return root;
+ }
+
+ @Override
+ public boolean hasRuleForSpan(int startIndex, int endIndex, int pathLength) {
+ return (spanLimit == -1 || pathLength <= spanLimit);
+ }
+
+ @Override
+ public int getNumRules() {
+ int num_rules = 0;
+ for (PackedSlice ps : slices)
+ num_rules += ps.featureSize;
+ return num_rules;
+ }
+
+ @Override
+ public int getNumDenseFeatures() {
+ return encoding.getNumDenseFeatures();
+ }
+
+ /**
+ * Computes the MD5 checksum of the vocabulary file.
+ * Can be used for comparing vocabularies across multiple packedGrammars.
+ * @return the computed checksum
+ */
+ public String computeVocabularyChecksum() {
+ MessageDigest md;
+ try {
+ md = MessageDigest.getInstance("MD5");
+ } catch (NoSuchAlgorithmException e) {
+ throw new RuntimeException("Unknown checksum algorithm");
+ }
+ byte[] buffer = new byte[1024];
+ try (final InputStream is = Files.newInputStream(Paths.get(vocabFile.toString()));
+ DigestInputStream dis = new DigestInputStream(is, md)) {
+ while (dis.read(buffer) != -1) {}
+ } catch (IOException e) {
+ throw new RuntimeException("Can not find vocabulary file. This should not happen.");
+ }
+ byte[] digest = md.digest();
+ // convert the byte to hex format
+ StringBuffer sb = new StringBuffer("");
+ for (int i = 0; i < digest.length; i++) {
+ sb.append(Integer.toString((digest[i] & 0xff) + 0x100, 16).substring(1));
+ }
+ return sb.toString();
+ }
+
+ /**
+ * PackedRoot represents the root of the packed grammar trie.
+ * Tries for different source-side firstwords are organized in
+ * packedSlices on disk. A packedSlice can contain multiple trie
+ * roots (i.e. multiple source-side firstwords).
+ * The PackedRoot builds a lookup table, mapping from
+ * source-side firstwords to the addresses in the packedSlices
+ * that represent the subtrie for a particular firstword.
+ * If the GrammarPacker has to distribute rules for a
+ * source-side firstword across multiple slices, a
+ * SliceAggregatingTrie node is created that aggregates those
+ * tries to hide
+ * this additional complexity from the grammar interface
+ * This feature allows packing of grammars where the list of rules
+ * for a single source-side firstword would exceed the maximum array
+ * size of Java (2gb).
+ */
+ public final class PackedRoot implements Trie {
+
+ private final HashMap<Integer, Trie> lookup;
+
+ public PackedRoot(final List<PackedSlice> slices) {
+ final Map<Integer, List<Trie>> childTries = collectChildTries(slices);
+ lookup = buildLookupTable(childTries);
+ }
+
+ /**
+ * Determines whether trie nodes for source first-words are spread over
+ * multiple packedSlices by counting their occurrences.
+ * @param slices
+ * @return A mapping from first word ids to a list of trie nodes.
+ */
+ private Map<Integer, List<Trie>> collectChildTries(final List<PackedSlice> slices) {
+ final Map<Integer, List<Trie>> childTries = new HashMap<>();
+ for (PackedSlice packedSlice : slices) {
+
+ // number of tries stored in this packedSlice
+ final int num_children = packedSlice.source[0];
+ for (int i = 0; i < num_children; i++) {
+ final int id = packedSlice.source[2 * i + 1];
+
+ /* aggregate tries with same root id
+ * obtain a Trie node, already at the correct address in the packedSlice.
+ * In other words, the lookup index already points to the correct trie node in the packedSlice.
+ * packedRoot.match() thus can directly return the result of lookup.get(id);
+ */
+ if (!childTries.containsKey(id)) {
+ childTries.put(id, new ArrayList<Trie>(1));
+ }
+ final Trie trie = packedSlice.root().match(id);
+ childTries.get(id).add(trie);
+ }
+ }
+ return childTries;
+ }
+
+ /**
+ * Build a lookup table for children tries.
+ * If the list contains only a single child node, a regular trie node
+ * is inserted into the table; otherwise a SliceAggregatingTrie node is
+ * created that hides this partitioning into multiple packedSlices
+ * upstream.
+ */
+ private HashMap<Integer,Trie> buildLookupTable(final Map<Integer, List<Trie>> childTries) {
+ HashMap<Integer,Trie> lookup = new HashMap<>(childTries.size());
+ for (int id : childTries.keySet()) {
+ final List<Trie> tries = childTries.get(id);
+ if (tries.size() == 1) {
+ lookup.put(id, tries.get(0));
+ } else {
+ lookup.put(id, new SliceAggregatingTrie(tries));
+ }
+ }
+ return lookup;
+ }
+
+ @Override
+ public Trie match(int word_id) {
+ return lookup.get(word_id);
+ }
+
+ @Override
+ public boolean hasExtensions() {
+ return !lookup.isEmpty();
+ }
+
+ @Override
+ public HashMap<Integer, ? extends Trie> getChildren() {
+ return lookup;
+ }
+
+ @Override
+ public ArrayList<? extends Trie> getExtensions() {
+ return new ArrayList<>(lookup.values());
+ }
+
+ @Override
+ public boolean hasRules() {
+ return false;
+ }
+
+ @Override
+ public RuleCollection getRuleCollection() {
+ return new BasicRuleCollection(0, new int[0]);
+ }
+
+ @Override
+ public Iterator<Integer> getTerminalExtensionIterator() {
+ return new ExtensionIterator(lookup, true);
+ }
+
+ @Override
+ public Iterator<Integer> getNonterminalExtensionIterator() {
+ return new ExtensionIterator(lookup, false);
+ }
+ }
+
+ public final class PackedSlice {
+ private final String name;
+
+ private final int[] source;
+ private final IntBuffer target;
+ private final ByteBuffer features;
+ private final ByteBuffer alignments;
+
+ private final int[] targetLookup;
+ private int featureSize;
+ private float[] estimated;
+ private float[] precomputable;
+
+ private final static int BUFFER_HEADER_POSITION = 8;
+
+ /**
+ * Provides a cache of packedTrie nodes to be used in getTrie.
+ */
+ private HashMap<Integer, PackedTrie> tries;
+
+ public PackedSlice(String prefix) throws IOException {
+ name = prefix;
+
+ File source_file = new File(prefix + ".source");
+ File target_file = new File(prefix + ".target");
+ File target_lookup_file = new File(prefix + ".target.lookup");
+ File feature_file = new File(prefix + ".features");
+ File alignment_file = new File(prefix + ".alignments");
+
+ source = fullyLoadFileToArray(source_file);
+ // First int specifies the size of this file, load from 1st int on
+ targetLookup = fullyLoadFileToArray(target_lookup_file, 1);
+
+ target = associateMemoryMappedFile(target_file).asIntBuffer();
+ features = associateMemoryMappedFile(feature_file);
+ initializeFeatureStructures();
+
+ if (alignment_file.exists()) {
+ alignments = associateMemoryMappedFile(alignment_file);
+ } else {
+ alignments = null;
+ }
+
+ tries = new HashMap<Integer, PackedTrie>();
+ }
+
+ /**
+ * Helper function to help create all the structures which describe features
+ * in the Slice. Only called during object construction.
+ */
+ private void initializeFeatureStructures() {
+ int num_blocks = features.getInt(0);
+ estimated = new float[num_blocks];
+ precomputable = new float[num_blocks];
+ Arrays.fill(estimated, Float.NEGATIVE_INFINITY);
+ Arrays.fill(precomputable, Float.NEGATIVE_INFINITY);
+ featureSize = features.getInt(4);
+ }
+
+ private int getIntFromByteBuffer(int position, ByteBuffer buffer) {
+ return buffer.getInt(BUFFER_HEADER_POSITION + (4 * position));
+ }
+
+ private int[] fullyLoadFileToArray(File file) throws IOException {
+ return fullyLoadFileToArray(file, 0);
+ }
+
+ /**
+ * This function will use a bulk loading method to fully populate a target
+ * array from file.
+ *
+ * @param file
+ * File that will be read from disk.
+ * @param startIndex
+ * an offset into the read file.
+ * @return an int array of size length(file) - offset containing ints in the
+ * file.
+ * @throws IOException
+ */
+ private int[] fullyLoadFileToArray(File file, int startIndex) throws IOException {
+ IntBuffer buffer = associateMemoryMappedFile(file).asIntBuffer();
+ int size = (int) (file.length() - (4 * startIndex))/4;
+ int[] result = new int[size];
+ buffer.position(startIndex);
+ buffer.get(result, 0, size);
+ return result;
+ }
+
+ private ByteBuffer associateMemoryMappedFile(File file) throws IOException {
+ try(FileInputStream fileInputStream = new FileInputStream(file)) {
+ FileChannel fileChannel = fileInputStream.getChannel();
+ int size = (int) fileChannel.size();
+ MappedByteBuffer result = fileChannel.map(MapMode.READ_ONLY, 0, size);
+ return result;
+ }
+ }
+
+ private final int[] getTarget(int pointer) {
+ // Figure out level.
+ int tgt_length = 1;
+ while (tgt_length < (targetLookup.length + 1) && targetLookup[tgt_length] <= pointer)
+ tgt_length++;
+ int[] tgt = new int[tgt_length];
+ int index = 0;
+ int parent;
+ do {
+ parent = target.get(pointer);
+ if (parent != -1)
+ tgt[index++] = target.get(pointer + 1);
+ pointer = parent;
+ } while (pointer != -1);
+ return tgt;
+ }
+
+ private synchronized PackedTrie getTrie(final int node_address) {
+ PackedTrie t = tries.get(node_address);
+ if (t == null) {
+ t = new PackedTrie(node_address);
+ tries.put(node_address, t);
+ }
+ return t;
+ }
+
+ private synchronized PackedTrie getTrie(int node_address, int[] parent_src, int parent_arity,
+ int symbol) {
+ PackedTrie t = tries.get(node_address);
+ if (t == null) {
+ t = new PackedTrie(node_address, parent_src, parent_arity, symbol);
+ tries.put(node_address, t);
+ }
+ return t;
+ }
+
+ /**
+ * Returns the FeatureVector associated with a rule (represented as a block ID).
+ * These features are in the form "feature1=value feature2=value...". By default, unlabeled
+ * features are named using the pattern.
+ * @param block_id
+ * @return feature vector
+ */
+
+ private final FeatureVector loadFeatureVector(int block_id) {
+ int featurePosition = getIntFromByteBuffer(block_id, features);
+ final int numFeatures = encoding.readId(features, featurePosition);
+
+ featurePosition += EncoderConfiguration.ID_SIZE;
+ final FeatureVector featureVector = new FeatureVector();
+ FloatEncoder encoder;
+ String featureName;
+
+ for (int i = 0; i < numFeatures; i++) {
+ final int innerId = encoding.readId(features, featurePosition);
+ final int outerId = encoding.outerId(innerId);
+ encoder = encoding.encoder(innerId);
+ // TODO (fhieber): why on earth are dense feature ids (ints) encoded in the vocabulary?
+ featureName = Vocabulary.word(outerId);
+ final float value = encoder.read(features, featurePosition);
+ try {
+ int index = Integer.parseInt(featureName);
+ featureVector.increment(index, -value);
+ } catch (NumberFormatException e) {
+ featureVector.increment(featureName, value);
+ }
+ featurePosition += EncoderConfiguration.ID_SIZE + encoder.size();
+ }
+
+ return featureVector;
+ }
+
+ /**
+ * We need to synchronize this method as there is a many to one ratio between
+ * PackedRule/PhrasePair and this class (PackedSlice). This means during concurrent first
+ * getAlignments calls to PackedRule objects they could alter each other's positions within the
+ * buffer before calling read on the buffer.
+ */
+ private synchronized final byte[] getAlignmentArray(int block_id) {
+ if (alignments == null)
+ throw new RuntimeException("No alignments available.");
+ int alignment_position = getIntFromByteBuffer(block_id, alignments);
+ int num_points = (int) alignments.get(alignment_position);
+ byte[] alignment = new byte[num_points * 2];
+
+ alignments.position(alignment_position + 1);
+ try {
+ alignments.get(alignment, 0, num_points * 2);
+ } catch (BufferUnderflowException bue) {
+ LOG.warn("Had an exception when accessing alignment mapped byte buffer");
+ LOG.warn("Attempting to access alignments at position: {}", alignment_position + 1);
+ LOG.warn("And to read this many bytes: {}", num_points * 2);
+ LOG.warn("Buffer capacity is : {}", alignments.capacity());
+ LOG.warn("Buffer position is : {}", alignments.position());
+ LOG.warn("Buffer limit is : {}", alignments.limit());
+ throw bue;
+ }
+ return alignment;
+ }
+
+ private final PackedTrie root() {
+ return getTrie(0);
+ }
+
+ public String toString() {
+ return name;
+ }
+
+ /**
+ * A trie node within the grammar slice. Identified by its position within the source array,
+ * and, as a supplement, the source string leading from the trie root to the node.
+ *
+ * @author jg
+ *
+ */
+ public class PackedTrie implements Trie, RuleCollection {
+
+ private final int position;
+
+ private boolean sorted = false;
+
+ private int[] src;
+ private int arity;
+
+ private PackedTrie(int position) {
+ this.position = position;
+ src = new int[0];
+ arity = 0;
+ }
+
+ private PackedTrie(int position, int[] parent_src, int parent_arity, int symbol) {
+ this.position = position;
+ src = new int[parent_src.length + 1];
+ System.arraycopy(parent_src, 0, src, 0, parent_src.length);
+ src[src.length - 1] = symbol;
+ arity = parent_arity;
+ if (FormatUtils.isNonterminal(symbol))
+ arity++;
+ }
+
+ @Override
+ public final Trie match(int token_id) {
+ int num_children = source[position];
+ if (num_children == 0)
+ return null;
+ if (num_children == 1 && token_id == source[position + 1])
+ return getTrie(source[position + 2], src, arity, token_id);
+ int top = 0;
+ int bottom = num_children - 1;
+ while (true) {
+ int candidate = (top + bottom) / 2;
+ int candidate_position = position + 1 + 2 * candidate;
+ int read_token = source[candidate_position];
+ if (read_token == token_id) {
+ return getTrie(source[candidate_position + 1], src, arity, token_id);
+ } else if (top == bottom) {
+ return null;
+ } else if (read_token > token_id) {
+ top = candidate + 1;
+ } else {
+ bottom = candidate - 1;
+ }
+ if (bottom < top)
+ return null;
+ }
+ }
+
+ @Override
+ public HashMap<Integer, ? extends Trie> getChildren() {
+ HashMap<Integer, Trie> children = new HashMap<Integer, Trie>();
+ int num_children = source[position];
+ for (int i = 0; i < num_children; i++) {
+ int symbol = source[position + 1 + 2 * i];
+ int address = source[position + 2 + 2 * i];
+ children.put(symbol, getTrie(address, src, arity, symbol));
+ }
+ return children;
+ }
+
+ @Override
+ public boolean hasExtensions() {
+ return (source[position] != 0);
+ }
+
+ @Override
+ public ArrayList<? extends Trie> getExtensions() {
+ int num_children = source[position];
+ ArrayList<PackedTrie> tries = new ArrayList<PackedTrie>(num_children);
+
+ for (int i = 0; i < num_children; i++) {
+ int symbol = source[position + 1 + 2 * i];
+ int address = source[position + 2 + 2 * i];
+ tries.add(getTrie(address, src, arity, symbol));
+ }
+
+ return tries;
+ }
+
+ @Override
+ public boolean hasRules() {
+ int num_children = source[position];
+ return (source[position + 1 + 2 * num_children] != 0);
+ }
+
+ @Override
+ public RuleCollection getRuleCollection() {
+ return this;
+ }
+
+ @Override
+ public List<Rule> getRules() {
+ List<Rule> rules = cached_rules.getIfPresent(this);
+ if (rules != null) {
+ return rules;
+ }
+
+ int num_children = source[position];
+ int rule_position = position + 2 * (num_children + 1);
+ int num_rules = source[rule_position - 1];
+
+ rules = new ArrayList<Rule>(num_rules);
+ for (int i = 0; i < num_rules; i++) {
+ rules.add(new PackedRule(rule_position + 3 * i));
+ }
+
+ cached_rules.put(this, rules);
+ return rules;
+ }
+
+ /**
+ * We determine if the Trie is sorted by checking if the estimated cost of the first rule in
+ * the trie has been set.
+ */
+ @Override
+ public boolean isSorted() {
+ return sorted;
+ }
+
+ private synchronized void sortRules(List<FeatureFunction> models) {
+ int num_children = source[position];
+ int rule_position = position + 2 * (num_children + 1);
+ int num_rules = source[rule_position - 1];
+ if (num_rules == 0) {
+ this.sorted = true;
+ return;
+ }
+ Integer[] rules = new Integer[num_rules];
+
+ int target_address;
+ int block_id;
+ for (int i = 0; i < num_rules; ++i) {
+ target_address = source[rule_position + 1 + 3 * i];
+ rules[i] = rule_position + 2 + 3 * i;
+ block_id = source[rules[i]];
+
+ Rule rule = new Rule(source[rule_position + 3 * i], src,
+ getTarget(target_address), loadFeatureVector(block_id), arity, owner);
+ estimated[block_id] = rule.estimateRuleCost(models);
+ precomputable[block_id] = rule.getPrecomputableCost();
+ }
+
+ Arrays.sort(rules, new Comparator<Integer>() {
+ public int compare(Integer a, Integer b) {
+ float a_cost = estimated[source[a]];
+ float b_cost = estimated[source[b]];
+ if (a_cost == b_cost)
+ return 0;
+ return (a_cost > b_cost ? -1 : 1);
+ }
+ });
+
+ int[] sorted = new int[3 * num_rules];
+ int j = 0;
+ for (int i = 0; i < rules.length; i++) {
+ int address = rules[i];
+ sorted[j++] = source[address - 2];
+ sorted[j++] = source[address - 1];
+ sorted[j++] = source[address];
+ }
+ for (int i = 0; i < sorted.length; i++)
+ source[rule_position + i] = sorted[i];
+
+ // Replace rules in cache with their sorted values on next getRules()
+ cached_rules.invalidate(this);
+ this.sorted = true;
+ }
+
+ @Override
+ public List<Rule> getSortedRules(List<FeatureFunction> featureFunctions) {
+ if (!isSorted())
+ sortRules(featureFunctions);
+ return getRules();
+ }
+
+ @Override
+ public int[] getSourceSide() {
+ return src;
+ }
+
+ @Override
+ public int getArity() {
+ return arity;
+ }
+
+ @Override
+ public Iterator<Integer> getTerminalExtensionIterator() {
+ return new PackedChildIterator(position, true);
+ }
+
+ @Override
+ public Iterator<Integer> getNonterminalExtensionIterator() {
+ return new PackedChildIterator(position, false);
+ }
+
+ public final class PackedChildIterator implements Iterator<Integer> {
+
+ private int current;
+ private boolean terminal;
+ private boolean done;
+ private int last;
+
+ PackedChildIterator(int position, boolean terminal) {
+ this.terminal = terminal;
+ int num_children = source[position];
+ done = (num_children == 0);
+ if (!done) {
+ current = (terminal ? position + 1 : position - 1 + 2 * num_children);
+ last = (terminal ? position - 1 + 2 * num_children : position + 1);
+ }
+ }
+
+ @Override
+ public boolean hasNext() {
+ if (done)
+ return false;
+ int next = (terminal ? current + 2 : current - 2);
+ if (next == last)
+ return false;
+ return (terminal ? source[next] > 0 : source[next] < 0);
+ }
+
+ @Override
+ public Integer next() {
+ if (done)
+ throw new RuntimeException("No more symbols!");
+ int symbol = source[current];
+ if (current == last)
+ done = true;
+ if (!done) {
+ current = (terminal ? current + 2 : current - 2);
+ done = (terminal ? source[current] < 0 : source[current] > 0);
+ }
+ return symbol;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ /**
+ * A packed phrase pair represents a rule of the form of a phrase pair, packed with the
+ * grammar-packer.pl script, which simply adds a nonterminal [X] to the left-hand side of
+ * all phrase pairs (and converts the Moses features). The packer then packs these. We have
+ * to then put a nonterminal on the source and target sides to treat the phrase pairs like
+ * left-branching rules, which is how Joshua deals with phrase decoding.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ *
+ */
+ public final class PackedPhrasePair extends PackedRule {
+
+ private final Supplier<int[]> englishSupplier;
+ private final Supplier<byte[]> alignmentSupplier;
+
+ public PackedPhrasePair(int address) {
+ super(address);
+ englishSupplier = initializeEnglishSupplier();
+ alignmentSupplier = initializeAlignmentSupplier();
+ }
+
+ @Override
+ public int getArity() {
+ return PackedTrie.this.getArity() + 1;
+ }
+
+ /**
+ * Initialize a number of suppliers which get evaluated when their respective getters
+ * are called.
+ * Inner lambda functions are guaranteed to only be called once, because of this underlying
+ * structures are accessed in a threadsafe way.
+ * Guava's implementation makes sure only one read of a volatile variable occurs per get.
+ * This means this implementation should be as thread-safe and performant as possible.
+ */
+
+ private Supplier<int[]> initializeEnglishSupplier(){
+ Supplier<int[]> result = Suppliers.memoize(() ->{
+ int[] phrase = getTarget(source[address + 1]);
+ int[] tgt = new int[phrase.length + 1];
+ tgt[0] = -1;
+ for (int i = 0; i < phrase.length; i++)
+ tgt[i+1] = phrase[i];
+ return tgt;
+ });
+ return result;
+ }
+
+ private Supplier<byte[]> initializeAlignmentSupplier(){
+ Supplier<byte[]> result = Suppliers.memoize(() ->{
+ byte[] raw_alignment = getAlignmentArray(source[address + 2]);
+ byte[] points = new byte[raw_alignment.length + 2];
+ points[0] = points[1] = 0;
+ for (int i = 0; i < raw_alignment.length; i++)
+ points[i + 2] = (byte) (raw_alignment[i] + 1);
+ return points;
+ });
+ return result;
+ }
+
+ /**
+ * Take the English phrase of the underlying rule and prepend an [X].
+ *
+ * @return the augmented phrase
+ */
+ @Override
+ public int[] getEnglish() {
+ return this.englishSupplier.get();
+ }
+
+ /**
+ * Take the French phrase of the underlying rule and prepend an [X].
+ *
+ * @return the augmented French phrase
+ */
+ @Override
+ public int[] getFrench() {
+ int phrase[] = new int[src.length + 1];
+ int ntid = Vocabulary.id(PackedGrammar.this.joshuaConfiguration.default_non_terminal);
+ phrase[0] = ntid;
+ System.arraycopy(src, 0, phrase, 1, src.length);
+ return phrase;
+ }
+
+ /**
+ * Similarly the alignment array needs to be shifted over by one.
+ *
+ * @return the byte[] alignment
+ */
+ @Override
+ public byte[] getAlignment() {
+ // if no alignments in grammar do not fail
+ if (alignments == null) {
+ return null;
+ }
+
+ return this.alignmentSupplier.get();
+ }
+ }
+
+ public class PackedRule extends Rule {
+ protected final int address;
+ private final Supplier<int[]> englishSupplier;
+ private final Supplier<FeatureVector> featureVectorSupplier;
+ private final Supplier<byte[]> alignmentsSupplier;
+
+ public PackedRule(int address) {
+ this.address = address;
+ this.englishSupplier = intializeEnglishSupplier();
+ this.featureVectorSupplier = initializeFeatureVectorSupplier();
+ this.alignmentsSupplier = initializeAlignmentsSupplier();
+ }
+
+ private Supplier<int[]> intializeEnglishSupplier(){
+ Supplier<int[]> result = Suppliers.memoize(() ->{
+ return getTarget(source[address + 1]);
+ });
+ return result;
+ }
+
+ private Supplier<FeatureVector> initializeFeatureVectorSupplier(){
+ Supplier<FeatureVector> result = Suppliers.memoize(() ->{
+ return loadFeatureVector(source[address + 2]);
+ });
+ return result;
+ }
+
+ private Supplier<byte[]> initializeAlignmentsSupplier(){
+ Supplier<byte[]> result = Suppliers.memoize(()->{
+ // if no alignments in grammar do not fail
+ if (alignments == null){
+ return null;
+ }
+ return getAlignmentArray(source[address + 2]);
+ });
+ return result;
+ }
+
+ @Override
+ public void setArity(int arity) {
+ }
+
+ @Override
+ public int getArity() {
+ return PackedTrie.this.getArity();
+ }
+
+ @Override
- public void setOwner(int ow) {
++ public void setOwner(OwnerId owner) {
+ }
+
+ @Override
- public int getOwner() {
++ public OwnerId getOwner() {
+ return owner;
+ }
+
+ @Override
+ public void setLHS(int lhs) {
+ }
+
+ @Override
+ public int getLHS() {
+ return source[address];
+ }
+
+ @Override
+ public void setEnglish(int[] eng) {
+ }
+
+ @Override
+ public int[] getEnglish() {
+ return this.englishSupplier.get();
+ }
+
+ @Override
+ public void setFrench(int[] french) {
+ }
+
+ @Override
+ public int[] getFrench() {
+ return src;
+ }
+
+ @Override
+ public FeatureVector getFeatureVector() {
+ return this.featureVectorSupplier.get();
+ }
+
+ @Override
+ public byte[] getAlignment() {
+ return this.alignmentsSupplier.get();
+ }
+
+ @Override
+ public String getAlignmentString() {
+ throw new RuntimeException("AlignmentString not implemented for PackedRule!");
+ }
+
+ @Override
+ public float getEstimatedCost() {
+ return estimated[source[address + 2]];
+ }
+
+ // @Override
+ // public void setPrecomputableCost(float cost) {
+ // precomputable[source[address + 2]] = cost;
+ // }
+
+ @Override
+ public float getPrecomputableCost() {
+ return precomputable[source[address + 2]];
+ }
+
+ @Override
+ public float estimateRuleCost(List<FeatureFunction> models) {
+ return estimated[source[address + 2]];
+ }
+
+ @Override
+ public String toString() {
+ StringBuffer sb = new StringBuffer();
+ sb.append(Vocabulary.word(this.getLHS()));
+ sb.append(" ||| ");
+ sb.append(getFrenchWords());
+ sb.append(" ||| ");
+ sb.append(getEnglishWords());
+ sb.append(" |||");
+ sb.append(" " + getFeatureVector());
+ sb.append(String.format(" ||| %.3f", getEstimatedCost()));
+ return sb.toString();
+ }
+ }
+ }
+ }
+
+ @Override
+ public void addOOVRules(int word, List<FeatureFunction> featureFunctions) {
+ throw new RuntimeException("PackedGrammar.addOOVRules(): I can't add OOV rules");
+ }
+
+ @Override
+ public void addRule(Rule rule) {
+ throw new RuntimeException("PackedGrammar.addRule(): I can't add rules");
+ }
+
+ /**
+ * Read the config file
+ *
+ * TODO: this should be rewritten using typeconfig.
+ *
+ * @param config
+ * @throws IOException
+ */
+ private void readConfig(String config) throws IOException {
+ int version = 0;
+
+ for (String line: new LineReader(config)) {
+ String[] tokens = line.split(" = ");
+ if (tokens[0].equals("max-source-len"))
+ this.maxSourcePhraseLength = Integer.parseInt(tokens[1]);
+ else if (tokens[0].equals("version")) {
+ version = Integer.parseInt(tokens[1]);
+ }
+ }
+
+ if (version != 3) {
+ String message = String.format("The grammar at %s was packed with packer version %d, but the earliest supported version is %d",
+ this.grammarDir, version, SUPPORTED_VERSION);
+ throw new RuntimeException(message);
+ }
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/GrammarBuilderWalkerFunction.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/GrammarBuilderWalkerFunction.java
index 0000000,a6edddd..c5d2398
mode 000000,100644..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/GrammarBuilderWalkerFunction.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/GrammarBuilderWalkerFunction.java
@@@ -1,0 -1,180 +1,179 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.decoder.hypergraph;
+
+ import java.io.PrintStream;
+ import java.util.HashSet;
+
+ import org.apache.joshua.corpus.Vocabulary;
+ import org.apache.joshua.decoder.JoshuaConfiguration;
+ import org.apache.joshua.decoder.ff.tm.Grammar;
+ import org.apache.joshua.decoder.ff.tm.Rule;
+ import org.apache.joshua.decoder.ff.tm.format.HieroFormatReader;
+ import org.apache.joshua.decoder.ff.tm.hash_based.MemoryBasedBatchGrammar;
+ import org.apache.joshua.util.FormatUtils;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
+ /**
+ * This walker function builds up a new context-free grammar by visiting each node in a hypergraph.
+ * For a quick overview, see Chris Dyer's 2010 NAACL paper
+ * "Two monlingual parses are better than one (synchronous parse)".
+ * <p>
+ * From a functional-programming point of view, this walker really wants to calculate a fold over
+ * the entire hypergraph: the initial value is an empty grammar, and as we visit each node, we add
+ * more rules to the grammar. After we have traversed the whole hypergraph, the resulting grammar
+ * will contain all rules needed for synchronous parsing.
+ * <p>
+ * These rules look just like the rules already present in the hypergraph, except that each
+ * non-terminal symbol is annotated with the span of its node.
+ */
+ public class GrammarBuilderWalkerFunction implements WalkerFunction {
+
+ private static final Logger LOG = LoggerFactory.getLogger(GrammarBuilderWalkerFunction.class);
+
+ private MemoryBasedBatchGrammar grammar;
+ private static HieroFormatReader reader = new HieroFormatReader();
+ private PrintStream outStream;
+ private int goalSymbol;
+ private HashSet<Rule> rules;
+
+ public GrammarBuilderWalkerFunction(String goal,JoshuaConfiguration joshuaConfiguration) {
- grammar = new MemoryBasedBatchGrammar(reader,joshuaConfiguration);
- grammar.setSpanLimit(1000);
++ grammar = new MemoryBasedBatchGrammar(reader, joshuaConfiguration, 1000);
+ outStream = null;
+ goalSymbol = Vocabulary.id(goal);
+ rules = new HashSet<Rule>();
+ }
+
+ public GrammarBuilderWalkerFunction(String goal, PrintStream out,JoshuaConfiguration joshuaConfiguration) {
+ this(goal,joshuaConfiguration);
+ outStream = out;
+ }
+
+ public void apply(HGNode node, int index) {
+ // System.err.printf("VISITING NODE: %s\n", getLabelWithSpan(node));
+ for (HyperEdge e : node.hyperedges) {
+ Rule r = getRuleWithSpans(e, node);
+ if (r != null && !rules.contains(r)) {
+ if (outStream != null) outStream.println(r);
+ grammar.addRule(r);
+ rules.add(r);
+ }
+ }
+ }
+
+ private static int getLabelWithSpan(HGNode node) {
+ return Vocabulary.id(getLabelWithSpanAsString(node));
+ }
+
+ private static String getLabelWithSpanAsString(HGNode node) {
+ String label = Vocabulary.word(node.lhs);
+ String unBracketedCleanLabel = label.substring(1, label.length() - 1);
+ return String.format("[%d-%s-%d]", node.i, unBracketedCleanLabel, node.j);
+ }
+
+ private boolean nodeHasGoalSymbol(HGNode node) {
+ return node.lhs == goalSymbol;
+ }
+
+ private Rule getRuleWithSpans(HyperEdge edge, HGNode head) {
+ Rule edgeRule = edge.getRule();
+ int headLabel = getLabelWithSpan(head);
+ // System.err.printf("Head label: %s\n", headLabel);
+ // if (edge.getAntNodes() != null) {
+ // for (HGNode n : edge.getAntNodes())
+ // System.err.printf("> %s\n", getLabelWithSpan(n));
+ // }
+ int[] source = getNewSource(nodeHasGoalSymbol(head), edge);
+ // if this would be unary abstract, getNewSource will be null
+ if (source == null) return null;
+ int[] target = getNewTargetFromSource(source);
+ Rule result =
+ new Rule(headLabel, source, target, edgeRule.getFeatureString(), edgeRule.getArity());
+ // System.err.printf("new rule is %s\n", result);
+ return result;
+ }
+
+ private static int[] getNewSource(boolean isGlue, HyperEdge edge) {
+ Rule rule = edge.getRule();
+ int[] english = rule.getEnglish();
+ // if this is a unary abstract rule, just return null
+ // TODO: except glue rules!
+ if (english.length == 1 && english[0] < 0 && !isGlue) return null;
+ int[] result = new int[english.length];
+ for (int i = 0; i < english.length; i++) {
+ int curr = english[i];
+ if (! FormatUtils.isNonterminal(curr)) {
+ // If it's a terminal symbol, we just copy it into the new rule.
+ result[i] = curr;
+ } else {
+ // If it's a nonterminal, its value is -N, where N is the index
+ // of the nonterminal on the source side.
+ //
+ // That is, if we would call a nonterminal "[X,2]", the value of
+ // curr at this point is -2. And the tail node that it points at
+ // is #1 (since getTailNodes() is 0-indexed).
+ int index = -curr - 1;
+ result[i] = getLabelWithSpan(edge.getTailNodes().get(index));
+ }
+ }
+ // System.err.printf("source: %s\n", result);
+ return result;
+ }
+
+ private static int[] getNewTargetFromSource(int[] source) {
+ int[] result = new int[source.length];
+ int currNT = -1; // value to stick into NT slots
+ for (int i = 0; i < source.length; i++) {
+ result[i] = source[i];
+ if (FormatUtils.isNonterminal(result[i])) {
+ result[i] = currNT;
+ currNT--;
+ }
+ }
+ // System.err.printf("target: %s\n", result);
+ return result;
+ }
+
+ private static HGNode getGoalSymbolNode(HGNode root) {
+ if (root.hyperedges == null || root.hyperedges.size() == 0) {
+ LOG.error("getGoalSymbolNode: root node has no hyperedges");
+ return null;
+ }
+ return root.hyperedges.get(0).getTailNodes().get(0);
+ }
+
+
+ public static int goalSymbol(HyperGraph hg) {
+ if (hg.goalNode == null) {
+ LOG.error("goalSymbol: goalNode of hypergraph is null");
+ return -1;
+ }
+ HGNode symbolNode = getGoalSymbolNode(hg.goalNode);
+ if (symbolNode == null) return -1;
+ // System.err.printf("goalSymbol: %s\n", result);
+ // System.err.printf("symbol node LHS is %d\n", symbolNode.lhs);
+ // System.err.printf("i = %d, j = %d\n", symbolNode.i, symbolNode.j);
+ return getLabelWithSpan(symbolNode);
+ }
+
+ public Grammar getGrammar() {
+ return grammar;
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/HyperGraphPruning.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/HyperGraphPruning.java
index 0000000,27f5525..51bd9d6
mode 000000,100644..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/HyperGraphPruning.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/hypergraph/HyperGraphPruning.java
@@@ -1,0 -1,176 +1,176 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.decoder.hypergraph;
+
+ import java.util.HashMap;
+
+ import org.apache.joshua.corpus.Vocabulary;
+
+ /**
+ * during the pruning process, many Item/Deductions may not be explored at all due to the early-stop
+ * in pruning_deduction
+ *
+ * @author Zhifei Li, zhifei.work@gmail.com
+ * @version $LastChangedDate$
+ */
+ public class HyperGraphPruning extends TrivialInsideOutside {
+
+ HashMap<HGNode, Boolean> processedNodesTbl = new HashMap<HGNode, Boolean>();
+ double bestLogProb;// viterbi unnormalized log prob in the hypergraph
+
+ boolean ViterbiPruning = false;// Viterbi or Posterior pruning
+
+ boolean fixThresholdPruning = true;
+ double THRESHOLD_GENERAL = 10;// if the merit is worse than the best_log_prob by this number, then
+ // prune
+ double THRESHOLD_GLUE = 10;// if the merit is worse than the best_log_prob by this number, then
+ // prune
+
+ int numSurvivedEdges = 0;
+ int numSurvivedNodes = 0;
+
+ int glueGrammarOwner = 0;// TODO
+
+
+ public HyperGraphPruning(boolean fixThreshold, double thresholdGeneral, double thresholdGlue) {
+ fixThresholdPruning = fixThreshold;
+ THRESHOLD_GENERAL = thresholdGeneral;
+ THRESHOLD_GLUE = thresholdGlue;
+ glueGrammarOwner = Vocabulary.id("glue");// TODO
+ }
+
+ public void clearState() {
+ processedNodesTbl.clear();
+ super.clearState();
+ }
+
+
+ // ######################### pruning here ##############
+ public void pruningHG(HyperGraph hg) {
+
+ runInsideOutside(hg, 2, 1, 1.0);// viterbi-max, log-semiring
+
+ if (fixThresholdPruning) {
+ pruningHGHelper(hg);
+ super.clearState();
+ } else {
+ throw new RuntimeException("wrong call");
+ }
+ }
+
+ private void pruningHGHelper(HyperGraph hg) {
+
+ this.bestLogProb = getLogNormalizationConstant();// set the best_log_prob
+
+ numSurvivedEdges = 0;
+ numSurvivedNodes = 0;
+ processedNodesTbl.clear();
+ pruningNode(hg.goalNode);
+
+ // clear up
+ processedNodesTbl.clear();
+
+ System.out.println("Item suvived ratio: " + numSurvivedNodes * 1.0 / hg.numNodes + " = "
+ + numSurvivedNodes + "/" + hg.numNodes);
+ System.out.println("Deduct suvived ratio: " + numSurvivedEdges * 1.0 / hg.numEdges + " = "
+ + numSurvivedEdges + "/" + hg.numEdges);
+ }
+
+
+ private void pruningNode(HGNode it) {
+
+ if (processedNodesTbl.containsKey(it)) return;
+
+ processedNodesTbl.put(it, true);
+ boolean shouldSurvive = false;
+
+ // ### recursive call on each deduction
+ for (int i = 0; i < it.hyperedges.size(); i++) {
+ HyperEdge dt = it.hyperedges.get(i);
+ boolean survived = pruningEdge(dt, it);// deduction-specifc operation
+ if (survived) {
+ shouldSurvive = true; // at least one deduction survive
+ } else {
+ it.hyperedges.remove(i);
+ i--;
+ }
+ }
+ // TODO: now we simply remove the pruned deductions, but in general, we may want to update the
+ // variables mainted in the item (e.g., best_deduction); this depends on the pruning method used
+
+ /*
+ * by defintion: "should_surive==false" should be impossible, since if I got called, then my
+ * upper-deduction must survive, then i will survive because there must be one way to reach me
+ * from lower part in order for my upper-deduction survive
+ */
+ if (!shouldSurvive) {
+ throw new RuntimeException("item explored but does not survive");
+ // TODO: since we always keep the best_deduction, this should never be true
+ } else {
+ numSurvivedNodes++;
+ }
+ }
+
+
+ // if survive, return true
+ // best-deduction is always kept
+ private boolean pruningEdge(HyperEdge dt, HGNode parent) {
+
+ /**
+ * TODO: theoretically, if an item is get called, then its best deduction should always be kept
+ * even just by the threshold-checling. In reality, due to precision of Double, the
+ * threshold-checking may not be perfect
+ */
+ if (dt != parent.bestHyperedge) { // best deduction should always survive if the Item is get
+ // called
+ // ### prune?
+ if (shouldPruneHyperedge(dt, parent)) {
+ return false; // early stop
+ }
+ }
+
+ // ### still survive, recursive call all my ant-items
+ if (null != dt.getTailNodes()) {
+ for (HGNode ant_it : dt.getTailNodes()) {
+ pruningNode(ant_it); // recursive call on each ant item, note: the ant_it will not be pruned
+ // as I need it
+ }
+ }
+
+ // ### if get to here, then survive; remember: if I survive, then my upper-item must survive
+ numSurvivedEdges++;
+ return true; // survive
+ }
+
+ private boolean shouldPruneHyperedge(HyperEdge dt, HGNode parent) {
+
+ // ### get merit
+ double postLogProb = getEdgeUnormalizedPosteriorLogProb(dt, parent);
+
+
- if (dt.getRule() != null && dt.getRule().getOwner() == glueGrammarOwner
++ if (dt.getRule() != null && dt.getRule().getOwner().equals(glueGrammarOwner)
+ && dt.getRule().getArity() == 2) { // specicial rule: S->S X
+ // TODO
+ return (postLogProb - this.bestLogProb < THRESHOLD_GLUE);
+ } else {
+ return (postLogProb - this.bestLogProb < THRESHOLD_GENERAL);
+ }
+ }
+
+ }
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/main/java/org/apache/joshua/decoder/io/JSONMessage.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/io/JSONMessage.java
index 0000000,9c3899e..90a550b
mode 000000,100644..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/io/JSONMessage.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/io/JSONMessage.java
@@@ -1,0 -1,159 +1,159 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.decoder.io;
+
+ import java.util.ArrayList;
+ import java.util.List;
+
+ import com.google.gson.Gson;
+ import com.google.gson.GsonBuilder;
+
+ import org.apache.joshua.decoder.StructuredTranslation;
+ import org.apache.joshua.decoder.Translation;
+
+ /**
+ * Represents a JSON object returned by the server. The object has the format
+ *
+ * { data: {
+ * translations: [
+ * { annotatedSource: "",
+ * translatedText: "",
+ * raw_nbest: [
+ * { hyp: "",
+ * totalScore: 0.0, } ]
+ * tokenization: { ... }
+ * translatedTextRaw: "",
+ * nbest: [
+ * { translatedText: "",
+ * translatedTextRaw: "",
+ * tokenization: { ... } } ] } ] } }
+ *
+ * @author post
+ */
+
+ public class JSONMessage {
+ public Data data = null;
+ public List<String> metadata = null;
+ public JSONMessage() {
+ metadata = new ArrayList<String>();
+ }
+
+ public class Data {
+ public List<TranslationItem> translations;
+
+ public Data() {
+ translations = new ArrayList<TranslationItem>();
+ }
+ }
+ //
+ // public class Metadata {
+ // public String metadata = null;
+ // public List<String> rules = null;
+ //
+ // public Metadata() {
+ // rules = new ArrayList<String>();
+ // }
+ // }
+
+ public void addTranslation(Translation translation) {
+ String viterbi = translation.getStructuredTranslations().get(0).getTranslationString();
+
+ TranslationItem item = addTranslation(viterbi);
+
+ for (StructuredTranslation hyp: translation.getStructuredTranslations()) {
- String text = hyp.getFormattedTranslationString();
++ String text = hyp.getTranslationString();
+ float score = hyp.getTranslationScore();
+
+ item.addHypothesis(text, score);
+ }
+
+ // old string-based k-best output
+ // String[] results = translation.toString().split("\\n");
+ // if (results.length > 0) {
+ // String rawTranslation = results[0].split(" \\|\\|\\| ")[1];
+ // JSONMessage.TranslationItem item = message.addTranslation(rawTranslation);
+ //
+ // for (String result: results) {
+ // String[] tokens = result.split(" \\|\\|\\| ");
+ // String rawResult = tokens[1];
+ // float score = Float.parseFloat(tokens[3]);
+ // item.addHypothesis(rawResult, score);
+ // }
+ // }
+ }
+
+ /**
+ * Adds a new Translation to the JSON object. A Translation represents one or more hypotheses
+ * (or k-best items)
+ *
- * @param text
++ * @param text the translation text
+ * @return the new TranslationItem object
+ */
+ public TranslationItem addTranslation(String text) {
+ if (data == null)
+ data = new Data();
+
+ TranslationItem newItem = new TranslationItem(text);
+ data.translations.add(newItem);
+ return newItem;
+ }
+
+ public void addMetaData(String msg) {
+ this.metadata.add(msg);
+ }
+
+ public class TranslationItem {
+ public String translatedText;
+ public List<NBestItem> raw_nbest;
+
+ public TranslationItem(String value) {
+ this.translatedText = value;
+ this.raw_nbest = new ArrayList<NBestItem>();
+ }
+
+ /**
+ * Adds a new item to the translation's list of k-best items
+ *
+ * @param hyp the hypothesis
+ * @param score its score
+ */
+ public void addHypothesis(String hyp, float score) {
+ this.raw_nbest.add(new NBestItem(hyp, score));
+ }
+ }
+
+ public class NBestItem {
+ public String hyp;
+ public float totalScore;
+
+ public NBestItem(String hyp, float score) {
+ this.hyp = hyp;
+ this.totalScore = score;
+ }
+ }
+
+ public void addRule(String rule) {
+ metadata.add("custom_rule " + rule);
+ }
+
+ public String toString() {
+ Gson gson = new GsonBuilder().setPrettyPrinting().create();
+ return gson.toJson(this) + "\n";
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/PhraseTable.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/phrase/PhraseTable.java
index 0000000,27f92ac..0e03dba
mode 000000,100644..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/PhraseTable.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/PhraseTable.java
@@@ -1,0 -1,181 +1,183 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.decoder.phrase;
+
++import static org.apache.joshua.decoder.ff.tm.OwnerMap.UNKNOWN_OWNER;
++
+ import java.io.File;
+ import java.io.IOException;
+ import java.util.List;
+
+ import org.apache.joshua.corpus.Vocabulary;
+ import org.apache.joshua.decoder.JoshuaConfiguration;
+ import org.apache.joshua.decoder.ff.FeatureFunction;
+ import org.apache.joshua.decoder.ff.tm.Grammar;
++import org.apache.joshua.decoder.ff.tm.OwnerId;
+ import org.apache.joshua.decoder.ff.tm.Rule;
+ import org.apache.joshua.decoder.ff.tm.RuleCollection;
+ import org.apache.joshua.decoder.ff.tm.Trie;
+ import org.apache.joshua.decoder.ff.tm.hash_based.MemoryBasedBatchGrammar;
+ import org.apache.joshua.decoder.ff.tm.packed.PackedGrammar;
+
+ /**
+ * Represents a phrase table, and is implemented as a wrapper around either a {@link PackedGrammar}
+ * or a {@link MemoryBasedBatchGrammar}.
+ *
+ * TODO: this should all be implemented as a two-level trie (source trie and target trie).
+ */
+ public class PhraseTable implements Grammar {
+
+ private JoshuaConfiguration config;
+ private Grammar backend;
+
+ /**
+ * Chain to the super with a number of defaults. For example, we only use a single nonterminal,
+ * and there is no span limit.
+ *
+ * @param grammarFile file path parent directory
+ * @param owner used to set phrase owners
+ * @param type the grammar specification keyword (e.g., "thrax" or "moses")
+ * @param config a populated {@link org.apache.joshua.decoder.JoshuaConfiguration}
+ * @throws IOException if there is an error reading the grammar file
+ */
+ public PhraseTable(String grammarFile, String owner, String type, JoshuaConfiguration config)
+ throws IOException {
+ this.config = config;
+ int spanLimit = 0;
+
+ if (grammarFile != null && new File(grammarFile).isDirectory()) {
+ this.backend = new PackedGrammar(grammarFile, spanLimit, owner, type, config);
+ if (this.backend.getMaxSourcePhraseLength() == -1) {
+ String msg = "FATAL: Using a packed grammar for a phrase table backend requires that you "
+ + "packed the grammar with Joshua 6.0.2 or greater";
+ throw new RuntimeException(msg);
+ }
+
+ } else {
+ this.backend = new MemoryBasedBatchGrammar(type, grammarFile, owner, "[X]", spanLimit, config);
+ }
+ }
+
+ public PhraseTable(String owner, JoshuaConfiguration config) {
+ this.config = config;
-
- this.backend = new MemoryBasedBatchGrammar(owner, config);
++ this.backend = new MemoryBasedBatchGrammar(owner, config, 20);
+ }
+
+ /**
+ * Returns the longest source phrase read. Because phrases have a dummy nonterminal prepended to
+ * them, we need to subtract 1.
+ *
+ * @return the longest source phrase read.
+ */
+ @Override
+ public int getMaxSourcePhraseLength() {
+ return this.backend.getMaxSourcePhraseLength() - 1;
+ }
+
+ /**
+ * Collect the set of target-side phrases associated with a source phrase.
+ *
+ * @param sourceWords the sequence of source words
+ * @return the rules
+ */
+ public RuleCollection getPhrases(int[] sourceWords) {
+ if (sourceWords.length != 0) {
+ Trie pointer = getTrieRoot();
+ pointer = pointer.match(Vocabulary.id("[X]"));
+ int i = 0;
+ while (pointer != null && i < sourceWords.length)
+ pointer = pointer.match(sourceWords[i++]);
+
+ if (pointer != null && pointer.hasRules()) {
+ return pointer.getRuleCollection();
+ }
+ }
+
+ return null;
+ }
+
+ /**
+ * Adds a rule to the grammar. Only supported when the backend is a MemoryBasedBatchGrammar.
+ *
+ * @param rule the rule to add
+ */
+ public void addRule(Rule rule) {
+ ((MemoryBasedBatchGrammar)backend).addRule(rule);
+ }
+
+ @Override
+ public void addOOVRules(int sourceWord, List<FeatureFunction> featureFunctions) {
+ // TODO: _OOV shouldn't be outright added, since the word might not be OOV for the LM (but now almost
+ // certainly is)
+ int targetWord = config.mark_oovs
+ ? Vocabulary.id(Vocabulary.word(sourceWord) + "_OOV")
+ : sourceWord;
+
+ int nt_i = Vocabulary.id("[X]");
+ Rule oovRule = new Rule(nt_i, new int[] { nt_i, sourceWord },
- new int[] { -1, targetWord }, "", 1, null);
++ new int[] { -1, targetWord }, "", 1, UNKNOWN_OWNER);
+ addRule(oovRule);
+ oovRule.estimateRuleCost(featureFunctions);
+
+ // String ruleString = String.format("[X] ||| [X,1] %s ||| [X,1] %s",
+ // Vocabulary.word(sourceWord), Vocabulary.word(targetWord));
+ // Rule oovRule = new HieroFormatReader().parseLine(ruleString);
+ // oovRule.setOwner(Vocabulary.id("oov"));
+ // addRule(oovRule);
+ // oovRule.estimateRuleCost(featureFunctions);
+ }
+
+ @Override
+ public Trie getTrieRoot() {
+ return backend.getTrieRoot();
+ }
+
+ @Override
+ public void sortGrammar(List<FeatureFunction> models) {
+ backend.sortGrammar(models);
+ }
+
+ @Override
+ public boolean isSorted() {
+ return backend.isSorted();
+ }
+
+ /**
+ * This should never be called.
+ */
+ @Override
+ public boolean hasRuleForSpan(int startIndex, int endIndex, int pathLength) {
+ return true;
+ }
+
+ @Override
+ public int getNumRules() {
+ return backend.getNumRules();
+ }
+
+ @Override
- public int getOwner() {
++ public OwnerId getOwner() {
+ return backend.getOwner();
+ }
+
+ @Override
+ public int getNumDenseFeatures() {
+ return backend.getNumDenseFeatures();
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Stacks.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Stacks.java
index 0000000,c688b2c..8c092ec
mode 000000,100644..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Stacks.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/phrase/Stacks.java
@@@ -1,0 -1,269 +1,271 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.decoder.phrase;
+
+ /***
+ * Entry point for phrase-based decoding, analogous to {@link Chart} for the CKY algorithm. This
+ * class organizes all the stacks used for decoding, and is responsible for building them. Stack
+ * construction is stack-centric: that is, we loop over the number of source words in increasing sizes;
+ * at each step of this iteration, we break the search between smaller stack sizes and source-side
+ * phrase sizes.
+ *
+ * The end result of decoding is a {@link Hypergraph} with the same format as hierarchical decoding.
+ * Phrases are treating as left-branching rules, and the span information (i,j) is overloaded so
+ * that i means nothing and j represents the index of the last-translated source word in each
+ * hypothesis. This means that most hypergraph code can work without modification. The algorithm
+ * ensures that the coverage vector is consistent but the resulting hypergraph may not be projective,
+ * which is different from the CKY algorithm, which does produce projective derivations.
+ *
+ * TODO Lattice decoding is not yet supported (March 2015).
+ */
+
++import static org.apache.joshua.decoder.ff.tm.OwnerMap.UNKNOWN_OWNER;
++
+ import java.util.ArrayList;
+ import java.util.List;
+
+ import org.apache.joshua.corpus.Span;
+ import org.apache.joshua.decoder.JoshuaConfiguration;
+ import org.apache.joshua.decoder.chart_parser.ComputeNodeResult;
+ import org.apache.joshua.decoder.ff.FeatureFunction;
+ import org.apache.joshua.decoder.ff.tm.AbstractGrammar;
+ import org.apache.joshua.decoder.ff.tm.Grammar;
+ import org.apache.joshua.decoder.hypergraph.HGNode;
+ import org.apache.joshua.decoder.hypergraph.HyperEdge;
+ import org.apache.joshua.decoder.hypergraph.HyperGraph;
+ import org.apache.joshua.decoder.segment_file.Sentence;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
+ public class Stacks {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Stacks.class);
+
+ // The list of stacks, grouped according to number of source words covered
+ private List<Stack> stacks;
+
+ // The end state
+ private Hypothesis end;
+
+ List<FeatureFunction> featureFunctions;
+
+ private Sentence sentence;
+
+ private JoshuaConfiguration config;
+
+ /* Contains all the phrase tables */
+ private PhraseChart chart;
+
+ /**
+ * Entry point. Initialize everything. Create pass-through (OOV) phrase table and glue phrase
+ * table (with start-of-sentence and end-of-sentence rules).
+ *
+ * @param sentence input to {@link org.apache.joshua.lattice.Lattice}
+ * @param featureFunctions {@link java.util.List} of {@link org.apache.joshua.decoder.ff.FeatureFunction}'s
+ * @param grammars an array of {@link org.apache.joshua.decoder.ff.tm.Grammar}'s
+ * @param config a populated {@link org.apache.joshua.decoder.JoshuaConfiguration}
+ */
+ public Stacks(Sentence sentence, List<FeatureFunction> featureFunctions, Grammar[] grammars,
+ JoshuaConfiguration config) {
+
+ this.sentence = sentence;
+ this.featureFunctions = featureFunctions;
+ this.config = config;
+
+ int num_phrase_tables = 0;
+ for (int i = 0; i < grammars.length; i++)
+ if (grammars[i] instanceof PhraseTable)
+ ++num_phrase_tables;
+
+ PhraseTable[] phraseTables = new PhraseTable[num_phrase_tables + 2];
+ for (int i = 0, j = 0; i < grammars.length; i++)
+ if (grammars[i] instanceof PhraseTable)
+ phraseTables[j++] = (PhraseTable) grammars[i];
+
- phraseTables[phraseTables.length - 2] = new PhraseTable("null", config);
++ phraseTables[phraseTables.length - 2] = new PhraseTable(UNKNOWN_OWNER, config);
+ phraseTables[phraseTables.length - 2].addRule(Hypothesis.END_RULE);
+
+ phraseTables[phraseTables.length - 1] = new PhraseTable("oov", config);
+ AbstractGrammar.addOOVRules(phraseTables[phraseTables.length - 1], sentence.getLattice(), featureFunctions, config.true_oovs_only);
+
+ this.chart = new PhraseChart(phraseTables, featureFunctions, sentence, config.num_translation_options);
+ }
+
+
+ /**
+ * The main algorithm. Returns a hypergraph representing the search space.
+ *
+ * @return a {@link org.apache.joshua.decoder.hypergraph.HyperGraph} representing the search space
+ */
+ public HyperGraph search() {
+
+ long startTime = System.currentTimeMillis();
+
+ Future future = new Future(chart);
+ stacks = new ArrayList<Stack>();
+
+ // <s> counts as the first word. Pushing null lets us count from one.
+ stacks.add(null);
+
+ // Initialize root hypothesis with <s> context and future cost for everything.
+ ComputeNodeResult result = new ComputeNodeResult(this.featureFunctions, Hypothesis.BEGIN_RULE,
+ null, -1, 1, null, this.sentence);
+ Stack firstStack = new Stack(featureFunctions, sentence, config);
+ firstStack.add(new Hypothesis(result.getDPStates(), future.Full()));
+ stacks.add(firstStack);
+
+ // Decode with increasing numbers of source words.
+ for (int source_words = 2; source_words <= sentence.length(); ++source_words) {
+ Stack targetStack = new Stack(featureFunctions, sentence, config);
+ stacks.add(targetStack);
+
+ // Iterate over stacks to continue from.
+ for (int phrase_length = 1; phrase_length <= Math.min(source_words - 1, chart.MaxSourcePhraseLength());
+ phrase_length++) {
+ int from_stack = source_words - phrase_length;
+ Stack tailStack = stacks.get(from_stack);
+
+ LOG.debug("WORDS {} MAX {} (STACK {} phrase_length {})", source_words,
+ chart.MaxSourcePhraseLength(), from_stack, phrase_length);
+
+ // Iterate over antecedents in this stack.
+ for (Coverage coverage: tailStack.getCoverages()) {
+ ArrayList<Hypothesis> hypotheses = tailStack.get(coverage);
+
+ // the index of the starting point of the first possible phrase
+ int begin = coverage.firstZero();
+
+ // the absolute position of the ending spot of the last possible phrase
+ int last_end = Math.min(coverage.firstZero() + config.reordering_limit, chart.SentenceLength());
+ int last_begin = (last_end > phrase_length) ? (last_end - phrase_length) : 0;
+
+ for (begin = coverage.firstZero(); begin <= last_begin; begin++) {
+ if (!coverage.compatible(begin, begin + phrase_length) ||
+ ! permissible(coverage, begin, begin + phrase_length)) {
+ continue;
+ }
+
+ Span span = new Span(begin, begin + phrase_length);
+
+ // Don't append </s> until the end
+ if (begin == sentence.length() - 1 && source_words != sentence.length())
+ continue;
+
+ TargetPhrases phrases = chart.getRange(begin, begin + phrase_length);
+ if (phrases == null)
+ continue;
+
+
+ LOG.debug("Applying {} target phrases over [{}, {}]",
+ phrases.size(), begin, begin + phrase_length);
+
+ // TODO: could also compute some number of features here (e.g., non-LM ones)
+ // float score_delta = context.GetScorer().transition(ant, phrases, begin, begin + phrase_length);
+
+ // Future costs: remove span to be filled.
+ float future_delta = future.Change(coverage, begin, begin + phrase_length);
+
+ /* This associates with each span a set of hypotheses that can be extended by
+ * phrases from that span. The hypotheses are wrapped in HypoState objects, which
+ * augment the hypothesis score with a future cost.
+ */
+ Candidate cand = new Candidate(hypotheses, phrases, span, future_delta);
+ targetStack.addCandidate(cand);
+ }
+ }
+ }
+
+ /* At this point, every vertex contains a list of all existing hypotheses that the target
+ * phrases in that vertex could extend. Now we need to create the search object, which
+ * implements cube pruning. There are up to O(n^2) cubes, n the size of the current stack,
+ * one cube each over each span of the input. Each "cube" has two dimensions: one representing
+ * the target phrases over the span, and one representing all of these incoming hypotheses.
+ * We seed the chart with the best item in each cube, and then repeatedly pop and extend.
+ */
+
+ // System.err.println(String.format("\nBuilding cube-pruning chart for %d words", source_words));
+
+ targetStack.search();
+ }
+
+ LOG.info("Input {}: Search took {} seconds", sentence.id(),
+ (System.currentTimeMillis() - startTime) / 1000.0f);
+
+ return createGoalNode();
+ }
+
+ /**
+ * Enforces reordering constraints. Our version of Moses' ReorderingConstraint::Check() and
+ * SearchCubePruning::CheckDistortion().
+ *
+ * @param coverage
+ * @param begin
+ * @param i
+ * @return
+ */
+ private boolean permissible(Coverage coverage, int begin, int end) {
+ int firstZero = coverage.firstZero();
+
+ if (config.reordering_limit < 0)
+ return true;
+
+ /* We can always start with the first zero since it doesn't create a reordering gap
+ */
+ if (begin == firstZero)
+ return true;
+
+ /* If a gap is created by applying this phrase, make sure that you can reach the first
+ * zero later on without violating the distortion constraint.
+ */
+ if (end - firstZero > config.reordering_limit) {
+ return false;
+ }
+
+ return true;
+ }
+
+
+ /**
+ * Searches through the goal stack, calling the final transition function on each node, and then returning
+ * the best item. Usually the final transition code doesn't add anything, because all features
+ * have already computed everything they need to. The standard exception is language models that
+ * have not yet computed their prefix probabilities (which is not the case with KenLM, the default).
+ *
+ * @return
+ */
+ private HyperGraph createGoalNode() {
+ Stack lastStack = stacks.get(sentence.length());
+
+ for (Hypothesis hyp: lastStack) {
+ float score = hyp.getScore();
+ List<HGNode> tailNodes = new ArrayList<HGNode>();
+ tailNodes.add(hyp);
+
+ float finalTransitionScore = ComputeNodeResult.computeFinalCost(featureFunctions, tailNodes, 0, sentence.length(), null, sentence);
+
+ if (null == this.end)
+ this.end = new Hypothesis(null, score + finalTransitionScore, hyp, sentence.length(), null);
+
+ HyperEdge edge = new HyperEdge(null, score + finalTransitionScore, finalTransitionScore, tailNodes, null);
+ end.addHyperedgeInNode(edge);
+ }
+
+ return new HyperGraph(end, -1, -1, this.sentence);
+ }
+ }
[18/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/zmert/MertCore.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/zmert/MertCore.java b/joshua-core/src/main/java/org/apache/joshua/zmert/MertCore.java
new file mode 100644
index 0000000..c0d470d
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/zmert/MertCore.java
@@ -0,0 +1,3191 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.zmert;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutputStream;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.text.DecimalFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Random;
+import java.util.Scanner;
+import java.util.TreeSet;
+import java.util.Vector;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Semaphore;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+
+import org.apache.joshua.decoder.Decoder;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.metrics.EvaluationMetric;
+import org.apache.joshua.util.StreamGobbler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This code was originally written by Omar Zaidan. In September of 2012, it was augmented to support
+ * a sparse feature implementation.
+ *
+ * @author Omar Zaidan
+ */
+
+public class MertCore {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MertCore.class);
+
+ private final JoshuaConfiguration joshuaConfiguration;
+ private TreeSet<Integer>[] indicesOfInterest_all;
+
+ private final static DecimalFormat f4 = new DecimalFormat("###0.0000");
+ private final Runtime myRuntime = Runtime.getRuntime();
+
+ private final static double NegInf = (-1.0 / 0.0);
+ private final static double PosInf = (+1.0 / 0.0);
+ private final static double epsilon = 1.0 / 1000000;
+
+ private int verbosity; // anything of priority <= verbosity will be printed
+ // (lower value for priority means more important)
+
+ private Random randGen;
+ private int generatedRands;
+
+ private int numSentences;
+ // number of sentences in the dev set
+ // (aka the "MERT training" set)
+
+ private int numDocuments;
+ // number of documents in the dev set
+ // this should be 1, unless doing doc-level optimization
+
+ private int[] docOfSentence;
+ // docOfSentence[i] stores which document contains the i'th sentence.
+ // docOfSentence is 0-indexed, as are the documents (i.e. first doc is indexed 0)
+
+ private int[] docSubsetInfo;
+ // stores information regarding which subset of the documents are evaluated
+ // [0]: method (0-6)
+ // [1]: first (1-indexed)
+ // [2]: last (1-indexed)
+ // [3]: size
+ // [4]: center
+ // [5]: arg1
+ // [6]: arg2
+ // [1-6] are 0 for method 0, [6] is 0 for methods 1-4 as well
+ // only [1] and [2] are needed for optimization. The rest are only needed for an output message.
+
+ private int refsPerSen;
+ // number of reference translations per sentence
+
+ private int textNormMethod;
+ // 0: no normalization, 1: "NIST-style" tokenization, and also rejoin 'm, 're, *'s, 've, 'll, 'd,
+ // and n't,
+ // 2: apply 1 and also rejoin dashes between letters, 3: apply 1 and also drop non-ASCII
+ // characters
+ // 4: apply 1+2+3
+
+ private int numParams;
+ // number of features for the log-linear model
+
+ private double[] normalizationOptions;
+ // How should a lambda[] vector be normalized (before decoding)?
+ // nO[0] = 0: no normalization
+ // nO[0] = 1: scale so that parameter nO[2] has absolute value nO[1]
+ // nO[0] = 2: scale so that the maximum absolute value is nO[1]
+ // nO[0] = 3: scale so that the minimum absolute value is nO[1]
+ // nO[0] = 4: scale so that the L-nO[1] norm equals nO[2]
+
+ /* *********************************************************** */
+ /* NOTE: indexing starts at 1 in the following few arrays: */
+ /* *********************************************************** */
+
+ private String[] paramNames;
+ // feature names, needed to read/create config file
+
+ private double[] lambda;
+ // the current weight vector. NOTE: indexing starts at 1.
+
+ private boolean[] isOptimizable;
+ // isOptimizable[c] = true iff lambda[c] should be optimized
+
+ private double[] minThValue;
+ private double[] maxThValue;
+ // when investigating thresholds along the lambda[c] dimension, only values
+ // in the [minThValue[c],maxThValue[c]] range will be considered.
+ // (*) minThValue and maxThValue can be real values as well as -Infinity and +Infinity
+ // (coded as -Inf and +Inf, respectively, in an input file)
+
+ private double[] minRandValue;
+ private double[] maxRandValue;
+ // when choosing a random value for the lambda[c] parameter, it will be
+ // chosen from the [minRandValue[c],maxRandValue[c]] range.
+ // (*) minRandValue and maxRandValue must be real values, but not -Inf or +Inf
+
+ private int damianos_method;
+ private double damianos_param;
+ private double damianos_mult;
+
+ private double[] defaultLambda;
+ // "default" parameter values; simply the values read in the parameter file
+
+ /* *********************************************************** */
+ /* *********************************************************** */
+
+ private Decoder myDecoder;
+ // COMMENT OUT if decoder is not Joshua
+
+ private String decoderCommand;
+ // the command that runs the decoder; read from decoderCommandFileName
+
+ private int decVerbosity;
+ // verbosity level for decoder output. If 0, decoder output is ignored.
+ // If 1, decoder output is printed.
+
+ private int validDecoderExitValue;
+ // return value from running the decoder command that indicates success
+
+ private int numOptThreads;
+ // number of threads to run things in parallel
+
+ private int saveInterFiles;
+ // 0: nothing, 1: only configs, 2: only n-bests, 3: both configs and n-bests
+
+ private int compressFiles;
+ // should Z-MERT gzip the large files? If 0, no compression takes place.
+ // If 1, compression is performed on: decoder output files, temp sents files,
+ // and temp feats files.
+
+ private int sizeOfNBest;
+ // size of N-best list generated by decoder at each iteration
+ // (aka simply N, but N is a bad variable name)
+
+ private long seed;
+ // seed used to create random number generators
+
+ private boolean randInit;
+ // if true, parameters are initialized randomly. If false, parameters
+ // are initialized using values from parameter file.
+
+ private int initsPerIt;
+ // number of intermediate initial points per iteration
+
+ private int maxMERTIterations, minMERTIterations, prevMERTIterations;
+ // max: maximum number of MERT iterations
+ // min: minimum number of MERT iterations before an early MERT exit
+ // prev: number of previous MERT iterations from which to consider candidates (in addition to
+ // the candidates from the current iteration)
+
+ private double stopSigValue;
+ // early MERT exit if no weight changes by more than stopSigValue
+ // (but see minMERTIterations above and stopMinIts below)
+
+ private int stopMinIts;
+ // some early stopping criterion must be satisfied in stopMinIts *consecutive* iterations
+ // before an early exit (but see minMERTIterations above)
+
+ private boolean oneModificationPerIteration;
+ // if true, each MERT iteration performs at most one parameter modification.
+ // If false, a new MERT iteration starts (i.e. a new N-best list is
+ // generated) only after the previous iteration reaches a local maximum.
+
+ private String metricName;
+ // name of evaluation metric optimized by MERT
+
+ private String metricName_display;
+ // name of evaluation metric optimized by MERT, possibly with "doc-level " prefixed
+
+ private String[] metricOptions;
+ // options for the evaluation metric (e.g. for BLEU, maxGramLength and effLengthMethod)
+
+ private EvaluationMetric evalMetric;
+ // the evaluation metric used by MERT
+
+ private int suffStatsCount;
+ // number of sufficient statistics for the evaluation metric
+
+ private String tmpDirPrefix;
+ // prefix for the ZMERT.temp.* files
+
+ private boolean passIterationToDecoder;
+ // should the iteration number be passed as an argument to decoderCommandFileName?
+ // If 1, iteration number is passed. If 0, launch with no arguments.
+
+ private String dirPrefix; // where are all these files located?
+ private String paramsFileName, docInfoFileName, finalLambdaFileName;
+ private String sourceFileName, refFileName, decoderOutFileName;
+ private String decoderConfigFileName, decoderCommandFileName;
+ private String fakeFileNameTemplate, fakeFileNamePrefix, fakeFileNameSuffix;
+
+ // e.g. output.it[1-x].someOldRun would be specified as:
+ // output.it?.someOldRun
+ // and we'd have prefix = "output.it" and suffix = ".sameOldRun"
+
+ // private int useDisk;
+
+ public MertCore(JoshuaConfiguration joshuaConfiguration)
+ {
+ this.joshuaConfiguration = joshuaConfiguration;
+ }
+
+ public MertCore(String[] args, JoshuaConfiguration joshuaConfiguration) {
+ this.joshuaConfiguration = joshuaConfiguration;
+ EvaluationMetric.set_knownMetrics();
+ processArgsArray(args);
+ initialize(0);
+ }
+
+ public MertCore(String configFileName,JoshuaConfiguration joshuaConfiguration) {
+ this.joshuaConfiguration = joshuaConfiguration;
+ EvaluationMetric.set_knownMetrics();
+ processArgsArray(cfgFileToArgsArray(configFileName));
+ initialize(0);
+ }
+
+ private void initialize(int randsToSkip) {
+ println("NegInf: " + NegInf + ", PosInf: " + PosInf + ", epsilon: " + epsilon, 4);
+
+ randGen = new Random(seed);
+ for (int r = 1; r <= randsToSkip; ++r) {
+ randGen.nextDouble();
+ }
+ generatedRands = randsToSkip;
+
+ if (randsToSkip == 0) {
+ println("----------------------------------------------------", 1);
+ println("Initializing...", 1);
+ println("----------------------------------------------------", 1);
+ println("", 1);
+
+ println("Random number generator initialized using seed: " + seed, 1);
+ println("", 1);
+ }
+
+ if (refsPerSen > 1) {
+ String refFile = refFileName + "0";
+ if (! new File(refFile).exists())
+ refFile = refFileName + ".0";
+ if (! new File(refFile).exists()) {
+ throw new RuntimeException(String.format("* FATAL: can't find first reference file '%s{0,.0}'", refFileName));
+ }
+
+ numSentences = countLines(refFile);
+ } else {
+ numSentences = countLines(refFileName);
+ }
+
+ processDocInfo();
+ // sets numDocuments and docOfSentence[]
+
+ if (numDocuments > 1) metricName_display = "doc-level " + metricName;
+
+ set_docSubsetInfo(docSubsetInfo);
+
+
+
+ numParams = countNonEmptyLines(paramsFileName) - 1;
+ // the parameter file contains one line per parameter
+ // and one line for the normalization method
+
+
+ paramNames = new String[1 + numParams];
+ lambda = new double[1 + numParams]; // indexing starts at 1 in these arrays
+ isOptimizable = new boolean[1 + numParams];
+ minThValue = new double[1 + numParams];
+ maxThValue = new double[1 + numParams];
+ minRandValue = new double[1 + numParams];
+ maxRandValue = new double[1 + numParams];
+ // precision = new double[1+numParams];
+ defaultLambda = new double[1 + numParams];
+ normalizationOptions = new double[3];
+
+ try {
+ // read parameter names
+ BufferedReader inFile_names = new BufferedReader(new FileReader(paramsFileName));
+
+ for (int c = 1; c <= numParams; ++c) {
+ String line = "";
+ while (line != null && line.length() == 0) { // skip empty lines
+ line = inFile_names.readLine();
+ }
+ String paramName = (line.substring(0, line.indexOf("|||"))).trim();
+ paramNames[c] = paramName;
+ }
+
+ inFile_names.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ processParamFile();
+ // sets the arrays declared just above
+
+ // SentenceInfo.createV(); // uncomment ONLY IF using vocabulary implementation of SentenceInfo
+
+
+ String[][] refSentences = new String[numSentences][refsPerSen];
+
+ try {
+
+ // read in reference sentences
+ BufferedReader reference_readers[] = new BufferedReader[refsPerSen];
+ if (refsPerSen == 1) {
+ reference_readers[0] = new BufferedReader(new InputStreamReader(new FileInputStream(new File(refFileName)), "utf8"));
+ } else {
+ for (int i = 0; i < refsPerSen; i++) {
+ String refFile = refFileName + i;
+ if (! new File(refFile).exists())
+ refFile = refFileName + "." + i;
+ if (! new File(refFile).exists()) {
+ throw new RuntimeException(String.format("* FATAL: can't find reference file '%s'", refFile));
+ }
+
+ reference_readers[i] = new BufferedReader(new InputStreamReader(new FileInputStream(new File(refFile)), "utf8"));
+ }
+ }
+
+ for (int i = 0; i < numSentences; ++i) {
+ for (int r = 0; r < refsPerSen; ++r) {
+ // read the rth reference translation for the ith sentence
+ refSentences[i][r] = normalize(reference_readers[r].readLine(), textNormMethod);
+ }
+ }
+
+ // close all the reference files
+ for (int i = 0; i < refsPerSen; i++)
+ reference_readers[i].close();
+
+ // read in decoder command, if any
+ decoderCommand = null;
+ if (decoderCommandFileName != null) {
+ if (fileExists(decoderCommandFileName)) {
+ BufferedReader inFile_comm = new BufferedReader(new FileReader(decoderCommandFileName));
+ decoderCommand = inFile_comm.readLine();
+ inFile_comm.close();
+ }
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+
+ // set static data members for the EvaluationMetric class
+ EvaluationMetric.set_numSentences(numSentences);
+ EvaluationMetric.set_numDocuments(numDocuments);
+ EvaluationMetric.set_refsPerSen(refsPerSen);
+ EvaluationMetric.set_refSentences(refSentences);
+ EvaluationMetric.set_tmpDirPrefix(tmpDirPrefix);
+
+ evalMetric = EvaluationMetric.getMetric(metricName, metricOptions);
+
+ suffStatsCount = evalMetric.get_suffStatsCount();
+
+ // set static data members for the IntermediateOptimizer class
+ IntermediateOptimizer.set_MERTparams(numSentences, numDocuments, docOfSentence, docSubsetInfo,
+ numParams, normalizationOptions, isOptimizable, minThValue, maxThValue,
+ oneModificationPerIteration, evalMetric, tmpDirPrefix, verbosity);
+
+
+
+ if (randsToSkip == 0) { // i.e. first iteration
+ println("Number of sentences: " + numSentences, 1);
+ println("Number of documents: " + numDocuments, 1);
+ println("Optimizing " + metricName_display, 1);
+
+ print("docSubsetInfo: {", 1);
+ for (int f = 0; f < 6; ++f)
+ print(docSubsetInfo[f] + ", ", 1);
+ println(docSubsetInfo[6] + "}", 1);
+
+ println("Number of features: " + numParams, 1);
+ print("Feature names: {", 1);
+ for (int c = 1; c <= numParams; ++c) {
+ print("\"" + paramNames[c] + "\"", 1);
+ if (c < numParams) print(",", 1);
+ }
+ println("}", 1);
+ println("", 1);
+
+ println("c Default value\tOptimizable?\tCrit. val. range\tRand. val. range", 1);
+
+ for (int c = 1; c <= numParams; ++c) {
+ print(c + " " + f4.format(lambda[c]) + "\t\t", 1);
+ if (!isOptimizable[c]) {
+ println(" No", 1);
+ } else {
+ print(" Yes\t\t", 1);
+ // print("[" + minThValue[c] + "," + maxThValue[c] + "] @ " + precision[c] +
+ // " precision",1);
+ print(" [" + minThValue[c] + "," + maxThValue[c] + "]", 1);
+ print("\t\t", 1);
+ print(" [" + minRandValue[c] + "," + maxRandValue[c] + "]", 1);
+ println("", 1);
+ }
+ }
+
+ println("", 1);
+ print("Weight vector normalization method: ", 1);
+ if (normalizationOptions[0] == 0) {
+ println("none.", 1);
+ } else if (normalizationOptions[0] == 1) {
+ println("weights will be scaled so that the \"" + paramNames[(int) normalizationOptions[1]]
+ + "\" weight has an absolute value of " + normalizationOptions[2] + ".", 1);
+ } else if (normalizationOptions[0] == 2) {
+ println("weights will be scaled so that the maximum absolute value is "
+ + normalizationOptions[1] + ".", 1);
+ } else if (normalizationOptions[0] == 3) {
+ println("weights will be scaled so that the minimum absolute value is "
+ + normalizationOptions[1] + ".", 1);
+ } else if (normalizationOptions[0] == 4) {
+ println("weights will be scaled so that the L-" + normalizationOptions[1] + " norm is "
+ + normalizationOptions[2] + ".", 1);
+ }
+
+ println("", 1);
+
+ println("----------------------------------------------------", 1);
+ println("", 1);
+
+ // rename original config file so it doesn't get overwritten
+ // (original name will be restored in finish())
+ renameFile(decoderConfigFileName, decoderConfigFileName + ".ZMERT.orig");
+
+ } // if (randsToSkip == 0)
+
+
+ if (decoderCommand == null && fakeFileNameTemplate == null) {
+ println("Loading Joshua decoder...", 1);
+ myDecoder = new Decoder(joshuaConfiguration, decoderConfigFileName + ".ZMERT.orig");
+ println("...finished loading @ " + (new Date()), 1);
+ println("");
+ } else {
+ myDecoder = null;
+ }
+
+
+
+ @SuppressWarnings("unchecked")
+ TreeSet<Integer>[] temp_TSA = new TreeSet[numSentences];
+ indicesOfInterest_all = temp_TSA;
+
+ for (int i = 0; i < numSentences; ++i) {
+ indicesOfInterest_all[i] = new TreeSet<Integer>();
+ }
+
+
+ } // void initialize(...)
+
+ public void run_MERT() {
+ run_MERT(minMERTIterations, maxMERTIterations, prevMERTIterations);
+ }
+
+ public void run_MERT(int minIts, int maxIts, int prevIts) {
+ println("----------------------------------------------------", 1);
+ println("Z-MERT run started @ " + (new Date()), 1);
+ // printMemoryUsage();
+ println("----------------------------------------------------", 1);
+ println("", 1);
+
+ if (randInit) {
+ println("Initializing lambda[] randomly.", 1);
+
+ // initialize optimizable parameters randomly (sampling uniformly from
+ // that parameter's random value range)
+ lambda = randomLambda();
+ }
+
+ println("Initial lambda[]: " + lambdaToString(lambda), 1);
+ println("", 1);
+
+ double FINAL_score = evalMetric.worstPossibleScore();
+
+
+ // int[] lastUsedIndex = new int[numSentences];
+ int[] maxIndex = new int[numSentences];
+ // used to grow featVal_array dynamically
+ // HashMap<Integer,int[]>[] suffStats_array = new HashMap[numSentences];
+ // suffStats_array[i] maps candidates of interest for sentence i to an array
+ // storing the sufficient statistics for that candidate
+ for (int i = 0; i < numSentences; ++i) {
+ // lastUsedIndex[i] = -1;
+ maxIndex[i] = sizeOfNBest - 1;
+ // suffStats_array[i] = new HashMap<Integer,int[]>();
+ }
+ /*
+ * double[][][] featVal_array = new double[1+numParams][][]; // indexed by
+ * [param][sentence][candidate] featVal_array[0] = null; // param indexing starts at 1 for (int
+ * c = 1; c <= numParams; ++c) { featVal_array[c] = new double[numSentences][]; for (int i = 0;
+ * i < numSentences; ++i) { featVal_array[c][i] = new double[maxIndex[i]]; // will grow
+ * dynamically as needed } }
+ */
+ int earlyStop = 0;
+ // number of consecutive iteration an early stopping criterion was satisfied
+
+ for (int iteration = 1;; ++iteration) {
+
+ double[] A = run_single_iteration(iteration, minIts, maxIts, prevIts, earlyStop, maxIndex);
+ if (A != null) {
+ FINAL_score = A[0];
+ earlyStop = (int) A[1];
+ if (A[2] == 1) break;
+ } else {
+ break;
+ }
+
+ } // for (iteration)
+
+ println("", 1);
+
+ println("----------------------------------------------------", 1);
+ println("Z-MERT run ended @ " + (new Date()), 1);
+ // printMemoryUsage();
+ println("----------------------------------------------------", 1);
+ println("", 1);
+ println("FINAL lambda: " + lambdaToString(lambda) + " (" + metricName_display + ": "
+ + FINAL_score + ")", 1);
+ // check if a lambda is outside its threshold range
+ for (int c = 1; c <= numParams; ++c) {
+ if (lambda[c] < minThValue[c] || lambda[c] > maxThValue[c]) {
+ println("Warning: after normalization, lambda[" + c + "]=" + f4.format(lambda[c])
+ + " is outside its critical value range.", 1);
+ }
+ }
+ println("", 1);
+
+ // delete intermediate .temp.*.it* decoder output files
+ for (int iteration = 1; iteration <= maxIts; ++iteration) {
+ if (compressFiles == 1) {
+ deleteFile(tmpDirPrefix + "temp.sents.it" + iteration + ".gz");
+ deleteFile(tmpDirPrefix + "temp.feats.it" + iteration + ".gz");
+ if (fileExists(tmpDirPrefix + "temp.stats.it" + iteration + ".copy.gz")) {
+ deleteFile(tmpDirPrefix + "temp.stats.it" + iteration + ".copy.gz");
+ } else {
+ deleteFile(tmpDirPrefix + "temp.stats.it" + iteration + ".gz");
+ }
+ } else {
+ deleteFile(tmpDirPrefix + "temp.sents.it" + iteration);
+ deleteFile(tmpDirPrefix + "temp.feats.it" + iteration);
+ if (fileExists(tmpDirPrefix + "temp.stats.it" + iteration + ".copy")) {
+ deleteFile(tmpDirPrefix + "temp.stats.it" + iteration + ".copy");
+ } else {
+ deleteFile(tmpDirPrefix + "temp.stats.it" + iteration);
+ }
+ }
+ }
+
+ } // void run_MERT(int maxIts)
+
+
+ @SuppressWarnings("unchecked")
+ public double[] run_single_iteration(int iteration, int minIts, int maxIts, int prevIts,
+ int earlyStop, int[] maxIndex) {
+ double FINAL_score = 0;
+
+ double[] retA = new double[3];
+ // retA[0]: FINAL_score
+ // retA[1]: earlyStop
+ // retA[2]: should this be the last iteration?
+
+ boolean done = false;
+ retA[2] = 1; // will only be made 0 if we don't break from the following loop
+
+
+ double[][][] featVal_array = new double[1 + numParams][][];
+ // indexed by [param][sentence][candidate]
+ featVal_array[0] = null; // param indexing starts at 1
+ for (int c = 1; c <= numParams; ++c) {
+ featVal_array[c] = new double[numSentences][];
+ for (int i = 0; i < numSentences; ++i) {
+ featVal_array[c][i] = new double[maxIndex[i] + 1];
+ // will grow dynamically as needed
+ }
+ }
+
+
+ while (!done) { // NOTE: this "loop" will only be carried out once
+ println("--- Starting Z-MERT iteration #" + iteration + " @ " + (new Date()) + " ---", 1);
+
+ // printMemoryUsage();
+
+ // run the decoder on all the sentences, producing for each sentence a set of
+ // sizeOfNBest candidates, with numParams feature values for each candidate
+
+ /******************************/
+ // CREATE DECODER CONFIG FILE //
+ /******************************/
+
+ createConfigFile(lambda, decoderConfigFileName, decoderConfigFileName + ".ZMERT.orig");
+ // i.e. use the original config file as a template
+
+ /***************/
+ // RUN DECODER //
+ /***************/
+
+ if (iteration == 1) {
+ println("Decoding using initial weight vector " + lambdaToString(lambda), 1);
+ } else {
+ println("Redecoding using weight vector " + lambdaToString(lambda), 1);
+ }
+
+ String[] decRunResult = run_decoder(iteration); // iteration passed in case fake decoder will
+ // be used
+ // [0] name of file to be processed
+ // [1] indicates how the output file was obtained:
+ // 1: external decoder
+ // 2: fake decoder
+ // 3: internal decoder
+
+ if (!decRunResult[1].equals("2")) {
+ println("...finished decoding @ " + (new Date()), 1);
+ }
+
+ checkFile(decRunResult[0]);
+
+ println("Producing temp files for iteration " + iteration, 3);
+
+ produceTempFiles(decRunResult[0], iteration);
+
+ if (saveInterFiles == 1 || saveInterFiles == 3) { // make copy of intermediate config file
+ if (!copyFile(decoderConfigFileName, decoderConfigFileName + ".ZMERT.it" + iteration)) {
+ println("Warning: attempt to make copy of decoder config file (to create"
+ + decoderConfigFileName + ".ZMERT.it" + iteration + ") was unsuccessful!", 1);
+ }
+ }
+ if (saveInterFiles == 2 || saveInterFiles == 3) { // make copy of intermediate decoder output
+ // file...
+
+ if (!decRunResult[1].equals("2")) { // ...but only if no fake decoder
+ if (!decRunResult[0].endsWith(".gz")) {
+ if (!copyFile(decRunResult[0], decRunResult[0] + ".ZMERT.it" + iteration)) {
+ println("Warning: attempt to make copy of decoder output file (to create"
+ + decRunResult[0] + ".ZMERT.it" + iteration + ") was unsuccessful!", 1);
+ }
+ } else {
+ String prefix = decRunResult[0].substring(0, decRunResult[0].length() - 3);
+ if (!copyFile(prefix + ".gz", prefix + ".ZMERT.it" + iteration + ".gz")) {
+ println("Warning: attempt to make copy of decoder output file (to create" + prefix
+ + ".ZMERT.it" + iteration + ".gz" + ") was unsuccessful!", 1);
+ }
+ }
+
+ if (compressFiles == 1 && !decRunResult[0].endsWith(".gz")) {
+ gzipFile(decRunResult[0] + ".ZMERT.it" + iteration);
+ }
+ } // if (!fake)
+
+ }
+
+ int[] candCount = new int[numSentences];
+ int[] lastUsedIndex = new int[numSentences];
+ ConcurrentHashMap<Integer, int[]>[] suffStats_array = new ConcurrentHashMap[numSentences];
+ for (int i = 0; i < numSentences; ++i) {
+ candCount[i] = 0;
+ lastUsedIndex[i] = -1;
+ // suffStats_array[i].clear();
+ suffStats_array[i] = new ConcurrentHashMap<Integer, int[]>();
+ }
+
+ double[][] initialLambda = new double[1 + initsPerIt][1 + numParams];
+ // the intermediate "initial" lambdas
+ double[][] finalLambda = new double[1 + initsPerIt][1 + numParams];
+ // the intermediate "final" lambdas
+
+ // set initialLambda[][]
+ System.arraycopy(lambda, 1, initialLambda[1], 1, numParams);
+ for (int j = 2; j <= initsPerIt; ++j) {
+ if (damianos_method == 0) {
+ initialLambda[j] = randomLambda();
+ } else {
+ initialLambda[j] =
+ randomPerturbation(initialLambda[1], iteration, damianos_method, damianos_param,
+ damianos_mult);
+ }
+ }
+
+// double[] initialScore = new double[1 + initsPerIt];
+ double[] finalScore = new double[1 + initsPerIt];
+
+ int[][][] best1Cand_suffStats = new int[1 + initsPerIt][numSentences][suffStatsCount];
+ double[][] best1Score = new double[1 + initsPerIt][numSentences];
+ // Those two arrays are used to calculate initialScore[]
+ // (the "score" in best1Score refers to that assigned by the
+ // decoder; the "score" in initialScore refers to that
+ // assigned by the evaluation metric)
+
+ int firstIt = Math.max(1, iteration - prevIts);
+ // i.e. only process candidates from the current iteration and candidates
+ // from up to prevIts previous iterations.
+ println("Reading candidate translations from iterations " + firstIt + "-" + iteration, 1);
+ println("(and computing " + metricName
+ + " sufficient statistics for previously unseen candidates)", 1);
+ print(" Progress: ");
+
+ int[] newCandidatesAdded = new int[1 + iteration];
+ for (int it = 1; it <= iteration; ++it) {
+ newCandidatesAdded[it] = 0;
+ }
+
+
+
+ try {
+
+ // each inFile corresponds to the output of an iteration
+ // (index 0 is not used; no corresponding index for the current iteration)
+ BufferedReader[] inFile_sents = new BufferedReader[iteration];
+ BufferedReader[] inFile_feats = new BufferedReader[iteration];
+ BufferedReader[] inFile_stats = new BufferedReader[iteration];
+
+ for (int it = firstIt; it < iteration; ++it) {
+ InputStream inStream_sents, inStream_feats, inStream_stats;
+ if (compressFiles == 0) {
+ inStream_sents = new FileInputStream(tmpDirPrefix + "temp.sents.it" + it);
+ inStream_feats = new FileInputStream(tmpDirPrefix + "temp.feats.it" + it);
+ inStream_stats = new FileInputStream(tmpDirPrefix + "temp.stats.it" + it);
+ } else {
+ inStream_sents =
+ new GZIPInputStream(
+ new FileInputStream(tmpDirPrefix + "temp.sents.it" + it + ".gz"));
+ inStream_feats =
+ new GZIPInputStream(
+ new FileInputStream(tmpDirPrefix + "temp.feats.it" + it + ".gz"));
+ inStream_stats =
+ new GZIPInputStream(
+ new FileInputStream(tmpDirPrefix + "temp.stats.it" + it + ".gz"));
+ }
+
+ inFile_sents[it] = new BufferedReader(new InputStreamReader(inStream_sents, "utf8"));
+ inFile_feats[it] = new BufferedReader(new InputStreamReader(inStream_feats, "utf8"));
+ inFile_stats[it] = new BufferedReader(new InputStreamReader(inStream_stats, "utf8"));
+ }
+
+
+ InputStream inStream_sentsCurrIt, inStream_featsCurrIt, inStream_statsCurrIt;
+ if (compressFiles == 0) {
+ inStream_sentsCurrIt = new FileInputStream(tmpDirPrefix + "temp.sents.it" + iteration);
+ inStream_featsCurrIt = new FileInputStream(tmpDirPrefix + "temp.feats.it" + iteration);
+ } else {
+ inStream_sentsCurrIt =
+ new GZIPInputStream(new FileInputStream(tmpDirPrefix + "temp.sents.it" + iteration
+ + ".gz"));
+ inStream_featsCurrIt =
+ new GZIPInputStream(new FileInputStream(tmpDirPrefix + "temp.feats.it" + iteration
+ + ".gz"));
+ }
+
+ BufferedReader inFile_sentsCurrIt =
+ new BufferedReader(new InputStreamReader(inStream_sentsCurrIt, "utf8"));
+ BufferedReader inFile_featsCurrIt =
+ new BufferedReader(new InputStreamReader(inStream_featsCurrIt, "utf8"));
+
+ BufferedReader inFile_statsCurrIt = null; // will only be used if statsCurrIt_exists below
+ // is set to true
+ PrintWriter outFile_statsCurrIt = null; // will only be used if statsCurrIt_exists below is
+ // set to false
+ boolean statsCurrIt_exists = false;
+ if (fileExists(tmpDirPrefix + "temp.stats.it" + iteration)) {
+ inStream_statsCurrIt = new FileInputStream(tmpDirPrefix + "temp.stats.it" + iteration);
+ inFile_statsCurrIt =
+ new BufferedReader(new InputStreamReader(inStream_statsCurrIt, "utf8"));
+ statsCurrIt_exists = true;
+ copyFile(tmpDirPrefix + "temp.stats.it" + iteration, tmpDirPrefix + "temp.stats.it"
+ + iteration + ".copy");
+ } else if (fileExists(tmpDirPrefix + "temp.stats.it" + iteration + ".gz")) {
+ inStream_statsCurrIt =
+ new GZIPInputStream(new FileInputStream(tmpDirPrefix + "temp.stats.it" + iteration
+ + ".gz"));
+ inFile_statsCurrIt =
+ new BufferedReader(new InputStreamReader(inStream_statsCurrIt, "utf8"));
+ statsCurrIt_exists = true;
+ copyFile(tmpDirPrefix + "temp.stats.it" + iteration + ".gz", tmpDirPrefix
+ + "temp.stats.it" + iteration + ".copy.gz");
+ } else {
+ outFile_statsCurrIt = new PrintWriter(tmpDirPrefix + "temp.stats.it" + iteration);
+ }
+
+ PrintWriter outFile_statsMerged = new PrintWriter(tmpDirPrefix + "temp.stats.merged");
+ // write sufficient statistics from all the sentences
+ // from the output files into a single file
+ PrintWriter outFile_statsMergedKnown =
+ new PrintWriter(tmpDirPrefix + "temp.stats.mergedKnown");
+ // write sufficient statistics from all the sentences
+ // from the output files into a single file
+
+ FileOutputStream outStream_unknownCands =
+ new FileOutputStream(tmpDirPrefix + "temp.currIt.unknownCands", false);
+ OutputStreamWriter outStreamWriter_unknownCands =
+ new OutputStreamWriter(outStream_unknownCands, "utf8");
+ BufferedWriter outFile_unknownCands = new BufferedWriter(outStreamWriter_unknownCands);
+
+ PrintWriter outFile_unknownIndices =
+ new PrintWriter(tmpDirPrefix + "temp.currIt.unknownIndices");
+
+
+ String sents_str, feats_str, stats_str;
+
+ // BUG: this assumes a candidate string cannot be produced for two
+ // different source sentences, which is not necessarily true
+ // (It's not actually a bug, but only because existingCandStats gets
+ // cleared before moving to the next source sentence.)
+ // FIX: should be made an array, indexed by i
+ HashMap<String, String> existingCandStats = new HashMap<String, String>();
+ // Stores precalculated sufficient statistics for candidates, in case
+ // the same candidate is seen again. (SS stored as a String.)
+ // Q: Why do we care? If we see the same candidate again, aren't we going
+ // to ignore it? So, why do we care about the SS of this repeat candidate?
+ // A: A "repeat" candidate may not be a repeat candidate in later
+ // iterations if the user specifies a value for prevMERTIterations
+ // that causes MERT to skip candidates from early iterations.
+ double[] currFeatVal = new double[1 + numParams];
+ String[] featVal_str;
+
+ int totalCandidateCount = 0;
+
+
+
+ int[] sizeUnknown_currIt = new int[numSentences];
+
+
+
+ for (int i = 0; i < numSentences; ++i) {
+
+ for (int j = 1; j <= initsPerIt; ++j) {
+ best1Score[j][i] = NegInf;
+ }
+
+ for (int it = firstIt; it < iteration; ++it) {
+ // Why up to but *excluding* iteration?
+ // Because the last iteration is handled a little differently, since
+ // the SS must be claculated (and the corresponding file created),
+ // which is not true for previous iterations.
+
+ for (int n = 0; n <= sizeOfNBest; ++n) {
+ // Why up to and *including* sizeOfNBest?
+ // So that it would read the "||||||" separator even if there is
+ // a complete list of sizeOfNBest candidates.
+
+ // for the nth candidate for the ith sentence, read the sentence, feature values,
+ // and sufficient statistics from the various temp files
+
+ sents_str = inFile_sents[it].readLine();
+ feats_str = inFile_feats[it].readLine();
+ stats_str = inFile_stats[it].readLine();
+
+ if (sents_str.equals("||||||")) {
+ n = sizeOfNBest + 1;
+ } else if (!existingCandStats.containsKey(sents_str)) {
+
+ outFile_statsMergedKnown.println(stats_str);
+
+ featVal_str = feats_str.split("\\s+");
+
+ /* Sparse (labeled) feature version */
+ if (feats_str.indexOf('=') != -1) {
+ for (String featurePair: featVal_str) {
+ String[] pair = featurePair.split("=");
+ String name = pair[0];
+ Double value = Double.parseDouble(pair[1]);
+ currFeatVal[c_fromParamName(name)] = value;
+ }
+ } else {
+ for (int c = 1; c <= numParams; ++c) {
+ try {
+ currFeatVal[c] = Double.parseDouble(featVal_str[c - 1]);
+ } catch (Exception e) {
+ currFeatVal[c] = 0.0;
+ }
+ // print("fV[" + c + "]=" + currFeatVal[c] + " ",4);
+ }
+ // println("",4);
+ }
+
+
+ for (int j = 1; j <= initsPerIt; ++j) {
+ double score = 0; // i.e. score assigned by decoder
+ for (int c = 1; c <= numParams; ++c) {
+ score += initialLambda[j][c] * currFeatVal[c];
+ }
+ if (score > best1Score[j][i]) {
+ best1Score[j][i] = score;
+ String[] tempStats = stats_str.split("\\s+");
+ for (int s = 0; s < suffStatsCount; ++s)
+ best1Cand_suffStats[j][i][s] = Integer.parseInt(tempStats[s]);
+ }
+ } // for (j)
+
+ existingCandStats.put(sents_str, stats_str);
+
+ setFeats(featVal_array, i, lastUsedIndex, maxIndex, currFeatVal);
+ candCount[i] += 1;
+
+ newCandidatesAdded[it] += 1;
+
+ } // if unseen candidate
+
+ } // for (n)
+
+ } // for (it)
+
+ outFile_statsMergedKnown.println("||||||");
+
+
+ // now process the candidates of the current iteration
+ // now determine the new candidates of the current iteration
+
+ /*
+ * remember: BufferedReader inFile_sentsCurrIt BufferedReader inFile_featsCurrIt
+ * PrintWriter outFile_statsCurrIt
+ */
+
+ String[] sentsCurrIt_currSrcSent = new String[sizeOfNBest + 1];
+
+ Vector<String> unknownCands_V = new Vector<String>();
+ // which candidates (of the i'th source sentence) have not been seen before
+ // this iteration?
+
+ for (int n = 0; n <= sizeOfNBest; ++n) {
+ // Why up to and *including* sizeOfNBest?
+ // So that it would read the "||||||" separator even if there is
+ // a complete list of sizeOfNBest candidates.
+
+ // for the nth candidate for the ith sentence, read the sentence,
+ // and store it in the sentsCurrIt_currSrcSent array
+
+ sents_str = inFile_sentsCurrIt.readLine();
+ sentsCurrIt_currSrcSent[n] = sents_str; // Note: possibly "||||||"
+
+ if (sents_str.equals("||||||")) {
+ n = sizeOfNBest + 1;
+ } else if (!existingCandStats.containsKey(sents_str)) {
+ unknownCands_V.add(sents_str);
+ writeLine(sents_str, outFile_unknownCands);
+ outFile_unknownIndices.println(i);
+ newCandidatesAdded[iteration] += 1;
+ existingCandStats.put(sents_str, "U"); // i.e. unknown
+ // we add sents_str to avoid duplicate entries in unknownCands_V
+ }
+
+ } // for (n)
+
+
+
+ // now unknownCands_V has the candidates for which we need to calculate
+ // sufficient statistics (for the i'th source sentence)
+ int sizeUnknown = unknownCands_V.size();
+ sizeUnknown_currIt[i] = sizeUnknown;
+
+ /*********************************************/
+ /*
+ * String[] unknownCands = new String[sizeUnknown]; unknownCands_V.toArray(unknownCands);
+ * int[] indices = new int[sizeUnknown]; for (int d = 0; d < sizeUnknown; ++d) {
+ * existingCandStats.remove(unknownCands[d]); // remove the (unknownCands[d],"U") entry
+ * from existingCandStats // (we had added it while constructing unknownCands_V to avoid
+ * duplicate entries) indices[d] = i; }
+ */
+ /*********************************************/
+
+ existingCandStats.clear();
+
+ } // for (i)
+
+ /*
+ * int[][] newSuffStats = null; if (!statsCurrIt_exists && sizeUnknown > 0) { newSuffStats =
+ * evalMetric.suffStats(unknownCands, indices); }
+ */
+
+ outFile_statsMergedKnown.close();
+ outFile_unknownCands.close();
+ outFile_unknownIndices.close();
+
+
+ for (int it = firstIt; it < iteration; ++it) {
+ inFile_sents[it].close();
+ inFile_stats[it].close();
+
+ InputStream inStream_sents, inStream_stats;
+ if (compressFiles == 0) {
+ inStream_sents = new FileInputStream(tmpDirPrefix + "temp.sents.it" + it);
+ inStream_stats = new FileInputStream(tmpDirPrefix + "temp.stats.it" + it);
+ } else {
+ inStream_sents =
+ new GZIPInputStream(
+ new FileInputStream(tmpDirPrefix + "temp.sents.it" + it + ".gz"));
+ inStream_stats =
+ new GZIPInputStream(
+ new FileInputStream(tmpDirPrefix + "temp.stats.it" + it + ".gz"));
+ }
+
+ inFile_sents[it] = new BufferedReader(new InputStreamReader(inStream_sents, "utf8"));
+ inFile_stats[it] = new BufferedReader(new InputStreamReader(inStream_stats, "utf8"));
+ }
+
+ inFile_sentsCurrIt.close();
+ if (compressFiles == 0) {
+ inStream_sentsCurrIt = new FileInputStream(tmpDirPrefix + "temp.sents.it" + iteration);
+ } else {
+ inStream_sentsCurrIt =
+ new GZIPInputStream(new FileInputStream(tmpDirPrefix + "temp.sents.it" + iteration
+ + ".gz"));
+ }
+ inFile_sentsCurrIt =
+ new BufferedReader(new InputStreamReader(inStream_sentsCurrIt, "utf8"));
+
+
+
+ // calculate SS for unseen candidates and write them to file
+ FileInputStream inStream_statsCurrIt_unknown = null;
+ BufferedReader inFile_statsCurrIt_unknown = null;
+
+ if (!statsCurrIt_exists && newCandidatesAdded[iteration] > 0) {
+ // create the file...
+ evalMetric.createSuffStatsFile(tmpDirPrefix + "temp.currIt.unknownCands", tmpDirPrefix
+ + "temp.currIt.unknownIndices", tmpDirPrefix + "temp.stats.unknown", sizeOfNBest);
+
+ // ...and open it
+ inStream_statsCurrIt_unknown = new FileInputStream(tmpDirPrefix + "temp.stats.unknown");
+ inFile_statsCurrIt_unknown =
+ new BufferedReader(new InputStreamReader(inStream_statsCurrIt_unknown, "utf8"));
+ }
+
+ // OPEN mergedKnown file
+ FileInputStream instream_statsMergedKnown =
+ new FileInputStream(tmpDirPrefix + "temp.stats.mergedKnown");
+ BufferedReader inFile_statsMergedKnown =
+ new BufferedReader(new InputStreamReader(instream_statsMergedKnown, "utf8"));
+
+ for (int i = 0; i < numSentences; ++i) {
+
+ // reprocess candidates from previous iterations
+ for (int it = firstIt; it < iteration; ++it) {
+ for (int n = 0; n <= sizeOfNBest; ++n) {
+
+ sents_str = inFile_sents[it].readLine();
+ stats_str = inFile_stats[it].readLine();
+
+ if (sents_str.equals("||||||")) {
+ n = sizeOfNBest + 1;
+ } else if (!existingCandStats.containsKey(sents_str)) {
+ existingCandStats.put(sents_str, stats_str);
+ } // if unseen candidate
+
+ } // for (n)
+ } // for (it)
+
+ // copy relevant portion from mergedKnown to the merged file
+ String line_mergedKnown = inFile_statsMergedKnown.readLine();
+ while (!line_mergedKnown.equals("||||||")) {
+ outFile_statsMerged.println(line_mergedKnown);
+ line_mergedKnown = inFile_statsMergedKnown.readLine();
+ }
+
+ int[] stats = new int[suffStatsCount];
+
+ for (int n = 0; n <= sizeOfNBest; ++n) {
+ // Why up to and *including* sizeOfNBest?
+ // So that it would read the "||||||" separator even if there is
+ // a complete list of sizeOfNBest candidates.
+
+ // for the nth candidate for the ith sentence, read the sentence, feature values,
+ // and sufficient statistics from the various temp files
+
+ sents_str = inFile_sentsCurrIt.readLine();
+ feats_str = inFile_featsCurrIt.readLine();
+
+ if (sents_str.equals("||||||")) {
+ n = sizeOfNBest + 1;
+ } else if (!existingCandStats.containsKey(sents_str)) {
+
+ if (!statsCurrIt_exists) {
+ stats_str = inFile_statsCurrIt_unknown.readLine();
+
+ String[] temp_stats = stats_str.split("\\s+");
+ for (int s = 0; s < suffStatsCount; ++s) {
+ stats[s] = Integer.parseInt(temp_stats[s]);
+ }
+
+ /*
+ * stats_str = ""; for (int s = 0; s < suffStatsCount-1; ++s) { stats[s] =
+ * newSuffStats[d][s]; stats_str += (stats[s] + " "); } stats[suffStatsCount-1] =
+ * newSuffStats[d][suffStatsCount-1]; stats_str += stats[suffStatsCount-1];
+ */
+
+ outFile_statsCurrIt.println(stats_str);
+ } else {
+ stats_str = inFile_statsCurrIt.readLine();
+ String[] temp_stats = stats_str.split("\\s+");
+ for (int s = 0; s < suffStatsCount; ++s) {
+ try {
+ stats[s] = Integer.parseInt(temp_stats[s]);
+ } catch (Exception e) {
+ stats[s] = 0;
+ }
+ }
+ }
+
+ outFile_statsMerged.println(stats_str);
+
+ featVal_str = feats_str.split("\\s+");
+
+ if (feats_str.indexOf('=') != -1) {
+ for (String featurePair: featVal_str) {
+ String[] pair = featurePair.split("=");
+ String name = pair[0];
+ Double value = Double.parseDouble(pair[1]);
+ currFeatVal[c_fromParamName(name)] = value;
+ }
+ } else {
+ for (int c = 1; c <= numParams; ++c) {
+ try {
+ currFeatVal[c] = Double.parseDouble(featVal_str[c - 1]);
+ } catch (Exception e) {
+ // NumberFormatException, ArrayIndexOutOfBoundsException
+ currFeatVal[c] = 0.0;
+ }
+
+ // print("fV[" + c + "]=" + currFeatVal[c] + " ",4);
+ }
+ }
+ // println("",4);
+
+
+ for (int j = 1; j <= initsPerIt; ++j) {
+ double score = 0; // i.e. score assigned by decoder
+ for (int c = 1; c <= numParams; ++c) {
+ score += initialLambda[j][c] * currFeatVal[c];
+ }
+ if (score > best1Score[j][i]) {
+ best1Score[j][i] = score;
+ for (int s = 0; s < suffStatsCount; ++s)
+ best1Cand_suffStats[j][i][s] = stats[s];
+ }
+ } // for (j)
+
+ existingCandStats.put(sents_str, stats_str);
+
+ setFeats(featVal_array, i, lastUsedIndex, maxIndex, currFeatVal);
+ candCount[i] += 1;
+
+ // newCandidatesAdded[iteration] += 1;
+ // moved to code above detecting new candidates
+
+ } else {
+ if (statsCurrIt_exists)
+ inFile_statsCurrIt.readLine();
+ else {
+ // write SS to outFile_statsCurrIt
+ stats_str = existingCandStats.get(sents_str);
+ outFile_statsCurrIt.println(stats_str);
+ }
+ }
+
+ } // for (n)
+
+ // now d = sizeUnknown_currIt[i] - 1
+
+ if (statsCurrIt_exists)
+ inFile_statsCurrIt.readLine();
+ else
+ outFile_statsCurrIt.println("||||||");
+
+ existingCandStats.clear();
+ totalCandidateCount += candCount[i];
+
+ if ((i + 1) % 500 == 0) {
+ print((i + 1) + "\n" + " ", 1);
+ } else if ((i + 1) % 100 == 0) {
+ print("+", 1);
+ } else if ((i + 1) % 25 == 0) {
+ print(".", 1);
+ }
+
+ } // for (i)
+
+ inFile_statsMergedKnown.close();
+ outFile_statsMerged.close();
+
+ println("", 1); // finish progress line
+
+ for (int it = firstIt; it < iteration; ++it) {
+ inFile_sents[it].close();
+ inFile_feats[it].close();
+ inFile_stats[it].close();
+ }
+
+ inFile_sentsCurrIt.close();
+ inFile_featsCurrIt.close();
+ if (statsCurrIt_exists)
+ inFile_statsCurrIt.close();
+ else
+ outFile_statsCurrIt.close();
+
+ if (compressFiles == 1 && !statsCurrIt_exists) {
+ gzipFile(tmpDirPrefix + "temp.stats.it" + iteration);
+ }
+
+ deleteFile(tmpDirPrefix + "temp.currIt.unknownCands");
+ deleteFile(tmpDirPrefix + "temp.currIt.unknownIndices");
+ deleteFile(tmpDirPrefix + "temp.stats.unknown");
+ deleteFile(tmpDirPrefix + "temp.stats.mergedKnown");
+
+ // cleanupMemory();
+
+ println("Processed " + totalCandidateCount + " distinct candidates " + "(about "
+ + totalCandidateCount / numSentences + " per sentence):", 1);
+ for (int it = firstIt; it <= iteration; ++it) {
+ println("newCandidatesAdded[it=" + it + "] = " + newCandidatesAdded[it] + " (about "
+ + newCandidatesAdded[it] / numSentences + " per sentence)", 1);
+ }
+
+ println("", 1);
+
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+
+ if (newCandidatesAdded[iteration] == 0) {
+ if (!oneModificationPerIteration) {
+ println("No new candidates added in this iteration; exiting Z-MERT.", 1);
+ println("", 1);
+ println("--- Z-MERT iteration #" + iteration + " ending @ " + (new Date()) + " ---", 1);
+ println("", 1);
+ return null; // THIS MEANS THAT THE OLD VALUES SHOULD BE KEPT BY THE CALLER
+ } else {
+ println("Note: No new candidates added in this iteration.", 1);
+ }
+ }
+
+ // run the initsPerIt optimizations, in parallel, across numOptThreads threads
+ ExecutorService pool = Executors.newFixedThreadPool(numOptThreads);
+ Semaphore blocker = new Semaphore(0);
+ Vector<String>[] threadOutput = new Vector[initsPerIt + 1];
+
+ for (int j = 1; j <= initsPerIt; ++j) {
+ threadOutput[j] = new Vector<String>();
+ pool.execute(new IntermediateOptimizer(j, blocker, threadOutput[j], initialLambda[j],
+ finalLambda[j], best1Cand_suffStats[j], finalScore, candCount, featVal_array,
+ suffStats_array));
+ }
+
+ pool.shutdown();
+
+ try {
+ blocker.acquire(initsPerIt);
+ } catch (java.lang.InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+
+ // extract output from threadOutput[]
+ for (int j = 1; j <= initsPerIt; ++j) {
+ for (String str : threadOutput[j]) {
+ println(str); // no verbosity check needed; thread already checked
+ }
+ }
+
+ int best_j = 1;
+ double bestFinalScore = finalScore[1];
+ for (int j = 2; j <= initsPerIt; ++j) {
+ if (evalMetric.isBetter(finalScore[j], bestFinalScore)) {
+ best_j = j;
+ bestFinalScore = finalScore[j];
+ }
+ }
+
+ if (initsPerIt > 1) {
+ println("Best final lambda is lambda[j=" + best_j + "] " + "(" + metricName_display + ": "
+ + f4.format(bestFinalScore) + ").", 1);
+ println("", 1);
+ }
+
+ FINAL_score = bestFinalScore;
+
+ boolean anyParamChanged = false;
+ boolean anyParamChangedSignificantly = false;
+
+ for (int c = 1; c <= numParams; ++c) {
+ if (finalLambda[best_j][c] != lambda[c]) {
+ anyParamChanged = true;
+ }
+ if (Math.abs(finalLambda[best_j][c] - lambda[c]) > stopSigValue) {
+ anyParamChangedSignificantly = true;
+ }
+ }
+
+ System.arraycopy(finalLambda[best_j], 1, lambda, 1, numParams);
+ println("--- Z-MERT iteration #" + iteration + " ending @ " + (new Date()) + " ---", 1);
+ println("", 1);
+
+ if (!anyParamChanged) {
+ println("No parameter value changed in this iteration; exiting Z-MERT.", 1);
+ println("", 1);
+ break; // exit for (iteration) loop preemptively
+ }
+
+ // check if a lambda is outside its threshold range
+ for (int c = 1; c <= numParams; ++c) {
+ if (lambda[c] < minThValue[c] || lambda[c] > maxThValue[c]) {
+ println("Warning: after normalization, lambda[" + c + "]=" + f4.format(lambda[c])
+ + " is outside its critical value range.", 1);
+ }
+ }
+
+ // was an early stopping criterion satisfied?
+ boolean critSatisfied = false;
+ if (!anyParamChangedSignificantly && stopSigValue >= 0) {
+ println("Note: No parameter value changed significantly " + "(i.e. by more than "
+ + stopSigValue + ") in this iteration.", 1);
+ critSatisfied = true;
+ }
+
+ if (critSatisfied) {
+ ++earlyStop;
+ println("", 1);
+ } else {
+ earlyStop = 0;
+ }
+
+ // if min number of iterations executed, investigate if early exit should happen
+ if (iteration >= minIts && earlyStop >= stopMinIts) {
+ println("Some early stopping criteria has been observed " + "in " + stopMinIts
+ + " consecutive iterations; exiting Z-MERT.", 1);
+ println("", 1);
+ break; // exit for (iteration) loop preemptively
+ }
+
+ // if max number of iterations executed, exit
+ if (iteration >= maxIts) {
+ println("Maximum number of MERT iterations reached; exiting Z-MERT.", 1);
+ println("", 1);
+ break; // exit for (iteration) loop
+ }
+
+ println("Next iteration will decode with lambda: " + lambdaToString(lambda), 1);
+ println("", 1);
+
+ // printMemoryUsage();
+ for (int i = 0; i < numSentences; ++i) {
+ suffStats_array[i].clear();
+ }
+ // cleanupMemory();
+ // println("",2);
+
+
+ retA[2] = 0; // i.e. this should NOT be the last iteration
+ done = true;
+
+ } // while (!done) // NOTE: this "loop" will only be carried out once
+
+
+ // delete .temp.stats.merged file, since it is not needed in the next
+ // iteration (it will be recreated from scratch)
+ deleteFile(tmpDirPrefix + "temp.stats.merged");
+
+ retA[0] = FINAL_score;
+ retA[1] = earlyStop;
+ return retA;
+
+ } // run_single_iteration
+
+ private String lambdaToString(double[] lambdaA) {
+ String retStr = "{";
+ for (int c = 1; c <= numParams - 1; ++c) {
+ retStr += "" + lambdaA[c] + ", ";
+ }
+ retStr += "" + lambdaA[numParams] + "}";
+
+ return retStr;
+ }
+
+ private String[] run_decoder(int iteration) {
+ String[] retSA = new String[2];
+ // [0] name of file to be processed
+ // [1] indicates how the output file was obtained:
+ // 1: external decoder
+ // 2: fake decoder
+ // 3: internal decoder
+
+ if (fakeFileNameTemplate != null
+ && fileExists(fakeFileNamePrefix + iteration + fakeFileNameSuffix)) {
+ String fakeFileName = fakeFileNamePrefix + iteration + fakeFileNameSuffix;
+ println("Not running decoder; using " + fakeFileName + " instead.", 1);
+ /*
+ * if (fakeFileName.endsWith(".gz")) { copyFile(fakeFileName,decoderOutFileName+".gz");
+ * gunzipFile(decoderOutFileName+".gz"); } else { copyFile(fakeFileName,decoderOutFileName); }
+ */
+ retSA[0] = fakeFileName;
+ retSA[1] = "2";
+
+ } else {
+ println("Running external decoder...", 1);
+
+ try {
+ ArrayList<String> cmd = new ArrayList<String>();
+ cmd.add(decoderCommandFileName);
+
+ if (passIterationToDecoder)
+ cmd.add(Integer.toString(iteration));
+
+ ProcessBuilder pb = new ProcessBuilder(cmd);
+ // this merges the error and output streams of the subprocess
+ pb.redirectErrorStream(true);
+ Process p = pb.start();
+
+ // capture the sub-command's output
+ StreamGobbler outputGobbler = new StreamGobbler(p.getInputStream(), decVerbosity);
+ outputGobbler.start();
+
+ int decStatus = p.waitFor();
+ if (decStatus != validDecoderExitValue) {
+ throw new RuntimeException("Call to decoder returned " + decStatus + "; was expecting "
+ + validDecoderExitValue + ".");
+ }
+ } catch (IOException| InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+
+ retSA[0] = decoderOutFileName;
+ retSA[1] = "1";
+ }
+
+ return retSA;
+
+ }
+
+ private void produceTempFiles(String nbestFileName, int iteration) {
+ try {
+ String sentsFileName = tmpDirPrefix + "temp.sents.it" + iteration;
+ String featsFileName = tmpDirPrefix + "temp.feats.it" + iteration;
+
+ FileOutputStream outStream_sents = new FileOutputStream(sentsFileName, false);
+ OutputStreamWriter outStreamWriter_sents = new OutputStreamWriter(outStream_sents, "utf8");
+ BufferedWriter outFile_sents = new BufferedWriter(outStreamWriter_sents);
+
+ PrintWriter outFile_feats = new PrintWriter(featsFileName);
+
+
+ InputStream inStream_nbest = null;
+ if (nbestFileName.endsWith(".gz")) {
+ inStream_nbest = new GZIPInputStream(new FileInputStream(nbestFileName));
+ } else {
+ inStream_nbest = new FileInputStream(nbestFileName);
+ }
+ BufferedReader inFile_nbest =
+ new BufferedReader(new InputStreamReader(inStream_nbest, "utf8"));
+
+ String line; // , prevLine;
+ String candidate_str = "";
+ String feats_str = "";
+
+ int i = 0;
+ int n = 0;
+ line = inFile_nbest.readLine();
+
+ while (line != null) {
+
+ // skip blank lines
+ if (line.equals("")) continue;
+
+ // skip lines that aren't formatted correctly
+ if (line.indexOf("|||") == -1)
+ continue;
+
+ /*
+ * line format:
+ *
+ * i ||| words of candidate translation . ||| feat-1_val feat-2_val ... feat-numParams_val
+ * .*
+ *
+ * Updated September 2012: features can now be named (for sparse feature compatibility).
+ * You must name all features or none of them.
+ */
+
+ // in a well formed file, we'd find the nth candidate for the ith sentence
+
+ int read_i = Integer.parseInt((line.substring(0, line.indexOf("|||"))).trim());
+
+ if (read_i != i) {
+ writeLine("||||||", outFile_sents);
+ outFile_feats.println("||||||");
+ n = 0;
+ ++i;
+ }
+
+ line = (line.substring(line.indexOf("|||") + 3)).trim(); // get rid of initial text
+
+ candidate_str = (line.substring(0, line.indexOf("|||"))).trim();
+ feats_str = (line.substring(line.indexOf("|||") + 3)).trim();
+ // get rid of candidate string
+
+ int junk_i = feats_str.indexOf("|||");
+ if (junk_i >= 0) {
+ feats_str = (feats_str.substring(0, junk_i)).trim();
+ }
+
+ writeLine(normalize(candidate_str, textNormMethod), outFile_sents);
+ outFile_feats.println(feats_str);
+
+ ++n;
+ if (n == sizeOfNBest) {
+ writeLine("||||||", outFile_sents);
+ outFile_feats.println("||||||");
+ n = 0;
+ ++i;
+ }
+
+ line = inFile_nbest.readLine();
+ }
+
+ if (i != numSentences) { // last sentence had too few candidates
+ writeLine("||||||", outFile_sents);
+ outFile_feats.println("||||||");
+ }
+
+ inFile_nbest.close();
+ outFile_sents.close();
+ outFile_feats.close();
+
+ if (compressFiles == 1) {
+ gzipFile(sentsFileName);
+ gzipFile(featsFileName);
+ }
+
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ }
+
+ private void createConfigFile(double[] params, String cfgFileName, String templateFileName) {
+ try {
+ // i.e. create cfgFileName, which is similar to templateFileName, but with
+ // params[] as parameter values
+
+ BufferedReader inFile = new BufferedReader(new FileReader(templateFileName));
+ PrintWriter outFile = new PrintWriter(cfgFileName);
+
+ String line = inFile.readLine();
+
+ while (line != null) {
+ int c_match = -1;
+ for (int c = 1; c <= numParams; ++c) {
+ if (line.startsWith(paramNames[c] + " ")) {
+ c_match = c;
+ break;
+ }
+ }
+
+ if (c_match == -1) {
+ outFile.println(line);
+ } else {
+ outFile.println(paramNames[c_match] + " " + params[c_match]);
+ }
+
+ line = inFile.readLine();
+ }
+
+ inFile.close();
+ outFile.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void processParamFile() {
+ // process parameter file
+ Scanner inFile_init = null;
+ try {
+ inFile_init = new Scanner(new FileReader(paramsFileName));
+ } catch (FileNotFoundException e) {
+ throw new RuntimeException("FileNotFoundException in MertCore.processParamFile(): " + e.getMessage());
+ }
+
+ String dummy = "";
+
+ // initialize lambda[] and other related arrays
+ for (int c = 1; c <= numParams; ++c) {
+ // skip parameter name
+ while (!dummy.equals("|||")) {
+ dummy = inFile_init.next();
+ }
+
+ // read default value
+ lambda[c] = inFile_init.nextDouble();
+ defaultLambda[c] = lambda[c];
+
+ // read isOptimizable
+ dummy = inFile_init.next();
+ if (dummy.equals("Opt")) {
+ isOptimizable[c] = true;
+ } else if (dummy.equals("Fix")) {
+ isOptimizable[c] = false;
+ } else {
+ throw new RuntimeException("Unknown isOptimizable string " + dummy
+ + " (must be either Opt or Fix)");
+ }
+
+ if (!isOptimizable[c]) { // skip next four values
+ dummy = inFile_init.next();
+ dummy = inFile_init.next();
+ dummy = inFile_init.next();
+ dummy = inFile_init.next();
+ } else {
+ // set minThValue[c] and maxThValue[c] (range for thresholds to investigate)
+ dummy = inFile_init.next();
+ if (dummy.equals("-Inf")) {
+ minThValue[c] = NegInf;
+ } else if (dummy.equals("+Inf")) {
+ throw new RuntimeException("minThValue[" + c + "] cannot be +Inf!");
+ } else {
+ minThValue[c] = Double.parseDouble(dummy);
+ }
+
+ dummy = inFile_init.next();
+ if (dummy.equals("-Inf")) {
+ throw new RuntimeException("maxThValue[" + c + "] cannot be -Inf!");
+ } else if (dummy.equals("+Inf")) {
+ maxThValue[c] = PosInf;
+ } else {
+ maxThValue[c] = Double.parseDouble(dummy);
+ }
+
+ // set minRandValue[c] and maxRandValue[c] (range for random values)
+ dummy = inFile_init.next();
+ if (dummy.equals("-Inf") || dummy.equals("+Inf")) {
+ throw new RuntimeException("minRandValue[" + c + "] cannot be -Inf or +Inf!");
+ } else {
+ minRandValue[c] = Double.parseDouble(dummy);
+ }
+
+ dummy = inFile_init.next();
+ if (dummy.equals("-Inf") || dummy.equals("+Inf")) {
+ throw new RuntimeException("maxRandValue[" + c + "] cannot be -Inf or +Inf!");
+ } else {
+ maxRandValue[c] = Double.parseDouble(dummy);
+ }
+
+
+ // check for illogical values
+ if (minThValue[c] > maxThValue[c]) {
+ throw new RuntimeException("minThValue[" + c + "]=" + minThValue[c]
+ + " > " + maxThValue[c] + "=maxThValue[" + c + "]!");
+ }
+ if (minRandValue[c] > maxRandValue[c]) {
+ throw new RuntimeException("minRandValue[" + c + "]=" + minRandValue[c]
+ + " > " + maxRandValue[c] + "=maxRandValue[" + c + "]!");
+ }
+
+ // check for odd values
+ if (!(minThValue[c] <= lambda[c] && lambda[c] <= maxThValue[c])) {
+ println("Warning: lambda[" + c + "] has initial value (" + lambda[c] + ")", 1);
+ println(" that is outside its critical value range " + "[" + minThValue[c] + ","
+ + maxThValue[c] + "]", 1);
+ }
+
+ if (minThValue[c] == maxThValue[c]) {
+ println("Warning: lambda[" + c + "] has " + "minThValue = maxThValue = " + minThValue[c]
+ + ".", 1);
+ }
+
+ if (minRandValue[c] == maxRandValue[c]) {
+ println("Warning: lambda[" + c + "] has " + "minRandValue = maxRandValue = "
+ + minRandValue[c] + ".", 1);
+ }
+
+ if (minRandValue[c] < minThValue[c] || minRandValue[c] > maxThValue[c]
+ || maxRandValue[c] < minThValue[c] || maxRandValue[c] > maxThValue[c]) {
+ println("Warning: The random value range for lambda[" + c + "] is not contained", 1);
+ println(" within its critical value range.", 1);
+ }
+
+ } // if (!isOptimizable[c])
+
+ /*
+ * precision[c] = inFile_init.nextDouble(); if (precision[c] < 0) { println("precision[" + c +
+ * "]=" + precision[c] + " < 0! Must be non-negative."); System.exit(21); }
+ */
+
+ }
+
+ // set normalizationOptions[]
+ String origLine = "";
+ while (origLine != null && origLine.length() == 0) {
+ origLine = inFile_init.nextLine();
+ }
+
+
+ // How should a lambda[] vector be normalized (before decoding)?
+ // nO[0] = 0: no normalization
+ // nO[0] = 1: scale so that parameter nO[2] has absolute value nO[1]
+ // nO[0] = 2: scale so that the maximum absolute value is nO[1]
+ // nO[0] = 3: scale so that the minimum absolute value is nO[1]
+ // nO[0] = 4: scale so that the L-nO[1] norm equals nO[2]
+
+ // normalization = none
+ // normalization = absval 1 lm
+ // normalization = maxabsval 1
+ // normalization = minabsval 1
+ // normalization = LNorm 2 1
+
+ dummy = (origLine.substring(origLine.indexOf("=") + 1)).trim();
+ String[] dummyA = dummy.split("\\s+");
+
+ if (dummyA[0].equals("none")) {
+ normalizationOptions[0] = 0;
+ } else if (dummyA[0].equals("absval")) {
+ normalizationOptions[0] = 1;
+ normalizationOptions[1] = Double.parseDouble(dummyA[1]);
+ String pName = dummyA[2];
+ for (int i = 3; i < dummyA.length; ++i) { // in case parameter name has multiple words
+ pName = pName + " " + dummyA[i];
+ }
+ normalizationOptions[2] = c_fromParamName(pName);;
+
+ if (normalizationOptions[1] <= 0) {
+ throw new RuntimeException("Value for the absval normalization method must be positive.");
+ }
+ if (normalizationOptions[2] == 0) {
+ throw new RuntimeException("Unrecognized feature name " + normalizationOptions[2]
+ + " for absval normalization method.");
+ }
+ } else if (dummyA[0].equals("maxabsval")) {
+ normalizationOptions[0] = 2;
+ normalizationOptions[1] = Double.parseDouble(dummyA[1]);
+ if (normalizationOptions[1] <= 0) {
+ throw new RuntimeException("Value for the maxabsval normalization method must be positive.");
+ }
+ } else if (dummyA[0].equals("minabsval")) {
+ normalizationOptions[0] = 3;
+ normalizationOptions[1] = Double.parseDouble(dummyA[1]);
+ if (normalizationOptions[1] <= 0) {
+ throw new RuntimeException("Value for the minabsval normalization method must be positive.");
+ }
+ } else if (dummyA[0].equals("LNorm")) {
+ normalizationOptions[0] = 4;
+ normalizationOptions[1] = Double.parseDouble(dummyA[1]);
+ normalizationOptions[2] = Double.parseDouble(dummyA[2]);
+ if (normalizationOptions[1] <= 0 || normalizationOptions[2] <= 0) {
+ throw new RuntimeException("Both values for the LNorm normalization method must be positive.");
+ }
+ } else {
+ throw new RuntimeException("Unrecognized normalization method " + dummyA[0] + "; "
+ + "must be one of none, absval, maxabsval, and LNorm.");
+ } // if (dummyA[0])
+
+ inFile_init.close();
+ }
+
+ private void processDocInfo() {
+ // sets numDocuments and docOfSentence[]
+ docOfSentence = new int[numSentences];
+
+ if (docInfoFileName == null) {
+ for (int i = 0; i < numSentences; ++i)
+ docOfSentence[i] = 0;
+ numDocuments = 1;
+ } else {
+
+ try {
+
+ // 4 possible formats:
+ // 1) List of numbers, one per document, indicating # sentences in each document.
+ // 2) List of "docName size" pairs, one per document, indicating name of document and #
+ // sentences.
+ // 3) List of docName's, one per sentence, indicating which doument each sentence belongs
+ // to.
+ // 4) List of docName_number's, one per sentence, indicating which doument each sentence
+ // belongs to,
+ // and its order in that document. (can also use '-' instead of '_')
+
+ int docInfoSize = countNonEmptyLines(docInfoFileName);
+
+ if (docInfoSize < numSentences) { // format #1 or #2
+ numDocuments = docInfoSize;
+ int i = 0;
+
+ BufferedReader inFile = new BufferedReader(new FileReader(docInfoFileName));
+ String line = inFile.readLine();
+ boolean format1 = (!(line.contains(" ")));
+
+ for (int doc = 0; doc < numDocuments; ++doc) {
+
+ if (doc != 0) line = inFile.readLine();
+
+ int docSize = 0;
+ if (format1) {
+ docSize = Integer.parseInt(line);
+ } else {
+ docSize = Integer.parseInt(line.split("\\s+")[1]);
+ }
+
+ for (int i2 = 1; i2 <= docSize; ++i2) {
+ docOfSentence[i] = doc;
+ ++i;
+ }
+
+ }
+
+ // now i == numSentences
+
+ inFile.close();
+
+ } else if (docInfoSize == numSentences) { // format #3 or #4
+
+ boolean format3 = false;
+
+ HashSet<String> seenStrings = new HashSet<String>();
+ BufferedReader inFile = new BufferedReader(new FileReader(docInfoFileName));
+ for (int i = 0; i < numSentences; ++i) {
+ // set format3 = true if a duplicate is found
+ String line = inFile.readLine();
+ if (seenStrings.contains(line)) format3 = true;
+ seenStrings.add(line);
+ }
+
+ inFile.close();
+
+ HashSet<String> seenDocNames = new HashSet<String>();
+ HashMap<String, Integer> docOrder = new HashMap<String, Integer>();
+ // maps a document name to the order (0-indexed) in which it was seen
+
+ inFile = new BufferedReader(new FileReader(docInfoFileName));
+ for (int i = 0; i < numSentences; ++i) {
+ String line = inFile.readLine();
+
+ String docName = "";
+ if (format3) {
+ docName = line;
+ } else {
+ int sep_i = Math.max(line.lastIndexOf('_'), line.lastIndexOf('-'));
+ docName = line.substring(0, sep_i);
+ }
+
+ if (!seenDocNames.contains(docName)) {
+ seenDocNames.add(docName);
+ docOrder.put(docName, seenDocNames.size() - 1);
+ }
+
+ int docOrder_i = docOrder.get(docName);
+
+ docOfSentence[i] = docOrder_i;
+
+ }
+
+ inFile.close();
+
+ numDocuments = seenDocNames.size();
+
+ } else { // badly formatted
+
+ }
+
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ }
+
+ private boolean copyFile(String origFileName, String newFileName) {
+ try {
+ File inputFile = new File(origFileName);
+ File outputFile = new File(newFileName);
+
+ InputStream in = new FileInputStream(inputFile);
+ OutputStream out = new FileOutputStream(outputFile);
+
+ byte[] buffer = new byte[1024];
+ int len;
+ while ((len = in.read(buffer)) > 0) {
+ out.write(buffer, 0, len);
+ }
+ in.close();
+ out.close();
+
+ /*
+ * InputStream inStream = new FileInputStream(new File(origFileName)); BufferedReader inFile =
+ * new BufferedReader(new InputStreamReader(inStream, "utf8"));
+ *
+ * FileOutputStream outStream = new FileOutputStream(newFileName, false); OutputStreamWriter
+ * outStreamWriter = new OutputStreamWriter(outStream, "utf8"); BufferedWriter outFile = new
+ * BufferedWriter(outStreamWriter);
+ *
+ * String line; while(inFile.ready()) { line = inFile.readLine(); writeLine(line, outFile); }
+ *
+ * inFile.close(); outFile.close();
+ */
+ return true;
+ } catch (IOException e) {
+ LOG.error(e.getMessage(), e);
+ return false;
+ }
+ }
+
+ private void renameFile(String origFileName, String newFileName) {
+ if (fileExists(origFileName)) {
+ deleteFile(newFileName);
+ File oldFile = new File(origFileName);
+ File newFile = new File(newFileName);
+ if (!oldFile.renameTo(newFile)) {
+ println("Warning: attempt to rename " + origFileName + " to " + newFileName
+ + " was unsuccessful!", 1);
+ }
+ } else {
+ println("Warning: file " + origFileName + " does not exist! (in MertCore.renameFile)", 1);
+ }
+ }
+
+ private void deleteFile(String fileName) {
+ if (fileExists(fileName)) {
+ File fd = new File(fileName);
+ if (!fd.delete()) {
+ println("Warning: attempt to delete " + fileName + " was unsuccessful!", 1);
+ }
+ }
+ }
+
+ private void writeLine(String line, BufferedWriter writer) throws IOException {
+ writer.write(line, 0, line.length());
+ writer.newLine();
+ writer.flush();
+ }
+
+ public void finish() {
+ if (myDecoder != null) {
+ myDecoder.cleanUp();
+ }
+
+ // create config file with final values
+ createConfigFile(lambda, decoderConfigFileName + ".ZMERT.final", decoderConfigFileName
+ + ".ZMERT.orig");
+
+ // delete current decoder config file and decoder output
+ deleteFile(decoderConfigFileName);
+ deleteFile(decoderOutFileName);
+
+ // restore original name for config file (name was changed
+ // in initialize() so it doesn't get overwritten)
+ renameFile(decoderConfigFileName + ".ZMERT.orig", decoderConfigFileName);
+
+ if (finalLambdaFileName != null) {
+ try {
+ PrintWriter outFile_lambdas = new PrintWriter(finalLambdaFileName);
+ for (int c = 1; c <= numParams; ++c) {
+ outFile_lambdas.println(paramNames[c] + " ||| " + lambda[c]);
+ }
+ outFile_lambdas.close();
+
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ }
+
+ private String[] cfgFileToArgsArray(String fileName) {
+ checkFile(fileName);
+
+ Vector<String> argsVector = new Vector<String>();
+
+ BufferedReader inFile = null;
+ try {
+ inFile = new BufferedReader(new FileReader(fileName));
+ String line, origLine;
+ do {
+ line = inFile.readLine();
+ origLine = line; // for error reporting purposes
+
+ if (line != null && line.length() > 0 && line.charAt(0) != '#') {
+
+ if (line.indexOf("#") != -1) { // discard comment
+ line = line.substring(0, line.indexOf("#"));
+ }
+
+ line = line.trim();
+
+ // now line should look like "-xxx XXX"
+
+ String[] paramA = line.split("\\s+");
+
+ if (paramA.length == 2 && paramA[0].charAt(0) == '-') {
+ argsVector.add(paramA[0]);
+ argsVector.add(paramA[1]);
+ } else if (paramA.length > 2
+ && (paramA[0].equals("-m") || paramA[0].equals("-docSet") || paramA[0]
+ .equals("-damianos"))) {
+ // -m (metricName), -docSet, and -damianos are allowed to have extra optinos
+ for (int opt = 0; opt < paramA.length; ++opt) {
+ argsVector.add(paramA[opt]);
+ }
+ } else {
+ println("Malformed line in config file:");
+ println(origLine);
+ System.exit(70);
+ }
+
+ }
+ } while (line != null);
+
+ inFile.close();
+ } catch (FileNotFoundException e) {
+ throw new RuntimeException("Z-MERT configuration file " + fileName + " was not found!", e);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ String[] argsArray = new String[argsVector.size()];
+
+ for (int i = 0; i < argsVector.size(); ++i) {
+ argsArray[i] = argsVector.elementAt(i);
+ }
+
+ return argsArray;
+ }
+
+ private void processArgsArray(String[] args) {
+ processArgsArray(args, true);
+ }
+
+ private void processArgsArray(String[] args, boolean firstTime) {
+ /* set default values */
+ // Relevant files
+ dirPrefix = null;
+ sourceFileName = null;
+ refFileName = "reference.txt";
+ refsPerSen = 1;
+ textNormMethod = 1;
+ paramsFileName = "params.txt";
+ docInfoFileName = null;
+ finalLambdaFileName = null;
+ // MERT specs
+ metricName = "BLEU";
+ metricName_display = metricName;
+ metricOptions = new String[2];
+ metricOptions[0] = "4";
+ metricOptions[1] = "closest";
+ docSubsetInfo = new int[7];
+ docSubsetInfo[0] = 0;
+ maxMERTIterations = 20;
+ prevMERTIterations = 20;
+ minMERTIterations = 5;
+ stopMinIts = 3;
+ stopSigValue = -1;
+ //
+ // /* possibly other early stopping criteria here */
+ //
+ numOptThreads = 1;
+ saveInterFiles = 3;
+ compressFiles = 0;
+ initsPerIt = 20;
+ oneModificationPerIteration = false;
+ randInit = false;
+ seed = System.currentTimeMillis();
+ // useDisk = 2;
+ // Decoder specs
+ decoderCommandFileName = null;
+ passIterationToDecoder = false;
+ decoderOutFileName = "output.nbest";
+ validDecoderExitValue = 0;
+ decoderConfigFileName = "dec_cfg.txt";
+ sizeOfNBest = 100;
+ fakeFileNameTemplate = null;
+ fakeFileNamePrefix = null;
+ fakeFileNameSuffix = null;
+ // Output specs
+ verbosity = 1;
+ decVerbosity = 1;
+
+ damianos_method = 0;
+ damianos_param = 0.0;
+ damianos_mult = 0.0;
+
+ int i = 0;
+
+ while (i < args.length) {
+ String option = args[i];
+ // Relevant files
+ if (option.equals("-dir")) {
+ dirPrefix = args[i + 1];
+ } else if (option.equals("-s")) {
+ sourceFileName = args[i + 1];
+ } else if (option.equals("-r")) {
+ refFileName = args[i + 1];
+ } else if (option.equals("-rps")) {
+ refsPerSen = Integer.parseInt(args[i + 1]);
+ if (refsPerSen < 1) {
+ throw new RuntimeException("refsPerSen must be positive.");
+ }
+ } else if (option.equals("-txtNrm")) {
+ textNormMethod = Integer.parseInt(args[i + 1]);
+ if (textNormMethod < 0 || textNormMethod > 4) {
+ throw new RuntimeException("textNormMethod should be between 0 and 4");
+ }
+ } else if (option.equals("-p")) {
+ paramsFileName = args[i + 1];
+ } else if (option.equals("-docInfo")) {
+ docInfoFileName = args[i + 1];
+ } else if (option.equals("-fin")) {
+ finalLambdaFileName = args[i + 1];
+ // MERT specs
+ } else if (option.equals("-m")) {
+ metricName = args[i + 1];
+ metricName_display = metricName;
+ if (EvaluationMetric.knownMetricName(metricName)) {
+ int optionCount = EvaluationMetric.metricOptionCount(metricName);
+ metricOptions = new String[optionCount];
+ for (int opt = 0; opt < optionCount; ++opt) {
+ metricOptions[opt] = args[i + opt + 2];
+ }
+ i += optionCount;
+ } else {
+ throw new RuntimeException("Unknown metric name " + metricName + ".");
+ }
+ } else if (option.equals("-docSet")) {
+ String method = args[i + 1];
+
+ if (method.equals("all")) {
+ docSubsetInfo[0] = 0;
+ i += 0;
+ } else if (method.equals("bottom")) {
+ String a = args[i + 2];
+ if (a.endsWith("d")) {
+ docSubsetInfo[0] = 1;
+ a = a.substring(0, a.indexOf("d"));
+ } else {
+ docSubsetInfo[0] = 2;
+ a = a.substring(0, a.indexOf("%"));
+ }
+ docSubsetInfo[5] = Integer.parseInt(a);
+ i += 1;
+ } else if (method.equals("top")) {
+ String a = args[i + 2];
+ if (a.endsWith("d")) {
+ docSubsetInfo[0] = 3;
+ a = a.substring(0, a.indexOf("d"));
+ } else {
+ docSubsetInfo[0] = 4;
+ a = a.substring(0, a.indexOf("%"));
+ }
+ docSubsetInfo[5] = Integer.parseInt(a);
+ i += 1;
+ } else if (method.equals("window")) {
+ String a1 = args[i + 2];
+ a1 = a1.substring(0, a1.indexOf("d")); // size of window
+ String a2 = args[i + 4];
+ if (a2.indexOf("p") > 0) {
+ docSubsetInfo[0] = 5;
+ a2 = a2.substring(0, a2.indexOf("p"));
+ } else {
+ docSubsetInfo[0] = 6;
+ a2 = a2.substring(0, a2.indexOf("r"));
+ }
+ docSubsetInfo[5] = Integer.parseInt(a1);
+ docSubsetInfo[6] = Integer.parseInt(a2);
+ i += 3;
+ } else {
+ throw new RuntimeException("Unknown docSet method " + method + ".");
+ }
+ } else if (option.equals("-maxIt")) {
+ maxMERTIterations = Integer.parseInt(args[i + 1]);
+ if (maxMERTIterations < 1) {
+ throw new RuntimeException("maxMERTIts must be positive.");
+ }
+ } else if (option.equals("-minIt")) {
+ minMERTIterations = Integer.parseInt(args[i + 1]);
+ if (minMERTIterations < 1) {
+ throw new RuntimeException("minMERTIts must be positive.");
+ }
+ } else if (option.equals("-prevIt")) {
+ prevMERTIterations = Integer.parseInt(args[i + 1]);
+ if (prevMERTIterations < 0) {
+ throw new RuntimeException("prevMERTIts must be non-negative.");
+ }
+ } else if (option.equals("-stopIt")) {
+ stopMinIts = Integer.parseInt(args[i + 1]);
+ if (stopMinIts < 1) {
+ throw new RuntimeException("stopMinIts must be positive.");
+ }
+ } else if (option.equals("-stopSig")) {
+ stopSigValue = Double.parseDouble(args[i + 1]);
+ }
+ //
+ // /* possibly other early stopping criteria here */
+ //
+ else if (option.equals("-thrCnt")) {
+ numOptThreads = Integer.parseInt(args[i + 1]);
+ if (numOptThreads < 1) {
+ throw new RuntimeException("threadCount must be positive.");
+ }
+ } else if (option.equals("-save")) {
+ saveInterFiles = Integer.parseInt(args[i + 1]);
+ if (saveInterFiles < 0 || saveInterFiles > 3) {
+ throw new RuntimeException("save should be between 0 and 3");
+ }
+ } else if (option.equals("-compress")) {
+ compressFiles = Integer.parseInt(args[i + 1]);
+ if (compressFiles < 0 || compressFiles > 1) {
+ throw new RuntimeException("compressFiles should be either 0 or 1");
+ }
+ } else if (option.equals("-ipi")) {
+ initsPerIt = Integer.parseInt(args[i + 1]);
+ if (initsPerIt < 1) {
+ throw new RuntimeException("initsPerIt must be positive.");
+ }
+ } else if (option.equals("-opi")) {
+ int opi = Integer.parseInt(args[i + 1]);
+ if (opi == 1) {
+ oneModificationPerIteration = true;
+ } else if (opi == 0) {
+ oneModificationPerIteration = false;
+ } else {
+ throw new RuntimeException("oncePerIt must be either 0 or 1.");
+ }
+ } else if (option.equals("-rand")) {
+ int rand = Integer.parseInt(args[i + 1]);
+ if (rand == 1) {
+ randInit = true;
+ } else if (rand == 0) {
+ randInit = false;
+ } else {
+ throw new RuntimeException("randInit must be either 0 or 1.");
+ }
+ } else if (option.equals("-seed")) {
+ if (args[i + 1].equals("time")) {
+ seed = System.currentTimeMillis();
+ } else {
+ seed = Long.parseLong(args[i + 1]);
+ }
+ }
+ /*
+ * else if (option.equals("-ud")) { useDisk = Integer.parseInt(args[i+1]); if (useDisk < 0 ||
+ * useDisk > 2) { println("useDisk should be between 0 and 2"); System.exit(10); } }
+ */
+ // Decoder specs
+ else if (option.equals("-cmd")) {
+ decoderCommandFileName = args[i + 1];
+ } else if (option.equals("-passIt")) {
+ int val = Integer.parseInt(args[i + 1]);
+ if (val < 0 || val > 1) {
+ throw new RuntimeException("passIterationToDecoder should be either 0 or 1");
+ }
+ passIterationToDecoder = (val == 1) ? true : false;
+ } else if (option.equals("-decOut")) {
+ decoderOutFileName = args[i + 1];
+ } else if (option.equals("-decExit")) {
+ validDecoderExitValue = Integer.parseInt(args[i + 1]);
+ } else if (option.equals("-dcfg")) {
+ decoderConfigFileName = args[i + 1];
+ } else if (option.equals("-N")) {
+ sizeOfNBest = Integer.parseInt(args[i + 1]);
+ if (sizeOfNBest < 1) {
+ throw new RuntimeException("N must be positive.");
+ }
+ }
+ // Output specs
+ else if (option.equals("-v")) {
+ verbosity = Integer.parseInt(args[i + 1]);
+ if (verbosity < 0 || verbosity > 4) {
+ throw new RuntimeException("verbosity should be between 0 and 4");
+ }
+ } else if (option.equals("-decV")) {
+ decVerbosity = Integer.parseInt(args[i + 1]);
+ if (decVerbosity < 0 || decVerbosity > 1) {
+ throw new RuntimeException("decVerbosity should be either 0 or 1");
+ }
+ } else if (option.equals("-fake")) {
+ fakeFileNameTemplate = args[i + 1];
+ int QM_i = fakeFileNameTemplate.indexOf("?");
+ if (QM_i <= 0) {
+ throw new RuntimeException("fakeFileNameTemplate must contain '?' to indicate position of iteration number");
+ }
+ fakeFileNamePrefix = fakeFileNameTemplate.substring(0, QM_i);
+ fakeFileNameSuffix = fakeFileNameTemplate.substring(QM_i + 1);
+ } else if (option.equals("-damianos")) {
+ damianos_method = Integer.parseInt(args[i + 1]);
+ if (damianos_method < 0 || damianos_method > 3) {
+ throw new RuntimeException("damianos_method should be between 0 and 3");
+ }
+ damianos_param = Double.parseDouble(args[i + 2]);
+ damianos_mult = Double.parseDouble(args[i + 3]);
+ i += 2;
+ } else {
+ throw new RuntimeException("Unknown option " + option);
+ }
+
+ i += 2;
+
+ } // wh
<TRUNCATED>
[03/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/packed/reference.en.0
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/packed/reference.en.0 b/joshua-core/src/test/resources/bn-en/packed/reference.en.0
new file mode 100644
index 0000000..a91dd56
--- /dev/null
+++ b/joshua-core/src/test/resources/bn-en/packed/reference.en.0
@@ -0,0 +1,100 @@
+rabindranath was born in a pirali brahmin family of kolkata .
+recently the relationship between india and united stated has improved .
+mathematics is , hence , the language of science .
+from this it can easily be understood that this metric will be frw metric .
+at the same time this novel had also indicated to the fall of the land basis feudalism in bengal .
+mujib and his party could attain the absolute majority in the elections .
+hitlar continued his work in the city of munich at bavariar .
+besides tux there are os-tan and some other characters representing linux but are not much popular .
+it explains the conventional rules for decision making followed by the decision maker in the field of sports in relation to the social background
+annual rainfall is 2540 millimeter .
+he provided his main keynote speech in the national conference of his democrat party in the year 2004 in the boston city of machechuest states .
+since the distribution of powers was in favor of the east pakistan if the population was taken into the account , therefore the west pakistan introduced a unique idea under the name of " one unit theory " whereby the entire pakistan was considered as one province .
+measurement theory .
+external connections
+videsh sanchar nigam limited bhavan of tata communications , this is a major telecom service provider in the city .
+that year on the 4th november , he became victorious in the general election and got elected as the 44th president of the united states .
+many indian species originated from gondwana born taxa .
+plays , novels , stories , and recently screenplays written by the british writers are acknowledged worldwide .
+on 1919 it was published in a magazine called swagat .
+in 2005 , the women 's tennis association tour 's tier-iii tournament , sunfeast open was held in netaji indoor stadium .
+several provisions are taken to avoid this possibility
+in lahore , a national conference of the opponent parties was held on 5th february , 1955 .
+bangladesh became the member of the organization of the islamic conference and islamic development bank .
+special class : world dictionary
+russia , france and israel are the main countries supplying arms to india and defense associates .
+this is our familiar imaginary unit which relegates theory of mathematics to concern itself with set of complex numbers from that of real numbers .
+</address>
+september
+according to this philosophy you can not disagree universe , though the logic has been accepted .
+france is the most important country in europe in agriculture ; it export mainly food crop , wine , cheese , and other agricultural product to europe and the world .
+arithmetic was prevalent in their mathematics .
+the countries are : france , hong kong , china , belgium , switzerland , germany , denmark , sweden , austria , czechoslovakia , argentina , italy , norway , hungary , yugoslavia , bulgaria , romania , greece , egypt , singapore , indonesia , thailand , japan , burma , holland , soviet russia , iran , iraq , and sri lanka .
+in the place of this basilica bank of england is located now .
+to the north and south of spain there are bay of biscay and the gibraltar respectively , morocco lies to south of gibraltar and the atlantic ocean is situated at the west and south-west part of this country .
+except that , in this situation , the inability of united nations to take decision quickly in emergency situation was realized .
+this was popularised by karl marx
+its subject is often borrowed from hindu mythology , medieval romances and news of social and political events .
+depending on the three measures , the age of the universe has been found to be 13.7 � 0.2 billion years .
+east russia is close by , which is situated on the opposite side of the sea of okhotsk and sea of japan .
+the indian national library in kolkata is the leading public library of the country .
+mycology
+the secretary-general of the un at present is ban ki-moon .
+the creator of minix was andrew tunnenbom who was a famous teacher of operating system designing .
+the times of india wrote that " it is absurd to compare it with any other indian cinema ... pather panchali is pure cinema " .
+after that in 1953 , during the month of may , nazrul and his wife pramila devi were sent to london for better treatment .
+it has got plain lands in its south and east sides and rough hills and mountains in its west and north sides .
+trademark
+lord wellesley , the governor general between 1797 \u2013 1805 , was largely responsible for the growth of the city
+complex numbers are indispensible for solving many important and real problems .
+an important consequence of the big bang is that the present state of the universe is completely different from its past and future states .
+windows millennium
+although rabindranath had subjugated a number of styles .
+labor economy
+britain was once the most powerful and advance economic power of the world .
+he is highly admired due to the movement against the military ruler of pakistan and the protest against the racial inequities in pakistan and moving the movement of bengalees towards independence .
+though it is thought that the meaning of internet and world wide web but actually they refer two completely different things .
+the two relevant polar coordinates of the z are r = 1
+november
+the first electronics mail used was in 1972 ad in the arpanet .
+the section of biology which discusses fungi and its practical source is called mycology .
+the water was carried up with some pur , this was actually a method of pulling up water by animals with an arrangement of ropes and buckets .
+these are tribal dance , folk dance , classical dance etc .
+the indian literature was spread earlier as orally and later as written
+his direction in his film " gana satru " in 1989 was weak and it was considered as satyajit 's comeback in making movie after returning from a long-lasting illness .
+mechanics .
+linux is different from various angle than the other right-protected operating systems such as windows and mac os .
+according to the language of the asian times ,
+open source means free distribution of computer software source code .
+dhaka in bangladesh online
+in first war world germany was defeated .
+but in order to understand this subject experiments are going on at present also .
+super h
+he was declared unfit for the armed force .
+threatening to boycott the assembly bhutto announced that , if yahya khan call mujib to form the government he will not do accept that government .
+and the word computer stands for the machine that does the work of counting .
+on 4th of july , 1776 these colonies introduced a declaration of independence .
+germany -lrb- in german language : deutschland -rrb- is a country of middle europe .
+christianity is the main religion of russia .
+but the development of the educational systems of the gols into roman styles was a bit slow .
+subject : foundation
+it also studies the financial and revenue strategies .
+among them there were : ' hoyto tomar pabo dekha ' -lrb- perhaps i will meet you -rrb- , ' ore e kon sneho-surdhani ' -lrb- what an affectionate music is this -rrb- .
+he died on 23 april 1992 .
+the medical reports of nazrul was sent to the renowned doctors of vienna at this time .
+apart from acting rani mukherjee is involved with many charitable organisations .
+on december 9 , 1974 , dhaka university honored him with d.lit degree , for his excellent contribution in the field of bengali culture and literature .
+durgapuja in kolkata is a tourist attraction too .
+but when the millions of east germans started to emigrate into the economically developed and democratic country of west germany , the government of east germany had built a wall in berlin and protected the borders of the country in 1962 .
+the first one is the first seven speechless minutes of the film which expresses the monotonous life of charu and second one is " the scene of the cradle in the garden " where charu faces with her love for amal .
+a group of 18th century thinkers and writers , developed the idea of the economy as a circular flow of income and output .
+foreplay is a set of intimate psychological and physical acts and sexual arousal activities before penetrating sex organ .
+this virus could only be found in the pigs which would cause influenza among the pigs .
+it constitutes mycelium together .
+russia is currently a democratic country .
+sex
+this state of intercourse is called orgasm
+several large empires had been built here in different course of history .
+macro economics
+computer interface is the most visible part to a computer user .
+details : the temporary government of bangladesh of 1971
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/packed/reference.en.1
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/packed/reference.en.1 b/joshua-core/src/test/resources/bn-en/packed/reference.en.1
new file mode 100644
index 0000000..7f83452
--- /dev/null
+++ b/joshua-core/src/test/resources/bn-en/packed/reference.en.1
@@ -0,0 +1,100 @@
+rabindranath was born in a " pirali brahmin " family in kolkata .
+recently , the relation between india and united states developed .
+therefore mathematics is the language of science .
+this is verygood machine which can be used
+simultaneously , a presage of decline of bengal 's landed feudal system is also found in this novel .
+mujibur and his party got an absolute majority in the election .
+hitler also worked at the city of munich in bavaria .
+other than tux there were many other characters like os tyan but these are not so popular .
+it explains the common rules for the decision maker in taking decisions in the playground while dealing with the opponents in the face of social condition .
+rain : total amount of rain in a year is 2540 milli meter .
+in 2008 he gave the keynote speech in national conference of democrat party organised in boston , massachusetts .
+as the division of power based on population favored east pakistn , west pakistan proposed a novel idea of ' one unit theory ' wherein entire west pakistan was considered as a province .
+measurement function
+outer communication
+tata communication bidesh sanchar nigam ltd bhavan , this is one of the unique telecommunication service in the city .
+he won in the national election on that year on 4th november and became as the 44th president of the united states .
+many indian races have been emerged from the texa community of the gondoana lands .
+dramas , novels , stories and recent screenplays of british authors are appreciated worldwide .
+it published in may , year 1919 in shwagat magazine .
+in year 2005 sunfeast open of women tennis association was organized in netaji indoor stadium .
+many steps are taken to eradicate this likely occurance .
+on february 5,1966 a national conference of the opposition parties was held in lahore .
+accepted membership of bangladesh organization of the islamic conference and islamic development bank .
+subject : world cell
+russia , france and italy are the main countries to supply arms and are helpful neighborhood countries .
+this is our known imaginary unit with the help of which mathematical theories get promoted to the set of complex numbers from the set of real numbers .
+<address>
+september
+this theory , however , does not oppose big bang theory rather supports it .
+france has historically been an important producer of agricultural products ; basically they export crops , wine , cheese and other agricultural products to europe and other parts of the world .
+arithmetic was the most important in their mathematics .
+these countries are : france , hong kong , china , belgium , switzerland , germany , denmark , sweden , austria , czechoslovakia , argentina , italy , norway , hungary , yugoslavia , bulgaria , romania , greece , egypt , singapore , indonesia , thailand , japan , myanmar , holland , soviet russia , iran , iraq and sri lanka .
+presently , the bank of london is situated in the place of basilica .
+the bay of biscay is in its north and strait of gibraltarto is in the south and morocco is in the south of the strait and the atlantic ocean in the west and south-west .
+besides , this also demonstrate the inability of the united nations to take quick decisions at the moment of emergency .
+it derives from the work of karl marx .
+its subjects are adopted sometimes from mythologies , sometimes from love affairs of the middle age and even from present day social and political events .
+on the basis of three measurements , the age has been determined as almost 13.7 � 0.2 billion years .
+there is east russia nearby , which is situated on the other side of sea of okhotsk and sea of japan .
+national library of india located in kolkata is one of the leading libraries in the country .
+mycology
+ban ki moon is the secretary general of the united nations .
+the creator of minix was andrew tanenbaum , a famous teacher of operating system design .
+in the times of india it was written that , " it is absurd to compare it with any other indian cinema . pather panchali is pure cinema " .
+after that in 1953 of may , nazrul and pamila devi were sent to london for treatment .
+there are wide flat lands in its southern and eastern side ; while there are wild hills and mountains in the western and northern side .
+trademark
+during the rule of lord wellesley -lrb- governor general 1797-1805 -rrb- there had been a considerable growth of cities .
+complex numbers are must for solving many important and real problems .
+one mention worthy result of big bang theory is that , the recent condition of the universe is completely different from that of the past and future .
+windows millennium
+rabindranath , however , mastered more than one style .
+labor economy .
+britain was once the prime and aggressive economic power of the world .
+he was appreciated for taking forward the protest of the bengali community towards the independence against pakistan military government and clearing the differences within the groups .
+it should be mentioned here that although the terms internet and world wide web are often used in everyday speech without much distinction these are not one and the same .
+the two accompanying polar co-ordinates of z are r = -pipe-
+november
+in 1972 bc at the mean time electronic mail is sent at first in orpanet .
+in that branch of biology where there is discussion about fungus is called as mycology .
+water was drawn from the river by a series of purs , an animal-powered rope and bucket mechanism .
+among them there are : tribal dance , folk dance , traditional or classical dance etc .
+the oldest literature in india became popular initially in oral form and then in writing .
+in 1989 he made ganashatru but his direction was comparably week and was considered as a retry of making film after a long suffering illness .
+mechanics
+linux is different in many ways from other licenses protected operating systems like windows and mac os .
+according to " asia times " : -
+open source means the free distribution of source code of author 's computer software
+dhaka on bangladesh online
+germans lost the 1st world war
+it only takes a few hours to be completed.the social customs are certain
+super h
+he was declared unfit for the military force .
+bhutto declared by threatening to boycott the assembly that he would not recognize that government if yahya khan would call mujib to form the government .
+and the meaning of " computer " is enumerator .
+on 4th july of 1776 , these colonies issued a declaration of independence .
+germany -lrb- deutschland in german -rrb- is a country of the middle europe .
+christianity is the main religion for russian .
+but the romanization of education was slower for the gaul .
+subject : gonu foundation
+it also studies effects of monetary policy and fiscal policy .
+these contain " haito tomar pabo dekha , " o re e kone sneha-surdhuni
+on 23th april 1992 his life came to an end .
+at this time nazrul 's medical report was sent to the famous doctors in vienna .
+other than acting rani mukherjee was found many times to get attached to many of the charitable institutions .
+on 9th december1974 , the bangladesh university honored him with d. lit as recognition of his contribution in bengali literacy and culture .
+kolkata is also the most attractive place for durga puja .
+but lakhs of german started to come in democratic western germany for its rich heritage and wealth from the year 1961 and so the government built up a wall in the southern germany and made the boundary of the country stronger .
+the first one is the silent seven minutes at the beginning which reflects the monotony of charu 's life and the second one is the " swinging in a cradle in the garden " where charu realised her love for amal .
+some thinkers and writers of 18th century by the cycling of income and production made development in economical thoughts .
+the sexual activities before intercourse is called foreplay .
+it was mainly seen in pigs by which the pigs were being infected .
+all of these together formed mycelium .
+russia is at present a democratic country .
+penis
+this is called organism .
+different vast empires were established here in different periods of history .
+micro economics
+to a user , the most visible form of an operating system is the computer interface .
+descriptive : temporary bangladesh government of 1971
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/packed/reference.en.2
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/packed/reference.en.2 b/joshua-core/src/test/resources/bn-en/packed/reference.en.2
new file mode 100644
index 0000000..9588db6
--- /dev/null
+++ b/joshua-core/src/test/resources/bn-en/packed/reference.en.2
@@ -0,0 +1,100 @@
+rabindranath was born of a pirali brahmin family of calcutta .
+recently the relationship between india and united states has improved .
+so mathematics is the language of science .
+it can be easily understood from it that this metric would be frw metric .
+at the same time indication of the end of bengal 's land centric feudalism is also obtained from this novel .
+mujib and his party got absolute majority in the election .
+hitler also started his work in the state named bavaria miunik
+other than task there are some other western and some other agencies of linux are also present but these are not very much introduced .
+it explains the rules of decision making in a game with one 's rival in the context of social condition .
+rain : yearly 2540 millimeter
+in 2004 he delivered the keynote speech in the national conference of the democrat party in the boston city of massachusetts .
+since a straightforward system of representation based on population would have concentrated political power in east pakistan , the west pakistani establishment came up with the " one unit " scheme , where all of west pakistan was considered one province
+quantum theory
+outer link
+bidesh sanchar nigam limited bhavan of tata communication is one of the main telecom service provider of the city .
+he won the national election on 4th november that year and was elected as the 44th us president .
+today , every one of us is in a problem of ageing in life.quick tips to look fresh and glamorous
+drama , novel , story and recently screenplay written by british writers are adored throughout the world .
+it was published in the sowgat in 1919
+sunfeast open , the tier three tournament of women \u2019 s tennis association tour was held in netaji indoor stadium in 2005 .
+to remove this possibility various steps are taken .
+on 5th february 1966 a religional meeting was arranged by the opposition party .
+he became the members of bangladesh organization of the islamic conference and islamic development bank .
+topics : encyclopedia
+russia , france and israel are the main suppliers of arms and also help in the defence system .
+this is our known imaginary number with which the mathematical theory to deduce complex number set from real number set .
+<address>
+september
+though the theory of big bang can not be opposed through this theory , rather it can be supported .
+france is the most important country of europe ; mainly it exports wine , paneer , food grain to whole europe and other part of the world .
+the dominance of arithmetics can be found in their mathematics .
+the countries are : france , hong kong , china , belgium , switzerland , germany , denmark , sweden , austria , czechoslovakia , argentina , italy , norway , hungary , yugoslavia , bulgaria , romania , greece , egypt , singapore , indonesia , thailand , japan , burma , holland , soviet russia , iran , iraq and sri lanka .
+at the place of that basilica , bank of england is now situated .
+north side of this country has bay of biskay , south side has gibralta and in south west side has atlantic ocean .
+moreover in this urgent situation , it declares the inability of leage of nations for fast decision .
+it is conducted through the works of karl marx .
+it \u2019 s content are taken sometime from mythology , love stories of middle ages or sometime from social and political incidents of recent time .
+the age of the universe that is derived based on three measurements is almost 13.7 � 0.2 billion years
+eastern russia is near , which is situated opposite of the okhotsk sea and japan sea .
+national library of india which is in kolkata is the oldest library .
+mycology
+the president of united nation is " wan ki moon " .
+the writer of minix was andrew tanenbaum , a famous operating system design teacher .
+the times of india wrote that " it is absurd to compare it with any other indian cinema ... pather panchali is pure cinema " .
+after this , in the month of may , 1953b.c nazrul and prameladevi were sent to london for treatment .
+in its south are vast plains , in the west and north are rough hills and mountains .
+trademark
+notable development of the city happens in time of lord wellesley -lrb- governor general 1797-1805 -rrb- .
+complex numbers are indispensable to solve many important and real life problems .
+one important result of the big bang is completely different situations of the past , present and future universe .
+windows millennium
+though rabindranath brought multiple saili within range .
+labour economics
+once upon a time in whole world britain has the most power full and fastest economy .
+he is greatly praised for organizing movement against the military rule of pakistan and protesting against racial discrimination by pakistan and for channelizing the movement of bengalis towards attaining freedom .
+it must be mentioned here that nevertheless most of us consider the internet and the www to be the same thing but these two words , infact , direct to completely different subjects .
+polar coordinate corresponding to z are r = -pipe-
+novewrmber
+electronic mail was sent for the first time with arpanet in 1972 .
+mycology is a science in which mushroom and its uses are described .
+water was collected from the river by the help of rope and domestic animals . rope was tied with a bucket and that was pulled up by domestic animals . this is an old process of collecting water from river
+tribal dance , folk dance , classical dance etc are few among them .
+early works of indian literature was practiced orally at first and later in written form .
+the first , ganashatru is considered the weakest of the three and endeavor to restore his energy to make films
+mechanics
+linux is different in many ways from other proprietary softwares like windows and mac .
+according to the asia times ,
+the meaning of open source is freely publishing the source code of computer software .
+dhaka in bangladesh online .
+germany lost the first world war .
+however , researches are even now going on to have a further understanding of the subject .
+not shown here ...
+he was declared unfit for the army .
+threatening to boycott assembly , he declared that he will not accept the government , if yahia khan calls mujib to form government
+and the meaning of the word ' computer ' is calculating machine .
+these colonies announced declaration of independence on july 4 , 1716 .
+germany -lrb- in german language : deutschland , do-yoch-lant -rrb- is a country of middle europe .
+christianity is the principal religion of russia .
+but the romanization of the education of gauls was slow moving .
+category : gnu foundation .
+monetary and fiscal policies are studied .
+among these were ' hayta tomar pabo dekha ' and ' ore e kon sneha-suradhuni ' .
+he died on 23 april 1992 .
+at this time , nazrul 's medical report was sent to famous surgeons in vienna .
+apart from acting rani mukherjee has kept herself attached with several charitable organizations in different times .
+on 9th december , year 1974 dhaka university gave him d.lit degree to give him respect .
+the durgapuja festival of calcutta is also a tourists ' attraction .
+but when lakhs of east germans started moving to the west german because it was economically developed and a republic , the east german government strengthened the boarders by constructing a wall in berlin in 1961 .
+first one is the speechless first seven minutes , which portrayed the monotony of charu 's life , and the second one is the scene of the " swing in the garden " , where charu first found his love for amol .
+in the 18th century a group of intellectuals improved the economic ideologies by incorporating the circle of income and manufacturing .
+before copulation the things done to get sex is called as sringer .
+it was generally got in between the pigs which attacked the pigs or its kinds .
+all this together form mycelium .
+presently russia is a democratic country .
+penis
+this condition is called as ragmochon .
+in different phases of history , a number of huge empires were established here .
+micro economics
+user can see the operating system as computer interface .
+in details : the temporary government of bangladesh of 1971 .
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/packed/reference.en.3
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/packed/reference.en.3 b/joshua-core/src/test/resources/bn-en/packed/reference.en.3
new file mode 100644
index 0000000..20a8c75
--- /dev/null
+++ b/joshua-core/src/test/resources/bn-en/packed/reference.en.3
@@ -0,0 +1,100 @@
+robindranath was born in a pirali brahman family .
+recently relation of india with the united states has improved .
+mathematics is thus the language of science .
+it easily understood from it that this metric is frw metric .
+the decline of imperilism is also reflected from this novel
+mujib and his party secured sweeping majority in the election .
+hitler continued to operate at the city of munich in bavaria .
+other than tux , there are o s tan and few more character to represent linux , but those are not very popular .
+this is not an identical task to complete with but to do some thing incredible and spectacular
+rainfall : annually 2580 mm
+in 2004 he give keynote speech in national assembly of democrat party in boston city of massachusetts province .
+division of political power according to the popoulation was favouring eest pakistan , therfore they invented an unique " one unit " theory in which the entire west pakistan was considered as a seperate state .
+measurement theory
+bringing together the relation with outside
+videsh sanchar nigam limited of tata communications is one of the main provider of cities telecommunication service .
+he won the national election on november , 4 in the same year and elected the 44th us president
+many indian tribes came up from the gondwana 's tribe teska .
+drama , story , literature , art of london is still very famous .
+in may 1919 it was published in ' sawgath ' magazine .
+the 2005 sunfeast open , the tier three tournament of the women 's tennis association tours , was organized in the netaji indoor stadium .
+to make this probability zero many kind pf protection is taken during sex .
+on 5th february , 1966 a national conference held of the all opposition parties in lahore .
+bangladesh take the membership of organization of the islamic conference and the islamic development bank .
+subject : encyclopedia
+russia , france and israel are the main ammunition supplier and security assisting countries to india
+this is our known imaginary unit through which the theories of the mathematics are developed to the set of complex numbers from the the set of real numbers .
+<address>
+september
+through this theory the big bang can not be protested but can be supported .
+france is one of the most important countries of europe in agriculture ; it mainly exports different agricultural products including wheat , wines , cottage cheese etc to europe and other parts of the world .
+their mathematics was basically airthmatic .
+the countries are : france , hongkong , china , belgium , switzerland , germany . denmark , sweden , austria , czechoslovakia , argentina , italy , norway , hungary , yugoslavia bulgaria , rumania , greece , singapore , indonesia , thiland , japan , burma , holland , soviet russia , iraq and srilanka
+in the place of this basilica now the bank of england is situated .
+on the north , it borders bay of biscay ; on the south , it borders gibraltar and morocco , and the atlantic ocean on the northwest and southwest .
+moreover , uno 's incapability to initiate quick action in the emergencies was also came into the lights .
+there is a scenario of togetherness spread at every every corner .
+the subject matters are taken sometimes from puran , sometimes from middle age love stories and sometimes from the modern social and political incidents .
+depending upon the three measurements the age that is calculated of the universe is 13.7 + - 0.2 billion years .
+in the near by there is east russia which is at the opposite banks of the okhotsk sea and japan sea .
+the national library of india is situated in kolkata and is the country \u2019 s leading public library .
+mycology
+the secretary general of the united nation is ban ki moon .
+the writer of minx was andrew tannenbaum , a renowned teacher of operating design .
+in the times of india it was written that " it is absurd to compare with any other indian cinema .. pather panchali is pure cinema " .
+thereafter in may 1953 nazrul and promila devi were sent to london for medical treatment .
+in south and east the vast plane and in west and north mountain are found in this state .
+trademark .
+during the ruling period of lord welesly -lrb- governor-general 1917-1805 -rrb- in the city mentionable increment took place .
+to solve several important and practical problems , the complex numbers are inevitable .
+a mentionable fact about great explosive theory is that , the present state of the earth is very much dissimilar from the past and the future state of the earth .
+windows millennium
+though rabindranath mastered more than one style .
+labour economics
+once britain was the prime advanced economic power of the world .
+revolt against the inequality in group and to enforce the bengali movement and also to revolt against the pakisthani military rule , for all this activity mujib is honored by all .
+it is necessary to mention that many people considers internet and world wide web as similar meaning words but in actuality they are different subjects .
+two polar coordinate of .z is r = -pipe- .
+november
+in 1972 the first electronic mail was sent on arpanet .
+mycology is the branch of biology concerned with the study of fungi and their use to humans .
+water used to be drawn from the river with thick rope and bucket pulled by the animals .
+among these are : tribal dance , folk dance , classical dance etc .
+the oldest literature of india were first in vogue through verbally and the in written .
+his direction was comparatively weak in his film ' ganashatru ' -lrb- the enemy of the people -rrb- in 1989 and this film has been considered as his trial of making films after recovering from long illness .
+powerful
+linux is different in many ways from other copy righted operating systems such as windows and mac oc
+according the news of the asia times ,
+however , the company 's army led by robert clive was able to recapture kolkata .
+dhaka is in bangladesh .
+germany was defeated in the first world war .
+but to understand this aspect the research works are progressing .
+superh
+he was declared unfit for the army .
+the day has since been observed as language movement day in bangladesh ,
+and the meaning of the word computer is machine to calculate .
+on the 4th july , 1776 these colonies had collectively issued a declaration of independence .
+germany -lrb- deutschland in german language -rrb- is a country of central europe .
+christ is the main religion of russia .
+but the speed of the romanization of gall 's education system was slow .
+subject : gonu foundation
+it also studies economic and revenue policy .
+among this are : hoyto tomar pap dekha , o hein a kon sneho sur dhoni .
+on the 23rd april , 1992 satyajit died .
+this time medical report of najrul was sent to the famous doctor of vienna .
+in addition to acting , rani mukerji remained involved with various charity organizations .
+for his great contribution to the bengali literature and culture , he was conferred on with the honorary d.lit. by the dhaka university on 9th december , 1974 .
+' durga puja ' is also a tourist attraction of kolkata city .
+but when many people started to migrate to the economically enriched and democratic west germany , east germany built a wall in berlin in 1961 and tightened the country 's border .
+the first one is the silent first seven minutes where the boring life of charu was shown and the second one is the scene of the cradle at the garden where charu realised her love for amal .
+in 18th century a group of intellectuals and writers made advancement in the field of economic thinking using the cyclic movement of earning and production .
+the sexually stimulating activities carried on before actual penetration are called foreplay .
+this virus was primarily found in pigs , which caused the flu in the pigs .
+these are collectively called a mycelium
+now russia is a democratic country .
+at that time , conflict between the french and the company occurred frequently .
+this is termed as orgasm .
+in different phases of history , multiple vast empires were established here .
+microeconomics
+to the user , computer interface is the most visible impersonation of operating systems .
+details : temporary government of bangladesh , 1971
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/packed/reference.en.all
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/packed/reference.en.all b/joshua-core/src/test/resources/bn-en/packed/reference.en.all
new file mode 100644
index 0000000..5aa465f
--- /dev/null
+++ b/joshua-core/src/test/resources/bn-en/packed/reference.en.all
@@ -0,0 +1,400 @@
+rabindranath was born in a pirali brahmin family of kolkata .
+rabindranath was born in a " pirali brahmin " family in kolkata .
+rabindranath was born of a pirali brahmin family of calcutta .
+robindranath was born in a pirali brahman family .
+recently the relationship between india and united stated has improved .
+recently , the relation between india and united states developed .
+recently the relationship between india and united states has improved .
+recently relation of india with the united states has improved .
+mathematics is , hence , the language of science .
+therefore mathematics is the language of science .
+so mathematics is the language of science .
+mathematics is thus the language of science .
+from this it can easily be understood that this metric will be frw metric .
+this is verygood machine which can be used
+it can be easily understood from it that this metric would be frw metric .
+it easily understood from it that this metric is frw metric .
+at the same time this novel had also indicated to the fall of the land basis feudalism in bengal .
+simultaneously , a presage of decline of bengal 's landed feudal system is also found in this novel .
+at the same time indication of the end of bengal 's land centric feudalism is also obtained from this novel .
+the decline of imperilism is also reflected from this novel
+mujib and his party could attain the absolute majority in the elections .
+mujibur and his party got an absolute majority in the election .
+mujib and his party got absolute majority in the election .
+mujib and his party secured sweeping majority in the election .
+hitlar continued his work in the city of munich at bavariar .
+hitler also worked at the city of munich in bavaria .
+hitler also started his work in the state named bavaria miunik
+hitler continued to operate at the city of munich in bavaria .
+besides tux there are os-tan and some other characters representing linux but are not much popular .
+other than tux there were many other characters like os tyan but these are not so popular .
+other than task there are some other western and some other agencies of linux are also present but these are not very much introduced .
+other than tux , there are o s tan and few more character to represent linux , but those are not very popular .
+it explains the conventional rules for decision making followed by the decision maker in the field of sports in relation to the social background
+it explains the common rules for the decision maker in taking decisions in the playground while dealing with the opponents in the face of social condition .
+it explains the rules of decision making in a game with one 's rival in the context of social condition .
+this is not an identical task to complete with but to do some thing incredible and spectacular
+annual rainfall is 2540 millimeter .
+rain : total amount of rain in a year is 2540 milli meter .
+rain : yearly 2540 millimeter
+rainfall : annually 2580 mm
+he provided his main keynote speech in the national conference of his democrat party in the year 2004 in the boston city of machechuest states .
+in 2008 he gave the keynote speech in national conference of democrat party organised in boston , massachusetts .
+in 2004 he delivered the keynote speech in the national conference of the democrat party in the boston city of massachusetts .
+in 2004 he give keynote speech in national assembly of democrat party in boston city of massachusetts province .
+since the distribution of powers was in favor of the east pakistan if the population was taken into the account , therefore the west pakistan introduced a unique idea under the name of " one unit theory " whereby the entire pakistan was considered as one province .
+as the division of power based on population favored east pakistn , west pakistan proposed a novel idea of ' one unit theory ' wherein entire west pakistan was considered as a province .
+since a straightforward system of representation based on population would have concentrated political power in east pakistan , the west pakistani establishment came up with the " one unit " scheme , where all of west pakistan was considered one province
+division of political power according to the popoulation was favouring eest pakistan , therfore they invented an unique " one unit " theory in which the entire west pakistan was considered as a seperate state .
+measurement theory .
+measurement function
+quantum theory
+measurement theory
+external connections
+outer communication
+outer link
+bringing together the relation with outside
+videsh sanchar nigam limited bhavan of tata communications , this is a major telecom service provider in the city .
+tata communication bidesh sanchar nigam ltd bhavan , this is one of the unique telecommunication service in the city .
+bidesh sanchar nigam limited bhavan of tata communication is one of the main telecom service provider of the city .
+videsh sanchar nigam limited of tata communications is one of the main provider of cities telecommunication service .
+that year on the 4th november , he became victorious in the general election and got elected as the 44th president of the united states .
+he won in the national election on that year on 4th november and became as the 44th president of the united states .
+he won the national election on 4th november that year and was elected as the 44th us president .
+he won the national election on november , 4 in the same year and elected the 44th us president
+many indian species originated from gondwana born taxa .
+many indian races have been emerged from the texa community of the gondoana lands .
+today , every one of us is in a problem of ageing in life.quick tips to look fresh and glamorous
+many indian tribes came up from the gondwana 's tribe teska .
+plays , novels , stories , and recently screenplays written by the british writers are acknowledged worldwide .
+dramas , novels , stories and recent screenplays of british authors are appreciated worldwide .
+drama , novel , story and recently screenplay written by british writers are adored throughout the world .
+drama , story , literature , art of london is still very famous .
+on 1919 it was published in a magazine called swagat .
+it published in may , year 1919 in shwagat magazine .
+it was published in the sowgat in 1919
+in may 1919 it was published in ' sawgath ' magazine .
+in 2005 , the women 's tennis association tour 's tier-iii tournament , sunfeast open was held in netaji indoor stadium .
+in year 2005 sunfeast open of women tennis association was organized in netaji indoor stadium .
+sunfeast open , the tier three tournament of women \u2019 s tennis association tour was held in netaji indoor stadium in 2005 .
+the 2005 sunfeast open , the tier three tournament of the women 's tennis association tours , was organized in the netaji indoor stadium .
+several provisions are taken to avoid this possibility
+many steps are taken to eradicate this likely occurance .
+to remove this possibility various steps are taken .
+to make this probability zero many kind pf protection is taken during sex .
+in lahore , a national conference of the opponent parties was held on 5th february , 1955 .
+on february 5,1966 a national conference of the opposition parties was held in lahore .
+on 5th february 1966 a religional meeting was arranged by the opposition party .
+on 5th february , 1966 a national conference held of the all opposition parties in lahore .
+bangladesh became the member of the organization of the islamic conference and islamic development bank .
+accepted membership of bangladesh organization of the islamic conference and islamic development bank .
+he became the members of bangladesh organization of the islamic conference and islamic development bank .
+bangladesh take the membership of organization of the islamic conference and the islamic development bank .
+special class : world dictionary
+subject : world cell
+topics : encyclopedia
+subject : encyclopedia
+russia , france and israel are the main countries supplying arms to india and defense associates .
+russia , france and italy are the main countries to supply arms and are helpful neighborhood countries .
+russia , france and israel are the main suppliers of arms and also help in the defence system .
+russia , france and israel are the main ammunition supplier and security assisting countries to india
+this is our familiar imaginary unit which relegates theory of mathematics to concern itself with set of complex numbers from that of real numbers .
+this is our known imaginary unit with the help of which mathematical theories get promoted to the set of complex numbers from the set of real numbers .
+this is our known imaginary number with which the mathematical theory to deduce complex number set from real number set .
+this is our known imaginary unit through which the theories of the mathematics are developed to the set of complex numbers from the the set of real numbers .
+</address>
+<address>
+<address>
+<address>
+september
+september
+september
+september
+according to this philosophy you can not disagree universe , though the logic has been accepted .
+this theory , however , does not oppose big bang theory rather supports it .
+though the theory of big bang can not be opposed through this theory , rather it can be supported .
+through this theory the big bang can not be protested but can be supported .
+france is the most important country in europe in agriculture ; it export mainly food crop , wine , cheese , and other agricultural product to europe and the world .
+france has historically been an important producer of agricultural products ; basically they export crops , wine , cheese and other agricultural products to europe and other parts of the world .
+france is the most important country of europe ; mainly it exports wine , paneer , food grain to whole europe and other part of the world .
+france is one of the most important countries of europe in agriculture ; it mainly exports different agricultural products including wheat , wines , cottage cheese etc to europe and other parts of the world .
+arithmetic was prevalent in their mathematics .
+arithmetic was the most important in their mathematics .
+the dominance of arithmetics can be found in their mathematics .
+their mathematics was basically airthmatic .
+the countries are : france , hong kong , china , belgium , switzerland , germany , denmark , sweden , austria , czechoslovakia , argentina , italy , norway , hungary , yugoslavia , bulgaria , romania , greece , egypt , singapore , indonesia , thailand , japan , burma , holland , soviet russia , iran , iraq , and sri lanka .
+these countries are : france , hong kong , china , belgium , switzerland , germany , denmark , sweden , austria , czechoslovakia , argentina , italy , norway , hungary , yugoslavia , bulgaria , romania , greece , egypt , singapore , indonesia , thailand , japan , myanmar , holland , soviet russia , iran , iraq and sri lanka .
+the countries are : france , hong kong , china , belgium , switzerland , germany , denmark , sweden , austria , czechoslovakia , argentina , italy , norway , hungary , yugoslavia , bulgaria , romania , greece , egypt , singapore , indonesia , thailand , japan , burma , holland , soviet russia , iran , iraq and sri lanka .
+the countries are : france , hongkong , china , belgium , switzerland , germany . denmark , sweden , austria , czechoslovakia , argentina , italy , norway , hungary , yugoslavia bulgaria , rumania , greece , singapore , indonesia , thiland , japan , burma , holland , soviet russia , iraq and srilanka
+in the place of this basilica bank of england is located now .
+presently , the bank of london is situated in the place of basilica .
+at the place of that basilica , bank of england is now situated .
+in the place of this basilica now the bank of england is situated .
+to the north and south of spain there are bay of biscay and the gibraltar respectively , morocco lies to south of gibraltar and the atlantic ocean is situated at the west and south-west part of this country .
+the bay of biscay is in its north and strait of gibraltarto is in the south and morocco is in the south of the strait and the atlantic ocean in the west and south-west .
+north side of this country has bay of biskay , south side has gibralta and in south west side has atlantic ocean .
+on the north , it borders bay of biscay ; on the south , it borders gibraltar and morocco , and the atlantic ocean on the northwest and southwest .
+except that , in this situation , the inability of united nations to take decision quickly in emergency situation was realized .
+besides , this also demonstrate the inability of the united nations to take quick decisions at the moment of emergency .
+moreover in this urgent situation , it declares the inability of leage of nations for fast decision .
+moreover , uno 's incapability to initiate quick action in the emergencies was also came into the lights .
+this was popularised by karl marx
+it derives from the work of karl marx .
+it is conducted through the works of karl marx .
+there is a scenario of togetherness spread at every every corner .
+its subject is often borrowed from hindu mythology , medieval romances and news of social and political events .
+its subjects are adopted sometimes from mythologies , sometimes from love affairs of the middle age and even from present day social and political events .
+it \u2019 s content are taken sometime from mythology , love stories of middle ages or sometime from social and political incidents of recent time .
+the subject matters are taken sometimes from puran , sometimes from middle age love stories and sometimes from the modern social and political incidents .
+depending on the three measures , the age of the universe has been found to be 13.7 � 0.2 billion years .
+on the basis of three measurements , the age has been determined as almost 13.7 � 0.2 billion years .
+the age of the universe that is derived based on three measurements is almost 13.7 � 0.2 billion years
+depending upon the three measurements the age that is calculated of the universe is 13.7 + - 0.2 billion years .
+east russia is close by , which is situated on the opposite side of the sea of okhotsk and sea of japan .
+there is east russia nearby , which is situated on the other side of sea of okhotsk and sea of japan .
+eastern russia is near , which is situated opposite of the okhotsk sea and japan sea .
+in the near by there is east russia which is at the opposite banks of the okhotsk sea and japan sea .
+the indian national library in kolkata is the leading public library of the country .
+national library of india located in kolkata is one of the leading libraries in the country .
+national library of india which is in kolkata is the oldest library .
+the national library of india is situated in kolkata and is the country \u2019 s leading public library .
+mycology
+mycology
+mycology
+mycology
+the secretary-general of the un at present is ban ki-moon .
+ban ki moon is the secretary general of the united nations .
+the president of united nation is " wan ki moon " .
+the secretary general of the united nation is ban ki moon .
+the creator of minix was andrew tunnenbom who was a famous teacher of operating system designing .
+the creator of minix was andrew tanenbaum , a famous teacher of operating system design .
+the writer of minix was andrew tanenbaum , a famous operating system design teacher .
+the writer of minx was andrew tannenbaum , a renowned teacher of operating design .
+the times of india wrote that " it is absurd to compare it with any other indian cinema ... pather panchali is pure cinema " .
+in the times of india it was written that , " it is absurd to compare it with any other indian cinema . pather panchali is pure cinema " .
+the times of india wrote that " it is absurd to compare it with any other indian cinema ... pather panchali is pure cinema " .
+in the times of india it was written that " it is absurd to compare with any other indian cinema .. pather panchali is pure cinema " .
+after that in 1953 , during the month of may , nazrul and his wife pramila devi were sent to london for better treatment .
+after that in 1953 of may , nazrul and pamila devi were sent to london for treatment .
+after this , in the month of may , 1953b.c nazrul and prameladevi were sent to london for treatment .
+thereafter in may 1953 nazrul and promila devi were sent to london for medical treatment .
+it has got plain lands in its south and east sides and rough hills and mountains in its west and north sides .
+there are wide flat lands in its southern and eastern side ; while there are wild hills and mountains in the western and northern side .
+in its south are vast plains , in the west and north are rough hills and mountains .
+in south and east the vast plane and in west and north mountain are found in this state .
+trademark
+trademark
+trademark
+trademark .
+lord wellesley , the governor general between 1797 \u2013 1805 , was largely responsible for the growth of the city
+during the rule of lord wellesley -lrb- governor general 1797-1805 -rrb- there had been a considerable growth of cities .
+notable development of the city happens in time of lord wellesley -lrb- governor general 1797-1805 -rrb- .
+during the ruling period of lord welesly -lrb- governor-general 1917-1805 -rrb- in the city mentionable increment took place .
+complex numbers are indispensible for solving many important and real problems .
+complex numbers are must for solving many important and real problems .
+complex numbers are indispensable to solve many important and real life problems .
+to solve several important and practical problems , the complex numbers are inevitable .
+an important consequence of the big bang is that the present state of the universe is completely different from its past and future states .
+one mention worthy result of big bang theory is that , the recent condition of the universe is completely different from that of the past and future .
+one important result of the big bang is completely different situations of the past , present and future universe .
+a mentionable fact about great explosive theory is that , the present state of the earth is very much dissimilar from the past and the future state of the earth .
+windows millennium
+windows millennium
+windows millennium
+windows millennium
+although rabindranath had subjugated a number of styles .
+rabindranath , however , mastered more than one style .
+though rabindranath brought multiple saili within range .
+though rabindranath mastered more than one style .
+labor economy
+labor economy .
+labour economics
+labour economics
+britain was once the most powerful and advance economic power of the world .
+britain was once the prime and aggressive economic power of the world .
+once upon a time in whole world britain has the most power full and fastest economy .
+once britain was the prime advanced economic power of the world .
+he is highly admired due to the movement against the military ruler of pakistan and the protest against the racial inequities in pakistan and moving the movement of bengalees towards independence .
+he was appreciated for taking forward the protest of the bengali community towards the independence against pakistan military government and clearing the differences within the groups .
+he is greatly praised for organizing movement against the military rule of pakistan and protesting against racial discrimination by pakistan and for channelizing the movement of bengalis towards attaining freedom .
+revolt against the inequality in group and to enforce the bengali movement and also to revolt against the pakisthani military rule , for all this activity mujib is honored by all .
+though it is thought that the meaning of internet and world wide web but actually they refer two completely different things .
+it should be mentioned here that although the terms internet and world wide web are often used in everyday speech without much distinction these are not one and the same .
+it must be mentioned here that nevertheless most of us consider the internet and the www to be the same thing but these two words , infact , direct to completely different subjects .
+it is necessary to mention that many people considers internet and world wide web as similar meaning words but in actuality they are different subjects .
+the two relevant polar coordinates of the z are r = 1
+the two accompanying polar co-ordinates of z are r = -pipe-
+polar coordinate corresponding to z are r = -pipe-
+two polar coordinate of .z is r = -pipe- .
+november
+november
+novewrmber
+november
+the first electronics mail used was in 1972 ad in the arpanet .
+in 1972 bc at the mean time electronic mail is sent at first in orpanet .
+electronic mail was sent for the first time with arpanet in 1972 .
+in 1972 the first electronic mail was sent on arpanet .
+the section of biology which discusses fungi and its practical source is called mycology .
+in that branch of biology where there is discussion about fungus is called as mycology .
+mycology is a science in which mushroom and its uses are described .
+mycology is the branch of biology concerned with the study of fungi and their use to humans .
+the water was carried up with some pur , this was actually a method of pulling up water by animals with an arrangement of ropes and buckets .
+water was drawn from the river by a series of purs , an animal-powered rope and bucket mechanism .
+water was collected from the river by the help of rope and domestic animals . rope was tied with a bucket and that was pulled up by domestic animals . this is an old process of collecting water from river
+water used to be drawn from the river with thick rope and bucket pulled by the animals .
+these are tribal dance , folk dance , classical dance etc .
+among them there are : tribal dance , folk dance , traditional or classical dance etc .
+tribal dance , folk dance , classical dance etc are few among them .
+among these are : tribal dance , folk dance , classical dance etc .
+the indian literature was spread earlier as orally and later as written
+the oldest literature in india became popular initially in oral form and then in writing .
+early works of indian literature was practiced orally at first and later in written form .
+the oldest literature of india were first in vogue through verbally and the in written .
+his direction in his film " gana satru " in 1989 was weak and it was considered as satyajit 's comeback in making movie after returning from a long-lasting illness .
+in 1989 he made ganashatru but his direction was comparably week and was considered as a retry of making film after a long suffering illness .
+the first , ganashatru is considered the weakest of the three and endeavor to restore his energy to make films
+his direction was comparatively weak in his film ' ganashatru ' -lrb- the enemy of the people -rrb- in 1989 and this film has been considered as his trial of making films after recovering from long illness .
+mechanics .
+mechanics
+mechanics
+powerful
+linux is different from various angle than the other right-protected operating systems such as windows and mac os .
+linux is different in many ways from other licenses protected operating systems like windows and mac os .
+linux is different in many ways from other proprietary softwares like windows and mac .
+linux is different in many ways from other copy righted operating systems such as windows and mac oc
+according to the language of the asian times ,
+according to " asia times " : -
+according to the asia times ,
+according the news of the asia times ,
+open source means free distribution of computer software source code .
+open source means the free distribution of source code of author 's computer software
+the meaning of open source is freely publishing the source code of computer software .
+however , the company 's army led by robert clive was able to recapture kolkata .
+dhaka in bangladesh online
+dhaka on bangladesh online
+dhaka in bangladesh online .
+dhaka is in bangladesh .
+in first war world germany was defeated .
+germans lost the 1st world war
+germany lost the first world war .
+germany was defeated in the first world war .
+but in order to understand this subject experiments are going on at present also .
+it only takes a few hours to be completed.the social customs are certain
+however , researches are even now going on to have a further understanding of the subject .
+but to understand this aspect the research works are progressing .
+super h
+super h
+not shown here ...
+superh
+he was declared unfit for the armed force .
+he was declared unfit for the military force .
+he was declared unfit for the army .
+he was declared unfit for the army .
+threatening to boycott the assembly bhutto announced that , if yahya khan call mujib to form the government he will not do accept that government .
+bhutto declared by threatening to boycott the assembly that he would not recognize that government if yahya khan would call mujib to form the government .
+threatening to boycott assembly , he declared that he will not accept the government , if yahia khan calls mujib to form government
+the day has since been observed as language movement day in bangladesh ,
+and the word computer stands for the machine that does the work of counting .
+and the meaning of " computer " is enumerator .
+and the meaning of the word ' computer ' is calculating machine .
+and the meaning of the word computer is machine to calculate .
+on 4th of july , 1776 these colonies introduced a declaration of independence .
+on 4th july of 1776 , these colonies issued a declaration of independence .
+these colonies announced declaration of independence on july 4 , 1716 .
+on the 4th july , 1776 these colonies had collectively issued a declaration of independence .
+germany -lrb- in german language : deutschland -rrb- is a country of middle europe .
+germany -lrb- deutschland in german -rrb- is a country of the middle europe .
+germany -lrb- in german language : deutschland , do-yoch-lant -rrb- is a country of middle europe .
+germany -lrb- deutschland in german language -rrb- is a country of central europe .
+christianity is the main religion of russia .
+christianity is the main religion for russian .
+christianity is the principal religion of russia .
+christ is the main religion of russia .
+but the development of the educational systems of the gols into roman styles was a bit slow .
+but the romanization of education was slower for the gaul .
+but the romanization of the education of gauls was slow moving .
+but the speed of the romanization of gall 's education system was slow .
+subject : foundation
+subject : gonu foundation
+category : gnu foundation .
+subject : gonu foundation
+it also studies the financial and revenue strategies .
+it also studies effects of monetary policy and fiscal policy .
+monetary and fiscal policies are studied .
+it also studies economic and revenue policy .
+among them there were : ' hoyto tomar pabo dekha ' -lrb- perhaps i will meet you -rrb- , ' ore e kon sneho-surdhani ' -lrb- what an affectionate music is this -rrb- .
+these contain " haito tomar pabo dekha , " o re e kone sneha-surdhuni
+among these were ' hayta tomar pabo dekha ' and ' ore e kon sneha-suradhuni ' .
+among this are : hoyto tomar pap dekha , o hein a kon sneho sur dhoni .
+he died on 23 april 1992 .
+on 23th april 1992 his life came to an end .
+he died on 23 april 1992 .
+on the 23rd april , 1992 satyajit died .
+the medical reports of nazrul was sent to the renowned doctors of vienna at this time .
+at this time nazrul 's medical report was sent to the famous doctors in vienna .
+at this time , nazrul 's medical report was sent to famous surgeons in vienna .
+this time medical report of najrul was sent to the famous doctor of vienna .
+apart from acting rani mukherjee is involved with many charitable organisations .
+other than acting rani mukherjee was found many times to get attached to many of the charitable institutions .
+apart from acting rani mukherjee has kept herself attached with several charitable organizations in different times .
+in addition to acting , rani mukerji remained involved with various charity organizations .
+on december 9 , 1974 , dhaka university honored him with d.lit degree , for his excellent contribution in the field of bengali culture and literature .
+on 9th december1974 , the bangladesh university honored him with d. lit as recognition of his contribution in bengali literacy and culture .
+on 9th december , year 1974 dhaka university gave him d.lit degree to give him respect .
+for his great contribution to the bengali literature and culture , he was conferred on with the honorary d.lit. by the dhaka university on 9th december , 1974 .
+durgapuja in kolkata is a tourist attraction too .
+kolkata is also the most attractive place for durga puja .
+the durgapuja festival of calcutta is also a tourists ' attraction .
+' durga puja ' is also a tourist attraction of kolkata city .
+but when the millions of east germans started to emigrate into the economically developed and democratic country of west germany , the government of east germany had built a wall in berlin and protected the borders of the country in 1962 .
+but lakhs of german started to come in democratic western germany for its rich heritage and wealth from the year 1961 and so the government built up a wall in the southern germany and made the boundary of the country stronger .
+but when lakhs of east germans started moving to the west german because it was economically developed and a republic , the east german government strengthened the boarders by constructing a wall in berlin in 1961 .
+but when many people started to migrate to the economically enriched and democratic west germany , east germany built a wall in berlin in 1961 and tightened the country 's border .
+the first one is the first seven speechless minutes of the film which expresses the monotonous life of charu and second one is " the scene of the cradle in the garden " where charu faces with her love for amal .
+the first one is the silent seven minutes at the beginning which reflects the monotony of charu 's life and the second one is the " swinging in a cradle in the garden " where charu realised her love for amal .
+first one is the speechless first seven minutes , which portrayed the monotony of charu 's life , and the second one is the scene of the " swing in the garden " , where charu first found his love for amol .
+the first one is the silent first seven minutes where the boring life of charu was shown and the second one is the scene of the cradle at the garden where charu realised her love for amal .
+a group of 18th century thinkers and writers , developed the idea of the economy as a circular flow of income and output .
+some thinkers and writers of 18th century by the cycling of income and production made development in economical thoughts .
+in the 18th century a group of intellectuals improved the economic ideologies by incorporating the circle of income and manufacturing .
+in 18th century a group of intellectuals and writers made advancement in the field of economic thinking using the cyclic movement of earning and production .
+foreplay is a set of intimate psychological and physical acts and sexual arousal activities before penetrating sex organ .
+the sexual activities before intercourse is called foreplay .
+before copulation the things done to get sex is called as sringer .
+the sexually stimulating activities carried on before actual penetration are called foreplay .
+this virus could only be found in the pigs which would cause influenza among the pigs .
+it was mainly seen in pigs by which the pigs were being infected .
+it was generally got in between the pigs which attacked the pigs or its kinds .
+this virus was primarily found in pigs , which caused the flu in the pigs .
+it constitutes mycelium together .
+all of these together formed mycelium .
+all this together form mycelium .
+these are collectively called a mycelium
+russia is currently a democratic country .
+russia is at present a democratic country .
+presently russia is a democratic country .
+now russia is a democratic country .
+sex
+penis
+penis
+at that time , conflict between the french and the company occurred frequently .
+this state of intercourse is called orgasm
+this is called organism .
+this condition is called as ragmochon .
+this is termed as orgasm .
+several large empires had been built here in different course of history .
+different vast empires were established here in different periods of history .
+in different phases of history , a number of huge empires were established here .
+in different phases of history , multiple vast empires were established here .
+macro economics
+micro economics
+micro economics
+microeconomics
+computer interface is the most visible part to a computer user .
+to a user , the most visible form of an operating system is the computer interface .
+user can see the operating system as computer interface .
+to the user , computer interface is the most visible impersonation of operating systems .
+details : the temporary government of bangladesh of 1971
+descriptive : temporary bangladesh government of 1971
+in details : the temporary government of bangladesh of 1971 .
+details : temporary government of bangladesh , 1971
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/resources/bn-en/packed/test.sh
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/resources/bn-en/packed/test.sh b/joshua-core/src/test/resources/bn-en/packed/test.sh
new file mode 100755
index 0000000..e7a775e
--- /dev/null
+++ b/joshua-core/src/test/resources/bn-en/packed/test.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -u
+
+cat input.bn | $JOSHUA/bin/joshua-decoder -m 1g -threads 2 -c joshua.config > output 2> log
+
+# Extract the translations and model scores
+cat output | awk -F\| '{print $4 " ||| " $10}' > output.scores
+
+# Compare
+diff -u output.scores output.scores.gold > diff
+
+if [ $? -eq 0 ]; then
+ rm -f diff output log output.scores
+ exit 0
+else
+ exit 1
+fi
+
+
[27/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/pro/PROCore.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/pro/PROCore.java b/joshua-core/src/main/java/org/apache/joshua/pro/PROCore.java
new file mode 100755
index 0000000..290000d
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/pro/PROCore.java
@@ -0,0 +1,3027 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.pro;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.text.DecimalFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Random;
+import java.util.Scanner;
+import java.util.TreeSet;
+import java.util.Vector;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.Decoder;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.metrics.EvaluationMetric;
+import org.apache.joshua.util.StreamGobbler;
+
+import EDU.oswego.cs.dl.util.concurrent.ConcurrentHashMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This code was originally written by Yuan Cao, who copied the MERT code to produce this file.
+ */
+
+public class PROCore {
+
+ private static final Logger LOG = LoggerFactory.getLogger(PROCore.class);
+
+ private final JoshuaConfiguration joshuaConfiguration;
+ private TreeSet<Integer>[] indicesOfInterest_all;
+
+ private final static DecimalFormat f4 = new DecimalFormat("###0.0000");
+ private final Runtime myRuntime = Runtime.getRuntime();
+
+ private final static double NegInf = (-1.0 / 0.0);
+ private final static double PosInf = (+1.0 / 0.0);
+ private final static double epsilon = 1.0 / 1000000;
+
+ private int progress;
+
+ private int verbosity; // anything of priority <= verbosity will be printed
+ // (lower value for priority means more important)
+
+ private Random randGen;
+ private int generatedRands;
+
+ private int numSentences;
+ // number of sentences in the dev set
+ // (aka the "MERT training" set)
+
+ private int numDocuments;
+ // number of documents in the dev set
+ // this should be 1, unless doing doc-level optimization
+
+ private int[] docOfSentence;
+ // docOfSentence[i] stores which document contains the i'th sentence.
+ // docOfSentence is 0-indexed, as are the documents (i.e. first doc is indexed 0)
+
+ private int[] docSubsetInfo;
+ // stores information regarding which subset of the documents are evaluated
+ // [0]: method (0-6)
+ // [1]: first (1-indexed)
+ // [2]: last (1-indexed)
+ // [3]: size
+ // [4]: center
+ // [5]: arg1
+ // [6]: arg2
+ // [1-6] are 0 for method 0, [6] is 0 for methods 1-4 as well
+ // only [1] and [2] are needed for optimization. The rest are only needed for an output message.
+
+ private int refsPerSen;
+ // number of reference translations per sentence
+
+ private int textNormMethod;
+ // 0: no normalization, 1: "NIST-style" tokenization, and also rejoin 'm, 're, *'s, 've, 'll, 'd,
+ // and n't,
+ // 2: apply 1 and also rejoin dashes between letters, 3: apply 1 and also drop non-ASCII
+ // characters
+ // 4: apply 1+2+3
+
+ private int numParams;
+ // total number of firing features
+ // this number may increase overtime as new n-best lists are decoded
+ // initially it is equal to the # of params in the parameter config file
+ private int numParamsOld;
+ // number of features before observing the new features fired in the current iteration
+
+ private double[] normalizationOptions;
+ // How should a lambda[] vector be normalized (before decoding)?
+ // nO[0] = 0: no normalization
+ // nO[0] = 1: scale so that parameter nO[2] has absolute value nO[1]
+ // nO[0] = 2: scale so that the maximum absolute value is nO[1]
+ // nO[0] = 3: scale so that the minimum absolute value is nO[1]
+ // nO[0] = 4: scale so that the L-nO[1] norm equals nO[2]
+
+ /* *********************************************************** */
+ /* NOTE: indexing starts at 1 in the following few arrays: */
+ /* *********************************************************** */
+
+ // private double[] lambda;
+ private ArrayList<Double> lambda = new ArrayList<Double>();
+ // the current weight vector. NOTE: indexing starts at 1.
+ private ArrayList<Double> bestLambda = new ArrayList<Double>();
+ // the best weight vector across all iterations
+
+ private boolean[] isOptimizable;
+ // isOptimizable[c] = true iff lambda[c] should be optimized
+
+ private double[] minRandValue;
+ private double[] maxRandValue;
+ // when choosing a random value for the lambda[c] parameter, it will be
+ // chosen from the [minRandValue[c],maxRandValue[c]] range.
+ // (*) minRandValue and maxRandValue must be real values, but not -Inf or +Inf
+
+ private double[] defaultLambda;
+ // "default" parameter values; simply the values read in the parameter file
+ // USED FOR NON-OPTIMIZABLE (FIXED) FEATURES
+
+ /* *********************************************************** */
+ /* *********************************************************** */
+
+ private Decoder myDecoder;
+ // COMMENT OUT if decoder is not Joshua
+
+ private String decoderCommand;
+ // the command that runs the decoder; read from decoderCommandFileName
+
+ private int decVerbosity;
+ // verbosity level for decoder output. If 0, decoder output is ignored.
+ // If 1, decoder output is printed.
+
+ private int validDecoderExitValue;
+ // return value from running the decoder command that indicates success
+
+ private int numOptThreads;
+ // number of threads to run things in parallel
+
+ private int saveInterFiles;
+ // 0: nothing, 1: only configs, 2: only n-bests, 3: both configs and n-bests
+
+ private int compressFiles;
+ // should PRO gzip the large files? If 0, no compression takes place.
+ // If 1, compression is performed on: decoder output files, temp sents files,
+ // and temp feats files.
+
+ private int sizeOfNBest;
+ // size of N-best list generated by decoder at each iteration
+ // (aka simply N, but N is a bad variable name)
+
+ private long seed;
+ // seed used to create random number generators
+
+ private boolean randInit;
+ // if true, parameters are initialized randomly. If false, parameters
+ // are initialized using values from parameter file.
+
+ private int maxMERTIterations, minMERTIterations, prevMERTIterations;
+ // max: maximum number of MERT iterations
+ // min: minimum number of MERT iterations before an early MERT exit
+ // prev: number of previous MERT iterations from which to consider candidates (in addition to
+ // the candidates from the current iteration)
+
+ private double stopSigValue;
+ // early MERT exit if no weight changes by more than stopSigValue
+ // (but see minMERTIterations above and stopMinIts below)
+
+ private int stopMinIts;
+ // some early stopping criterion must be satisfied in stopMinIts *consecutive* iterations
+ // before an early exit (but see minMERTIterations above)
+
+ private boolean oneModificationPerIteration;
+ // if true, each MERT iteration performs at most one parameter modification.
+ // If false, a new MERT iteration starts (i.e. a new N-best list is
+ // generated) only after the previous iteration reaches a local maximum.
+
+ private String metricName;
+ // name of evaluation metric optimized by MERT
+
+ private String metricName_display;
+ // name of evaluation metric optimized by MERT, possibly with "doc-level " prefixed
+
+ private String[] metricOptions;
+ // options for the evaluation metric (e.g. for BLEU, maxGramLength and effLengthMethod)
+
+ private EvaluationMetric evalMetric;
+ // the evaluation metric used by MERT
+
+ private int suffStatsCount;
+ // number of sufficient statistics for the evaluation metric
+
+ private String tmpDirPrefix;
+ // prefix for the PRO.temp.* files
+
+ private boolean passIterationToDecoder;
+ // should the iteration number be passed as an argument to decoderCommandFileName?
+
+ // used for pro
+ private String classifierAlg; // the classification algorithm(percep, megam, maxent ...)
+ private String[] classifierParams = null; // the param array for each classifier
+ private int Tau;
+ private int Xi;
+ private double interCoef;
+ private double metricDiff;
+ private double prevMetricScore = 0; // final metric score of the previous iteration, used only
+ // when returnBest = true
+ private boolean returnBest = false; // return the best weight during tuning
+
+ private String dirPrefix; // where are all these files located?
+ private String paramsFileName, docInfoFileName, finalLambdaFileName;
+ private String sourceFileName, refFileName, decoderOutFileName;
+ private String decoderConfigFileName, decoderCommandFileName;
+ private String fakeFileNameTemplate, fakeFileNamePrefix, fakeFileNameSuffix;
+
+ // e.g. output.it[1-x].someOldRun would be specified as:
+ // output.it?.someOldRun
+ // and we'd have prefix = "output.it" and suffix = ".sameOldRun"
+
+ // private int useDisk;
+
+ public PROCore(JoshuaConfiguration joshuaConfiguration) {
+ this.joshuaConfiguration = joshuaConfiguration;
+ }
+
+ public PROCore(String[] args, JoshuaConfiguration joshuaConfiguration) {
+ this.joshuaConfiguration = joshuaConfiguration;
+ EvaluationMetric.set_knownMetrics();
+ processArgsArray(args);
+ initialize(0);
+ }
+
+ public PROCore(String configFileName, JoshuaConfiguration joshuaConfiguration) {
+ this.joshuaConfiguration = joshuaConfiguration;
+ EvaluationMetric.set_knownMetrics();
+ processArgsArray(cfgFileToArgsArray(configFileName));
+ initialize(0);
+ }
+
+ private void initialize(int randsToSkip) {
+ println("NegInf: " + NegInf + ", PosInf: " + PosInf + ", epsilon: " + epsilon, 4);
+
+ randGen = new Random(seed);
+ for (int r = 1; r <= randsToSkip; ++r) {
+ randGen.nextDouble();
+ }
+ generatedRands = randsToSkip;
+
+ if (randsToSkip == 0) {
+ println("----------------------------------------------------", 1);
+ println("Initializing...", 1);
+ println("----------------------------------------------------", 1);
+ println("", 1);
+
+ println("Random number generator initialized using seed: " + seed, 1);
+ println("", 1);
+ }
+
+ // COUNT THE TOTAL NUM OF SENTENCES TO BE DECODED, refFileName IS THE COMBINED REFERENCE FILE
+ // NAME(AUTO GENERATED)
+ numSentences = countLines(refFileName) / refsPerSen;
+
+ // ??
+ processDocInfo();
+ // sets numDocuments and docOfSentence[]
+
+ if (numDocuments > 1)
+ metricName_display = "doc-level " + metricName;
+
+ // ??
+ set_docSubsetInfo(docSubsetInfo);
+
+ // count the number of initial features
+ numParams = countNonEmptyLines(paramsFileName) - 1;
+ numParamsOld = numParams;
+
+ // read parameter config file
+ try {
+ // read dense parameter names
+ BufferedReader inFile_names = new BufferedReader(new FileReader(paramsFileName));
+
+ for (int c = 1; c <= numParams; ++c) {
+ String line = "";
+ while (line != null && line.length() == 0) { // skip empty lines
+ line = inFile_names.readLine();
+ }
+
+ // save feature names
+ String paramName = (line.substring(0, line.indexOf("|||"))).trim();
+ Vocabulary.id(paramName);
+ // System.err.println(String.format("VOCAB(%s) = %d", paramName, id));
+ }
+
+ inFile_names.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ // the parameter file contains one line per parameter
+ // and one line for the normalization method
+ // indexing starts at 1 in these arrays
+ for (int p = 0; p <= numParams; ++p)
+ lambda.add(new Double(0));
+ bestLambda.add(new Double(0));
+ // why only lambda is a list? because the size of lambda
+ // may increase over time, but other arrays are specified in
+ // the param config file, only used for initialization
+ isOptimizable = new boolean[1 + numParams];
+ minRandValue = new double[1 + numParams];
+ maxRandValue = new double[1 + numParams];
+ defaultLambda = new double[1 + numParams];
+ normalizationOptions = new double[3];
+
+ // read initial param values
+ processParamFile();
+ // sets the arrays declared just above
+
+ // SentenceInfo.createV(); // uncomment ONLY IF using vocabulary implementation of SentenceInfo
+
+ String[][] refSentences = new String[numSentences][refsPerSen];
+
+ try {
+
+ // read in reference sentences
+ InputStream inStream_refs = new FileInputStream(new File(refFileName));
+ BufferedReader inFile_refs = new BufferedReader(new InputStreamReader(inStream_refs, "utf8"));
+
+ for (int i = 0; i < numSentences; ++i) {
+ for (int r = 0; r < refsPerSen; ++r) {
+ // read the rth reference translation for the ith sentence
+ refSentences[i][r] = inFile_refs.readLine();
+ }
+ }
+
+ inFile_refs.close();
+
+ // normalize reference sentences
+ for (int i = 0; i < numSentences; ++i) {
+ for (int r = 0; r < refsPerSen; ++r) {
+ // normalize the rth reference translation for the ith sentence
+ refSentences[i][r] = normalize(refSentences[i][r], textNormMethod);
+ }
+ }
+
+ // read in decoder command, if any
+ decoderCommand = null;
+ if (decoderCommandFileName != null) {
+ if (fileExists(decoderCommandFileName)) {
+ BufferedReader inFile_comm = new BufferedReader(new FileReader(decoderCommandFileName));
+ decoderCommand = inFile_comm.readLine(); // READ IN DECODE COMMAND
+ inFile_comm.close();
+ }
+ }
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ // set static data members for the EvaluationMetric class
+ EvaluationMetric.set_numSentences(numSentences);
+ EvaluationMetric.set_numDocuments(numDocuments);
+ EvaluationMetric.set_refsPerSen(refsPerSen);
+ EvaluationMetric.set_refSentences(refSentences);
+ EvaluationMetric.set_tmpDirPrefix(tmpDirPrefix);
+
+ evalMetric = EvaluationMetric.getMetric(metricName, metricOptions);
+ // used only if returnBest = true
+ prevMetricScore = evalMetric.getToBeMinimized() ? PosInf : NegInf;
+
+ // length of sufficient statistics
+ // for bleu: suffstatscount=8 (2*ngram+2)
+ suffStatsCount = evalMetric.get_suffStatsCount();
+
+ // set static data members for the IntermediateOptimizer class
+ /*
+ * IntermediateOptimizer.set_MERTparams(numSentences, numDocuments, docOfSentence,
+ * docSubsetInfo, numParams, normalizationOptions, isOptimizable oneModificationPerIteration,
+ * evalMetric, tmpDirPrefix, verbosity);
+ */
+
+ // print info
+ if (randsToSkip == 0) { // i.e. first iteration
+ println("Number of sentences: " + numSentences, 1);
+ println("Number of documents: " + numDocuments, 1);
+ println("Optimizing " + metricName_display, 1);
+
+ /*
+ * print("docSubsetInfo: {", 1); for (int f = 0; f < 6; ++f) print(docSubsetInfo[f] + ", ",
+ * 1); println(docSubsetInfo[6] + "}", 1);
+ */
+
+ println("Number of initial features: " + numParams, 1);
+ print("Initial feature names: {", 1);
+
+ for (int c = 1; c <= numParams; ++c)
+ print("\"" + Vocabulary.word(c) + "\"", 1);
+ println("}", 1);
+ println("", 1);
+
+ // TODO just print the correct info
+ println("c Default value\tOptimizable?\tRand. val. range", 1);
+
+ for (int c = 1; c <= numParams; ++c) {
+ print(c + " " + f4.format(lambda.get(c).doubleValue()) + "\t\t", 1);
+
+ if (!isOptimizable[c]) {
+ println(" No", 1);
+ } else {
+ print(" Yes\t\t", 1);
+ print(" [" + minRandValue[c] + "," + maxRandValue[c] + "]", 1);
+ println("", 1);
+ }
+ }
+
+ println("", 1);
+ print("Weight vector normalization method: ", 1);
+ if (normalizationOptions[0] == 0) {
+ println("none.", 1);
+ } else if (normalizationOptions[0] == 1) {
+ println(
+ "weights will be scaled so that the \""
+ + Vocabulary.word((int) normalizationOptions[2])
+ + "\" weight has an absolute value of " + normalizationOptions[1] + ".", 1);
+ } else if (normalizationOptions[0] == 2) {
+ println("weights will be scaled so that the maximum absolute value is "
+ + normalizationOptions[1] + ".", 1);
+ } else if (normalizationOptions[0] == 3) {
+ println("weights will be scaled so that the minimum absolute value is "
+ + normalizationOptions[1] + ".", 1);
+ } else if (normalizationOptions[0] == 4) {
+ println("weights will be scaled so that the L-" + normalizationOptions[1] + " norm is "
+ + normalizationOptions[2] + ".", 1);
+ }
+
+ println("", 1);
+
+ println("----------------------------------------------------", 1);
+ println("", 1);
+
+ // rename original config file so it doesn't get overwritten
+ // (original name will be restored in finish())
+ renameFile(decoderConfigFileName, decoderConfigFileName + ".PRO.orig");
+ } // if (randsToSkip == 0)
+
+ // by default, load joshua decoder
+ if (decoderCommand == null && fakeFileNameTemplate == null) {
+ println("Loading Joshua decoder...", 1);
+ myDecoder = new Decoder(joshuaConfiguration, decoderConfigFileName + ".PRO.orig");
+ println("...finished loading @ " + (new Date()), 1);
+ println("");
+ } else {
+ myDecoder = null;
+ }
+
+ @SuppressWarnings("unchecked")
+ TreeSet<Integer>[] temp_TSA = new TreeSet[numSentences];
+ indicesOfInterest_all = temp_TSA;
+
+ for (int i = 0; i < numSentences; ++i) {
+ indicesOfInterest_all[i] = new TreeSet<Integer>();
+ }
+ } // void initialize(...)
+
+ // -------------------------
+
+ public void run_PRO() {
+ run_PRO(minMERTIterations, maxMERTIterations, prevMERTIterations);
+ }
+
+ public void run_PRO(int minIts, int maxIts, int prevIts) {
+ // FIRST, CLEAN ALL PREVIOUS TEMP FILES
+ String dir;
+ int k = tmpDirPrefix.lastIndexOf("/");
+ if (k >= 0) {
+ dir = tmpDirPrefix.substring(0, k + 1);
+ } else {
+ dir = "./";
+ }
+ String files;
+ File folder = new File(dir);
+
+ if (folder.exists()) {
+ File[] listOfFiles = folder.listFiles();
+
+ for (int i = 0; i < listOfFiles.length; i++) {
+ if (listOfFiles[i].isFile()) {
+ files = listOfFiles[i].getName();
+ if (files.startsWith("PRO.temp")) {
+ deleteFile(files);
+ }
+ }
+ }
+ }
+
+ println("----------------------------------------------------", 1);
+ println("PRO run started @ " + (new Date()), 1);
+ // printMemoryUsage();
+ println("----------------------------------------------------", 1);
+ println("", 1);
+
+ // if no default lambda is provided
+ if (randInit) {
+ println("Initializing lambda[] randomly.", 1);
+ // initialize optimizable parameters randomly (sampling uniformly from
+ // that parameter's random value range)
+ lambda = randomLambda();
+ }
+
+ println("Initial lambda[]: " + lambdaToString(lambda), 1);
+ println("", 1);
+
+ int[] maxIndex = new int[numSentences];
+
+ // HashMap<Integer,int[]>[] suffStats_array = new HashMap[numSentences];
+ // suffStats_array[i] maps candidates of interest for sentence i to an array
+ // storing the sufficient statistics for that candidate
+
+ int earlyStop = 0;
+ // number of consecutive iteration an early stopping criterion was satisfied
+
+ for (int iteration = 1;; ++iteration) {
+
+ // what does "A" contain?
+ // retA[0]: FINAL_score
+ // retA[1]: earlyStop
+ // retA[2]: should this be the last iteration?
+ double[] A = run_single_iteration(iteration, minIts, maxIts, prevIts, earlyStop, maxIndex);
+ if (A != null) {
+ earlyStop = (int) A[1];
+ if (A[2] == 1)
+ break;
+ } else {
+ break;
+ }
+
+ } // for (iteration)
+
+ println("", 1);
+
+ println("----------------------------------------------------", 1);
+ println("PRO run ended @ " + (new Date()), 1);
+ // printMemoryUsage();
+ println("----------------------------------------------------", 1);
+ println("", 1);
+
+ if (!returnBest)
+ println("FINAL lambda: " + lambdaToString(lambda), 1);
+ // + " (" + metricName_display + ": " + FINAL_score + ")",1);
+ else
+ println("BEST lambda: " + lambdaToString(lambda), 1);
+ // + " (" + metricName_display + ": " + FINAL_score + ")",1);
+
+ // delete intermediate .temp.*.it* decoder output files
+ for (int iteration = 1; iteration <= maxIts; ++iteration) {
+ if (compressFiles == 1) {
+ deleteFile(tmpDirPrefix + "temp.sents.it" + iteration + ".gz");
+ deleteFile(tmpDirPrefix + "temp.feats.it" + iteration + ".gz");
+ if (fileExists(tmpDirPrefix + "temp.stats.it" + iteration + ".copy.gz")) {
+ deleteFile(tmpDirPrefix + "temp.stats.it" + iteration + ".copy.gz");
+ } else {
+ deleteFile(tmpDirPrefix + "temp.stats.it" + iteration + ".gz");
+ }
+ } else {
+ deleteFile(tmpDirPrefix + "temp.sents.it" + iteration);
+ deleteFile(tmpDirPrefix + "temp.feats.it" + iteration);
+ if (fileExists(tmpDirPrefix + "temp.stats.it" + iteration + ".copy")) {
+ deleteFile(tmpDirPrefix + "temp.stats.it" + iteration + ".copy");
+ } else {
+ deleteFile(tmpDirPrefix + "temp.stats.it" + iteration);
+ }
+ }
+ }
+ } // void run_PRO(int maxIts)
+
+ // this is the key function!
+ @SuppressWarnings("unchecked")
+ public double[] run_single_iteration(int iteration, int minIts, int maxIts, int prevIts,
+ int earlyStop, int[] maxIndex) {
+ double FINAL_score = 0;
+
+ double[] retA = new double[3];
+ // retA[0]: FINAL_score
+ // retA[1]: earlyStop
+ // retA[2]: should this be the last iteration?
+
+ boolean done = false;
+ retA[2] = 1; // will only be made 0 if we don't break from the following loop
+
+ // save feats and stats for all candidates(old & new)
+ HashMap<String, String>[] feat_hash = new HashMap[numSentences];
+ for (int i = 0; i < numSentences; i++)
+ feat_hash[i] = new HashMap<String, String>();
+
+ HashMap<String, String>[] stats_hash = new HashMap[numSentences];
+ for (int i = 0; i < numSentences; i++)
+ stats_hash[i] = new HashMap<String, String>();
+
+ while (!done) { // NOTE: this "loop" will only be carried out once
+ println("--- Starting PRO iteration #" + iteration + " @ " + (new Date()) + " ---", 1);
+
+ // printMemoryUsage();
+
+ /******************************/
+ // CREATE DECODER CONFIG FILE //
+ /******************************/
+
+ createConfigFile(lambda, decoderConfigFileName, decoderConfigFileName + ".PRO.orig");
+ // i.e. use the original config file as a template
+
+ /***************/
+ // RUN DECODER //
+ /***************/
+
+ if (iteration == 1) {
+ println("Decoding using initial weight vector " + lambdaToString(lambda), 1);
+ } else {
+ println("Redecoding using weight vector " + lambdaToString(lambda), 1);
+ }
+
+ // generate the n-best file after decoding
+ String[] decRunResult = run_decoder(iteration); // iteration passed in case fake decoder will
+ // be used
+ // [0] name of file to be processed
+ // [1] indicates how the output file was obtained:
+ // 1: external decoder
+ // 2: fake decoder
+ // 3: internal decoder
+
+ if (!decRunResult[1].equals("2")) {
+ println("...finished decoding @ " + (new Date()), 1);
+ }
+
+ checkFile(decRunResult[0]);
+
+ /************* END OF DECODING **************/
+
+ println("Producing temp files for iteration " + iteration, 3);
+
+ produceTempFiles(decRunResult[0], iteration);
+
+ // save intermedidate output files
+ // save joshua.config.pro.it*
+ if (saveInterFiles == 1 || saveInterFiles == 3) { // make copy of intermediate config file
+ if (!copyFile(decoderConfigFileName, decoderConfigFileName + ".PRO.it" + iteration)) {
+ println("Warning: attempt to make copy of decoder config file (to create"
+ + decoderConfigFileName + ".PRO.it" + iteration + ") was unsuccessful!", 1);
+ }
+ }
+
+ // save output.nest.PRO.it*
+ if (saveInterFiles == 2 || saveInterFiles == 3) { // make copy of intermediate decoder output
+ // file...
+
+ if (!decRunResult[1].equals("2")) { // ...but only if no fake decoder
+ if (!decRunResult[0].endsWith(".gz")) {
+ if (!copyFile(decRunResult[0], decRunResult[0] + ".PRO.it" + iteration)) {
+ println("Warning: attempt to make copy of decoder output file (to create"
+ + decRunResult[0] + ".PRO.it" + iteration + ") was unsuccessful!", 1);
+ }
+ } else {
+ String prefix = decRunResult[0].substring(0, decRunResult[0].length() - 3);
+ if (!copyFile(prefix + ".gz", prefix + ".PRO.it" + iteration + ".gz")) {
+ println("Warning: attempt to make copy of decoder output file (to create" + prefix
+ + ".PRO.it" + iteration + ".gz" + ") was unsuccessful!", 1);
+ }
+ }
+
+ if (compressFiles == 1 && !decRunResult[0].endsWith(".gz")) {
+ gzipFile(decRunResult[0] + ".PRO.it" + iteration);
+ }
+ } // if (!fake)
+ }
+
+ // ------------- end of saving .pro.it* files ---------------
+
+ int[] candCount = new int[numSentences];
+ int[] lastUsedIndex = new int[numSentences];
+
+ ConcurrentHashMap[] suffStats_array = new ConcurrentHashMap[numSentences];
+ for (int i = 0; i < numSentences; ++i) {
+ candCount[i] = 0;
+ lastUsedIndex[i] = -1;
+ // suffStats_array[i].clear();
+ suffStats_array[i] = new ConcurrentHashMap();
+ }
+
+ // initLambda[0] is not used!
+ double[] initialLambda = new double[1 + numParams];
+ for (int i = 1; i <= numParams; ++i)
+ initialLambda[i] = lambda.get(i);
+
+ // the "score" in initialScore refers to that
+ // assigned by the evaluation metric)
+
+ // you may consider all candidates from iter 1, or from iter (iteration-prevIts) to current
+ // iteration
+ int firstIt = Math.max(1, iteration - prevIts);
+ // i.e. only process candidates from the current iteration and candidates
+ // from up to prevIts previous iterations.
+ println("Reading candidate translations from iterations " + firstIt + "-" + iteration, 1);
+ println("(and computing " + metricName
+ + " sufficient statistics for previously unseen candidates)", 1);
+ print(" Progress: ");
+
+ int[] newCandidatesAdded = new int[1 + iteration];
+ for (int it = 1; it <= iteration; ++it)
+ newCandidatesAdded[it] = 0;
+
+ try {
+ // read temp files from all past iterations
+ // 3 types of temp files:
+ // 1. output hypo at iter i
+ // 2. feature value of each hypo at iter i
+ // 3. suff stats of each hypo at iter i
+
+ // each inFile corresponds to the output of an iteration
+ // (index 0 is not used; no corresponding index for the current iteration)
+ BufferedReader[] inFile_sents = new BufferedReader[iteration];
+ BufferedReader[] inFile_feats = new BufferedReader[iteration];
+ BufferedReader[] inFile_stats = new BufferedReader[iteration];
+
+ // temp file(array) from previous iterations
+ for (int it = firstIt; it < iteration; ++it) {
+ InputStream inStream_sents, inStream_feats, inStream_stats;
+ if (compressFiles == 0) {
+ inStream_sents = new FileInputStream(tmpDirPrefix + "temp.sents.it" + it);
+ inStream_feats = new FileInputStream(tmpDirPrefix + "temp.feats.it" + it);
+ inStream_stats = new FileInputStream(tmpDirPrefix + "temp.stats.it" + it);
+ } else {
+ inStream_sents = new GZIPInputStream(new FileInputStream(tmpDirPrefix + "temp.sents.it"
+ + it + ".gz"));
+ inStream_feats = new GZIPInputStream(new FileInputStream(tmpDirPrefix + "temp.feats.it"
+ + it + ".gz"));
+ inStream_stats = new GZIPInputStream(new FileInputStream(tmpDirPrefix + "temp.stats.it"
+ + it + ".gz"));
+ }
+
+ inFile_sents[it] = new BufferedReader(new InputStreamReader(inStream_sents, "utf8"));
+ inFile_feats[it] = new BufferedReader(new InputStreamReader(inStream_feats, "utf8"));
+ inFile_stats[it] = new BufferedReader(new InputStreamReader(inStream_stats, "utf8"));
+ }
+
+ InputStream inStream_sentsCurrIt, inStream_featsCurrIt, inStream_statsCurrIt;
+ // temp file for current iteration!
+ if (compressFiles == 0) {
+ inStream_sentsCurrIt = new FileInputStream(tmpDirPrefix + "temp.sents.it" + iteration);
+ inStream_featsCurrIt = new FileInputStream(tmpDirPrefix + "temp.feats.it" + iteration);
+ } else {
+ inStream_sentsCurrIt = new GZIPInputStream(new FileInputStream(tmpDirPrefix
+ + "temp.sents.it" + iteration + ".gz"));
+ inStream_featsCurrIt = new GZIPInputStream(new FileInputStream(tmpDirPrefix
+ + "temp.feats.it" + iteration + ".gz"));
+ }
+
+ BufferedReader inFile_sentsCurrIt = new BufferedReader(new InputStreamReader(
+ inStream_sentsCurrIt, "utf8"));
+ BufferedReader inFile_featsCurrIt = new BufferedReader(new InputStreamReader(
+ inStream_featsCurrIt, "utf8"));
+
+ BufferedReader inFile_statsCurrIt = null; // will only be used if statsCurrIt_exists below
+ // is set to true
+ PrintWriter outFile_statsCurrIt = null; // will only be used if statsCurrIt_exists below is
+ // set to false
+
+ // just to check if temp.stat.it.iteration exists
+ boolean statsCurrIt_exists = false;
+
+ if (fileExists(tmpDirPrefix + "temp.stats.it" + iteration)) {
+ inStream_statsCurrIt = new FileInputStream(tmpDirPrefix + "temp.stats.it" + iteration);
+ inFile_statsCurrIt = new BufferedReader(new InputStreamReader(inStream_statsCurrIt,
+ "utf8"));
+ statsCurrIt_exists = true;
+ copyFile(tmpDirPrefix + "temp.stats.it" + iteration, tmpDirPrefix + "temp.stats.it"
+ + iteration + ".copy");
+ } else if (fileExists(tmpDirPrefix + "temp.stats.it" + iteration + ".gz")) {
+ inStream_statsCurrIt = new GZIPInputStream(new FileInputStream(tmpDirPrefix
+ + "temp.stats.it" + iteration + ".gz"));
+ inFile_statsCurrIt = new BufferedReader(new InputStreamReader(inStream_statsCurrIt,
+ "utf8"));
+ statsCurrIt_exists = true;
+ copyFile(tmpDirPrefix + "temp.stats.it" + iteration + ".gz", tmpDirPrefix
+ + "temp.stats.it" + iteration + ".copy.gz");
+ } else {
+ outFile_statsCurrIt = new PrintWriter(tmpDirPrefix + "temp.stats.it" + iteration);
+ }
+
+ // output the 4^th temp file: *.temp.stats.merged
+ PrintWriter outFile_statsMerged = new PrintWriter(tmpDirPrefix + "temp.stats.merged");
+ // write sufficient statistics from all the sentences
+ // from the output files into a single file
+ PrintWriter outFile_statsMergedKnown = new PrintWriter(tmpDirPrefix
+ + "temp.stats.mergedKnown");
+ // write sufficient statistics from all the sentences
+ // from the output files into a single file
+
+ // output the 5^th 6^th temp file, but will be deleted at the end of the function
+ FileOutputStream outStream_unknownCands = new FileOutputStream(tmpDirPrefix
+ + "temp.currIt.unknownCands", false);
+ OutputStreamWriter outStreamWriter_unknownCands = new OutputStreamWriter(
+ outStream_unknownCands, "utf8");
+ BufferedWriter outFile_unknownCands = new BufferedWriter(outStreamWriter_unknownCands);
+
+ PrintWriter outFile_unknownIndices = new PrintWriter(tmpDirPrefix
+ + "temp.currIt.unknownIndices");
+
+ String sents_str, feats_str, stats_str;
+
+ // BUG: this assumes a candidate string cannot be produced for two
+ // different source sentences, which is not necessarily true
+ // (It's not actually a bug, but only because existingCandStats gets
+ // cleared before moving to the next source sentence.)
+ // FIX: should be made an array, indexed by i
+ HashMap<String, String> existingCandStats = new HashMap<String, String>();
+ // VERY IMPORTANT:
+ // A CANDIDATE X MAY APPEARED IN ITER 1, ITER 3
+ // BUT IF THE USER SPECIFIED TO CONSIDER ITERATIONS FROM ONLY ITER 2, THEN
+ // X IS NOT A "REPEATED" CANDIDATE IN ITER 3. THEREFORE WE WANT TO KEEP THE
+ // SUFF STATS FOR EACH CANDIDATE(TO SAVE COMPUTATION IN THE FUTURE)
+
+ // Stores precalculated sufficient statistics for candidates, in case
+ // the same candidate is seen again. (SS stored as a String.)
+ // Q: Why do we care? If we see the same candidate again, aren't we going
+ // to ignore it? So, why do we care about the SS of this repeat candidate?
+ // A: A "repeat" candidate may not be a repeat candidate in later
+ // iterations if the user specifies a value for prevMERTIterations
+ // that causes MERT to skip candidates from early iterations.
+
+ String[] featVal_str;
+
+ int totalCandidateCount = 0;
+
+ // new candidate size for each sentence
+ int[] sizeUnknown_currIt = new int[numSentences];
+
+ for (int i = 0; i < numSentences; ++i) {
+ // process candidates from previous iterations
+ // low efficiency? for each iteration, it reads in all previous iteration outputs
+ // therefore a lot of overlapping jobs
+ // this is an easy implementation to deal with the situation in which user only specified
+ // "previt" and hopes to consider only the previous previt
+ // iterations, then for each iteration the existing candadites will be different
+ for (int it = firstIt; it < iteration; ++it) {
+ // Why up to but *excluding* iteration?
+ // Because the last iteration is handled a little differently, since
+ // the SS must be calculated (and the corresponding file created),
+ // which is not true for previous iterations.
+
+ for (int n = 0; n <= sizeOfNBest; ++n) {
+ // note that in all temp files, "||||||" is a separator between 2 n-best lists
+
+ // Why up to and *including* sizeOfNBest?
+ // So that it would read the "||||||" separator even if there is
+ // a complete list of sizeOfNBest candidates.
+
+ // for the nth candidate for the ith sentence, read the sentence, feature values,
+ // and sufficient statistics from the various temp files
+
+ // read one line of temp.sent, temp.feat, temp.stats from iteration it
+ sents_str = inFile_sents[it].readLine();
+ feats_str = inFile_feats[it].readLine();
+ stats_str = inFile_stats[it].readLine();
+
+ if (sents_str.equals("||||||")) {
+ n = sizeOfNBest + 1; // move on to the next n-best list
+ } else if (!existingCandStats.containsKey(sents_str)) // if this candidate does not
+ // exist
+ {
+ outFile_statsMergedKnown.println(stats_str);
+
+ // save feats & stats
+ feat_hash[i].put(sents_str, feats_str);
+ stats_hash[i].put(sents_str, stats_str);
+
+ // extract feature value
+ featVal_str = feats_str.split("\\s+");
+
+ if (feats_str.indexOf('=') != -1) {
+ for (String featurePair : featVal_str) {
+ String[] pair = featurePair.split("=");
+ String name = pair[0];
+ Double value = Double.parseDouble(pair[1]);
+ int featId = Vocabulary.id(name);
+ // need to identify newly fired feats here
+ if (featId > numParams) {
+ ++numParams;
+ lambda.add(new Double(0));
+ }
+ }
+ }
+ existingCandStats.put(sents_str, stats_str);
+ candCount[i] += 1;
+ newCandidatesAdded[it] += 1;
+ } // if unseen candidate
+ } // for (n)
+ } // for (it)
+
+ outFile_statsMergedKnown.println("||||||");
+
+ // ---------- end of processing previous iterations ----------
+ // ---------- now start processing new candidates ----------
+
+ // now process the candidates of the current iteration
+ // now determine the new candidates of the current iteration
+
+ /*
+ * remember: BufferedReader inFile_sentsCurrIt BufferedReader inFile_featsCurrIt
+ * PrintWriter outFile_statsCurrIt
+ */
+
+ String[] sentsCurrIt_currSrcSent = new String[sizeOfNBest + 1];
+
+ Vector<String> unknownCands_V = new Vector<String>();
+ // which candidates (of the i'th source sentence) have not been seen before
+ // this iteration?
+
+ for (int n = 0; n <= sizeOfNBest; ++n) {
+ // Why up to and *including* sizeOfNBest?
+ // So that it would read the "||||||" separator even if there is
+ // a complete list of sizeOfNBest candidates.
+
+ // for the nth candidate for the ith sentence, read the sentence,
+ // and store it in the sentsCurrIt_currSrcSent array
+
+ sents_str = inFile_sentsCurrIt.readLine(); // read one candidate from the current
+ // iteration
+ sentsCurrIt_currSrcSent[n] = sents_str; // Note: possibly "||||||"
+
+ if (sents_str.equals("||||||")) {
+ n = sizeOfNBest + 1;
+ } else if (!existingCandStats.containsKey(sents_str)) {
+ unknownCands_V.add(sents_str); // NEW CANDIDATE FROM THIS ITERATION
+ writeLine(sents_str, outFile_unknownCands);
+ outFile_unknownIndices.println(i); // INDEX OF THE NEW CANDIDATES
+ newCandidatesAdded[iteration] += 1;
+ existingCandStats.put(sents_str, "U"); // i.e. unknown
+ // we add sents_str to avoid duplicate entries in unknownCands_V
+ }
+ } // for (n)
+
+ // only compute suff stats for new candidates
+ // now unknownCands_V has the candidates for which we need to calculate
+ // sufficient statistics (for the i'th source sentence)
+ int sizeUnknown = unknownCands_V.size();
+ sizeUnknown_currIt[i] = sizeUnknown;
+
+ existingCandStats.clear();
+
+ } // for (i) each sentence
+
+ // ---------- end of merging candidates stats from previous iterations
+ // and finding new candidates ------------
+
+ /*
+ * int[][] newSuffStats = null; if (!statsCurrIt_exists && sizeUnknown > 0) { newSuffStats =
+ * evalMetric.suffStats(unknownCands, indices); }
+ */
+
+ outFile_statsMergedKnown.close();
+ outFile_unknownCands.close();
+ outFile_unknownIndices.close();
+
+ // want to re-open all temp files and start from scratch again?
+ for (int it = firstIt; it < iteration; ++it) // previous iterations temp files
+ {
+ inFile_sents[it].close();
+ inFile_stats[it].close();
+
+ InputStream inStream_sents, inStream_stats;
+ if (compressFiles == 0) {
+ inStream_sents = new FileInputStream(tmpDirPrefix + "temp.sents.it" + it);
+ inStream_stats = new FileInputStream(tmpDirPrefix + "temp.stats.it" + it);
+ } else {
+ inStream_sents = new GZIPInputStream(new FileInputStream(tmpDirPrefix + "temp.sents.it"
+ + it + ".gz"));
+ inStream_stats = new GZIPInputStream(new FileInputStream(tmpDirPrefix + "temp.stats.it"
+ + it + ".gz"));
+ }
+
+ inFile_sents[it] = new BufferedReader(new InputStreamReader(inStream_sents, "utf8"));
+ inFile_stats[it] = new BufferedReader(new InputStreamReader(inStream_stats, "utf8"));
+ }
+
+ inFile_sentsCurrIt.close();
+ // current iteration temp files
+ if (compressFiles == 0) {
+ inStream_sentsCurrIt = new FileInputStream(tmpDirPrefix + "temp.sents.it" + iteration);
+ } else {
+ inStream_sentsCurrIt = new GZIPInputStream(new FileInputStream(tmpDirPrefix
+ + "temp.sents.it" + iteration + ".gz"));
+ }
+ inFile_sentsCurrIt = new BufferedReader(new InputStreamReader(inStream_sentsCurrIt, "utf8"));
+
+ // calculate SS for unseen candidates and write them to file
+ FileInputStream inStream_statsCurrIt_unknown = null;
+ BufferedReader inFile_statsCurrIt_unknown = null;
+
+ if (!statsCurrIt_exists && newCandidatesAdded[iteration] > 0) {
+ // create the file...
+ evalMetric.createSuffStatsFile(tmpDirPrefix + "temp.currIt.unknownCands", tmpDirPrefix
+ + "temp.currIt.unknownIndices", tmpDirPrefix + "temp.stats.unknown", sizeOfNBest);
+
+ // ...and open it
+ inStream_statsCurrIt_unknown = new FileInputStream(tmpDirPrefix + "temp.stats.unknown");
+ inFile_statsCurrIt_unknown = new BufferedReader(new InputStreamReader(
+ inStream_statsCurrIt_unknown, "utf8"));
+ }
+
+ // open mergedKnown file
+ // newly created by the big loop above
+ FileInputStream instream_statsMergedKnown = new FileInputStream(tmpDirPrefix
+ + "temp.stats.mergedKnown");
+ BufferedReader inFile_statsMergedKnown = new BufferedReader(new InputStreamReader(
+ instream_statsMergedKnown, "utf8"));
+
+ // num of features before observing new firing features from this iteration
+ numParamsOld = numParams;
+
+ for (int i = 0; i < numSentences; ++i) {
+ // reprocess candidates from previous iterations
+ for (int it = firstIt; it < iteration; ++it) {
+ for (int n = 0; n <= sizeOfNBest; ++n) {
+ sents_str = inFile_sents[it].readLine();
+ stats_str = inFile_stats[it].readLine();
+
+ if (sents_str.equals("||||||")) {
+ n = sizeOfNBest + 1;
+ } else if (!existingCandStats.containsKey(sents_str)) {
+ existingCandStats.put(sents_str, stats_str);
+ } // if unseen candidate
+ } // for (n)
+ } // for (it)
+
+ // copy relevant portion from mergedKnown to the merged file
+ String line_mergedKnown = inFile_statsMergedKnown.readLine();
+ while (!line_mergedKnown.equals("||||||")) {
+ outFile_statsMerged.println(line_mergedKnown);
+ line_mergedKnown = inFile_statsMergedKnown.readLine();
+ }
+
+ int[] stats = new int[suffStatsCount];
+
+ for (int n = 0; n <= sizeOfNBest; ++n) {
+ sents_str = inFile_sentsCurrIt.readLine();
+ feats_str = inFile_featsCurrIt.readLine();
+
+ if (sents_str.equals("||||||")) {
+ n = sizeOfNBest + 1;
+ } else if (!existingCandStats.containsKey(sents_str)) {
+
+ if (!statsCurrIt_exists) {
+ stats_str = inFile_statsCurrIt_unknown.readLine();
+
+ String[] temp_stats = stats_str.split("\\s+");
+ for (int s = 0; s < suffStatsCount; ++s) {
+ stats[s] = Integer.parseInt(temp_stats[s]);
+ }
+
+ outFile_statsCurrIt.println(stats_str);
+ } else {
+ stats_str = inFile_statsCurrIt.readLine();
+
+ String[] temp_stats = stats_str.split("\\s+");
+ for (int s = 0; s < suffStatsCount; ++s) {
+ stats[s] = Integer.parseInt(temp_stats[s]);
+ }
+ }
+
+ outFile_statsMerged.println(stats_str);
+
+ // save feats & stats
+ // System.out.println(sents_str+" "+feats_str);
+
+ feat_hash[i].put(sents_str, feats_str);
+ stats_hash[i].put(sents_str, stats_str);
+
+ featVal_str = feats_str.split("\\s+");
+
+ if (feats_str.indexOf('=') != -1) {
+ for (String featurePair : featVal_str) {
+ String[] pair = featurePair.split("=");
+ String name = pair[0];
+ int featId = Vocabulary.id(name);
+ // need to identify newly fired feats here
+ if (featId > numParams) {
+ ++numParams;
+ lambda.add(new Double(0));
+ }
+ }
+ }
+ existingCandStats.put(sents_str, stats_str);
+ candCount[i] += 1;
+
+ // newCandidatesAdded[iteration] += 1;
+ // moved to code above detecting new candidates
+ } else {
+ if (statsCurrIt_exists)
+ inFile_statsCurrIt.readLine();
+ else {
+ // write SS to outFile_statsCurrIt
+ stats_str = existingCandStats.get(sents_str);
+ outFile_statsCurrIt.println(stats_str);
+ }
+ }
+
+ } // for (n)
+
+ // now d = sizeUnknown_currIt[i] - 1
+
+ if (statsCurrIt_exists)
+ inFile_statsCurrIt.readLine();
+ else
+ outFile_statsCurrIt.println("||||||");
+
+ existingCandStats.clear();
+ totalCandidateCount += candCount[i];
+
+ // output sentence progress
+ if ((i + 1) % 500 == 0) {
+ print((i + 1) + "\n" + " ", 1);
+ } else if ((i + 1) % 100 == 0) {
+ print("+", 1);
+ } else if ((i + 1) % 25 == 0) {
+ print(".", 1);
+ }
+
+ } // for (i)
+
+ inFile_statsMergedKnown.close();
+ outFile_statsMerged.close();
+
+ // for testing
+ /*
+ * int total_sent = 0; for( int i=0; i<numSentences; i++ ) {
+ * System.out.println(feat_hash[i].size()+" "+candCount[i]); total_sent +=
+ * feat_hash[i].size(); feat_hash[i].clear(); }
+ * System.out.println("----------------total sent: "+total_sent); total_sent = 0; for( int
+ * i=0; i<numSentences; i++ ) { System.out.println(stats_hash[i].size()+" "+candCount[i]);
+ * total_sent += stats_hash[i].size(); stats_hash[i].clear(); }
+ * System.out.println("*****************total sent: "+total_sent);
+ */
+
+ println("", 1); // finish progress line
+
+ for (int it = firstIt; it < iteration; ++it) {
+ inFile_sents[it].close();
+ inFile_feats[it].close();
+ inFile_stats[it].close();
+ }
+
+ inFile_sentsCurrIt.close();
+ inFile_featsCurrIt.close();
+ if (statsCurrIt_exists)
+ inFile_statsCurrIt.close();
+ else
+ outFile_statsCurrIt.close();
+
+ if (compressFiles == 1 && !statsCurrIt_exists) {
+ gzipFile(tmpDirPrefix + "temp.stats.it" + iteration);
+ }
+
+ // clear temp files
+ deleteFile(tmpDirPrefix + "temp.currIt.unknownCands");
+ deleteFile(tmpDirPrefix + "temp.currIt.unknownIndices");
+ deleteFile(tmpDirPrefix + "temp.stats.unknown");
+ deleteFile(tmpDirPrefix + "temp.stats.mergedKnown");
+
+ // cleanupMemory();
+
+ println("Processed " + totalCandidateCount + " distinct candidates " + "(about "
+ + totalCandidateCount / numSentences + " per sentence):", 1);
+ for (int it = firstIt; it <= iteration; ++it) {
+ println("newCandidatesAdded[it=" + it + "] = " + newCandidatesAdded[it] + " (about "
+ + newCandidatesAdded[it] / numSentences + " per sentence)", 1);
+ }
+
+ println("", 1);
+
+ println("Number of features observed so far: " + numParams);
+ println("", 1);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ // n-best list converges
+ if (newCandidatesAdded[iteration] == 0) {
+ if (!oneModificationPerIteration) {
+ println("No new candidates added in this iteration; exiting PRO.", 1);
+ println("", 1);
+ println("--- PRO iteration #" + iteration + " ending @ " + (new Date()) + " ---", 1);
+ println("", 1);
+ deleteFile(tmpDirPrefix + "temp.stats.merged");
+
+ if (returnBest) {
+ // note that bestLambda.size() <= lambda.size()
+ for (int p = 1; p < bestLambda.size(); ++p)
+ lambda.set(p, bestLambda.get(p));
+ // and set the rest of lambda to be 0
+ for (int p = 0; p < lambda.size() - bestLambda.size(); ++p)
+ lambda.set(p + bestLambda.size(), new Double(0));
+ }
+
+ return null; // this means that the old values should be kept by the caller
+ } else {
+ println("Note: No new candidates added in this iteration.", 1);
+ }
+ }
+
+ /************* start optimization **************/
+
+ /*
+ * for( int v=1; v<initialLambda[1].length; v++ ) System.out.print(initialLambda[1][v]+" ");
+ * System.exit(0);
+ */
+
+ Vector<String> output = new Vector<String>();
+
+ // note: initialLambda[] has length = numParamsOld
+ // augmented with new feature weights, initial values are 0
+ double[] initialLambdaNew = new double[1 + numParams];
+ System.arraycopy(initialLambda, 1, initialLambdaNew, 1, numParamsOld);
+
+ // finalLambda[] has length = numParams (considering new features)
+ double[] finalLambda = new double[1 + numParams];
+
+ Optimizer opt = new Optimizer(seed + iteration, isOptimizable, output, initialLambdaNew,
+ feat_hash, stats_hash, evalMetric, Tau, Xi, metricDiff, normalizationOptions,
+ classifierAlg, classifierParams);
+ finalLambda = opt.run_Optimizer();
+
+ if (returnBest) {
+ double metricScore = opt.getMetricScore();
+ if (!evalMetric.getToBeMinimized()) {
+ if (metricScore > prevMetricScore) {
+ prevMetricScore = metricScore;
+ for (int p = 1; p < bestLambda.size(); ++p)
+ bestLambda.set(p, finalLambda[p]);
+ if (1 + numParams > bestLambda.size()) {
+ for (int p = bestLambda.size(); p <= numParams; ++p)
+ bestLambda.add(p, finalLambda[p]);
+ }
+ }
+ } else {
+ if (metricScore < prevMetricScore) {
+ prevMetricScore = metricScore;
+ for (int p = 1; p < bestLambda.size(); ++p)
+ bestLambda.set(p, finalLambda[p]);
+ if (1 + numParams > bestLambda.size()) {
+ for (int p = bestLambda.size(); p <= numParams; ++p)
+ bestLambda.add(p, finalLambda[p]);
+ }
+ }
+ }
+ }
+
+ // System.out.println(finalLambda.length);
+ // for( int i=0; i<finalLambda.length-1; i++ )
+ // System.out.print(finalLambda[i+1]+" ");
+ // System.out.println();
+
+ /************* end optimization **************/
+
+ for (int i = 0; i < output.size(); i++)
+ println(output.get(i));
+
+ // check if any parameter has been updated
+ boolean anyParamChanged = false;
+ boolean anyParamChangedSignificantly = false;
+
+ for (int c = 1; c <= numParams; ++c) {
+ if (finalLambda[c] != lambda.get(c)) {
+ anyParamChanged = true;
+ }
+ if (Math.abs(finalLambda[c] - lambda.get(c)) > stopSigValue) {
+ anyParamChangedSignificantly = true;
+ }
+ }
+
+ // System.arraycopy(finalLambda,1,lambda,1,numParams);
+
+ println("--- PRO iteration #" + iteration + " ending @ " + (new Date()) + " ---", 1);
+ println("", 1);
+
+ if (!anyParamChanged) {
+ println("No parameter value changed in this iteration; exiting PRO.", 1);
+ println("", 1);
+ break; // exit for (iteration) loop preemptively
+ }
+
+ // was an early stopping criterion satisfied?
+ boolean critSatisfied = false;
+ if (!anyParamChangedSignificantly && stopSigValue >= 0) {
+ println("Note: No parameter value changed significantly " + "(i.e. by more than "
+ + stopSigValue + ") in this iteration.", 1);
+ critSatisfied = true;
+ }
+
+ if (critSatisfied) {
+ ++earlyStop;
+ println("", 1);
+ } else {
+ earlyStop = 0;
+ }
+
+ // if min number of iterations executed, investigate if early exit should happen
+ if (iteration >= minIts && earlyStop >= stopMinIts) {
+ println("Some early stopping criteria has been observed " + "in " + stopMinIts
+ + " consecutive iterations; exiting PRO.", 1);
+ println("", 1);
+
+ if (returnBest) {
+ // note that numParams >= bestLamba.size()-1 here!
+ for (int f = 1; f <= bestLambda.size() - 1; ++f)
+ lambda.set(f, bestLambda.get(f));
+ } else {
+ for (int f = 1; f <= numParams; ++f)
+ lambda.set(f, finalLambda[f]);
+ }
+
+ break; // exit for (iteration) loop preemptively
+ }
+
+ // if max number of iterations executed, exit
+ if (iteration >= maxIts) {
+ println("Maximum number of PRO iterations reached; exiting PRO.", 1);
+ println("", 1);
+
+ if (returnBest) {
+ // note that numParams >= bestLamba.size()-1 here!
+ for (int f = 1; f <= bestLambda.size() - 1; ++f)
+ lambda.set(f, bestLambda.get(f));
+ } else {
+ for (int f = 1; f <= numParams; ++f)
+ lambda.set(f, finalLambda[f]);
+ }
+
+ break; // exit for (iteration) loop
+ }
+
+ // use the new wt vector to decode the next iteration
+ // (interpolation with previous wt vector)
+ for (int i = 1; i <= numParams; i++)
+ lambda.set(i, interCoef * finalLambda[i] + (1 - interCoef) * lambda.get(i).doubleValue());
+
+ println("Next iteration will decode with lambda: " + lambdaToString(lambda), 1);
+ println("", 1);
+
+ // printMemoryUsage();
+ for (int i = 0; i < numSentences; ++i) {
+ suffStats_array[i].clear();
+ }
+ // cleanupMemory();
+ // println("",2);
+
+ retA[2] = 0; // i.e. this should NOT be the last iteration
+ done = true;
+
+ } // while (!done) // NOTE: this "loop" will only be carried out once
+
+ // delete .temp.stats.merged file, since it is not needed in the next
+ // iteration (it will be recreated from scratch)
+ deleteFile(tmpDirPrefix + "temp.stats.merged");
+
+ retA[0] = FINAL_score;
+ retA[1] = earlyStop;
+ return retA;
+
+ } // run_single_iteration
+
+ private String lambdaToString(ArrayList<Double> lambdaA) {
+ String retStr = "{";
+ int featToPrint = numParams > 15 ? 15 : numParams;
+ // print at most the first 15 features
+
+ retStr += "(listing the first " + featToPrint + " lambdas)";
+ for (int c = 1; c <= featToPrint - 1; ++c) {
+ retStr += "" + String.format("%.4f", lambdaA.get(c).doubleValue()) + ", ";
+ }
+ retStr += "" + String.format("%.4f", lambdaA.get(numParams).doubleValue()) + "}";
+
+ return retStr;
+ }
+
+ private String[] run_decoder(int iteration) {
+ String[] retSA = new String[2];
+
+ // retsa saves the output file name(nbest-file)
+ // and the decoder type
+
+ // [0] name of file to be processed
+ // [1] indicates how the output file was obtained:
+ // 1: external decoder
+ // 2: fake decoder
+ // 3: internal decoder
+
+ // use fake decoder
+ if (fakeFileNameTemplate != null
+ && fileExists(fakeFileNamePrefix + iteration + fakeFileNameSuffix)) {
+ String fakeFileName = fakeFileNamePrefix + iteration + fakeFileNameSuffix;
+ println("Not running decoder; using " + fakeFileName + " instead.", 1);
+ /*
+ * if (fakeFileName.endsWith(".gz")) { copyFile(fakeFileName,decoderOutFileName+".gz");
+ * gunzipFile(decoderOutFileName+".gz"); } else { copyFile(fakeFileName,decoderOutFileName); }
+ */
+ retSA[0] = fakeFileName;
+ retSA[1] = "2";
+
+ } else {
+ println("Running external decoder...", 1);
+
+ try {
+ ArrayList<String> cmd = new ArrayList<String>();
+ cmd.add(decoderCommandFileName);
+
+ if (passIterationToDecoder)
+ cmd.add(Integer.toString(iteration));
+
+ ProcessBuilder pb = new ProcessBuilder(cmd);
+ // this merges the error and output streams of the subprocess
+ pb.redirectErrorStream(true);
+ Process p = pb.start();
+
+ // capture the sub-command's output
+ new StreamGobbler(p.getInputStream(), decVerbosity).start();
+
+ int decStatus = p.waitFor();
+ if (decStatus != validDecoderExitValue) {
+ throw new RuntimeException("Call to decoder returned " + decStatus + "; was expecting "
+ + validDecoderExitValue + ".");
+ }
+ } catch (IOException | InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+
+ retSA[0] = decoderOutFileName;
+ retSA[1] = "1";
+
+ }
+
+ return retSA;
+ }
+
+ private void produceTempFiles(String nbestFileName, int iteration) {
+ try {
+ String sentsFileName = tmpDirPrefix + "temp.sents.it" + iteration;
+ String featsFileName = tmpDirPrefix + "temp.feats.it" + iteration;
+
+ FileOutputStream outStream_sents = new FileOutputStream(sentsFileName, false);
+ OutputStreamWriter outStreamWriter_sents = new OutputStreamWriter(outStream_sents, "utf8");
+ BufferedWriter outFile_sents = new BufferedWriter(outStreamWriter_sents);
+
+ PrintWriter outFile_feats = new PrintWriter(featsFileName);
+
+ InputStream inStream_nbest = null;
+ if (nbestFileName.endsWith(".gz")) {
+ inStream_nbest = new GZIPInputStream(new FileInputStream(nbestFileName));
+ } else {
+ inStream_nbest = new FileInputStream(nbestFileName);
+ }
+ BufferedReader inFile_nbest = new BufferedReader(
+ new InputStreamReader(inStream_nbest, "utf8"));
+
+ String line; // , prevLine;
+ String candidate_str = "";
+ String feats_str = "";
+
+ int i = 0;
+ int n = 0;
+ line = inFile_nbest.readLine();
+
+ while (line != null) {
+
+ /*
+ * line format:
+ *
+ * i ||| words of candidate translation . ||| feat-1_val feat-2_val ... feat-numParams_val
+ * .*
+ */
+
+ // in a well formed file, we'd find the nth candidate for the ith sentence
+
+ int read_i = Integer.parseInt((line.substring(0, line.indexOf("|||"))).trim());
+
+ if (read_i != i) {
+ writeLine("||||||", outFile_sents);
+ outFile_feats.println("||||||");
+ n = 0;
+ ++i;
+ }
+
+ line = (line.substring(line.indexOf("|||") + 3)).trim(); // get rid of initial text
+
+ candidate_str = (line.substring(0, line.indexOf("|||"))).trim();
+ feats_str = (line.substring(line.indexOf("|||") + 3)).trim();
+ // get rid of candidate string
+
+ int junk_i = feats_str.indexOf("|||");
+ if (junk_i >= 0) {
+ feats_str = (feats_str.substring(0, junk_i)).trim();
+ }
+
+ writeLine(normalize(candidate_str, textNormMethod), outFile_sents);
+ outFile_feats.println(feats_str);
+
+ ++n;
+ if (n == sizeOfNBest) {
+ writeLine("||||||", outFile_sents);
+ outFile_feats.println("||||||");
+ n = 0;
+ ++i;
+ }
+
+ line = inFile_nbest.readLine();
+ }
+
+ if (i != numSentences) { // last sentence had too few candidates
+ writeLine("||||||", outFile_sents);
+ outFile_feats.println("||||||");
+ }
+
+ inFile_nbest.close();
+ outFile_sents.close();
+ outFile_feats.close();
+
+ if (compressFiles == 1) {
+ gzipFile(sentsFileName);
+ gzipFile(featsFileName);
+ }
+
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ }
+
+ private void createConfigFile(ArrayList<Double> params, String cfgFileName,
+ String templateFileName) {
+ try {
+ // i.e. create cfgFileName, which is similar to templateFileName, but with
+ // params[] as parameter values
+
+ BufferedReader inFile = new BufferedReader(new FileReader(templateFileName));
+ PrintWriter outFile = new PrintWriter(cfgFileName);
+
+ BufferedReader inFeatDefFile = null;
+ PrintWriter outFeatDefFile = null;
+ int origFeatNum = 0; // feat num in the template file
+
+ String line = inFile.readLine();
+ while (line != null) {
+ int c_match = -1;
+ for (int c = 1; c <= numParams; ++c) {
+ if (line.startsWith(Vocabulary.word(c) + " ")) {
+ c_match = c;
+ ++origFeatNum;
+ break;
+ }
+ }
+
+ if (c_match == -1) {
+ outFile.println(line);
+ } else {
+ if (Math.abs(params.get(c_match).doubleValue()) > 1e-20)
+ outFile.println(Vocabulary.word(c_match) + " " + params.get(c_match));
+ }
+
+ line = inFile.readLine();
+ }
+
+ // now append weights of new features
+ for (int c = origFeatNum + 1; c <= numParams; ++c) {
+ if (Math.abs(params.get(c).doubleValue()) > 1e-20)
+ outFile.println(Vocabulary.word(c) + " " + params.get(c));
+ }
+
+ inFile.close();
+ outFile.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ private void processParamFile() {
+ // process parameter file
+ Scanner inFile_init = null;
+ try {
+ inFile_init = new Scanner(new FileReader(paramsFileName));
+ } catch (FileNotFoundException e) {
+ throw new RuntimeException(e);
+ }
+
+ String dummy = "";
+
+ // initialize lambda[] and other related arrays
+ for (int c = 1; c <= numParams; ++c) {
+ // skip parameter name
+ while (!dummy.equals("|||")) {
+ dummy = inFile_init.next();
+ }
+
+ // read default value
+ lambda.set(c, inFile_init.nextDouble());
+ defaultLambda[c] = lambda.get(c).doubleValue();
+
+ // read isOptimizable
+ dummy = inFile_init.next();
+ if (dummy.equals("Opt")) {
+ isOptimizable[c] = true;
+ } else if (dummy.equals("Fix")) {
+ isOptimizable[c] = false;
+ } else {
+ throw new RuntimeException("Unknown isOptimizable string " + dummy + " (must be either Opt or Fix)");
+ }
+
+ if (!isOptimizable[c]) { // skip next two values
+ dummy = inFile_init.next();
+ dummy = inFile_init.next();
+ dummy = inFile_init.next();
+ dummy = inFile_init.next();
+ } else {
+ // the next two values are not used, only to be consistent with ZMERT's params file format
+ dummy = inFile_init.next();
+ dummy = inFile_init.next();
+ // set minRandValue[c] and maxRandValue[c] (range for random values)
+ dummy = inFile_init.next();
+ if (dummy.equals("-Inf") || dummy.equals("+Inf")) {
+ throw new RuntimeException("minRandValue[" + c + "] cannot be -Inf or +Inf!");
+ } else {
+ minRandValue[c] = Double.parseDouble(dummy);
+ }
+
+ dummy = inFile_init.next();
+ if (dummy.equals("-Inf") || dummy.equals("+Inf")) {
+ throw new RuntimeException("maxRandValue[" + c + "] cannot be -Inf or +Inf!");
+ } else {
+ maxRandValue[c] = Double.parseDouble(dummy);
+ }
+
+ // check for illogical values
+ if (minRandValue[c] > maxRandValue[c]) {
+ throw new RuntimeException("minRandValue[" + c + "]=" + minRandValue[c] + " > " + maxRandValue[c]
+ + "=maxRandValue[" + c + "]!");
+ }
+
+ // check for odd values
+ if (minRandValue[c] == maxRandValue[c]) {
+ println("Warning: lambda[" + c + "] has " + "minRandValue = maxRandValue = "
+ + minRandValue[c] + ".", 1);
+ }
+ } // if (!isOptimizable[c])
+
+ /*
+ * precision[c] = inFile_init.nextDouble(); if (precision[c] < 0) { println("precision[" + c +
+ * "]=" + precision[c] + " < 0! Must be non-negative."); System.exit(21); }
+ */
+
+ }
+
+ // set normalizationOptions[]
+ String origLine = "";
+ while (origLine != null && origLine.length() == 0) {
+ origLine = inFile_init.nextLine();
+ }
+
+ // How should a lambda[] vector be normalized (before decoding)?
+ // nO[0] = 0: no normalization
+ // nO[0] = 1: scale so that parameter nO[2] has absolute value nO[1]
+ // nO[0] = 2: scale so that the maximum absolute value is nO[1]
+ // nO[0] = 3: scale so that the minimum absolute value is nO[1]
+ // nO[0] = 4: scale so that the L-nO[1] norm equals nO[2]
+
+ // normalization = none
+ // normalization = absval 1 lm
+ // normalization = maxabsval 1
+ // normalization = minabsval 1
+ // normalization = LNorm 2 1
+
+ dummy = (origLine.substring(origLine.indexOf("=") + 1)).trim();
+ String[] dummyA = dummy.split("\\s+");
+
+ if (dummyA[0].equals("none")) {
+ normalizationOptions[0] = 0;
+ } else if (dummyA[0].equals("absval")) {
+ normalizationOptions[0] = 1;
+ normalizationOptions[1] = Double.parseDouble(dummyA[1]);
+ String pName = dummyA[2];
+ for (int i = 3; i < dummyA.length; ++i) { // in case parameter name has multiple words
+ pName = pName + " " + dummyA[i];
+ }
+ normalizationOptions[2] = Vocabulary.id(pName);
+
+ if (normalizationOptions[1] <= 0) {
+ throw new RuntimeException("Value for the absval normalization method must be positive.");
+ }
+ if (normalizationOptions[2] == 0) {
+ throw new RuntimeException("Unrecognized feature name " + normalizationOptions[2]
+ + " for absval normalization method.");
+ }
+ } else if (dummyA[0].equals("maxabsval")) {
+ normalizationOptions[0] = 2;
+ normalizationOptions[1] = Double.parseDouble(dummyA[1]);
+ if (normalizationOptions[1] <= 0) {
+ throw new RuntimeException("Value for the maxabsval normalization method must be positive.");
+ }
+ } else if (dummyA[0].equals("minabsval")) {
+ normalizationOptions[0] = 3;
+ normalizationOptions[1] = Double.parseDouble(dummyA[1]);
+ if (normalizationOptions[1] <= 0) {
+ throw new RuntimeException("Value for the minabsval normalization method must be positive.");
+ }
+ } else if (dummyA[0].equals("LNorm")) {
+ normalizationOptions[0] = 4;
+ normalizationOptions[1] = Double.parseDouble(dummyA[1]);
+ normalizationOptions[2] = Double.parseDouble(dummyA[2]);
+ if (normalizationOptions[1] <= 0 || normalizationOptions[2] <= 0) {
+ throw new RuntimeException("Both values for the LNorm normalization method must be positive.");
+ }
+ } else {
+ throw new RuntimeException("Unrecognized normalization method " + dummyA[0] + "; "
+ + "must be one of none, absval, maxabsval, and LNorm.");
+ } // if (dummyA[0])
+
+ inFile_init.close();
+ } // processParamFile()
+
+ private void processDocInfo() {
+ // sets numDocuments and docOfSentence[]
+ docOfSentence = new int[numSentences];
+
+ if (docInfoFileName == null) {
+ for (int i = 0; i < numSentences; ++i)
+ docOfSentence[i] = 0;
+ numDocuments = 1;
+ } else {
+
+ try {
+
+ // 4 possible formats:
+ // 1) List of numbers, one per document, indicating # sentences in each document.
+ // 2) List of "docName size" pairs, one per document, indicating name of document and #
+ // sentences.
+ // 3) List of docName's, one per sentence, indicating which doument each sentence belongs
+ // to.
+ // 4) List of docName_number's, one per sentence, indicating which doument each sentence
+ // belongs to,
+ // and its order in that document. (can also use '-' instead of '_')
+
+ int docInfoSize = countNonEmptyLines(docInfoFileName);
+
+ if (docInfoSize < numSentences) { // format #1 or #2
+ numDocuments = docInfoSize;
+ int i = 0;
+
+ BufferedReader inFile = new BufferedReader(new FileReader(docInfoFileName));
+ String line = inFile.readLine();
+ boolean format1 = (!(line.contains(" ")));
+
+ for (int doc = 0; doc < numDocuments; ++doc) {
+
+ if (doc != 0)
+ line = inFile.readLine();
+
+ int docSize = 0;
+ if (format1) {
+ docSize = Integer.parseInt(line);
+ } else {
+ docSize = Integer.parseInt(line.split("\\s+")[1]);
+ }
+
+ for (int i2 = 1; i2 <= docSize; ++i2) {
+ docOfSentence[i] = doc;
+ ++i;
+ }
+
+ }
+
+ // now i == numSentences
+
+ inFile.close();
+
+ } else if (docInfoSize == numSentences) { // format #3 or #4
+
+ boolean format3 = false;
+
+ HashSet<String> seenStrings = new HashSet<String>();
+ BufferedReader inFile = new BufferedReader(new FileReader(docInfoFileName));
+ for (int i = 0; i < numSentences; ++i) {
+ // set format3 = true if a duplicate is found
+ String line = inFile.readLine();
+ if (seenStrings.contains(line))
+ format3 = true;
+ seenStrings.add(line);
+ }
+
+ inFile.close();
+
+ HashSet<String> seenDocNames = new HashSet<String>();
+ HashMap<String, Integer> docOrder = new HashMap<String, Integer>();
+ // maps a document name to the order (0-indexed) in which it was seen
+
+ inFile = new BufferedReader(new FileReader(docInfoFileName));
+ for (int i = 0; i < numSentences; ++i) {
+ String line = inFile.readLine();
+
+ String docName = "";
+ if (format3) {
+ docName = line;
+ } else {
+ int sep_i = Math.max(line.lastIndexOf('_'), line.lastIndexOf('-'));
+ docName = line.substring(0, sep_i);
+ }
+
+ if (!seenDocNames.contains(docName)) {
+ seenDocNames.add(docName);
+ docOrder.put(docName, seenDocNames.size() - 1);
+ }
+
+ int docOrder_i = docOrder.get(docName);
+
+ docOfSentence[i] = docOrder_i;
+
+ }
+
+ inFile.close();
+
+ numDocuments = seenDocNames.size();
+
+ } else { // badly formatted
+
+ }
+
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ }
+
+ private boolean copyFile(String origFileName, String newFileName) {
+ try {
+ File inputFile = new File(origFileName);
+ File outputFile = new File(newFileName);
+
+ InputStream in = new FileInputStream(inputFile);
+ OutputStream out = new FileOutputStream(outputFile);
+
+ byte[] buffer = new byte[1024];
+ int len;
+ while ((len = in.read(buffer)) > 0) {
+ out.write(buffer, 0, len);
+ }
+ in.close();
+ out.close();
+
+ /*
+ * InputStream inStream = new FileInputStream(new File(origFileName)); BufferedReader inFile =
+ * new BufferedReader(new InputStreamReader(inStream, "utf8"));
+ *
+ * FileOutputStream outStream = new FileOutputStream(newFileName, false); OutputStreamWriter
+ * outStreamWriter = new OutputStreamWriter(outStream, "utf8"); BufferedWriter outFile = new
+ * BufferedWriter(outStreamWriter);
+ *
+ * String line; while(inFile.ready()) { line = inFile.readLine(); writeLine(line, outFile); }
+ *
+ * inFile.close(); outFile.close();
+ */
+ return true;
+ } catch (IOException e) {
+ LOG.error(e.getMessage(), e);
+ return false;
+ }
+ }
+
+ private void renameFile(String origFileName, String newFileName) {
+ if (fileExists(origFileName)) {
+ deleteFile(newFileName);
+ File oldFile = new File(origFileName);
+ File newFile = new File(newFileName);
+ if (!oldFile.renameTo(newFile)) {
+ println("Warning: attempt to rename " + origFileName + " to " + newFileName
+ + " was unsuccessful!", 1);
+ }
+ } else {
+ println("Warning: file " + origFileName + " does not exist! (in PROCore.renameFile)", 1);
+ }
+ }
+
+ private void deleteFile(String fileName) {
+ if (fileExists(fileName)) {
+ File fd = new File(fileName);
+ if (!fd.delete()) {
+ println("Warning: attempt to delete " + fileName + " was unsuccessful!", 1);
+ }
+ }
+ }
+
+ private void writeLine(String line, BufferedWriter writer) throws IOException {
+ writer.write(line, 0, line.length());
+ writer.newLine();
+ writer.flush();
+ }
+
+ // need to re-write to handle different forms of lambda
+ public void finish() {
+ if (myDecoder != null) {
+ myDecoder.cleanUp();
+ }
+
+ // create config file with final values
+ createConfigFile(lambda, decoderConfigFileName + ".PRO.final", decoderConfigFileName
+ + ".PRO.orig");
+
+ // delete current decoder config file and decoder output
+ deleteFile(decoderConfigFileName);
+ deleteFile(decoderOutFileName);
+
+ // restore original name for config file (name was changed
+ // in initialize() so it doesn't get overwritten)
+ renameFile(decoderConfigFileName + ".PRO.orig", decoderConfigFileName);
+
+ if (finalLambdaFileName != null) {
+ try {
+ PrintWriter outFile_lambdas = new PrintWriter(finalLambdaFileName);
+ for (int c = 1; c <= numParams; ++c) {
+ outFile_lambdas.println(Vocabulary.word(c) + " ||| " + lambda.get(c).doubleValue());
+ }
+ outFile_lambdas.close();
+
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ }
+
+ private String[] cfgFileToArgsArray(String fileName) {
+ checkFile(fileName);
+
+ Vector<String> argsVector = new Vector<String>();
+
+ BufferedReader inFile = null;
+ try {
+ inFile = new BufferedReader(new FileReader(fileName));
+ String line, origLine;
+ do {
+ line = inFile.readLine();
+ origLine = line; // for error reporting purposes
+
+ if (line != null && line.length() > 0 && line.charAt(0) != '#') {
+
+ if (line.indexOf("#") != -1) { // discard comment
+ line = line.substring(0, line.indexOf("#"));
+ }
+
+ line = line.trim();
+
+ // now line should look like "-xxx XXX"
+
+ /*
+ * OBSOLETE MODIFICATION //SPECIAL HANDLING FOR PRO CLASSIFIER PARAMETERS String[] paramA
+ * = line.split("\\s+");
+ *
+ * if( paramA[0].equals("-classifierParams") ) { String classifierParam = ""; for(int p=1;
+ * p<=paramA.length-1; p++) classifierParam += paramA[p]+" ";
+ *
+ * if(paramA.length>=2) { String[] tmpParamA = new String[2]; tmpParamA[0] = paramA[0];
+ * tmpParamA[1] = classifierParam; paramA = tmpParamA; } else {
+ * println("Malformed line in config file:"); println(origLine); System.exit(70); } }//END
+ * MODIFICATION
+ */
+
+ // CMU MODIFICATION(FROM METEOR FOR ZMERT)
+ // Parse args
+ ArrayList<String> argList = new ArrayList<String>();
+ StringBuilder arg = new StringBuilder();
+ boolean quoted = false;
+ for (int i = 0; i < line.length(); i++) {
+ if (Character.isWhitespace(line.charAt(i))) {
+ if (quoted)
+ arg.append(line.charAt(i));
+ else if (arg.length() > 0) {
+ argList.add(arg.toString());
+ arg = new StringBuilder();
+ }
+ } else if (line.charAt(i) == '\'') {
+ if (quoted) {
+ argList.add(arg.toString());
+ arg = new StringBuilder();
+ }
+ quoted = !quoted;
+ } else
+ arg.append(line.charAt(i));
+ }
+ if (arg.length() > 0)
+ argList.add(arg.toString());
+ // Create paramA
+ String[] paramA = new String[argList.size()];
+ for (int i = 0; i < paramA.length; paramA[i] = argList.get(i++))
+ ;
+ // END CMU MODIFICATION
+
+ if (paramA.length == 2 && paramA[0].charAt(0) == '-') {
+ argsVector.add(paramA[0]);
+ argsVector.add(paramA[1]);
+ } else if (paramA.length > 2 && (paramA[0].equals("-m") || paramA[0].equals("-docSet"))) {
+ // -m (metricName), -docSet are allowed to have extra optinos
+ for (int opt = 0; opt < paramA.length; ++opt) {
+ argsVector.add(paramA[opt]);
+ }
+ } else {
+ throw new RuntimeException("Malformed line in config file:" + origLine);
+ }
+
+ }
+ } while (line != null);
+
+ inFile.close();
+ } catch (FileNotFoundException e) {
+ println("PRO configuration file " + fileName + " was not found!");
+ throw new RuntimeException(e);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ String[] argsArray = new String[argsVector.size()];
+
+ for (int i = 0; i < argsVector.size(); ++i) {
+ argsArray[i] = argsVector.elementAt(i);
+ }
+
+ return argsArray;
+ }
+
+ private void processArgsArray(String[] args) {
+ processArgsArray(args, true);
+ }
+
+ private void processArgsArray(String[] args, boolean firstTime) {
+ /* set default values */
+ // Relevant files
+ dirPrefix = null;
+ sourceFileName = null;
+ refFileName = "reference.txt";
+ refsPerSen = 1;
+ textNormMethod = 1;
+ paramsFileName = "params.txt";
+ docInfoFileName = null;
+ finalLambdaFileName = null;
+ // MERT specs
+ metricName = "BLEU";
+ metricName_display = metricName;
+ metricOptions = new String[2];
+ metricOptions[0] = "4";
+ metricOptions[1] = "closest";
+ docSubsetInfo = new int[7];
+ docSubsetInfo[0] = 0;
+ maxMERTIterations = 20;
+ prevMERTIterations = 20;
+ minMERTIterations = 5;
+ stopMinIts = 3;
+ stopSigValue = -1;
+ //
+ // /* possibly other early stopping criteria here */
+ //
+ numOptThreads = 1;
+ saveInterFiles = 3;
+ compressFiles = 0;
+ oneModificationPerIteration = false;
+ randInit = false;
+ seed = System.currentTimeMillis();
+ // useDisk = 2;
+ // Decoder specs
+ decoderCommandFileName = null;
+ passIterationToDecoder = false;
+ decoderOutFileName = "output.nbest";
+ validDecoderExitValue = 0;
+ decoderConfigFileName = "dec_cfg.txt";
+ sizeOfNBest = 100;
+ fakeFileNameTemplate = null;
+ fakeFileNamePrefix = null;
+ fakeFileNameSuffix = null;
+ // Output specs
+ verbosity = 1;
+ decVerbosity = 0;
+
+ int i = 0;
+
+ while (i < args.length) {
+ String option = args[i];
+ // Relevant files
+ if (option.equals("-dir")) {
+ dirPrefix = args[i + 1];
+ } else if (option.equals("-s")) {
+ sourceFileName = args[i + 1];
+ } else if (option.equals("-r")) {
+ refFileName = args[i + 1];
+ } else if (option.equals("-rps")) {
+ refsPerSen = Integer.parseInt(args[i + 1]);
+ if (refsPerSen < 1) {
+ throw new RuntimeException("refsPerSen must be positive.");
+ }
+ } else if (option.equals("-txtNrm")) {
+ textNormMethod = Integer.parseInt(args[i + 1]);
+ if (textNormMethod < 0 || textNormMethod > 4) {
+ throw new RuntimeException("textNormMethod should be between 0 and 4");
+ }
+ } else if (option.equals("-p")) {
+ paramsFileName = args[i + 1];
+ } else if (option.equals("-docInfo")) {
+ docInfoFileName = args[i + 1];
+ } else if (option.equals("-fin")) {
+ finalLambdaFileName = args[i + 1];
+ // MERT specs
+ } else if (option.equals("-m")) {
+ metricName = args[i + 1];
+ metricName_display = metricName;
+ if (EvaluationMetric.knownMetricName(metricName)) {
+ int optionCount = EvaluationMetric.metricOptionCount(metricName);
+ metricOptions = new String[optionCount];
+ for (int opt = 0; opt < optionCount; ++opt) {
+ metricOptions[opt] = args[i + opt + 2];
+ }
+ i += optionCount;
+ } else {
+ throw new RuntimeException("Unknown metric name " + metricName + ".");
+ }
+ } else if (option.equals("-docSet")) {
+ String method = args[i + 1];
+
+ if (method.equals("all")) {
+ docSubsetInfo[0] = 0;
+ i += 0;
+ } else if (method.equals("bottom")) {
+ String a = args[i + 2];
+ if (a.endsWith("d")) {
+ docSubsetInfo[0] = 1;
+ a = a.substring(0, a.indexOf("d"));
+ } else {
+ docSubsetInfo[0] = 2;
+ a = a.substring(0, a.indexOf("%"));
+ }
+ docSubsetInfo[5] = Integer.parseInt(a);
+ i += 1;
+ } else if (method.equals("top")) {
+ String a = args[i + 2];
+ if (a.endsWith("d")) {
+ docSubsetInfo[0] = 3;
+ a = a.substring(0, a.indexOf("d"));
+ } else {
+ docSubsetInfo[0] = 4;
+ a = a.substring(0, a.indexOf("%"));
+ }
+ docSubsetInfo[5] = Integer.parseInt(a);
+ i += 1;
+ } else if (method.equals("window")) {
+ String a1 = args[i + 2];
+ a1 = a1.substring(0, a1.indexOf("d")); // size of window
+ String a2 = args[i + 4];
+ if (a2.indexOf("p") > 0) {
+ docSubsetInfo[0] = 5;
+ a2 = a2.substring(0, a2.indexOf("p"));
+ } else {
+ docSubsetInfo[0] = 6;
+ a2 = a2.substring(0, a2.indexOf("r"));
+ }
+ docSubsetInfo[5] = Integer.parseInt(a1);
+ docSubsetInfo[6] = Integer.parseInt(a2);
+ i += 3;
+ } else {
+ throw new RuntimeException("Unknown docSet method " + method + ".");
+ }
+ } else if (option.equals("-maxIt")) {
+ maxMERTIterations = Integer.parseInt(args[i + 1]);
+ if (maxMERTIterations < 1) {
+ throw new RuntimeException("maxMERTIts must be positive.");
+ }
+ } else if (option.equals("-minIt")) {
+ minMERTIterations = Integer.parseInt(args[i + 1]);
+ if (minMERTIterations < 1) {
+ throw new RuntimeException("minMERTIts must be positive.");
+ }
+ } else if (option.equals("-prevIt")) {
+ prevMERTIterations = Integer.parseInt(args[i + 1]);
+ if (prevMERTIterations < 0) {
+ throw new RuntimeException("prevMERTIts must be non-negative.");
+ }
+ } else if (option.equals("-stopIt")) {
+ stopMinIts = Integer.parseInt(args[i + 1]);
+ if (stopMinIts < 1) {
+ throw new RuntimeException("stopMinIts must be positive.");
+ }
+ } else if (option.equals("-stopSig")) {
+ stopSigValue = Double.parseDouble(args[i + 1]);
+ }
+ //
+ // /* possibly other early stopping criteria here */
+ //
+ else if (option.equals("-thrCnt")) {
+ numOptThreads = Integer.parseInt(args[i + 1]);
+ if (numOptThreads < 1) {
+ throw new RuntimeException("threadCount must be positive.");
+ }
+ } else if (option.equals("-save")) {
+ saveInterFiles = Integer.parseInt(args[i + 1]);
+ if (saveInterFiles < 0 || saveInterFiles > 3) {
+ throw new RuntimeException("save should be between 0 and 3");
+ }
+ } else if (option.equals("-compress")) {
+ compressFiles = Integer.parseInt(args[i + 1]);
+ if (compressFiles < 0 || compressFiles > 1) {
+ throw new RuntimeException("compressFiles should be either 0 or 1");
+ }
+ } else if (option.equals("-opi")) {
+ int opi = Integer.parseInt(args[i + 1]);
+ if (opi == 1) {
+ oneModificationPerIteration = true;
+ } else if (opi == 0) {
+ oneModificationPerIteration = false;
+ } else {
+ throw new RuntimeException("oncePerIt must be either 0 or 1.");
+ }
+ } else if (option.equals("-rand")) {
+ int rand = Integer.parseInt(args[i + 1]);
+ if (rand == 1) {
+ randInit = true;
+ } else if (rand == 0) {
+ randInit = false;
+ } else {
+ throw new RuntimeException("randInit must be either 0 or 1.");
+ }
+ } else if (option.equals("-seed")) {
+ if (args[i + 1].equals("time")) {
+ seed = System.currentTimeMillis();
+ } else {
+ seed = Long.parseLong(args[i + 1]);
+ }
+ }
+ /*
+ * else if (option.equals("-ud")) { useDisk = Integer.parseInt(args[i+1]); if (useDisk < 0 ||
+ * useDisk > 2) { println("useDisk should be between 0 and 2"); System.exit(10); } }
+ */
+
+ // for pro:
+ // classification algorithm class path
+ else if (option.equals("-classifierClass")) {
+ classifierAlg = args[i + 1];
+ }
+ // params for the specified classifier
+ else if (option.equals("-classifierParams")) {
+ classifierParams = args[i + 1].split("\\s+");
+ }
+ // tau: num of randomly generated candidates
+ else if (option.equals("-Tau")) {
+ Tau = Integer.parseInt(args[i + 1]);
+ }
+ // xi: top-xi candidates to be accepted
+ else if (option.equals("-Xi")) {
+ Xi = Integer.parseInt(args[i + 1]);
+ }
+ // return the best weight during tuning or not
+ else if (option.equals("-returnBest")) {
+ int retBest = Integer.parseInt(args[i + 1]);
+ if (retBest == 1)
+ returnBest = true;
+ else if (retBest == 0)
+ returnBest = false;
+ else {
+ throw new RuntimeException("-returnBest must be either 0 or 1.");
+ }
+ }
+ // interpolation coefficient between current & previous weights
+ else if (option.equals("-interCoef")) {
+ interCoef = Double.parseDouble(args[i + 1]);
+ }
+ // metric(eg. bleu) diff threshold(to select sampled candidates)
+ else if (option.equals("-metricDiff")) {
+ metricDiff = Double.parseDouble(
<TRUNCATED>
[31/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/metrics/PrecisMinusSourceBLEU.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/metrics/PrecisMinusSourceBLEU.java b/joshua-core/src/main/java/org/apache/joshua/metrics/PrecisMinusSourceBLEU.java
new file mode 100644
index 0000000..bfe15d0
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/metrics/PrecisMinusSourceBLEU.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.metrics;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
+
+public class PrecisMinusSourceBLEU extends EvaluationMetric {
+
+ private Precis myPrecis;
+ private SourceBLEU mySourceBLEU;
+
+ private double bleuWeight;
+
+ private int precisCount;
+ private int sourceBleuCount;
+
+ public PrecisMinusSourceBLEU(String[] options) {
+ // Automatically deactivate Levenshtein penalty for Precis.
+ bleuWeight = Double.parseDouble(options[5]);
+ options[5] = "0";
+
+ myPrecis = new Precis(options);
+ mySourceBLEU =
+ new SourceBLEU(Integer.parseInt(options[0]), options[1], Integer.parseInt(options[2]),
+ false);
+
+ initialize();
+ }
+
+ protected void initialize() {
+ metricName = "PRECIS-SRC_BLEU";
+ toBeMinimized = false;
+ precisCount = myPrecis.suffStatsCount;
+ sourceBleuCount = mySourceBLEU.suffStatsCount;
+ suffStatsCount = precisCount + sourceBleuCount;
+ }
+
+ public double bestPossibleScore() {
+ return 1.0;
+ }
+
+ public double worstPossibleScore() {
+ return -1.0;
+ }
+
+ public int[] suffStats(String cand_str, int i) {
+ return null;
+ }
+
+ public int[][] suffStats(String[] cand_strings, int[] cand_indices) {
+ int candCount = cand_strings.length;
+ if (cand_indices.length != candCount) {
+ System.out.println("Array lengths mismatch in suffStats(String[],int[]); returning null.");
+ return null;
+ }
+
+ int[][] stats = new int[candCount][suffStatsCount];
+
+ int[][] precis_stats = myPrecis.suffStats(cand_strings, cand_indices);
+ int[][] source_bleu_stats = mySourceBLEU.suffStats(cand_strings, cand_indices);
+
+ for (int d = 0; d < candCount; ++d) {
+ int s = 0;
+ for (int s_T = 0; s_T < precisCount; s_T++) {
+ stats[d][s] = precis_stats[d][s_T];
+ ++s;
+ }
+ for (int s_B = 0; s_B < sourceBleuCount; s_B++) {
+ stats[d][s] = source_bleu_stats[d][s_B];
+ ++s;
+ }
+ }
+ return stats;
+ }
+
+ public void createSuffStatsFile(String cand_strings_fileName, String cand_indices_fileName,
+ String outputFileName, int maxBatchSize) {
+ try {
+ myPrecis.createSuffStatsFile(cand_strings_fileName, cand_indices_fileName, outputFileName
+ + ".PRECIS", maxBatchSize);
+ mySourceBLEU.createSuffStatsFile(cand_strings_fileName, cand_indices_fileName, outputFileName
+ + ".SRC_BLEU", maxBatchSize);
+
+ PrintWriter outFile = new PrintWriter(outputFileName);
+
+ FileInputStream inStream_Precis = new FileInputStream(outputFileName + ".PRECIS");
+ BufferedReader inFile_Precis =
+ new BufferedReader(new InputStreamReader(inStream_Precis, "utf8"));
+
+ FileInputStream inStream_SourceBLEU = new FileInputStream(outputFileName + ".SRC_BLEU");
+ BufferedReader inFile_SourceBLEU =
+ new BufferedReader(new InputStreamReader(inStream_SourceBLEU, "utf8"));
+
+ String line_Precis = inFile_Precis.readLine();
+ String line_SourceBLEU = inFile_SourceBLEU.readLine();
+
+ // combine the two files into one
+ while (line_Precis != null) {
+ outFile.println(line_Precis + " " + line_SourceBLEU);
+ line_Precis = inFile_Precis.readLine();
+ line_SourceBLEU = inFile_SourceBLEU.readLine();
+ }
+
+ inFile_Precis.close();
+ inFile_SourceBLEU.close();
+ outFile.close();
+
+ File fd;
+ fd = new File(outputFileName + ".PRECIS");
+ if (fd.exists()) fd.delete();
+ fd = new File(outputFileName + ".SRC_BLEU");
+ if (fd.exists()) fd.delete();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public double score(int[] stats) {
+ if (stats.length != suffStatsCount) {
+ throw new RuntimeException("Mismatch between stats.length and suffStatsCount (" + stats.length
+ + " vs. " + suffStatsCount + ") in PrecisMinusSourceBLEU.score(int[])");
+ }
+
+ double sc = 0.0;
+
+ int[] stats_Precis = new int[precisCount];
+ int[] stats_SourceBLEU = new int[sourceBleuCount];
+ for (int s = 0; s < precisCount; ++s) {
+ stats_Precis[s] = stats[s];
+ }
+ for (int s = 0; s < sourceBleuCount; ++s) {
+ stats_SourceBLEU[s] = stats[s + precisCount];
+ }
+
+ double sc_T = myPrecis.score(stats_Precis);
+ double sc_B = mySourceBLEU.score(stats_SourceBLEU);
+
+ sc = sc_T - (bleuWeight * sc_B);
+
+ return sc;
+ }
+
+ public void printDetailedScore_fromStats(int[] stats, boolean oneLiner) {
+ int[] stats_Precis = new int[precisCount];
+ int[] stats_SourceBLEU = new int[sourceBleuCount];
+ for (int s = 0; s < precisCount; ++s) {
+ stats_Precis[s] = stats[s];
+ }
+ for (int s = 0; s < sourceBleuCount; ++s) {
+ stats_SourceBLEU[s] = stats[s + precisCount];
+ }
+
+ System.out.println("---PRECIS---");
+ myPrecis.printDetailedScore_fromStats(stats_Precis, oneLiner);
+ System.out.println("---SRC_BLEU---");
+ mySourceBLEU.printDetailedScore_fromStats(stats_SourceBLEU, oneLiner);
+ System.out.println("---------");
+ System.out.println(" => " + metricName + " = " + f4.format(score(stats)));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/metrics/SARI.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/metrics/SARI.java b/joshua-core/src/main/java/org/apache/joshua/metrics/SARI.java
new file mode 100644
index 0000000..129e4af
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/metrics/SARI.java
@@ -0,0 +1,681 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License
+ * is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
+ * or implied. See the License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.joshua.metrics;
+
+// Changed PROCore.java (text normalization function) and EvaluationMetric too
+
+import java.util.Map;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.logging.Logger;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+
+/***
+ * Implementation of the SARI metric for text-to-text correction.
+ *
+ * \@article{xu2016optimizing,
+ * title={Optimizing statistical machine translation for text simplification},
+ * author={Xu, Wei and Napoles, Courtney and Pavlick, Ellie and Chen, Quanze and Callison-Burch, Chris},
+ * journal={Transactions of the Association for Computational Linguistics},
+ * volume={4},
+ * year={2016}}
+ *
+ * @author Wei Xu
+ */
+public class SARI extends EvaluationMetric {
+ private static final Logger logger = Logger.getLogger(SARI.class.getName());
+
+ // The maximum n-gram we care about
+ protected int maxGramLength;
+ protected String[] srcSentences;
+ protected double[] weights;
+ protected HashMap<String, Integer>[][] refNgramCounts;
+ protected HashMap<String, Integer>[][] srcNgramCounts;
+
+ /*
+ * You already have access to these data members of the parent class (EvaluationMetric): int
+ * numSentences; number of sentences in the MERT set int refsPerSen; number of references per
+ * sentence String[][] refSentences; refSentences[i][r] stores the r'th reference of the i'th
+ * source sentence (both indices are 0-based)
+ */
+
+ public SARI(String[] Metric_options) {
+ int mxGrmLn = Integer.parseInt(Metric_options[0]);
+ if (mxGrmLn >= 1) {
+ maxGramLength = mxGrmLn;
+ } else {
+ logger.severe("Maximum gram length must be positive");
+ System.exit(1);
+ }
+
+ try {
+ loadSources(Metric_options[1]);
+ } catch (IOException e) {
+ logger.severe("Error loading the source sentences from " + Metric_options[1]);
+ System.exit(1);
+ }
+
+ initialize(); // set the data members of the metric
+
+ }
+
+ protected void initialize() {
+ metricName = "SARI";
+ toBeMinimized = false;
+ suffStatsCount = StatIndex.values().length * maxGramLength + 1;
+
+ set_weightsArray();
+ set_refNgramCounts();
+ set_srcNgramCounts();
+
+ }
+
+ public double bestPossibleScore() {
+ return 1.0;
+ }
+
+ public double worstPossibleScore() {
+ return 0.0;
+ }
+
+ /**
+ * Sets the BLEU weights for each n-gram level to uniform.
+ */
+ protected void set_weightsArray() {
+ weights = new double[1 + maxGramLength];
+ for (int n = 1; n <= maxGramLength; ++n) {
+ weights[n] = 1.0 / maxGramLength;
+ }
+ }
+
+ /**
+ * Computes the sum of ngram counts in references for each sentence (storing them in
+ * <code>refNgramCounts</code>), which are used for clipping n-gram counts.
+ */
+ protected void set_refNgramCounts() {
+ @SuppressWarnings("unchecked")
+
+ HashMap<String, Integer>[][] temp_HMA = new HashMap[numSentences][maxGramLength];
+ refNgramCounts = temp_HMA;
+
+ String gram = "";
+ int oldCount = 0, nextCount = 0;
+
+ for (int i = 0; i < numSentences; ++i) {
+ refNgramCounts[i] = getNgramCountsArray(refSentences[i][0]);
+ // initialize to ngramCounts[n] of the first reference translation...
+
+ // ...and update as necessary from the other reference translations
+ for (int r = 1; r < refsPerSen; ++r) {
+
+ HashMap<String, Integer>[] nextNgramCounts = getNgramCountsArray(refSentences[i][r]);
+
+ for (int n = 1; n <= maxGramLength; ++n) {
+
+ Iterator<String> it = (nextNgramCounts[n].keySet()).iterator();
+
+ while (it.hasNext()) {
+ gram = it.next();
+ nextCount = nextNgramCounts[n].get(gram);
+
+ if (refNgramCounts[i][n].containsKey(gram)) { // update if necessary
+ oldCount = refNgramCounts[i][n].get(gram);
+ refNgramCounts[i][n].put(gram, oldCount + nextCount);
+ } else { // add it
+ refNgramCounts[i][n].put(gram, nextCount);
+ }
+
+ }
+
+ } // for (n)
+
+ } // for (r)
+
+ } // for (i)
+
+ }
+
+ protected void set_srcNgramCounts() {
+ @SuppressWarnings("unchecked")
+
+ HashMap<String, Integer>[][] temp_HMA = new HashMap[numSentences][maxGramLength];
+ srcNgramCounts = temp_HMA;
+
+ for (int i = 0; i < numSentences; ++i) {
+ srcNgramCounts[i] = getNgramCountsArray(srcSentences[i]);
+ } // for (i)
+ }
+
+ // set contents of stats[] here!
+ public int[] suffStats(String cand_str, int i) {
+ int[] stats = new int[suffStatsCount];
+
+ HashMap<String, Integer>[] candNgramCounts = getNgramCountsArray(cand_str);
+
+ for (int n = 1; n <= maxGramLength; ++n) {
+
+ // ADD OPERATIONS
+ HashMap cand_sub_src = substractHashMap(candNgramCounts[n], srcNgramCounts[i][n]);
+ HashMap cand_and_ref_sub_src = intersectHashMap(cand_sub_src, refNgramCounts[i][n]);
+ HashMap ref_sub_src = substractHashMap(refNgramCounts[i][n], srcNgramCounts[i][n]);
+
+ stats[StatIndex.values().length * (n - 1)
+ + StatIndex.ADDBOTH.ordinal()] = cand_and_ref_sub_src.keySet().size();
+ stats[StatIndex.values().length * (n - 1) + StatIndex.ADDCAND.ordinal()] = cand_sub_src
+ .keySet().size();
+ stats[StatIndex.values().length * (n - 1) + StatIndex.ADDREF.ordinal()] = ref_sub_src.keySet()
+ .size();
+
+ // System.out.println("src_and_cand_sub_ref" + cand_and_ref_sub_src +
+ // cand_and_ref_sub_src.keySet().size());
+ // System.out.println("cand_sub_src" + cand_sub_src + cand_sub_src.keySet().size());
+ // System.out.println("ref_sub_src" + ref_sub_src + ref_sub_src.keySet().size());
+
+ // DELETION OPERATIONS
+ HashMap src_sub_cand = substractHashMap(srcNgramCounts[i][n], candNgramCounts[n],
+ this.refsPerSen, this.refsPerSen);
+ HashMap src_sub_ref = substractHashMap(srcNgramCounts[i][n], refNgramCounts[i][n],
+ this.refsPerSen, 1);
+ HashMap src_sub_cand_sub_ref = intersectHashMap(src_sub_cand, src_sub_ref, 1, 1);
+
+ stats[StatIndex.values().length * (n - 1) + StatIndex.DELBOTH.ordinal()] = sumHashMapByValues(
+ src_sub_cand_sub_ref);
+ stats[StatIndex.values().length * (n - 1) + StatIndex.DELCAND.ordinal()] = sumHashMapByValues(
+ src_sub_cand);
+ stats[StatIndex.values().length * (n - 1) + StatIndex.DELREF.ordinal()] = sumHashMapByValues(
+ src_sub_ref);
+
+ // System.out.println("src_sub_cand_sub_ref" + src_sub_cand_sub_ref +
+ // sumHashMapByValues(src_sub_cand_sub_ref));
+ // System.out.println("src_sub_cand" + src_sub_cand + sumHashMapByValues(src_sub_cand));
+ // System.out.println("src_sub_ref" + src_sub_ref + sumHashMapByValues(src_sub_ref));
+
+ stats[StatIndex.values().length * (n - 1) + StatIndex.DELREF.ordinal()] = src_sub_ref.keySet()
+ .size() * this.refsPerSen;
+
+ // KEEP OPERATIONS
+ HashMap src_and_cand = intersectHashMap(srcNgramCounts[i][n], candNgramCounts[n],
+ this.refsPerSen, this.refsPerSen);
+ HashMap src_and_ref = intersectHashMap(srcNgramCounts[i][n], refNgramCounts[i][n],
+ this.refsPerSen, 1);
+ HashMap src_and_cand_and_ref = intersectHashMap(src_and_cand, src_and_ref, 1, 1);
+
+ stats[StatIndex.values().length * (n - 1)
+ + StatIndex.KEEPBOTH.ordinal()] = sumHashMapByValues(src_and_cand_and_ref);
+ stats[StatIndex.values().length * (n - 1)
+ + StatIndex.KEEPCAND.ordinal()] = sumHashMapByValues(src_and_cand);
+ stats[StatIndex.values().length * (n - 1) + StatIndex.KEEPREF.ordinal()] = sumHashMapByValues(
+ src_and_ref);
+
+ stats[StatIndex.values().length * (n - 1) + StatIndex.KEEPBOTH.ordinal()] = (int) (1000000
+ * sumHashMapByDoubleValues(divideHashMap(src_and_cand_and_ref, src_and_cand)));
+ stats[StatIndex.values().length * (n - 1)
+ + StatIndex.KEEPCAND.ordinal()] = (int) sumHashMapByDoubleValues(
+ divideHashMap(src_and_cand_and_ref, src_and_ref));
+ stats[StatIndex.values().length * (n - 1) + StatIndex.KEEPREF.ordinal()] = src_and_ref
+ .keySet().size();
+
+ // System.out.println("src_and_cand_and_ref" + src_and_cand_and_ref);
+ // System.out.println("src_and_cand" + src_and_cand);
+ // System.out.println("src_and_ref" + src_and_ref);
+
+ // stats[StatIndex.values().length * (n - 1) + StatIndex.KEEPBOTH2.ordinal()] = (int)
+ // sumHashMapByDoubleValues(divideHashMap(src_and_cand_and_ref,src_and_ref)) * 100000000 /
+ // src_and_ref.keySet().size() ;
+ // stats[StatIndex.values().length * (n - 1) + StatIndex.KEEPREF.ordinal()] =
+ // src_and_ref.keySet().size() * 8;
+
+ // System.out.println("src_and_cand_and_ref" + src_and_cand_and_ref);
+ // System.out.println("src_and_cand" + src_and_cand);
+ // System.out.println("divide" + divideHashMap(src_and_cand_and_ref,src_and_cand));
+ // System.out.println(sumHashMapByDoubleValues(divideHashMap(src_and_cand_and_ref,src_and_cand)));
+
+ }
+
+ int n = 1;
+
+ // System.out.println("CAND: " + candNgramCounts[n]);
+ // System.out.println("SRC: " + srcNgramCounts[i][n]);
+ // System.out.println("REF: " + refNgramCounts[i][n]);
+
+ HashMap src_and_cand = intersectHashMap(srcNgramCounts[i][n], candNgramCounts[n],
+ this.refsPerSen, this.refsPerSen);
+ HashMap src_and_ref = intersectHashMap(srcNgramCounts[i][n], refNgramCounts[i][n],
+ this.refsPerSen, 1);
+ HashMap src_and_cand_and_ref = intersectHashMap(src_and_cand, src_and_ref, 1, 1);
+ // System.out.println("SRC&CAND&REF : " + src_and_cand_and_ref);
+
+ HashMap cand_sub_src = substractHashMap(candNgramCounts[n], srcNgramCounts[i][n]);
+ HashMap cand_and_ref_sub_src = intersectHashMap(cand_sub_src, refNgramCounts[i][n]);
+ // System.out.println("CAND&REF-SRC : " + cand_and_ref_sub_src);
+
+ HashMap src_sub_cand = substractHashMap(srcNgramCounts[i][n], candNgramCounts[n],
+ this.refsPerSen, this.refsPerSen);
+ HashMap src_sub_ref = substractHashMap(srcNgramCounts[i][n], refNgramCounts[i][n],
+ this.refsPerSen, 1);
+ HashMap src_sub_cand_sub_ref = intersectHashMap(src_sub_cand, src_sub_ref, 1, 1);
+ // System.out.println("SRC-REF-CAND : " + src_sub_cand_sub_ref);
+
+ // System.out.println("DEBUG:" + Arrays.toString(stats));
+ // System.out.println("REF-SRC: " + substractHashMap(refNgramCounts[i], srcNgramCounts[i][0],
+ // (double)refsPerSen));
+
+ return stats;
+ }
+
+ public double score(int[] stats) {
+ if (stats.length != suffStatsCount) {
+ System.out.println("Mismatch between stats.length and suffStatsCount (" + stats.length
+ + " vs. " + suffStatsCount + ") in NewMetric.score(int[])");
+ System.exit(1);
+ }
+
+ double sc = 0.0;
+
+ for (int n = 1; n <= maxGramLength; ++n) {
+
+ int addCandCorrectNgram = stats[StatIndex.values().length * (n - 1)
+ + StatIndex.ADDBOTH.ordinal()];
+ int addCandTotalNgram = stats[StatIndex.values().length * (n - 1)
+ + StatIndex.ADDCAND.ordinal()];
+ int addRefTotalNgram = stats[StatIndex.values().length * (n - 1)
+ + StatIndex.ADDREF.ordinal()];
+
+ double prec_add_n = 0.0;
+ if (addCandTotalNgram > 0) {
+ prec_add_n = addCandCorrectNgram / (double) addCandTotalNgram;
+ }
+
+ double recall_add_n = 0.0;
+ if (addRefTotalNgram > 0) {
+ recall_add_n = addCandCorrectNgram / (double) addRefTotalNgram;
+ }
+
+ // System.out.println("\nDEBUG-SARI:" + addCandCorrectNgram + " " + addCandTotalNgram + " " +
+ // addRefTotalNgram);
+
+ double f1_add_n = meanHarmonic(prec_add_n, recall_add_n);
+
+ sc += weights[n] * f1_add_n;
+
+ int delCandCorrectNgram = stats[StatIndex.values().length * (n - 1)
+ + StatIndex.DELBOTH.ordinal()];
+ int delCandTotalNgram = stats[StatIndex.values().length * (n - 1)
+ + StatIndex.DELCAND.ordinal()];
+ int delRefTotalNgram = stats[StatIndex.values().length * (n - 1)
+ + StatIndex.DELREF.ordinal()];
+
+ double prec_del_n = 0.0;
+ if (delCandTotalNgram > 0) {
+ prec_del_n = delCandCorrectNgram / (double) delCandTotalNgram;
+ }
+
+ double recall_del_n = 0.0;
+ if (delRefTotalNgram > 0) {
+ recall_del_n = delCandCorrectNgram / (double) delRefTotalNgram;
+ }
+
+ // System.out.println("\nDEBUG-SARI:" + delCandCorrectNgram + " " + delRefTotalNgram);
+
+ double f1_del_n = meanHarmonic(prec_del_n, recall_del_n);
+
+ // sc += weights[n] * f1_del_n;
+ sc += weights[n] * prec_del_n;
+
+ int keepCandCorrectNgram = stats[StatIndex.values().length * (n - 1)
+ + StatIndex.KEEPBOTH.ordinal()];
+ // int keepCandCorrectNgram2 = stats[StatIndex.values().length * (n - 1) +
+ // StatIndex.KEEPBOTH2.ordinal()];
+ int keepCandTotalNgram = stats[StatIndex.values().length * (n - 1)
+ + StatIndex.KEEPCAND.ordinal()];
+ int keepRefTotalNgram = stats[StatIndex.values().length * (n - 1)
+ + StatIndex.KEEPREF.ordinal()];
+
+ double prec_keep_n = 0.0;
+ if (keepCandTotalNgram > 0) {
+ prec_keep_n = keepCandCorrectNgram / (double) (1000000 * keepCandTotalNgram);
+ }
+
+ double recall_keep_n = 0.0;
+ if (keepRefTotalNgram > 0) {
+ recall_keep_n = keepCandTotalNgram / (double) keepRefTotalNgram;
+ }
+
+ // System.out.println("\nDEBUG-SARI-KEEP: " + n + " " + keepCandCorrectNgram + " " +
+ // keepCandTotalNgram + " " + keepRefTotalNgram);
+
+ double f1_keep_n = meanHarmonic(prec_keep_n, recall_keep_n);
+
+ sc += weights[n] * f1_keep_n;
+
+ // System.out.println("\nDEBUG-SARI: " + n + " " + prec_add_n + " " + recall_add_n + " " +
+ // prec_del_n + " " + recall_del_n + " " + prec_keep_n + " " + recall_keep_n);
+
+ // System.out.println("\nDEBUG-SARI-KEEP: " + n + " " + keepCandCorrectNgram + " " +
+ // keepCandTotalNgram + " " + keepRefTotalNgram);
+ }
+
+ sc = sc / 3.0;
+ //
+ //
+ // set sc here!
+ //
+ //
+
+ return sc;
+ }
+
+ public double meanHarmonic(double precision, double recall) {
+
+ if (precision > 0 && recall > 0) {
+ return (2.0 * precision * recall) / (precision + recall);
+ }
+ return 0.0;
+ }
+
+ public void loadSources(String filepath) throws IOException {
+ srcSentences = new String[numSentences];
+ // BufferedReader br = new BufferedReader(new FileReader(filepath));
+ InputStream inStream = new FileInputStream(new File(filepath));
+ BufferedReader br = new BufferedReader(new InputStreamReader(inStream, "utf8"));
+
+ String line;
+ int i = 0;
+ while (i < numSentences && (line = br.readLine()) != null) {
+ srcSentences[i] = line.trim();
+ i++;
+ }
+ br.close();
+ }
+
+ public double sumHashMapByDoubleValues(HashMap<String, Double> counter) {
+ double sumcounts = 0;
+
+ for (Map.Entry<String, Double> e : counter.entrySet()) {
+ sumcounts += (double) e.getValue();
+ }
+
+ return sumcounts;
+ }
+
+ public int sumHashMapByValues(HashMap<String, Integer> counter) {
+ int sumcounts = 0;
+
+ for (Map.Entry<String, Integer> e : counter.entrySet()) {
+ sumcounts += (int) e.getValue();
+ }
+
+ return sumcounts;
+ }
+
+ public HashMap<String, Integer> substractHashMap(HashMap<String, Integer> counter1,
+ HashMap<String, Integer> counter2) {
+ HashMap<String, Integer> newcounter = new HashMap<String, Integer>();
+
+ for (Map.Entry<String, Integer> e : counter1.entrySet()) {
+ String ngram = e.getKey();
+ int count1 = e.getValue();
+ int count2 = counter2.containsKey(ngram) ? counter2.get(ngram) : 0;
+ if (count2 == 0) {
+ newcounter.put(ngram, 1);
+ }
+ }
+
+ return newcounter;
+ }
+
+ // HashMap result = counter1*ratio1 - counter2*ratio2
+ public HashMap<String, Integer> substractHashMap(HashMap<String, Integer> counter1,
+ HashMap<String, Integer> counter2, int ratio1, int ratio2) {
+ HashMap<String, Integer> newcounter = new HashMap<String, Integer>();
+
+ for (Map.Entry<String, Integer> e : counter1.entrySet()) {
+ String ngram = e.getKey();
+ int count1 = e.getValue();
+ int count2 = counter2.containsKey(ngram) ? counter2.get(ngram) : 0;
+ int newcount = count1 * ratio1 - count2 * ratio2;
+ if (newcount > 0) {
+ newcounter.put(ngram, newcount);
+ }
+ }
+
+ return newcounter;
+ }
+
+ public HashMap<String, Double> divideHashMap(HashMap<String, Integer> counter1,
+ HashMap<String, Integer> counter2) {
+ HashMap<String, Double> newcounter = new HashMap<String, Double>();
+
+ for (Map.Entry<String, Integer> e : counter1.entrySet()) {
+ String ngram = e.getKey();
+ int count1 = e.getValue();
+ int count2 = counter2.containsKey(ngram) ? counter2.get(ngram) : 0;
+ if (count2 != 0) {
+ newcounter.put(ngram, (double) count1 / (double) count2);
+ }
+ }
+
+ return newcounter;
+ }
+
+ public HashMap<String, Integer> intersectHashMap(HashMap<String, Integer> counter1,
+ HashMap<String, Integer> counter2) {
+ HashMap<String, Integer> newcounter = new HashMap<String, Integer>();
+
+ for (Map.Entry<String, Integer> e : counter1.entrySet()) {
+ String ngram = e.getKey();
+ int count1 = e.getValue();
+ int count2 = counter2.containsKey(ngram) ? counter2.get(ngram) : 0;
+ if (count2 > 0) {
+ newcounter.put(ngram, 1);
+ }
+ }
+
+ return newcounter;
+ }
+
+ // HashMap result = (counter1*ratio1) & (counter2*ratio2)
+ public HashMap<String, Integer> intersectHashMap(HashMap<String, Integer> counter1,
+ HashMap<String, Integer> counter2, int ratio1, int ratio2) {
+ HashMap<String, Integer> newcounter = new HashMap<String, Integer>();
+
+ for (Map.Entry<String, Integer> e : counter1.entrySet()) {
+ String ngram = e.getKey();
+ int count1 = e.getValue();
+ int count2 = counter2.containsKey(ngram) ? counter2.get(ngram) : 0;
+ int newcount = Math.min(count1 * ratio1, count2 * ratio2);
+ if (newcount > 0) {
+ newcounter.put(ngram, newcount);
+ }
+ }
+
+ return newcounter;
+ }
+
+ protected int wordCount(String cand_str) {
+ if (!cand_str.equals("")) {
+ return cand_str.split("\\s+").length;
+ } else {
+ return 0;
+ }
+ }
+
+ public HashMap<String, Integer>[] getNgramCountsArray(String cand_str) {
+ if (!cand_str.equals("")) {
+ return getNgramCountsArray(cand_str.split("\\s+"));
+ } else {
+ return getNgramCountsArray(new String[0]);
+ }
+ }
+
+ public HashMap<String, Integer>[] getNgramCountsArray(String[] words) {
+ @SuppressWarnings("unchecked")
+ HashMap<String, Integer>[] ngramCountsArray = new HashMap[1 + maxGramLength];
+ ngramCountsArray[0] = null;
+ for (int n = 1; n <= maxGramLength; ++n) {
+ ngramCountsArray[n] = new HashMap<String, Integer>();
+ }
+
+ int len = words.length;
+ String gram;
+ int st = 0;
+
+ for (; st <= len - maxGramLength; ++st) {
+
+ gram = words[st];
+ if (ngramCountsArray[1].containsKey(gram)) {
+ int oldCount = ngramCountsArray[1].get(gram);
+ ngramCountsArray[1].put(gram, oldCount + 1);
+ } else {
+ ngramCountsArray[1].put(gram, 1);
+ }
+
+ for (int n = 2; n <= maxGramLength; ++n) {
+ gram = gram + " " + words[st + n - 1];
+ if (ngramCountsArray[n].containsKey(gram)) {
+ int oldCount = ngramCountsArray[n].get(gram);
+ ngramCountsArray[n].put(gram, oldCount + 1);
+ } else {
+ ngramCountsArray[n].put(gram, 1);
+ }
+ } // for (n)
+
+ } // for (st)
+
+ // now st is either len-maxGramLength+1 or zero (if above loop never entered, which
+ // happens with sentences that have fewer than maxGramLength words)
+
+ for (; st < len; ++st) {
+
+ gram = words[st];
+ if (ngramCountsArray[1].containsKey(gram)) {
+ int oldCount = ngramCountsArray[1].get(gram);
+ ngramCountsArray[1].put(gram, oldCount + 1);
+ } else {
+ ngramCountsArray[1].put(gram, 1);
+ }
+
+ int n = 2;
+ for (int fin = st + 1; fin < len; ++fin) {
+ gram = gram + " " + words[st + n - 1];
+
+ if (ngramCountsArray[n].containsKey(gram)) {
+ int oldCount = ngramCountsArray[n].get(gram);
+ ngramCountsArray[n].put(gram, oldCount + 1);
+ } else {
+ ngramCountsArray[n].put(gram, 1);
+ }
+ ++n;
+ } // for (fin)
+
+ } // for (st)
+
+ return ngramCountsArray;
+
+ }
+
+ public HashMap<String, Integer> getNgramCountsAll(String cand_str) {
+ if (!cand_str.equals("")) {
+ return getNgramCountsAll(cand_str.split("\\s+"));
+ } else {
+ return getNgramCountsAll(new String[0]);
+ }
+ }
+
+ public HashMap<String, Integer> getNgramCountsAll(String[] words) {
+ HashMap<String, Integer> ngramCountsAll = new HashMap<String, Integer>();
+
+ int len = words.length;
+ String gram;
+ int st = 0;
+
+ for (; st <= len - maxGramLength; ++st) {
+
+ gram = words[st];
+ if (ngramCountsAll.containsKey(gram)) {
+ int oldCount = ngramCountsAll.get(gram);
+ ngramCountsAll.put(gram, oldCount + 1);
+ } else {
+ ngramCountsAll.put(gram, 1);
+ }
+
+ for (int n = 2; n <= maxGramLength; ++n) {
+ gram = gram + " " + words[st + n - 1];
+ if (ngramCountsAll.containsKey(gram)) {
+ int oldCount = ngramCountsAll.get(gram);
+ ngramCountsAll.put(gram, oldCount + 1);
+ } else {
+ ngramCountsAll.put(gram, 1);
+ }
+ } // for (n)
+
+ } // for (st)
+
+ // now st is either len-maxGramLength+1 or zero (if above loop never entered, which
+ // happens with sentences that have fewer than maxGramLength words)
+
+ for (; st < len; ++st) {
+
+ gram = words[st];
+ if (ngramCountsAll.containsKey(gram)) {
+ int oldCount = ngramCountsAll.get(gram);
+ ngramCountsAll.put(gram, oldCount + 1);
+ } else {
+ ngramCountsAll.put(gram, 1);
+ }
+
+ int n = 2;
+ for (int fin = st + 1; fin < len; ++fin) {
+ gram = gram + " " + words[st + n - 1];
+
+ if (ngramCountsAll.containsKey(gram)) {
+ int oldCount = ngramCountsAll.get(gram);
+ ngramCountsAll.put(gram, oldCount + 1);
+ } else {
+ ngramCountsAll.put(gram, 1);
+ }
+ ++n;
+ } // for (fin)
+
+ } // for (st)
+
+ return ngramCountsAll;
+
+ }
+
+ public void printDetailedScore_fromStats(int[] stats, boolean oneLiner) {
+ System.out.println(metricName + " = " + score(stats));
+
+ // for (Map.Entry<String, Integer> entry : refNgramCounts.) {
+ // System.out.println(entry.getKey()+" : "+ entry.getValue());
+ // }
+ //
+ //
+ // optional (for debugging purposes)
+ //
+ //
+ }
+
+ private enum StatIndex {
+ KEEPBOTH, KEEPCAND, KEEPREF, DELBOTH, DELCAND, DELREF, ADDBOTH, ADDCAND, ADDREF, KEEPBOTH2
+ };
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/metrics/SourceBLEU.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/metrics/SourceBLEU.java b/joshua-core/src/main/java/org/apache/joshua/metrics/SourceBLEU.java
new file mode 100644
index 0000000..f594954
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/metrics/SourceBLEU.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.metrics;
+
+import java.util.HashMap;
+
+public class SourceBLEU extends BLEU {
+ // We assume that the source for the paraphrasing run is
+ // part of the set of references
+ private int sourceReferenceIndex;
+
+ private int[] sourceWordCount;
+ private boolean useBrevityPenalty;
+
+ public SourceBLEU() {
+ super();
+ this.sourceReferenceIndex = 0;
+ this.useBrevityPenalty = true;
+ initialize();
+ }
+
+ public SourceBLEU(String[] options) {
+ super(options);
+ this.sourceReferenceIndex = Integer.parseInt(options[2]);
+ this.useBrevityPenalty = Boolean.parseBoolean(options[3]);
+ initialize();
+ }
+
+ public SourceBLEU(int num_references, String method, int source_index, boolean use_brevity_penalty) {
+ super(num_references, method);
+ this.sourceReferenceIndex = source_index;
+ this.useBrevityPenalty = use_brevity_penalty;
+ initialize();
+ }
+
+ protected void initialize() {
+ metricName = "SRC_BLEU";
+ toBeMinimized = true;
+ suffStatsCount = 2 * getMaxGramLength() + 2;
+
+ set_weightsArray();
+ set_maxNgramCounts();
+ }
+
+ public double bestPossibleScore() {
+ return 0.0;
+ }
+
+ public double worstPossibleScore() {
+ return 1.0;
+ }
+
+ protected void set_maxNgramCounts() {
+ @SuppressWarnings("unchecked")
+ HashMap<String, Integer>[] temp_HMA = new HashMap[numSentences];
+ maxNgramCounts = temp_HMA;
+ sourceWordCount = new int[numSentences];
+
+ for (int i = 0; i < numSentences; ++i) {
+ sourceWordCount[i] = wordCount(refSentences[i][sourceReferenceIndex]);
+ maxNgramCounts[i] = getNgramCountsAll(refSentences[i][sourceReferenceIndex]);
+ }
+ }
+
+ public int[] suffStats(String cand_str, int i) {
+ int[] stats = new int[suffStatsCount];
+
+ String[] candidate_words;
+ if (!cand_str.equals(""))
+ candidate_words = cand_str.split("\\s+");
+ else
+ candidate_words = new String[0];
+
+ set_prec_suffStats(stats, candidate_words, i);
+ if (this.useBrevityPenalty)
+ stats[suffStatsCount - 1] = effLength(candidate_words.length, i);
+ else
+ stats[suffStatsCount - 1] = candidate_words.length;
+ stats[suffStatsCount - 2] = candidate_words.length;
+
+ return stats;
+ }
+
+ public int effLength(int candLength, int i) {
+ return sourceWordCount[i];
+ }
+
+ public void printDetailedScore_fromStats(int[] stats, boolean oneLiner) {
+ System.out.println(String.format("SRC_BLEU = %.4f", score(stats)));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/metrics/TER.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/metrics/TER.java b/joshua-core/src/main/java/org/apache/joshua/metrics/TER.java
new file mode 100644
index 0000000..0dcf9d9
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/metrics/TER.java
@@ -0,0 +1,460 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.metrics;
+
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Semaphore;
+
+import org.apache.joshua.util.StreamGobbler;
+
+public class TER extends EvaluationMetric {
+ private boolean caseSensitive;
+ private boolean withPunctuation;
+ private int beamWidth;
+ private int maxShiftDist;
+ private String tercomJarFileName;
+ private int numScoringThreads;
+
+ public TER(String[] Metric_options) {
+ // M_o[0]: case sensitivity, case/nocase
+ // M_o[1]: with-punctuation, punc/nopunc
+ // M_o[2]: beam width, positive integer
+ // M_o[3]: maximum shift distance, positive integer
+ // M_o[4]: filename of tercom jar file
+ // M_o[5]: number of threads to use for TER scoring (= number of tercom processes launched)
+
+ // for 0-3, default values in tercom-0.7.25 are: nocase, punc, 20, 50
+
+ if (Metric_options[0].equals("case")) {
+ caseSensitive = true;
+ } else if (Metric_options[0].equals("nocase")) {
+ caseSensitive = false;
+ } else {
+ String msg = "Unknown case sensitivity string " + Metric_options[0]
+ + ". Should be one of case or nocase.";
+ throw new RuntimeException(msg);
+ }
+
+ if (Metric_options[1].equals("punc")) {
+ withPunctuation = true;
+ } else if (Metric_options[1].equals("nopunc")) {
+ withPunctuation = false;
+ } else {
+ String msg = "Unknown with-punctuation string " + Metric_options[1]
+ + ". Should be one of punc or nopunc.";
+ throw new RuntimeException(msg);
+ }
+
+ beamWidth = Integer.parseInt(Metric_options[2]);
+ if (beamWidth < 1) {
+ throw new RuntimeException("Beam width must be positive");
+ }
+
+ maxShiftDist = Integer.parseInt(Metric_options[3]);
+ if (maxShiftDist < 1) {
+ throw new RuntimeException("Maximum shift distance must be positive");
+ }
+
+ tercomJarFileName = Metric_options[4];
+
+ if (tercomJarFileName == null || tercomJarFileName.equals("")) {
+ throw new RuntimeException("Problem processing tercom's jar filename");
+ } else {
+ File checker = new File(tercomJarFileName);
+ if (!checker.exists()) {
+ String msg = "Could not find tercom jar file " + tercomJarFileName
+ + "(Please make sure you use the full path in the filename)";
+ throw new RuntimeException(msg);
+ }
+ }
+
+ numScoringThreads = Integer.parseInt(Metric_options[5]);
+ if (numScoringThreads < 1) {
+ throw new RuntimeException("Number of TER scoring threads must be positive");
+ }
+
+
+ TercomRunner.set_TercomParams(caseSensitive, withPunctuation, beamWidth, maxShiftDist,
+ tercomJarFileName);
+
+
+ initialize(); // set the data members of the metric
+ }
+
+ protected void initialize() {
+ metricName = "TER";
+ toBeMinimized = true;
+ suffStatsCount = 2;
+ }
+
+ public double bestPossibleScore() {
+ return 0.0;
+ }
+
+ public double worstPossibleScore() {
+ return (+1.0 / 0.0);
+ }
+
+ public int[] suffStats(String cand_str, int i) {
+ // this method should never be used when the metric is TER,
+ // because TER.java overrides createSuffStatsFile below,
+ // which is the only method that calls suffStats(String,int).
+ return null;
+ }
+
+ public int[][] suffStats(String[] cand_strings, int[] cand_indices) {
+ // calculate sufficient statistics for each sentence in an arbitrary set of candidates
+
+ int candCount = cand_strings.length;
+ if (cand_indices.length != candCount) {
+ System.out.println("Array lengths mismatch in suffStats(String[],int[]); returning null.");
+ return null;
+ }
+
+ int[][] stats = new int[candCount][suffStatsCount];
+
+ try {
+
+ // 1) Create input files for tercom
+
+ // 1a) Create hypothesis file
+ FileOutputStream outStream = new FileOutputStream("hyp.txt.TER", false); // false: don't
+ // append
+ OutputStreamWriter outStreamWriter = new OutputStreamWriter(outStream, "utf8");
+ BufferedWriter outFile = new BufferedWriter(outStreamWriter);
+
+ for (int d = 0; d < candCount; ++d) {
+ writeLine(cand_strings[d] + " (ID" + d + ")", outFile);
+ }
+
+ outFile.close();
+
+ // 1b) Create reference file
+ outStream = new FileOutputStream("ref.txt.TER", false); // false: don't append
+ outStreamWriter = new OutputStreamWriter(outStream, "utf8");
+ outFile = new BufferedWriter(outStreamWriter);
+
+ for (int d = 0; d < candCount; ++d) {
+ for (int r = 0; r < refsPerSen; ++r) {
+ writeLine(refSentences[cand_indices[d]][r] + " (ID" + d + ")", outFile);
+ }
+ }
+
+ outFile.close();
+
+ // 2) Launch tercom as an external process
+
+ runTercom("ref.txt.TER", "hyp.txt.TER", "TER_out", 500);
+
+ // 3) Read SS from output file produced by tercom.7.25.jar
+
+ BufferedReader inFile = new BufferedReader(new FileReader("TER_out.ter"));
+ String line = "";
+
+ line = inFile.readLine(); // skip hyp line
+ line = inFile.readLine(); // skip ref line
+
+ for (int d = 0; d < candCount; ++d) {
+ line = inFile.readLine(); // read info
+ String[] strA = line.split("\\s+");
+
+ stats[d][0] = (int) Double.parseDouble(strA[1]);
+ stats[d][1] = (int) Double.parseDouble(strA[2]);
+ }
+
+ inFile.close();
+
+ // 4) Delete TER files
+
+ File fd;
+ fd = new File("hyp.txt.TER");
+ if (fd.exists()) fd.delete();
+ fd = new File("ref.txt.TER");
+ if (fd.exists()) fd.delete();
+ fd = new File("TER_out.ter");
+ if (fd.exists()) fd.delete();
+
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ return stats;
+ }
+
+ public void createSuffStatsFile(String cand_strings_fileName, String cand_indices_fileName,
+ String outputFileName, int maxBatchSize) {
+
+ try {
+ int batchCount = 0;
+
+ FileInputStream inStream_cands = new FileInputStream(cand_strings_fileName);
+ BufferedReader inFile_cands =
+ new BufferedReader(new InputStreamReader(inStream_cands, "utf8"));
+
+ FileInputStream inStream_indices = new FileInputStream(cand_indices_fileName);
+ BufferedReader inFile_indices =
+ new BufferedReader(new InputStreamReader(inStream_indices, "utf8"));
+
+ while (true) {
+ ++batchCount;
+ int readCount =
+ createTercomHypFile(inFile_cands, tmpDirPrefix + "hyp.txt.TER.batch" + batchCount,
+ 10000);
+ createTercomRefFile(inFile_indices, tmpDirPrefix + "ref.txt.TER.batch" + batchCount, 10000);
+
+ if (readCount == 0) {
+ --batchCount;
+ break;
+ } else if (readCount < 10000) {
+ break;
+ }
+ }
+
+ // score the batchCount batches of candidates, in parallel, across numThreads threads
+ ExecutorService pool = Executors.newFixedThreadPool(numScoringThreads);
+ Semaphore blocker = new Semaphore(0);
+
+ for (int b = 1; b <= batchCount; ++b) {
+ pool.execute(new TercomRunner(blocker, tmpDirPrefix + "ref.txt.TER.batch" + b, tmpDirPrefix
+ + "hyp.txt.TER.batch" + b, tmpDirPrefix + "TER_out.batch" + b, 500));
+ // Each thread scores the candidates, creating a tercom output file,
+ // and then deletes the .hyp. and .ref. files, which are not needed
+ // for other batches.
+ }
+
+ pool.shutdown();
+
+ try {
+ blocker.acquire(batchCount);
+ } catch (java.lang.InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+
+ PrintWriter outFile = new PrintWriter(outputFileName);
+ for (int b = 1; b <= batchCount; ++b) {
+ copySS(tmpDirPrefix + "TER_out.batch" + b + ".ter", outFile);
+ File fd;
+ fd = new File(tmpDirPrefix + "TER_out.batch" + b + ".ter");
+ if (fd.exists()) fd.delete();
+ // .hyp. and .ref. already deleted by individual threads
+ }
+ outFile.close();
+
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ }
+
+ public int createTercomHypFile(BufferedReader inFile_cands, String hypFileName, int numCands) {
+ // returns # lines read
+
+ int readCount = 0;
+
+ try {
+ FileOutputStream outStream = new FileOutputStream(hypFileName, false); // false: don't append
+ OutputStreamWriter outStreamWriter = new OutputStreamWriter(outStream, "utf8");
+ BufferedWriter outFile = new BufferedWriter(outStreamWriter);
+
+ String line_cand = "";
+
+ if (numCands > 0) {
+ for (int d = 0; d < numCands; ++d) {
+ line_cand = inFile_cands.readLine();
+ if (line_cand != null) {
+ ++readCount;
+ writeLine(line_cand + " (ID" + d + ")", outFile);
+ } else {
+ break;
+ }
+ }
+ } else {
+ line_cand = inFile_cands.readLine();
+ int d = -1;
+ while (line_cand != null) {
+ ++readCount;
+ ++d;
+ writeLine(line_cand + " (ID" + d + ")", outFile);
+ line_cand = inFile_cands.readLine();
+ }
+ }
+
+ outFile.close();
+
+ } catch (IOException e) {
+ throw new RuntimeException("IOException in TER.createTercomHypFile(...): " + e.getMessage(), e);
+ }
+
+ return readCount;
+
+ }
+
+ public int createTercomRefFile(BufferedReader inFile_indices, String refFileName, int numIndices) {
+ // returns # lines read
+
+ int readCount = 0;
+
+ try {
+ FileOutputStream outStream = new FileOutputStream(refFileName, false); // false: don't append
+ OutputStreamWriter outStreamWriter = new OutputStreamWriter(outStream, "utf8");
+ BufferedWriter outFile = new BufferedWriter(outStreamWriter);
+
+ String line_index = "";
+
+ if (numIndices > 0) {
+ for (int d = 0; d < numIndices; ++d) {
+ line_index = inFile_indices.readLine();
+ if (line_index != null) {
+ ++readCount;
+ int index = Integer.parseInt(line_index);
+ for (int r = 0; r < refsPerSen; ++r) {
+ writeLine(refSentences[index][r] + " (ID" + d + ")", outFile);
+ }
+ } else {
+ break;
+ }
+ }
+ } else {
+ line_index = inFile_indices.readLine();
+ int d = -1;
+ while (line_index != null) {
+ ++readCount;
+ ++d;
+ int index = Integer.parseInt(line_index);
+ for (int r = 0; r < refsPerSen; ++r) {
+ writeLine(refSentences[index][r] + " (ID" + d + ")", outFile);
+ }
+ line_index = inFile_indices.readLine();
+ }
+ }
+
+ outFile.close();
+
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+
+ return readCount;
+
+ }
+
+ public int runTercom(String refFileName, String hypFileName, String outFileNamePrefix, int memSize) {
+ int exitValue = -1;
+
+ try {
+
+ String cmd_str =
+ "java -Xmx" + memSize + "m -Dfile.encoding=utf8 -jar " + tercomJarFileName + " -r "
+ + refFileName + " -h " + hypFileName + " -o ter -n " + outFileNamePrefix;
+ cmd_str += " -b " + beamWidth;
+ cmd_str += " -d " + maxShiftDist;
+ if (caseSensitive) {
+ cmd_str += " -s";
+ }
+ if (!withPunctuation) {
+ cmd_str += " -P";
+ }
+ /*
+ * From tercom's README: -s case sensitivity, optional, default is insensitive -P no
+ * punctuations, default is with punctuations.
+ */
+
+ Runtime rt = Runtime.getRuntime();
+ Process p = rt.exec(cmd_str);
+
+ StreamGobbler errorGobbler = new StreamGobbler(p.getErrorStream(), 0);
+ StreamGobbler outputGobbler = new StreamGobbler(p.getInputStream(), 0);
+
+ errorGobbler.start();
+ outputGobbler.start();
+
+ exitValue = p.waitFor();
+
+ } catch (IOException | InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+
+ return exitValue;
+
+ }
+
+ public void copySS(String inputFileName, PrintWriter outFile) {
+ try {
+ BufferedReader inFile = new BufferedReader(new FileReader(inputFileName));
+ String line = "";
+
+ line = inFile.readLine(); // skip hyp line
+ line = inFile.readLine(); // skip ref line
+
+ line = inFile.readLine(); // read info for first line
+
+ while (line != null) {
+ String[] strA = line.split("\\s+");
+ outFile
+ .println((int) Double.parseDouble(strA[1]) + " " + (int) Double.parseDouble(strA[2]));
+ line = inFile.readLine(); // read info for next line
+ }
+
+ inFile.close();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ public double score(int[] stats) {
+ if (stats.length != suffStatsCount) {
+ throw new RuntimeException("Mismatch between stats.length and suffStatsCount (" + stats.length
+ + " vs. " + suffStatsCount + ") in TER.score(int[])");
+ }
+
+ double sc = 0.0;
+
+ sc = stats[0] / (double) stats[1];
+
+ return sc;
+ }
+
+ public void printDetailedScore_fromStats(int[] stats, boolean oneLiner) {
+ if (oneLiner) {
+ System.out.println("TER = " + stats[0] + " / " + stats[1] + " = " + f4.format(score(stats)));
+ } else {
+ System.out.println("# edits = " + stats[0]);
+ System.out.println("Reference length = " + stats[1]);
+ System.out.println("TER = " + stats[0] + " / " + stats[1] + " = " + f4.format(score(stats)));
+ }
+ }
+
+ private void writeLine(String line, BufferedWriter writer) throws IOException {
+ writer.write(line, 0, line.length());
+ writer.newLine();
+ writer.flush();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/metrics/TERMinusBLEU.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/metrics/TERMinusBLEU.java b/joshua-core/src/main/java/org/apache/joshua/metrics/TERMinusBLEU.java
new file mode 100644
index 0000000..bd40140
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/metrics/TERMinusBLEU.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.metrics;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.PrintWriter;
+
+public class TERMinusBLEU extends EvaluationMetric {
+ // individual components
+ private TER myTER;
+ private BLEU myBLEU;
+ private int suffStatsCount_TER;
+ private int suffStatsCount_BLEU;
+
+ public TERMinusBLEU(String[] Metric_options) {
+ // M_o[0]: case sensitivity, case/nocase
+ // M_o[1]: with-punctuation, punc/nopunc
+ // M_o[2]: beam width, positive integer
+ // M_o[3]: maximum shift distance, positive integer
+ // M_o[4]: filename of tercom jar file
+ // M_o[5]: number of threads to use for TER scoring (= number of tercom processes launched)
+ // M_o[6]: maximum gram length, positive integer
+ // M_o[7]: effective length calculation method, closest/shortest/average
+
+ // for 0-3, default values in tercom-0.7.25 are: nocase, punc, 20, 50
+
+ myTER = new TER(Metric_options);
+ myBLEU = new BLEU(Integer.parseInt(Metric_options[6]), Metric_options[7]);
+
+ initialize(); // set the data members of the metric
+ }
+
+ protected void initialize() {
+ metricName = "TER-BLEU";
+ toBeMinimized = true;
+ suffStatsCount_TER = myTER.get_suffStatsCount();
+ suffStatsCount_BLEU = myBLEU.get_suffStatsCount();
+ suffStatsCount = suffStatsCount_TER + suffStatsCount_BLEU;
+ }
+
+ public double bestPossibleScore() {
+ return -1.0;
+ }
+
+ public double worstPossibleScore() {
+ return (+1.0 / 0.0);
+ }
+
+ public int[] suffStats(String cand_str, int i) {
+ // this method should never be used when the metric is TER-BLEU,
+ // because TERMinusBLEU.java overrides suffStats(String[],int[]) below,
+ // which is the only method that calls suffStats(Sting,int).
+ return null;
+ }
+
+ public int[][] suffStats(String[] cand_strings, int[] cand_indices) {
+ // calculate sufficient statistics for each sentence in an arbitrary set of candidates
+
+ int candCount = cand_strings.length;
+ if (cand_indices.length != candCount) {
+ System.out.println("Array lengths mismatch in suffStats(String[],int[]); returning null.");
+ return null;
+ }
+
+ int[][] stats = new int[candCount][suffStatsCount];
+ // size candCount x suffStatsCount
+ // = candCount x (suffStatsCount_TER + suffStatsCount_BLEU)
+
+ int[][] stats_TER = myTER.suffStats(cand_strings, cand_indices);
+ // size candCount x suffStatsCount_TER
+ int[][] stats_BLEU = myBLEU.suffStats(cand_strings, cand_indices);
+ // size candCount x suffStatsCount_BLEU
+
+ for (int d = 0; d < candCount; ++d) {
+ int s = 0;
+ for (int s_T = 0; s_T < suffStatsCount_TER; ++s_T) {
+ stats[d][s] = stats_TER[d][s_T];
+ ++s;
+ }
+
+ for (int s_B = 0; s_B < suffStatsCount_BLEU; ++s_B) {
+ stats[d][s] = stats_BLEU[d][s_B];
+ ++s;
+ }
+ }
+
+ return stats;
+
+ }
+
+ public void createSuffStatsFile(String cand_strings_fileName, String cand_indices_fileName,
+ String outputFileName, int maxBatchSize) {
+ try {
+ myTER.createSuffStatsFile(cand_strings_fileName, cand_indices_fileName, outputFileName
+ + ".TER", maxBatchSize);
+ myBLEU.createSuffStatsFile(cand_strings_fileName, cand_indices_fileName, outputFileName
+ + ".BLEU", maxBatchSize);
+
+ PrintWriter outFile = new PrintWriter(outputFileName);
+
+ FileInputStream inStream_TER = new FileInputStream(outputFileName + ".TER");
+ BufferedReader inFile_TER = new BufferedReader(new InputStreamReader(inStream_TER, "utf8"));
+
+ FileInputStream inStream_BLEU = new FileInputStream(outputFileName + ".BLEU");
+ BufferedReader inFile_BLEU = new BufferedReader(new InputStreamReader(inStream_BLEU, "utf8"));
+
+ String line_TER = inFile_TER.readLine();
+ String line_BLEU = inFile_BLEU.readLine();
+
+ // combine the two files into one
+ while (line_TER != null) {
+ outFile.println(line_TER + " " + line_BLEU);
+ line_TER = inFile_TER.readLine();
+ line_BLEU = inFile_BLEU.readLine();
+ }
+
+ inFile_TER.close();
+ inFile_BLEU.close();
+ outFile.close();
+
+ File fd;
+ fd = new File(outputFileName + ".TER");
+ if (fd.exists()) fd.delete();
+ fd = new File(outputFileName + ".BLEU");
+ if (fd.exists()) fd.delete();
+ } catch (IOException e) {
+ throw new RuntimeException("IOException in TER.createTercomHypFile(...): " + e.getMessage());
+ }
+ }
+
+ public double score(int[] stats) {
+ if (stats.length != suffStatsCount) {
+ throw new RuntimeException("Mismatch between stats.length and suffStatsCount (" + stats.length
+ + " vs. " + suffStatsCount + ") in TERMinusBLEU.score(int[])");
+ }
+
+ double sc = 0.0;
+
+ int[] stats_TER = new int[suffStatsCount_TER];
+ int[] stats_BLEU = new int[suffStatsCount_BLEU];
+ for (int s = 0; s < suffStatsCount_TER; ++s) {
+ stats_TER[s] = stats[s];
+ }
+ for (int s = 0; s < suffStatsCount_BLEU; ++s) {
+ stats_BLEU[s] = stats[s + suffStatsCount_TER];
+ }
+
+ double sc_T = myTER.score(stats_TER);
+ double sc_B = myBLEU.score(stats_BLEU);
+
+ sc = sc_T - sc_B;
+
+ return sc;
+ }
+
+ public void printDetailedScore_fromStats(int[] stats, boolean oneLiner) {
+ int[] stats_TER = new int[suffStatsCount_TER];
+ int[] stats_BLEU = new int[suffStatsCount_BLEU];
+ for (int s = 0; s < suffStatsCount_TER; ++s) {
+ stats_TER[s] = stats[s];
+ }
+ for (int s = 0; s < suffStatsCount_BLEU; ++s) {
+ stats_BLEU[s] = stats[s + suffStatsCount_TER];
+ }
+
+ System.out.println("---TER---");
+ myTER.printDetailedScore_fromStats(stats_TER, oneLiner);
+ System.out.println("---BLEU---");
+ myBLEU.printDetailedScore_fromStats(stats_BLEU, oneLiner);
+ System.out.println("---------");
+ System.out.println(" => " + metricName + " = " + f4.format(score(stats)));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/metrics/TercomRunner.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/metrics/TercomRunner.java b/joshua-core/src/main/java/org/apache/joshua/metrics/TercomRunner.java
new file mode 100644
index 0000000..d7eeae5
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/metrics/TercomRunner.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.metrics;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.concurrent.Semaphore;
+
+import org.apache.joshua.util.StreamGobbler;
+
+
+public class TercomRunner implements Runnable {
+ /* non-static data members */
+ private Semaphore blocker;
+
+ private String refFileName;
+ private String hypFileName;
+ private String outFileNamePrefix;
+ private int memSize;
+
+ /* static data members */
+ private static boolean caseSensitive;
+ private static boolean withPunctuation;
+ private static int beamWidth;
+ private static int maxShiftDist;
+ private static String tercomJarFileName;
+
+ public static void set_TercomParams(boolean in_caseSensitive, boolean in_withPunctuation,
+ int in_beamWidth, int in_maxShiftDist, String in_tercomJarFileName) {
+ caseSensitive = in_caseSensitive;
+ withPunctuation = in_withPunctuation;
+ beamWidth = in_beamWidth;
+ maxShiftDist = in_maxShiftDist;
+ tercomJarFileName = in_tercomJarFileName;
+ }
+
+ public TercomRunner(Semaphore in_blocker, String in_refFileName, String in_hypFileName,
+ String in_outFileNamePrefix, int in_memSize) {
+ blocker = in_blocker;
+ refFileName = in_refFileName;
+ hypFileName = in_hypFileName;
+ outFileNamePrefix = in_outFileNamePrefix;
+ memSize = in_memSize;
+ }
+
+ private void real_run() {
+
+ try {
+
+ String cmd_str =
+ "java -Xmx" + memSize + "m -Dfile.encoding=utf8 -jar " + tercomJarFileName + " -r "
+ + refFileName + " -h " + hypFileName + " -o ter -n " + outFileNamePrefix;
+ cmd_str += " -b " + beamWidth;
+ cmd_str += " -d " + maxShiftDist;
+ if (caseSensitive) {
+ cmd_str += " -s";
+ }
+ if (!withPunctuation) {
+ cmd_str += " -P";
+ }
+ /*
+ * From tercom's README: -s case sensitivity, optional, default is insensitive -P no
+ * punctuation, default is with punctuation.
+ */
+
+ Runtime rt = Runtime.getRuntime();
+ Process p = rt.exec(cmd_str);
+
+ StreamGobbler errorGobbler = new StreamGobbler(p.getErrorStream(), 0);
+ StreamGobbler outputGobbler = new StreamGobbler(p.getInputStream(), 0);
+
+ errorGobbler.start();
+ outputGobbler.start();
+
+ p.waitFor();
+
+ File fd;
+ fd = new File(hypFileName);
+ if (fd.exists()) fd.delete();
+ fd = new File(refFileName);
+ if (fd.exists()) fd.delete();
+
+ } catch (IOException| InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+
+ blocker.release();
+
+ }
+
+ public void run() {
+ try {
+ real_run();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/metrics/ZeroOneLoss.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/metrics/ZeroOneLoss.java b/joshua-core/src/main/java/org/apache/joshua/metrics/ZeroOneLoss.java
new file mode 100644
index 0000000..aee6bcc
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/metrics/ZeroOneLoss.java
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.metrics;
+
+public class ZeroOneLoss extends EvaluationMetric {
+ public ZeroOneLoss() {
+ initialize();
+ }
+
+ public ZeroOneLoss(String[] ZOL_options) {
+ this();
+ }
+
+ protected void initialize() {
+ metricName = "01LOSS";
+ toBeMinimized = true;
+ suffStatsCount = 2;
+ }
+
+ public double bestPossibleScore() {
+ return 0.0;
+ }
+
+ public double worstPossibleScore() {
+ return 1.0;
+ }
+
+ public int[] suffStats(String cand_str, int i) {
+ int[] stats = new int[suffStatsCount];
+
+ boolean matchFound = false;
+
+ for (int r = 0; r < refsPerSen; ++r) {
+ if (cand_str.equals(refSentences[i][r])) {
+ matchFound = true;
+ break;
+ }
+ }
+
+ if (matchFound) {
+ stats[0] = 1;
+ } else {
+ stats[0] = 0;
+ }
+
+ stats[1] = 1;
+
+ return stats;
+ }
+
+ public double score(int[] stats) {
+ if (stats.length != suffStatsCount) {
+ throw new RuntimeException("Mismatch between stats.length and suffStatsCount (" + stats.length
+ + " vs. " + suffStatsCount + ") in ZeroOneLoss.score(int[])");
+ }
+
+ return 1.0 - (stats[0] / (double) stats[1]);
+ }
+
+ public void printDetailedScore_fromStats(int[] stats, boolean oneLiner) {
+ if (oneLiner) {
+ System.out.println("01LOSS = 1.0 - " + stats[0] + "/" + stats[1] + " = "
+ + f4.format(1.0 - (stats[0] / (double) stats[1])));
+ } else {
+ System.out.println("# correct = " + stats[0]);
+ System.out.println("# sentences = " + stats[1]);
+ System.out.println("01LOSS = 1.0 - " + stats[0] + "/" + stats[1] + " = "
+ + f4.format(1.0 - (stats[0] / (double) stats[1])));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/mira/MIRA.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/mira/MIRA.java b/joshua-core/src/main/java/org/apache/joshua/mira/MIRA.java
new file mode 100755
index 0000000..fb1f5e2
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/mira/MIRA.java
@@ -0,0 +1,160 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.mira;
+
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.util.FileUtility;
+import org.apache.joshua.util.StreamGobbler;
+
+public class MIRA {
+ public static void main(String[] args) throws Exception {
+ JoshuaConfiguration joshuaConfiguration = new JoshuaConfiguration();
+ boolean external = false; // should each MIRA iteration be launched externally?
+
+ if (args.length == 1) {
+ if (args[0].equals("-h")) {
+ printMIRAUsage(args.length, true);
+ System.exit(2);
+ } else {
+ external = false;
+ }
+ } else if (args.length == 3) {
+ external = true;
+ } else {
+ printMIRAUsage(args.length, false);
+ System.exit(1);
+ }
+
+ if (!external) {
+ MIRACore myMIRA = new MIRACore(args[0], joshuaConfiguration);
+ myMIRA.run_MIRA(); // optimize lambda[]
+ myMIRA.finish();
+ } else {
+
+ int maxMem = Integer.parseInt(args[1]);
+ String configFileName = args[2];
+ String stateFileName = FileUtility.dirname(configFileName) + "/MIRA.temp.state";
+ String cp = System.getProperty("java.class.path");
+ boolean done = false;
+ int iteration = 0;
+
+ while (!done) {
+ ++iteration;
+ Runtime rt = Runtime.getRuntime();
+ Process p =
+ rt.exec("java -Xmx" + maxMem + "m -cp " + cp + " org.apache.joshua.mira.MIRACore " + configFileName
+ + " " + stateFileName + " " + iteration);
+ /*
+ * BufferedReader br_i = new BufferedReader(new InputStreamReader(p.getInputStream()));
+ * BufferedReader br_e = new BufferedReader(new InputStreamReader(p.getErrorStream()));
+ * String dummy_line = null; while ((dummy_line = br_i.readLine()) != null) {
+ * System.out.println(dummy_line); } while ((dummy_line = br_e.readLine()) != null) {
+ * System.out.println(dummy_line); }
+ */
+ StreamGobbler errorGobbler = new StreamGobbler(p.getErrorStream(), 1);
+ StreamGobbler outputGobbler = new StreamGobbler(p.getInputStream(), 1);
+
+ errorGobbler.start();
+ outputGobbler.start();
+
+ int status = p.waitFor();
+
+ if (status == 90) {
+ done = true;
+ } else if (status == 91) {
+ done = false;
+ } else {
+ System.out.println("MIRA exiting prematurely (MIRACore returned " + status + ")...");
+ break;
+ }
+ }
+ }
+
+ System.exit(0);
+
+ } // main(String[] args)
+
+ public static void printMIRAUsage(int argsLen, boolean detailed) {
+ if (!detailed) {
+ println("Oops, you provided " + argsLen + " args!");
+ println("");
+ println("Usage:");
+ println(" MIRA -maxMem maxMemoryInMB MIRA_configFile");
+ println("");
+ println("Where -maxMem specifies the maximum amount of memory (in MB) MIRA is");
+ println("allowed to use when performing its calculations (no memroy is needed while");
+ println("the decoder is running),");
+ println("and the config file contains any subset of MIRA's 20-some parameters,");
+ println("one per line. Run MIRA -h for more details on those parameters.");
+ } else {
+ println("Usage:");
+ println(" MIRA -maxMem maxMemoryInMB MIRA_configFile");
+ println("");
+ println("Where -maxMem specifies the maximum amount of memory (in MB) MIRA is");
+ println("allowed to use when performing its calculations (no memroy is needed while");
+ println("the decoder is running),");
+ println("and the config file contains any subset of MIRA's 20-some parameters,");
+ println("one per line. Those parameters, and their default values, are:");
+ println("");
+ println("Relevant files:");
+ println(" -dir dirPrefix: working directory\n [[default: null string (i.e. they are in the current directory)]]");
+ println(" -s sourceFile: source sentences (foreign sentences) of the MIRA dataset\n [[default: null string (i.e. file name is not needed by MIRA)]]");
+ println(" -r refFile: target sentences (reference translations) of the MIRA dataset\n [[default: reference.txt]]");
+ println(" -rps refsPerSen: number of reference translations per sentence\n [[default: 1]]");
+ //println(" -txtNrm textNormMethod: how should text be normalized?\n (0) don't normalize text,\n or (1) \"NIST-style\", and also rejoin 're, *'s, n't, etc,\n or (2) apply 1 and also rejoin dashes between letters,\n or (3) apply 1 and also drop non-ASCII characters,\n or (4) apply 1+2+3\n [[default: 1]]");
+ println(" -p paramsFile: file containing parameter names, initial values, and ranges\n [[default: params.txt]]");
+ //println(" -docInfo documentInfoFile: file informing MIRA which document each\n sentence belongs to\n [[default: null string (i.e. all sentences are in one 'document')]]");
+ println(" -fin finalLambda: file name for final lambda[] values\n [[default: null string (i.e. no such file will be created)]]");
+ println("");
+ println("MIRA specs:");
+ println(" -m metricName metric options: name of evaluation metric and its options\n [[default: BLEU 4 closest]]");
+ println(" -maxIt maxMIRAIts: maximum number of MIRA iterations\n [[default: 20]]");
+ println(" -prevIt prevMIRAIts: maximum number of previous MIRA iterations to\n construct candidate sets from\n [[default: 20]]");
+ println(" -minIt minMIRAIts: number of iterations before considering an early exit\n [[default: 5]]");
+ println(" -stopIt stopMinIts: some early stopping criterion must be satisfied in\n stopMinIts *consecutive* iterations before an early exit\n [[default: 3]]");
+ println(" -stopSig sigValue: early MIRA exit if no weight changes by more than sigValue\n [[default: -1 (i.e. this criterion is never investigated)]]");
+ //println(" -thrCnt threadCount: number of threads to run in parallel when optimizing\n [[default: 1]]");
+ println(" -save saveInter: save intermediate cfg files (1) or decoder outputs (2)\n or both (3) or neither (0)\n [[default: 3]]");
+ println(" -compress compressFiles: should MIRA compress the files it produces (1)\n or not (0)\n [[default: 0]]");
+ //println(" -ipi initsPerIt: number of intermediate initial points per iteration\n [[default: 20]]");
+ //println(" -opi oncePerIt: modify a parameter only once per iteration (1) or not (0)\n [[default: 0]]");
+ //println(" -rand randInit: choose initial point randomly (1) or from paramsFile (0)\n [[default: 0]]");
+ //println(" -seed seed: seed used to initialize random number generator\n [[default: time (i.e. value returned by System.currentTimeMillis()]]");
+ // println(" -ud useDisk: reliance on disk (0-2; higher value => more reliance)\n [[default: 2]]");
+ println("");
+ println("Decoder specs:");
+ println(" -cmd commandFile: name of file containing commands to run the decoder\n [[default: null string (i.e. decoder is a JoshuaDecoder object)]]");
+ println(" -passIt passIterationToDecoder: should iteration number be passed\n to command file (1) or not (0)\n [[default: 0]]");
+ println(" -decOut decoderOutFile: name of the output file produced by the decoder\n [[default: output.nbest]]");
+ println(" -decExit validExit: value returned by decoder to indicate success\n [[default: 0]]");
+ println(" -dcfg decConfigFile: name of decoder config file\n [[default: dec_cfg.txt]]");
+ println(" -N N: size of N-best list (per sentence) generated in each MIRA iteration\n [[default: 100]]");
+ println("");
+ println("Output specs:");
+ println(" -v verbosity: MIRA verbosity level (0-2; higher value => more verbose)\n [[default: 1]]");
+ println(" -decV decVerbosity: should decoder output be printed (1) or ignored (0)\n [[default: 0]]");
+ println("");
+ }
+ }
+
+ private static void println(Object obj) {
+ System.out.println(obj);
+ }
+
+}
[21/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/UnicodeCharacterName.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/UnicodeCharacterName.java b/joshua-core/src/main/java/org/apache/joshua/util/UnicodeCharacterName.java
new file mode 100644
index 0000000..93c759e
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/UnicodeCharacterName.java
@@ -0,0 +1,22466 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util;
+
+import java.util.HashMap;
+import java.util.Map;
+
+public class UnicodeCharacterName {
+
+ private final Map<Character, String> map;
+
+ public UnicodeCharacterName() {
+
+ int expectedSize = 21927;
+
+ map = new HashMap<Character, String>(expectedSize);
+
+ basicLatin(map);
+ latin1Supplement(map);
+ latinExtendedA(map);
+ latinExtendedB(map);
+ ipaExtensions(map);
+ spacingModifierLetters(map);
+ combiningDiacriticMarks(map);
+ greekAndCoptic(map);
+ cyrillic(map);
+ cyrillicSupplement(map);
+ armenian(map);
+ hebrew(map);
+ arabic(map);
+ syriac(map);
+ arabicSupplement(map);
+ thaana(map);
+ nko(map);
+ samaritan(map);
+ devanagari(map);
+ bengali(map);
+ gurmukhi(map);
+ gujarati(map);
+ oriya(map);
+ tamil(map);
+ telugu(map);
+ kannada(map);
+ malayalam(map);
+ sinhala(map);
+ thai(map);
+ lao(map);
+ tibetan(map);
+ myanmar(map);
+ georgian(map);
+ hangulJamo(map);
+ ethiopic(map);
+ ethiopicSupplement(map);
+ cherokee(map);
+ canadianAboriginalSyllabics(map);
+ ogham(map);
+ runic(map);
+ tagalog(map);
+ hanunoo(map);
+ buhid(map);
+ tagbanwa(map);
+ khmer(map);
+ mongolian(map);
+ canadianSyllabics(map);
+ limbu(map);
+ taiLe(map);
+ newTaiLue(map);
+ khmerSymbols(map);
+ buginese(map);
+ balinese(map);
+ sudanese(map);
+ lepcha(map);
+ olChiki(map);
+ vedic(map);
+ phoneticExtensions(map);
+ phoneticExtensionsSupplement(map);
+ combiningDiacriticalMarksSupplement(map);
+ latinExtendedAdditional(map);
+ greekExtended(map);
+ generalPunctuation(map);
+ cjkSymbolsAndPunctuation(map);
+ hangulSyllables(map);
+ }
+
+ public static final void basicLatin(Map<Character, String> map) {
+
+ // C0 Controls
+ map.put('\u0000', "NULL");
+ map.put('\u0001', "START OF HEADING");
+ map.put('\u0002', "START OF TEXT");
+ map.put('\u0003', "END OF TEXT");
+ map.put('\u0004', "END OF TRANSMISSION");
+ map.put('\u0005', "ENQUIRY");
+ map.put('\u0006', "ACKNOWLEDGE");
+ map.put('\u0007', "BELL");
+ map.put('\u0008', "BACKSPACE");
+ map.put('\u0009', "CHARACTER TABULATION");
+ map.put('\u000B', "LINE TABULATION");
+ map.put('\u000C', "FORM FEED");
+ map.put('\u000E', "SHIFT OUT");
+ map.put('\u000F', "SHIFT IN");
+ map.put('\u0010', "DATA LINK ESCAPE");
+ map.put('\u0011', "DEVICE CONTROL ONE");
+ map.put('\u0012', "DEVICE CONTROL TWO");
+ map.put('\u0013', "DEVICE CONTROL THREE");
+ map.put('\u0014', "DEVICE CONTROL FOUR");
+ map.put('\u0015', "NEGATIVE ACKNOWLEDGE");
+ map.put('\u0016', "SYNCHRONOUS IDLE");
+ map.put('\u0017', "END OF TRANSMISSION BLOCK");
+ map.put('\u0018', "CANCEL");
+ map.put('\u0019', "END OF MEDIUM");
+ map.put('\u001A', "SUBSTITUTE");
+ map.put('\u001B', "ESCAPE");
+ map.put('\u001C', "FILE SEPARATOR");
+ map.put('\u001D', "GROUP SEPARATOR");
+ map.put('\u001E', "RECORD SEPARATOR");
+ map.put('\u001F', "UNIT SEPARATOR");
+
+ // ASCII punctuation and symbols
+ map.put('\u0020', "SPACE");
+ map.put('\u0021', "EXCLAMATION MARK");
+ map.put('\u0022', "QUOTATION MARK");
+ map.put('\u0023', "NUMBER SIGN");
+ map.put('\u0024', "DOLLAR SIGN");
+ map.put('\u0025', "PERCENT SIGN");
+ map.put('\u0026', "AMPERSAND");
+ map.put('\'', "APOSTROPHE");
+ map.put('\u0028', "LEFT PARENTHESIS");
+ map.put('\u0029', "RIGHT PARENTHESIS");
+ map.put('\u002A', "ASTERISK");
+ map.put('\u002B', "PLUS SIGN");
+ map.put('\u002C', "COMMA");
+ map.put('\u002D', "HYPHEN-MINUS");
+ map.put('\u002E', "FULL STOP");
+ map.put('\u002F', "SOLIDUS");
+
+ // ASCII digits
+ map.put('\u0030', "DIGIT ZERO");
+ map.put('\u0031', "DIGIT ONE");
+ map.put('\u0032', "DIGIT TWO");
+ map.put('\u0033', "DIGIT THREE");
+ map.put('\u0034', "DIGIT FOUR");
+ map.put('\u0035', "DIGIT FIVE");
+ map.put('\u0036', "DIGIT SIX");
+ map.put('\u0037', "DIGIT SEVEN");
+ map.put('\u0038', "DIGIT EIGHT");
+ map.put('\u0039', "DIGIT NINE");
+
+ // ASCII punctuation and symbols
+ map.put('\u003A', "COLON");
+ map.put('\u003B', "SEMICOLON");
+ map.put('\u003C', "LESS-THAN SIGN");
+ map.put('\u003D', "EQUALS SIGN");
+ map.put('\u003E', "GREATER-THAN SIGN");
+ map.put('\u003F', "QUESTION MARK");
+ map.put('\u0040', "COMMERCIAL AT");
+
+ // Uppercase Latin alphabet
+ map.put('\u0041', "LATIN CAPITAL LETTER A");
+ map.put('\u0042', "LATIN CAPITAL LETTER B");
+ map.put('\u0043', "LATIN CAPITAL LETTER C");
+ map.put('\u0044', "LATIN CAPITAL LETTER D");
+ map.put('\u0045', "LATIN CAPITAL LETTER E");
+ map.put('\u0046', "LATIN CAPITAL LETTER F");
+ map.put('\u0047', "LATIN CAPITAL LETTER G");
+ map.put('\u0048', "LATIN CAPITAL LETTER H");
+ map.put('\u0049', "LATIN CAPITAL LETTER I");
+ map.put('\u004A', "LATIN CAPITAL LETTER J");
+ map.put('\u004B', "LATIN CAPITAL LETTER K");
+ map.put('\u004C', "LATIN CAPITAL LETTER L");
+ map.put('\u004D', "LATIN CAPITAL LETTER M");
+ map.put('\u004E', "LATIN CAPITAL LETTER N");
+ map.put('\u004F', "LATIN CAPITAL LETTER O");
+ map.put('\u0050', "LATIN CAPITAL LETTER P");
+ map.put('\u0051', "LATIN CAPITAL LETTER Q");
+ map.put('\u0052', "LATIN CAPITAL LETTER R");
+ map.put('\u0053', "LATIN CAPITAL LETTER S");
+ map.put('\u0054', "LATIN CAPITAL LETTER T");
+ map.put('\u0055', "LATIN CAPITAL LETTER U");
+ map.put('\u0056', "LATIN CAPITAL LETTER V");
+ map.put('\u0057', "LATIN CAPITAL LETTER W");
+ map.put('\u0058', "LATIN CAPITAL LETTER X");
+ map.put('\u0059', "LATIN CAPITAL LETTER Y");
+ map.put('\u005A', "LATIN CAPITAL LETTER Z");
+
+ // ASCII punctuation and symbols
+ map.put('\u005B', "LEFT SQUARE BRACKET");
+ map.put('\\', "REVERSE SOLIDUS");
+ map.put('\u005D', "RIGHT SQUARE BRACKET");
+ map.put('\u005E', "CIRCUMFLEX ACCENT");
+ map.put('\u005F', "LOW LINE");
+ map.put('\u0060', "GRAVE ACCENT");
+
+ // Lowercase Latin alphabet
+ map.put('\u0061', "LATIN SMALL LETTER A");
+ map.put('\u0062', "LATIN SMALL LETTER B");
+ map.put('\u0063', "LATIN SMALL LETTER C");
+ map.put('\u0064', "LATIN SMALL LETTER D");
+ map.put('\u0065', "LATIN SMALL LETTER E");
+ map.put('\u0066', "LATIN SMALL LETTER F");
+ map.put('\u0067', "LATIN SMALL LETTER G");
+ map.put('\u0068', "LATIN SMALL LETTER H");
+ map.put('\u0069', "LATIN SMALL LETTER I");
+ map.put('\u006A', "LATIN SMALL LETTER J");
+ map.put('\u006B', "LATIN SMALL LETTER K");
+ map.put('\u006C', "LATIN SMALL LETTER L");
+ map.put('\u006D', "LATIN SMALL LETTER M");
+ map.put('\u006E', "LATIN SMALL LETTER N");
+ map.put('\u006F', "LATIN SMALL LETTER O");
+ map.put('\u0070', "LATIN SMALL LETTER P");
+ map.put('\u0071', "LATIN SMALL LETTER Q");
+ map.put('\u0072', "LATIN SMALL LETTER R");
+ map.put('\u0073', "LATIN SMALL LETTER S");
+ map.put('\u0074', "LATIN SMALL LETTER T");
+ map.put('\u0075', "LATIN SMALL LETTER U");
+ map.put('\u0076', "LATIN SMALL LETTER V");
+ map.put('\u0077', "LATIN SMALL LETTER W");
+ map.put('\u0078', "LATIN SMALL LETTER X");
+ map.put('\u0079', "LATIN SMALL LETTER Y");
+ map.put('\u007A', "LATIN SMALL LETTER Z");
+
+ // ASCII punctuation and symbols
+ map.put('\u007B', "LEFT CURLY BRACKET");
+ map.put('\u007C', "VERTICAL LINE");
+ map.put('\u007D', "RIGHT CURLY BRACKET");
+ map.put('\u007E', "TILDE");
+
+ // Control character
+ map.put('\u007F', "DELETE");
+
+ }
+
+ public static final void latin1Supplement(Map<Character, String> map) {
+
+ // C1 controls
+ map.put('\u0080', "<control>");
+ map.put('\u0081', "<control>");
+ map.put('\u0082', "BREAK PERMITTED HERE");
+ map.put('\u0083', "NO BREAK HERE");
+ map.put('\u0084', "<control>");
+ map.put('\u0085', "NEXT LINE");
+ map.put('\u0086', "START OF SELECTED AREA");
+ map.put('\u0087', "END OF SELECTED AREA");
+ map.put('\u0088', "CHARACTER TABULATION SET");
+ map.put('\u0089', "CHARACTER TABULATION WITH JUSTIFICATION");
+ map.put('\u008A', "LINE TABULATION SET");
+ map.put('\u008B', "PARTIAL LINE FORWARD");
+ map.put('\u008C', "PARTIAL LINE BACKWARD");
+ map.put('\u008D', "REVERSE LINE FEED");
+ map.put('\u008E', "SINGLE SHIFT TWO");
+ map.put('\u008F', "SINGLE SHIFT THREE");
+ map.put('\u0090', "DEVICE CONTROL STRING");
+ map.put('\u0091', "PRIVATE USE ONE");
+ map.put('\u0092', "PRIVATE USE TWO");
+ map.put('\u0093', "SET TRANSMIT STATE");
+ map.put('\u0094', "CANCEL CHARACTER");
+ map.put('\u0095', "MESSAGE WAITING");
+ map.put('\u0096', "START OF GUARDED AREA");
+ map.put('\u0097', "END OF GUARDED AREA");
+ map.put('\u0098', "START OF STRING");
+ map.put('\u0099', "<control>");
+ map.put('\u009A', "SINGLE CHARACTER INTRODUCER");
+ map.put('\u009B', "CONTROL SEQUENCE INTRODUCER");
+ map.put('\u009C', "STRING TERMINATOR");
+ map.put('\u009D', "OPERATING SYSTEM COMMAND");
+ map.put('\u009E', "PRIVACY MESSAGE");
+ map.put('\u009F', "APPLICATION PROGRAM COMMAND");
+
+ // Latin-1 punctuation and symbols
+ map.put('\u00A0', "NO-BREAK SPACE");
+ map.put('\u00A1', "INVERTED EXCLAMATION MARK");
+ map.put('\u00A2', "CENT SIGN");
+ map.put('\u00A3', "POUND SIGN");
+ map.put('\u00A4', "CURRENCY SIGN");
+ map.put('\u00A5', "YEN SIGN");
+ map.put('\u00A6', "BROKEN BAR");
+ map.put('\u00A7', "SECTION SIGN");
+ map.put('\u00A8', "DIAERESIS");
+ map.put('\u00A9', "COPYRIGHT SIGN");
+ map.put('\u00AA', "FEMININE ORDINAL INDICATOR");
+ map.put('\u00AB', "LEFT-POINTING DOUBLE ANGLE QUOTATION MARK");
+ map.put('\u00AC', "NOT SIGN");
+ map.put('\u00AD', "SOFT HYPHEN");
+ map.put('\u00AE', "REGISTERED SIGN");
+ map.put('\u00AF', "MACRON");
+ map.put('\u00B0', "DEGREE SIGN");
+ map.put('\u00B1', "PLUS-MINUS SIGN");
+ map.put('\u00B2', "SUPERSCRIPT TWO");
+ map.put('\u00B3', "SUPERSCRIPT THREE");
+ map.put('\u00B4', "ACUTE ACCENT");
+ map.put('\u00B5', "MICRO SIGN");
+ map.put('\u00B6', "PILCROW SIGN");
+ map.put('\u00B7', "MIDDLE DOT");
+ map.put('\u00B8', "CEDILLA");
+ map.put('\u00B9', "SUPERSCRIPT ONE");
+ map.put('\u00BA', "MASCULINE ORDINAL INDICATOR");
+ map.put('\u00BB', "RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK");
+ map.put('\u00BC', "VULGAR FRACTION ONE QUARTER");
+ map.put('\u00BD', "VULGAR FRACTION ONE HALF");
+ map.put('\u00BE', "VULGAR FRACTION THREE QUARTERS");
+ map.put('\u00BF', "INVERTED QUESTION MARK");
+
+ // Letters
+ map.put('\u00C0', "LATIN CAPITAL LETTER A WITH GRAVE");
+ map.put('\u00C1', "LATIN CAPITAL LETTER A WITH ACUTE");
+ map.put('\u00C2', "LATIN CAPITAL LETTER A WITH CIRCUMFLEX");
+ map.put('\u00C3', "LATIN CAPITAL LETTER A WITH TILDE");
+ map.put('\u00C4', "LATIN CAPITAL LETTER A WITH DIAERESIS");
+ map.put('\u00C5', "LATIN CAPITAL LETTER A WITH RING ABOVE");
+ map.put('\u00C6', "LATIN CAPITAL LETTER AE");
+ map.put('\u00C7', "LATIN CAPITAL LETTER C WITH CEDILLA");
+ map.put('\u00C8', "LATIN CAPITAL LETTER E WITH GRAVE");
+ map.put('\u00C9', "LATIN CAPITAL LETTER E WITH ACUTE");
+ map.put('\u00CA', "LATIN CAPITAL LETTER E WITH CIRCUMFLEX");
+ map.put('\u00CB', "LATIN CAPITAL LETTER E WITH DIAERESIS");
+ map.put('\u00CC', "LATIN CAPITAL LETTER I WITH GRAVE");
+ map.put('\u00CD', "LATIN CAPITAL LETTER I WITH ACUTE");
+ map.put('\u00CE', "LATIN CAPITAL LETTER I WITH CIRCUMFLEX");
+ map.put('\u00CF', "LATIN CAPITAL LETTER I WITH DIAERESIS");
+ map.put('\u00D0', "LATIN CAPITAL LETTER ETH");
+ map.put('\u00D1', "LATIN CAPITAL LETTER N WITH TILDE");
+ map.put('\u00D2', "LATIN CAPITAL LETTER O WITH GRAVE");
+ map.put('\u00D3', "LATIN CAPITAL LETTER O WITH ACUTE");
+ map.put('\u00D4', "LATIN CAPITAL LETTER O WITH CIRCUMFLEX");
+ map.put('\u00D5', "LATIN CAPITAL LETTER O WITH TILDE");
+ map.put('\u00D6', "LATIN CAPITAL LETTER O WITH DIAERESIS");
+
+ // Mathematical operator
+ map.put('\u00D7', "MULTIPLICATION SIGN");
+
+ // Letters
+ map.put('\u00D8', "LATIN CAPITAL LETTER O WITH STROKE");
+ map.put('\u00D9', "LATIN CAPITAL LETTER U WITH GRAVE");
+ map.put('\u00DA', "LATIN CAPITAL LETTER U WITH ACUTE");
+ map.put('\u00DB', "LATIN CAPITAL LETTER U WITH CIRCUMFLEX");
+ map.put('\u00DC', "LATIN CAPITAL LETTER U WITH DIAERESIS");
+ map.put('\u00DD', "LATIN CAPITAL LETTER Y WITH ACUTE");
+ map.put('\u00DE', "LATIN CAPITAL LETTER THORN");
+ map.put('\u00DF', "LATIN SMALL LETTER SHARP S");
+ map.put('\u00E0', "LATIN SMALL LETTER A WITH GRAVE");
+ map.put('\u00E1', "LATIN SMALL LETTER A WITH ACUTE");
+ map.put('\u00E2', "LATIN SMALL LETTER A WITH CIRCUMFLEX");
+ map.put('\u00E3', "LATIN SMALL LETTER A WITH TILDE");
+ map.put('\u00E4', "LATIN SMALL LETTER A WITH DIAERESIS");
+ map.put('\u00E5', "LATIN SMALL LETTER A WITH RING ABOVE");
+ map.put('\u00E6', "LATIN SMALL LETTER AE");
+ map.put('\u00E7', "LATIN SMALL LETTER C WITH CEDILLA");
+ map.put('\u00E8', "LATIN SMALL LETTER E WITH GRAVE");
+ map.put('\u00E9', "LATIN SMALL LETTER E WITH ACUTE");
+ map.put('\u00EA', "LATIN SMALL LETTER E WITH CIRCUMFLEX");
+ map.put('\u00EB', "LATIN SMALL LETTER E WITH DIAERESIS");
+ map.put('\u00EC', "LATIN SMALL LETTER I WITH GRAVE");
+ map.put('\u00ED', "LATIN SMALL LETTER I WITH ACUTE");
+ map.put('\u00EE', "LATIN SMALL LETTER I WITH CIRCUMFLEX");
+ map.put('\u00EF', "LATIN SMALL LETTER I WITH DIAERESIS");
+ map.put('\u00F0', "LATIN SMALL LETTER ETH");
+ map.put('\u00F1', "LATIN SMALL LETTER N WITH TILDE");
+ map.put('\u00F2', "LATIN SMALL LETTER O WITH GRAVE");
+ map.put('\u00F3', "LATIN SMALL LETTER O WITH ACUTE");
+ map.put('\u00F4', "LATIN SMALL LETTER O WITH CIRCUMFLEX");
+ map.put('\u00F5', "LATIN SMALL LETTER O WITH TILDE");
+ map.put('\u00F6', "LATIN SMALL LETTER O WITH DIAERESIS");
+
+ // Mathematical operator
+ map.put('\u00F7', "DIVISION SIGN");
+
+ // Letters
+ map.put('\u00F8', "LATIN SMALL LETTER O WITH STROKE");
+ map.put('\u00F9', "LATIN SMALL LETTER U WITH GRAVE");
+ map.put('\u00FA', "LATIN SMALL LETTER U WITH ACUTE");
+ map.put('\u00FB', "LATIN SMALL LETTER U WITH CIRCUMFLEX");
+ map.put('\u00FC', "LATIN SMALL LETTER U WITH DIAERESIS");
+ map.put('\u00FD', "LATIN SMALL LETTER Y WITH ACUTE");
+ map.put('\u00FE', "LATIN SMALL LETTER THORN");
+ map.put('\u00FF', "LATIN SMALL LETTER Y WITH DIAERESIS");
+
+ }
+
+ public static final void latinExtendedA(Map<Character, String> map) {
+
+ // European Latin
+ map.put('\u0100', "LATIN CAPITAL LETTER A WITH MACRON");
+ map.put('\u0101', "LATIN SMALL LETTER A WITH MACRON");
+ map.put('\u0102', "LATIN CAPITAL LETTER A WITH BREVE");
+ map.put('\u0103', "LATIN SMALL LETTER A WITH BREVE");
+ map.put('\u0104', "LATIN CAPITAL LETTER A WITH OGONEK");
+ map.put('\u0105', "LATIN SMALL LETTER A WITH OGONEK");
+ map.put('\u0106', "LATIN CAPITAL LETTER C WITH ACUTE");
+ map.put('\u0107', "LATIN SMALL LETTER C WITH ACUTE");
+ map.put('\u0108', "LATIN CAPITAL LETTER C WITH CIRCUMFLEX");
+ map.put('\u0109', "LATIN SMALL LETTER C WITH CIRCUMFLEX");
+ map.put('\u010A', "LATIN CAPITAL LETTER C WITH DOT ABOVE");
+ map.put('\u010B', "LATIN SMALL LETTER C WITH DOT ABOVE");
+ map.put('\u010C', "LATIN CAPITAL LETTER C WITH CARON");
+ map.put('\u010D', "LATIN SMALL LETTER C WITH CARON");
+ map.put('\u010E', "LATIN CAPITAL LETTER D WITH CARON");
+ map.put('\u010F', "LATIN SMALL LETTER D WITH CARON");
+ map.put('\u0110', "LATIN CAPITAL LETTER D WITH STROKE");
+ map.put('\u0111', "LATIN SMALL LETTER D WITH STROKE");
+ map.put('\u0112', "LATIN CAPITAL LETTER E WITH MACRON");
+ map.put('\u0113', "LATIN SMALL LETTER E WITH MACRON");
+ map.put('\u0114', "LATIN CAPITAL LETTER E WITH BREVE");
+ map.put('\u0115', "LATIN SMALL LETTER E WITH BREVE");
+ map.put('\u0116', "LATIN CAPITAL LETTER E WITH DOT ABOVE");
+ map.put('\u0117', "LATIN SMALL LETTER E WITH DOT ABOVE");
+ map.put('\u0118', "LATIN CAPITAL LETTER E WITH OGONEK");
+ map.put('\u0119', "LATIN SMALL LETTER E WITH OGONEK");
+ map.put('\u011A', "LATIN CAPITAL LETTER E WITH CARON");
+ map.put('\u011B', "LATIN SMALL LETTER E WITH CARON");
+ map.put('\u011C', "LATIN CAPITAL LETTER G WITH CIRCUMFLEX");
+ map.put('\u011D', "LATIN SMALL LETTER G WITH CIRCUMFLEX");
+ map.put('\u011E', "LATIN CAPITAL LETTER G WITH BREVE");
+ map.put('\u011F', "LATIN SMALL LETTER G WITH BREVE");
+ map.put('\u0120', "LATIN CAPITAL LETTER G WITH DOT ABOVE");
+ map.put('\u0121', "LATIN SMALL LETTER G WITH DOT ABOVE");
+ map.put('\u0122', "LATIN CAPITAL LETTER G WITH CEDILLA");
+ map.put('\u0123', "LATIN SMALL LETTER G WITH CEDILLA");
+ map.put('\u0124', "LATIN CAPITAL LETTER H WITH CIRCUMFLEX");
+ map.put('\u0125', "LATIN SMALL LETTER H WITH CIRCUMFLEX");
+ map.put('\u0126', "LATIN CAPITAL LETTER H WITH STROKE");
+ map.put('\u0127', "LATIN SMALL LETTER H WITH STROKE");
+ map.put('\u0128', "LATIN CAPITAL LETTER I WITH TILDE");
+ map.put('\u0129', "LATIN SMALL LETTER I WITH TILDE");
+ map.put('\u012A', "LATIN CAPITAL LETTER I WITH MACRON");
+ map.put('\u012B', "LATIN SMALL LETTER I WITH MACRON");
+ map.put('\u012C', "LATIN CAPITAL LETTER I WITH BREVE");
+ map.put('\u012D', "LATIN SMALL LETTER I WITH BREVE");
+ map.put('\u012E', "LATIN CAPITAL LETTER I WITH OGONEK");
+ map.put('\u012F', "LATIN SMALL LETTER I WITH OGONEK");
+ map.put('\u0130', "LATIN CAPITAL LETTER I WITH DOT ABOVE");
+ map.put('\u0131', "LATIN SMALL LETTER DOTLESS I");
+ map.put('\u0132', "LATIN CAPITAL LIGATURE IJ");
+ map.put('\u0133', "LATIN SMALL LIGATURE IJ");
+ map.put('\u0134', "LATIN CAPITAL LETTER J WITH CIRCUMFLEX");
+ map.put('\u0135', "LATIN SMALL LETTER J WITH CIRCUMFLEX");
+ map.put('\u0136', "LATIN CAPITAL LETTER K WITH CEDILLA");
+ map.put('\u0137', "LATIN SMALL LETTER K WITH CEDILLA");
+ map.put('\u0138', "LATIN SMALL LETTER KRA");
+ map.put('\u0139', "LATIN CAPITAL LETTER L WITH ACUTE");
+ map.put('\u013A', "LATIN SMALL LETTER L WITH ACUTE");
+ map.put('\u013B', "LATIN CAPITAL LETTER L WITH CEDILLA");
+ map.put('\u013C', "LATIN SMALL LETTER L WITH CEDILLA");
+ map.put('\u013D', "LATIN CAPITAL LETTER L WITH CARON");
+ map.put('\u013E', "LATIN SMALL LETTER L WITH CARON");
+ map.put('\u013F', "LATIN CAPITAL LETTER L WITH MIDDLE DOT");
+ map.put('\u0140', "LATIN SMALL LETTER L WITH MIDDLE DOT");
+ map.put('\u0141', "LATIN CAPITAL LETTER L WITH STROKE");
+ map.put('\u0142', "LATIN SMALL LETTER L WITH STROKE");
+ map.put('\u0143', "LATIN CAPITAL LETTER N WITH ACUTE");
+ map.put('\u0144', "LATIN SMALL LETTER N WITH ACUTE");
+ map.put('\u0145', "LATIN CAPITAL LETTER N WITH CEDILLA");
+ map.put('\u0146', "LATIN SMALL LETTER N WITH CEDILLA");
+ map.put('\u0147', "LATIN CAPITAL LETTER N WITH CARON");
+ map.put('\u0148', "LATIN SMALL LETTER N WITH CARON");
+
+ // Deprecated letter
+ map.put('\u0149', "LATIN SMALL LETTER N PRECEDED BY APOSTROPHE");
+
+ // European Latin
+ map.put('\u014A', "LATIN CAPITAL LETTER ENG");
+ map.put('\u014B', "LATIN SMALL LETTER ENG");
+ map.put('\u014C', "LATIN CAPITAL LETTER O WITH MACRON");
+ map.put('\u014D', "LATIN SMALL LETTER O WITH MACRON");
+ map.put('\u014E', "LATIN CAPITAL LETTER O WITH BREVE");
+ map.put('\u014F', "LATIN SMALL LETTER O WITH BREVE");
+ map.put('\u0150', "LATIN CAPITAL LETTER O WITH DOUBLE ACUTE");
+ map.put('\u0151', "LATIN SMALL LETTER O WITH DOUBLE ACUTE");
+ map.put('\u0152', "LATIN CAPITAL LIGATURE OE");
+ map.put('\u0153', "LATIN SMALL LIGATURE OE");
+ map.put('\u0154', "LATIN CAPITAL LETTER R WITH ACUTE");
+ map.put('\u0155', "LATIN SMALL LETTER R WITH ACUTE");
+ map.put('\u0156', "LATIN CAPITAL LETTER R WITH CEDILLA");
+ map.put('\u0157', "LATIN SMALL LETTER R WITH CEDILLA");
+ map.put('\u0158', "LATIN CAPITAL LETTER R WITH CARON");
+ map.put('\u0159', "LATIN SMALL LETTER R WITH CARON");
+ map.put('\u015A', "LATIN CAPITAL LETTER S WITH ACUTE");
+ map.put('\u015B', "LATIN SMALL LETTER S WITH ACUTE");
+ map.put('\u015C', "LATIN CAPITAL LETTER S WITH CIRCUMFLEX");
+ map.put('\u015D', "LATIN SMALL LETTER S WITH CIRCUMFLEX");
+ map.put('\u015E', "LATIN CAPITAL LETTER S WITH CEDILLA");
+ map.put('\u015F', "LATIN SMALL LETTER S WITH CEDILLA");
+ map.put('\u0160', "LATIN CAPITAL LETTER S WITH CARON");
+ map.put('\u0161', "LATIN SMALL LETTER S WITH CARON");
+ map.put('\u0162', "LATIN CAPITAL LETTER T WITH CEDILLA");
+ map.put('\u0163', "LATIN SMALL LETTER T WITH CEDILLA");
+ map.put('\u0164', "LATIN CAPITAL LETTER T WITH CARON");
+ map.put('\u0165', "LATIN SMALL LETTER T WITH CARON");
+ map.put('\u0166', "LATIN CAPITAL LETTER T WITH STROKE");
+ map.put('\u0167', "LATIN SMALL LETTER T WITH STROKE");
+ map.put('\u0168', "LATIN CAPITAL LETTER U WITH TILDE");
+ map.put('\u0169', "LATIN SMALL LETTER U WITH TILDE");
+ map.put('\u016A', "LATIN CAPITAL LETTER U WITH MACRON");
+ map.put('\u016B', "LATIN SMALL LETTER U WITH MACRON");
+ map.put('\u016C', "LATIN CAPITAL LETTER U WITH BREVE");
+ map.put('\u016D', "LATIN SMALL LETTER U WITH BREVE");
+ map.put('\u016E', "LATIN CAPITAL LETTER U WITH RING ABOVE");
+ map.put('\u016F', "LATIN SMALL LETTER U WITH RING ABOVE");
+ map.put('\u0170', "LATIN CAPITAL LETTER U WITH DOUBLE ACUTE");
+ map.put('\u0171', "LATIN SMALL LETTER U WITH DOUBLE ACUTE");
+ map.put('\u0172', "LATIN CAPITAL LETTER U WITH OGONEK");
+ map.put('\u0173', "LATIN SMALL LETTER U WITH OGONEK");
+ map.put('\u0174', "LATIN CAPITAL LETTER W WITH CIRCUMFLEX");
+ map.put('\u0175', "LATIN SMALL LETTER W WITH CIRCUMFLEX");
+ map.put('\u0176', "LATIN CAPITAL LETTER Y WITH CIRCUMFLEX");
+ map.put('\u0177', "LATIN SMALL LETTER Y WITH CIRCUMFLEX");
+ map.put('\u0178', "LATIN CAPITAL LETTER Y WITH DIAERESIS");
+ map.put('\u0179', "LATIN CAPITAL LETTER Z WITH ACUTE");
+ map.put('\u017A', "LATIN SMALL LETTER Z WITH ACUTE");
+ map.put('\u017B', "LATIN CAPITAL LETTER Z WITH DOT ABOVE");
+ map.put('\u017C', "LATIN SMALL LETTER Z WITH DOT ABOVE");
+ map.put('\u017D', "LATIN CAPITAL LETTER Z WITH CARON");
+ map.put('\u017E', "LATIN SMALL LETTER Z WITH CARON");
+ map.put('\u017F', "LATIN SMALL LETTER LONG S");
+
+ }
+
+ public static final void latinExtendedB(Map<Character, String> map) {
+
+ // Non-European and historic Latin
+ map.put('\u0180', "LATIN SMALL LETTER B WITH STROKE");
+ map.put('\u0181', "LATIN CAPITAL LETTER B WITH HOOK");
+ map.put('\u0182', "LATIN CAPITAL LETTER B WITH TOPBAR");
+ map.put('\u0183', "LATIN SMALL LETTER B WITH TOPBAR");
+ map.put('\u0184', "LATIN CAPITAL LETTER TONE SIX");
+ map.put('\u0185', "LATIN SMALL LETTER TONE SIX");
+ map.put('\u0186', "LATIN CAPITAL LETTER OPEN O");
+ map.put('\u0187', "LATIN CAPITAL LETTER C WITH HOOK");
+ map.put('\u0188', "LATIN SMALL LETTER C WITH HOOK");
+ map.put('\u0189', "LATIN CAPITAL LETTER AFRICAN D");
+ map.put('\u018A', "LATIN CAPITAL LETTER D WITH HOOK");
+ map.put('\u018B', "LATIN CAPITAL LETTER D WITH TOPBAR");
+ map.put('\u018C', "LATIN SMALL LETTER D WITH TOPBAR");
+ map.put('\u018D', "LATIN SMALL LETTER TURNED DELTA");
+ map.put('\u018E', "LATIN CAPITAL LETTER REVERSED E");
+ map.put('\u018F', "LATIN CAPITAL LETTER SCHWA");
+ map.put('\u0190', "LATIN CAPITAL LETTER OPEN E");
+ map.put('\u0191', "LATIN CAPITAL LETTER F WITH HOOK");
+ map.put('\u0192', "LATIN SMALL LETTER F WITH HOOK");
+ map.put('\u0193', "LATIN CAPITAL LETTER G WITH HOOK");
+ map.put('\u0194', "LATIN CAPITAL LETTER GAMMA");
+ map.put('\u0195', "LATIN SMALL LETTER HV");
+ map.put('\u0196', "LATIN CAPITAL LETTER IOTA");
+ map.put('\u0197', "LATIN CAPITAL LETTER I WITH STROKE");
+ map.put('\u0198', "LATIN CAPITAL LETTER K WITH HOOK");
+ map.put('\u0199', "LATIN SMALL LETTER K WITH HOOK");
+ map.put('\u019A', "LATIN SMALL LETTER L WITH BAR");
+ map.put('\u019B', "LATIN SMALL LETTER LAMBDA WITH STROKE");
+ map.put('\u019C', "LATIN CAPITAL LETTER TURNED M");
+ map.put('\u019D', "LATIN CAPITAL LETTER N WITH LEFT HOOK");
+ map.put('\u019E', "LATIN SMALL LETTER N WITH LONG RIGHT LEG");
+ map.put('\u019F', "LATIN CAPITAL LETTER O WITH MIDDLE TILDE");
+ map.put('\u01A0', "LATIN CAPITAL LETTER O WITH HORN");
+ map.put('\u01A1', "LATIN SMALL LETTER O WITH HORN");
+ map.put('\u01A2', "LATIN CAPITAL LETTER OI");
+ map.put('\u01A3', "LATIN SMALL LETTER OI");
+ map.put('\u01A4', "LATIN CAPITAL LETTER P WITH HOOK");
+ map.put('\u01A5', "LATIN SMALL LETTER P WITH HOOK");
+ map.put('\u01A6', "LATIN LETTER YR");
+ map.put('\u01A7', "LATIN CAPITAL LETTER TONE TWO");
+ map.put('\u01A8', "LATIN SMALL LETTER TONE TWO");
+ map.put('\u01A9', "LATIN CAPITAL LETTER ESH");
+ map.put('\u01AA', "LATIN LETTER REVERSED ESH LOOP");
+ map.put('\u01AB', "LATIN SMALL LETTER T WITH PALATAL HOOK");
+ map.put('\u01AC', "LATIN CAPITAL LETTER T WITH HOOK");
+ map.put('\u01AD', "LATIN SMALL LETTER T WITH HOOK");
+ map.put('\u01AE', "LATIN CAPITAL LETTER T WITH RETROFLEX HOOK");
+ map.put('\u01AF', "LATIN CAPITAL LETTER U WITH HORN");
+ map.put('\u01B0', "LATIN SMALL LETTER U WITH HORN");
+ map.put('\u01B1', "LATIN CAPITAL LETTER UPSILON");
+ map.put('\u01B2', "LATIN CAPITAL LETTER V WITH HOOK");
+ map.put('\u01B3', "LATIN CAPITAL LETTER Y WITH HOOK");
+ map.put('\u01B4', "LATIN SMALL LETTER Y WITH HOOK");
+ map.put('\u01B5', "LATIN CAPITAL LETTER Z WITH STROKE");
+ map.put('\u01B6', "LATIN SMALL LETTER Z WITH STROKE");
+ map.put('\u01B7', "LATIN CAPITAL LETTER EZH");
+ map.put('\u01B8', "LATIN CAPITAL LETTER EZH REVERSED");
+ map.put('\u01B9', "LATIN SMALL LETTER EZH REVERSED");
+ map.put('\u01BA', "LATIN SMALL LETTER EZH WITH TAIL");
+ map.put('\u01BB', "LATIN LETTER TWO WITH STROKE");
+ map.put('\u01BC', "LATIN CAPITAL LETTER TONE FIVE");
+ map.put('\u01BD', "LATIN SMALL LETTER TONE FIVE");
+ map.put('\u01BE', "LATIN LETTER INVERTED GLOTTAL STOP WITH STROKE");
+ map.put('\u01BF', "LATIN LETTER WYNN");
+
+ // African letters for clicks
+ map.put('\u01C0', "LATIN LETTER DENTAL CLICK");
+ map.put('\u01C1', "LATIN LETTER LATERAL CLICK");
+ map.put('\u01C2', "LATIN LETTER ALVEOLAR CLICK");
+ map.put('\u01C3', "LATIN LETTER RETROFLEX CLICK");
+
+ // Croatian digraphs matching Serbian Cyrillic letters
+ map.put('\u01C4', "LATIN CAPITAL LETTER DZ WITH CARON");
+ map.put('\u01C5', "LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON");
+ map.put('\u01C6', "LATIN SMALL LETTER DZ WITH CARON");
+ map.put('\u01C7', "LATIN CAPITAL LETTER LJ");
+ map.put('\u01C8', "LATIN CAPITAL LETTER L WITH SMALL LETTER J");
+ map.put('\u01C9', "LATIN SMALL LETTER LJ");
+ map.put('\u01CA', "LATIN CAPITAL LETTER NJ");
+ map.put('\u01CB', "LATIN CAPITAL LETTER N WITH SMALL LETTER J");
+ map.put('\u01CC', "LATIN SMALL LETTER NJ");
+
+ // Pinyin diacritic-vowel combinations
+ map.put('\u01CD', "LATIN CAPITAL LETTER A WITH CARON");
+ map.put('\u01CE', "LATIN SMALL LETTER A WITH CARON");
+ map.put('\u01CF', "LATIN CAPITAL LETTER I WITH CARON");
+ map.put('\u01D0', "LATIN SMALL LETTER I WITH CARON");
+ map.put('\u01D1', "LATIN CAPITAL LETTER O WITH CARON");
+ map.put('\u01D2', "LATIN SMALL LETTER O WITH CARON");
+ map.put('\u01D3', "LATIN CAPITAL LETTER U WITH CARON");
+ map.put('\u01D4', "LATIN SMALL LETTER U WITH CARON");
+ map.put('\u01D5', "LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON");
+ map.put('\u01D6', "LATIN SMALL LETTER U WITH DIAERESIS AND MACRON");
+ map.put('\u01D7', "LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE");
+ map.put('\u01D8', "LATIN SMALL LETTER U WITH DIAERESIS AND ACUTE");
+ map.put('\u01D9', "LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON");
+ map.put('\u01DA', "LATIN SMALL LETTER U WITH DIAERESIS AND CARON");
+ map.put('\u01DB', "LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE");
+ map.put('\u01DC', "LATIN SMALL LETTER U WITH DIAERESIS AND GRAVE");
+
+ // Phonetic and historic letters
+ map.put('\u01DD', "LATIN SMALL LETTER TURNED E");
+ map.put('\u01DE', "LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON");
+ map.put('\u01DF', "LATIN SMALL LETTER A WITH DIAERESIS AND MACRON");
+ map.put('\u01E0', "LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON");
+ map.put('\u01E1', "LATIN SMALL LETTER A WITH DOT ABOVE AND MACRON");
+ map.put('\u01E2', "LATIN CAPITAL LETTER AE WITH MACRON");
+ map.put('\u01E3', "LATIN SMALL LETTER AE WITH MACRON");
+ map.put('\u01E4', "LATIN CAPITAL LETTER G WITH STROKE");
+ map.put('\u01E5', "LATIN SMALL LETTER G WITH STROKE");
+ map.put('\u01E6', "LATIN CAPITAL LETTER G WITH CARON");
+ map.put('\u01E7', "LATIN SMALL LETTER G WITH CARON");
+ map.put('\u01E8', "LATIN CAPITAL LETTER K WITH CARON");
+ map.put('\u01E9', "LATIN SMALL LETTER K WITH CARON");
+ map.put('\u01EA', "LATIN CAPITAL LETTER O WITH OGONEK");
+ map.put('\u01EB', "LATIN SMALL LETTER O WITH OGONEK");
+ map.put('\u01EC', "LATIN CAPITAL LETTER O WITH OGONEK AND MACRON");
+ map.put('\u01ED', "LATIN SMALL LETTER O WITH OGONEK AND MACRON");
+ map.put('\u01EE', "LATIN CAPITAL LETTER EZH WITH CARON");
+ map.put('\u01EF', "LATIN SMALL LETTER EZH WITH CARON");
+ map.put('\u01F0', "LATIN SMALL LETTER J WITH CARON");
+ map.put('\u01F1', "LATIN CAPITAL LETTER DZ");
+ map.put('\u01F2', "LATIN CAPITAL LETTER D WITH SMALL LETTER Z");
+ map.put('\u01F3', "LATIN SMALL LETTER DZ");
+ map.put('\u01F4', "LATIN CAPITAL LETTER G WITH ACUTE");
+ map.put('\u01F5', "LATIN SMALL LETTER G WITH ACUTE");
+ map.put('\u01F6', "LATIN CAPITAL LETTER HWAIR");
+ map.put('\u01F7', "LATIN CAPITAL LETTER WYNN");
+ map.put('\u01F8', "LATIN CAPITAL LETTER N WITH GRAVE");
+ map.put('\u01F9', "LATIN SMALL LETTER N WITH GRAVE");
+ map.put('\u01FA', "LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE");
+ map.put('\u01FB', "LATIN SMALL LETTER A WITH RING ABOVE AND ACUTE");
+ map.put('\u01FC', "LATIN CAPITAL LETTER AE WITH ACUTE");
+ map.put('\u01FD', "LATIN SMALL LETTER AE WITH ACUTE");
+ map.put('\u01FE', "LATIN CAPITAL LETTER O WITH STROKE AND ACUTE");
+ map.put('\u01FF', "LATIN SMALL LETTER O WITH STROKE AND ACUTE");
+
+ // Additions for Slovenian and Croatian
+ map.put('\u0200', "LATIN CAPITAL LETTER A WITH DOUBLE GRAVE");
+ map.put('\u0201', "LATIN SMALL LETTER A WITH DOUBLE GRAVE");
+ map.put('\u0202', "LATIN CAPITAL LETTER A WITH INVERTED BREVE");
+ map.put('\u0203', "LATIN SMALL LETTER A WITH INVERTED BREVE");
+ map.put('\u0204', "LATIN CAPITAL LETTER E WITH DOUBLE GRAVE");
+ map.put('\u0205', "LATIN SMALL LETTER E WITH DOUBLE GRAVE");
+ map.put('\u0206', "LATIN CAPITAL LETTER E WITH INVERTED BREVE");
+ map.put('\u0207', "LATIN SMALL LETTER E WITH INVERTED BREVE");
+ map.put('\u0208', "LATIN CAPITAL LETTER I WITH DOUBLE GRAVE");
+ map.put('\u0209', "LATIN SMALL LETTER I WITH DOUBLE GRAVE");
+ map.put('\u020A', "LATIN CAPITAL LETTER I WITH INVERTED BREVE");
+ map.put('\u020B', "LATIN SMALL LETTER I WITH INVERTED BREVE");
+ map.put('\u020C', "LATIN CAPITAL LETTER O WITH DOUBLE GRAVE");
+ map.put('\u020D', "LATIN SMALL LETTER O WITH DOUBLE GRAVE");
+ map.put('\u020E', "LATIN CAPITAL LETTER O WITH INVERTED BREVE");
+ map.put('\u020F', "LATIN SMALL LETTER O WITH INVERTED BREVE");
+ map.put('\u0210', "LATIN CAPITAL LETTER R WITH DOUBLE GRAVE");
+ map.put('\u0211', "LATIN SMALL LETTER R WITH DOUBLE GRAVE");
+ map.put('\u0212', "LATIN CAPITAL LETTER R WITH INVERTED BREVE");
+ map.put('\u0213', "LATIN SMALL LETTER R WITH INVERTED BREVE");
+ map.put('\u0214', "LATIN CAPITAL LETTER U WITH DOUBLE GRAVE");
+ map.put('\u0215', "LATIN SMALL LETTER U WITH DOUBLE GRAVE");
+ map.put('\u0216', "LATIN CAPITAL LETTER U WITH INVERTED BREVE");
+ map.put('\u0217', "LATIN SMALL LETTER U WITH INVERTED BREVE");
+
+ // Additions for Romanian
+ map.put('\u0218', "LATIN CAPITAL LETTER S WITH COMMA BELOW");
+ map.put('\u0219', "LATIN SMALL LETTER S WITH COMMA BELOW");
+ map.put('\u021A', "LATIN CAPITAL LETTER T WITH COMMA BELOW");
+ map.put('\u021B', "LATIN SMALL LETTER T WITH COMMA BELOW");
+
+ // Miscellaneous additions
+ map.put('\u021C', "LATIN CAPITAL LETTER YOGH");
+ map.put('\u021D', "LATIN SMALL LETTER YOGH");
+ map.put('\u021E', "LATIN CAPITAL LETTER H WITH CARON");
+ map.put('\u021F', "LATIN SMALL LETTER H WITH CARON");
+ map.put('\u0220', "LATIN CAPITAL LETTER N WITH LONG RIGHT LEG");
+ map.put('\u0221', "LATIN SMALL LETTER D WITH CURL");
+ map.put('\u0222', "LATIN CAPITAL LETTER OU");
+ map.put('\u0223', "LATIN SMALL LETTER OU");
+ map.put('\u0224', "LATIN CAPITAL LETTER Z WITH HOOK");
+ map.put('\u0225', "LATIN SMALL LETTER Z WITH HOOK");
+ map.put('\u0226', "LATIN CAPITAL LETTER A WITH DOT ABOVE");
+ map.put('\u0227', "LATIN SMALL LETTER A WITH DOT ABOVE");
+ map.put('\u0228', "LATIN CAPITAL LETTER E WITH CEDILLA");
+ map.put('\u0229', "LATIN SMALL LETTER E WITH CEDILLA");
+
+ // Additions for Livonian
+ map.put('\u022A', "LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON");
+ map.put('\u022B', "LATIN SMALL LETTER O WITH DIAERESIS AND MACRON");
+ map.put('\u022C', "LATIN CAPITAL LETTER O WITH TILDE AND MACRON");
+ map.put('\u022D', "LATIN SMALL LETTER O WITH TILDE AND MACRON");
+ map.put('\u022E', "LATIN CAPITAL LETTER O WITH DOT ABOVE");
+ map.put('\u022F', "LATIN SMALL LETTER O WITH DOT ABOVE");
+ map.put('\u0230', "LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON");
+ map.put('\u0231', "LATIN SMALL LETTER O WITH DOT ABOVE AND MACRON");
+ map.put('\u0232', "LATIN CAPITAL LETTER Y WITH MACRON");
+ map.put('\u0233', "LATIN SMALL LETTER Y WITH MACRON");
+
+ // Additions for Sinology
+ map.put('\u0234', "LATIN SMALL LETTER L WITH CURL");
+ map.put('\u0235', "LATIN SMALL LETTER N WITH CURL");
+ map.put('\u0236', "LATIN SMALL LETTER T WITH CURL");
+
+ // Miscellaneous additions
+ map.put('\u0237', "LATIN SMALL LETTER DOTLESS J");
+ map.put('\u0238', "LATIN SMALL LETTER DB DIGRAPH");
+ map.put('\u0239', "LATIN SMALL LETTER QP DIGRAPH");
+ map.put('\u023A', "LATIN CAPITAL LETTER A WITH STROKE");
+ map.put('\u023B', "LATIN CAPITAL LETTER C WITH STROKE");
+ map.put('\u023C', "LATIN SMALL LETTER C WITH STROKE");
+ map.put('\u023D', "LATIN CAPITAL LETTER L WITH BAR");
+ map.put('\u023E', "LATIN CAPITAL LETTER T WITH DIAGONAL STROKE");
+ map.put('\u023F', "LATIN SMALL LETTER S WITH SWASH TAIL");
+ map.put('\u0240', "LATIN SMALL LETTER Z WITH SWASH TAIL");
+ map.put('\u0241', "LATIN CAPITAL LETTER GLOTTAL STOP");
+ map.put('\u0242', "LATIN SMALL LETTER GLOTTAL STOP");
+ map.put('\u0243', "LATIN CAPITAL LETTER B WITH STROKE");
+ map.put('\u0244', "LATIN CAPITAL LETTER U BAR");
+ map.put('\u0245', "LATIN CAPITAL LETTER TURNED V");
+ map.put('\u0246', "LATIN CAPITAL LETTER E WITH STROKE");
+ map.put('\u0247', "LATIN SMALL LETTER E WITH STROKE");
+ map.put('\u0248', "LATIN CAPITAL LETTER J WITH STROKE");
+ map.put('\u0249', "LATIN SMALL LETTER J WITH STROKE");
+ map.put('\u024A', "LATIN CAPITAL LETTER SMALL Q WITH HOOK TAIL");
+ map.put('\u024B', "LATIN SMALL LETTER Q WITH HOOK TAIL");
+ map.put('\u024C', "LATIN CAPITAL LETTER R WITH STROKE");
+ map.put('\u024D', "LATIN SMALL LETTER R WITH STROKE");
+ map.put('\u024E', "LATIN CAPITAL LETTER Y WITH STROKE");
+ map.put('\u024F', "LATIN SMALL LETTER Y WITH STROKE");
+
+ }
+
+ public static final void ipaExtensions(Map<Character, String> map) {
+
+ // IPA extensions
+ map.put('\u0250', "LATIN SMALL LETTER TURNED A");
+ map.put('\u0251', "LATIN SMALL LETTER ALPHA");
+ map.put('\u0252', "LATIN SMALL LETTER TURNED ALPHA");
+ map.put('\u0253', "LATIN SMALL LETTER B WITH HOOK");
+ map.put('\u0254', "LATIN SMALL LETTER OPEN O");
+ map.put('\u0255', "LATIN SMALL LETTER C WITH CURL");
+ map.put('\u0256', "LATIN SMALL LETTER D WITH TAIL");
+ map.put('\u0257', "LATIN SMALL LETTER D WITH HOOK");
+ map.put('\u0258', "LATIN SMALL LETTER REVERSED E");
+ map.put('\u0259', "LATIN SMALL LETTER SCHWA");
+ map.put('\u025A', "LATIN SMALL LETTER SCHWA WITH HOOK");
+ map.put('\u025B', "LATIN SMALL LETTER OPEN E");
+ map.put('\u025C', "LATIN SMALL LETTER REVERSED OPEN E");
+ map.put('\u025D', "LATIN SMALL LETTER REVERSED OPEN E WITH HOOK");
+ map.put('\u025E', "LATIN SMALL LETTER CLOSED REVERSED OPEN E");
+ map.put('\u025F', "LATIN SMALL LETTER DOTLESS J WITH STROKE");
+ map.put('\u0260', "LATIN SMALL LETTER G WITH HOOK");
+ map.put('\u0261', "LATIN SMALL LETTER SCRIPT G");
+ map.put('\u0262', "LATIN LETTER SMALL CAPITAL G");
+ map.put('\u0263', "LATIN SMALL LETTER GAMMA");
+ map.put('\u0264', "LATIN SMALL LETTER RAMS HORN");
+ map.put('\u0265', "LATIN SMALL LETTER TURNED H");
+ map.put('\u0266', "LATIN SMALL LETTER H WITH HOOK");
+ map.put('\u0267', "LATIN SMALL LETTER HENG WITH HOOK");
+ map.put('\u0268', "LATIN SMALL LETTER I WITH STROKE");
+ map.put('\u0269', "LATIN SMALL LETTER IOTA");
+ map.put('\u026A', "LATIN LETTER SMALL CAPITAL I");
+ map.put('\u026B', "LATIN SMALL LETTER L WITH MIDDLE TILDE");
+ map.put('\u026C', "LATIN SMALL LETTER L WITH BELT");
+ map.put('\u026D', "LATIN SMALL LETTER L WITH RETROFLEX HOOK");
+ map.put('\u026E', "LATIN SMALL LETTER LEZH");
+ map.put('\u026F', "LATIN SMALL LETTER TURNED M");
+ map.put('\u0270', "LATIN SMALL LETTER TURNED M WITH LONG LEG");
+ map.put('\u0271', "LATIN SMALL LETTER M WITH HOOK");
+ map.put('\u0272', "LATIN SMALL LETTER N WITH LEFT HOOK");
+ map.put('\u0273', "LATIN SMALL LETTER N WITH RETROFLEX HOOK");
+ map.put('\u0274', "LATIN LETTER SMALL CAPITAL N");
+ map.put('\u0275', "LATIN SMALL LETTER BARRED O");
+ map.put('\u0276', "LATIN LETTER SMALL CAPITAL OE");
+ map.put('\u0277', "LATIN SMALL LETTER CLOSED OMEGA");
+ map.put('\u0278', "LATIN SMALL LETTER PHI");
+ map.put('\u0279', "LATIN SMALL LETTER TURNED R");
+ map.put('\u027A', "LATIN SMALL LETTER TURNED R WITH LONG LEG");
+ map.put('\u027B', "LATIN SMALL LETTER TURNED R WITH HOOK");
+ map.put('\u027C', "LATIN SMALL LETTER R WITH LONG LEG");
+ map.put('\u027D', "LATIN SMALL LETTER R WITH TAIL");
+ map.put('\u027E', "LATIN SMALL LETTER R WITH FISHHOOK");
+ map.put('\u027F', "LATIN SMALL LETTER REVERSED R WITH FISHHOOK");
+ map.put('\u0280', "LATIN LETTER SMALL CAPITAL R");
+ map.put('\u0281', "LATIN LETTER SMALL CAPITAL INVERTED R");
+ map.put('\u0282', "LATIN SMALL LETTER S WITH HOOK");
+ map.put('\u0283', "LATIN SMALL LETTER ESH");
+ map.put('\u0284', "LATIN SMALL LETTER DOTLESS J WITH STROKE AND HOOK");
+ map.put('\u0285', "LATIN SMALL LETTER SQUAT REVERSED ESH");
+ map.put('\u0286', "LATIN SMALL LETTER ESH WITH CURL");
+ map.put('\u0287', "LATIN SMALL LETTER TURNED T");
+ map.put('\u0288', "LATIN SMALL LETTER T WITH RETROFLEX HOOK");
+ map.put('\u0289', "LATIN SMALL LETTER U BAR");
+ map.put('\u028A', "LATIN SMALL LETTER UPSILON");
+ map.put('\u028B', "LATIN SMALL LETTER V WITH HOOK");
+ map.put('\u028C', "LATIN SMALL LETTER TURNED V");
+ map.put('\u028D', "LATIN SMALL LETTER TURNED W");
+ map.put('\u028E', "LATIN SMALL LETTER TURNED Y");
+ map.put('\u028F', "LATIN LETTER SMALL CAPITAL Y");
+ map.put('\u0290', "LATIN SMALL LETTER Z WITH RETROFLEX HOOK");
+ map.put('\u0291', "LATIN SMALL LETTER Z WITH CURL");
+ map.put('\u0292', "LATIN SMALL LETTER EZH");
+ map.put('\u0293', "LATIN SMALL LETTER EZH WITH CURL");
+ map.put('\u0294', "LATIN LETTER GLOTTAL STOP");
+ map.put('\u0295', "LATIN LETTER PHARYNGEAL VOICED FRICATIVE");
+ map.put('\u0296', "LATIN LETTER INVERTED GLOTTAL STOP");
+ map.put('\u0297', "LATIN LETTER STRETCHED C");
+ map.put('\u0298', "LATIN LETTER BILABIAL CLICK");
+ map.put('\u0299', "LATIN LETTER SMALL CAPITAL B");
+ map.put('\u029A', "LATIN SMALL LETTER CLOSED OPEN E");
+ map.put('\u029B', "LATIN LETTER SMALL CAPITAL G WITH HOOK");
+ map.put('\u029C', "LATIN LETTER SMALL CAPITAL H");
+ map.put('\u029D', "LATIN SMALL LETTER J WITH CROSSED-TAIL");
+ map.put('\u029E', "LATIN SMALL LETTER TURNED K");
+ map.put('\u029F', "LATIN LETTER SMALL CAPITAL L");
+ map.put('\u02A0', "LATIN SMALL LETTER Q WITH HOOK");
+ map.put('\u02A1', "LATIN LETTER GLOTTAL STOP WITH STROKE");
+ map.put('\u02A2', "LATIN LETTER REVERSED GLOTTAL STOP WITH STROKE");
+ map.put('\u02A3', "LATIN SMALL LETTER DZ DIGRAPH");
+ map.put('\u02A4', "LATIN SMALL LETTER DEZH DIGRAPH");
+ map.put('\u02A5', "LATIN SMALL LETTER DZ DIGRAPH WITH CURL");
+ map.put('\u02A6', "LATIN SMALL LETTER TS DIGRAPH");
+ map.put('\u02A7', "LATIN SMALL LETTER TESH DIGRAPH");
+ map.put('\u02A8', "LATIN SMALL LETTER TC DIGRAPH WITH CURL");
+
+ // IPA characters for disordered speech
+ map.put('\u02A9', "LATIN SMALL LETTER FENG DIGRAPH");
+ map.put('\u02AA', "LATIN SMALL LETTER LS DIGRAPH");
+ map.put('\u02AB', "LATIN SMALL LETTER LZ DIGRAPH");
+ map.put('\u02AC', "LATIN LETTER BILABIAL PERCUSSIVE");
+ map.put('\u02AD', "LATIN LETTER BIDENTAL PERCUSSIVE");
+
+ // Additions for Sinology
+ map.put('\u02AE', "LATIN SMALL LETTER TURNED H WITH FISHHOOK");
+ map.put('\u02AF', "LATIN SMALL LETTER TURNED H WITH FISHHOOK AND TAIL");
+
+ }
+
+ public static final void spacingModifierLetters(Map<Character, String> map) {
+
+ // Latin superscript modifier letters
+ map.put('\u02B0', "MODIFIER LETTER SMALL H");
+ map.put('\u02B1', "MODIFIER LETTER SMALL H WITH HOOK");
+ map.put('\u02B2', "MODIFIER LETTER SMALL J");
+ map.put('\u02B3', "MODIFIER LETTER SMALL R");
+ map.put('\u02B4', "MODIFIER LETTER SMALL TURNED R");
+ map.put('\u02B5', "MODIFIER LETTER SMALL TURNED R WITH HOOK");
+ map.put('\u02B6', "MODIFIER LETTER SMALL CAPITAL INVERTED R");
+ map.put('\u02B7', "MODIFIER LETTER SMALL W");
+ map.put('\u02B8', "MODIFIER LETTER SMALL Y");
+
+ // Miscellaneous phonetic modifiers
+ map.put('\u02B9', "MODIFIER LETTER PRIME");
+ map.put('\u02BA', "MODIFIER LETTER DOUBLE PRIME");
+ map.put('\u02BB', "MODIFIER LETTER TURNED COMMA");
+ map.put('\u02BC', "MODIFIER LETTER APOSTROPHE");
+ map.put('\u02BD', "MODIFIER LETTER REVERSED COMMA");
+ map.put('\u02BE', "MODIFIER LETTER RIGHT HALF RING");
+ map.put('\u02BF', "MODIFIER LETTER LEFT HALF RING");
+ map.put('\u02C0', "MODIFIER LETTER GLOTTAL STOP");
+ map.put('\u02C1', "MODIFIER LETTER REVERSED GLOTTAL STOP");
+ map.put('\u02C2', "MODIFIER LETTER LEFT ARROWHEAD");
+ map.put('\u02C3', "MODIFIER LETTER RIGHT ARROWHEAD");
+ map.put('\u02C4', "MODIFIER LETTER UP ARROWHEAD");
+ map.put('\u02C5', "MODIFIER LETTER DOWN ARROWHEAD");
+ map.put('\u02C6', "MODIFIER LETTER CIRCUMFLEX ACCENT");
+ map.put('\u02C7', "CARON");
+ map.put('\u02C8', "MODIFIER LETTER VERTICAL LINE");
+ map.put('\u02C9', "MODIFIER LETTER MACRON");
+ map.put('\u02CA', "MODIFIER LETTER ACUTE ACCENT");
+ map.put('\u02CB', "MODIFIER LETTER GRAVE ACCENT");
+ map.put('\u02CC', "MODIFIER LETTER LOW VERTICAL LINE");
+ map.put('\u02CD', "MODIFIER LETTER LOW MACRON");
+ map.put('\u02CE', "MODIFIER LETTER LOW GRAVE ACCENT");
+ map.put('\u02CF', "MODIFIER LETTER LOW ACUTE ACCENT");
+ map.put('\u02D0', "MODIFIER LETTER TRIANGULAR COLON");
+ map.put('\u02D1', "MODIFIER LETTER HALF TRIANGULAR COLON");
+ map.put('\u02D2', "MODIFIER LETTER CENTRED RIGHT HALF RING");
+ map.put('\u02D3', "MODIFIER LETTER CENTRED LEFT HALF RING");
+ map.put('\u02D4', "MODIFIER LETTER UP TACK");
+ map.put('\u02D5', "MODIFIER LETTER DOWN TACK");
+ map.put('\u02D6', "MODIFIER LETTER PLUS SIGN");
+ map.put('\u02D7', "MODIFIER LETTER MINUS SIGN");
+
+ // Spacing clones of diacritics
+ map.put('\u02D8', "BREVE");
+ map.put('\u02D9', "DOT ABOVE");
+ map.put('\u02DA', "RING ABOVE");
+ map.put('\u02DB', "OGONEK");
+ map.put('\u02DC', "SMALL TILDE");
+ map.put('\u02DD', "DOUBLE ACUTE ACCENT");
+
+ // Additions based on 1989 IPA
+ map.put('\u02DE', "MODIFIER LETTER RHOTIC HOOK");
+ map.put('\u02DF', "MODIFIER LETTER CROSS ACCENT");
+ map.put('\u02E0', "MODIFIER LETTER SMALL GAMMA");
+ map.put('\u02E1', "MODIFIER LETTER SMALL L");
+ map.put('\u02E2', "MODIFIER LETTER SMALL S");
+ map.put('\u02E3', "MODIFIER LETTER SMALL X");
+ map.put('\u02E4', "MODIFIER LETTER SMALL REVERSED GLOTTAL STOP");
+
+ // Tone letters
+ map.put('\u02E5', "MODIFIER LETTER EXTRA-HIGH TONE BAR");
+ map.put('\u02E6', "MODIFIER LETTER HIGH TONE BAR");
+ map.put('\u02E7', "MODIFIER LETTER MID TONE BAR");
+ map.put('\u02E8', "MODIFIER LETTER LOW TONE BAR");
+ map.put('\u02E9', "MODIFIER LETTER EXTRA-LOW TONE BAR");
+
+ // Extended Bopomofo tone marks
+ map.put('\u02EA', "MODIFIER LETTER YIN DEPARTING TONE MARK");
+ map.put('\u02EB', "MODIFIER LETTER YANG DEPARTING TONE MARK");
+
+ // IPA modifiers
+ map.put('\u02EC', "MODIFIER LETTER VOICING");
+ map.put('\u02ED', "MODIFIER LETTER UNASPIRATED");
+
+ // Other modifier letter
+ map.put('\u02EE', "MODIFIER LETTER DOUBLE APOSTROPHE");
+
+ // UPA modifiers
+ map.put('\u02EF', "MODIFIER LETTER LOW DOWN ARROWHEAD");
+ map.put('\u02F0', "MODIFIER LETTER LOW UP ARROWHEAD");
+ map.put('\u02F1', "MODIFIER LETTER LOW LEFT ARROWHEAD");
+ map.put('\u02F2', "MODIFIER LETTER LOW RIGHT ARROWHEAD");
+ map.put('\u02F3', "MODIFIER LETTER LOW RING");
+ map.put('\u02F4', "MODIFIER LETTER MIDDLE GRAVE ACCENT");
+ map.put('\u02F5', "MODIFIER LETTER MIDDLE DOUBLE GRAVE ACCENT");
+ map.put('\u02F6', "MODIFIER LETTER MIDDLE DOUBLE ACUTE ACCENT");
+ map.put('\u02F7', "MODIFIER LETTER LOW TILDE");
+ map.put('\u02F8', "MODIFIER LETTER RAISED COLON");
+ map.put('\u02F9', "MODIFIER LETTER BEGIN HIGH TONE");
+ map.put('\u02FA', "MODIFIER LETTER END HIGH TONE");
+ map.put('\u02FB', "MODIFIER LETTER BEGIN LOW TONE");
+ map.put('\u02FC', "MODIFIER LETTER END LOW TONE");
+ map.put('\u02FD', "MODIFIER LETTER SHELF");
+ map.put('\u02FE', "MODIFIER LETTER OPEN SHELF");
+ map.put('\u02FF', "MODIFIER LETTER LOW LEFT ARROW");
+
+ }
+
+ public static final void combiningDiacriticMarks(Map<Character, String> map) {
+
+ // Ordinary diacritics
+ map.put('\u0300', "COMBINING GRAVE ACCENT");
+ map.put('\u0301', "COMBINING ACUTE ACCENT");
+ map.put('\u0302', "COMBINING CIRCUMFLEX ACCENT");
+ map.put('\u0303', "COMBINING TILDE");
+ map.put('\u0304', "COMBINING MACRON");
+ map.put('\u0305', "COMBINING OVERLINE");
+ map.put('\u0306', "COMBINING BREVE");
+ map.put('\u0307', "COMBINING DOT ABOVE");
+ map.put('\u0308', "COMBINING DIAERESIS");
+ map.put('\u0309', "COMBINING HOOK ABOVE");
+ map.put('\u030A', "COMBINING RING ABOVE");
+ map.put('\u030B', "COMBINING DOUBLE ACUTE ACCENT");
+ map.put('\u030C', "COMBINING CARON");
+ map.put('\u030D', "COMBINING VERTICAL LINE ABOVE");
+ map.put('\u030E', "COMBINING DOUBLE VERTICAL LINE ABOVE");
+ map.put('\u030F', "COMBINING DOUBLE GRAVE ACCENT");
+ map.put('\u0310', "COMBINING CANDRABINDU");
+ map.put('\u0311', "COMBINING INVERTED BREVE");
+ map.put('\u0312', "COMBINING TURNED COMMA ABOVE");
+ map.put('\u0313', "COMBINING COMMA ABOVE");
+ map.put('\u0314', "COMBINING REVERSED COMMA ABOVE");
+ map.put('\u0315', "COMBINING COMMA ABOVE RIGHT");
+ map.put('\u0316', "COMBINING GRAVE ACCENT BELOW");
+ map.put('\u0317', "COMBINING ACUTE ACCENT BELOW");
+ map.put('\u0318', "COMBINING LEFT TACK BELOW");
+ map.put('\u0319', "COMBINING RIGHT TACK BELOW");
+ map.put('\u031A', "COMBINING LEFT ANGLE ABOVE");
+ map.put('\u031B', "COMBINING HORN");
+ map.put('\u031C', "COMBINING LEFT HALF RING BELOW");
+ map.put('\u031D', "COMBINING UP TACK BELOW");
+ map.put('\u031E', "COMBINING DOWN TACK BELOW");
+ map.put('\u031F', "COMBINING PLUS SIGN BELOW");
+ map.put('\u0320', "COMBINING MINUS SIGN BELOW");
+ map.put('\u0321', "COMBINING PALATALIZED HOOK BELOW");
+ map.put('\u0322', "COMBINING RETROFLEX HOOK BELOW");
+ map.put('\u0323', "COMBINING DOT BELOW");
+ map.put('\u0324', "COMBINING DIAERESIS BELOW");
+ map.put('\u0325', "COMBINING RING BELOW");
+ map.put('\u0326', "COMBINING COMMA BELOW");
+ map.put('\u0327', "COMBINING CEDILLA");
+ map.put('\u0328', "COMBINING OGONEK");
+ map.put('\u0329', "COMBINING VERTICAL LINE BELOW");
+ map.put('\u032A', "COMBINING BRIDGE BELOW");
+ map.put('\u032B', "COMBINING INVERTED DOUBLE ARCH BELOW");
+ map.put('\u032C', "COMBINING CARON BELOW");
+ map.put('\u032D', "COMBINING CIRCUMFLEX ACCENT BELOW");
+ map.put('\u032E', "COMBINING BREVE BELOW");
+ map.put('\u032F', "COMBINING INVERTED BREVE BELOW");
+ map.put('\u0330', "COMBINING TILDE BELOW");
+ map.put('\u0331', "COMBINING MACRON BELOW");
+ map.put('\u0332', "COMBINING LOW LINE");
+ map.put('\u0333', "COMBINING DOUBLE LOW LINE");
+
+ // Overstruck diacritics
+ map.put('\u0334', "COMBINING TILDE OVERLAY");
+ map.put('\u0335', "COMBINING SHORT STROKE OVERLAY");
+ map.put('\u0336', "COMBINING LONG STROKE OVERLAY");
+ map.put('\u0337', "COMBINING SHORT SOLIDUS OVERLAY");
+ map.put('\u0338', "COMBINING LONG SOLIDUS OVERLAY");
+
+ // Additions
+ map.put('\u0339', "COMBINING RIGHT HALF RING BELOW");
+ map.put('\u033A', "COMBINING INVERTED BRIDGE BELOW");
+ map.put('\u033B', "COMBINING SQUARE BELOW");
+ map.put('\u033C', "COMBINING SEAGULL BELOW");
+ map.put('\u033D', "COMBINING X ABOVE");
+ map.put('\u033E', "COMBINING VERTICAL TILDE");
+ map.put('\u033F', "COMBINING DOUBLE OVERLINE");
+
+ // Vietnamese tone marks
+ map.put('\u0340', "COMBINING GRAVE TONE MARK");
+ map.put('\u0341', "COMBINING ACUTE TONE MARK");
+
+ // Additions for Greek
+ map.put('\u0342', "COMBINING GREEK PERISPOMENI");
+ map.put('\u0343', "COMBINING GREEK KORONIS");
+ map.put('\u0344', "COMBINING GREEK DIALYTIKA TONOS");
+ map.put('\u0345', "COMBINING GREEK YPOGEGRAMMENI");
+
+ // Additions for IPA
+ map.put('\u0346', "COMBINING BRIDGE ABOVE");
+ map.put('\u0347', "COMBINING EQUALS SIGN BELOW");
+ map.put('\u0348', "COMBINING DOUBLE VERTICAL LINE BELOW");
+ map.put('\u0349', "COMBINING LEFT ANGLE BELOW");
+ map.put('\u034A', "COMBINING NOT TILDE ABOVE");
+
+ // IPA diacritics for disordered speech
+ map.put('\u034B', "COMBINING HOMOTHETIC ABOVE");
+ map.put('\u034C', "COMBINING ALMOST EQUAL TO ABOVE");
+ map.put('\u034D', "COMBINING LEFT RIGHT ARROW BELOW");
+ map.put('\u034E', "COMBINING UPWARDS ARROW BELOW");
+
+ // Grapheme joiner
+ map.put('\u034F', "COMBINING GRAPHEME JOINER");
+
+ // Additions for the Uralic Phonetic Alphabet
+ map.put('\u0350', "COMBINING RIGHT ARROWHEAD ABOVE");
+ map.put('\u0351', "COMBINING LEFT HALF RING ABOVE");
+ map.put('\u0352', "COMBINING FERMATA");
+ map.put('\u0353', "COMBINING X BELOW");
+ map.put('\u0354', "COMBINING LEFT ARROWHEAD BELOW");
+ map.put('\u0355', "COMBINING RIGHT ARROWHEAD BELOW");
+ map.put('\u0356', "COMBINING RIGHT ARROWHEAD AND UP ARROWHEAD BELOW");
+ map.put('\u0357', "COMBINING RIGHT HALF RING ABOVE");
+
+ // Miscellaneous additions
+ map.put('\u0358', "COMBINING DOT ABOVE RIGHT");
+ map.put('\u0359', "COMBINING ASTERISK BELOW");
+ map.put('\u035A', "COMBINING DOUBLE RING BELOW");
+ map.put('\u035B', "COMBINING ZIGZAG ABOVE");
+
+ // Double diacritics
+ map.put('\u035C', "COMBINING DOUBLE BREVE BELOW");
+ map.put('\u035D', "COMBINING DOUBLE BREVE");
+ map.put('\u035E', "COMBINING DOUBLE MACRON");
+ map.put('\u035F', "COMBINING DOUBLE MACRON BELOW");
+ map.put('\u0360', "COMBINING DOUBLE TILDE");
+ map.put('\u0361', "COMBINING DOUBLE INVERTED BREVE");
+ map.put('\u0362', "COMBINING DOUBLE RIGHTWARDS ARROW BELOW");
+
+ // Medieval superscript letter diacritics
+ map.put('\u0363', "COMBINING LATIN SMALL LETTER A");
+ map.put('\u0364', "COMBINING LATIN SMALL LETTER E");
+ map.put('\u0365', "COMBINING LATIN SMALL LETTER I");
+ map.put('\u0366', "COMBINING LATIN SMALL LETTER O");
+ map.put('\u0367', "COMBINING LATIN SMALL LETTER U");
+ map.put('\u0368', "COMBINING LATIN SMALL LETTER C");
+ map.put('\u0369', "COMBINING LATIN SMALL LETTER D");
+ map.put('\u036A', "COMBINING LATIN SMALL LETTER H");
+ map.put('\u036B', "COMBINING LATIN SMALL LETTER M");
+ map.put('\u036C', "COMBINING LATIN SMALL LETTER R");
+ map.put('\u036D', "COMBINING LATIN SMALL LETTER T");
+ map.put('\u036E', "COMBINING LATIN SMALL LETTER V");
+ map.put('\u036F', "COMBINING LATIN SMALL LETTER X");
+
+ }
+
+ public static final void greekAndCoptic(Map<Character, String> map) {
+
+ map.put('\u0370', "GREEK CAPITAL LETTER HETA");
+ map.put('\u0371', "GREEK SMALL LETTER HETA");
+ map.put('\u0372', "GREEK CAPITAL LETTER ARCHAIC SAMPI");
+ map.put('\u0373', "GREEK SMALL LETTER ARCHAIC SAMPI");
+ map.put('\u0374', "GREEK NUMERAL SIGN");
+ map.put('\u0375', "GREEK LOWER NUMERAL SIGN");
+ map.put('\u0376', "GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA");
+ map.put('\u0377', "GREEK SMALL LETTER PAMPHYLIAN DIGAMMA");
+ map.put('\u037A', "GREEK YPOGEGRAMMENI");
+ map.put('\u037B', "GREEK SMALL REVERSED LUNATE SIGMA SYMBOL");
+ map.put('\u037C', "GREEK SMALL DOTTED LUNATE SIGMA SYMBOL");
+ map.put('\u037D', "GREEK SMALL REVERSED DOTTED LUNATE SIGMA SYMBOL");
+ map.put('\u037E', "GREEK QUESTION MARK");
+ map.put('\u0384', "GREEK TONOS");
+ map.put('\u0385', "GREEK DIALYTIKA TONOS");
+ map.put('\u0386', "GREEK CAPITAL LETTER ALPHA WITH TONOS");
+ map.put('\u0387', "GREEK ANO TELEIA");
+ map.put('\u0388', "GREEK CAPITAL LETTER EPSILON WITH TONOS");
+ map.put('\u0389', "GREEK CAPITAL LETTER ETA WITH TONOS");
+ map.put('\u038A', "GREEK CAPITAL LETTER IOTA WITH TONOS");
+ map.put('\u038C', "GREEK CAPITAL LETTER OMICRON WITH TONOS");
+ map.put('\u038E', "GREEK CAPITAL LETTER UPSILON WITH TONOS");
+ map.put('\u038F', "GREEK CAPITAL LETTER OMEGA WITH TONOS");
+ map.put('\u0390', "GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS");
+ map.put('\u0391', "GREEK CAPITAL LETTER ALPHA");
+ map.put('\u0392', "GREEK CAPITAL LETTER BETA");
+ map.put('\u0393', "GREEK CAPITAL LETTER GAMMA");
+ map.put('\u0394', "GREEK CAPITAL LETTER DELTA");
+ map.put('\u0395', "GREEK CAPITAL LETTER EPSILON");
+ map.put('\u0396', "GREEK CAPITAL LETTER ZETA");
+ map.put('\u0397', "GREEK CAPITAL LETTER ETA");
+ map.put('\u0398', "GREEK CAPITAL LETTER THETA");
+ map.put('\u0399', "GREEK CAPITAL LETTER IOTA");
+ map.put('\u039A', "GREEK CAPITAL LETTER KAPPA");
+ map.put('\u039B', "GREEK CAPITAL LETTER LAMDA");
+ map.put('\u039C', "GREEK CAPITAL LETTER MU");
+ map.put('\u039D', "GREEK CAPITAL LETTER NU");
+ map.put('\u039E', "GREEK CAPITAL LETTER XI");
+ map.put('\u039F', "GREEK CAPITAL LETTER OMICRON");
+ map.put('\u03A0', "GREEK CAPITAL LETTER PI");
+ map.put('\u03A1', "GREEK CAPITAL LETTER RHO");
+ map.put('\u03A3', "GREEK CAPITAL LETTER SIGMA");
+ map.put('\u03A4', "GREEK CAPITAL LETTER TAU");
+ map.put('\u03A5', "GREEK CAPITAL LETTER UPSILON");
+ map.put('\u03A6', "GREEK CAPITAL LETTER PHI");
+ map.put('\u03A7', "GREEK CAPITAL LETTER CHI");
+ map.put('\u03A8', "GREEK CAPITAL LETTER PSI");
+ map.put('\u03A9', "GREEK CAPITAL LETTER OMEGA");
+ map.put('\u03AA', "GREEK CAPITAL LETTER IOTA WITH DIALYTIKA");
+ map.put('\u03AB', "GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA");
+ map.put('\u03AC', "GREEK SMALL LETTER ALPHA WITH TONOS");
+ map.put('\u03AD', "GREEK SMALL LETTER EPSILON WITH TONOS");
+ map.put('\u03AE', "GREEK SMALL LETTER ETA WITH TONOS");
+ map.put('\u03AF', "GREEK SMALL LETTER IOTA WITH TONOS");
+ map.put('\u03B0', "GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS");
+ map.put('\u03B1', "GREEK SMALL LETTER ALPHA");
+ map.put('\u03B2', "GREEK SMALL LETTER BETA");
+ map.put('\u03B3', "GREEK SMALL LETTER GAMMA");
+ map.put('\u03B4', "GREEK SMALL LETTER DELTA");
+ map.put('\u03B5', "GREEK SMALL LETTER EPSILON");
+ map.put('\u03B6', "GREEK SMALL LETTER ZETA");
+ map.put('\u03B7', "GREEK SMALL LETTER ETA");
+ map.put('\u03B8', "GREEK SMALL LETTER THETA");
+ map.put('\u03B9', "GREEK SMALL LETTER IOTA");
+ map.put('\u03BA', "GREEK SMALL LETTER KAPPA");
+ map.put('\u03BB', "GREEK SMALL LETTER LAMDA");
+ map.put('\u03BC', "GREEK SMALL LETTER MU");
+ map.put('\u03BD', "GREEK SMALL LETTER NU");
+ map.put('\u03BE', "GREEK SMALL LETTER XI");
+ map.put('\u03BF', "GREEK SMALL LETTER OMICRON");
+ map.put('\u03C0', "GREEK SMALL LETTER PI");
+ map.put('\u03C1', "GREEK SMALL LETTER RHO");
+ map.put('\u03C2', "GREEK SMALL LETTER FINAL SIGMA");
+ map.put('\u03C3', "GREEK SMALL LETTER SIGMA");
+ map.put('\u03C4', "GREEK SMALL LETTER TAU");
+ map.put('\u03C5', "GREEK SMALL LETTER UPSILON");
+ map.put('\u03C6', "GREEK SMALL LETTER PHI");
+ map.put('\u03C7', "GREEK SMALL LETTER CHI");
+ map.put('\u03C8', "GREEK SMALL LETTER PSI");
+ map.put('\u03C9', "GREEK SMALL LETTER OMEGA");
+ map.put('\u03CA', "GREEK SMALL LETTER IOTA WITH DIALYTIKA");
+ map.put('\u03CB', "GREEK SMALL LETTER UPSILON WITH DIALYTIKA");
+ map.put('\u03CC', "GREEK SMALL LETTER OMICRON WITH TONOS");
+ map.put('\u03CD', "GREEK SMALL LETTER UPSILON WITH TONOS");
+ map.put('\u03CE', "GREEK SMALL LETTER OMEGA WITH TONOS");
+ map.put('\u03CF', "GREEK CAPITAL KAI SYMBOL");
+ map.put('\u03D0', "GREEK BETA SYMBOL");
+ map.put('\u03D1', "GREEK THETA SYMBOL");
+ map.put('\u03D2', "GREEK UPSILON WITH HOOK SYMBOL");
+ map.put('\u03D3', "GREEK UPSILON WITH ACUTE AND HOOK SYMBOL");
+ map.put('\u03D4', "GREEK UPSILON WITH DIAERESIS AND HOOK SYMBOL");
+ map.put('\u03D5', "GREEK PHI SYMBOL");
+ map.put('\u03D6', "GREEK PI SYMBOL");
+ map.put('\u03D7', "GREEK KAI SYMBOL");
+ map.put('\u03D8', "GREEK LETTER ARCHAIC KOPPA");
+ map.put('\u03D9', "GREEK SMALL LETTER ARCHAIC KOPPA");
+ map.put('\u03DA', "GREEK LETTER STIGMA");
+ map.put('\u03DB', "GREEK SMALL LETTER STIGMA");
+ map.put('\u03DC', "GREEK LETTER DIGAMMA");
+ map.put('\u03DD', "GREEK SMALL LETTER DIGAMMA");
+ map.put('\u03DE', "GREEK LETTER KOPPA");
+ map.put('\u03DF', "GREEK SMALL LETTER KOPPA");
+ map.put('\u03E0', "GREEK LETTER SAMPI");
+ map.put('\u03E1', "GREEK SMALL LETTER SAMPI");
+ map.put('\u03E2', "COPTIC CAPITAL LETTER SHEI");
+ map.put('\u03E3', "COPTIC SMALL LETTER SHEI");
+ map.put('\u03E4', "COPTIC CAPITAL LETTER FEI");
+ map.put('\u03E5', "COPTIC SMALL LETTER FEI");
+ map.put('\u03E6', "COPTIC CAPITAL LETTER KHEI");
+ map.put('\u03E7', "COPTIC SMALL LETTER KHEI");
+ map.put('\u03E8', "COPTIC CAPITAL LETTER HORI");
+ map.put('\u03E9', "COPTIC SMALL LETTER HORI");
+ map.put('\u03EA', "COPTIC CAPITAL LETTER GANGIA");
+ map.put('\u03EB', "COPTIC SMALL LETTER GANGIA");
+ map.put('\u03EC', "COPTIC CAPITAL LETTER SHIMA");
+ map.put('\u03ED', "COPTIC SMALL LETTER SHIMA");
+ map.put('\u03EE', "COPTIC CAPITAL LETTER DEI");
+ map.put('\u03EF', "COPTIC SMALL LETTER DEI");
+ map.put('\u03F0', "GREEK KAPPA SYMBOL");
+ map.put('\u03F1', "GREEK RHO SYMBOL");
+ map.put('\u03F2', "GREEK LUNATE SIGMA SYMBOL");
+ map.put('\u03F3', "GREEK LETTER YOT");
+ map.put('\u03F4', "GREEK CAPITAL THETA SYMBOL");
+ map.put('\u03F5', "GREEK LUNATE EPSILON SYMBOL");
+ map.put('\u03F6', "GREEK REVERSED LUNATE EPSILON SYMBOL");
+ map.put('\u03F7', "GREEK CAPITAL LETTER SHO");
+ map.put('\u03F8', "GREEK SMALL LETTER SHO");
+ map.put('\u03F9', "GREEK CAPITAL LUNATE SIGMA SYMBOL");
+ map.put('\u03FA', "GREEK CAPITAL LETTER SAN");
+ map.put('\u03FB', "GREEK SMALL LETTER SAN");
+ map.put('\u03FC', "GREEK RHO WITH STROKE SYMBOL");
+ map.put('\u03FD', "GREEK CAPITAL REVERSED LUNATE SIGMA SYMBOL");
+ map.put('\u03FE', "GREEK CAPITAL DOTTED LUNATE SIGMA SYMBOL");
+ map.put('\u03FF', "GREEK CAPITAL REVERSED DOTTED LUNATE SIGMA SYMBOL");
+
+ }
+
+ public static final void cyrillic(Map<Character, String> map) {
+
+ map.put('\u0400', "CYRILLIC CAPITAL LETTER IE WITH GRAVE");
+ map.put('\u0401', "CYRILLIC CAPITAL LETTER IO");
+ map.put('\u0402', "CYRILLIC CAPITAL LETTER DJE");
+ map.put('\u0403', "CYRILLIC CAPITAL LETTER GJE");
+ map.put('\u0404', "CYRILLIC CAPITAL LETTER UKRAINIAN IE");
+ map.put('\u0405', "CYRILLIC CAPITAL LETTER DZE");
+ map.put('\u0406', "CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I");
+ map.put('\u0407', "CYRILLIC CAPITAL LETTER YI");
+ map.put('\u0408', "CYRILLIC CAPITAL LETTER JE");
+ map.put('\u0409', "CYRILLIC CAPITAL LETTER LJE");
+ map.put('\u040A', "CYRILLIC CAPITAL LETTER NJE");
+ map.put('\u040B', "CYRILLIC CAPITAL LETTER TSHE");
+ map.put('\u040C', "CYRILLIC CAPITAL LETTER KJE");
+ map.put('\u040D', "CYRILLIC CAPITAL LETTER I WITH GRAVE");
+ map.put('\u040E', "CYRILLIC CAPITAL LETTER SHORT U");
+ map.put('\u040F', "CYRILLIC CAPITAL LETTER DZHE");
+ map.put('\u0410', "CYRILLIC CAPITAL LETTER A");
+ map.put('\u0411', "CYRILLIC CAPITAL LETTER BE");
+ map.put('\u0412', "CYRILLIC CAPITAL LETTER VE");
+ map.put('\u0413', "CYRILLIC CAPITAL LETTER GHE");
+ map.put('\u0414', "CYRILLIC CAPITAL LETTER DE");
+ map.put('\u0415', "CYRILLIC CAPITAL LETTER IE");
+ map.put('\u0416', "CYRILLIC CAPITAL LETTER ZHE");
+ map.put('\u0417', "CYRILLIC CAPITAL LETTER ZE");
+ map.put('\u0418', "CYRILLIC CAPITAL LETTER I");
+ map.put('\u0419', "CYRILLIC CAPITAL LETTER SHORT I");
+ map.put('\u041A', "CYRILLIC CAPITAL LETTER KA");
+ map.put('\u041B', "CYRILLIC CAPITAL LETTER EL");
+ map.put('\u041C', "CYRILLIC CAPITAL LETTER EM");
+ map.put('\u041D', "CYRILLIC CAPITAL LETTER EN");
+ map.put('\u041E', "CYRILLIC CAPITAL LETTER O");
+ map.put('\u041F', "CYRILLIC CAPITAL LETTER PE");
+ map.put('\u0420', "CYRILLIC CAPITAL LETTER ER");
+ map.put('\u0421', "CYRILLIC CAPITAL LETTER ES");
+ map.put('\u0422', "CYRILLIC CAPITAL LETTER TE");
+ map.put('\u0423', "CYRILLIC CAPITAL LETTER U");
+ map.put('\u0424', "CYRILLIC CAPITAL LETTER EF");
+ map.put('\u0425', "CYRILLIC CAPITAL LETTER HA");
+ map.put('\u0426', "CYRILLIC CAPITAL LETTER TSE");
+ map.put('\u0427', "CYRILLIC CAPITAL LETTER CHE");
+ map.put('\u0428', "CYRILLIC CAPITAL LETTER SHA");
+ map.put('\u0429', "CYRILLIC CAPITAL LETTER SHCHA");
+ map.put('\u042A', "CYRILLIC CAPITAL LETTER HARD SIGN");
+ map.put('\u042B', "CYRILLIC CAPITAL LETTER YERU");
+ map.put('\u042C', "CYRILLIC CAPITAL LETTER SOFT SIGN");
+ map.put('\u042D', "CYRILLIC CAPITAL LETTER E");
+ map.put('\u042E', "CYRILLIC CAPITAL LETTER YU");
+ map.put('\u042F', "CYRILLIC CAPITAL LETTER YA");
+ map.put('\u0430', "CYRILLIC SMALL LETTER A");
+ map.put('\u0431', "CYRILLIC SMALL LETTER BE");
+ map.put('\u0432', "CYRILLIC SMALL LETTER VE");
+ map.put('\u0433', "CYRILLIC SMALL LETTER GHE");
+ map.put('\u0434', "CYRILLIC SMALL LETTER DE");
+ map.put('\u0435', "CYRILLIC SMALL LETTER IE");
+ map.put('\u0436', "CYRILLIC SMALL LETTER ZHE");
+ map.put('\u0437', "CYRILLIC SMALL LETTER ZE");
+ map.put('\u0438', "CYRILLIC SMALL LETTER I");
+ map.put('\u0439', "CYRILLIC SMALL LETTER SHORT I");
+ map.put('\u043A', "CYRILLIC SMALL LETTER KA");
+ map.put('\u043B', "CYRILLIC SMALL LETTER EL");
+ map.put('\u043C', "CYRILLIC SMALL LETTER EM");
+ map.put('\u043D', "CYRILLIC SMALL LETTER EN");
+ map.put('\u043E', "CYRILLIC SMALL LETTER O");
+ map.put('\u043F', "CYRILLIC SMALL LETTER PE");
+ map.put('\u0440', "CYRILLIC SMALL LETTER ER");
+ map.put('\u0441', "CYRILLIC SMALL LETTER ES");
+ map.put('\u0442', "CYRILLIC SMALL LETTER TE");
+ map.put('\u0443', "CYRILLIC SMALL LETTER U");
+ map.put('\u0444', "CYRILLIC SMALL LETTER EF");
+ map.put('\u0445', "CYRILLIC SMALL LETTER HA");
+ map.put('\u0446', "CYRILLIC SMALL LETTER TSE");
+ map.put('\u0447', "CYRILLIC SMALL LETTER CHE");
+ map.put('\u0448', "CYRILLIC SMALL LETTER SHA");
+ map.put('\u0449', "CYRILLIC SMALL LETTER SHCHA");
+ map.put('\u044A', "CYRILLIC SMALL LETTER HARD SIGN");
+ map.put('\u044B', "CYRILLIC SMALL LETTER YERU");
+ map.put('\u044C', "CYRILLIC SMALL LETTER SOFT SIGN");
+ map.put('\u044D', "CYRILLIC SMALL LETTER E");
+ map.put('\u044E', "CYRILLIC SMALL LETTER YU");
+ map.put('\u044F', "CYRILLIC SMALL LETTER YA");
+ map.put('\u0450', "CYRILLIC SMALL LETTER IE WITH GRAVE");
+ map.put('\u0451', "CYRILLIC SMALL LETTER IO");
+ map.put('\u0452', "CYRILLIC SMALL LETTER DJE");
+ map.put('\u0453', "CYRILLIC SMALL LETTER GJE");
+ map.put('\u0454', "CYRILLIC SMALL LETTER UKRAINIAN IE");
+ map.put('\u0455', "CYRILLIC SMALL LETTER DZE");
+ map.put('\u0456', "CYRILLIC SMALL LETTER BYELORUSSIAN-UKRAINIAN I");
+ map.put('\u0457', "CYRILLIC SMALL LETTER YI");
+ map.put('\u0458', "CYRILLIC SMALL LETTER JE");
+ map.put('\u0459', "CYRILLIC SMALL LETTER LJE");
+ map.put('\u045A', "CYRILLIC SMALL LETTER NJE");
+ map.put('\u045B', "CYRILLIC SMALL LETTER TSHE");
+ map.put('\u045C', "CYRILLIC SMALL LETTER KJE");
+ map.put('\u045D', "CYRILLIC SMALL LETTER I WITH GRAVE");
+ map.put('\u045E', "CYRILLIC SMALL LETTER SHORT U");
+ map.put('\u045F', "CYRILLIC SMALL LETTER DZHE");
+ map.put('\u0460', "CYRILLIC CAPITAL LETTER OMEGA");
+ map.put('\u0461', "CYRILLIC SMALL LETTER OMEGA");
+ map.put('\u0462', "CYRILLIC CAPITAL LETTER YAT");
+ map.put('\u0463', "CYRILLIC SMALL LETTER YAT");
+ map.put('\u0464', "CYRILLIC CAPITAL LETTER IOTIFIED E");
+ map.put('\u0465', "CYRILLIC SMALL LETTER IOTIFIED E");
+ map.put('\u0466', "CYRILLIC CAPITAL LETTER LITTLE YUS");
+ map.put('\u0467', "CYRILLIC SMALL LETTER LITTLE YUS");
+ map.put('\u0468', "CYRILLIC CAPITAL LETTER IOTIFIED LITTLE YUS");
+ map.put('\u0469', "CYRILLIC SMALL LETTER IOTIFIED LITTLE YUS");
+ map.put('\u046A', "CYRILLIC CAPITAL LETTER BIG YUS");
+ map.put('\u046B', "CYRILLIC SMALL LETTER BIG YUS");
+ map.put('\u046C', "CYRILLIC CAPITAL LETTER IOTIFIED BIG YUS");
+ map.put('\u046D', "CYRILLIC SMALL LETTER IOTIFIED BIG YUS");
+ map.put('\u046E', "CYRILLIC CAPITAL LETTER KSI");
+ map.put('\u046F', "CYRILLIC SMALL LETTER KSI");
+ map.put('\u0470', "CYRILLIC CAPITAL LETTER PSI");
+ map.put('\u0471', "CYRILLIC SMALL LETTER PSI");
+ map.put('\u0472', "CYRILLIC CAPITAL LETTER FITA");
+ map.put('\u0473', "CYRILLIC SMALL LETTER FITA");
+ map.put('\u0474', "CYRILLIC CAPITAL LETTER IZHITSA");
+ map.put('\u0475', "CYRILLIC SMALL LETTER IZHITSA");
+ map.put('\u0476', "CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT");
+ map.put('\u0477', "CYRILLIC SMALL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT");
+ map.put('\u0478', "CYRILLIC CAPITAL LETTER UK");
+ map.put('\u0479', "CYRILLIC SMALL LETTER UK");
+ map.put('\u047A', "CYRILLIC CAPITAL LETTER ROUND OMEGA");
+ map.put('\u047B', "CYRILLIC SMALL LETTER ROUND OMEGA");
+ map.put('\u047C', "CYRILLIC CAPITAL LETTER OMEGA WITH TITLO");
+ map.put('\u047D', "CYRILLIC SMALL LETTER OMEGA WITH TITLO");
+ map.put('\u047E', "CYRILLIC CAPITAL LETTER OT");
+ map.put('\u047F', "CYRILLIC SMALL LETTER OT");
+ map.put('\u0480', "CYRILLIC CAPITAL LETTER KOPPA");
+ map.put('\u0481', "CYRILLIC SMALL LETTER KOPPA");
+ map.put('\u0482', "CYRILLIC THOUSANDS SIGN");
+ map.put('\u0483', "COMBINING CYRILLIC TITLO");
+ map.put('\u0484', "COMBINING CYRILLIC PALATALIZATION");
+ map.put('\u0485', "COMBINING CYRILLIC DASIA PNEUMATA");
+ map.put('\u0486', "COMBINING CYRILLIC PSILI PNEUMATA");
+ map.put('\u0487', "COMBINING CYRILLIC POKRYTIE");
+ map.put('\u0488', "COMBINING CYRILLIC HUNDRED THOUSANDS SIGN");
+ map.put('\u0489', "COMBINING CYRILLIC MILLIONS SIGN");
+ map.put('\u048A', "CYRILLIC CAPITAL LETTER SHORT I WITH TAIL");
+ map.put('\u048B', "CYRILLIC SMALL LETTER SHORT I WITH TAIL");
+ map.put('\u048C', "CYRILLIC CAPITAL LETTER SEMISOFT SIGN");
+ map.put('\u048D', "CYRILLIC SMALL LETTER SEMISOFT SIGN");
+ map.put('\u048E', "CYRILLIC CAPITAL LETTER ER WITH TICK");
+ map.put('\u048F', "CYRILLIC SMALL LETTER ER WITH TICK");
+ map.put('\u0490', "CYRILLIC CAPITAL LETTER GHE WITH UPTURN");
+ map.put('\u0491', "CYRILLIC SMALL LETTER GHE WITH UPTURN");
+ map.put('\u0492', "CYRILLIC CAPITAL LETTER GHE WITH STROKE");
+ map.put('\u0493', "CYRILLIC SMALL LETTER GHE WITH STROKE");
+ map.put('\u0494', "CYRILLIC CAPITAL LETTER GHE WITH MIDDLE HOOK");
+ map.put('\u0495', "CYRILLIC SMALL LETTER GHE WITH MIDDLE HOOK");
+ map.put('\u0496', "CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER");
+ map.put('\u0497', "CYRILLIC SMALL LETTER ZHE WITH DESCENDER");
+ map.put('\u0498', "CYRILLIC CAPITAL LETTER ZE WITH DESCENDER");
+ map.put('\u0499', "CYRILLIC SMALL LETTER ZE WITH DESCENDER");
+ map.put('\u049A', "CYRILLIC CAPITAL LETTER KA WITH DESCENDER");
+ map.put('\u049B', "CYRILLIC SMALL LETTER KA WITH DESCENDER");
+ map.put('\u049C', "CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE");
+ map.put('\u049D', "CYRILLIC SMALL LETTER KA WITH VERTICAL STROKE");
+ map.put('\u049E', "CYRILLIC CAPITAL LETTER KA WITH STROKE");
+ map.put('\u049F', "CYRILLIC SMALL LETTER KA WITH STROKE");
+ map.put('\u04A0', "CYRILLIC CAPITAL LETTER BASHKIR KA");
+ map.put('\u04A1', "CYRILLIC SMALL LETTER BASHKIR KA");
+ map.put('\u04A2', "CYRILLIC CAPITAL LETTER EN WITH DESCENDER");
+ map.put('\u04A3', "CYRILLIC SMALL LETTER EN WITH DESCENDER");
+ map.put('\u04A4', "CYRILLIC CAPITAL LIGATURE EN GHE");
+ map.put('\u04A5', "CYRILLIC SMALL LIGATURE EN GHE");
+ map.put('\u04A6', "CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK");
+ map.put('\u04A7', "CYRILLIC SMALL LETTER PE WITH MIDDLE HOOK");
+ map.put('\u04A8', "CYRILLIC CAPITAL LETTER ABKHASIAN HA");
+ map.put('\u04A9', "CYRILLIC SMALL LETTER ABKHASIAN HA");
+ map.put('\u04AA', "CYRILLIC CAPITAL LETTER ES WITH DESCENDER");
+ map.put('\u04AB', "CYRILLIC SMALL LETTER ES WITH DESCENDER");
+ map.put('\u04AC', "CYRILLIC CAPITAL LETTER TE WITH DESCENDER");
+ map.put('\u04AD', "CYRILLIC SMALL LETTER TE WITH DESCENDER");
+ map.put('\u04AE', "CYRILLIC CAPITAL LETTER STRAIGHT U");
+ map.put('\u04AF', "CYRILLIC SMALL LETTER STRAIGHT U");
+ map.put('\u04B0', "CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE");
+ map.put('\u04B1', "CYRILLIC SMALL LETTER STRAIGHT U WITH STROKE");
+ map.put('\u04B2', "CYRILLIC CAPITAL LETTER HA WITH DESCENDER");
+ map.put('\u04B3', "CYRILLIC SMALL LETTER HA WITH DESCENDER");
+ map.put('\u04B4', "CYRILLIC CAPITAL LIGATURE TE TSE");
+ map.put('\u04B5', "CYRILLIC SMALL LIGATURE TE TSE");
+ map.put('\u04B6', "CYRILLIC CAPITAL LETTER CHE WITH DESCENDER");
+ map.put('\u04B7', "CYRILLIC SMALL LETTER CHE WITH DESCENDER");
+ map.put('\u04B8', "CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE");
+ map.put('\u04B9', "CYRILLIC SMALL LETTER CHE WITH VERTICAL STROKE");
+ map.put('\u04BA', "CYRILLIC CAPITAL LETTER SHHA");
+ map.put('\u04BB', "CYRILLIC SMALL LETTER SHHA");
+ map.put('\u04BC', "CYRILLIC CAPITAL LETTER ABKHASIAN CHE");
+ map.put('\u04BD', "CYRILLIC SMALL LETTER ABKHASIAN CHE");
+ map.put('\u04BE', "CYRILLIC CAPITAL LETTER ABKHASIAN CHE WITH DESCENDER");
+ map.put('\u04BF', "CYRILLIC SMALL LETTER ABKHASIAN CHE WITH DESCENDER");
+ map.put('\u04C0', "CYRILLIC LETTER PALOCHKA");
+ map.put('\u04C1', "CYRILLIC CAPITAL LETTER ZHE WITH BREVE");
+ map.put('\u04C2', "CYRILLIC SMALL LETTER ZHE WITH BREVE");
+ map.put('\u04C3', "CYRILLIC CAPITAL LETTER KA WITH HOOK");
+ map.put('\u04C4', "CYRILLIC SMALL LETTER KA WITH HOOK");
+ map.put('\u04C5', "CYRILLIC CAPITAL LETTER EL WITH TAIL");
+ map.put('\u04C6', "CYRILLIC SMALL LETTER EL WITH TAIL");
+ map.put('\u04C7', "CYRILLIC CAPITAL LETTER EN WITH HOOK");
+ map.put('\u04C8', "CYRILLIC SMALL LETTER EN WITH HOOK");
+ map.put('\u04C9', "CYRILLIC CAPITAL LETTER EN WITH TAIL");
+ map.put('\u04CA', "CYRILLIC SMALL LETTER EN WITH TAIL");
+ map.put('\u04CB', "CYRILLIC CAPITAL LETTER KHAKASSIAN CHE");
+ map.put('\u04CC', "CYRILLIC SMALL LETTER KHAKASSIAN CHE");
+ map.put('\u04CD', "CYRILLIC CAPITAL LETTER EM WITH TAIL");
+ map.put('\u04CE', "CYRILLIC SMALL LETTER EM WITH TAIL");
+ map.put('\u04CF', "CYRILLIC SMALL LETTER PALOCHKA");
+ map.put('\u04D0', "CYRILLIC CAPITAL LETTER A WITH BREVE");
+ map.put('\u04D1', "CYRILLIC SMALL LETTER A WITH BREVE");
+ map.put('\u04D2', "CYRILLIC CAPITAL LETTER A WITH DIAERESIS");
+ map.put('\u04D3', "CYRILLIC SMALL LETTER A WITH DIAERESIS");
+ map.put('\u04D4', "CYRILLIC CAPITAL LIGATURE A IE");
+ map.put('\u04D5', "CYRILLIC SMALL LIGATURE A IE");
+ map.put('\u04D6', "CYRILLIC CAPITAL LETTER IE WITH BREVE");
+ map.put('\u04D7', "CYRILLIC SMALL LETTER IE WITH BREVE");
+ map.put('\u04D8', "CYRILLIC CAPITAL LETTER SCHWA");
+ map.put('\u04D9', "CYRILLIC SMALL LETTER SCHWA");
+ map.put('\u04DA', "CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS");
+ map.put('\u04DB', "CYRILLIC SMALL LETTER SCHWA WITH DIAERESIS");
+ map.put('\u04DC', "CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS");
+ map.put('\u04DD', "CYRILLIC SMALL LETTER ZHE WITH DIAERESIS");
+ map.put('\u04DE', "CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS");
+ map.put('\u04DF', "CYRILLIC SMALL LETTER ZE WITH DIAERESIS");
+ map.put('\u04E0', "CYRILLIC CAPITAL LETTER ABKHASIAN DZE");
+ map.put('\u04E1', "CYRILLIC SMALL LETTER ABKHASIAN DZE");
+ map.put('\u04E2', "CYRILLIC CAPITAL LETTER I WITH MACRON");
+ map.put('\u04E3', "CYRILLIC SMALL LETTER I WITH MACRON");
+ map.put('\u04E4', "CYRILLIC CAPITAL LETTER I WITH DIAERESIS");
+ map.put('\u04E5', "CYRILLIC SMALL LETTER I WITH DIAERESIS");
+ map.put('\u04E6', "CYRILLIC CAPITAL LETTER O WITH DIAERESIS");
+ map.put('\u04E7', "CYRILLIC SMALL LETTER O WITH DIAERESIS");
+ map.put('\u04E8', "CYRILLIC CAPITAL LETTER BARRED O");
+ map.put('\u04E9', "CYRILLIC SMALL LETTER BARRED O");
+ map.put('\u04EA', "CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS");
+ map.put('\u04EB', "CYRILLIC SMALL LETTER BARRED O WITH DIAERESIS");
+ map.put('\u04EC', "CYRILLIC CAPITAL LETTER E WITH DIAERESIS");
+ map.put('\u04ED', "CYRILLIC SMALL LETTER E WITH DIAERESIS");
+ map.put('\u04EE', "CYRILLIC CAPITAL LETTER U WITH MACRON");
+ map.put('\u04EF', "CYRILLIC SMALL LETTER U WITH MACRON");
+ map.put('\u04F0', "CYRILLIC CAPITAL LETTER U WITH DIAERESIS");
+ map.put('\u04F1', "CYRILLIC SMALL LETTER U WITH DIAERESIS");
+ map.put('\u04F2', "CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE");
+ map.put('\u04F3', "CYRILLIC SMALL LETTER U WITH DOUBLE ACUTE");
+ map.put('\u04F4', "CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS");
+ map.put('\u04F5', "CYRILLIC SMALL LETTER CHE WITH DIAERESIS");
+ map.put('\u04F6', "CYRILLIC CAPITAL LETTER GHE WITH DESCENDER");
+ map.put('\u04F7', "CYRILLIC SMALL LETTER GHE WITH DESCENDER");
+ map.put('\u04F8', "CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS");
+ map.put('\u04F9', "CYRILLIC SMALL LETTER YERU WITH DIAERESIS");
+ map.put('\u04FA', "CYRILLIC CAPITAL LETTER GHE WITH STROKE AND HOOK");
+ map.put('\u04FB', "CYRILLIC SMALL LETTER GHE WITH STROKE AND HOOK");
+ map.put('\u04FC', "CYRILLIC CAPITAL LETTER HA WITH HOOK");
+ map.put('\u04FD', "CYRILLIC SMALL LETTER HA WITH HOOK");
+ map.put('\u04FE', "CYRILLIC CAPITAL LETTER HA WITH STROKE");
+ map.put('\u04FF', "CYRILLIC SMALL LETTER HA WITH STROKE");
+
+ }
+
+ public static final void cyrillicSupplement(Map<Character, String> map) {
+
+ map.put('\u0500', "CYRILLIC CAPITAL LETTER KOMI DE");
+ map.put('\u0501', "CYRILLIC SMALL LETTER KOMI DE");
+ map.put('\u0502', "CYRILLIC CAPITAL LETTER KOMI DJE");
+ map.put('\u0503', "CYRILLIC SMALL LETTER KOMI DJE");
+ map.put('\u0504', "CYRILLIC CAPITAL LETTER KOMI ZJE");
+ map.put('\u0505', "CYRILLIC SMALL LETTER KOMI ZJE");
+ map.put('\u0506', "CYRILLIC CAPITAL LETTER KOMI DZJE");
+ map.put('\u0507', "CYRILLIC SMALL LETTER KOMI DZJE");
+ map.put('\u0508', "CYRILLIC CAPITAL LETTER KOMI LJE");
+ map.put('\u0509', "CYRILLIC SMALL LETTER KOMI LJE");
+ map.put('\u050A', "CYRILLIC CAPITAL LETTER KOMI NJE");
+ map.put('\u050B', "CYRILLIC SMALL LETTER KOMI NJE");
+ map.put('\u050C', "CYRILLIC CAPITAL LETTER KOMI SJE");
+ map.put('\u050D', "CYRILLIC SMALL LETTER KOMI SJE");
+ map.put('\u050E', "CYRILLIC CAPITAL LETTER KOMI TJE");
+ map.put('\u050F', "CYRILLIC SMALL LETTER KOMI TJE");
+ map.put('\u0510', "CYRILLIC CAPITAL LETTER REVERSED ZE");
+ map.put('\u0511', "CYRILLIC SMALL LETTER REVERSED ZE");
+ map.put('\u0512', "CYRILLIC CAPITAL LETTER EL WITH HOOK");
+ map.put('\u0513', "CYRILLIC SMALL LETTER EL WITH HOOK");
+ map.put('\u0514', "CYRILLIC CAPITAL LETTER LHA");
+ map.put('\u0515', "CYRILLIC SMALL LETTER LHA");
+ map.put('\u0516', "CYRILLIC CAPITAL LETTER RHA");
+ map.put('\u0517', "CYRILLIC SMALL LETTER RHA");
+ map.put('\u0518', "CYRILLIC CAPITAL LETTER YAE");
+ map.put('\u0519', "CYRILLIC SMALL LETTER YAE");
+ map.put('\u051A', "CYRILLIC CAPITAL LETTER QA");
+ map.put('\u051B', "CYRILLIC SMALL LETTER QA");
+ map.put('\u051C', "CYRILLIC CAPITAL LETTER WE");
+ map.put('\u051D', "CYRILLIC SMALL LETTER WE");
+ map.put('\u051E', "CYRILLIC CAPITAL LETTER ALEUT KA");
+ map.put('\u051F', "CYRILLIC SMALL LETTER ALEUT KA");
+ map.put('\u0520', "CYRILLIC CAPITAL LETTER EL WITH MIDDLE HOOK");
+ map.put('\u0521', "CYRILLIC SMALL LETTER EL WITH MIDDLE HOOK");
+ map.put('\u0522', "CYRILLIC CAPITAL LETTER EN WITH MIDDLE HOOK");
+ map.put('\u0523', "CYRILLIC SMALL LETTER EN WITH MIDDLE HOOK");
+ map.put('\u0524', "CYRILLIC CAPITAL LETTER PE WITH DESCENDER");
+ map.put('\u0525', "CYRILLIC SMALL LETTER PE WITH DESCENDER");
+
+ }
+
+ public static final void armenian(Map<Character, String> map) {
+
+ map.put('\u0531', "ARMENIAN CAPITAL LETTER AYB");
+ map.put('\u0532', "ARMENIAN CAPITAL LETTER BEN");
+ map.put('\u0533', "ARMENIAN CAPITAL LETTER GIM");
+ map.put('\u0534', "ARMENIAN CAPITAL LETTER DA");
+ map.put('\u0535', "ARMENIAN CAPITAL LETTER ECH");
+ map.put('\u0536', "ARMENIAN CAPITAL LETTER ZA");
+ map.put('\u0537', "ARMENIAN CAPITAL LETTER EH");
+ map.put('\u0538', "ARMENIAN CAPITAL LETTER ET");
+ map.put('\u0539', "ARMENIAN CAPITAL LETTER TO");
+ map.put('\u053A', "ARMENIAN CAPITAL LETTER ZHE");
+ map.put('\u053B', "ARMENIAN CAPITAL LETTER INI");
+ map.put('\u053C', "ARMENIAN CAPITAL LETTER LIWN");
+ map.put('\u053D', "ARMENIAN CAPITAL LETTER XEH");
+ map.put('\u053E', "ARMENIAN CAPITAL LETTER CA");
+ map.put('\u053F', "ARMENIAN CAPITAL LETTER KEN");
+ map.put('\u0540', "ARMENIAN CAPITAL LETTER HO");
+ map.put('\u0541', "ARMENIAN CAPITAL LETTER JA");
+ map.put('\u0542', "ARMENIAN CAPITAL LETTER GHAD");
+ map.put('\u0543', "ARMENIAN CAPITAL LETTER CHEH");
+ map.put('\u0544', "ARMENIAN CAPITAL LETTER MEN");
+ map.put('\u0545', "ARMENIAN CAPITAL LETTER YI");
+ map.put('\u0546', "ARMENIAN CAPITAL LETTER NOW");
+ map.put('\u0547', "ARMENIAN CAPITAL LETTER SHA");
+ map.put('\u0548', "ARMENIAN CAPITAL LETTER VO");
+ map.put('\u0549', "ARMENIAN CAPITAL LETTER CHA");
+ map.put('\u054A', "ARMENIAN CAPITAL LETTER PEH");
+ map.put('\u054B', "ARMENIAN CAPITAL LETTER JHEH");
+ map.put('\u054C', "ARMENIAN CAPITAL LETTER RA");
+ map.put('\u054D', "ARMENIAN CAPITAL LETTER SEH");
+ map.put('\u054E', "ARMENIAN CAPITAL LETTER VEW");
+ map.put('\u054F', "ARMENIAN CAPITAL LETTER TIWN");
+ map.put('\u0550', "ARMENIAN CAPITAL LETTER REH");
+ map.put('\u0551', "ARMENIAN CAPITAL LETTER CO");
+ map.put('\u0552', "ARMENIAN CAPITAL LETTER YIWN");
+ map.put('\u0553', "ARMENIAN CAPITAL LETTER PIWR");
+ map.put('\u0554', "ARMENIAN CAPITAL LETTER KEH");
+ map.put('\u0555', "ARMENIAN CAPITAL LETTER OH");
+ map.put('\u0556', "ARMENIAN CAPITAL LETTER FEH");
+ map.put('\u0559', "ARMENIAN MODIFIER LETTER LEFT HALF RING");
+ map.put('\u055A', "ARMENIAN APOSTROPHE");
+ map.put('\u055B', "ARMENIAN EMPHASIS MARK");
+ map.put('\u055C', "ARMENIAN EXCLAMATION MARK");
+ map.put('\u055D', "ARMENIAN COMMA");
+ map.put('\u055E', "ARMENIAN QUESTION MARK");
+ map.put('\u055F', "ARMENIAN ABBREVIATION MARK");
+ map.put('\u0561', "ARMENIAN SMALL LETTER AYB");
+ map.put('\u0562', "ARMENIAN SMALL LETTER BEN");
+ map.put('\u0563', "ARMENIAN SMALL LETTER GIM");
+ map.put('\u0564', "ARMENIAN SMALL LETTER DA");
+ map.put('\u0565', "ARMENIAN SMALL LETTER ECH");
+ map.put('\u0566', "ARMENIAN SMALL LETTER ZA");
+ map.put('\u0567', "ARMENIAN SMALL LETTER EH");
+ map.put('\u0568', "ARMENIAN SMALL LETTER ET");
+ map.put('\u0569', "ARMENIAN SMALL LETTER TO");
+ map.put('\u056A', "ARMENIAN SMALL LETTER ZHE");
+ map.put('\u056B', "ARMENIAN SMALL LETTER INI");
+ map.put('\u056C', "ARMENIAN SMALL LETTER LIWN");
+ map.put('\u056D', "ARMENIAN SMALL LETTER XEH");
+ map.put('\u056E', "ARMENIAN SMALL LETTER CA");
+ map.put('\u056F', "ARMENIAN SMALL LETTER KEN");
+ map.put('\u0570', "ARMENIAN SMALL LETTER HO");
+ map.put('\u0571', "ARMENIAN SMALL LETTER JA");
+ map.put('\u0572', "ARMENIAN SMALL LETTER GHAD");
+ map.put('\u0573', "ARMENIAN SMALL LETTER CHEH");
+ map.put('\u0574', "ARMENIAN SMALL LETTER MEN");
+ map.put('\u0575', "ARMENIAN SMALL LETTER YI");
+ map.put('\u0576', "ARMENIAN SMALL LETTER NOW");
+ map.put('\u0577', "ARMENIAN SMALL LETTER SHA");
+ map.put('\u0578', "ARMENIAN SMALL LETTER VO");
+ map.put('\u0579', "ARMENIAN SMALL LETTER CHA");
+ map.put('\u057A', "ARMENIAN SMALL LETTER PEH");
+ map.put('\u057B', "ARMENIAN SMALL LETTER JHEH");
+ map.put('\u057C', "ARMENIAN SMALL LETTER RA");
+ map.put('\u057D', "ARMENIAN SMALL LETTER SEH");
+ map.put('\u057E', "ARMENIAN SMALL LETTER VEW");
+ map.put('\u057F', "ARMENIAN SMALL LETTER TIWN");
+ map.put('\u0580', "ARMENIAN SMALL LETTER REH");
+ map.put('\u0581', "ARMENIAN SMALL LETTER CO");
+ map.put('\u0582', "ARMENIAN SMALL LETTER YIWN");
+ map.put('\u0583', "ARMENIAN SMALL LETTER PIWR");
+ map.put('\u0584', "ARMENIAN SMALL LETTER KEH");
+ map.put('\u0585', "ARMENIAN SMALL LETTER OH");
+ map.put('\u0586', "ARMENIAN SMALL LETTER FEH");
+ map.put('\u0587', "ARMENIAN SMALL LIGATURE ECH YIWN");
+ map.put('\u0589', "ARMENIAN FULL STOP");
+ map.put('\u058A', "ARMENIAN HYPHEN");
+
+ }
+
+ public static final void hebrew(Map<Character, String> map) {
+
+ map.put('\u0591', "HEBREW ACCENT ETNAHTA");
+ map.put('\u0592', "HEBREW ACCENT SEGOL");
+ map.put('\u0593', "HEBREW ACCENT SHALSHELET");
+ map.put('\u0594', "HEBREW ACCENT ZAQEF QATAN");
+ map.put('\u0595', "HEBREW ACCENT ZAQEF GADOL");
+ map.put('\u0596', "HEBREW ACCENT TIPEHA");
+ map.put('\u0597', "HEBREW ACCENT REVIA");
+ map.put('\u0598', "HEBREW ACCENT ZARQA");
+ map.put('\u0599', "HEBREW ACCENT PASHTA");
+ map.put('\u059A', "HEBREW ACCENT YETIV");
+ map.put('\u059B', "HEBREW ACCENT TEVIR");
+ map.put('\u059C', "HEBREW ACCENT GERESH");
+ map.put('\u059D', "HEBREW ACCENT GERESH MUQDAM");
+ map.put('\u059E', "HEBREW ACCENT GERSHAYIM");
+ map.put('\u059F', "HEBREW ACCENT QARNEY PARA");
+ map.put('\u05A0', "HEBREW ACCENT TELISHA GEDOLA");
+ map.put('\u05A1', "HEBREW ACCENT PAZER");
+ map.put('\u05A2', "HEBREW ACCENT ATNAH HAFUKH");
+ map.put('\u05A3', "HEBREW ACCENT MUNAH");
+ map.put('\u05A4', "HEBREW ACCENT MAHAPAKH");
+ map.put('\u05A5', "HEBREW ACCENT MERKHA");
+ map.put('\u05A6', "HEBREW ACCENT MERKHA KEFULA");
+ map.put('\u05A7', "HEBREW ACCENT DARGA");
+ map.put('\u05A8', "HEBREW ACCENT QADMA");
+ map.put('\u05A9', "HEBREW ACCENT TELISHA QETANA");
+ map.put('\u05AA', "HEBREW ACCENT YERAH BEN YOMO");
+ map.put('\u05AB', "HEBREW ACCENT OLE");
+ map.put('\u05AC', "HEBREW ACCENT ILUY");
+ map.put('\u05AD', "HEBREW ACCENT DEHI");
+ map.put('\u05AE', "HEBREW ACCENT ZINOR");
+ map.put('\u05AF', "HEBREW MARK MASORA CIRCLE");
+ map.put('\u05B0', "HEBREW POINT SHEVA");
+ map.put('\u05B1', "HEBREW POINT HATAF SEGOL");
+ map.put('\u05B2', "HEBREW POINT HATAF PATAH");
+ map.put('\u05B3', "HEBREW POINT HATAF QAMATS");
+ map.put('\u05B4', "HEBREW POINT HIRIQ");
+ map.put('\u05B5', "HEBREW POINT TSERE");
+ map.put('\u05B6', "HEBREW POINT SEGOL");
+ map.put('\u05B7', "HEBREW POINT PATAH");
+ map.put('\u05B8', "HEBREW POINT QAMATS");
+ map.put('\u05B9', "HEBREW POINT HOLAM");
+ map.put('\u05BA', "HEBREW POINT HOLAM HASER FOR VAV");
+ map.put('\u05BB', "HEBREW POINT QUBUTS");
+ map.put('\u05BC', "HEBREW POINT DAGESH OR MAPIQ");
+ map.put('\u05BD', "HEBREW POINT METEG");
+ map.put('\u05BE', "HEBREW PUNCTUATION MAQAF");
+ map.put('\u05BF', "HEBREW POINT RAFE");
+ map.put('\u05C0', "HEBREW PUNCTUATION PASEQ");
+ map.put('\u05C1', "HEBREW POINT SHIN DOT");
+ map.put('\u05C2', "HEBREW POINT SIN DOT");
+ map.put('\u05C3', "HEBREW PUNCTUATION SOF PASUQ");
+ map.put('\u05C4', "HEBREW MARK UPPER DOT");
+ map.put('\u05C5', "HEBREW MARK LOWER DOT");
+ map.put('\u05C6', "HEBREW PUNCTUATION NUN HAFUKHA");
+ map.put('\u05C7', "HEBREW POINT QAMATS QATAN");
+ map.put('\u05D0', "HEBREW LETTER ALEF");
+ map.put('\u05D1', "HEBREW LETTER BET");
+ map.put('\u05D2', "HEBREW LETTER GIMEL");
+ map.put('\u05D3', "HEBREW LETTER DALET");
+ map.put('\u05D4', "HEBREW LETTER HE");
+ map.put('\u05D5', "HEBREW LETTER VAV");
+ map.put('\u05D6', "HEBREW LETTER ZAYIN");
+ map.put('\u05D7', "HEBREW LETTER HET");
+ map.put('\u05D8', "HEBREW LETTER TET");
+ map.put('\u05D9', "HEBREW LETTER YOD");
+ map.put('\u05DA', "HEBREW LETTER FINAL KAF");
+ map.put('\u05DB', "HEBREW LETTER KAF");
+ map.put('\u05DC', "HEBREW LETTER LAMED");
+ map.put('\u05DD', "HEBREW LETTER FINAL MEM");
+ map.put('\u05DE', "HEBREW LETTER MEM");
+ map.put('\u05DF', "HEBREW LETTER FINAL NUN");
+ map.put('\u05E0', "HEBREW LETTER NUN");
+ map.put('\u05E1', "HEBREW LETTER SAMEKH");
+ map.put('\u05E2', "HEBREW LETTER AYIN");
+ map.put('\u05E3', "HEBREW LETTER FINAL PE");
+ map.put('\u05E4', "HEBREW LETTER PE");
+ map.put('\u05E5', "HEBREW LETTER FINAL TSADI");
+ map.put('\u05E6', "HEBREW LETTER TSADI");
+ map.put('\u05E7', "HEBREW LETTER QOF");
+ map.put('\u05E8', "HEBREW LETTER RESH");
+ map.put('\u05E9', "HEBREW LETTER SHIN");
+ map.put('\u05EA', "HEBREW LETTER TAV");
+ map.put('\u05F0', "HEBREW LIGATURE YIDDISH DOUBLE VAV");
+ map.put('\u05F1', "HEBREW LIGATURE YIDDISH VAV YOD");
+ map.put('\u05F2', "HEBREW LIGATURE YIDDISH DOUBLE YOD");
+ map.put('\u05F3', "HEBREW PUNCTUATION GERESH");
+ map.put('\u05F4', "HEBREW PUNCTUATION GERSHAYIM");
+
+ }
+
+ public static final void arabic(Map<Character, String> map) {
+
+ map.put('\u0600', "ARABIC NUMBER S
<TRUNCATED>
[29/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/mira/Optimizer.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/mira/Optimizer.java b/joshua-core/src/main/java/org/apache/joshua/mira/Optimizer.java
new file mode 100755
index 0000000..6eaced4
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/mira/Optimizer.java
@@ -0,0 +1,643 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.mira;
+
+import java.util.Collection;
+import java.util.Collections;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.Vector;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.metrics.EvaluationMetric;
+
+// this class implements the MIRA algorithm
+public class Optimizer {
+ public Optimizer(Vector<String> _output, boolean[] _isOptimizable, double[] _initialLambda,
+ HashMap<String, String>[] _feat_hash, HashMap<String, String>[] _stats_hash) {
+ output = _output; // (not used for now)
+ isOptimizable = _isOptimizable;
+ initialLambda = _initialLambda; // initial weights array
+ paramDim = initialLambda.length - 1;
+ initialLambda = _initialLambda;
+ feat_hash = _feat_hash; // feature hash table
+ stats_hash = _stats_hash; // suff. stats hash table
+ finalLambda = new double[initialLambda.length];
+ for (int i = 0; i < finalLambda.length; i++)
+ finalLambda[i] = initialLambda[i];
+ }
+
+ // run MIRA for one epoch
+ public double[] runOptimizer() {
+ List<Integer> sents = new ArrayList<Integer>();
+ for (int i = 0; i < sentNum; ++i)
+ sents.add(i);
+ double[] avgLambda = new double[initialLambda.length]; // only needed if averaging is required
+ for (int i = 0; i < initialLambda.length; i++)
+ avgLambda[i] = 0.0;
+ double[] bestLambda = new double[initialLambda.length]; // only needed if averaging is required
+ for (int i = 0; i < initialLambda.length; i++)
+ bestLambda[i] = 0.0;
+ double bestMetricScore = evalMetric.getToBeMinimized() ? PosInf : NegInf;
+ int bestIter = 0;
+ for (int iter = 0; iter < miraIter; ++iter) {
+ System.arraycopy(finalLambda, 1, initialLambda, 1, paramDim);
+ if (needShuffle)
+ Collections.shuffle(sents);
+
+ double oraMetric, oraScore, predMetric, predScore;
+ double[] oraPredScore = new double[4];
+ double eta = 1.0; // learning rate, will not be changed if run percep
+ double avgEta = 0; // average eta, just for analysis
+ double loss = 0;
+ double diff = 0;
+ double featNorm = 0;
+ double sumMetricScore = 0;
+ double sumModelScore = 0;
+ String oraFeat = "";
+ String predFeat = "";
+ String[] oraPredFeat = new String[2];
+ String[] vecOraFeat;
+ String[] vecPredFeat;
+ String[] featInfo;
+ int thisBatchSize = 0;
+ int numBatch = 0;
+ int numUpdate = 0;
+ Iterator it;
+ Integer diffFeatId;
+
+ // update weights
+ Integer s;
+ int sentCount = 0;
+ while( sentCount < sentNum ) {
+ loss = 0;
+ thisBatchSize = batchSize;
+ ++numBatch;
+ HashMap<Integer, Double> featDiff = new HashMap<Integer, Double>();
+ for(int b = 0; b < batchSize; ++b ) {
+ //find out oracle and prediction
+ s = sents.get(sentCount);
+ // find out oracle and prediction
+ findOraPred(s, oraPredScore, oraPredFeat, finalLambda, featScale);
+
+ // the model scores here are already scaled in findOraPred
+ oraMetric = oraPredScore[0];
+ oraScore = oraPredScore[1];
+ predMetric = oraPredScore[2];
+ predScore = oraPredScore[3];
+ oraFeat = oraPredFeat[0];
+ predFeat = oraPredFeat[1];
+
+ // update the scale
+ if (needScale) { // otherwise featscale remains 1.0
+ sumMetricScore += java.lang.Math.abs(oraMetric + predMetric);
+ // restore the original model score
+ sumModelScore += java.lang.Math.abs(oraScore + predScore) / featScale;
+
+ if (sumModelScore / sumMetricScore > scoreRatio)
+ featScale = sumMetricScore / sumModelScore;
+ }
+
+ vecOraFeat = oraFeat.split("\\s+");
+ vecPredFeat = predFeat.split("\\s+");
+
+ //accumulate difference feature vector
+ if ( b == 0 ) {
+ for (int i = 0; i < vecOraFeat.length; i++) {
+ featInfo = vecOraFeat[i].split("=");
+ diffFeatId = Integer.parseInt(featInfo[0]);
+ featDiff.put(diffFeatId, Double.parseDouble(featInfo[1]));
+ }
+ for (int i = 0; i < vecPredFeat.length; i++) {
+ featInfo = vecPredFeat[i].split("=");
+ diffFeatId = Integer.parseInt(featInfo[0]);
+ if (featDiff.containsKey(diffFeatId)) { //overlapping features
+ diff = featDiff.get(diffFeatId)-Double.parseDouble(featInfo[1]);
+ if ( Math.abs(diff) > 1e-20 )
+ featDiff.put(diffFeatId, diff);
+ else
+ featDiff.remove(diffFeatId);
+ }
+ else //features only firing in the 2nd feature vector
+ featDiff.put(diffFeatId, -1.0*Double.parseDouble(featInfo[1]));
+ }
+ } else {
+ for (int i = 0; i < vecOraFeat.length; i++) {
+ featInfo = vecOraFeat[i].split("=");
+ diffFeatId = Integer.parseInt(featInfo[0]);
+ if (featDiff.containsKey(diffFeatId)) { //overlapping features
+ diff = featDiff.get(diffFeatId)+Double.parseDouble(featInfo[1]);
+ if ( Math.abs(diff) > 1e-20 )
+ featDiff.put(diffFeatId, diff);
+ else
+ featDiff.remove(diffFeatId);
+ }
+ else //features only firing in the new oracle feature vector
+ featDiff.put(diffFeatId, Double.parseDouble(featInfo[1]));
+ }
+ for (int i = 0; i < vecPredFeat.length; i++) {
+ featInfo = vecPredFeat[i].split("=");
+ diffFeatId = Integer.parseInt(featInfo[0]);
+ if (featDiff.containsKey(diffFeatId)) { //overlapping features
+ diff = featDiff.get(diffFeatId)-Double.parseDouble(featInfo[1]);
+ if ( Math.abs(diff) > 1e-20 )
+ featDiff.put(diffFeatId, diff);
+ else
+ featDiff.remove(diffFeatId);
+ }
+ else //features only firing in the new prediction feature vector
+ featDiff.put(diffFeatId, -1.0*Double.parseDouble(featInfo[1]));
+ }
+ }
+ if (!runPercep) { // otherwise eta=1.0
+ // remember the model scores here are already scaled
+ double singleLoss = evalMetric.getToBeMinimized() ?
+ (predMetric - oraMetric) - (oraScore - predScore) / featScale
+ : (oraMetric - predMetric) - (oraScore - predScore) / featScale;
+ loss += singleLoss;
+ }
+ ++sentCount;
+ if( sentCount >= sentNum ) {
+ thisBatchSize = b + 1;
+ break;
+ }
+ } //for(int b = 0; b < batchSize; ++b)
+
+ if (!runPercep) { // otherwise eta=1.0
+ featNorm = 0;
+ Collection<Double> allDiff = featDiff.values();
+ for (it = allDiff.iterator(); it.hasNext();) {
+ diff = (Double) it.next();
+ featNorm += diff * diff / ( thisBatchSize * thisBatchSize );
+ }
+ }
+ if( loss <= 0 )
+ eta = 0;
+ else {
+ loss /= thisBatchSize;
+ // feat vector not scaled before
+ eta = C < loss / featNorm ? C : loss / featNorm;
+ }
+ avgEta += eta;
+ Set<Integer> diffFeatSet = featDiff.keySet();
+ it = diffFeatSet.iterator();
+ if ( java.lang.Math.abs(eta) > 1e-20 ) {
+ while (it.hasNext()) {
+ diffFeatId = (Integer) it.next();
+ finalLambda[diffFeatId] =
+ finalLambda[diffFeatId] + eta * featDiff.get(diffFeatId) / thisBatchSize;
+ }
+ }
+ if (needAvg) {
+ for (int i = 0; i < avgLambda.length; ++i)
+ avgLambda[i] += finalLambda[i];
+ }
+ } //while( sentCount < sentNum )
+
+ avgEta /= numBatch;
+
+ /*
+ * for( int i=0; i<finalLambda.length; i++ ) System.out.print(finalLambda[i]+" ");
+ * System.out.println(); System.exit(0);
+ */
+
+ double initMetricScore;
+ if(iter == 0 ) {
+ initMetricScore = computeCorpusMetricScore(initialLambda);
+ if(needAvg)
+ finalMetricScore = computeCorpusMetricScore(avgLambda);
+ else
+ finalMetricScore = computeCorpusMetricScore(finalLambda);
+ } else {
+ initMetricScore = finalMetricScore;
+ if(needAvg)
+ finalMetricScore = computeCorpusMetricScore(avgLambda);
+ else
+ finalMetricScore = computeCorpusMetricScore(finalLambda);
+ }
+
+ if(evalMetric.getToBeMinimized()) {
+ if( finalMetricScore < bestMetricScore ) {
+ bestMetricScore = finalMetricScore;
+ bestIter = iter;
+ for( int i = 0; i < finalLambda.length; ++i )
+ bestLambda[i] = needAvg ? avgLambda[i] : finalLambda[i];
+ }
+ } else {
+ if( finalMetricScore > bestMetricScore ) {
+ bestMetricScore = finalMetricScore;
+ bestIter = iter;
+ for( int i = 0; i < finalLambda.length; ++i )
+ bestLambda[i] = needAvg ? avgLambda[i] : finalLambda[i];
+ }
+ }
+
+ if ( iter == miraIter - 1 ) {
+ for (int i = 0; i < finalLambda.length; ++i)
+ finalLambda[i] =
+ needAvg ? bestLambda[i] / ( numBatch * ( bestIter + 1 ) ) : bestLambda[i];
+ }
+
+ // prepare the printing info
+ String result = "Iter " + iter + ": Avg learning rate=" + String.format("%.4f", avgEta);
+ result += " Initial " + evalMetric.get_metricName() + "="
+ + String.format("%.4f", initMetricScore) + " Final " + evalMetric.get_metricName() + "="
+ + String.format("%.4f", finalMetricScore);
+ output.add(result);
+ } // for ( int iter = 0; iter < miraIter; ++iter )
+ String result = "Best " + evalMetric.get_metricName() + "="
+ + String.format("%.4f", bestMetricScore)
+ + " (iter = " + bestIter + ")\n";
+ output.add(result);
+ finalMetricScore = bestMetricScore;
+
+ // non-optimizable weights should remain unchanged
+ ArrayList<Double> cpFixWt = new ArrayList<Double>();
+ for (int i = 1; i < isOptimizable.length; ++i) {
+ if (!isOptimizable[i])
+ cpFixWt.add(finalLambda[i]);
+ }
+ normalizeLambda(finalLambda);
+ int countNonOpt = 0;
+ for (int i = 1; i < isOptimizable.length; ++i) {
+ if (!isOptimizable[i]) {
+ finalLambda[i] = cpFixWt.get(countNonOpt);
+ ++countNonOpt;
+ }
+ }
+ return finalLambda;
+ }
+
+ public double computeCorpusMetricScore(double[] finalLambda) {
+ int suffStatsCount = evalMetric.get_suffStatsCount();
+ double modelScore;
+ double maxModelScore;
+ Set<String> candSet;
+ String candStr;
+ String[] feat_str;
+ String[] tmpStatsVal = new String[suffStatsCount];
+ int[] corpusStatsVal = new int[suffStatsCount];
+ for (int i = 0; i < suffStatsCount; i++)
+ corpusStatsVal[i] = 0;
+
+ for (int i = 0; i < sentNum; i++) {
+ candSet = feat_hash[i].keySet();
+ // find out the 1-best candidate for each sentence
+ // this depends on the training mode
+ maxModelScore = NegInf;
+ for (Iterator it = candSet.iterator(); it.hasNext();) {
+ modelScore = 0.0;
+ candStr = it.next().toString();
+ feat_str = feat_hash[i].get(candStr).split("\\s+");
+ String[] feat_info;
+ for (int f = 0; f < feat_str.length; f++) {
+ feat_info = feat_str[f].split("=");
+ modelScore += Double.parseDouble(feat_info[1]) * finalLambda[Vocabulary.id(feat_info[0])];
+ }
+ if (maxModelScore < modelScore) {
+ maxModelScore = modelScore;
+ tmpStatsVal = stats_hash[i].get(candStr).split("\\s+"); // save the
+ // suff stats
+ }
+ }
+
+ for (int j = 0; j < suffStatsCount; j++)
+ corpusStatsVal[j] += Integer.parseInt(tmpStatsVal[j]); // accumulate
+ // corpus-leve
+ // suff stats
+ } // for( int i=0; i<sentNum; i++ )
+
+ return evalMetric.score(corpusStatsVal);
+ }
+
+ private void findOraPred(int sentId, double[] oraPredScore, String[] oraPredFeat,
+ double[] lambda, double featScale) {
+ double oraMetric = 0, oraScore = 0, predMetric = 0, predScore = 0;
+ String oraFeat = "", predFeat = "";
+ double candMetric = 0, candScore = 0; // metric and model scores for each cand
+ Set<String> candSet = stats_hash[sentId].keySet();
+ String cand = "";
+ String feats = "";
+ String oraCand = ""; // only used when BLEU/TER-BLEU is used as metric
+ String[] featStr;
+ String[] featInfo;
+
+ int actualFeatId;
+ double bestOraScore;
+ double worstPredScore;
+
+ if (oraSelectMode == 1)
+ bestOraScore = NegInf; // larger score will be selected
+ else {
+ if (evalMetric.getToBeMinimized())
+ bestOraScore = PosInf; // smaller score will be selected
+ else
+ bestOraScore = NegInf;
+ }
+
+ if (predSelectMode == 1 || predSelectMode == 2)
+ worstPredScore = NegInf; // larger score will be selected
+ else {
+ if (evalMetric.getToBeMinimized())
+ worstPredScore = NegInf; // larger score will be selected
+ else
+ worstPredScore = PosInf;
+ }
+
+ for (Iterator it = candSet.iterator(); it.hasNext();) {
+ cand = it.next().toString();
+ candMetric = computeSentMetric(sentId, cand); // compute metric score
+
+ // start to compute model score
+ candScore = 0;
+ featStr = feat_hash[sentId].get(cand).split("\\s+");
+ feats = "";
+
+ for (int i = 0; i < featStr.length; i++) {
+ featInfo = featStr[i].split("=");
+ actualFeatId = Vocabulary.id(featInfo[0]);
+ candScore += Double.parseDouble(featInfo[1]) * lambda[actualFeatId];
+ if ((actualFeatId < isOptimizable.length && isOptimizable[actualFeatId])
+ || actualFeatId >= isOptimizable.length)
+ feats += actualFeatId + "=" + Double.parseDouble(featInfo[1]) + " ";
+ }
+
+ candScore *= featScale; // scale the model score
+
+ // is this cand oracle?
+ if (oraSelectMode == 1) {// "hope", b=1, r=1
+ if (evalMetric.getToBeMinimized()) {// if the smaller the metric score, the better
+ if (bestOraScore <= (candScore - candMetric)) {
+ bestOraScore = candScore - candMetric;
+ oraMetric = candMetric;
+ oraScore = candScore;
+ oraFeat = feats;
+ oraCand = cand;
+ }
+ } else {
+ if (bestOraScore <= (candScore + candMetric)) {
+ bestOraScore = candScore + candMetric;
+ oraMetric = candMetric;
+ oraScore = candScore;
+ oraFeat = feats;
+ oraCand = cand;
+ }
+ }
+ } else {// best metric score(ex: max BLEU), b=1, r=0
+ if (evalMetric.getToBeMinimized()) {// if the smaller the metric score, the better
+ if (bestOraScore >= candMetric) {
+ bestOraScore = candMetric;
+ oraMetric = candMetric;
+ oraScore = candScore;
+ oraFeat = feats;
+ oraCand = cand;
+ }
+ } else {
+ if (bestOraScore <= candMetric) {
+ bestOraScore = candMetric;
+ oraMetric = candMetric;
+ oraScore = candScore;
+ oraFeat = feats;
+ oraCand = cand;
+ }
+ }
+ }
+
+ // is this cand prediction?
+ if (predSelectMode == 1) {// "fear"
+ if (evalMetric.getToBeMinimized()) {// if the smaller the metric score, the better
+ if (worstPredScore <= (candScore + candMetric)) {
+ worstPredScore = candScore + candMetric;
+ predMetric = candMetric;
+ predScore = candScore;
+ predFeat = feats;
+ }
+ } else {
+ if (worstPredScore <= (candScore - candMetric)) {
+ worstPredScore = candScore - candMetric;
+ predMetric = candMetric;
+ predScore = candScore;
+ predFeat = feats;
+ }
+ }
+ } else if (predSelectMode == 2) {// model prediction(max model score)
+ if (worstPredScore <= candScore) {
+ worstPredScore = candScore;
+ predMetric = candMetric;
+ predScore = candScore;
+ predFeat = feats;
+ }
+ } else {// worst metric score(ex: min BLEU)
+ if (evalMetric.getToBeMinimized()) {// if the smaller the metric score, the better
+ if (worstPredScore <= candMetric) {
+ worstPredScore = candMetric;
+ predMetric = candMetric;
+ predScore = candScore;
+ predFeat = feats;
+ }
+ } else {
+ if (worstPredScore >= candMetric) {
+ worstPredScore = candMetric;
+ predMetric = candMetric;
+ predScore = candScore;
+ predFeat = feats;
+ }
+ }
+ }
+ }
+
+ oraPredScore[0] = oraMetric;
+ oraPredScore[1] = oraScore;
+ oraPredScore[2] = predMetric;
+ oraPredScore[3] = predScore;
+ oraPredFeat[0] = oraFeat;
+ oraPredFeat[1] = predFeat;
+
+ // update the BLEU metric statistics if pseudo corpus is used to compute BLEU/TER-BLEU
+ if (evalMetric.get_metricName().equals("BLEU") && usePseudoBleu) {
+ String statString;
+ String[] statVal_str;
+ statString = stats_hash[sentId].get(oraCand);
+ statVal_str = statString.split("\\s+");
+
+ for (int j = 0; j < evalMetric.get_suffStatsCount(); j++)
+ bleuHistory[sentId][j] = R * bleuHistory[sentId][j] + Integer.parseInt(statVal_str[j]);
+ }
+
+ if (evalMetric.get_metricName().equals("TER-BLEU") && usePseudoBleu) {
+ String statString;
+ String[] statVal_str;
+ statString = stats_hash[sentId].get(oraCand);
+ statVal_str = statString.split("\\s+");
+
+ for (int j = 0; j < evalMetric.get_suffStatsCount() - 2; j++)
+ bleuHistory[sentId][j] = R * bleuHistory[sentId][j] + Integer.parseInt(statVal_str[j + 2]); // the
+ // first
+ // 2
+ // stats
+ // are
+ // TER
+ // stats
+ }
+ }
+
+ // compute *sentence-level* metric score for cand
+ private double computeSentMetric(int sentId, String cand) {
+ String statString;
+ String[] statVal_str;
+ int[] statVal = new int[evalMetric.get_suffStatsCount()];
+
+ statString = stats_hash[sentId].get(cand);
+ statVal_str = statString.split("\\s+");
+
+ if (evalMetric.get_metricName().equals("BLEU") && usePseudoBleu) {
+ for (int j = 0; j < evalMetric.get_suffStatsCount(); j++)
+ statVal[j] = (int) (Integer.parseInt(statVal_str[j]) + bleuHistory[sentId][j]);
+ } else if (evalMetric.get_metricName().equals("TER-BLEU") && usePseudoBleu) {
+ for (int j = 0; j < evalMetric.get_suffStatsCount() - 2; j++)
+ statVal[j + 2] = (int) (Integer.parseInt(statVal_str[j + 2]) + bleuHistory[sentId][j]); // only
+ // modify
+ // the
+ // BLEU
+ // stats
+ // part(TER
+ // has
+ // 2
+ // stats)
+ } else { // in all other situations, use normal stats
+ for (int j = 0; j < evalMetric.get_suffStatsCount(); j++)
+ statVal[j] = Integer.parseInt(statVal_str[j]);
+ }
+
+ return evalMetric.score(statVal);
+ }
+
+ // from ZMERT
+ private void normalizeLambda(double[] origLambda) {
+ // private String[] normalizationOptions;
+ // How should a lambda[] vector be normalized (before decoding)?
+ // nO[0] = 0: no normalization
+ // nO[0] = 1: scale so that parameter nO[2] has absolute value nO[1]
+ // nO[0] = 2: scale so that the maximum absolute value is nO[1]
+ // nO[0] = 3: scale so that the minimum absolute value is nO[1]
+ // nO[0] = 4: scale so that the L-nO[1] norm equals nO[2]
+
+ int normalizationMethod = (int) normalizationOptions[0];
+ double scalingFactor = 1.0;
+ if (normalizationMethod == 0) {
+ scalingFactor = 1.0;
+ } else if (normalizationMethod == 1) {
+ int c = (int) normalizationOptions[2];
+ scalingFactor = normalizationOptions[1] / Math.abs(origLambda[c]);
+ } else if (normalizationMethod == 2) {
+ double maxAbsVal = -1;
+ int maxAbsVal_c = 0;
+ for (int c = 1; c <= paramDim; ++c) {
+ if (Math.abs(origLambda[c]) > maxAbsVal) {
+ maxAbsVal = Math.abs(origLambda[c]);
+ maxAbsVal_c = c;
+ }
+ }
+ scalingFactor = normalizationOptions[1] / Math.abs(origLambda[maxAbsVal_c]);
+
+ } else if (normalizationMethod == 3) {
+ double minAbsVal = PosInf;
+ int minAbsVal_c = 0;
+
+ for (int c = 1; c <= paramDim; ++c) {
+ if (Math.abs(origLambda[c]) < minAbsVal) {
+ minAbsVal = Math.abs(origLambda[c]);
+ minAbsVal_c = c;
+ }
+ }
+ scalingFactor = normalizationOptions[1] / Math.abs(origLambda[minAbsVal_c]);
+
+ } else if (normalizationMethod == 4) {
+ double pow = normalizationOptions[1];
+ double norm = L_norm(origLambda, pow);
+ scalingFactor = normalizationOptions[2] / norm;
+ }
+
+ for (int c = 1; c <= paramDim; ++c) {
+ origLambda[c] *= scalingFactor;
+ }
+ }
+
+ // from ZMERT
+ private double L_norm(double[] A, double pow) {
+ // calculates the L-pow norm of A[]
+ // NOTE: this calculation ignores A[0]
+ double sum = 0.0;
+ for (int i = 1; i < A.length; ++i)
+ sum += Math.pow(Math.abs(A[i]), pow);
+
+ return Math.pow(sum, 1 / pow);
+ }
+
+ public static double getScale() {
+ return featScale;
+ }
+
+ public static void initBleuHistory(int sentNum, int statCount) {
+ bleuHistory = new double[sentNum][statCount];
+ for (int i = 0; i < sentNum; i++) {
+ for (int j = 0; j < statCount; j++) {
+ bleuHistory[i][j] = 0.0;
+ }
+ }
+ }
+
+ public double getMetricScore() {
+ return finalMetricScore;
+ }
+
+ private Vector<String> output;
+ private double[] initialLambda;
+ private double[] finalLambda;
+ private double finalMetricScore;
+ private HashMap<String, String>[] feat_hash;
+ private HashMap<String, String>[] stats_hash;
+ private int paramDim;
+ private boolean[] isOptimizable;
+ public static int sentNum;
+ public static int miraIter; // MIRA internal iterations
+ public static int oraSelectMode;
+ public static int predSelectMode;
+ public static int batchSize;
+ public static boolean needShuffle;
+ public static boolean needScale;
+ public static double scoreRatio;
+ public static boolean runPercep;
+ public static boolean needAvg;
+ public static boolean usePseudoBleu;
+ public static double featScale = 1.0; // scale the features in order to make the model score
+ // comparable with metric score
+ // updates in each epoch if necessary
+ public static double C; // relaxation coefficient
+ public static double R; // corpus decay(used only when pseudo corpus is used to compute BLEU)
+ public static EvaluationMetric evalMetric;
+ public static double[] normalizationOptions;
+ public static double[][] bleuHistory;
+
+ private final static double NegInf = (-1.0 / 0.0);
+ private final static double PosInf = (+1.0 / 0.0);
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/oracle/OracleExtractionHG.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/oracle/OracleExtractionHG.java b/joshua-core/src/main/java/org/apache/joshua/oracle/OracleExtractionHG.java
new file mode 100644
index 0000000..575515a
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/oracle/OracleExtractionHG.java
@@ -0,0 +1,797 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.oracle;
+
+import static org.apache.joshua.decoder.hypergraph.ViterbiExtractor.getViterbiString;
+import static org.apache.joshua.util.FormatUtils.removeSentenceMarkers;
+
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.Support;
+import org.apache.joshua.decoder.Decoder;
+import org.apache.joshua.decoder.hypergraph.HGNode;
+import org.apache.joshua.decoder.hypergraph.HyperEdge;
+import org.apache.joshua.decoder.hypergraph.HyperGraph;
+import org.apache.joshua.decoder.hypergraph.KBestExtractor;
+import org.apache.joshua.util.FileUtility;
+import org.apache.joshua.util.io.LineReader;
+import org.apache.joshua.util.FormatUtils;
+
+/**
+ * approximated BLEU (1) do not consider clipping effect (2) in the dynamic programming, do not
+ * maintain different states for different hyp length (3) brief penalty is calculated based on the
+ * avg ref length (4) using sentence-level BLEU, instead of doc-level BLEU
+ *
+ * @author Zhifei Li, zhifei.work@gmail.com (Johns Hopkins University)
+ */
+public class OracleExtractionHG extends SplitHg {
+ static String BACKOFF_LEFT_LM_STATE_SYM = "<lzfbo>";
+ public int BACKOFF_LEFT_LM_STATE_SYM_ID;// used for equivelant state
+
+ static String NULL_LEFT_LM_STATE_SYM = "<lzflnull>";
+ public int NULL_LEFT_LM_STATE_SYM_ID;// used for equivelant state
+
+ static String NULL_RIGHT_LM_STATE_SYM = "<lzfrnull>";
+ public int NULL_RIGHT_LM_STATE_SYM_ID;// used for equivelant state
+
+ // int[] ref_sentence;//reference string (not tree)
+ protected int src_sent_len = 0;
+ protected int ref_sent_len = 0;
+ protected int g_lm_order = 4; // only used for decide whether to get the LM state by this class or
+ // not in compute_state
+ static protected boolean do_local_ngram_clip = false;
+ static protected boolean maitain_length_state = false;
+ static protected int g_bleu_order = 4;
+
+ static boolean using_left_equiv_state = true;
+ static boolean using_right_equiv_state = true;
+
+ // TODO Add generics to hash tables in this class
+ HashMap<String, Boolean> tbl_suffix = new HashMap<String, Boolean>();
+ HashMap<String, Boolean> tbl_prefix = new HashMap<String, Boolean>();
+ static PrefixGrammar grammar_prefix = new PrefixGrammar();// TODO
+ static PrefixGrammar grammar_suffix = new PrefixGrammar();// TODO
+
+ // key: item; value: best_deduction, best_bleu, best_len, # of n-gram match where n is in [1,4]
+ protected HashMap<String, Integer> tbl_ref_ngrams = new HashMap<String, Integer>();
+
+ static boolean always_maintain_seperate_lm_state = true; // if true: the virtual item maintain its
+ // own lm state regardless whether
+ // lm_order>=g_bleu_order
+
+ int lm_feat_id = 0; // the baseline LM feature id
+
+ /**
+ * Constructs a new object capable of extracting a tree from a hypergraph that most closely
+ * matches a provided oracle sentence.
+ * <p>
+ * It seems that the symbol table here should only need to represent monolingual terminals, plus
+ * nonterminals.
+ *
+ * @param lm_feat_id_ a language model feature identifier
+ */
+ public OracleExtractionHG(int lm_feat_id_) {
+ this.lm_feat_id = lm_feat_id_;
+ this.BACKOFF_LEFT_LM_STATE_SYM_ID = Vocabulary.id(BACKOFF_LEFT_LM_STATE_SYM);
+ this.NULL_LEFT_LM_STATE_SYM_ID = Vocabulary.id(NULL_RIGHT_LM_STATE_SYM);
+ this.NULL_RIGHT_LM_STATE_SYM_ID = Vocabulary.id(NULL_RIGHT_LM_STATE_SYM);
+ }
+
+ /*
+ * for 919 sent, time_on_reading: 148797 time_on_orc_extract: 580286
+ */
+ @SuppressWarnings({ "unused" })
+ public static void main(String[] args) throws IOException {
+ JoshuaConfiguration joshuaConfiguration = new JoshuaConfiguration();
+ /*
+ * String f_hypergraphs="C:\\Users\\zli\\Documents\\mt03.src.txt.ss.nbest.hg.items"; String
+ * f_rule_tbl="C:\\Users\\zli\\Documents\\mt03.src.txt.ss.nbest.hg.rules"; String
+ * f_ref_files="C:\\Users\\zli\\Documents\\mt03.ref.txt.1"; String f_orc_out
+ * ="C:\\Users\\zli\\Documents\\mt03.orc.txt";
+ */
+ if (6 != args.length) {
+ System.out
+ .println("Usage: java Decoder f_hypergraphs f_rule_tbl f_ref_files f_orc_out lm_order orc_extract_nbest");
+ System.out.println("num of args is " + args.length);
+ for (int i = 0; i < args.length; i++) {
+ System.out.println("arg is: " + args[i]);
+ }
+ System.exit(1);
+ }
+ // String f_hypergraphs = args[0].trim();
+ // String f_rule_tbl = args[1].trim();
+ String f_ref_files = args[2].trim();
+ String f_orc_out = args[3].trim();
+ int lm_order = Integer.parseInt(args[4].trim());
+ boolean orc_extract_nbest = Boolean.valueOf(args[5].trim()); // oracle extraction from nbest or hg
+
+ int baseline_lm_feat_id = 0;
+
+ KBestExtractor kbest_extractor = null;
+ int topN = 300;// TODO
+ joshuaConfiguration.use_unique_nbest = true;
+ joshuaConfiguration.include_align_index = false;
+ boolean do_ngram_clip_nbest = true; // TODO
+ if (orc_extract_nbest) {
+ System.out.println("oracle extraction from nbest list");
+
+ kbest_extractor = new KBestExtractor(null, null, Decoder.weights, false, joshuaConfiguration);
+ }
+
+ BufferedWriter orc_out = FileUtility.getWriteFileStream(f_orc_out);
+
+ long start_time0 = System.currentTimeMillis();
+ long time_on_reading = 0;
+ long time_on_orc_extract = 0;
+ // DiskHyperGraph dhg_read = new DiskHyperGraph(baseline_lm_feat_id, true, null);
+
+ // dhg_read.initRead(f_hypergraphs, f_rule_tbl, null);
+
+ OracleExtractionHG orc_extractor = new OracleExtractionHG(baseline_lm_feat_id);
+ long start_time = System.currentTimeMillis();
+ int sent_id = 0;
+ for (String ref_sent: new LineReader(f_ref_files)) {
+ System.out.println("############Process sentence " + sent_id);
+ start_time = System.currentTimeMillis();
+ sent_id++;
+ // if(sent_id>10)break;
+
+ // HyperGraph hg = dhg_read.readHyperGraph();
+ HyperGraph hg = null;
+ if (hg == null)
+ continue;
+
+ // System.out.println("read disk hyp: " + (System.currentTimeMillis()-start_time));
+ time_on_reading += System.currentTimeMillis() - start_time;
+ start_time = System.currentTimeMillis();
+
+ String orc_sent = null;
+ double orc_bleu = 0;
+ if (orc_extract_nbest) {
+ Object[] res = orc_extractor.oracle_extract_nbest(kbest_extractor, hg, topN,
+ do_ngram_clip_nbest, ref_sent);
+ orc_sent = (String) res[0];
+ orc_bleu = (Double) res[1];
+ } else {
+ HyperGraph hg_oracle = orc_extractor.oracle_extract_hg(hg, hg.sentLen(), lm_order, ref_sent);
+ orc_sent = removeSentenceMarkers(getViterbiString(hg_oracle));
+ orc_bleu = orc_extractor.get_best_goal_cost(hg, orc_extractor.g_tbl_split_virtual_items);
+
+ time_on_orc_extract += System.currentTimeMillis() - start_time;
+ System.out.println("num_virtual_items: " + orc_extractor.g_num_virtual_items
+ + " num_virtual_dts: " + orc_extractor.g_num_virtual_deductions);
+ // System.out.println("oracle extract: " + (System.currentTimeMillis()-start_time));
+ }
+
+ orc_out.write(orc_sent + "\n");
+ System.out.println("orc bleu is " + orc_bleu);
+ }
+ orc_out.close();
+
+ System.out.println("time_on_reading: " + time_on_reading);
+ System.out.println("time_on_orc_extract: " + time_on_orc_extract);
+ System.out.println("total running time: " + (System.currentTimeMillis() - start_time0));
+ }
+
+ // find the oracle hypothesis in the nbest list
+ public Object[] oracle_extract_nbest(KBestExtractor kbest_extractor, HyperGraph hg, int n,
+ boolean do_ngram_clip, String ref_sent) {
+ if (hg.goalNode == null)
+ return null;
+ kbest_extractor.resetState();
+ int next_n = 0;
+ double orc_bleu = -1;
+ String orc_sent = null;
+ while (true) {
+ String hyp_sent = kbest_extractor.getKthHyp(hg.goalNode, ++next_n);// ?????????
+ if (hyp_sent == null || next_n > n)
+ break;
+ double t_bleu = compute_sentence_bleu(ref_sent, hyp_sent, do_ngram_clip, 4);
+ if (t_bleu > orc_bleu) {
+ orc_bleu = t_bleu;
+ orc_sent = hyp_sent;
+ }
+ }
+ System.out.println("Oracle sent: " + orc_sent);
+ System.out.println("Oracle bleu: " + orc_bleu);
+ Object[] res = new Object[2];
+ res[0] = orc_sent;
+ res[1] = orc_bleu;
+ return res;
+ }
+
+ public HyperGraph oracle_extract_hg(HyperGraph hg, int src_sent_len_in, int lm_order,
+ String ref_sent_str) {
+ int[] ref_sent = Vocabulary.addAll(ref_sent_str);
+ g_lm_order = lm_order;
+ src_sent_len = src_sent_len_in;
+ ref_sent_len = ref_sent.length;
+
+ tbl_ref_ngrams.clear();
+ get_ngrams(tbl_ref_ngrams, g_bleu_order, ref_sent, false);
+ if (using_left_equiv_state || using_right_equiv_state) {
+ tbl_prefix.clear();
+ tbl_suffix.clear();
+ setup_prefix_suffix_tbl(ref_sent, g_bleu_order, tbl_prefix, tbl_suffix);
+ setup_prefix_suffix_grammar(ref_sent, g_bleu_order, grammar_prefix, grammar_suffix);// TODO
+ }
+ split_hg(hg);
+
+ // System.out.println("best bleu is " + get_best_goal_cost( hg, g_tbl_split_virtual_items));
+ return get_1best_tree_hg(hg, g_tbl_split_virtual_items);
+ }
+
+ /*
+ * This procedure does (1) identify all possible match (2) add a new deduction for each matches
+ */
+ protected void process_one_combination_axiom(HGNode parent_item,
+ HashMap<String, VirtualItem> virtual_item_sigs, HyperEdge cur_dt) {
+ if (null == cur_dt.getRule()) {
+ throw new RuntimeException("error null rule in axiom");
+ }
+ double avg_ref_len = (parent_item.j - parent_item.i >= src_sent_len) ? ref_sent_len
+ : (parent_item.j - parent_item.i) * ref_sent_len * 1.0 / src_sent_len;// avg len?
+ double bleu_score[] = new double[1];
+ DPStateOracle dps = compute_state(parent_item, cur_dt, null, tbl_ref_ngrams,
+ do_local_ngram_clip, g_lm_order, avg_ref_len, bleu_score, tbl_suffix, tbl_prefix);
+ VirtualDeduction t_dt = new VirtualDeduction(cur_dt, null, -bleu_score[0]);// cost: -best_bleu
+ g_num_virtual_deductions++;
+ add_deduction(parent_item, virtual_item_sigs, t_dt, dps, true);
+ }
+
+ /*
+ * This procedure does (1) create a new deduction (based on cur_dt and ant_virtual_item) (2) find
+ * whether an Item can contain this deduction (based on virtual_item_sigs which is a hashmap
+ * specific to a parent_item) (2.1) if yes, add the deduction, (2.2) otherwise (2.2.1) create a
+ * new item (2.2.2) and add the item into virtual_item_sigs
+ */
+ protected void process_one_combination_nonaxiom(HGNode parent_item,
+ HashMap<String, VirtualItem> virtual_item_sigs, HyperEdge cur_dt,
+ ArrayList<VirtualItem> l_ant_virtual_item) {
+ if (null == l_ant_virtual_item) {
+ throw new RuntimeException("wrong call in process_one_combination_nonaxiom");
+ }
+ double avg_ref_len = (parent_item.j - parent_item.i >= src_sent_len) ? ref_sent_len
+ : (parent_item.j - parent_item.i) * ref_sent_len * 1.0 / src_sent_len;// avg len?
+ double bleu_score[] = new double[1];
+ DPStateOracle dps = compute_state(parent_item, cur_dt, l_ant_virtual_item, tbl_ref_ngrams,
+ do_local_ngram_clip, g_lm_order, avg_ref_len, bleu_score, tbl_suffix, tbl_prefix);
+ VirtualDeduction t_dt = new VirtualDeduction(cur_dt, l_ant_virtual_item, -bleu_score[0]);// cost:
+ // -best_bleu
+ g_num_virtual_deductions++;
+ add_deduction(parent_item, virtual_item_sigs, t_dt, dps, true);
+ }
+
+ // DPState maintain all the state information at an item that is required during dynamic
+ // programming
+ protected static class DPStateOracle extends DPState {
+ int best_len; // this may not be used in the signature
+ int[] ngram_matches;
+ int[] left_lm_state;
+ int[] right_lm_state;
+
+ public DPStateOracle(int blen, int[] matches, int[] left, int[] right) {
+ best_len = blen;
+ ngram_matches = matches;
+ left_lm_state = left;
+ right_lm_state = right;
+ }
+
+ protected String get_signature() {
+ StringBuffer res = new StringBuffer();
+ if (maitain_length_state) {
+ res.append(best_len);
+ res.append(' ');
+ }
+ if (null != left_lm_state) { // goal-item have null state
+ for (int i = 0; i < left_lm_state.length; i++) {
+ res.append(left_lm_state[i]);
+ res.append(' ');
+ }
+ }
+ res.append("lzf ");
+
+ if (null != right_lm_state) { // goal-item have null state
+ for (int i = 0; i < right_lm_state.length; i++) {
+ res.append(right_lm_state[i]);
+ res.append(' ');
+ }
+ }
+ // if(left_lm_state==null || right_lm_state==null)System.out.println("sig is: " +
+ // res.toString());
+ return res.toString();
+ }
+
+ protected void print() {
+ StringBuffer res = new StringBuffer();
+ res.append("DPstate: best_len: ");
+ res.append(best_len);
+ for (int i = 0; i < ngram_matches.length; i++) {
+ res.append("; ngram: ");
+ res.append(ngram_matches[i]);
+ }
+ System.out.println(res.toString());
+ }
+ }
+
+ // ########################## commmon funcions #####################
+ // based on tbl_oracle_states, tbl_ref_ngrams, and dt, get the state
+ // get the new state: STATE_BEST_DEDUCT STATE_BEST_BLEU STATE_BEST_LEN NGRAM_MATCH_COUNTS
+ protected DPStateOracle compute_state(HGNode parent_item, HyperEdge dt,
+ ArrayList<VirtualItem> l_ant_virtual_item, HashMap<String, Integer> tbl_ref_ngrams,
+ boolean do_local_ngram_clip, int lm_order, double ref_len, double[] bleu_score,
+ HashMap<String, Boolean> tbl_suffix, HashMap<String, Boolean> tbl_prefix) {
+ // ##### deductions under "goal item" does not have rule
+ if (null == dt.getRule()) {
+ if (l_ant_virtual_item.size() != 1) {
+ throw new RuntimeException("error deduction under goal item have more than one item");
+ }
+ bleu_score[0] = -l_ant_virtual_item.get(0).best_virtual_deduction.best_cost;
+ return new DPStateOracle(0, null, null, null); // no DPState at all
+ }
+
+ // ################## deductions *not* under "goal item"
+ HashMap<String, Integer> new_ngram_counts = new HashMap<String, Integer>();// new ngrams created
+ // due to the
+ // combination
+ HashMap<String, Integer> old_ngram_counts = new HashMap<String, Integer>();// the ngram that has
+ // already been
+ // computed
+ int total_hyp_len = 0;
+ int[] num_ngram_match = new int[g_bleu_order];
+ int[] en_words = dt.getRule().getEnglish();
+
+ // ####calulate new and old ngram counts, and len
+
+ ArrayList<Integer> words = new ArrayList<Integer>();
+
+ // used for compute left- and right- lm state
+ ArrayList<Integer> left_state_sequence = null;
+ // used for compute left- and right- lm state
+ ArrayList<Integer> right_state_sequence = null;
+
+ int correct_lm_order = lm_order;
+ if (always_maintain_seperate_lm_state || lm_order < g_bleu_order) {
+ left_state_sequence = new ArrayList<Integer>();
+ right_state_sequence = new ArrayList<Integer>();
+ correct_lm_order = g_bleu_order; // if lm_order is smaller than g_bleu_order, we will get the
+ // lm state by ourself
+ }
+
+ // #### get left_state_sequence, right_state_sequence, total_hyp_len, num_ngram_match
+ for (int c = 0; c < en_words.length; c++) {
+ int c_id = en_words[c];
+ if (FormatUtils.isNonterminal(c_id)) {
+ int index = -(c_id + 1);
+ DPStateOracle ant_state = (DPStateOracle) l_ant_virtual_item.get(index).dp_state;
+ total_hyp_len += ant_state.best_len;
+ for (int t = 0; t < g_bleu_order; t++) {
+ num_ngram_match[t] += ant_state.ngram_matches[t];
+ }
+ int[] l_context = ant_state.left_lm_state;
+ int[] r_context = ant_state.right_lm_state;
+ for (int t : l_context) { // always have l_context
+ words.add(t);
+ if (null != left_state_sequence && left_state_sequence.size() < g_bleu_order - 1) {
+ left_state_sequence.add(t);
+ }
+ }
+ get_ngrams(old_ngram_counts, g_bleu_order, l_context, true);
+ if (r_context.length >= correct_lm_order - 1) { // the right and left are NOT overlapping
+ get_ngrams(new_ngram_counts, g_bleu_order, words, true);
+ get_ngrams(old_ngram_counts, g_bleu_order, r_context, true);
+ words.clear();// start a new chunk
+ if (null != right_state_sequence) {
+ right_state_sequence.clear();
+ }
+ for (int t : r_context) {
+ words.add(t);
+ }
+ }
+ if (null != right_state_sequence) {
+ for (int t : r_context) {
+ right_state_sequence.add(t);
+ }
+ }
+ } else {
+ words.add(c_id);
+ total_hyp_len += 1;
+ if (null != left_state_sequence && left_state_sequence.size() < g_bleu_order - 1) {
+ left_state_sequence.add(c_id);
+ }
+ if (null != right_state_sequence) {
+ right_state_sequence.add(c_id);
+ }
+ }
+ }
+ get_ngrams(new_ngram_counts, g_bleu_order, words, true);
+
+ // ####now deduct ngram counts
+ for (String ngram : new_ngram_counts.keySet()) {
+ if (tbl_ref_ngrams.containsKey(ngram)) {
+ int final_count = (Integer) new_ngram_counts.get(ngram);
+ if (old_ngram_counts.containsKey(ngram)) {
+ final_count -= (Integer) old_ngram_counts.get(ngram);
+ // BUG: Whoa, is that an actual hard-coded ID in there? :)
+ if (final_count < 0) {
+ throw new RuntimeException("negative count for ngram: " + Vocabulary.word(11844)
+ + "; new: " + new_ngram_counts.get(ngram) + "; old: " + old_ngram_counts.get(ngram));
+ }
+ }
+ if (final_count > 0) { // TODO: not correct/global ngram clip
+ if (do_local_ngram_clip) {
+ // BUG: use joshua.util.Regex.spaces.split(...)
+ num_ngram_match[ngram.split("\\s+").length - 1] += Support.findMin(final_count,
+ (Integer) tbl_ref_ngrams.get(ngram));
+ } else {
+ // BUG: use joshua.util.Regex.spaces.split(...)
+ num_ngram_match[ngram.split("\\s+").length - 1] += final_count; // do not do any cliping
+ }
+ }
+ }
+ }
+
+ // ####now calculate the BLEU score and state
+ int[] left_lm_state = null;
+ int[] right_lm_state = null;
+ left_lm_state = get_left_equiv_state(left_state_sequence, tbl_suffix);
+ right_lm_state = get_right_equiv_state(right_state_sequence, tbl_prefix);
+
+ // debug
+ // System.out.println("lm_order is " + lm_order);
+ // compare_two_int_arrays(left_lm_state,
+ // (int[])parent_item.tbl_states.get(Symbol.LM_L_STATE_SYM_ID));
+ // compare_two_int_arrays(right_lm_state,
+ // (int[])parent_item.tbl_states.get(Symbol.LM_R_STATE_SYM_ID));
+ // end
+
+ bleu_score[0] = compute_bleu(total_hyp_len, ref_len, num_ngram_match, g_bleu_order);
+ // System.out.println("blue score is " + bleu_score[0]);
+ return new DPStateOracle(total_hyp_len, num_ngram_match, left_lm_state, right_lm_state);
+ }
+
+ private int[] get_left_equiv_state(ArrayList<Integer> left_state_sequence,
+ HashMap<String, Boolean> tbl_suffix) {
+ int l_size = (left_state_sequence.size() < g_bleu_order - 1) ? left_state_sequence.size()
+ : (g_bleu_order - 1);
+ int[] left_lm_state = new int[l_size];
+ if (!using_left_equiv_state || l_size < g_bleu_order - 1) { // regular
+ for (int i = 0; i < l_size; i++) {
+ left_lm_state[i] = left_state_sequence.get(i);
+ }
+ } else {
+ for (int i = l_size - 1; i >= 0; i--) { // right to left
+ if (is_a_suffix_in_tbl(left_state_sequence, 0, i, tbl_suffix)) {
+ // if(is_a_suffix_in_grammar(left_state_sequence, 0, i, grammar_suffix)){
+ for (int j = i; j >= 0; j--) {
+ left_lm_state[j] = left_state_sequence.get(j);
+ }
+ break;
+ } else {
+ left_lm_state[i] = this.NULL_LEFT_LM_STATE_SYM_ID;
+ }
+ }
+ // System.out.println("origi left:" + Symbol.get_string(left_state_sequence) + "; equiv left:"
+ // + Symbol.get_string(left_lm_state));
+ }
+ return left_lm_state;
+ }
+
+ private boolean is_a_suffix_in_tbl(ArrayList<Integer> left_state_sequence, int start_pos,
+ int end_pos, HashMap<String, Boolean> tbl_suffix) {
+ if ((Integer) left_state_sequence.get(end_pos) == this.NULL_LEFT_LM_STATE_SYM_ID) {
+ return false;
+ }
+ StringBuffer suffix = new StringBuffer();
+ for (int i = end_pos; i >= start_pos; i--) { // right-most first
+ suffix.append(left_state_sequence.get(i));
+ if (i > start_pos)
+ suffix.append(' ');
+ }
+ return (Boolean) tbl_suffix.containsKey(suffix.toString());
+ }
+
+ private int[] get_right_equiv_state(ArrayList<Integer> right_state_sequence,
+ HashMap<String, Boolean> tbl_prefix) {
+ int r_size = (right_state_sequence.size() < g_bleu_order - 1) ? right_state_sequence.size()
+ : (g_bleu_order - 1);
+ int[] right_lm_state = new int[r_size];
+ if (!using_right_equiv_state || r_size < g_bleu_order - 1) { // regular
+ for (int i = 0; i < r_size; i++) {
+ right_lm_state[i] = (Integer) right_state_sequence.get(right_state_sequence.size() - r_size
+ + i);
+ }
+ } else {
+ for (int i = 0; i < r_size; i++) { // left to right
+ if (is_a_prefix_in_tbl(right_state_sequence, right_state_sequence.size() - r_size + i,
+ right_state_sequence.size() - 1, tbl_prefix)) {
+ // if(is_a_prefix_in_grammar(right_state_sequence, right_state_sequence.size()-r_size+i,
+ // right_state_sequence.size()-1, grammar_prefix)){
+ for (int j = i; j < r_size; j++) {
+ right_lm_state[j] = (Integer) right_state_sequence.get(right_state_sequence.size()
+ - r_size + j);
+ }
+ break;
+ } else {
+ right_lm_state[i] = this.NULL_RIGHT_LM_STATE_SYM_ID;
+ }
+ }
+ // System.out.println("origi right:" + Symbol.get_string(right_state_sequence)+
+ // "; equiv right:" + Symbol.get_string(right_lm_state));
+ }
+ return right_lm_state;
+ }
+
+ private boolean is_a_prefix_in_tbl(ArrayList<Integer> right_state_sequence, int start_pos,
+ int end_pos, HashMap<String, Boolean> tbl_prefix) {
+ if (right_state_sequence.get(start_pos) == this.NULL_RIGHT_LM_STATE_SYM_ID) {
+ return false;
+ }
+ StringBuffer prefix = new StringBuffer();
+ for (int i = start_pos; i <= end_pos; i++) {
+ prefix.append(right_state_sequence.get(i));
+ if (i < end_pos)
+ prefix.append(' ');
+ }
+ return (Boolean) tbl_prefix.containsKey(prefix.toString());
+ }
+
+ public static void compare_two_int_arrays(int[] a, int[] b) {
+ if (a.length != b.length) {
+ throw new RuntimeException("two arrays do not have same size");
+ }
+ for (int i = 0; i < a.length; i++) {
+ if (a[i] != b[i]) {
+ throw new RuntimeException("elements in two arrays are not same");
+ }
+ }
+ }
+
+ // sentence-bleu: BLEU= bp * prec; where prec = exp (sum 1/4 * log(prec[order]))
+ public static double compute_bleu(int hyp_len, double ref_len, int[] num_ngram_match,
+ int bleu_order) {
+ if (hyp_len <= 0 || ref_len <= 0) {
+ throw new RuntimeException("ref or hyp is zero len");
+ }
+ double res = 0;
+ double wt = 1.0 / bleu_order;
+ double prec = 0;
+ double smooth_factor = 1.0;
+ for (int t = 0; t < bleu_order && t < hyp_len; t++) {
+ if (num_ngram_match[t] > 0) {
+ prec += wt * Math.log(num_ngram_match[t] * 1.0 / (hyp_len - t));
+ } else {
+ smooth_factor *= 0.5;// TODO
+ prec += wt * Math.log(smooth_factor / (hyp_len - t));
+ }
+ }
+ double bp = (hyp_len >= ref_len) ? 1.0 : Math.exp(1 - ref_len / hyp_len);
+ res = bp * Math.exp(prec);
+ // System.out.println("hyp_len: " + hyp_len + "; ref_len:" + ref_len + "prec: " + Math.exp(prec)
+ // + "; bp: " + bp + "; bleu: " + res);
+ return res;
+ }
+
+ // accumulate ngram counts into tbl
+ public void get_ngrams(HashMap<String, Integer> tbl, int order, int[] wrds,
+ boolean ignore_null_equiv_symbol) {
+ for (int i = 0; i < wrds.length; i++) {
+ for (int j = 0; j < order && j + i < wrds.length; j++) { // ngram: [i,i+j]
+ boolean contain_null = false;
+ StringBuffer ngram = new StringBuffer();
+ for (int k = i; k <= i + j; k++) {
+ if (wrds[k] == this.NULL_LEFT_LM_STATE_SYM_ID
+ || wrds[k] == this.NULL_RIGHT_LM_STATE_SYM_ID) {
+ contain_null = true;
+ if (ignore_null_equiv_symbol)
+ break;
+ }
+ ngram.append(wrds[k]);
+ if (k < i + j)
+ ngram.append(' ');
+ }
+ if (ignore_null_equiv_symbol && contain_null)
+ continue; // skip this ngram
+ String ngram_str = ngram.toString();
+ if (tbl.containsKey(ngram_str)) {
+ tbl.put(ngram_str, (Integer) tbl.get(ngram_str) + 1);
+ } else {
+ tbl.put(ngram_str, 1);
+ }
+ }
+ }
+ }
+
+ /**
+ * accumulate ngram counts into tbl.
+ * @param tbl a {@link java.util.HashMap} which is used to store ngram counts
+ * @param order todo
+ * @param wrds an {@link java.util.ArrayList} containing {@link java.lang.Integer} word representations
+ * @param ignore_null_equiv_symbol set to true to skip some nGrams
+ */
+ public void get_ngrams(HashMap<String, Integer> tbl, int order, ArrayList<Integer> wrds,
+ boolean ignore_null_equiv_symbol) {
+ for (int i = 0; i < wrds.size(); i++) {
+ // ngram: [i,i+j]
+ for (int j = 0; j < order && j + i < wrds.size(); j++) {
+ boolean contain_null = false;
+ StringBuffer ngram = new StringBuffer();
+ for (int k = i; k <= i + j; k++) {
+ int t_wrd = (Integer) wrds.get(k);
+ if (t_wrd == this.NULL_LEFT_LM_STATE_SYM_ID || t_wrd == this.NULL_RIGHT_LM_STATE_SYM_ID) {
+ contain_null = true;
+ if (ignore_null_equiv_symbol)
+ break;
+ }
+ ngram.append(t_wrd);
+ if (k < i + j)
+ ngram.append(' ');
+ }
+ // skip this ngram
+ if (ignore_null_equiv_symbol && contain_null)
+ continue;
+
+ String ngram_str = ngram.toString();
+ if (tbl.containsKey(ngram_str)) {
+ tbl.put(ngram_str, (Integer) tbl.get(ngram_str) + 1);
+ } else {
+ tbl.put(ngram_str, 1);
+ }
+ }
+ }
+ }
+
+ // do_ngram_clip: consider global n-gram clip
+ public double compute_sentence_bleu(String ref_sent, String hyp_sent, boolean do_ngram_clip,
+ int bleu_order) {
+ // BUG: use joshua.util.Regex.spaces.split(...)
+ int[] numeric_ref_sent = Vocabulary.addAll(ref_sent);
+ int[] numeric_hyp_sent = Vocabulary.addAll(hyp_sent);
+ return compute_sentence_bleu(numeric_ref_sent, numeric_hyp_sent, do_ngram_clip, bleu_order);
+ }
+
+ public double compute_sentence_bleu(int[] ref_sent, int[] hyp_sent, boolean do_ngram_clip,
+ int bleu_order) {
+ double res_bleu = 0;
+ int order = 4;
+ HashMap<String, Integer> ref_ngram_tbl = new HashMap<String, Integer>();
+ get_ngrams(ref_ngram_tbl, order, ref_sent, false);
+ HashMap<String, Integer> hyp_ngram_tbl = new HashMap<String, Integer>();
+ get_ngrams(hyp_ngram_tbl, order, hyp_sent, false);
+
+ int[] num_ngram_match = new int[order];
+ for (String ngram : hyp_ngram_tbl.keySet()) {
+ if (ref_ngram_tbl.containsKey(ngram)) {
+ if (do_ngram_clip) {
+ // BUG: use joshua.util.Regex.spaces.split(...)
+ num_ngram_match[ngram.split("\\s+").length - 1] += Support.findMin(
+ (Integer) ref_ngram_tbl.get(ngram), (Integer) hyp_ngram_tbl.get(ngram)); // ngram clip
+ } else {
+ // BUG: use joshua.util.Regex.spaces.split(...)
+ num_ngram_match[ngram.split("\\s+").length - 1] += (Integer) hyp_ngram_tbl.get(ngram);// without
+ // ngram
+ // count
+ // clipping
+ }
+ }
+ }
+ res_bleu = compute_bleu(hyp_sent.length, ref_sent.length, num_ngram_match, bleu_order);
+ // System.out.println("hyp_len: " + hyp_sent.length + "; ref_len:" + ref_sent.length +
+ // "; bleu: " + res_bleu +" num_ngram_matches: " + num_ngram_match[0] + " " +num_ngram_match[1]+
+ // " " + num_ngram_match[2] + " " +num_ngram_match[3]);
+
+ return res_bleu;
+ }
+
+ // #### equivalent lm stuff ############
+ public static void setup_prefix_suffix_tbl(int[] wrds, int order,
+ HashMap<String, Boolean> prefix_tbl, HashMap<String, Boolean> suffix_tbl) {
+ for (int i = 0; i < wrds.length; i++) {
+ for (int j = 0; j < order && j + i < wrds.length; j++) { // ngram: [i,i+j]
+ StringBuffer ngram = new StringBuffer();
+ // ### prefix
+ for (int k = i; k < i + j; k++) { // all ngrams [i,i+j-1]
+ ngram.append(wrds[k]);
+ prefix_tbl.put(ngram.toString(), true);
+ ngram.append(' ');
+ }
+ // ### suffix: right-most wrd first
+ ngram = new StringBuffer();
+ for (int k = i + j; k > i; k--) { // all ngrams [i+1,i+j]: reverse order
+ ngram.append(wrds[k]);
+ suffix_tbl.put(ngram.toString(), true);// stored in reverse order
+ ngram.append(' ');
+ }
+ }
+ }
+ }
+
+ // #### equivalent lm stuff ############
+ public static void setup_prefix_suffix_grammar(int[] wrds, int order, PrefixGrammar prefix_gr,
+ PrefixGrammar suffix_gr) {
+ for (int i = 0; i < wrds.length; i++) {
+ for (int j = 0; j < order && j + i < wrds.length; j++) { // ngram: [i,i+j]
+ // ### prefix
+ prefix_gr.add_ngram(wrds, i, i + j - 1);// ngram: [i,i+j-1]
+
+ // ### suffix: right-most wrd first
+ int[] reverse_wrds = new int[j];
+ for (int k = i + j, t = 0; k > i; k--) { // all ngrams [i+1,i+j]: reverse order
+ reverse_wrds[t++] = wrds[k];
+ }
+ suffix_gr.add_ngram(reverse_wrds, 0, j - 1);
+ }
+ }
+ }
+
+ /*
+ * a backoff node is a hashtable, it may include: (1) probabilititis for next words (2) pointers
+ * to a next-layer backoff node (hashtable) (3) backoff weight for this node (4) suffix/prefix
+ * flag to indicate that there is ngrams start from this suffix
+ */
+ private static class PrefixGrammar {
+
+ private static class PrefixGrammarNode extends HashMap<Integer, PrefixGrammarNode> {
+ private static final long serialVersionUID = 1L;
+ };
+
+ PrefixGrammarNode root = new PrefixGrammarNode();
+
+ // add prefix information
+ public void add_ngram(int[] wrds, int start_pos, int end_pos) {
+ // ######### identify the position, and insert the trinodes if necessary
+ PrefixGrammarNode pos = root;
+ for (int k = start_pos; k <= end_pos; k++) {
+ int cur_sym_id = wrds[k];
+ PrefixGrammarNode next_layer = pos.get(cur_sym_id);
+
+ if (null != next_layer) {
+ pos = next_layer;
+ } else {
+ // next layer node
+ PrefixGrammarNode tmp = new PrefixGrammarNode();
+ pos.put(cur_sym_id, tmp);
+ pos = tmp;
+ }
+ }
+ }
+
+ @SuppressWarnings("unused")
+ public boolean contain_ngram(ArrayList<Integer> wrds, int start_pos, int end_pos) {
+ if (end_pos < start_pos)
+ return false;
+ PrefixGrammarNode pos = root;
+ for (int k = start_pos; k <= end_pos; k++) {
+ int cur_sym_id = wrds.get(k);
+ PrefixGrammarNode next_layer = pos.get(cur_sym_id);
+ if (next_layer != null) {
+ pos = next_layer;
+ } else {
+ return false;
+ }
+ }
+ return true;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/oracle/OracleExtractor.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/oracle/OracleExtractor.java b/joshua-core/src/main/java/org/apache/joshua/oracle/OracleExtractor.java
new file mode 100644
index 0000000..ef67905
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/oracle/OracleExtractor.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.oracle;
+
+import org.apache.joshua.decoder.hypergraph.HyperGraph;
+
+/**
+ * Convenience wrapper class for oracle extraction code.
+ *
+ * @author Lane Schwartz
+ */
+public class OracleExtractor {
+
+ private final OracleExtractionHG extractor;
+
+ /**
+ * Constructs an object capable of extracting an oracle hypergraph.
+ */
+ public OracleExtractor() {
+
+ int baselineLanguageModelFeatureID = 0;
+ this.extractor = new OracleExtractionHG(baselineLanguageModelFeatureID);
+
+ }
+
+ /**
+ * Extract a hypergraph that represents the translation from the original shared forest hypergraph
+ * that is closest to the reference translation.
+ *
+ * @param forest Original hypergraph representing a shared forest.
+ * @param lmOrder N-gram order of the language model.
+ * @param reference Reference sentence.
+ * @return Hypergraph closest to the reference.
+ */
+ public HyperGraph getOracle(HyperGraph forest, int lmOrder, String reference) {
+ if (reference != null)
+ return extractor.oracle_extract_hg(forest, forest.sentLen(), lmOrder, reference);
+
+ return null;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/oracle/SplitHg.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/oracle/SplitHg.java b/joshua-core/src/main/java/org/apache/joshua/oracle/SplitHg.java
new file mode 100644
index 0000000..9fcdd35
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/oracle/SplitHg.java
@@ -0,0 +1,300 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.oracle;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.joshua.decoder.hypergraph.HGNode;
+import org.apache.joshua.decoder.hypergraph.HyperEdge;
+import org.apache.joshua.decoder.hypergraph.HyperGraph;
+
+/**
+ * This class implements general ways of splitting the hypergraph based on coarse-to-fine idea input
+ * is a hypergraph output is another hypergraph that has changed state structures.
+ *
+ * @author Zhifei Li, zhifei.work@gmail.com (Johns Hopkins University)
+ */
+public abstract class SplitHg {
+
+ HashMap<HGNode, ArrayList<VirtualItem>> g_tbl_split_virtual_items = new HashMap<HGNode, ArrayList<VirtualItem>>();
+
+ // number of items or deductions after splitting the hypergraph
+ public int g_num_virtual_items = 0;
+ public int g_num_virtual_deductions = 0;
+
+ // Note: the implementation of the following two functions should call add_deduction
+ protected abstract void process_one_combination_axiom(HGNode parent_item,
+ HashMap<String, VirtualItem> virtual_item_sigs, HyperEdge cur_dt);
+
+ protected abstract void process_one_combination_nonaxiom(HGNode parent_item,
+ HashMap<String, VirtualItem> virtual_item_sigs, HyperEdge cur_dt,
+ ArrayList<VirtualItem> l_ant_virtual_item);
+
+ // #### all the functions should be called after running split_hg(), before clearing
+ // g_tbl_split_virtual_items
+ public double get_best_goal_cost(HyperGraph hg,
+ HashMap<HGNode, ArrayList<VirtualItem>> g_tbl_split_virtual_items) {
+ double res = get_virtual_goal_item(hg, g_tbl_split_virtual_items).best_virtual_deduction.best_cost;
+ // System.out.println("best bleu is " +res);
+ return res;
+ }
+
+ public VirtualItem get_virtual_goal_item(HyperGraph original_hg,
+ HashMap<HGNode, ArrayList<VirtualItem>> g_tbl_split_virtual_items) {
+ ArrayList<VirtualItem> l_virtual_items = g_tbl_split_virtual_items.get(original_hg.goalNode);
+
+ if (l_virtual_items.size() != 1) {
+ // TODO: log this properly, fail properly
+ throw new RuntimeException("number of virtual goal items is not equal to one");
+ }
+ return l_virtual_items.get(0);
+ }
+
+ // get the 1best tree hg, the 1-best is ranked by the split hypergraph, but the return hypergraph
+ // is in the form of the original hg
+ public HyperGraph get_1best_tree_hg(HyperGraph original_hg,
+ HashMap<HGNode, ArrayList<VirtualItem>> g_tbl_split_virtual_items) {
+ VirtualItem virutal_goal_item = get_virtual_goal_item(original_hg, g_tbl_split_virtual_items);
+ HGNode onebest_goal_item = clone_item_with_best_deduction(virutal_goal_item);
+ HyperGraph res = new HyperGraph(onebest_goal_item, -1, -1, null);
+ // TODO: number of items/deductions
+ get_1best_tree_item(virutal_goal_item, onebest_goal_item);
+ return res;
+ }
+
+ private void get_1best_tree_item(VirtualItem virtual_it, HGNode onebest_item) {
+ VirtualDeduction virtual_dt = virtual_it.best_virtual_deduction;
+ if (virtual_dt.l_ant_virtual_items != null)
+ for (int i = 0; i < virtual_dt.l_ant_virtual_items.size(); i++) {
+ VirtualItem ant_it = (VirtualItem) virtual_dt.l_ant_virtual_items.get(i);
+ HGNode new_it = clone_item_with_best_deduction(ant_it);
+ onebest_item.bestHyperedge.getTailNodes().set(i, new_it);
+ get_1best_tree_item(ant_it, new_it);
+ }
+ }
+
+ // TODO: tbl_states
+ private static HGNode clone_item_with_best_deduction(VirtualItem virtual_it) {
+ HGNode original_it = virtual_it.p_item;
+ ArrayList<HyperEdge> l_deductions = new ArrayList<HyperEdge>();
+ HyperEdge clone_dt = clone_deduction(virtual_it.best_virtual_deduction);
+ l_deductions.add(clone_dt);
+ return new HGNode(original_it.i, original_it.j, original_it.lhs, l_deductions, clone_dt,
+ original_it.getDPStates());
+ }
+
+ private static HyperEdge clone_deduction(VirtualDeduction virtual_dt) {
+ HyperEdge original_dt = virtual_dt.p_dt;
+ ArrayList<HGNode> l_ant_items = null;
+ // l_ant_items will be changed in get_1best_tree_item
+ if (original_dt.getTailNodes() != null)
+ l_ant_items = new ArrayList<HGNode>(original_dt.getTailNodes());
+ HyperEdge res = new HyperEdge(original_dt.getRule(), original_dt.getBestDerivationScore(),
+ original_dt.getTransitionLogP(false), l_ant_items, original_dt.getSourcePath());
+ return res;
+ }
+
+ // ############### split hg #####
+ public void split_hg(HyperGraph hg) {
+ // TODO: more pre-process in the extended class
+ g_tbl_split_virtual_items.clear();
+ g_num_virtual_items = 0;
+ g_num_virtual_deductions = 0;
+ split_item(hg.goalNode);
+ }
+
+ // for each original Item, get a list of VirtualItem
+ private void split_item(HGNode it) {
+ if (g_tbl_split_virtual_items.containsKey(it))
+ return;// already processed
+ HashMap<String, VirtualItem> virtual_item_sigs = new HashMap<String, VirtualItem>();
+ // ### recursive call on each deduction
+ if (speed_up_item(it)) {
+ for (HyperEdge dt : it.hyperedges) {
+ split_deduction(dt, virtual_item_sigs, it);
+ }
+ }
+ // ### item-specific operation
+ // a list of items result by splitting me
+ ArrayList<VirtualItem> l_virtual_items = new ArrayList<VirtualItem>();
+ for (String signature : virtual_item_sigs.keySet())
+ l_virtual_items.add(virtual_item_sigs.get(signature));
+ g_tbl_split_virtual_items.put(it, l_virtual_items);
+ g_num_virtual_items += l_virtual_items.size();
+ // if(virtual_item_sigs.size()!=1)System.out.println("num of split items is " +
+ // virtual_item_sigs.size());
+ // get_best_virtual_score(it);//debug
+ }
+
+ private void split_deduction(HyperEdge cur_dt, HashMap<String, VirtualItem> virtual_item_sigs,
+ HGNode parent_item) {
+ if (speed_up_deduction(cur_dt) == false)
+ return;// no need to continue
+
+ // ### recursively split all my ant items, get a l_split_items for each original item
+ if (cur_dt.getTailNodes() != null)
+ for (HGNode ant_it : cur_dt.getTailNodes())
+ split_item(ant_it);
+
+ // ### recombine the deduction
+ redo_combine(cur_dt, virtual_item_sigs, parent_item);
+ }
+
+ private void redo_combine(HyperEdge cur_dt, HashMap<String, VirtualItem> virtual_item_sigs,
+ HGNode parent_item) {
+ List<HGNode> l_ant_items = cur_dt.getTailNodes();
+ if (l_ant_items != null) {
+ // arity: one
+ if (l_ant_items.size() == 1) {
+ HGNode it = l_ant_items.get(0);
+ ArrayList<VirtualItem> l_virtual_items = g_tbl_split_virtual_items.get(it);
+ for (VirtualItem ant_virtual_item : l_virtual_items) {
+ // used in combination
+ ArrayList<VirtualItem> l_ant_virtual_item = new ArrayList<VirtualItem>();
+ l_ant_virtual_item.add(ant_virtual_item);
+ process_one_combination_nonaxiom(parent_item, virtual_item_sigs, cur_dt,
+ l_ant_virtual_item);
+ }
+ // arity: two
+ } else if (l_ant_items.size() == 2) {
+ HGNode it1 = l_ant_items.get(0);
+ HGNode it2 = l_ant_items.get(1);
+ ArrayList<VirtualItem> l_virtual_items1 = g_tbl_split_virtual_items.get(it1);
+ ArrayList<VirtualItem> l_virtual_items2 = g_tbl_split_virtual_items.get(it2);
+ for (VirtualItem virtual_it1 : l_virtual_items1) {
+ for (VirtualItem virtual_it2 : l_virtual_items2) {
+ // used in combination
+ ArrayList<VirtualItem> l_ant_virtual_item = new ArrayList<VirtualItem>();
+ l_ant_virtual_item.add(virtual_it1);
+ l_ant_virtual_item.add(virtual_it2);
+ process_one_combination_nonaxiom(parent_item, virtual_item_sigs, cur_dt,
+ l_ant_virtual_item);
+ }
+ }
+ } else {
+ throw new RuntimeException(
+ "Sorry, we can only deal with rules with at most TWO non-terminals");
+ }
+ // axiom case: no nonterminal
+ } else {
+ process_one_combination_axiom(parent_item, virtual_item_sigs, cur_dt);
+ }
+ }
+
+ // this function should be called by
+ // process_one_combination_axiom/process_one_combination_nonaxiom
+ // virtual_item_sigs is specific to parent_item
+ protected void add_deduction(HGNode parent_item, HashMap<String, VirtualItem> virtual_item_sigs,
+ VirtualDeduction t_ded, DPState dpstate, boolean maintain_onebest_only) {
+ if (null == t_ded) {
+ throw new RuntimeException("deduction is null");
+ }
+ String sig = VirtualItem.get_signature(parent_item, dpstate);
+ VirtualItem t_virtual_item = (VirtualItem) virtual_item_sigs.get(sig);
+ if (t_virtual_item != null) {
+ t_virtual_item.add_deduction(t_ded, dpstate, maintain_onebest_only);
+ } else {
+ t_virtual_item = new VirtualItem(parent_item, dpstate, t_ded, maintain_onebest_only);
+ virtual_item_sigs.put(sig, t_virtual_item);
+ }
+ }
+
+ // return false if we can skip the item;
+ protected boolean speed_up_item(HGNode it) {
+ return true;// e.g., if the lm state is not valid, then no need to continue
+ }
+
+ // return false if we can skip the deduction;
+ protected boolean speed_up_deduction(HyperEdge dt) {
+ return true;// if the rule state is not valid, then no need to continue
+ }
+
+ protected abstract static class DPState {
+ protected abstract String get_signature();
+ };
+
+ /*
+ * In general, variables of items (1) list of hyperedges (2) best hyperedge (3) DP state (4)
+ * signature (operated on part/full of DP state)
+ */
+
+ protected static class VirtualItem {
+ HGNode p_item = null;// pointer to the true item
+ ArrayList<VirtualDeduction> l_virtual_deductions = null;
+ VirtualDeduction best_virtual_deduction = null;
+ DPState dp_state;// dynamic programming state: not all the variable in dp_state are in the
+ // signature
+
+ public VirtualItem(HGNode item, DPState dstate, VirtualDeduction fdt,
+ boolean maintain_onebest_only) {
+ p_item = item;
+ add_deduction(fdt, dstate, maintain_onebest_only);
+ }
+
+ public void add_deduction(VirtualDeduction fdt, DPState dstate, boolean maintain_onebest_only) {
+ if (maintain_onebest_only == false) {
+ if (l_virtual_deductions == null)
+ l_virtual_deductions = new ArrayList<VirtualDeduction>();
+ ;
+ l_virtual_deductions.add(fdt);
+ }
+ if (best_virtual_deduction == null || fdt.best_cost < best_virtual_deduction.best_cost) {
+ dp_state = dstate;
+ best_virtual_deduction = fdt;
+ }
+ }
+
+ // not all the variable in dp_state are in the signature
+ public String get_signature() {
+ return get_signature(p_item, dp_state);
+ }
+
+ public static String get_signature(HGNode item, DPState dstate) {
+ /*
+ * StringBuffer res = new StringBuffer(); //res.append(item); res.append(" ");//TODO:
+ * res.append(dstate.get_signature()); return res.toString();
+ */
+ return dstate.get_signature();
+ }
+ }
+
+ protected static class VirtualDeduction {
+ HyperEdge p_dt = null;// pointer to the true deduction
+ ArrayList<VirtualItem> l_ant_virtual_items = null;
+ double best_cost = Double.POSITIVE_INFINITY;// the 1-best cost of all possible derivation: best
+ // costs of ant items +
+ // non_stateless_transition_cost + r.statelesscost
+
+ public VirtualDeduction(HyperEdge dt, ArrayList<VirtualItem> ant_items, double best_cost_in) {
+ p_dt = dt;
+ l_ant_virtual_items = ant_items;
+ best_cost = best_cost_in;
+ }
+
+ public double get_transition_cost() {// note: transition_cost is already linearly interpolated
+ double res = best_cost;
+ if (l_ant_virtual_items != null)
+ for (VirtualItem ant_it : l_ant_virtual_items)
+ res -= ant_it.best_virtual_deduction.best_cost;
+ return res;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/oracle/package-info.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/oracle/package-info.java b/joshua-core/src/main/java/org/apache/joshua/oracle/package-info.java
new file mode 100644
index 0000000..ae14e82
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/oracle/package-info.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/**
+ * Provides for extracting the target string from a hypergraph
+ * that most closely matches a reference sentence. Much of the
+ * code in this package is based on descriptions in Adam
+ * Lopez's <a href="http://homepages.inf.ed.ac.uk/alopez/papers/adam.lopez.dissertation.pdf">
+ * doctoral thesis</a>.
+ */
+package org.apache.joshua.oracle;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/pro/ClassifierInterface.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/pro/ClassifierInterface.java b/joshua-core/src/main/java/org/apache/joshua/pro/ClassifierInterface.java
new file mode 100755
index 0000000..d6dca73
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/pro/ClassifierInterface.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.pro;
+
+import java.util.Vector;
+
+public interface ClassifierInterface {
+ /*
+ * Arguments required to train a binary linear classifier: Vector<String> samples: all training
+ * samples should use sparse feature value representation. Format: feat_id1:feat_val1
+ * feat_id2:feat_val2 ... label (1 or -1) Example: 3:0.2 6:2 8:0.5 -1 (only enumerate firing
+ * features) Note feat_id should start from 1 double[] initialLambda: the initial weight
+ * vector(doesn't have to be used, depending on the classifier - just ignore the array if not to
+ * be used). The length of the vector should be the same as feature dimension. Note the 0^th entry
+ * is not used, so array should have length featDim+1 (to be consistent with Z-MERT) int featDim:
+ * feature vector dimension
+ *
+ * Return value: double[]: a vector containing weights for all features after training(also should
+ * have length featDim+1)
+ */
+ double[] runClassifier(Vector<String> samples, double[] initialLambda, int featDim);
+
+ // Set classifier-specific parameters, like config file path, num of iterations, command line...
+ void setClassifierParam(String[] param);
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/pro/ClassifierMegaM.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/pro/ClassifierMegaM.java b/joshua-core/src/main/java/org/apache/joshua/pro/ClassifierMegaM.java
new file mode 100755
index 0000000..f75605f
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/pro/ClassifierMegaM.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.pro;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.util.Vector;
+
+import org.apache.joshua.util.StreamGobbler;
+import org.apache.joshua.util.io.LineReader;
+
+// sparse feature representation version
+public class ClassifierMegaM implements ClassifierInterface {
+ @Override
+ public double[] runClassifier(Vector<String> samples, double[] initialLambda, int featDim) {
+ double[] lambda = new double[featDim + 1];
+ System.out.println("------- MegaM training starts ------");
+
+ try {
+ // prepare training file for MegaM
+ PrintWriter prt = new PrintWriter(new FileOutputStream(trainingFilePath));
+ String[] feat;
+ String[] featInfo;
+
+ for (String line : samples) {
+ feat = line.split("\\s+");
+
+ if (feat[feat.length - 1].equals("1"))
+ prt.print("1 ");
+ else
+ prt.print("0 ");
+
+ // only for dense representation
+ // for(int i=0; i<feat.length-1; i++)
+ // prt.print( (i+1) + " " + feat[i]+" "); //feat id starts from 1!
+
+ for (int i = 0; i < feat.length - 1; i++) {
+ featInfo = feat[i].split(":");
+ prt.print(featInfo[0] + " " + featInfo[1] + " ");
+ }
+ prt.println();
+ }
+ prt.close();
+
+ // start running MegaM
+ Runtime rt = Runtime.getRuntime();
+ Process p = rt.exec(commandFilePath);
+
+ StreamGobbler errorGobbler = new StreamGobbler(p.getErrorStream(), 1);
+ StreamGobbler outputGobbler = new StreamGobbler(p.getInputStream(), 1);
+
+ errorGobbler.start();
+ outputGobbler.start();
+
+ int decStatus = p.waitFor();
+ if (decStatus != 0) {
+ throw new RuntimeException("Call to decoder returned " + decStatus + "; was expecting " + 0 + ".");
+ }
+
+ // read the weights
+ for (String line: new LineReader(weightFilePath)) {
+ String val[] = line.split("\\s+");
+ lambda[Integer.parseInt(val[0])] = Double.parseDouble(val[1]);
+ }
+
+ File file = new File(trainingFilePath);
+ file.delete();
+ file = new File(weightFilePath);
+ file.delete();
+ } catch (IOException | InterruptedException e) {
+ throw new RuntimeException(e);
+ }
+
+ System.out.println("------- MegaM training ends ------");
+
+ /*
+ * try { Thread.sleep(20000); } catch(InterruptedException e) { }
+ */
+
+ return lambda;
+ }
+
+ @Override
+ /*
+ * for MegaM classifier: param[0] = MegaM command file path param[1] = MegaM training data
+ * file(generated on the fly) path param[2] = MegaM weight file(generated after training) path
+ * note that the training and weight file path should be consistent with that specified in the
+ * command file
+ */
+ public void setClassifierParam(String[] param) {
+ if (param == null) {
+ throw new RuntimeException("ERROR: must provide parameters for MegaM classifier!");
+ } else {
+ commandFilePath = param[0];
+ trainingFilePath = param[1];
+ weightFilePath = param[2];
+ }
+ }
+
+ String commandFilePath;
+ String trainingFilePath;
+ String weightFilePath;
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/pro/ClassifierPerceptron.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/pro/ClassifierPerceptron.java b/joshua-core/src/main/java/org/apache/joshua/pro/ClassifierPerceptron.java
new file mode 100755
index 0000000..1b5d75c
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/pro/ClassifierPerceptron.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.pro;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Vector;
+
+// sparse feature representation version
+public class ClassifierPerceptron implements ClassifierInterface {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ClassifierPerceptron.class);
+
+ @Override
+ public double[] runClassifier(Vector<String> samples, double[] initialLambda, int featDim) {
+ System.out.println("------- Average-perceptron training starts ------");
+
+ int sampleSize = samples.size();
+ double score = 0; // model score
+ double label;
+ double[] lambda = new double[featDim + 1]; // in ZMERT lambda[0] is not used
+ double[] sum_lambda = new double[featDim + 1];
+ String[] featVal;
+
+ for (int i = 1; i <= featDim; i++) {
+ sum_lambda[i] = 0;
+ lambda[i] = initialLambda[i];
+ }
+
+ System.out.print("Perceptron iteration ");
+ int numError = 0;
+ // int numPosSamp = 0;
+ String[] feat_info;
+
+ for (int it = 0; it < maxIter; it++) {
+ System.out.print(it + " ");
+ numError = 0;
+ // numPosSamp = 0;
+
+ for (int s = 0; s < sampleSize; s++) {
+ featVal = samples.get(s).split("\\s+");
+
+ // only consider positive samples
+ // if( featVal[featDim].equals("1") )
+ // {
+ // numPosSamp++;
+ score = 0;
+ for (int d = 0; d < featVal.length - 1; d++) {
+ feat_info = featVal[d].split(":");
+ score += Double.parseDouble(feat_info[1]) * lambda[Integer.parseInt(feat_info[0])];
+ }
+
+ label = Double.parseDouble(featVal[featVal.length - 1]);
+ score *= label; // the last element is class label(+1/-1)
+
+ if (score <= bias) // incorrect classification
+ {
+ numError++;
+ for (int d = 0; d < featVal.length - 1; d++) {
+ feat_info = featVal[d].split(":");
+ int featID = Integer.parseInt(feat_info[0]);
+ lambda[featID] += learningRate * label * Double.parseDouble(feat_info[1]);
+ sum_lambda[featID] += lambda[featID];
+ }
+ }
+ // }//if( featVal[featDim].equals("1") )
+ }
+ if (numError == 0) break;
+ }
+
+ System.out.println("\n------- Average-perceptron training ends ------");
+
+ for (int i = 1; i <= featDim; i++)
+ sum_lambda[i] /= maxIter;
+
+ return sum_lambda;
+ }
+
+ @Override
+ /*
+ * for avg_perceptron: param[0] = maximum number of iterations param[1] = learning rate (step
+ * size) param[2] = bias (usually set to 0)
+ */
+ public void setClassifierParam(String[] param) {
+ if (param == null)
+ LOG.warn("no parameters specified for perceptron classifier, using default settings.");
+ else {
+ maxIter = Integer.parseInt(param[0]);
+ learningRate = Double.parseDouble(param[1]);
+ bias = Double.parseDouble(param[2]);
+ }
+ }
+
+ int maxIter = 20;
+ double learningRate = 0.5;
+ double bias = 0.0;
+}
[39/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/DefaultNGramLanguageModel.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/DefaultNGramLanguageModel.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/DefaultNGramLanguageModel.java
new file mode 100644
index 0000000..d5cf8e9
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/DefaultNGramLanguageModel.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.lm;
+
+import java.util.Arrays;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class provides a default implementation for the Equivalent LM State optimization (namely,
+ * don't back off anywhere). It also provides some default implementations for more general
+ * functions on the interface to fall back to more specific ones (e.g. from {@link java.util.ArrayList}
+ * of {@link java.lang.Integer}'s to int[]) and a default implementation for sentenceLogProbability
+ * which enumerates the n-grams and calls calls ngramLogProbability for each of them.
+ *
+ * @author Zhifei Li, zhifei.work@gmail.com
+ * @author wren ng thornton wren@users.sourceforge.net
+ */
+public abstract class DefaultNGramLanguageModel implements NGramLanguageModel {
+
+ private static final Logger LOG = LoggerFactory.getLogger(DefaultNGramLanguageModel.class);
+
+ protected final int ngramOrder;
+
+ protected float ceiling_cost = -100;
+
+ // ===============================================================
+ // Constructors
+ // ===============================================================
+ public DefaultNGramLanguageModel(int order, float ceiling_cost) {
+ this.ngramOrder = order;
+ this.ceiling_cost = ceiling_cost;
+ }
+
+ public DefaultNGramLanguageModel(int order) {
+ this.ngramOrder = order;
+ }
+
+
+ // ===============================================================
+ // Attributes
+ // ===============================================================
+ @Override
+ public final int getOrder() {
+ return this.ngramOrder;
+ }
+
+
+ // ===============================================================
+ // NGramLanguageModel Methods
+ // ===============================================================
+
+ @Override
+ public boolean registerWord(String token, int id) {
+ // No private LM ID mapping, do nothing
+ return false;
+ }
+
+ @Override
+ public float sentenceLogProbability(int[] sentence, int order, int startIndex) {
+ if (sentence == null) return 0.0f;
+ int sentenceLength = sentence.length;
+ if (sentenceLength <= 0) return 0.0f;
+
+ float probability = 0.0f;
+ // partial ngrams at the beginning
+ for (int j = startIndex; j < order && j <= sentenceLength; j++) {
+ // TODO: startIndex dependents on the order, e.g., this.ngramOrder-1 (in srilm, for 3-gram lm,
+ // start_index=2. othercase, need to check)
+ int[] ngram = Arrays.copyOfRange(sentence, 0, j);
+ double logProb = ngramLogProbability(ngram, order);
+
+ LOG.debug("\tlogp ({}) = {}", Vocabulary.getWords(ngram), logProb);
+ probability += logProb;
+ }
+
+ // regular-order ngrams
+ for (int i = 0; i <= sentenceLength - order; i++) {
+ int[] ngram = Arrays.copyOfRange(sentence, i, i + order);
+ double logProb = ngramLogProbability(ngram, order);
+ LOG.debug("\tlogp ({}) = {} ", Vocabulary.getWords(ngram), logProb);
+ probability += logProb;
+ }
+
+ return probability;
+ }
+
+ @Override
+ public float ngramLogProbability(int[] ngram) {
+ return this.ngramLogProbability(ngram, this.ngramOrder);
+ }
+
+ protected abstract float ngramLogProbability_helper(int[] ngram, int order);
+
+ @Override
+ public float ngramLogProbability(int[] ngram, int order) {
+ if (ngram.length > order) {
+ throw new RuntimeException("ngram length is greather than the max order");
+ }
+ // if (ngram.length==1 && "we".equals(Vocabulary.getWord(ngram[0]))) {
+ // System.err.println("Something weird is about to happen");
+ // }
+
+ int historySize = ngram.length - 1;
+ if (historySize >= order || historySize < 0) {
+ // BUG: use logger or exception. Don't zero default
+ throw new RuntimeException("Error: history size is " + historySize);
+ // return 0;
+ }
+ float probability = ngramLogProbability_helper(ngram, order);
+ if (probability < ceiling_cost) {
+ probability = ceiling_cost;
+ }
+ return probability;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/KenLM.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/KenLM.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/KenLM.java
new file mode 100644
index 0000000..0d45879
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/KenLM.java
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.lm;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.ff.lm.NGramLanguageModel;
+import org.apache.joshua.decoder.ff.state_maintenance.KenLMState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * JNI wrapper for KenLM. This version of KenLM supports two use cases, implemented by the separate
+ * feature functions KenLMFF and LanguageModelFF. KenLMFF uses the RuleScore() interface in
+ * lm/left.hh, returning a state pointer representing the KenLM state, while LangaugeModelFF handles
+ * state by itself and just passes in the ngrams for scoring.
+ *
+ * @author Kenneth Heafield
+ * @author Matt Post post@cs.jhu.edu
+ */
+
+public class KenLM implements NGramLanguageModel, Comparable<KenLM> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(KenLM.class);
+
+ static {
+ try {
+ System.loadLibrary("ken");
+ } catch (UnsatisfiedLinkError e) {
+ //TODO: send these prints to LOG.err
+ LOG.error("* FATAL: Can't find libken.so (libken.dylib on OS X) in $JOSHUA/lib");
+ LOG.error("* This probably means that the KenLM library didn't compile.");
+ LOG.error("* Make sure that BOOST_ROOT is set to the root of your boost");
+ LOG.error("* installation (it's not /opt/local/, the default), change to");
+ LOG.error("* $JOSHUA, and type 'ant kenlm'. If problems persist, see the");
+ LOG.error("* website (joshua-decoder.org)."); //FIXME: update link to newer url
+ throw new RuntimeException(e);
+ }
+ }
+
+ private final long pointer;
+
+ // this is read from the config file, used to set maximum order
+ private final int ngramOrder;
+ // inferred from model file (may be larger than ngramOrder)
+ private final int N;
+ // whether left-state minimization was requested
+ private boolean minimizing;
+
+ private final static native long construct(String file_name);
+
+ private final static native void destroy(long ptr);
+
+ private final static native int order(long ptr);
+
+ private final static native boolean registerWord(long ptr, String word, int id);
+
+ private final static native float prob(long ptr, int words[]);
+
+ private final static native float probForString(long ptr, String[] words);
+
+ private final static native boolean isKnownWord(long ptr, String word);
+
+ private final static native StateProbPair probRule(long ptr, long pool, long words[]);
+
+ private final static native float estimateRule(long ptr, long words[]);
+
+ private final static native float probString(long ptr, int words[], int start);
+
+ public final static native long createPool();
+ public final static native void destroyPool(long pointer);
+
+ public KenLM(int order, String file_name) {
+ ngramOrder = order;
+
+ pointer = construct(file_name);
+ N = order(pointer);
+ }
+
+ /**
+ * Constructor if order is not known.
+ * Order will be inferred from the model.
+ * @param file_name string path to an input file
+ */
+ public KenLM(String file_name) {
+ pointer = construct(file_name);
+ N = order(pointer);
+ ngramOrder = N;
+ }
+
+ public void destroy() {
+ destroy(pointer);
+ }
+
+ public int getOrder() {
+ return ngramOrder;
+ }
+
+ public boolean registerWord(String word, int id) {
+ return registerWord(pointer, word, id);
+ }
+
+ public float prob(int[] words) {
+ return prob(pointer, words);
+ }
+
+ /**
+ * Query for n-gram probability using strings.
+ * @param words a string array of words
+ * @return float value denoting probability
+ */
+ public float prob(String[] words) {
+ return probForString(pointer, words);
+ }
+
+ // Apparently Zhifei starts some array indices at 1. Change to 0-indexing.
+ public float probString(int words[], int start) {
+ return probString(pointer, words, start - 1);
+ }
+
+ /**
+ * This function is the bridge to the interface in kenlm/lm/left.hh, which has KenLM score the
+ * whole rule. It takes an array of words and states retrieved from tail nodes (nonterminals in the
+ * rule). Nonterminals have a negative value so KenLM can distinguish them. The sentence number is
+ * needed so KenLM knows which memory pool to use. When finished, it returns the updated KenLM
+ * state and the LM probability incurred along this rule.
+ *
+ * @param words array of words
+ * @param poolPointer todo
+ * @return the updated {@link org.apache.joshua.decoder.ff.lm.KenLM.StateProbPair} e.g.
+ * KenLM state and the LM probability incurred along this rule
+ */
+ public StateProbPair probRule(long[] words, long poolPointer) {
+
+ StateProbPair pair = null;
+ try {
+ pair = probRule(pointer, poolPointer, words);
+ } catch (NoSuchMethodError e) {
+ e.printStackTrace();
+ System.exit(1);
+ }
+
+ return pair;
+ }
+
+ /**
+ * Public facing function that estimates the cost of a rule, which value is used for sorting
+ * rules during cube pruning.
+ *
+ * @param words array of words
+ * @return the estimated cost of the rule (the (partial) n-gram probabilities of all words in the rule)
+ */
+ public float estimateRule(long[] words) {
+ float estimate = 0.0f;
+ try {
+ estimate = estimateRule(pointer, words);
+ } catch (NoSuchMethodError e) {
+ throw new RuntimeException(e);
+ }
+
+ return estimate;
+ }
+
+ /**
+ * The start symbol for a KenLM is the Vocabulary.START_SYM.
+ * @return "<s>"
+ */
+ public String getStartSymbol() {
+ return Vocabulary.START_SYM;
+ }
+
+ public boolean isKnownWord(String word) {
+ return isKnownWord(pointer, word);
+ }
+
+
+ /**
+ * Inner class used to hold the results returned from KenLM with left-state minimization. Note
+ * that inner classes have to be static to be accessible from the JNI!
+ */
+ public static class StateProbPair {
+ public KenLMState state = null;
+ public float prob = 0.0f;
+
+ public StateProbPair(long state, float prob) {
+ this.state = new KenLMState(state);
+ this.prob = prob;
+ }
+ }
+
+ @Override
+ public int compareTo(KenLM other) {
+ if (this == other)
+ return 0;
+ else
+ return -1;
+ }
+
+ /**
+ * These functions are used if KenLM is invoked under LanguageModelFF instead of KenLMFF.
+ */
+ @Override
+ public float sentenceLogProbability(int[] sentence, int order, int startIndex) {
+ return probString(sentence, startIndex);
+ }
+
+ @Override
+ public float ngramLogProbability(int[] ngram, int order) {
+ if (order != N && order != ngram.length)
+ throw new RuntimeException("Lower order not supported.");
+ return prob(ngram);
+ }
+
+ @Override
+ public float ngramLogProbability(int[] ngram) {
+ return prob(ngram);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/LanguageModelFF.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/LanguageModelFF.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/LanguageModelFF.java
new file mode 100644
index 0000000..3fea410
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/LanguageModelFF.java
@@ -0,0 +1,527 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.lm;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+
+import com.google.common.primitives.Ints;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.Support;
+import org.apache.joshua.decoder.chart_parser.SourcePath;
+import org.apache.joshua.decoder.ff.FeatureVector;
+import org.apache.joshua.decoder.ff.StatefulFF;
+import org.apache.joshua.decoder.ff.lm.berkeley_lm.LMGrammarBerkeley;
+import org.apache.joshua.decoder.ff.lm.KenLM;
+import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+import org.apache.joshua.decoder.ff.state_maintenance.NgramDPState;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.hypergraph.HGNode;
+import org.apache.joshua.decoder.segment_file.Sentence;
+import org.apache.joshua.util.FormatUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class performs the following:
+ * <ol>
+ * <li>Gets the additional LM score due to combinations of small items into larger ones by using
+ * rules</li>
+ * <li>Gets the LM state</li>
+ * <li>Gets the left-side LM state estimation score</li>
+ * </ol>
+ *
+ * @author Matt Post post@cs.jhu.edu
+ * @author Juri Ganitkevitch juri@cs.jhu.edu
+ * @author Zhifei Li, zhifei.work@gmail.com
+ */
+public class LanguageModelFF extends StatefulFF {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LanguageModelFF.class);
+
+ public static int LM_INDEX = 0;
+ private int startSymbolId;
+
+ /**
+ * N-gram language model. We assume the language model is in ARPA format for equivalent state:
+ *
+ * <ol>
+ * <li>We assume it is a backoff lm, and high-order ngram implies low-order ngram; absense of
+ * low-order ngram implies high-order ngram</li>
+ * <li>For a ngram, existence of backoffweight => existence a probability Two ways of dealing with
+ * low counts:
+ * <ul>
+ * <li>SRILM: don't multiply zeros in for unknown words</li>
+ * <li>Pharaoh: cap at a minimum score exp(-10), including unknown words</li>
+ * </ul>
+ * </li>
+ * </ol>
+ */
+ protected NGramLanguageModel languageModel;
+
+ /**
+ * We always use this order of ngram, though the LMGrammar may provide higher order probability.
+ */
+ protected final int ngramOrder;
+
+ /*
+ * We cache the weight of the feature since there is only one.
+ */
+ protected float weight;
+ protected String type;
+ protected String path;
+
+ /* Whether this is a class-based LM */
+ private boolean isClassLM;
+ private ClassMap classMap;
+
+ protected class ClassMap {
+
+ private final int OOV_id = Vocabulary.getUnknownId();
+ private HashMap<Integer, Integer> classMap;
+
+ public ClassMap(String file_name) throws IOException {
+ this.classMap = new HashMap<Integer, Integer>();
+ read(file_name);
+ }
+
+ public int getClassID(int wordID) {
+ return this.classMap.getOrDefault(wordID, OOV_id);
+ }
+
+ /**
+ * Reads a class map from file.
+ *
+ * @param file_name
+ * @throws IOException
+ */
+ private void read(String file_name) throws IOException {
+
+ int lineno = 0;
+ for (String line: new org.apache.joshua.util.io.LineReader(file_name, false)) {
+ lineno++;
+ String[] lineComp = line.trim().split("\\s+");
+ try {
+ this.classMap.put(Vocabulary.id(lineComp[0]), Vocabulary.id(lineComp[1]));
+ } catch (java.lang.ArrayIndexOutOfBoundsException e) {
+ LOG.warn("bad vocab line #{} '{}'", lineno, line);
+ LOG.warn(e.getMessage(), e);
+ }
+ }
+ }
+
+ }
+
+ public LanguageModelFF(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, String.format("lm_%d", LanguageModelFF.LM_INDEX++), args, config);
+
+ this.type = parsedArgs.get("lm_type");
+ this.ngramOrder = Integer.parseInt(parsedArgs.get("lm_order"));
+ this.path = parsedArgs.get("lm_file");
+
+ if (parsedArgs.containsKey("class_map"))
+ try {
+ this.isClassLM = true;
+ this.classMap = new ClassMap(parsedArgs.get("class_map"));
+ } catch (IOException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+
+ // The dense feature initialization hasn't happened yet, so we have to retrieve this as sparse
+ this.weight = weights.getSparse(name);
+
+ initializeLM();
+ }
+
+ @Override
+ public ArrayList<String> reportDenseFeatures(int index) {
+ denseFeatureIndex = index;
+
+ ArrayList<String> names = new ArrayList<String>();
+ names.add(name);
+ return names;
+ }
+
+ /**
+ * Initializes the underlying language model.
+ */
+ protected void initializeLM() {
+ if (type.equals("kenlm")) {
+ this.languageModel = new KenLM(ngramOrder, path);
+
+ } else if (type.equals("berkeleylm")) {
+ this.languageModel = new LMGrammarBerkeley(ngramOrder, path);
+
+ } else {
+ String msg = String.format("* FATAL: Invalid backend lm_type '%s' for LanguageModel", type)
+ + "* Permissible values for 'lm_type' are 'kenlm' and 'berkeleylm'";
+ throw new RuntimeException(msg);
+ }
+
+ Vocabulary.registerLanguageModel(this.languageModel);
+ Vocabulary.id(config.default_non_terminal);
+
+ startSymbolId = Vocabulary.id(Vocabulary.START_SYM);
+ }
+
+ public NGramLanguageModel getLM() {
+ return this.languageModel;
+ }
+
+ public String logString() {
+ if (languageModel != null)
+ return String.format("%s, order %d (weight %.3f)", name, languageModel.getOrder(), weight);
+ else
+ return "WHOA";
+ }
+
+ /**
+ * Computes the features incurred along this edge. Note that these features are unweighted costs
+ * of the feature; they are the feature cost, not the model cost, or the inner product of them.
+ */
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+
+ NgramDPState newState = null;
+ if (rule != null) {
+ if (config.source_annotations) {
+ // Get source side annotations and project them to the target side
+ newState = computeTransition(getTags(rule, i, j, sentence), tailNodes, acc);
+ }
+ else {
+ if (this.isClassLM) {
+ // Use a class language model
+ // Return target side classes
+ newState = computeTransition(getClasses(rule), tailNodes, acc);
+ }
+ else {
+ // Default LM
+ newState = computeTransition(rule.getEnglish(), tailNodes, acc);
+ }
+ }
+
+ }
+
+ return newState;
+ }
+
+ /**
+ * Input sentences can be tagged with information specific to the language model. This looks for
+ * such annotations by following a word's alignments back to the source words, checking for
+ * annotations, and replacing the surface word if such annotations are found.
+ * @param rule the {@link org.apache.joshua.decoder.ff.tm.Rule} to use
+ * @param begin todo
+ * @param end todo
+ * @param sentence {@link org.apache.joshua.lattice.Lattice} input
+ * @return todo
+ */
+ protected int[] getTags(Rule rule, int begin, int end, Sentence sentence) {
+ /* Very important to make a copy here, so the original rule is not modified */
+ int[] tokens = Arrays.copyOf(rule.getEnglish(), rule.getEnglish().length);
+ byte[] alignments = rule.getAlignment();
+
+ // System.err.println(String.format("getTags() %s", rule.getRuleString()));
+
+ /* For each target-side token, project it to each of its source-language alignments. If any of those
+ * are annotated, take the first annotation and quit.
+ */
+ if (alignments != null) {
+ for (int i = 0; i < tokens.length; i++) {
+ if (tokens[i] > 0) { // skip nonterminals
+ for (int j = 0; j < alignments.length; j += 2) {
+ if (alignments[j] == i) {
+ String annotation = sentence.getAnnotation((int)alignments[i] + begin, "class");
+ if (annotation != null) {
+ // System.err.println(String.format(" word %d source %d abs %d annotation %d/%s",
+ // i, alignments[i], alignments[i] + begin, annotation, Vocabulary.word(annotation)));
+ tokens[i] = Vocabulary.id(annotation);
+ break;
+ }
+ }
+ }
+ }
+ }
+ }
+
+ return tokens;
+ }
+
+ /**
+ * Sets the class map if this is a class LM
+ * @param fileName a string path to a file
+ * @throws IOException if there is an error reading the input file
+ */
+ public void setClassMap(String fileName) throws IOException {
+ this.classMap = new ClassMap(fileName);
+ }
+
+ /**
+ * Replace each word in a rule with the target side classes.
+ * @param rule {@link org.apache.joshua.decoder.ff.tm.Rule} to use when obtaining tokens
+ * @return int[] of tokens
+ */
+ protected int[] getClasses(Rule rule) {
+ if (this.classMap == null) {
+ throw new RuntimeException("The class map is not set. Cannot use the class LM ");
+ }
+ /* Very important to make a copy here, so the original rule is not modified */
+ int[] tokens = Arrays.copyOf(rule.getEnglish(), rule.getEnglish().length);
+ for (int i = 0; i < tokens.length; i++) {
+ if (tokens[i] > 0 ) {
+ tokens[i] = this.classMap.getClassID(tokens[i]);
+ }
+ }
+ return tokens;
+ }
+
+ @Override
+ public DPState computeFinal(HGNode tailNode, int i, int j, SourcePath sourcePath, Sentence sentence,
+ Accumulator acc) {
+ return computeFinalTransition((NgramDPState) tailNode.getDPState(stateIndex), acc);
+ }
+
+ /**
+ * This function computes all the complete n-grams found in the rule, as well as the incomplete
+ * n-grams on the left-hand side.
+ */
+ @Override
+ public float estimateCost(Rule rule, Sentence sentence) {
+
+ float estimate = 0.0f;
+ boolean considerIncompleteNgrams = true;
+
+ int[] enWords = rule.getEnglish();
+
+ List<Integer> words = new ArrayList<Integer>();
+ boolean skipStart = (enWords[0] == startSymbolId);
+
+ /*
+ * Move through the words, accumulating language model costs each time we have an n-gram (n >=
+ * 2), and resetting the series of words when we hit a nonterminal.
+ */
+ for (int c = 0; c < enWords.length; c++) {
+ int currentWord = enWords[c];
+ if (FormatUtils.isNonterminal(currentWord)) {
+ estimate += scoreChunkLogP(words, considerIncompleteNgrams, skipStart);
+ words.clear();
+ skipStart = false;
+ } else {
+ words.add(currentWord);
+ }
+ }
+ estimate += scoreChunkLogP(words, considerIncompleteNgrams, skipStart);
+
+ return weight * estimate;
+ }
+
+ /**
+ * Estimates the future cost of a rule. For the language model feature, this is the sum of the
+ * costs of the leftmost k-grams, k = [1..n-1].
+ */
+ @Override
+ public float estimateFutureCost(Rule rule, DPState currentState, Sentence sentence) {
+ NgramDPState state = (NgramDPState) currentState;
+
+ float estimate = 0.0f;
+ int[] leftContext = state.getLeftLMStateWords();
+
+ if (null != leftContext) {
+ boolean skipStart = true;
+ if (leftContext[0] != startSymbolId) {
+ skipStart = false;
+ }
+ estimate += scoreChunkLogP(leftContext, true, skipStart);
+ }
+ return weight * estimate;
+ }
+
+ /**
+ * Compute the cost of a rule application. The cost of applying a rule is computed by determining
+ * the n-gram costs for all n-grams created by this rule application, and summing them. N-grams
+ * are created when (a) terminal words in the rule string are followed by a nonterminal (b)
+ * terminal words in the rule string are preceded by a nonterminal (c) we encounter adjacent
+ * nonterminals. In all of these situations, the corresponding boundary words of the node in the
+ * hypergraph represented by the nonterminal must be retrieved.
+ *
+ * IMPORTANT: only complete n-grams are scored. This means that hypotheses with fewer words
+ * than the complete n-gram state remain *unscored*. This fact adds a lot of complication to the
+ * code, including the use of the computeFinal* family of functions, which correct this fact for
+ * sentences that are too short on the final transition.
+ */
+ private NgramDPState computeTransition(int[] enWords, List<HGNode> tailNodes, Accumulator acc) {
+
+ int[] current = new int[this.ngramOrder];
+ int[] shadow = new int[this.ngramOrder];
+ int ccount = 0;
+ float transitionLogP = 0.0f;
+ int[] left_context = null;
+
+ for (int c = 0; c < enWords.length; c++) {
+ int curID = enWords[c];
+
+ if (FormatUtils.isNonterminal(curID)) {
+ int index = -(curID + 1);
+
+ NgramDPState state = (NgramDPState) tailNodes.get(index).getDPState(stateIndex);
+ int[] left = state.getLeftLMStateWords();
+ int[] right = state.getRightLMStateWords();
+
+ // Left context.
+ for (int i = 0; i < left.length; i++) {
+ current[ccount++] = left[i];
+
+ if (left_context == null && ccount == this.ngramOrder - 1)
+ left_context = Arrays.copyOf(current, ccount);
+
+ if (ccount == this.ngramOrder) {
+ // Compute the current word probability, and remove it.
+ float prob = this.languageModel.ngramLogProbability(current, this.ngramOrder);
+ // System.err.println(String.format("-> prob(%s) = %f", Vocabulary.getWords(current), prob));
+ transitionLogP += prob;
+ System.arraycopy(current, 1, shadow, 0, this.ngramOrder - 1);
+ int[] tmp = current;
+ current = shadow;
+ shadow = tmp;
+ --ccount;
+ }
+ }
+ System.arraycopy(right, 0, current, ccount - right.length, right.length);
+ } else { // terminal words
+ current[ccount++] = curID;
+
+ if (left_context == null && ccount == this.ngramOrder - 1)
+ left_context = Arrays.copyOf(current, ccount);
+
+ if (ccount == this.ngramOrder) {
+ // Compute the current word probability, and remove it.s
+ float prob = this.languageModel.ngramLogProbability(current, this.ngramOrder);
+ // System.err.println(String.format("-> prob(%s) = %f", Vocabulary.getWords(current), prob));
+ transitionLogP += prob;
+ System.arraycopy(current, 1, shadow, 0, this.ngramOrder - 1);
+ int[] tmp = current;
+ current = shadow;
+ shadow = tmp;
+ --ccount;
+ }
+ }
+ }
+ // acc.add(name, transitionLogP);
+ acc.add(denseFeatureIndex, transitionLogP);
+
+ if (left_context != null) {
+ return new NgramDPState(left_context, Arrays.copyOfRange(current, ccount - this.ngramOrder
+ + 1, ccount));
+ } else {
+ int[] context = Arrays.copyOf(current, ccount);
+ return new NgramDPState(context, context);
+ }
+ }
+
+ /**
+ * This function differs from regular transitions because we incorporate the cost of incomplete
+ * left-hand ngrams, as well as including the start- and end-of-sentence markers (if they were
+ * requested when the object was created).
+ *
+ * @param state the dynamic programming state
+ * @return the final transition probability (including incomplete n-grams)
+ */
+ private NgramDPState computeFinalTransition(NgramDPState state, Accumulator acc) {
+
+ // System.err.println(String.format("LanguageModel::computeFinalTransition()"));
+
+ float res = 0.0f;
+ LinkedList<Integer> currentNgram = new LinkedList<Integer>();
+ int[] leftContext = state.getLeftLMStateWords();
+ int[] rightContext = state.getRightLMStateWords();
+
+ for (int i = 0; i < leftContext.length; i++) {
+ int t = leftContext[i];
+ currentNgram.add(t);
+
+ if (currentNgram.size() >= 2) { // start from bigram
+ float prob = this.languageModel.ngramLogProbability(Support.toArray(currentNgram),
+ currentNgram.size());
+ res += prob;
+ }
+ if (currentNgram.size() == this.ngramOrder)
+ currentNgram.removeFirst();
+ }
+
+ // Tell the accumulator
+ // acc.add(name, res);
+ acc.add(denseFeatureIndex, res);
+
+ // State is the same
+ return new NgramDPState(leftContext, rightContext);
+ }
+
+
+ /**
+ * Compatibility method for {@link #scoreChunkLogP(int[], boolean, boolean)}
+ */
+ private float scoreChunkLogP(List<Integer> words, boolean considerIncompleteNgrams,
+ boolean skipStart) {
+ return scoreChunkLogP(Ints.toArray(words), considerIncompleteNgrams, skipStart);
+ }
+
+ /**
+ * This function is basically a wrapper for NGramLanguageModel::sentenceLogProbability(). It
+ * computes the probability of a phrase ("chunk"), using lower-order n-grams for the first n-1
+ * words.
+ *
+ * @param words
+ * @param considerIncompleteNgrams
+ * @param skipStart
+ * @return the phrase log probability
+ */
+ private float scoreChunkLogP(int[] words, boolean considerIncompleteNgrams,
+ boolean skipStart) {
+
+ float score = 0.0f;
+ if (words.length > 0) {
+ int startIndex;
+ if (!considerIncompleteNgrams) {
+ startIndex = this.ngramOrder;
+ } else if (skipStart) {
+ startIndex = 2;
+ } else {
+ startIndex = 1;
+ }
+ score = this.languageModel.sentenceLogProbability(words, this.ngramOrder, startIndex);
+ }
+
+ return score;
+ }
+
+ /**
+ * Public method to set LM_INDEX back to 0.
+ * Required if multiple instances of the JoshuaDecoder live in the same JVM.
+ */
+ public static void resetLmIndex() {
+ LM_INDEX = 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/NGramLanguageModel.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/NGramLanguageModel.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/NGramLanguageModel.java
new file mode 100644
index 0000000..882424b
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/NGramLanguageModel.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.lm;
+
+/**
+ * An interface for new language models to implement. An object of this type is passed to
+ * LanguageModelFF, which will handle all the dynamic programming and state maintenance.
+ *
+ * @author wren ng thornton wren@users.sourceforge.net
+ * @author Zhifei Li, zhifei.work@gmail.com
+ * @author Matt Post post@cs.jhu.edu
+ * @author Juri Ganitkevitch juri@cs.jhu.edu
+ */
+public interface NGramLanguageModel {
+
+ // ===============================================================
+ // Attributes
+ // ===============================================================
+ int getOrder();
+
+ // ===============================================================
+ // Methods
+ // ===============================================================
+
+ /**
+ * Language models may have their own private vocabulary mapping strings to integers; for example,
+ * if they make use of a compile format (as KenLM and BerkeleyLM do). This mapping is likely
+ * different from the global mapping containing in {@link org.apache.joshua.corpus.Vocabulary}, which is used to
+ * convert the input string and grammars. This function is used to tell the language model what
+ * the global mapping is, so that the language model can convert it into its own private mapping.
+ *
+ * @param token string token to be registered
+ * @param id to associate with this word
+ * @return Whether any collisions were detected.
+ */
+ boolean registerWord(String token, int id);
+
+ /**
+ * @param sentence the sentence to be scored
+ * @param order the order of N-grams for the LM
+ * @param startIndex the index of first event-word we want to get its probability; if we want to
+ * get the prob for the whole sentence, then startIndex should be 1
+ * @return the LogP of the whole sentence
+ */
+ float sentenceLogProbability(int[] sentence, int order, int startIndex);
+
+ /**
+ * Compute the probability of a single word given its context.
+ *
+ * @param ngram the NGram for which we wish to compute the probability
+ * @param order NGram order/context
+ * @return float representing the probability
+ */
+ float ngramLogProbability(int[] ngram, int order);
+
+ float ngramLogProbability(int[] ngram);
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/StateMinimizingLanguageModel.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/StateMinimizingLanguageModel.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/StateMinimizingLanguageModel.java
new file mode 100644
index 0000000..533365c
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/StateMinimizingLanguageModel.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.lm;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.chart_parser.SourcePath;
+import org.apache.joshua.decoder.ff.FeatureVector;
+import org.apache.joshua.decoder.ff.lm.KenLM;
+import org.apache.joshua.decoder.ff.lm.KenLM.StateProbPair;
+import org.apache.joshua.decoder.ff.state_maintenance.DPState;
+import org.apache.joshua.decoder.ff.state_maintenance.KenLMState;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.hypergraph.HGNode;
+import org.apache.joshua.decoder.segment_file.Sentence;
+import org.apache.joshua.util.FormatUtils;
+
+/**
+ * Wrapper for KenLM LMs with left-state minimization. We inherit from the regular
+ *
+ * @author Matt Post post@cs.jhu.edu
+ * @author Juri Ganitkevitch juri@cs.jhu.edu
+ */
+public class StateMinimizingLanguageModel extends LanguageModelFF {
+
+ // maps from sentence numbers to KenLM-side pools used to allocate state
+ private static final ConcurrentHashMap<Integer, Long> poolMap = new ConcurrentHashMap<Integer, Long>();
+
+ public StateMinimizingLanguageModel(FeatureVector weights, String[] args, JoshuaConfiguration config) {
+ super(weights, args, config);
+ this.type = "kenlm";
+ if (parsedArgs.containsKey("lm_type") && ! parsedArgs.get("lm_type").equals("kenlm")) {
+ String msg = "* FATAL: StateMinimizingLanguageModel only supports 'kenlm' lm_type backend"
+ + "* Remove lm_type from line or set to 'kenlm'";
+ throw new RuntimeException(msg);
+ }
+ }
+
+ @Override
+ public ArrayList<String> reportDenseFeatures(int index) {
+ denseFeatureIndex = index;
+
+ ArrayList<String> names = new ArrayList<String>();
+ names.add(name);
+ return names;
+ }
+
+ /**
+ * Initializes the underlying language model.
+ */
+ @Override
+ public void initializeLM() {
+
+ // Override type (only KenLM supports left-state minimization)
+ this.languageModel = new KenLM(ngramOrder, path);
+
+ Vocabulary.registerLanguageModel(this.languageModel);
+ Vocabulary.id(config.default_non_terminal);
+
+ }
+
+ /**
+ * Estimates the cost of a rule. We override here since KenLM can do it more efficiently
+ * than the default {@link LanguageModelFF} class.
+ *
+ * Most of this function implementation is redundant with compute().
+ */
+ @Override
+ public float estimateCost(Rule rule, Sentence sentence) {
+
+ int[] ruleWords = rule.getEnglish();
+
+ // The IDs we'll pass to KenLM
+ long[] words = new long[ruleWords.length];
+
+ for (int x = 0; x < ruleWords.length; x++) {
+ int id = ruleWords[x];
+
+ if (FormatUtils.isNonterminal(id)) {
+ // For the estimate, we can just mark negative values
+ words[x] = -1;
+
+ } else {
+ // Terminal: just add it
+ words[x] = id;
+ }
+ }
+
+ // Get the probability of applying the rule and the new state
+ return weight * ((KenLM) languageModel).estimateRule(words);
+ }
+
+ /**
+ * Computes the features incurred along this edge. Note that these features are unweighted costs
+ * of the feature; they are the feature cost, not the model cost, or the inner product of them.
+ */
+ @Override
+ public DPState compute(Rule rule, List<HGNode> tailNodes, int i, int j, SourcePath sourcePath,
+ Sentence sentence, Accumulator acc) {
+
+ int[] ruleWords = config.source_annotations
+ ? getTags(rule, i, j, sentence)
+ : rule.getEnglish();
+
+ // The IDs we'll pass to KenLM
+ long[] words = new long[ruleWords.length];
+
+ for (int x = 0; x < ruleWords.length; x++) {
+ int id = ruleWords[x];
+
+ if (FormatUtils.isNonterminal(id)) {
+ // Nonterminal: retrieve the KenLM long that records the state
+ int index = -(id + 1);
+ KenLMState state = (KenLMState) tailNodes.get(index).getDPState(stateIndex);
+ words[x] = -state.getState();
+
+ } else {
+ // Terminal: just add it
+ words[x] = id;
+ }
+ }
+
+ int sentID = sentence.id();
+ // Since sentId is unique across threads, next operations are safe, but not atomic!
+ if (!poolMap.containsKey(sentID)) {
+ poolMap.put(sentID, KenLM.createPool());
+ }
+
+ // Get the probability of applying the rule and the new state
+ StateProbPair pair = ((KenLM) languageModel).probRule(words, poolMap.get(sentID));
+
+ // Record the prob
+// acc.add(name, pair.prob);
+ acc.add(denseFeatureIndex, pair.prob);
+
+ // Return the state
+ return pair.state;
+ }
+
+ /**
+ * Destroys the pool created to allocate state for this sentence. Called from the
+ * {@link org.apache.joshua.decoder.Translation} class after outputting the sentence or k-best list. Hosting
+ * this map here in KenLMFF statically allows pools to be shared across KenLM instances.
+ *
+ * @param sentId a key in the poolmap table to destroy
+ */
+ public void destroyPool(int sentId) {
+ if (poolMap.containsKey(sentId))
+ KenLM.destroyPool(poolMap.get(sentId));
+ poolMap.remove(sentId);
+ }
+
+ /**
+ * This function differs from regular transitions because we incorporate the cost of incomplete
+ * left-hand ngrams, as well as including the start- and end-of-sentence markers (if they were
+ * requested when the object was created).
+ *
+ * KenLM already includes the prefix probabilities (of shorter n-grams on the left-hand side), so
+ * there's nothing that needs to be done.
+ */
+ @Override
+ public DPState computeFinal(HGNode tailNode, int i, int j, SourcePath sourcePath, Sentence sentence,
+ Accumulator acc) {
+
+ // KenLMState state = (KenLMState) tailNode.getDPState(getStateIndex());
+
+ // This is unnecessary
+ // acc.add(name, 0.0f);
+
+ // The state is the same since no rule was applied
+ return new KenLMState();
+ }
+
+ /**
+ * KenLM probs already include the prefix probabilities (they are substracted out when merging
+ * states), so this doesn't need to do anything.
+ */
+ @Override
+ public float estimateFutureCost(Rule rule, DPState currentState, Sentence sentence) {
+ return 0.0f;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/LICENSE
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/LICENSE b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/LICENSE
new file mode 100644
index 0000000..2aaeb08
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/LICENSE
@@ -0,0 +1,13 @@
+Copyright 2013 University of California, Berkeley
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/LMGrammarBerkeley.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/LMGrammarBerkeley.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/LMGrammarBerkeley.java
new file mode 100644
index 0000000..8e54534
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/LMGrammarBerkeley.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.lm.berkeley_lm;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.logging.Handler;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.ff.lm.DefaultNGramLanguageModel;
+
+import com.google.common.annotations.VisibleForTesting;
+
+import edu.berkeley.nlp.lm.ArrayEncodedNgramLanguageModel;
+import edu.berkeley.nlp.lm.ConfigOptions;
+import edu.berkeley.nlp.lm.StringWordIndexer;
+import edu.berkeley.nlp.lm.WordIndexer;
+import edu.berkeley.nlp.lm.cache.ArrayEncodedCachingLmWrapper;
+import edu.berkeley.nlp.lm.io.LmReaders;
+import edu.berkeley.nlp.lm.util.StrUtils;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class wraps Berkeley LM.
+ *
+ * @author adpauls@gmail.com
+ */
+public class LMGrammarBerkeley extends DefaultNGramLanguageModel {
+
+ public static final org.slf4j.Logger LOG = LoggerFactory.getLogger(LMGrammarBerkeley.class);
+
+ private ArrayEncodedNgramLanguageModel<String> lm;
+
+ private static final Logger logger = Logger.getLogger(LMGrammarBerkeley.class.getName());
+
+ private int[] vocabIdToMyIdMapping;
+
+ private ThreadLocal<int[]> arrayScratch = new ThreadLocal<int[]>() {
+
+ @Override
+ protected int[] initialValue() {
+ return new int[5];
+ }
+ };
+
+ private int mappingLength = 0;
+
+ private final int unkIndex;
+
+ private static boolean logRequests = false;
+
+ private static Handler logHandler = null;
+
+ public LMGrammarBerkeley(int order, String lm_file) {
+ super(order);
+ vocabIdToMyIdMapping = new int[10];
+
+ if (!new File(lm_file).exists()) {
+ throw new RuntimeException("Can't read lm_file '" + lm_file + "'");
+ }
+
+ if (logRequests) {
+ logger.addHandler(logHandler);
+ logger.setLevel(Level.FINEST);
+ logger.setUseParentHandlers(false);
+ }
+
+ try { // try binary format (even gzipped)
+ lm = (ArrayEncodedNgramLanguageModel<String>) LmReaders.<String>readLmBinary(lm_file);
+ LOG.info("Loading Berkeley LM from binary {}", lm_file);
+ } catch (RuntimeException e) {
+ ConfigOptions opts = new ConfigOptions();
+ LOG.info("Loading Berkeley LM from ARPA file {}", lm_file);
+ final StringWordIndexer wordIndexer = new StringWordIndexer();
+ ArrayEncodedNgramLanguageModel<String> berkeleyLm =
+ LmReaders.readArrayEncodedLmFromArpa(lm_file, false, wordIndexer, opts, order);
+
+ lm = ArrayEncodedCachingLmWrapper.wrapWithCacheThreadSafe(berkeleyLm);
+ }
+ this.unkIndex = lm.getWordIndexer().getOrAddIndex(lm.getWordIndexer().getUnkSymbol());
+ }
+
+ @Override
+ public boolean registerWord(String token, int id) {
+ int myid = lm.getWordIndexer().getIndexPossiblyUnk(token);
+ if (myid < 0) return false;
+ if (id >= vocabIdToMyIdMapping.length) {
+ vocabIdToMyIdMapping =
+ Arrays.copyOf(vocabIdToMyIdMapping, Math.max(id + 1, vocabIdToMyIdMapping.length * 2));
+
+ }
+ mappingLength = Math.max(mappingLength, id + 1);
+ vocabIdToMyIdMapping[id] = myid;
+
+ return false;
+ }
+
+ @Override
+ public float sentenceLogProbability(int[] sentence, int order, int startIndex) {
+ if (sentence == null) return 0;
+ int sentenceLength = sentence.length;
+ if (sentenceLength <= 0) return 0;
+
+ float probability = 0;
+ // partial ngrams at the begining
+ for (int j = startIndex; j < order && j <= sentenceLength; j++) {
+ // TODO: startIndex dependens on the order, e.g., this.ngramOrder-1 (in srilm, for 3-gram lm,
+ // start_index=2. othercase, need to check)
+ double logProb = ngramLogProbability_helper(sentence, 0, j, false);
+ if (logger.isLoggable(Level.FINE)) {
+ int[] ngram = Arrays.copyOfRange(sentence, 0, j);
+ String words = Vocabulary.getWords(ngram);
+ logger.fine("\tlogp ( " + words + " ) = " + logProb);
+ }
+ probability += logProb;
+ }
+
+ // regular-order ngrams
+ for (int i = 0; i <= sentenceLength - order; i++) {
+ double logProb = ngramLogProbability_helper(sentence, i, order, false);
+ if (logger.isLoggable(Level.FINE)) {
+ int[] ngram = Arrays.copyOfRange(sentence, i, i + order);
+ String words = Vocabulary.getWords(ngram);
+ logger.fine("\tlogp ( " + words + " ) = " + logProb);
+ }
+ probability += logProb;
+ }
+
+ return probability;
+ }
+
+ @Override
+ public float ngramLogProbability_helper(int[] ngram, int order) {
+ return ngramLogProbability_helper(ngram, false);
+ }
+
+ protected float ngramLogProbability_helper(int[] ngram, boolean log) {
+ return ngramLogProbability_helper(ngram, 0, ngram.length, log);
+ }
+
+ protected float ngramLogProbability_helper(int sentence[], int ngramStartPos, int ngramLength, boolean log) {
+ int[] mappedNgram = arrayScratch.get();
+ if (mappedNgram.length < ngramLength) {
+ mappedNgram = new int[mappedNgram.length * 2];
+ arrayScratch.set(mappedNgram);
+ }
+ for (int i = 0; i < ngramLength; ++i) {
+ mappedNgram[i] = vocabIdToMyIdMapping[sentence[ngramStartPos + i]];
+ }
+
+ if (log && logRequests) {
+ dumpBuffer(mappedNgram, ngramLength);
+ }
+
+ return lm.getLogProb(mappedNgram, 0, ngramLength);
+ }
+
+ public static void setLogRequests(Handler handler) {
+ logRequests = true;
+ logHandler = handler;
+ }
+
+ @Override
+ public float ngramLogProbability(int[] ngram) {
+ return ngramLogProbability_helper(ngram,true);
+ }
+
+ @Override
+ public float ngramLogProbability(int[] ngram, int order) {
+ return ngramLogProbability(ngram);
+ }
+
+ private void dumpBuffer(int[] buffer, int len) {
+ final int[] copyOf = Arrays.copyOf(buffer, len);
+ for (int i = 0; i < copyOf.length; ++i) {
+ if (copyOf[i] < 0) {
+ copyOf[i] = unkIndex;
+ }
+ }
+ logger.finest(StrUtils.join(WordIndexer.StaticMethods.toList(lm.getWordIndexer(), copyOf)));
+ }
+
+ @VisibleForTesting
+ ArrayEncodedNgramLanguageModel<String> getLM() {
+ return lm;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/README
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/README b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/README
new file mode 100644
index 0000000..82bb473
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/README
@@ -0,0 +1,5 @@
+To build a binary for Berkeley LM, you need to do the following:
+
+java -cp [berkelylm jar file] -server -mx[lots of memory] edu.berkeley.nlp.lm.io.MakeLmBinaryFromArpa [ARPA file] [output file]
+
+Both input and output will be appropriately GZipped if they have a .gz extension. Note that MakeLmBinaryFromArpa has options for e.g. enabling compression.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/SymbolTableWrapper.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/SymbolTableWrapper.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/SymbolTableWrapper.java
new file mode 100644
index 0000000..e22e6d1
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/berkeley_lm/SymbolTableWrapper.java
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.lm.berkeley_lm;
+
+import org.apache.joshua.corpus.Vocabulary;
+import edu.berkeley.nlp.lm.WordIndexer;
+
+class SymbolTableWrapper implements WordIndexer<String> {
+ /**
+ *
+ */
+ private static final long serialVersionUID = 1L;
+
+ private String startSymbol;
+
+ private String endSymbol;
+
+ private String unkSymbol;
+
+ int size = -1;
+
+ public SymbolTableWrapper() {
+
+ }
+
+ @Override
+ public int getOrAddIndex(String word) {
+ return Vocabulary.id(word);
+ }
+
+ @Override
+ public int getOrAddIndexFromString(String word) {
+ return Vocabulary.id(word);
+ }
+
+ @Override
+ public String getWord(int index) {
+ return Vocabulary.word(index);
+ }
+
+ @Override
+ public int numWords() {
+ return Vocabulary.size();
+ }
+
+ @Override
+ public String getStartSymbol() {
+ return startSymbol;
+ }
+
+ @Override
+ public String getEndSymbol() {
+ return endSymbol;
+ }
+
+ @Override
+ public String getUnkSymbol() {
+ return unkSymbol;
+ }
+
+ @Override
+ public void setStartSymbol(String sym) {
+ startSymbol = sym;
+ }
+
+ @Override
+ public void setEndSymbol(String sym) {
+ endSymbol = sym;
+ }
+
+ @Override
+ public void setUnkSymbol(String sym) {
+ unkSymbol = sym;
+ }
+
+ @Override
+ public void trimAndLock() {
+
+ }
+
+ @Override
+ public int getIndexPossiblyUnk(String word) {
+ return Vocabulary.id(word);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/bloomfilter_lm/BloomFilter.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/bloomfilter_lm/BloomFilter.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/bloomfilter_lm/BloomFilter.java
new file mode 100644
index 0000000..a66fa44
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/bloomfilter_lm/BloomFilter.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.lm.bloomfilter_lm;
+
+import java.io.Externalizable;
+import java.io.IOException;
+import java.io.ObjectInput;
+import java.io.ObjectOutput;
+import java.math.BigInteger;
+import java.util.BitSet;
+import java.util.Random;
+
+/**
+ * A Bloom filter: a lossy data structure for set representation. A Bloom filter consists of a bit
+ * set and a set of hash functions. A Bloom filter has two operations: add and query. We can add an
+ * object to a Bloom filter to indicate that it should be considered part of the set that the Bloom
+ * filter represents. We can query the Bloom filter to see if a given object is considered part of
+ * its set.
+ * <p>
+ * An object is added by sending it through a number of hash functions, each of which returns an
+ * index into the bit set. The bit at each of the indices is flipped on. We can query for an abject
+ * by sending it through the same hash functions. Then we look the bit at each index that was
+ * returned by a hash function. If any of the bits is unset, we know that the object is not in the
+ * Bloom filter (for otherwise all the bits should have already been set). If all the bits are set,
+ * we assume that the object is present in the Bloom filter.
+ * <p>
+ * We cannot know for sure that an object is in the bloom filter just because all its bits were set.
+ * There may be many collisions in the hash space, and all the bits for some object might be set by
+ * chance, rather than by adding that particular object.
+ * <p>
+ * The advantage of a Bloom filter is that its set representation can be stored in a significantly
+ * smaller space than information-theoretic lossless lower bounds. The price we pay for this is a
+ * certain amount of error in the query function. One nice feature of the Bloom filter is that its
+ * error is one-sided. This means that while the query function may return false positives (saying
+ * an object is present when it really isn't), it can never return false negatives (saying that an
+ * object is not present when it was already added.
+ */
+public class BloomFilter implements Externalizable {
+ /**
+ * The main bit set of the Bloom filter.
+ */
+ private BitSet bitSet;
+
+ /**
+ * The number of objects expected to be stored in the Bloom filter. The optimal number of hash
+ * functions depends on this number.
+ */
+ int expectedNumberOfObjects;
+
+ /**
+ * A prime number that should be bigger than the size of the bit set.
+ */
+ long bigPrime;
+
+ /**
+ * The size of the bit set, in bits.
+ */
+ int filterSize;
+
+ /**
+ * A random number generator for building hash functions.
+ */
+ transient private Random RANDOM = new Random();
+
+ /**
+ * Builds an empty Bloom filter, ready to build hash functions and store objects.
+ *
+ * @param filterSize the size of Bloom filter to make, in bits
+ * @param expectedNumberOfObjects the number of objects expected to be stored in the Bloom filter
+ */
+ public BloomFilter(int filterSize, int expectedNumberOfObjects) {
+ bitSet = new BitSet(filterSize);
+ this.filterSize = filterSize;
+ this.expectedNumberOfObjects = expectedNumberOfObjects;
+ bigPrime = getPrimeLargerThan(filterSize);
+ }
+
+ /**
+ * Adds an item (represented by an integer) to the bloom filter.
+ *
+ * @param objectToAdd the object to add
+ * @param hashFunctions an array of pairs of long, representing the hash functions to be used on
+ * the object
+ */
+ public void add(int objectToAdd, long[][] hashFunctions) {
+ for (long[] h : hashFunctions) {
+ int i = hash(h, (long) objectToAdd);
+ bitSet.set(i);
+ }
+ }
+
+ public void add(long objectToAdd, long[][] hashFunctions) {
+ for (long[] h : hashFunctions) {
+ int i = hash(h, objectToAdd);
+ bitSet.set(i);
+ }
+ }
+
+ /**
+ * Determines whether an item (represented by an integer) is present in the bloom filter.
+ *
+ * @param objectToQuery the object we want to query for membership
+ * @param hashFunctions an array of pairs of long, representing the hash functions to be used
+ *
+ * @return true if the objects is assumed to be present in the Bloom filter, false if it is
+ * definitely not present
+ */
+ public boolean query(int objectToQuery, long[][] hashFunctions) {
+ for (long[] h : hashFunctions) {
+ int i = hash(h, (long) objectToQuery);
+ if (!bitSet.get(i)) return false;
+ }
+ return true;
+ }
+
+ public boolean query(long objectToQuery, long[][] hashFunctions) {
+ for (long[] h : hashFunctions) {
+ int i = hash(h, objectToQuery);
+ if (!bitSet.get(i)) return false;
+ }
+ return true;
+ }
+
+ /**
+ * Builds an array of pairs of long that can be used as hash functions for this Bloom filter.
+ *
+ * @return an array of pairs of long suitable for use as hash functions
+ */
+ public long[][] initializeHashFunctions() {
+ int numberOfHashFunctions;
+ int bigPrimeInt = (int) bigPrime;
+ numberOfHashFunctions =
+ (int) Math.floor(Math.log(2) * bitSet.length() / expectedNumberOfObjects);
+ if (numberOfHashFunctions == 0) numberOfHashFunctions = 1;
+ long[][] hashFunctions = new long[numberOfHashFunctions][2];
+ for (long[] h : hashFunctions) {
+ h[0] = (long) RANDOM.nextInt(bigPrimeInt) + 1;
+ h[1] = (long) RANDOM.nextInt(bigPrimeInt) + 1;
+ }
+ return hashFunctions;
+ }
+
+ /**
+ * Determines which bit of the bit set should be either set, for add operations, or checked, for
+ * query operations.
+ *
+ * @param h a length-2 array of long used as a hash function
+ * @param objectToHash the object of interest
+ *
+ * @return an index into the bit set of the Bloom filter
+ */
+ private int hash(long[] h, long objectToHash) {
+ long obj = (objectToHash < Integer.MAX_VALUE) ? objectToHash : objectToHash - bigPrime;
+ long h0 = h[0];
+ long h1 = (h[1] < (Long.MAX_VALUE / 2)) ? h[1] : h[1] - bigPrime;
+ long ret = (obj * h0) % bigPrime;
+ ret = (ret < (Long.MAX_VALUE / 2)) ? ret : ret - bigPrime;
+ return (int) (((ret + h1) % bigPrime) % (long) filterSize);
+ }
+
+ /**
+ * Finds a prime number that is larger than the given number. This is used to find bigPrime, a
+ * prime that has to be larger than the size of the Bloom filter.
+ *
+ * @param n an integer
+ *
+ * @return a prime number larger than n
+ */
+ private long getPrimeLargerThan(int n) {
+ BigInteger ret;
+ BigInteger maxLong = BigInteger.valueOf(Long.MAX_VALUE);
+ int numBits = BigInteger.valueOf(n).bitLength() + 1;
+ do {
+ ret = BigInteger.probablePrime(numBits, RANDOM);
+ } while (ret.compareTo(maxLong) > 1);
+ return ret.longValue();
+ }
+
+ /*
+ * functions for interface externalizable
+ */
+
+ public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ expectedNumberOfObjects = in.readInt();
+ filterSize = in.readInt();
+ bigPrime = in.readLong();
+ bitSet = (BitSet) in.readObject();
+ }
+
+ public void writeExternal(ObjectOutput out) throws IOException {
+ out.writeInt(expectedNumberOfObjects);
+ out.writeInt(filterSize);
+ out.writeLong(bigPrime);
+ out.writeObject(bitSet);
+ }
+
+ // only used for reconstruction via Externalizable
+ public BloomFilter() {}
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/bloomfilter_lm/BloomFilterLanguageModel.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/bloomfilter_lm/BloomFilterLanguageModel.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/bloomfilter_lm/BloomFilterLanguageModel.java
new file mode 100644
index 0000000..90d90b8
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/bloomfilter_lm/BloomFilterLanguageModel.java
@@ -0,0 +1,560 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.ff.lm.bloomfilter_lm;
+
+import java.io.Externalizable;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.ObjectInput;
+import java.io.ObjectInputStream;
+import java.io.ObjectOutput;
+import java.io.ObjectOutputStream;
+import java.util.HashMap;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.apache.joshua.decoder.ff.lm.DefaultNGramLanguageModel;
+import org.apache.joshua.util.Regex;
+import org.apache.joshua.util.io.LineReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * An n-gram language model with linearly-interpolated Witten-Bell smoothing, using a Bloom filter
+ * as its main data structure. A Bloom filter is a lossy data structure that can be used to test for
+ * set membership.
+ */
+public class BloomFilterLanguageModel extends DefaultNGramLanguageModel implements Externalizable {
+ /**
+ * An initial value used for hashing n-grams so that they can be stored in a bloom filter.
+ */
+ public static final int HASH_SEED = 17;
+
+ /**
+ * Another value used in the process of hashing n-grams.
+ */
+ public static final int HASH_OFFSET = 37;
+
+ /**
+ * The maximum score that a language model feature function can return to the Joshua decoder.
+ */
+ public static final double MAX_SCORE = 100.0;
+
+ /**
+ * The logger for this class.
+ */
+ private static final Logger LOG = LoggerFactory.getLogger(BloomFilterLanguageModel.class);
+
+ /**
+ * The Bloom filter data structure itself.
+ */
+ private BloomFilter bf;
+
+ /**
+ * The base of the logarithm used to quantize n-gram counts. N-gram counts are quantized
+ * logarithmically to reduce the number of times we need to query the Bloom filter.
+ */
+ private double quantizationBase;
+
+ /**
+ * Natural log of the number of tokens seen in the training corpus.
+ */
+ private double numTokens;
+
+ /**
+ * An array of pairs of long, used as hash functions for storing or retreiving the count of an
+ * n-gram in the Bloom filter.
+ */
+ private long[][] countFuncs;
+ /**
+ * An array of pairs of long, used as hash functions for storing or retreiving the number of
+ * distinct types observed after an n-gram.
+ */
+ private long[][] typesFuncs;
+
+ /**
+ * The smoothed probability of an unseen n-gram. This is also the probability of any n-gram under
+ * the zeroth-order model.
+ */
+ transient private double p0;
+
+ /**
+ * The interpolation constant between Witten-Bell models of order zero and one. Stored in a field
+ * because it can be calculated ahead of time; it doesn't depend on the particular n-gram.
+ */
+ transient private double lambda0;
+
+ /**
+ * The maximum possible quantized count of any n-gram stored in the Bloom filter. Used as an upper
+ * bound on the count that could be returned when querying the Bloom filter.
+ */
+ transient private int maxQ; // max quantized count
+
+ /**
+ * Constructor called from the Joshua decoder. This constructor assumes that the LM has already
+ * been built, and takes the name of the file where the LM is stored.
+ *
+ * @param order the order of the language model
+ * @param filename path to the file where the language model is stored
+ * @throws IOException if the bloom filter language model cannot be rebuilt from the input file
+ */
+ public BloomFilterLanguageModel(int order, String filename) throws IOException {
+ super(order);
+ try {
+ readExternal(new ObjectInputStream(new GZIPInputStream(new FileInputStream(filename))));
+ } catch (ClassNotFoundException e) {
+ IOException ioe = new IOException("Could not rebuild bloom filter LM from file " + filename);
+ ioe.initCause(e);
+ throw ioe;
+ }
+
+ int vocabSize = Vocabulary.size();
+ p0 = -Math.log(vocabSize + 1);
+ double oneMinusLambda0 = numTokens - logAdd(Math.log(vocabSize), numTokens);
+ p0 += oneMinusLambda0;
+ lambda0 = Math.log(vocabSize) - logAdd(Math.log(vocabSize), numTokens);
+ maxQ = quantize((long) Math.exp(numTokens));
+ }
+
+ /**
+ * Constructor to be used by the main function. This constructor is used to build a new language
+ * model from scratch. An LM should be built with the main function before using it in the Joshua
+ * decoder.
+ *
+ * @param filename path to the file of training corpus statistics
+ * @param order the order of the language model
+ * @param size the size of the Bloom filter, in bits
+ * @param base a double. The base of the logarithm for quantization.
+ */
+ private BloomFilterLanguageModel(String filename, int order, int size, double base) {
+ super(order);
+ quantizationBase = base;
+ populateBloomFilter(size, filename);
+ }
+
+ /**
+ * calculates the linearly-interpolated Witten-Bell probability for a given ngram. this is
+ * calculated as: p(w|h) = pML(w|h)L(h) - (1 - L(h))p(w|h') where: w is a word and h is a history
+ * h' is the history h with the first word removed pML is the maximum-likelihood estimate of the
+ * probability L(.) is lambda, the interpolation factor, which depends only on the history h: L(h)
+ * = s(h) / s(h) + c(h) where s(.) is the observed number of distinct types after h, and c is the
+ * observed number of counts of h in the training corpus.
+ * <p>
+ * in fact this model calculates the probability starting from the lowest order and working its
+ * way up, to take advantage of the one- sided error rate inherent in using a bloom filter data
+ * structure.
+ *
+ * @param ngram the ngram whose probability is to be calculated
+ * @param ngramOrder the order of the ngram.
+ *
+ * @return the linearly-interpolated Witten-Bell smoothed probability of an ngram
+ */
+ private float wittenBell(int[] ngram, int ngramOrder) {
+ int end = ngram.length;
+ double p = p0; // current calculated probability
+ // note that p0 and lambda0 are independent of the given
+ // ngram so they are calculated ahead of time.
+ int MAX_QCOUNT = getCount(ngram, ngram.length - 1, ngram.length, maxQ);
+ if (MAX_QCOUNT == 0) // OOV!
+ return (float) p;
+ double pML = Math.log(unQuantize(MAX_QCOUNT)) - numTokens;
+
+ // p += lambda0 * pML;
+ p = logAdd(p, (lambda0 + pML));
+ if (ngram.length == 1) { // if it's a unigram, we're done
+ return (float) p;
+ }
+ // otherwise we calculate the linear interpolation
+ // with higher order models.
+ for (int i = end - 2; i >= end - ngramOrder && i >= 0; i--) {
+ int historyCnt = getCount(ngram, i, end, MAX_QCOUNT);
+ // if the count for the history is zero, all higher
+ // terms in the interpolation must be zero, so we
+ // are done here.
+ if (historyCnt == 0) {
+ return (float) p;
+ }
+ int historyTypesAfter = getTypesAfter(ngram, i, end, historyCnt);
+ // unQuantize the counts we got from the BF
+ double HC = unQuantize(historyCnt);
+ double HTA = 1 + unQuantize(historyTypesAfter);
+ // interpolation constant
+ double lambda = Math.log(HTA) - Math.log(HTA + HC);
+ double oneMinusLambda = Math.log(HC) - Math.log(HTA + HC);
+ // p *= 1 - lambda
+ p += oneMinusLambda;
+ int wordCount = getCount(ngram, i + 1, end, historyTypesAfter);
+ double WC = unQuantize(wordCount);
+ // p += lambda * p_ML(w|h)
+ if (WC == 0) return (float) p;
+ p = logAdd(p, lambda + Math.log(WC) - Math.log(HC));
+ MAX_QCOUNT = wordCount;
+ }
+ return (float) p;
+ }
+
+ /**
+ * Retrieve the count of a ngram from the Bloom filter. That is, how many times did we see this
+ * ngram in the training corpus? This corresponds roughly to algorithm 2 in Talbot and Osborne's
+ * "Tera-Scale LMs on the Cheap."
+ *
+ * @param ngram array containing the ngram as a sub-array
+ * @param start the index of the first word of the ngram
+ * @param end the index after the last word of the ngram
+ * @param qcount the maximum possible count to be returned
+ *
+ * @return the number of times the ngram was seen in the training corpus, quantized
+ */
+ private int getCount(int[] ngram, int start, int end, int qcount) {
+ for (int i = 1; i <= qcount; i++) {
+ int hash = hashNgram(ngram, start, end, i);
+ if (!bf.query(hash, countFuncs)) {
+ return i - 1;
+ }
+ }
+ return qcount;
+ }
+
+ /**
+ * Retrieve the number of distinct types that follow an ngram in the training corpus.
+ *
+ * This is another version of algorithm 2. As noted in the paper, we have different algorithms for
+ * getting ngram counts versus suffix counts because c(x) = 1 is a proxy item for s(x) = 1
+ *
+ * @param ngram an array the contains the ngram as a sub-array
+ * @param start the index of the first word of the ngram
+ * @param end the index after the last word of the ngram
+ * @param qcount the maximum possible return value
+ *
+ * @return the number of distinct types observed to follow an ngram in the training corpus,
+ * quantized
+ */
+ private int getTypesAfter(int[] ngram, int start, int end, int qcount) {
+ // first we check c(x) >= 1
+ int hash = hashNgram(ngram, start, end, 1);
+ if (!bf.query(hash, countFuncs)) {
+ return 0;
+ }
+ // if c(x) >= 1, we check for the stored suffix count
+ for (int i = 1; i < qcount; i++) {
+ hash = hashNgram(ngram, start, end, i);
+ if (!bf.query(hash, typesFuncs)) {
+ return i - 1;
+ }
+ }
+ return qcount;
+ }
+
+ /**
+ * Logarithmically quantizes raw counts. The quantization scheme is described in Talbot and
+ * Osborne's paper "Tera-Scale LMs on the Cheap."
+ *
+ * @param x long giving the raw count to be quantized
+ *
+ * @return the quantized count
+ */
+ private int quantize(long x) {
+ return 1 + (int) Math.floor(Math.log(x) / Math.log(quantizationBase));
+ }
+
+ /**
+ * Unquantizes a quantized count.
+ *
+ * @param x the quantized count
+ *
+ * @return the expected raw value of the quantized count
+ */
+ private double unQuantize(int x) {
+ if (x == 0) {
+ return 0;
+ } else {
+ return ((quantizationBase + 1) * Math.pow(quantizationBase, x - 1) - 1) / 2;
+ }
+ }
+
+ /**
+ * Converts an n-gram and a count into a value that can be stored into a Bloom filter. This is
+ * adapted directly from <code>AbstractPhrase.hashCode()</code> elsewhere in the Joshua code base.
+ *
+ * @param ngram an array containing the ngram as a sub-array
+ * @param start the index of the first word of the ngram
+ * @param end the index after the last word of the ngram
+ * @param val the count of the ngram
+ *
+ * @return a value suitable to be stored in a Bloom filter
+ */
+ private int hashNgram(int[] ngram, int start, int end, int val) {
+ int result = HASH_OFFSET * HASH_SEED + val;
+ for (int i = start; i < end; i++)
+ result = HASH_OFFSET * result + ngram[i];
+ return result;
+ }
+
+ /**
+ * Adds two numbers that are in the log domain, avoiding underflow.
+ *
+ * @param x one summand
+ * @param y the other summand
+ *
+ * @return the log of the sum of the exponent of the two numbers.
+ */
+ private static double logAdd(double x, double y) {
+ if (y <= x) {
+ return x + Math.log1p(Math.exp(y - x));
+ } else {
+ return y + Math.log1p(Math.exp(x - y));
+ }
+ }
+
+ /**
+ * Builds a language model and stores it in a file.
+ *
+ * @param argv command-line arguments
+ */
+ public static void main(String[] argv) {
+ if (argv.length < 5) {
+ String msg = "usage: BloomFilterLanguageModel <statistics file> <order> <size>"
+ + " <quantization base> <output file>";
+ System.err.println(msg);
+ LOG.error(msg);
+ return;
+ }
+ int order = Integer.parseInt(argv[1]);
+ int size = (int) (Integer.parseInt(argv[2]) * Math.pow(2, 23));
+ double base = Double.parseDouble(argv[3]);
+
+ try {
+ BloomFilterLanguageModel lm = new BloomFilterLanguageModel(argv[0], order, size, base);
+
+ ObjectOutputStream out =
+ new ObjectOutputStream(new GZIPOutputStream(new FileOutputStream(argv[4])));
+
+ lm.writeExternal(out);
+ out.close(); //TODO: try-with-resources
+ } catch (IOException e) {
+ LOG.error(e.getMessage(), e);
+ }
+ }
+
+ /**
+ * Adds ngram counts and counts of distinct types after ngrams, read from a file, to the Bloom
+ * filter.
+ * <p>
+ * The file format should look like this: ngram1 count types-after ngram2 count types-after ...
+ *
+ * @param bloomFilterSize the size of the Bloom filter, in bits
+ * @param filename path to the statistics file
+ */
+ private void populateBloomFilter(int bloomFilterSize, String filename) {
+ HashMap<String, Long> typesAfter = new HashMap<String, Long>();
+ try {
+ FileInputStream file_in = new FileInputStream(filename);
+ FileInputStream file_in_copy = new FileInputStream(filename);
+ InputStream in;
+ InputStream estimateStream;
+ if (filename.endsWith(".gz")) {
+ in = new GZIPInputStream(file_in);
+ estimateStream = new GZIPInputStream(file_in_copy);
+ } else {
+ in = file_in;
+ estimateStream = file_in_copy;
+ }
+ int numObjects = estimateNumberOfObjects(estimateStream);
+ LOG.debug("Estimated number of objects: {}", numObjects);
+ bf = new BloomFilter(bloomFilterSize, numObjects);
+ countFuncs = bf.initializeHashFunctions();
+ populateFromInputStream(in, typesAfter);
+ in.close();
+ } catch (IOException e) {
+ LOG.error(e.getMessage(), e);
+ return;
+ }
+ typesFuncs = bf.initializeHashFunctions();
+ for (String history : typesAfter.keySet()) {
+ String[] toks = Regex.spaces.split(history);
+ int[] hist = new int[toks.length];
+ for (int i = 0; i < toks.length; i++)
+ hist[i] = Vocabulary.id(toks[i]);
+ add(hist, typesAfter.get(history), typesFuncs);
+ }
+ return;
+ }
+
+ /**
+ * Estimate the number of objects that will be stored in the Bloom filter. The optimum number of
+ * hash functions depends on the number of items that will be stored, so we want a guess before we
+ * begin to read the statistics file and store it.
+ *
+ * @param source an InputStream pointing to the training corpus stats
+ *
+ * @return an estimate of the number of objects to be stored in the Bloom filter
+ */
+ private int estimateNumberOfObjects(InputStream source) {
+ int numLines = 0;
+ long maxCount = 0;
+ for (String line: new LineReader(source)) {
+ if (line.trim().equals("")) continue;
+ String[] toks = Regex.spaces.split(line);
+ if (toks.length > ngramOrder + 1) continue;
+ try {
+ long cnt = Long.parseLong(toks[toks.length - 1]);
+ if (cnt > maxCount) maxCount = cnt;
+ } catch (NumberFormatException e) {
+ LOG.error(e.getMessage(), e);
+ break;
+ }
+ numLines++;
+ }
+ double estimate = Math.log(maxCount) / Math.log(quantizationBase);
+ return (int) Math.round(numLines * estimate);
+ }
+
+ /**
+ * Reads the statistics from a source and stores them in the Bloom filter. The ngram counts are
+ * stored immediately in the Bloom filter, but the counts of distinct types following each ngram
+ * are accumulated from the file as we go.
+ *
+ * @param source an InputStream pointing to the statistics
+ * @param types a HashMap that will stores the accumulated counts of distinct types observed to
+ * follow each ngram
+ */
+ private void populateFromInputStream(InputStream source, HashMap<String, Long> types) {
+ numTokens = Double.NEGATIVE_INFINITY; // = log(0)
+ for (String line: new LineReader(source)) {
+ String[] toks = Regex.spaces.split(line);
+ if ((toks.length < 2) || (toks.length > ngramOrder + 1)) continue;
+ int[] ngram = new int[toks.length - 1];
+ StringBuilder history = new StringBuilder();
+ for (int i = 0; i < toks.length - 1; i++) {
+ ngram[i] = Vocabulary.id(toks[i]);
+ if (i < toks.length - 2) history.append(toks[i]).append(" ");
+ }
+
+ long cnt = Long.parseLong(toks[toks.length - 1]);
+ add(ngram, cnt, countFuncs);
+ if (toks.length == 2) { // unigram
+ numTokens = logAdd(numTokens, Math.log(cnt));
+ // no need to count types after ""
+ // that's what vocabulary.size() is for.
+ continue;
+ }
+ if (types.get(history) == null)
+ types.put(history.toString(), 1L);
+ else {
+ long x = (Long) types.get(history);
+ types.put(history.toString(), x + 1);
+ }
+ }
+ return;
+ }
+
+ /**
+ * Adds an ngram, along with an associated value, to the Bloom filter. This corresponds to Talbot
+ * and Osborne's "Tera-scale LMs on the cheap", algorithm 1.
+ *
+ * @param ngram an array representing the ngram
+ * @param value the value to be associated with the ngram
+ * @param funcs an array of long to be used as hash functions
+ */
+ private void add(int[] ngram, long value, long[][] funcs) {
+ if (ngram == null) return;
+ int qValue = quantize(value);
+ for (int i = 1; i <= qValue; i++) {
+ int hash = hashNgram(ngram, 0, ngram.length, i);
+ bf.add(hash, funcs);
+ }
+ }
+
+ /**
+ * Read a Bloom filter LM from an external file.
+ *
+ * @param in an ObjectInput stream to read from
+ */
+ public void readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
+ int vocabSize = in.readInt();
+ for (int i = 0; i < vocabSize; i++) {
+ String line = in.readUTF();
+ Vocabulary.id(line);
+ }
+ numTokens = in.readDouble();
+ countFuncs = new long[in.readInt()][2];
+ for (int i = 0; i < countFuncs.length; i++) {
+ countFuncs[i][0] = in.readLong();
+ countFuncs[i][1] = in.readLong();
+ }
+ typesFuncs = new long[in.readInt()][2];
+ for (int i = 0; i < typesFuncs.length; i++) {
+ typesFuncs[i][0] = in.readLong();
+ typesFuncs[i][1] = in.readLong();
+ }
+ quantizationBase = in.readDouble();
+ bf = new BloomFilter();
+ bf.readExternal(in);
+ }
+
+ /**
+ * Write a Bloom filter LM to some external location.
+ *
+ * @param out an ObjectOutput stream to write to
+ *
+ * @throws IOException if an input or output exception occurred
+ */
+ public void writeExternal(ObjectOutput out) throws IOException {
+ out.writeInt(Vocabulary.size());
+ for (int i = 0; i < Vocabulary.size(); i++) {
+ // out.writeBytes(vocabulary.getWord(i));
+ // out.writeChar('\n'); // newline
+ out.writeUTF(Vocabulary.word(i));
+ }
+ out.writeDouble(numTokens);
+ out.writeInt(countFuncs.length);
+ for (int i = 0; i < countFuncs.length; i++) {
+ out.writeLong(countFuncs[i][0]);
+ out.writeLong(countFuncs[i][1]);
+ }
+ out.writeInt(typesFuncs.length);
+ for (int i = 0; i < typesFuncs.length; i++) {
+ out.writeLong(typesFuncs[i][0]);
+ out.writeLong(typesFuncs[i][1]);
+ }
+ out.writeDouble(quantizationBase);
+ bf.writeExternal(out);
+ }
+
+ /**
+ * Returns the language model score for an n-gram. This is called from the rest of the Joshua
+ * decoder.
+ *
+ * @param ngram the ngram to score
+ * @param order the order of the model
+ *
+ * @return the language model score of the ngram
+ */
+ @Override
+ protected float ngramLogProbability_helper(int[] ngram, int order) {
+ int[] lm_ngram = new int[ngram.length];
+ for (int i = 0; i < ngram.length; i++) {
+ lm_ngram[i] = Vocabulary.id(Vocabulary.word(ngram[i]));
+ }
+ return wittenBell(lm_ngram, order);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/bloomfilter_lm/package-info.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/bloomfilter_lm/package-info.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/bloomfilter_lm/package-info.java
new file mode 100644
index 0000000..19fa695
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/bloomfilter_lm/package-info.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * Provides an implementation of a bloom filter language model, and
+ * an associated implementation of the language model feature function typically used in
+ * hierarchical phrase-based decoding for statistical machine translation.
+ */
+package org.apache.joshua.decoder.ff.lm.bloomfilter_lm;
[16/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/decoder/io/DeNormalizeTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/io/DeNormalizeTest.java b/joshua-core/src/test/java/org/apache/joshua/decoder/io/DeNormalizeTest.java
new file mode 100644
index 0000000..88b2350
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/io/DeNormalizeTest.java
@@ -0,0 +1,273 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.io;
+
+import static org.testng.Assert.assertEquals;
+
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+/**
+ *
+ */
+public class DeNormalizeTest {
+
+ private String tokenized;
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @BeforeMethod
+ protected void setUp() throws Exception {
+ tokenized = "my son 's friend , however , plays a high - risk game .";
+ }
+
+ /**
+ * Test method for {@link joshua.decoder.io.DeNormalize#processSingleLine(java.lang.String)}.
+ */
+ @Test(enabled = true)
+ public void testProcessSingleLine() {
+ tokenized =
+ "my son 's friend , ( dr . -rrb- robotnik , phd , however , wo n't play a high - risk game .";
+ String expected = "My son's friend, (Dr.) robotnik, PhD, however, won't play a high-risk game.";
+ String actual = DeNormalize.processSingleLine(tokenized);
+ assertEquals(actual, expected);
+ }
+
+ /**
+ * Test method for {@link joshua.decoder.io.DeNormalize#processSingleLine(java.lang.String)}.
+ */
+ @Test
+ public void testProcessSingleLine_interspersed() {
+ tokenized = "phd mphil";
+ String expected = "PhD MPhil";
+ String actual = DeNormalize.processSingleLine(tokenized);
+ assertEquals(actual, expected);
+ }
+
+ /**
+ * Test method for
+ * {@link joshua.decoder.io.DeNormalize#capitalizeLineFirstLetter(java.lang.String)}.
+ */
+ @Test
+ public void testCapitalizeLineFirstLetter() throws Exception {
+ String actual = DeNormalize.capitalizeLineFirstLetter(tokenized);
+ String expected = "My son 's friend , however , plays a high - risk game .";
+ assertEquals(actual, expected);
+ }
+
+ /**
+ * Test method for
+ * {@link joshua.decoder.io.DeNormalize#capitalizeLineFirstLetter(java.lang.String)}.
+ */
+ @Test
+ public void testCapitalizeLineFirstLetter_empty() throws Exception {
+ String actual = DeNormalize.capitalizeLineFirstLetter("");
+ String expected = "";
+ assertEquals(actual, expected);
+ }
+
+ /**
+ * Test method for
+ * {@link joshua.decoder.io.DeNormalize#capitalizeLineFirstLetter(java.lang.String)}.
+ */
+ @Test
+ public void testCapitalizeLineFirstLetter_singleNumberCharacter() throws Exception {
+ String actual = DeNormalize.capitalizeLineFirstLetter("1");
+ String expected = "1";
+ assertEquals(actual, expected);
+ }
+
+ /**
+ * Test method for
+ * {@link joshua.decoder.io.DeNormalize#capitalizeLineFirstLetter(java.lang.String)}.
+ */
+ @Test
+ public void testCapitalizeLineFirstLetter_singleLetterCharacter() throws Exception {
+ String actual = DeNormalize.capitalizeLineFirstLetter("a");
+ String expected = "A";
+ assertEquals(actual, expected);
+ }
+
+ /**
+ * Test method for {@link joshua.decoder.io.DeNormalize#joinPunctuationMarks(java.lang.String)}.
+ */
+ @Test
+ public void testJoinPunctuationMarks() throws Exception {
+ String actual = DeNormalize.joinPunctuationMarks(tokenized);
+ String expected = "my son 's friend, however, plays a high - risk game.";
+ assertEquals(actual, expected);
+ }
+
+ /**
+ * Test method for {@link joshua.decoder.io.DeNormalize#joinPunctuationMarks(java.lang.String)}.
+ */
+ @Test
+ public void testJoinPunctuationMarks_empty() throws Exception {
+ String actual = DeNormalize.joinPunctuationMarks("");
+ String expected = "";
+ assertEquals(actual, expected);
+ }
+
+ /**
+ * Test method for {@link joshua.decoder.io.DeNormalize#joinHyphen(java.lang.String)}.
+ */
+ @Test
+ public void testJoinHyphen() throws Exception {
+ String actual = DeNormalize.joinHyphen(tokenized);
+ String expected = "my son 's friend , however , plays a high-risk game .";
+ assertEquals(actual, expected);
+ }
+
+ /**
+ * Test method for {@link joshua.decoder.io.DeNormalize#joinHyphen(java.lang.String)}.
+ */
+ @Test
+ public void testJoinHypen_empty() throws Exception {
+ String actual = DeNormalize.joinHyphen("");
+ String expected = "";
+ assertEquals(actual, expected);
+ }
+
+ /**
+ * Test method for {@link joshua.decoder.io.DeNormalize#joinHyphen(java.lang.String)}.
+ */
+ @Test
+ public void testJoinHyphen_1space_btw_2hyphens() throws Exception {
+ String actual = DeNormalize.joinHyphen("a - - b");
+ String expected = "a-- b";
+ assertEquals(actual, expected);
+ }
+
+ /**
+ * Test method for {@link joshua.decoder.io.DeNormalize#joinHyphen(java.lang.String)}.
+ */
+ @Test
+ public void testJoinHyphen_2spaces_btw_2hyphens() throws Exception {
+ String actual = DeNormalize.joinHyphen("a - - b");
+ String expected = "a--b";
+ assertEquals(actual, expected);
+ }
+
+ /**
+ * Test method for {@link joshua.decoder.io.DeNormalize#joinContractions(java.lang.String)}.
+ */
+ @Test
+ public void testJoinContractions() throws Exception {
+ tokenized = "my son 's friend , however , wo n't play a high - risk game .";
+ String actual = DeNormalize.joinContractions(tokenized);
+ String expected = "my son's friend , however , won't play a high - risk game .";
+ assertEquals(actual, expected);
+ }
+
+ /**
+ * Test method for {@link joshua.decoder.io.DeNormalize#joinContractions(java.lang.String)}.
+ */
+ @Test
+ public void testJoinContractions_empty() throws Exception {
+ String actual = DeNormalize.joinContractions("");
+ String expected = "";
+ assertEquals(actual, expected);
+ }
+
+ /**
+ * Test method for
+ * {@link joshua.decoder.io.DeNormalize#capitalizeNameTitleAbbrvs(java.lang.String)}.
+ */
+ @Test
+ public void testCapitalizeNameTitleAbbrvs() throws Exception {
+ String actual, expected;
+ tokenized =
+ "my son 's friend , dr . robotnik , phd , however , wo n't play a high - risk game .";
+ expected =
+ "my son 's friend , Dr . robotnik , PhD , however , wo n't play a high - risk game .";
+ actual = DeNormalize.capitalizeNameTitleAbbrvs(tokenized);
+ assertEquals(actual, expected);
+
+ tokenized = "mr mrs ms miss dr prof";
+ expected = "Mr Mrs Ms Miss Dr Prof";
+ actual = DeNormalize.capitalizeNameTitleAbbrvs(tokenized);
+ assertEquals(actual, expected);
+ }
+
+ /**
+ * Test method for {@link joshua.decoder.io.DeNormalize#capitalizeI(java.lang.String)}.
+ */
+ @Test
+ public void testCapitalizeI() throws Exception {
+ String expected, actual;
+
+ tokenized = "sam i am";
+ expected = "sam I am";
+ actual = DeNormalize.capitalizeI(tokenized);
+ assertEquals(actual, expected);
+
+ tokenized = "sam iam";
+ expected = "sam iam";
+ actual = DeNormalize.capitalizeI(tokenized);
+ assertEquals(actual, expected);
+
+ tokenized = "sami am";
+ expected = "sami am";
+ actual = DeNormalize.capitalizeI(tokenized);
+ assertEquals(actual, expected);
+
+ tokenized = "samiam";
+ expected = "samiam";
+ actual = DeNormalize.capitalizeI(tokenized);
+ assertEquals(actual, expected);
+ }
+
+ /**
+ * Test method for {@link joshua.decoder.io.DeNormalize#replaceBracketTokens(java.lang.String)}.
+ */
+ @Test
+ public void testReplaceBracketTokens() throws Exception {
+ String expected, actual;
+
+ tokenized = "-lrb- i -rrb-";
+ expected = "( i )";
+ actual = DeNormalize.replaceBracketTokens(tokenized);
+ assertEquals(actual, expected);
+
+ tokenized = "-LRB- i -RRB-";
+ expected = "( i )";
+ actual = DeNormalize.replaceBracketTokens(tokenized);
+ assertEquals(actual, expected);
+ }
+
+ /**
+ * Test method for {@link joshua.decoder.io.DeNormalize#detokenizeBracketTokens(java.lang.String)}
+ */
+ @Test
+ public void testDetokenizeBracketTokens() throws Exception {
+ String expected, actual;
+
+ tokenized = "( i )";
+ expected = "(i)";
+ actual = DeNormalize.joinPunctuationMarks(tokenized);
+ assertEquals(actual, expected);
+
+ tokenized = "[ i } j";
+ expected = "[i} j";
+ actual = DeNormalize.joinPunctuationMarks(tokenized);
+ assertEquals(actual, expected);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/decoder/io/TranslationRequestTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/io/TranslationRequestTest.java b/joshua-core/src/test/java/org/apache/joshua/decoder/io/TranslationRequestTest.java
new file mode 100644
index 0000000..5a1c3ab
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/io/TranslationRequestTest.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.io;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.Charset;
+
+import org.apache.joshua.decoder.JoshuaConfiguration;
+
+import org.testng.annotations.*;
+import static org.testng.Assert.*;
+import static org.mockito.Mockito.*;
+
+/**
+ * This class verifies the following behaviors:
+ *
+ * - A blank input, i.e. "", does not cause a translation to be created.
+ *
+ * - A non-blank input that is not followed by a newline, e.g. "1", causes a translation to be
+ * created.
+ *
+ * - An input that contains whitespace or nothing followed by a newline causes a translation to be
+ * created, with "" as the source.
+ */
+public class TranslationRequestTest {
+
+ private final JoshuaConfiguration joshuaConfiguration = new JoshuaConfiguration();
+ @BeforeMethod
+ public void createTranslationRequest() throws Exception {
+ }
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @BeforeMethod
+ protected void setUp() throws Exception {
+ }
+
+ /**
+ * @throws java.lang.Exception
+ */
+ @AfterMethod
+ protected void tearDown() throws Exception {
+ }
+
+ /**
+ * Test method for {@link joshua.decoder.io.TranslationRequest#TranslationRequest(java.io.InputStream)}.
+ */
+ @Test(enabled = false)
+ public void testTranslationRequest() {
+ fail("Not yet implemented");
+ }
+
+ /**
+ * Test method for {@link joshua.decoder.io.TranslationRequest#size()}.
+ */
+ @Test(enabled = true)
+ public void testSize_uponConstruction() {
+ InputStream in = mock(InputStream.class);
+ TranslationRequestStream request = new TranslationRequestStream(
+ new BufferedReader(new InputStreamReader(in, Charset.defaultCharset())), joshuaConfiguration);
+ assertEquals(request.size(), 0);
+ }
+
+ /**
+ * Test method for {@link joshua.decoder.io.TranslationRequest#size()}.
+ * @throws Exception
+ */
+ @Test(enabled = true)
+ public void testSize_1() throws Exception {
+ byte[] data = "1".getBytes();
+ ByteArrayInputStream input = new ByteArrayInputStream(data);
+ TranslationRequestStream request = new TranslationRequestStream(
+ new BufferedReader(new InputStreamReader(input, Charset.defaultCharset())), joshuaConfiguration);
+ request.next();
+ assertEquals(request.size(), 1);
+ }
+
+ /**
+ * Test method for {@link joshua.decoder.io.TranslationRequest#size()}.
+ * @throws Exception
+ */
+ @Test(enabled = true)
+ public void testSize_newline() throws Exception {
+ byte[] data = "\n".getBytes();
+ ByteArrayInputStream input = new ByteArrayInputStream(data);
+ TranslationRequestStream request = new TranslationRequestStream(
+ new BufferedReader(new InputStreamReader(input, Charset.defaultCharset())), joshuaConfiguration);
+ request.next();
+ assertEquals(request.size(), 1);
+ }
+
+ /**
+ * Test method for {@link joshua.decoder.io.TranslationRequest#size()}.
+ * @throws Exception
+ */
+ @Test(enabled = true)
+ public void testSize_2newlines() throws Exception {
+ byte[] data = "\n\n".getBytes();
+ ByteArrayInputStream input = new ByteArrayInputStream(data);
+ TranslationRequestStream request = new TranslationRequestStream(
+ new BufferedReader(new InputStreamReader(input, Charset.defaultCharset())), joshuaConfiguration);
+ request.next();
+ request.next();
+ assertEquals(request.size(), 2);
+ }
+
+ /**
+ * Test method for {@link joshua.decoder.io.TranslationRequest#next()}.
+ * @throws Exception
+ */
+ @Test(enabled = true)
+ public void testNext_2Newlines() throws Exception {
+ byte[] data = "\n\n".getBytes();
+ ByteArrayInputStream input = new ByteArrayInputStream(data);
+ TranslationRequestStream request = new TranslationRequestStream(
+ new BufferedReader(new InputStreamReader(input, Charset.defaultCharset())), joshuaConfiguration);
+ assertEquals(request.next().source(), "");
+ assertEquals(request.next().source(), "");
+ }
+
+ /**
+ * Test method for {@link joshua.decoder.io.TranslationRequest#remove()}.
+ */
+ @Test(enabled = false)
+ public void testRemove() {
+ fail("Not yet implemented");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/decoder/kbest_extraction/KBestExtractionTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/kbest_extraction/KBestExtractionTest.java b/joshua-core/src/test/java/org/apache/joshua/decoder/kbest_extraction/KBestExtractionTest.java
new file mode 100644
index 0000000..44ef35d
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/kbest_extraction/KBestExtractionTest.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.decoder.kbest_extraction;
+
+import static com.google.common.base.Charsets.UTF_8;
+import static java.nio.file.Files.readAllBytes;
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import org.apache.joshua.decoder.Decoder;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.Translation;
+import org.apache.joshua.decoder.segment_file.Sentence;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+/**
+ * Reimplements the kbest extraction regression test
+ * TODO (fhieber): this test strangely only works with StateMinimizing KenLM.
+ * This is to be investigated
+ */
+@Ignore("re-enable as soon as kenlm native library support will be in place")
+public class KBestExtractionTest {
+
+ private static final String CONFIG = "resources/kbest_extraction/joshua.config";
+ private static final String INPUT = "a b c d e";
+ private static final Path GOLD_PATH = Paths.get("resources/kbest_extraction/output.scores.gold");
+
+ private JoshuaConfiguration joshuaConfig = null;
+ private Decoder decoder = null;
+
+ @Before
+ public void setUp() throws Exception {
+ joshuaConfig = new JoshuaConfiguration();
+ joshuaConfig.readConfigFile(CONFIG);
+ joshuaConfig.outputFormat = "%i ||| %s ||| %c";
+ decoder = new Decoder(joshuaConfig, "");
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ decoder.cleanUp();
+ decoder = null;
+ }
+
+ @Test
+ public void givenInput_whenKbestExtraction_thenOutputIsAsExpected() throws IOException {
+ final String translation = decode(INPUT).toString();
+ final String gold = new String(readAllBytes(GOLD_PATH), UTF_8);
+ assertEquals(gold, translation);
+ }
+
+ private Translation decode(String input) {
+ final Sentence sentence = new Sentence(input, 0, joshuaConfig);
+ return decoder.decode(sentence);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/decoder/phrase/CoverageTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/phrase/CoverageTest.java b/joshua-core/src/test/java/org/apache/joshua/decoder/phrase/CoverageTest.java
new file mode 100644
index 0000000..7526b1f
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/phrase/CoverageTest.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.phrase;
+
+import static org.junit.Assert.*;
+
+import java.util.BitSet;
+
+import org.junit.Test;
+
+public class CoverageTest {
+
+ @Test
+ public void testSet() {
+ Coverage cov = new Coverage();
+ cov.set(1,2);
+ cov.set(3,4);
+ cov.set(2,3);
+ cov.set(0,1);
+
+ assertFalse(cov.compatible(0, 1));
+ assertFalse(cov.compatible(0, 5));
+ assertTrue(cov.compatible(4, 6));
+
+ assertEquals(cov.toString(), "4 ..........");
+ }
+
+ @Test
+ public void testPattern() {
+ Coverage cov = new Coverage();
+ cov.set(5,6);
+ cov.set(0,4);
+ BitSet bits = cov.pattern(4, 5);
+ BitSet answerBits = new BitSet();
+ answerBits.set(0);
+ assertEquals(bits, answerBits);
+ }
+
+ @Test
+ public void testCopyConstructor() {
+ Coverage a = new Coverage();
+ a.set(2,3);
+ Coverage b = new Coverage(a);
+ b.set(4,5);
+
+ assertFalse(a.toString().equals(b.toString()));
+ }
+
+ @Test
+ public void testCompatible() {
+ Coverage a = new Coverage();
+ a.set(10, 14);
+
+ assertTrue(a.compatible(14, 16));
+ assertTrue(a.compatible(6, 10));
+ assertTrue(a.compatible(1, 10));
+ assertTrue(a.compatible(1, 9));
+ assertFalse(a.compatible(9, 11));
+ assertFalse(a.compatible(13, 15));
+ assertFalse(a.compatible(9, 15));
+ assertFalse(a.compatible(9, 14));
+ assertFalse(a.compatible(10, 15));
+
+ a.set(0,9);
+
+ for (int width = 1; width <= 3; width++) {
+ for (int i = 0; i < 20; i++) {
+ int j = i + width;
+ if ((i == 9 && j == 10) || i >= 14)
+ assertTrue(a.compatible(i,j));
+ else {
+// System.err.println(String.format("%d,%d -> %s %s", i, j, a.compatible(i,j), a));
+ assertFalse(a.compatible(i,j));
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testFirstZero() {
+ Coverage cov = new Coverage();
+ cov.set(2, 5);
+ assertEquals(cov.firstZero(), 0);
+ cov.set(8,10);
+ assertEquals(cov.firstZero(), 0);
+ cov.set(0, 2);
+ assertEquals(cov.firstZero(), 5);
+ cov.set(5, 7);
+ assertEquals(cov.firstZero(), 7);
+ cov.set(7,8);
+ assertEquals(cov.firstZero(), 10);
+ }
+
+ @Test
+ public void testOpenings() {
+ Coverage cov = new Coverage();
+ cov.set(0, 2);
+ cov.set(8, 10);
+
+ for (int i = 2; i < 7; i++) {
+ assertEquals(cov.leftOpening(i), 2);
+ assertEquals(cov.rightOpening(i, 17), 8);
+ assertEquals(cov.rightOpening(i, 7), 7);
+ }
+ }
+
+ @Test
+ public void testEquals() {
+ Coverage cov = new Coverage();
+ cov.set(9, 11);
+ Coverage cov2 = new Coverage();
+ cov2.set(9,10);
+ cov2.set(10,11);
+ assertEquals(cov, cov2);
+ }
+
+ @Test
+ public void testToString() {
+ Coverage cov = new Coverage();
+ cov.set(0, 40);
+ cov.set(44, 49);
+ assertEquals(cov.toString(), "40 ....xxxxx.");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/decoder/phrase/constrained/ConstrainedPhraseDecodingTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/phrase/constrained/ConstrainedPhraseDecodingTest.java b/joshua-core/src/test/java/org/apache/joshua/decoder/phrase/constrained/ConstrainedPhraseDecodingTest.java
new file mode 100644
index 0000000..a99338a
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/phrase/constrained/ConstrainedPhraseDecodingTest.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.decoder.phrase.constrained;
+
+import static com.google.common.base.Charsets.UTF_8;
+import static java.nio.file.Files.readAllBytes;
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import org.apache.joshua.decoder.Decoder;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.Translation;
+import org.apache.joshua.decoder.segment_file.Sentence;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+/**
+ * Reimplements the constrained phrase decoding test
+ */
+@Ignore("re-enable as soon as kenlm native library support will be in place")
+public class ConstrainedPhraseDecodingTest {
+
+ private static final String CONFIG = "resources/phrase_decoder/constrained.config";
+ private static final String INPUT = "una estrategia republicana para obstaculizar la reelecci�n de Obama ||| President Obama to hinder a strategy for Republican re @-@ election";
+ private static final Path GOLD_PATH = Paths.get("resources/phrase_decoder/constrained.output.gold");
+
+ private JoshuaConfiguration joshuaConfig = null;
+ private Decoder decoder = null;
+
+ @Before
+ public void setUp() throws Exception {
+ joshuaConfig = new JoshuaConfiguration();
+ joshuaConfig.readConfigFile(CONFIG);
+ decoder = new Decoder(joshuaConfig, "");
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ decoder.cleanUp();
+ decoder = null;
+ }
+
+ @Test
+ public void givenInput_whenConstrainedPhraseDecoding_thenOutputIsAsExpected() throws IOException {
+ final String translation = decode(INPUT).toString();
+ final String gold = new String(readAllBytes(GOLD_PATH), UTF_8);
+ assertEquals(gold, translation);
+ }
+
+ private Translation decode(String input) {
+ final Sentence sentence = new Sentence(input, 0, joshuaConfig);
+ return decoder.decode(sentence);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/decoder/phrase/decode/PhraseDecodingTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/phrase/decode/PhraseDecodingTest.java b/joshua-core/src/test/java/org/apache/joshua/decoder/phrase/decode/PhraseDecodingTest.java
new file mode 100644
index 0000000..b5bd612
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/phrase/decode/PhraseDecodingTest.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.decoder.phrase.decode;
+
+import static com.google.common.base.Charsets.UTF_8;
+import static java.nio.file.Files.readAllBytes;
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import org.apache.joshua.decoder.Decoder;
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.apache.joshua.decoder.Translation;
+import org.apache.joshua.decoder.segment_file.Sentence;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
+/**
+ * Reimplements the constrained phrase decoding test
+ */
+@Ignore("re-enable as soon as kenlm native library support will be in place")
+public class PhraseDecodingTest {
+
+ private static final String CONFIG = "resources/phrase_decoder/config";
+ private static final String INPUT = "una estrategia republicana para obstaculizar la reelecci�n de Obama";
+ private static final Path GOLD_PATH = Paths.get("resources/phrase_decoder/output.gold");
+
+ private JoshuaConfiguration joshuaConfig = null;
+ private Decoder decoder = null;
+
+ @Before
+ public void setUp() throws Exception {
+ joshuaConfig = new JoshuaConfiguration();
+ joshuaConfig.readConfigFile(CONFIG);
+ decoder = new Decoder(joshuaConfig, "");
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ decoder.cleanUp();
+ decoder = null;
+ }
+
+ @Test
+ public void givenInput_whenPhraseDecoding_thenOutputIsAsExpected() throws IOException {
+ final String translation = decode(INPUT).toString();
+ final String gold = new String(readAllBytes(GOLD_PATH), UTF_8);
+ assertEquals(gold, translation);
+ }
+
+ private Translation decode(String input) {
+ final Sentence sentence = new Sentence(input, 0, joshuaConfig);
+ return decoder.decode(sentence);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/decoder/segment_file/AlmostTooLongSentenceTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/segment_file/AlmostTooLongSentenceTest.java b/joshua-core/src/test/java/org/apache/joshua/decoder/segment_file/AlmostTooLongSentenceTest.java
new file mode 100644
index 0000000..3b2852c
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/segment_file/AlmostTooLongSentenceTest.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.segment_file;
+
+import org.testng.annotations.Test;
+
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.AfterMethod;
+import static org.testng.Assert.*;
+
+import org.apache.joshua.decoder.JoshuaConfiguration;
+
+public class AlmostTooLongSentenceTest {
+ private JoshuaConfiguration joshuaConfiguration;
+ private String almostTooLongInput;
+ private Sentence sentencePlusTarget;
+
+ @BeforeMethod
+ public void setUp() {
+ joshuaConfiguration = new JoshuaConfiguration();
+ almostTooLongInput = concatStrings(".", joshuaConfiguration.maxlen);
+ sentencePlusTarget = new Sentence(this.almostTooLongInput + " ||| target side", 0,joshuaConfiguration);
+ }
+
+ @AfterMethod
+ public void tearDown() {
+ }
+
+ @Test
+ public void testConstructor() {
+ Sentence sent = new Sentence("", 0,joshuaConfiguration);
+ assertNotNull(sent);
+ }
+
+ @Test
+ public void testEmpty() {
+ assertTrue(new Sentence("", 0,joshuaConfiguration).isEmpty());
+ }
+
+ @Test
+ public void testNotEmpty() {
+ assertFalse(new Sentence("hello , world", 0, joshuaConfiguration).isEmpty());
+ }
+
+ /**
+ * Return a string consisting of repeatedToken concatenated MAX_SENTENCE_NODES times.
+ *
+ * @param repeatedToken
+ * @param repeatedTimes
+ * @return
+ */
+ private String concatStrings(String repeatedToken, int repeatedTimes) {
+ String result = "";
+ for (int i = 0; i < repeatedTimes; i++) {
+ result += repeatedToken;
+ }
+ return result;
+ }
+
+ @Test
+ public void testAlmostButNotTooManyTokensSourceOnlyNotEmpty() {
+ assertFalse(new Sentence(this.almostTooLongInput, 0, joshuaConfiguration).isEmpty());
+ }
+
+ @Test
+ public void testAlmostButNotTooManyTokensSourceOnlyTargetNull() {
+ assertNull(new Sentence(this.almostTooLongInput, 0, joshuaConfiguration).target);
+ }
+
+ @Test
+ public void testAlmostButNotTooManyTokensSourceAndTargetTargetIsNotEmpty() {
+ assertFalse(this.sentencePlusTarget.isEmpty());
+ }
+
+ @Test
+ public void testAlmostButNotTooManyTokensSourceAndTargetTargetNull() {
+ assertEquals(this.sentencePlusTarget.target, "target side");
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/decoder/segment_file/SentenceTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/decoder/segment_file/SentenceTest.java b/joshua-core/src/test/java/org/apache/joshua/decoder/segment_file/SentenceTest.java
new file mode 100644
index 0000000..8e0d171
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/decoder/segment_file/SentenceTest.java
@@ -0,0 +1,109 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.decoder.segment_file;
+
+import org.apache.joshua.decoder.JoshuaConfiguration;
+
+import org.testng.annotations.Test;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.AfterMethod;
+import static org.testng.Assert.*;
+
+public class SentenceTest {
+ private String tooLongInput;
+ private final JoshuaConfiguration joshuaConfiguration = new JoshuaConfiguration();
+
+
+
+ @BeforeMethod
+ public void setUp() {
+ tooLongInput = concatTokens("*", joshuaConfiguration.maxlen * 2);
+ }
+
+ @AfterMethod
+ public void tearDown() {
+ }
+
+ @Test
+ public void testConstructor() {
+ Sentence sent = new Sentence("", 0, joshuaConfiguration);
+ assertNotNull(sent);
+ }
+
+ @Test
+ public void testEmpty() {
+ assertTrue(new Sentence("", 0, joshuaConfiguration).isEmpty());
+ }
+
+ @Test
+ public void testNotEmpty() {
+ assertFalse(new Sentence("hello , world", 0, joshuaConfiguration).isEmpty());
+ }
+
+ /**
+ * Return a string consisting of repeatedToken concatenated MAX_SENTENCE_NODES times, joined by a
+ * space.
+ *
+ * @param repeatedToken
+ * @param repeatedTimes
+ * @return
+ */
+ private String concatTokens(String repeatedToken, int repeatedTimes) {
+ String result = "";
+ for (int i = 0; i < repeatedTimes - 1; i++) {
+ result += repeatedToken + " ";
+ }
+ result += repeatedToken;
+ return result;
+ }
+
+ /**
+ * The too long input sentence should be truncated from 799 to 202 characters
+ * TODO is this a bug? maxlen is defined as 200 not 202 characters
+ */
+ @Test
+ public void testTooManyTokensSourceTruncated() {
+ assertTrue(new Sentence(this.tooLongInput, 0, joshuaConfiguration).length() == 202);
+ }
+
+ @Test
+ public void testTooManyTokensSourceOnlyNotNull() {
+ assertNotNull(new Sentence(this.tooLongInput, 0, joshuaConfiguration));
+ }
+
+ @Test
+ public void testTooManyTokensSourceAndTargetIsEmpty() {
+ Sentence sentence = new Sentence(this.tooLongInput + " ||| target side", 0, joshuaConfiguration);
+ assertEquals(sentence.target, "");
+ }
+
+ @Test
+ public void testTooManyTokensSourceAndTargetTruncated() {
+ Sentence sentence = new Sentence(this.tooLongInput + " ||| target side", 0, joshuaConfiguration);
+ assertTrue(sentence.length() == 202);
+ }
+
+ @Test
+ public void testClearlyNotTooManyTokens() {
+ // Concatenate MAX_SENTENCE_NODES, each shorter than the average length, joined by a space.
+ String input = "token";
+ assertFalse(new Sentence(input, 0, joshuaConfiguration).isEmpty());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/lattice/ArcTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/lattice/ArcTest.java b/joshua-core/src/test/java/org/apache/joshua/lattice/ArcTest.java
new file mode 100644
index 0000000..a26a593
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/lattice/ArcTest.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.lattice;
+
+import org.apache.joshua.lattice.Arc;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+/**
+ * Unit tests for Arc class.
+ *
+ * @author Lane Schwartz
+ * @since 2008-07-09
+ * @version $LastChangedDate$
+ */
+@Test(groups = { "lattice_arc" })
+public class ArcTest {
+
+ private final Node<String> head = new Node<String>(1);
+ private final Node<String> tail = new Node<String>(2);
+ private final float cost = (float) Math.PI;
+ private final String label = "pi";
+
+ private Arc<String> arc;
+
+ @Test(dependsOnMethods = { "org.apache.joshua.lattice.NodeTest.constructNode" })
+ //@Test(dependsOnGroups = {"lattice_node" })
+ public void constructArc() {
+
+ arc = new Arc<String>(tail, head, (float)cost, label);
+
+ Assert.assertEquals(arc.getHead(), head);
+ Assert.assertEquals(arc.getTail(), tail);
+ Assert.assertEquals(arc.getCost(), cost);
+ Assert.assertEquals(arc.getLabel(), label);
+
+ }
+
+ @Test(dependsOnMethods = { "constructArc" })
+ public void getHead() {
+
+ Assert.assertEquals(arc.getHead(), head);
+
+ }
+
+
+ @Test(dependsOnMethods = { "constructArc" })
+ public void getTail() {
+
+ Assert.assertEquals(arc.getTail(), tail);
+
+ }
+
+
+ @Test(dependsOnMethods = { "constructArc" })
+ public void getCost() {
+
+ Assert.assertEquals(arc.getCost(), cost);
+
+ }
+
+
+ @Test(dependsOnMethods = { "constructArc" })
+ public void getLabel() {
+
+ Assert.assertEquals(arc.getLabel(), label);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/lattice/LatticeTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/lattice/LatticeTest.java b/joshua-core/src/test/java/org/apache/joshua/lattice/LatticeTest.java
new file mode 100644
index 0000000..1522120
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/lattice/LatticeTest.java
@@ -0,0 +1,192 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.lattice;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.joshua.decoder.JoshuaConfiguration;
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+/**
+ * Unit tests for Lattice class.
+ *
+ * @author Lane Schwartz
+ * @since 2008-07-09
+ * @version $LastChangedDate$
+ */
+@Test(groups = { "lattice" })
+public class LatticeTest {
+
+ @Test
+ public void allPairsShortestPath() {
+
+ List<Node<String>> nodes = new ArrayList<Node<String>>();
+ for (int i=0; i<4; i++) {
+ nodes.add(new Node<String>(i));
+ }
+
+ nodes.get(0).addArc(nodes.get(1), (float) 1.0, "x");
+ nodes.get(1).addArc(nodes.get(2), (float) 1.0, "y");
+ nodes.get(0).addArc(nodes.get(2), (float) 1.5, "a");
+ nodes.get(2).addArc(nodes.get(3), (float) 3.0, "b");
+ nodes.get(2).addArc(nodes.get(3), (float) 5.0, "c");
+
+ Lattice<String> graph = new Lattice<String>(nodes, new JoshuaConfiguration());
+
+ Assert.assertEquals(graph.getShortestPath(0, 1), 1);
+ Assert.assertEquals(graph.getShortestPath(0, 2), 1);
+ Assert.assertEquals(graph.getShortestPath(1, 2), 1);
+ Assert.assertEquals(graph.getShortestPath(0, 3), 2);
+ Assert.assertEquals(graph.getShortestPath(1, 3), 2);
+ Assert.assertEquals(graph.getShortestPath(2, 3), 1);
+ }
+
+ @Test
+ public void createFromString() {
+
+ String data =
+
+ // Start of lattice
+ "("+
+
+ // Node 0
+ "("+
+ "('A',1.0,5),"+ // Arc with label A and cost 1.0. Destination is Node 5 (Node 0 + span of 5)
+ "('B',1.0,2),"+ // Arc with label B and cost 1.0. Destination is Node 2 (Node 0 + span of 2)
+ "('C',1.0,3),"+ // Arc with label C and cost 1.0. Destination is Node 3 (Node 0 + span of 3)
+ "('D',1.0,1),"+ // Arc with label D and cost 1.0. Destination is Node 1 (Node 0 + span of 1)
+ ")," +
+
+ // Node 1
+ "(" +
+ "('E',1.0,4)," + // Arc with label E and cost 1.0. Destination is Node 5 (Node 1 + span of 4)
+ ")," +
+
+ // Node 2
+ "(" +
+ "('C',1.0,3)," + // Arc with label C and cost 1.0. Destination is Node 5 (Node 2 + span of 3)
+ ")," +
+
+ // Node 3
+ "(" +
+ "('D',1.0,1)," + // Arc with label D and cost 1.0. Destination is Node 4 (Node 3 + span of 1)
+ ")," +
+
+ // Node 4
+ "(" +
+ "('E',1.0,1)," + // Arc with label E and cost 1.0. Destination is Node 5 (Node 4 + span of 1)
+ ")," +
+
+ // Node 5
+ "(" +
+ "('X',1.0,1)," + // Arc with label X and cost 1.0. Destination is Node 6 (Node 5 + span of 1)
+ ")," +
+
+ // There is an implicit final state (Node 6).
+
+ ")"; // End of lattice
+
+
+ Lattice<String> lattice = Lattice.createFromString(data);
+
+ int numberOfNodes = 7;
+
+ Assert.assertEquals(lattice.size(), numberOfNodes);
+
+ Node<String> node0 = lattice.getNode(0);
+ Node<String> node1 = lattice.getNode(1);
+ Node<String> node2 = lattice.getNode(2);
+ Node<String> node3 = lattice.getNode(3);
+ Node<String> node4 = lattice.getNode(4);
+ Node<String> node5 = lattice.getNode(5);
+ Node<String> node6 = lattice.getNode(6);
+
+ Assert.assertEquals(node0.size(), 4);
+ Assert.assertEquals(node1.size(), 1);
+ Assert.assertEquals(node2.size(), 1);
+ Assert.assertEquals(node3.size(), 1);
+ Assert.assertEquals(node4.size(), 1);
+ Assert.assertEquals(node5.size(), 1);
+ Assert.assertEquals(node6.size(), 0);
+
+ // Node 0 outgoing arcs
+
+ Arc<String> arcA_0_5 = node0.getOutgoingArcs().get(0);
+ Assert.assertEquals(arcA_0_5.getLabel(), "A");
+ Assert.assertEquals(arcA_0_5.getHead(), node5);
+ Assert.assertEquals(arcA_0_5.getTail(), node0);
+
+ Assert.assertEquals(arcA_0_5.getCost(), (float) 1.0);
+
+ Arc<String> arcB_0_2 = node0.getOutgoingArcs().get(1);
+ Assert.assertEquals(arcB_0_2.getLabel(), "B");
+ Assert.assertEquals(arcB_0_2.getHead(), node2);
+ Assert.assertEquals(arcB_0_2.getTail(), node0);
+ Assert.assertEquals(arcB_0_2.getCost(), (float) 1.0);
+
+ Arc<String> arcC_0_3 = node0.getOutgoingArcs().get(2);
+ Assert.assertEquals(arcC_0_3.getLabel(), "C");
+ Assert.assertEquals(arcC_0_3.getHead(), node3);
+ Assert.assertEquals(arcC_0_3.getTail(), node0);
+ Assert.assertEquals(arcC_0_3.getCost(), (float) 1.0);
+
+ Arc<String> arcD_0_1 = node0.getOutgoingArcs().get(3);
+ Assert.assertEquals(arcD_0_1.getLabel(), "D");
+ Assert.assertEquals(arcD_0_1.getHead(), node1);
+ Assert.assertEquals(arcD_0_1.getTail(), node0);
+ Assert.assertEquals(arcD_0_1.getCost(), (float) 1.0);
+
+ // Node 1 outgoing arcs
+ Arc<String> arcE_1_5 = node1.getOutgoingArcs().get(0);
+ Assert.assertEquals(arcE_1_5.getLabel(), "E");
+ Assert.assertEquals(arcE_1_5.getHead(), node5);
+ Assert.assertEquals(arcE_1_5.getTail(), node1);
+ Assert.assertEquals(arcE_1_5.getCost(), (float) 1.0);
+
+ // Node 2 outgoing arcs
+ Arc<String> arcC_2_5 = node2.getOutgoingArcs().get(0);
+ Assert.assertEquals(arcC_2_5.getLabel(), "C");
+ Assert.assertEquals(arcC_2_5.getHead(), node5);
+ Assert.assertEquals(arcC_2_5.getTail(), node2);
+ Assert.assertEquals(arcC_2_5.getCost(), (float) 1.0);
+
+ // Node 3 outgoing arcs
+ Arc<String> arcD_3_4 = node3.getOutgoingArcs().get(0);
+ Assert.assertEquals(arcD_3_4.getLabel(), "D");
+ Assert.assertEquals(arcD_3_4.getHead(), node4);
+ Assert.assertEquals(arcD_3_4.getTail(), node3);
+ Assert.assertEquals(arcD_3_4.getCost(), (float) 1.0);
+
+ // Node 4 outgoing arcs
+ Arc<String> arcE_4_5 = node4.getOutgoingArcs().get(0);
+ Assert.assertEquals(arcE_4_5.getLabel(), "E");
+ Assert.assertEquals(arcE_4_5.getHead(), node5);
+ Assert.assertEquals(arcE_4_5.getTail(), node4);
+ Assert.assertEquals(arcE_1_5.getCost(), (float) 1.0);
+
+ // Node 5 outgoing arcs
+ Arc<String> arcX_5_6 = node5.getOutgoingArcs().get(0);
+ Assert.assertEquals(arcX_5_6.getLabel(), "X");
+ Assert.assertEquals(arcX_5_6.getHead(), node6);
+ Assert.assertEquals(arcX_5_6.getTail(), node5);
+ Assert.assertEquals(arcX_5_6.getCost(), (float) 1.0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/lattice/NodeTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/lattice/NodeTest.java b/joshua-core/src/test/java/org/apache/joshua/lattice/NodeTest.java
new file mode 100644
index 0000000..b58ba1e
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/lattice/NodeTest.java
@@ -0,0 +1,105 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.lattice;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+/**
+ * Unit tests for Node class.
+ *
+ * @author Lane Schwartz
+ * @since 2008-07-09
+ * @version $LastChangedDate$
+ */
+@Test(groups = { "lattice_node" })
+public class NodeTest {
+
+ private final int id = 12345;
+
+ private Node<String> node;
+
+ @Test
+ public void constructNode() {
+ node = new Node<String>(id);
+ Assert.assertEquals((int) node.id(), (int) id);
+ Assert.assertTrue(node.getOutgoingArcs().isEmpty());
+ Assert.assertEquals(node.size(), 0);
+ }
+
+
+ @Test(dependsOnMethods = { "constructNode" })
+ public void getNumber() {
+
+ Assert.assertEquals(node.getNumber(), id);
+
+ }
+
+
+ @Test(dependsOnMethods = { "constructNode" })
+ public void toStringTest() {
+
+ Assert.assertEquals(node.toString(), "Node-"+id);
+
+ }
+
+
+ @Test(dependsOnMethods = { "constructNode" })
+ public void addArc() {
+
+ Node<String> n2 = new Node<String>(2);
+ float w2 = (float) 0.123;
+ String l2 = "somthing cool";
+
+ Node<String> n3 = new Node<String>(3);
+ float w3 = (float) 124.78;
+ String l3 = "hurray!";
+
+ Node<String> n4 = new Node<String>(4);
+ float w4 = (float) Double.POSITIVE_INFINITY;
+ String l4 = "\u0000";
+
+ Assert.assertEquals(node.size(), 0);
+
+ node.addArc(n2,(float) w2, l2);
+ Assert.assertEquals(node.size(), 1);
+ Arc<String> a2 = node.getOutgoingArcs().get(0);
+ Assert.assertEquals(a2.getHead(), n2);
+ Assert.assertEquals(a2.getTail(), node);
+ Assert.assertEquals(a2.getCost(), w2);
+ Assert.assertEquals(a2.getLabel(), l2);
+
+ node.addArc(n3,(float) w3, l3);
+ Assert.assertEquals(node.size(), 2);
+ Arc<String> a3 = node.getOutgoingArcs().get(1);
+ Assert.assertEquals(a3.getHead(), n3);
+ Assert.assertEquals(a3.getTail(), node);
+ Assert.assertEquals(a3.getCost(), w3);
+ Assert.assertEquals(a3.getLabel(), l3);
+
+ node.addArc(n4, (float) w4, l4);
+ Assert.assertEquals(node.size(), 3);
+ Arc<String> a4 = node.getOutgoingArcs().get(2);
+ Assert.assertEquals(a4.getHead(), n4);
+ Assert.assertEquals(a4.getTail(), node);
+ Assert.assertEquals(a4.getCost(), w4);
+ Assert.assertEquals(a4.getLabel(), l4);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/packed/Benchmark.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/packed/Benchmark.java b/joshua-core/src/test/java/org/apache/joshua/packed/Benchmark.java
new file mode 100644
index 0000000..41cf2a0
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/packed/Benchmark.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.packed;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.IntBuffer;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileChannel.MapMode;
+import java.util.Random;
+
+/**
+ * This program runs a little benchmark to check reading speed on various data
+ * representations.
+ *
+ * Usage: java Benchmark PACKED_GRAMMAR_DIR TIMES
+ */
+
+public class Benchmark {
+
+
+ private static final Logger LOG = LoggerFactory.getLogger(Benchmark.class);
+
+ private IntBuffer intBuffer;
+ private MappedByteBuffer byteBuffer;
+ private int[] intArray;
+
+ public Benchmark(String dir) throws IOException {
+ File file = new File(dir + "/slice_00000.source");
+
+ FileChannel source_channel = new FileInputStream(file).getChannel();
+ int byte_size = (int) source_channel.size();
+ int int_size = byte_size / 4;
+
+ byteBuffer = source_channel.map(MapMode.READ_ONLY, 0, byte_size);
+ intBuffer = byteBuffer.asIntBuffer();
+
+ intArray = new int[int_size];
+ intBuffer.get(intArray);
+ }
+
+ public void benchmark(int times) {
+ LOG.info("Beginning benchmark.");
+
+ Random r = new Random();
+ r.setSeed(1234567890);
+ int[] positions = new int[1000];
+ for (int i = 0; i < positions.length; i++)
+ positions[i] = r.nextInt(intArray.length);
+
+ long sum;
+
+ long start_time = System.currentTimeMillis();
+
+ sum = 0;
+ for (int t = 0; t < times; t++)
+ for (int i = 0; i < positions.length; i++)
+ sum += byteBuffer.getInt(positions[i] * 4);
+ LOG.info("Sum: {}", sum);
+ long byte_time = System.currentTimeMillis();
+
+ sum = 0;
+ for (int t = 0; t < times; t++)
+ for (int i = 0; i < positions.length; i++)
+ sum += intBuffer.get(positions[i]);
+ LOG.info("Sum: {}", sum);
+ long int_time = System.currentTimeMillis();
+
+ sum = 0;
+ for (int t = 0; t < times; t++)
+ for (int i = 0; i < positions.length; i++)
+ sum += intArray[positions[i]];
+ LOG.info("Sum: {}", sum);
+ long array_time = System.currentTimeMillis();
+
+ sum = 0;
+ for (int t = 0; t < times; t++)
+ for (int i = 0; i < (intArray.length / 8); i++)
+ sum += intArray[i * 6] + intArray[i * 6 + 2];
+ LOG.info("Sum: {}", sum);
+ long mult_time = System.currentTimeMillis();
+
+ sum = 0;
+ for (int t = 0; t < times; t++) {
+ int index = 0;
+ for (int i = 0; i < (intArray.length / 8); i++) {
+ sum += intArray[index] + intArray[index + 2];
+ index += 6;
+ }
+ }
+ LOG.info("Sum: {}", sum);
+ long add_time = System.currentTimeMillis();
+
+ LOG.info("ByteBuffer: {}", (byte_time - start_time));
+ LOG.info("IntBuffer: {}", (int_time - byte_time));
+ LOG.info("Array: {}", (array_time - int_time));
+ LOG.info("Multiply: {}", (mult_time - array_time));
+ LOG.info("Add: {}", (add_time - mult_time));
+ }
+
+ public static void main(String args[]) throws IOException {
+ Benchmark pr = new Benchmark(args[0]);
+ pr.benchmark( Integer.parseInt(args[1]));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/packed/CountRules.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/packed/CountRules.java b/joshua-core/src/test/java/org/apache/joshua/packed/CountRules.java
new file mode 100644
index 0000000..5ada5ab
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/packed/CountRules.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.packed;
+
+import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileChannel.MapMode;
+
+import org.apache.joshua.corpus.Vocabulary;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+
+/**
+ * This program reads a packed representation and prints out some
+ * basic information about it.
+ *
+ * Usage: java CountRules PACKED_GRAMMAR_DIR
+ */
+
+public class CountRules {
+
+ public static void main(String args[]) {
+
+ String dir = args[0];
+
+ File file = new File(dir + "/chunk_00000.source");
+ FileInputStream stream = null;
+ FileChannel channel = null;
+ try {
+ // read the vocabulary
+ Vocabulary.read(new File(dir + "/vocabulary"));
+
+ // get the channel etc
+ stream = new FileInputStream(file);
+ channel = stream.getChannel();
+ int size = (int) channel.size();
+
+ MappedByteBuffer buffer = channel.map(MapMode.READ_ONLY, 0, size);
+ // byte[] bytes = new bytes[size];
+ // buffer.get(bytes);
+
+ // read the number of rules
+ int numRules = buffer.getInt();
+ System.out.println(String.format("There are %d source sides at the root", numRules));
+
+ // read the first symbol and its offset
+ for (int i = 0; i < numRules; i++) {
+ // String symbol = Vocabulary.word(buffer.getInt());
+ int symbol = buffer.getInt();
+ String string = Vocabulary.word(symbol);
+ int offset = buffer.getInt();
+ System.out.println(String.format("-> %s/%d [%d]", string, symbol, offset));
+ }
+
+ } catch (IOException e) {
+
+ e.printStackTrace();
+
+ } finally {
+ try {
+ if (stream != null)
+ stream.close();
+
+ if (channel != null)
+ channel.close();
+
+ } catch (IOException e) {
+
+ e.printStackTrace();
+
+ }
+ }
+
+
+ // // Read in the bytes
+ // int offset = 0;
+ // int numRead = 0;
+ // while (offset < bytes.length
+ // && (numRead=is.read(bytes, offset, bytes.length-offset)) >= 0) {
+ // offset += numRead;
+ // }
+
+ // // Ensure all the bytes have been read in
+ // if (offset < bytes.length) {
+ // throw new IOException("Could not completely read file "+file.getName());
+ // }
+
+ // // Close the input stream and return bytes
+ // is.close();
+ // return bytes;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/packed/PrintRules.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/packed/PrintRules.java b/joshua-core/src/test/java/org/apache/joshua/packed/PrintRules.java
new file mode 100644
index 0000000..af6507f
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/packed/PrintRules.java
@@ -0,0 +1,199 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.packed;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.nio.IntBuffer;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.channels.FileChannel.MapMode;
+
+import org.apache.joshua.util.quantization.Quantizer;
+import org.apache.joshua.util.quantization.QuantizerConfiguration;
+import org.apache.joshua.corpus.Vocabulary;
+
+/**
+ * This program reads a packed representation and prints out some basic
+ * information about it.
+ *
+ * Usage: java PrintRules PACKED_GRAMMAR_DIR
+ */
+
+public class PrintRules {
+
+ private QuantizerConfiguration quantization;
+
+ private int[] source;
+ private int[] target;
+ private MappedByteBuffer features;
+ private MappedByteBuffer alignments;
+
+ private int[] featureLookup;
+ private int[] alignmentLookup;
+
+ private boolean have_alignments;
+
+ public PrintRules(String dir) throws IOException {
+ File source_file = new File(dir + "/slice_00000.source");
+ File target_file = new File(dir + "/slice_00000.target");
+ File feature_file = new File(dir + "/slice_00000.features");
+ File alignment_file = new File(dir + "/slice_00000.alignments");
+
+ have_alignments = alignment_file.exists();
+
+ // Read the vocabulary.
+ Vocabulary.read(new File(dir + "/vocabulary"));
+
+ // Read the quantizer setup.
+ quantization = new QuantizerConfiguration();
+ quantization.read(dir + "/quantization");
+
+ // Get the channels etc.
+ @SuppressWarnings("resource")
+ FileChannel source_channel = new FileInputStream(source_file).getChannel();
+ int source_size = (int) source_channel.size();
+ IntBuffer source_buffer = source_channel.map(MapMode.READ_ONLY, 0,
+ source_size).asIntBuffer();
+ source = new int[source_size / 4];
+ source_buffer.get(source);
+
+ @SuppressWarnings("resource")
+ FileChannel target_channel = new FileInputStream(target_file).getChannel();
+ int target_size = (int) target_channel.size();
+ IntBuffer target_buffer = target_channel.map(MapMode.READ_ONLY, 0,
+ target_size).asIntBuffer();
+ target = new int[target_size / 4];
+ target_buffer.get(target);
+
+ @SuppressWarnings("resource")
+ FileChannel feature_channel = new FileInputStream(feature_file).getChannel();
+ int feature_size = (int) feature_channel.size();
+ features = feature_channel.map(MapMode.READ_ONLY, 0, feature_size);
+
+ if (have_alignments) {
+ @SuppressWarnings("resource")
+ FileChannel alignment_channel = new FileInputStream(alignment_file).getChannel();
+ int alignment_size = (int) alignment_channel.size();
+ alignments = alignment_channel.map(MapMode.READ_ONLY, 0, alignment_size);
+ }
+
+ int num_feature_blocks = features.getInt();
+ featureLookup = new int[num_feature_blocks];
+ // Read away data size.
+ features.getInt();
+ for (int i = 0; i < num_feature_blocks; i++)
+ featureLookup[i] = features.getInt();
+
+ int num_alignment_blocks = alignments.getInt();
+ alignmentLookup = new int[num_alignment_blocks];
+ // Read away data size.
+ alignments.getInt();
+ for (int i = 0; i < num_alignment_blocks; i++)
+ alignmentLookup[i] = alignments.getInt();
+
+ if (num_alignment_blocks != num_feature_blocks)
+ throw new RuntimeException("Number of blocks doesn't match up.");
+ }
+
+ public void traverse() {
+ traverse(0, "");
+ }
+
+ private void traverse(int position, String src_side) {
+ int num_children = source[position];
+ int[] addresses = new int[num_children];
+ int[] symbols = new int[num_children];
+ int j = position + 1;
+ for (int i = 0; i < num_children; i++) {
+ symbols[i] = source[j++];
+ addresses[i] = source[j++];
+ }
+ int num_rules = source[j++];
+ for (int i = 0; i < num_rules; i++) {
+ int lhs = source[j++];
+ int tgt_address = source[j++];
+ int data_address = source[j++];
+ printRule(src_side, lhs, tgt_address, data_address);
+ }
+ for (int i = 0; i < num_children; i++) {
+ traverse(addresses[i], src_side + " " + Vocabulary.word(symbols[i]));
+ }
+ }
+
+ private String getTarget(int pointer) {
+ StringBuilder sb = new StringBuilder();
+ do {
+ pointer = target[pointer];
+ if (pointer != -1) {
+ int symbol = target[pointer + 1];
+ if (symbol < 0)
+ sb.append(" ").append("NT" + symbol);
+ else
+ sb.append(" ").append(Vocabulary.word(symbol));
+ }
+ } while (pointer != -1);
+ return sb.toString();
+ }
+
+ private String getFeatures(int block_id) {
+ StringBuilder sb = new StringBuilder();
+
+ int data_position = featureLookup[block_id];
+ int num_features = features.getInt(data_position);
+ data_position += 4;
+ for (int i = 0; i < num_features; i++) {
+ int feature_id = features.getInt(data_position);
+ Quantizer quantizer = quantization.get(feature_id);
+ sb.append(" " + Vocabulary.word(feature_id) + "=" +
+ quantizer.read(features, data_position));
+ data_position += 4 + quantizer.size();
+ }
+ return sb.toString();
+ }
+
+ private String getAlignments(int block_id) {
+ StringBuilder sb = new StringBuilder();
+
+ int data_position = alignmentLookup[block_id];
+ byte num_points = alignments.get(data_position);
+ for (int i = 0; i < num_points; i++) {
+ byte src = alignments.get(data_position + 1 + 2 * i);
+ byte tgt = alignments.get(data_position + 2 + 2 * i);
+
+ sb.append(" " + src + "-" + tgt);
+ }
+ return sb.toString();
+ }
+
+ private void printRule(String src_side, int lhs, int tgt_address,
+ int data_address) {
+ System.out.println(Vocabulary.word(lhs) + " |||" +
+ src_side + " |||" +
+ getTarget(tgt_address) + " |||" +
+ getFeatures(data_address) +
+ (have_alignments ? " |||" + getAlignments(data_address) : ""));
+ }
+
+ public static void main(String args[]) throws IOException {
+ PrintRules pr = new PrintRules(args[0]);
+ pr.traverse();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/packed/README
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/packed/README b/joshua-core/src/test/java/org/apache/joshua/packed/README
new file mode 100644
index 0000000..3cb52b8
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/packed/README
@@ -0,0 +1,6 @@
+# This code generates the packed grammar representation from the grammar file
+rm -rf small_packed
+java -cp /home/hltcoe/mpost/code/joshua/bin:. joshua.tools.GrammarPacker packer.config small_packed small_grammar
+
+# This compiles and reads the grammar file
+java -cp $JOSHUA/bin:. CountRules small_packed
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/packed/VocabTest.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/packed/VocabTest.java b/joshua-core/src/test/java/org/apache/joshua/packed/VocabTest.java
new file mode 100644
index 0000000..523df4c
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/packed/VocabTest.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.packed;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.joshua.corpus.Vocabulary;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class VocabTest {
+
+ private static final Logger LOG = LoggerFactory.getLogger(VocabTest.class);
+
+ //FIXME: no main() in automated test case,
+ public static void main(String args[]) {
+
+ int numWords = 0;
+ try {
+ String dir = args[0];
+
+ boolean read = Vocabulary.read(new File(dir + "/vocabulary"));
+ if (! read) {
+ System.err.println("VocabTest: Failed to read the vocabulary.");
+ System.exit(1);
+ }
+
+ int id = 0;
+ while (Vocabulary.hasId(id)) {
+ String word = Vocabulary.word(id);
+ System.out.println(String.format("VOCAB: %d\t%s", id, word));
+ numWords++;
+ id++;
+ }
+ } catch (IOException e) {
+ LOG.error(e.getMessage(), e);
+ }
+
+ System.out.println("read " + numWords + " words");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/test/java/org/apache/joshua/packed/packer.config
----------------------------------------------------------------------
diff --git a/joshua-core/src/test/java/org/apache/joshua/packed/packer.config b/joshua-core/src/test/java/org/apache/joshua/packed/packer.config
new file mode 100644
index 0000000..73edb1a
--- /dev/null
+++ b/joshua-core/src/test/java/org/apache/joshua/packed/packer.config
@@ -0,0 +1,6 @@
+#chunk_size 30000
+chunk_size 2500000
+
+quantizer boolean Abstract,Adjacent,ContainsX,GlueRule,Lexical,Monotonic,TargetTerminalsButNoSource
+quantizer float LexprobSourceGivenTarget,LexprobTargetGivenSource,PhrasePenalty,RarityPenalty,SourcePhraseGivenTarget,SourceTerminalsButNoTarget,TargetPhraseGivenSource
+quantizer byte TargetWords
[60/60] incubator-joshua git commit: Merge branch
'maven-multi-module' of https://github.com/logogin/incubator-joshua into
maven-multi-module
Posted by mj...@apache.org.
Merge branch 'maven-multi-module' of https://github.com/logogin/incubator-joshua into maven-multi-module
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/1bb8a203
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/1bb8a203
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/1bb8a203
Branch: refs/heads/maven-multi-module
Commit: 1bb8a203727c8f7e1eb7090be9c785606c415d38
Parents: f178334 ae3a5df
Author: Matt Post <po...@cs.jhu.edu>
Authored: Thu Jun 23 13:19:18 2016 -0400
Committer: Matt Post <po...@cs.jhu.edu>
Committed: Thu Jun 23 13:19:18 2016 -0400
----------------------------------------------------------------------
joshua-core/.gitignore | 1 +
joshua-core/pom.xml | 143 +
joshua-core/resources/berkeley_lm/lm | 16 +
joshua-core/resources/berkeley_lm/lm.berkeleylm | Bin 0 -> 4294 bytes
.../resources/berkeley_lm/lm.berkeleylm.gz | Bin 0 -> 1786 bytes
joshua-core/resources/berkeley_lm/lm.gz | Bin 0 -> 162 bytes
joshua-core/resources/grammar.glue | 4 +
.../resources/kbest_extraction/glue-grammar | 3 +
joshua-core/resources/kbest_extraction/grammar | 25 +
.../resources/kbest_extraction/joshua.config | 27 +
joshua-core/resources/kbest_extraction/lm.gz | Bin 0 -> 2466496 bytes
.../resources/kbest_extraction/output.gold | 3126 +++
.../kbest_extraction/output.scores.gold | 3126 +++
joshua-core/resources/phrase_decoder/config | 29 +
.../resources/phrase_decoder/constrained.config | 28 +
.../phrase_decoder/constrained.output.gold | 5 +
joshua-core/resources/phrase_decoder/lm.1.gz | Bin 0 -> 2235 bytes
.../resources/phrase_decoder/output.gold | 1 +
joshua-core/resources/phrase_decoder/rules.1.gz | Bin 0 -> 2998042 bytes
joshua-core/resources/wa_grammar | 3 +
joshua-core/resources/wa_grammar.packed/config | 1 +
.../resources/wa_grammar.packed/encoding | Bin 0 -> 154 bytes
.../wa_grammar.packed/slice_00000.alignments | Bin 0 -> 45 bytes
.../wa_grammar.packed/slice_00000.features | Bin 0 -> 47 bytes
.../wa_grammar.packed/slice_00000.source | Bin 0 -> 204 bytes
.../wa_grammar.packed/slice_00000.target | Bin 0 -> 128 bytes
.../wa_grammar.packed/slice_00000.target.lookup | Bin 0 -> 32 bytes
.../resources/wa_grammar.packed/vocabulary | Bin 0 -> 238 bytes
.../java/org/apache/joshua/adagrad/AdaGrad.java | 160 +
.../org/apache/joshua/adagrad/AdaGradCore.java | 3127 +++
.../org/apache/joshua/adagrad/Optimizer.java | 728 +
.../apache/joshua/corpus/AbstractPhrase.java | 133 +
.../org/apache/joshua/corpus/BasicPhrase.java | 97 +
.../apache/joshua/corpus/ContiguousPhrase.java | 127 +
.../java/org/apache/joshua/corpus/Corpus.java | 160 +
.../java/org/apache/joshua/corpus/Phrase.java | 117 +
.../java/org/apache/joshua/corpus/Span.java | 175 +
.../org/apache/joshua/corpus/SymbolTable.java | 327 +
.../apache/joshua/corpus/TerminalIterator.java | 85 +
.../org/apache/joshua/corpus/Vocabulary.java | 301 +
.../joshua/corpus/syntax/ArraySyntaxTree.java | 411 +
.../apache/joshua/corpus/syntax/SyntaxTree.java | 34 +
.../org/apache/joshua/decoder/ArgsParser.java | 118 +
.../java/org/apache/joshua/decoder/BLEU.java | 562 +
.../java/org/apache/joshua/decoder/Decoder.java | 813 +
.../apache/joshua/decoder/DecoderThread.java | 201 +
.../joshua/decoder/JoshuaConfiguration.java | 729 +
.../apache/joshua/decoder/JoshuaDecoder.java | 148 +
.../joshua/decoder/NbestMinRiskReranker.java | 446 +
.../joshua/decoder/StructuredTranslation.java | 157 +
.../decoder/StructuredTranslationFactory.java | 120 +
.../java/org/apache/joshua/decoder/Support.java | 86 +
.../org/apache/joshua/decoder/Translation.java | 239 +
.../org/apache/joshua/decoder/Translations.java | 158 +
.../joshua/decoder/chart_parser/Cell.java | 294 +
.../joshua/decoder/chart_parser/Chart.java | 746 +
.../decoder/chart_parser/ComputeNodeResult.java | 225 +
.../decoder/chart_parser/CubePruneState.java | 114 +
.../joshua/decoder/chart_parser/DotChart.java | 476 +
.../joshua/decoder/chart_parser/SourcePath.java | 63 +
.../decoder/chart_parser/StateConstraint.java | 75 +
.../joshua/decoder/chart_parser/SuperNode.java | 62 +
.../decoder/chart_parser/package-info.java | 24 +
.../joshua/decoder/ff/ArityPhrasePenalty.java | 73 +
.../joshua/decoder/ff/FeatureFunction.java | 364 +
.../apache/joshua/decoder/ff/FeatureVector.java | 385 +
.../joshua/decoder/ff/LabelCombinationFF.java | 62 +
.../joshua/decoder/ff/LabelSubstitutionFF.java | 131 +
.../joshua/decoder/ff/LexicalFeatures.java | 152 +
.../apache/joshua/decoder/ff/OOVPenalty.java | 108 +
.../apache/joshua/decoder/ff/PhraseModel.java | 134 +
.../apache/joshua/decoder/ff/PhrasePenalty.java | 87 +
.../apache/joshua/decoder/ff/RuleCountBin.java | 77 +
.../org/apache/joshua/decoder/ff/RuleFF.java | 126 +
.../apache/joshua/decoder/ff/RuleLength.java | 52 +
.../decoder/ff/RulePropertiesQuerying.java | 49 +
.../org/apache/joshua/decoder/ff/RuleShape.java | 99 +
.../joshua/decoder/ff/SourceDependentFF.java | 29 +
.../apache/joshua/decoder/ff/SourcePathFF.java | 63 +
.../apache/joshua/decoder/ff/StatefulFF.java | 88 +
.../apache/joshua/decoder/ff/StatelessFF.java | 79 +
.../apache/joshua/decoder/ff/TargetBigram.java | 216 +
.../apache/joshua/decoder/ff/WordPenalty.java | 80 +
.../ff/fragmentlm/ConcatenationIterator.java | 93 +
.../decoder/ff/fragmentlm/FragmentLMFF.java | 368 +
.../ff/fragmentlm/PennTreebankReader.java | 135 +
.../joshua/decoder/ff/fragmentlm/Tree.java | 779 +
.../joshua/decoder/ff/fragmentlm/Trees.java | 270 +
.../apache/joshua/decoder/ff/lm/AbstractLM.java | 122 +
.../apache/joshua/decoder/ff/lm/ArpaFile.java | 328 +
.../apache/joshua/decoder/ff/lm/ArpaNgram.java | 73 +
.../ff/lm/DefaultNGramLanguageModel.java | 134 +
.../org/apache/joshua/decoder/ff/lm/KenLM.java | 233 +
.../joshua/decoder/ff/lm/LanguageModelFF.java | 495 +
.../decoder/ff/lm/NGramLanguageModel.java | 73 +
.../ff/lm/StateMinimizingLanguageModel.java | 193 +
.../joshua/decoder/ff/lm/berkeley_lm/LICENSE | 13 +
.../ff/lm/berkeley_lm/LMGrammarBerkeley.java | 205 +
.../joshua/decoder/ff/lm/berkeley_lm/README | 5 +
.../ff/lm/berkeley_lm/SymbolTableWrapper.java | 102 +
.../ff/lm/bloomfilter_lm/BloomFilter.java | 215 +
.../BloomFilterLanguageModel.java | 560 +
.../ff/lm/bloomfilter_lm/package-info.java | 25 +
.../joshua/decoder/ff/lm/buildin_lm/TrieLM.java | 331 +
.../decoder/ff/lm/buildin_lm/package-info.java | 19 +
.../joshua/decoder/ff/lm/package-info.java | 42 +
.../apache/joshua/decoder/ff/package-info.java | 42 +
.../joshua/decoder/ff/phrase/Distortion.java | 71 +
.../ff/similarity/EdgePhraseSimilarityFF.java | 279 +
.../decoder/ff/state_maintenance/DPState.java | 34 +
.../ff/state_maintenance/KenLMState.java | 56 +
.../ff/state_maintenance/NgramDPState.java | 100 +
.../joshua/decoder/ff/tm/AbstractGrammar.java | 229 +
.../decoder/ff/tm/BasicRuleCollection.java | 101 +
.../joshua/decoder/ff/tm/CreateGlueGrammar.java | 126 +
.../apache/joshua/decoder/ff/tm/Grammar.java | 120 +
.../joshua/decoder/ff/tm/GrammarReader.java | 158 +
.../org/apache/joshua/decoder/ff/tm/Rule.java | 635 +
.../joshua/decoder/ff/tm/RuleCollection.java | 76 +
.../decoder/ff/tm/SentenceFilteredGrammar.java | 366 +
.../org/apache/joshua/decoder/ff/tm/Trie.java | 108 +
.../ff/tm/UnsortedRuleCollectionException.java | 40 +
.../decoder/ff/tm/format/HieroFormatReader.java | 106 +
.../decoder/ff/tm/format/MosesFormatReader.java | 108 +
.../ff/tm/hash_based/ExtensionIterator.java | 73 +
.../tm/hash_based/MemoryBasedBatchGrammar.java | 279 +
.../ff/tm/hash_based/MemoryBasedRuleBin.java | 59 +
.../ff/tm/hash_based/MemoryBasedTrie.java | 88 +
.../decoder/ff/tm/hash_based/package-info.java | 23 +
.../joshua/decoder/ff/tm/package-info.java | 25 +
.../decoder/ff/tm/packed/PackedGrammar.java | 1064 +
.../ff/tm/packed/SliceAggregatingTrie.java | 236 +
.../decoder/hypergraph/AlignedSourceTokens.java | 112 +
.../decoder/hypergraph/AllSpansWalker.java | 63 +
.../hypergraph/DefaultInsideOutside.java | 407 +
.../hypergraph/FeatureVectorExtractor.java | 80 +
.../joshua/decoder/hypergraph/ForestWalker.java | 79 +
.../GrammarBuilderWalkerFunction.java | 179 +
.../joshua/decoder/hypergraph/HGNode.java | 331 +
.../joshua/decoder/hypergraph/HyperEdge.java | 101 +
.../joshua/decoder/hypergraph/HyperGraph.java | 163 +
.../decoder/hypergraph/HyperGraphPruning.java | 176 +
.../decoder/hypergraph/KBestExtractor.java | 1052 +
.../hypergraph/OutputStringExtractor.java | 195 +
.../hypergraph/StringToTreeConverter.java | 74 +
.../hypergraph/TrivialInsideOutside.java | 31 +
.../decoder/hypergraph/ViterbiExtractor.java | 178 +
.../decoder/hypergraph/WalkerFunction.java | 37 +
.../hypergraph/WordAlignmentExtractor.java | 134 +
.../decoder/hypergraph/WordAlignmentState.java | 192 +
.../joshua/decoder/hypergraph/package-info.java | 25 +
.../apache/joshua/decoder/io/DeNormalize.java | 203 +
.../apache/joshua/decoder/io/JSONMessage.java | 159 +
.../decoder/io/TranslationRequestStream.java | 179 +
.../org/apache/joshua/decoder/package-info.java | 26 +
.../apache/joshua/decoder/phrase/Candidate.java | 241 +
.../decoder/phrase/CandidateComparator.java | 28 +
.../apache/joshua/decoder/phrase/Coverage.java | 235 +
.../apache/joshua/decoder/phrase/Future.java | 119 +
.../apache/joshua/decoder/phrase/Header.java | 87 +
.../joshua/decoder/phrase/Hypothesis.java | 154 +
.../org/apache/joshua/decoder/phrase/Note.java | 44 +
.../joshua/decoder/phrase/PhraseChart.java | 197 +
.../joshua/decoder/phrase/PhraseTable.java | 183 +
.../org/apache/joshua/decoder/phrase/Stack.java | 229 +
.../apache/joshua/decoder/phrase/Stacks.java | 271 +
.../joshua/decoder/phrase/TargetPhrases.java | 80 +
.../decoder/segment_file/ConstraintRule.java | 100 +
.../decoder/segment_file/ConstraintSpan.java | 80 +
.../decoder/segment_file/ParseTreeInput.java | 40 +
.../decoder/segment_file/ParsedSentence.java | 56 +
.../joshua/decoder/segment_file/Sentence.java | 450 +
.../joshua/decoder/segment_file/Token.java | 158 +
.../decoder/segment_file/package-info.java | 25 +
.../java/org/apache/joshua/lattice/Arc.java | 117 +
.../java/org/apache/joshua/lattice/Lattice.java | 587 +
.../java/org/apache/joshua/lattice/Node.java | 159 +
.../lattice/NodeIdentifierComparator.java | 41 +
.../org/apache/joshua/lattice/package-info.java | 22 +
.../java/org/apache/joshua/metrics/BLEU.java | 575 +
.../org/apache/joshua/metrics/BLEU_SBP.java | 63 +
.../apache/joshua/metrics/EvaluationMetric.java | 409 +
.../apache/joshua/metrics/GradeLevelBLEU.java | 280 +
.../java/org/apache/joshua/metrics/METEOR.java | 235 +
.../joshua/metrics/MinimumChangeBLEU.java | 223 +
.../joshua/metrics/NewMetric.java.template | 134 +
.../java/org/apache/joshua/metrics/Precis.java | 334 +
.../joshua/metrics/PrecisMinusSourceBLEU.java | 182 +
.../java/org/apache/joshua/metrics/SARI.java | 681 +
.../org/apache/joshua/metrics/SourceBLEU.java | 107 +
.../java/org/apache/joshua/metrics/TER.java | 460 +
.../org/apache/joshua/metrics/TERMinusBLEU.java | 194 +
.../org/apache/joshua/metrics/TercomRunner.java | 115 +
.../org/apache/joshua/metrics/ZeroOneLoss.java | 88 +
.../main/java/org/apache/joshua/mira/MIRA.java | 160 +
.../java/org/apache/joshua/mira/MIRACore.java | 3112 +++
.../java/org/apache/joshua/mira/Optimizer.java | 643 +
.../joshua/oracle/OracleExtractionHG.java | 797 +
.../apache/joshua/oracle/OracleExtractor.java | 58 +
.../java/org/apache/joshua/oracle/SplitHg.java | 300 +
.../org/apache/joshua/oracle/package-info.java | 26 +
.../apache/joshua/pro/ClassifierInterface.java | 41 +
.../org/apache/joshua/pro/ClassifierMegaM.java | 121 +
.../apache/joshua/pro/ClassifierPerceptron.java | 114 +
.../org/apache/joshua/pro/ClassifierSVM.java | 140 +
.../java/org/apache/joshua/pro/Optimizer.java | 454 +
.../main/java/org/apache/joshua/pro/PRO.java | 159 +
.../java/org/apache/joshua/pro/PROCore.java | 3027 +++
.../org/apache/joshua/server/ServerThread.java | 294 +
.../org/apache/joshua/server/TcpServer.java | 65 +
.../joshua/subsample/AlignedSubsampler.java | 103 +
.../org/apache/joshua/subsample/Alignment.java | 92 +
.../org/apache/joshua/subsample/BiCorpus.java | 186 +
.../joshua/subsample/BiCorpusFactory.java | 83 +
.../org/apache/joshua/subsample/PhrasePair.java | 72 +
.../apache/joshua/subsample/PhraseReader.java | 46 +
.../apache/joshua/subsample/PhraseWriter.java | 79 +
.../org/apache/joshua/subsample/Subsampler.java | 246 +
.../apache/joshua/subsample/SubsamplerCLI.java | 141 +
.../apache/joshua/subsample/package-info.java | 25 +
.../org/apache/joshua/tools/GrammarPacker.java | 959 +
.../apache/joshua/tools/GrammarPackerCli.java | 156 +
.../org/apache/joshua/tools/LabelPhrases.java | 111 +
.../org/apache/joshua/tools/TestSetFilter.java | 383 +
.../java/org/apache/joshua/ui/Orientation.java | 23 +
.../org/apache/joshua/ui/StartupWindow.java | 87 +
.../java/org/apache/joshua/ui/package-info.java | 22 +
.../ui/tree_visualizer/DerivationTree.java | 103 +
.../ui/tree_visualizer/DerivationTreeEdge.java | 27 +
.../DerivationTreeTransformer.java | 117 +
.../ui/tree_visualizer/DerivationViewer.java | 128 +
.../tree_visualizer/DerivationViewerApplet.java | 51 +
.../apache/joshua/ui/tree_visualizer/Node.java | 59 +
.../ui/tree_visualizer/browser/Browser.java | 237 +
.../browser/DerivationTreeFrame.java | 253 +
.../browser/TranslationInfo.java | 56 +
.../joshua/ui/tree_visualizer/tree/Tree.java | 283 +
.../java/org/apache/joshua/util/Algorithms.java | 85 +
.../main/java/org/apache/joshua/util/Bits.java | 128 +
.../java/org/apache/joshua/util/BotMap.java | 94 +
.../main/java/org/apache/joshua/util/Cache.java | 176 +
.../java/org/apache/joshua/util/ChartSpan.java | 88 +
.../apache/joshua/util/CommandLineParser.java | 738 +
.../java/org/apache/joshua/util/Constants.java | 36 +
.../java/org/apache/joshua/util/Counted.java | 92 +
.../java/org/apache/joshua/util/Counts.java | 306 +
.../org/apache/joshua/util/ExtractTopCand.java | 188 +
.../org/apache/joshua/util/FileUtility.java | 318 +
.../org/apache/joshua/util/FormatUtils.java | 245 +
.../org/apache/joshua/util/IntegerPair.java | 36 +
.../java/org/apache/joshua/util/JoshuaEval.java | 624 +
.../java/org/apache/joshua/util/ListUtil.java | 95 +
.../main/java/org/apache/joshua/util/Lists.java | 567 +
.../apache/joshua/util/NBestListUtility.java | 74 +
.../main/java/org/apache/joshua/util/Ngram.java | 105 +
.../org/apache/joshua/util/NullIterator.java | 65 +
.../apache/joshua/util/PackedGrammarServer.java | 87 +
.../main/java/org/apache/joshua/util/Pair.java | 130 +
.../java/org/apache/joshua/util/Platform.java | 27 +
.../org/apache/joshua/util/QuietFormatter.java | 36 +
.../main/java/org/apache/joshua/util/Regex.java | 143 +
.../org/apache/joshua/util/ReverseOrder.java | 39 +
.../org/apache/joshua/util/SampledList.java | 69 +
.../org/apache/joshua/util/SocketUtility.java | 144 +
.../org/apache/joshua/util/StreamGobbler.java | 50 +
.../joshua/util/UnicodeCharacterName.java | 22466 +++++++++++++++++
.../apache/joshua/util/encoding/Analyzer.java | 235 +
.../joshua/util/encoding/EightBitQuantizer.java | 92 +
.../util/encoding/EncoderConfiguration.java | 160 +
.../joshua/util/encoding/EncoderFactory.java | 42 +
.../util/encoding/FeatureTypeAnalyzer.java | 254 +
.../joshua/util/encoding/FloatEncoder.java | 39 +
.../apache/joshua/util/encoding/IntEncoder.java | 39 +
.../util/encoding/PrimitiveFloatEncoder.java | 129 +
.../util/encoding/PrimitiveIntEncoder.java | 111 +
.../joshua/util/encoding/VariableQuantizer.java | 106 +
.../org/apache/joshua/util/io/BinaryIn.java | 91 +
.../org/apache/joshua/util/io/BinaryOut.java | 505 +
.../apache/joshua/util/io/IndexedReader.java | 155 +
.../org/apache/joshua/util/io/LineReader.java | 368 +
.../org/apache/joshua/util/io/NullReader.java | 63 +
.../joshua/util/io/ProgressInputStream.java | 82 +
.../java/org/apache/joshua/util/io/Reader.java | 51 +
.../org/apache/joshua/util/io/package-info.java | 22 +
.../org/apache/joshua/util/package-info.java | 22 +
.../util/quantization/BooleanQuantizer.java | 45 +
.../joshua/util/quantization/Quantizer.java | 45 +
.../quantization/QuantizerConfiguration.java | 119 +
.../util/quantization/QuantizerFactory.java | 50 +
.../util/quantization/StatelessQuantizer.java | 38 +
.../joshua/util/quantization/package-info.java | 19 +
.../joshua/zmert/IntermediateOptimizer.java | 991 +
.../java/org/apache/joshua/zmert/MertCore.java | 3191 +++
.../java/org/apache/joshua/zmert/ZMERT.java | 156 +
.../org/apache/joshua/zmert/package-info.java | 24 +
joshua-core/src/main/resources/log4j.properties | 20 +
joshua-core/src/overview.html | 41 +
.../apache/joshua/corpus/CorpusArrayTest.java | 177 +
.../java/org/apache/joshua/corpus/SpanTest.java | 47 +
.../apache/joshua/corpus/VocabularyTest.java | 135 +
.../joshua/corpus/vocab/VocabularyTest.java | 180 +
.../ArtificialGrammarAndCorpusCreater.java | 130 +
.../joshua/decoder/DecoderThreadTest.java | 172 +
.../joshua/decoder/JoshuaDecoderTest.java | 83 +
.../joshua/decoder/TestConfigFileCreater.java | 184 +
.../apache/joshua/decoder/TranslationsTest.java | 87 +
.../decoder/ff/ArityPhrasePenaltyFFTest.java | 64 +
.../joshua/decoder/ff/lm/ArpaFileTest.java | 226 +
.../decoder/ff/lm/LanguageModelFFTest.java | 95 +
.../LMBerkeleySentenceProbablityTest.java | 47 +
.../lm/berkeley_lm/LMGrammarBerkeleyTest.java | 80 +
.../joshua/decoder/io/DeNormalizeTest.java | 273 +
.../decoder/io/TranslationRequestTest.java | 149 +
.../kbest_extraction/KBestExtractionTest.java | 80 +
.../joshua/decoder/phrase/CoverageTest.java | 140 +
.../ConstrainedPhraseDecodingTest.java | 77 +
.../phrase/decode/PhraseDecodingTest.java | 77 +
.../segment_file/AlmostTooLongSentenceTest.java | 96 +
.../decoder/segment_file/SentenceTest.java | 109 +
.../java/org/apache/joshua/lattice/ArcTest.java | 86 +
.../org/apache/joshua/lattice/LatticeTest.java | 192 +
.../org/apache/joshua/lattice/NodeTest.java | 105 +
.../org/apache/joshua/packed/Benchmark.java | 126 +
.../org/apache/joshua/packed/CountRules.java | 110 +
.../org/apache/joshua/packed/PrintRules.java | 199 +
.../test/java/org/apache/joshua/packed/README | 6 +
.../org/apache/joshua/packed/VocabTest.java | 58 +
.../java/org/apache/joshua/packed/packer.config | 6 +
.../java/org/apache/joshua/packed/small_grammar | 20000 +++++++++++++++
.../test/java/org/apache/joshua/packed/test.sh | 20 +
.../apache/joshua/system/AlignmentMapTest.java | 73 +
.../org/apache/joshua/system/KenLmTest.java | 95 +
.../system/MultithreadedTranslationTests.java | 156 +
.../joshua/system/StructuredOutputTest.java | 118 +
.../system/StructuredTranslationTest.java | 274 +
.../ui/tree_visualizer/tree/TreeTest.java | 111 +
.../java/org/apache/joshua/util/BitsTest.java | 187 +
.../java/org/apache/joshua/util/CacheTest.java | 53 +
.../java/org/apache/joshua/util/CountsTest.java | 98 +
.../org/apache/joshua/util/FormatUtilsTest.java | 74 +
.../org/apache/joshua/util/io/BinaryTest.java | 74 +
.../java/org/apache/joshua/zmert/BLEUTest.java | 132 +
.../src/test/resources/bn-en/hiero/.gitignore | 4 +
.../src/test/resources/bn-en/hiero/class.map | 5140 ++++
.../resources/bn-en/hiero/class_lm_2gram.gz | Bin 0 -> 18052 bytes
.../resources/bn-en/hiero/class_lm_9gram.gz | Bin 0 -> 12733137 bytes
.../src/test/resources/bn-en/hiero/glue-grammar | 3 +
.../src/test/resources/bn-en/hiero/grammar.gz | Bin 0 -> 518164 bytes
.../src/test/resources/bn-en/hiero/input.bn | 100 +
.../bn-en/hiero/joshua-berkeleylm.config | 46 +
.../resources/bn-en/hiero/joshua-classlm.config | 51 +
.../test/resources/bn-en/hiero/joshua.config | 50 +
.../src/test/resources/bn-en/hiero/lm.gz | Bin 0 -> 2466496 bytes
.../resources/bn-en/hiero/output-classlm.gold | 887 +
.../src/test/resources/bn-en/hiero/output.gold | 805 +
.../test/resources/bn-en/hiero/output.gold.bleu | 14 +
.../bn-en/hiero/output.scores.berkeleylm.gold | 100 +
.../resources/bn-en/hiero/output.scores.gold | 805 +
.../test/resources/bn-en/hiero/reference.en.0 | 100 +
.../test/resources/bn-en/hiero/reference.en.1 | 100 +
.../test/resources/bn-en/hiero/reference.en.2 | 100 +
.../test/resources/bn-en/hiero/reference.en.3 | 100 +
.../resources/bn-en/hiero/test-berkeleylm.sh | 33 +
.../test/resources/bn-en/hiero/test-classlm.sh | 32 +
.../test/resources/bn-en/hiero/test-filter.sh | 35 +
.../src/test/resources/bn-en/hiero/test.sh | 35 +
.../src/test/resources/bn-en/hiero/topN.pl | 18 +
.../src/test/resources/bn-en/packed/.gitignore | 3 +
.../test/resources/bn-en/packed/grammar.glue | 5673 +++++
.../src/test/resources/bn-en/packed/grammar.gz | Bin 0 -> 3540984 bytes
.../bn-en/packed/grammar.packed/encoding | Bin 0 -> 767 bytes
.../packed/grammar.packed/slice_00000.features | Bin 0 -> 4631480 bytes
.../packed/grammar.packed/slice_00000.source | Bin 0 -> 4240012 bytes
.../packed/grammar.packed/slice_00000.target | Bin 0 -> 162776 bytes
.../grammar.packed/slice_00000.target.lookup | Bin 0 -> 56 bytes
.../bn-en/packed/grammar.packed/vocabulary | Bin 0 -> 136522 bytes
.../src/test/resources/bn-en/packed/input.bn | 100 +
.../test/resources/bn-en/packed/joshua.config | 47 +
.../src/test/resources/bn-en/packed/lm.gz | Bin 0 -> 2466496 bytes
.../src/test/resources/bn-en/packed/output.gold | 862 +
.../resources/bn-en/packed/output.scores.gold | 862 +
.../test/resources/bn-en/packed/reference.en.0 | 100 +
.../test/resources/bn-en/packed/reference.en.1 | 100 +
.../test/resources/bn-en/packed/reference.en.2 | 100 +
.../test/resources/bn-en/packed/reference.en.3 | 100 +
.../resources/bn-en/packed/reference.en.all | 400 +
.../src/test/resources/bn-en/packed/test.sh | 20 +
.../src/test/resources/bn-en/samt/grammar.glue | 5673 +++++
.../src/test/resources/bn-en/samt/grammar.gz | Bin 0 -> 3847934 bytes
.../src/test/resources/bn-en/samt/input.bn | 100 +
.../src/test/resources/bn-en/samt/joshua.config | 47 +
joshua-core/src/test/resources/bn-en/samt/lm.gz | Bin 0 -> 2466496 bytes
.../src/test/resources/bn-en/samt/output.gold | 0
.../test/resources/bn-en/samt/output.gold.bleu | 14 +
.../resources/bn-en/samt/output.scores.gold | 862 +
.../test/resources/bn-en/samt/reference.en.0 | 100 +
.../test/resources/bn-en/samt/reference.en.1 | 100 +
.../test/resources/bn-en/samt/reference.en.2 | 100 +
.../test/resources/bn-en/samt/reference.en.3 | 100 +
.../src/test/resources/bn-en/samt/test.sh | 35 +
joshua-core/src/test/resources/data/tiny.en | 5 +
.../resources/decoder/constrained/.gitignore | 4 +
.../resources/decoder/constrained/glue-grammar | 3 +
.../resources/decoder/constrained/gold.scores | 27 +
.../resources/decoder/constrained/grammar.gz | Bin 0 -> 518164 bytes
.../test/resources/decoder/constrained/input.bn | 8 +
.../resources/decoder/constrained/joshua.config | 45 +
.../test/resources/decoder/constrained/lm.gz | Bin 0 -> 2466496 bytes
.../resources/decoder/constrained/output.bleu | 0
.../resources/decoder/constrained/output.gold | 30 +
.../test/resources/decoder/constrained/test.sh | 30 +
.../test/resources/decoder/constrained/weights | 22 +
.../resources/decoder/denormalization/input.txt | 1 +
.../decoder/denormalization/output.expected | 1 +
.../resources/decoder/denormalization/test.sh | 30 +
.../src/test/resources/decoder/dont-crash/input | 10 +
.../test/resources/decoder/dont-crash/test.sh | 29 +
.../resources/decoder/empty-test/.gitignore | 3 +
.../src/test/resources/decoder/empty-test/input | 1 +
.../resources/decoder/empty-test/output.gold | 1 +
.../test/resources/decoder/empty-test/test.sh | 29 +
.../resources/decoder/fragmentlm/fragments.txt | 7 +
.../src/test/resources/decoder/fragmentlm/glue | 1 +
.../test/resources/decoder/fragmentlm/grammar | 4 +
.../src/test/resources/decoder/fragmentlm/input | 1 +
.../resources/decoder/fragmentlm/joshua.config | 109 +
.../resources/decoder/fragmentlm/mapping.txt | 4 +
.../test/resources/decoder/fragmentlm/test.sh | 30 +
.../decoder/k-best-extraction/glue-grammar | 3 +
.../resources/decoder/k-best-extraction/grammar | 25 +
.../decoder/k-best-extraction/input.txt | 1 +
.../decoder/k-best-extraction/joshua.config | 27 +
.../resources/decoder/k-best-extraction/lm.gz | Bin 0 -> 2466496 bytes
.../decoder/k-best-extraction/output.gold | 3126 +++
.../k-best-extraction/output.scores.gold | 3126 +++
.../resources/decoder/k-best-extraction/test.sh | 33 +
.../resources/decoder/left-state/glue-grammar | 3 +
.../resources/decoder/left-state/grammar.gz | Bin 0 -> 518164 bytes
.../test/resources/decoder/left-state/input.bn | 2 +
.../resources/decoder/left-state/joshua.config | 44 +
.../src/test/resources/decoder/left-state/lm.gz | Bin 0 -> 2466496 bytes
.../resources/decoder/left-state/output.gold | 600 +
.../decoder/left-state/output.scores.gold | 600 +
.../test/resources/decoder/left-state/test.sh | 33 +
.../test/resources/decoder/lowercaser/config | 140 +
.../resources/decoder/lowercaser/grammar.glue | 4 +
.../resources/decoder/lowercaser/grammar.test | 1 +
.../resources/decoder/lowercaser/output.gold | 5 +
.../test/resources/decoder/lowercaser/test.sh | 40 +
.../decoder/metadata/add_rule/output.gold | 4 +
.../resources/decoder/metadata/add_rule/test.sh | 32 +
.../resources/decoder/moses-compat/n-best.txt | 0
.../decoder/moses-compat/output.expected | 6 +
.../test/resources/decoder/moses-compat/test.sh | 40 +
.../test/resources/decoder/n-ary/glue-grammar | 3 +
.../test/resources/decoder/n-ary/gold.scores | 2 +
.../src/test/resources/decoder/n-ary/grammar | 9 +
.../src/test/resources/decoder/n-ary/input.txt | 2 +
.../test/resources/decoder/n-ary/joshua.config | 22 +
.../src/test/resources/decoder/n-ary/lm.gz | Bin 0 -> 2466496 bytes
.../test/resources/decoder/n-ary/output.bleu | 0
.../test/resources/decoder/n-ary/output.gold | 2 +
.../src/test/resources/decoder/n-ary/test.sh | 33 +
.../src/test/resources/decoder/n-ary/weights | 6 +
.../decoder/num_translation_options/README | 1 +
.../num_translation_options/glue-grammar | 3 +
.../decoder/num_translation_options/grammar.gz | Bin 0 -> 119 bytes
.../grammar.packed/encoding | Bin 0 -> 32 bytes
.../grammar.packed/slice_00000.features | Bin 0 -> 43 bytes
.../grammar.packed/slice_00000.source | Bin 0 -> 132 bytes
.../grammar.packed/slice_00000.target | Bin 0 -> 120 bytes
.../grammar.packed/slice_00000.target.lookup | Bin 0 -> 32 bytes
.../grammar.packed/vocabulary | Bin 0 -> 144 bytes
.../decoder/num_translation_options/input | 1 +
.../num_translation_options/joshua.config | 30 +
.../joshua.config.packed | 30 +
.../decoder/num_translation_options/lm.gz | Bin 0 -> 2466496 bytes
.../decoder/num_translation_options/output.gold | 12 +
.../decoder/num_translation_options/test.sh | 17 +
.../src/test/resources/decoder/oov-list/config | 29 +
.../resources/decoder/oov-list/glue-grammar | 3 +
.../src/test/resources/decoder/oov-list/grammar | 11 +
.../test/resources/decoder/oov-list/input.txt | 3 +
.../test/resources/decoder/oov-list/output.gold | 3 +
.../src/test/resources/decoder/oov-list/test.sh | 30 +
.../resources/decoder/phrase/constrained/config | 29 +
.../decoder/phrase/constrained/corpus.es | 1 +
.../decoder/phrase/constrained/glue.grammar | 3 +
.../decoder/phrase/constrained/output.gold | 5 +
.../decoder/phrase/constrained/test.sh | 32 +
.../test/resources/decoder/phrase/decode/config | 29 +
.../decoder/phrase/decode/config.packed | 29 +
.../resources/decoder/phrase/decode/corpus.es | 1 +
.../resources/decoder/phrase/decode/lm.1.gz | Bin 0 -> 2235 bytes
.../resources/decoder/phrase/decode/output.gold | 1 +
.../resources/decoder/phrase/decode/rules.1.gz | Bin 0 -> 2998042 bytes
.../decoder/phrase/decode/rules.packed/config | 2 +
.../decoder/phrase/decode/rules.packed/encoding | Bin 0 -> 87 bytes
.../decode/rules.packed/slice_00000.features | Bin 0 -> 4128858 bytes
.../decode/rules.packed/slice_00000.source | Bin 0 -> 1982244 bytes
.../decode/rules.packed/slice_00000.target | Bin 0 -> 2652936 bytes
.../rules.packed/slice_00000.target.lookup | Bin 0 -> 32 bytes
.../phrase/decode/rules.packed/vocabulary | Bin 0 -> 169236 bytes
.../decoder/phrase/decode/test-packed.sh | 32 +
.../resources/decoder/phrase/decode/test.sh | 17 +
.../decoder/phrase/include-align-index/README | 2 +
.../decoder/phrase/include-align-index/config | 29 +
.../phrase/include-align-index/corpus.es | 1 +
.../decoder/phrase/include-align-index/lm.1.gz | Bin 0 -> 2235 bytes
.../decoder/phrase/include-align-index/log | 50 +
.../decoder/phrase/include-align-index/output | 1 +
.../phrase/include-align-index/output.gold | 1 +
.../phrase/include-align-index/rules.1.gz | Bin 0 -> 2998042 bytes
.../decoder/phrase/include-align-index/test.sh | 32 +
.../decoder/phrase/unique-hypotheses/README | 1 +
.../decoder/phrase/unique-hypotheses/corpus.es | 1 +
.../phrase/unique-hypotheses/joshua.config | 23 +
.../decoder/phrase/unique-hypotheses/lm.1.gz | 1 +
.../phrase/unique-hypotheses/output.gold | 300 +
.../decoder/phrase/unique-hypotheses/rules.1.gz | 1 +
.../decoder/phrase/unique-hypotheses/test.sh | 32 +
.../resources/decoder/rescoring/glue-grammar | 3 +
.../test/resources/decoder/rescoring/grammar.gz | Bin 0 -> 177 bytes
.../test/resources/decoder/rescoring/input.txt | 2 +
.../resources/decoder/rescoring/joshua.config | 31 +
.../resources/decoder/rescoring/output.gold | 12 +
.../test/resources/decoder/rescoring/test.sh | 30 +
.../test/resources/decoder/segment-oovs/config | 41 +
.../resources/decoder/segment-oovs/input.txt | 1 +
.../decoder/segment-oovs/output.expected | 82 +
.../test/resources/decoder/segment-oovs/test.sh | 31 +
.../decoder/source-annotations/grammar | 5 +
.../decoder/source-annotations/grammar.glue | 3 +
.../decoder/source-annotations/input.txt | 1 +
.../decoder/source-annotations/joshua.config | 140 +
.../decoder/source-annotations/lm.kenlm | Bin 0 -> 25355958 bytes
.../decoder/source-annotations/output.gold | 2 +
.../decoder/source-annotations/test.sh | 36 +
.../resources/decoder/target-bigram/out.gold | 3 +
.../resources/decoder/target-bigram/test.sh | 32 +
.../test/resources/decoder/target-bigram/vocab | 4 +
.../test/resources/decoder/too-long/output.gold | 4 +
.../src/test/resources/decoder/too-long/test.sh | 36 +
.../decoder/tree-output/fragment-map.txt | 2 +
.../resources/decoder/tree-output/glue-grammar | 6 +
.../resources/decoder/tree-output/grammar.gz | Bin 0 -> 134 bytes
.../test/resources/decoder/tree-output/input | 5 +
.../resources/decoder/tree-output/joshua.config | 45 +
.../test/resources/decoder/tree-output/lm.gz | Bin 0 -> 2466496 bytes
.../resources/decoder/tree-output/output.gold | 5 +
.../test/resources/decoder/tree-output/test.sh | 30 +
.../resources/grammar/sparse-features/grammar | 1 +
.../grammar/sparse-features/grammar.glue | 3 +
.../sparse-features/grammar.packed/encoding | Bin 0 -> 118 bytes
.../grammar.packed/slice_00000.features | Bin 0 -> 18 bytes
.../grammar.packed/slice_00000.source | Bin 0 -> 52 bytes
.../grammar.packed/slice_00000.target | Bin 0 -> 24 bytes
.../grammar.packed/slice_00000.target.lookup | Bin 0 -> 16 bytes
.../sparse-features/grammar.packed/vocabulary | Bin 0 -> 104 bytes
.../sparse-features/joshua-packed.config | 12 +
.../grammar/sparse-features/joshua.config | 12 +
.../grammar/sparse-features/output.gold | 1 +
.../grammar/sparse-features/test-packed.sh | 32 +
.../resources/grammar/sparse-features/test.sh | 32 +
.../src/test/resources/joshua/README.broken | 1 +
.../src/test/resources/lattice-short/README | 3 +
.../test/resources/lattice-short/glue-grammar | 3 +
.../test/resources/lattice-short/grammar.test | 3 +
.../src/test/resources/lattice-short/input | 5 +
.../test/resources/lattice-short/joshua.config | 39 +
.../resources/lattice-short/output.expected | 18 +
.../src/test/resources/lattice-short/test.lm | 113 +
.../src/test/resources/lattice-short/test.sh | 31 +
.../src/test/resources/lattice/.gitignore | 3 +
joshua-core/src/test/resources/lattice/README | 4 +
.../src/test/resources/lattice/glue-grammar | 3 +
.../src/test/resources/lattice/grammar.test | 204 +
.../src/test/resources/lattice/joshua.config | 47 +
.../src/test/resources/lattice/output.expected | 33 +
.../src/test/resources/lattice/test-lattice.pdf | Bin 0 -> 10943 bytes
joshua-core/src/test/resources/lattice/test.lm | 113 +
joshua-core/src/test/resources/lattice/test.plf | 4 +
joshua-core/src/test/resources/lattice/test.sh | 37 +
joshua-core/src/test/resources/lm/berkeley/lm | 16 +
.../test/resources/lm/berkeley/lm.berkeleylm | Bin 0 -> 4294 bytes
.../test/resources/lm/berkeley/lm.berkeleylm.gz | Bin 0 -> 1786 bytes
.../src/test/resources/lm/berkeley/lm.gz | Bin 0 -> 162 bytes
.../src/test/resources/lm/berkeley/output.gold | 4 +
.../src/test/resources/lm/berkeley/test.sh | 30 +
.../test/resources/packed-grammar/.gitignore | 8 +
.../src/test/resources/packed-grammar/README | 2 +
.../test/resources/packed-grammar/grammar.gz | Bin 0 -> 576901 bytes
.../src/test/resources/packed-grammar/input.bn | 100 +
.../test/resources/packed-grammar/joshua.config | 46 +
.../src/test/resources/packed-grammar/lm.gz | Bin 0 -> 2466496 bytes
.../test/resources/packed-grammar/output.gold | 100 +
.../resources/packed-grammar/reference.en.0 | 100 +
.../resources/packed-grammar/reference.en.1 | 100 +
.../resources/packed-grammar/reference.en.2 | 100 +
.../resources/packed-grammar/reference.en.3 | 100 +
.../resources/packed-grammar/test-multiple.sh | 31 +
.../src/test/resources/packed-grammar/test.sh | 38 +
joshua-core/src/test/resources/parser/grammar | 11 +
.../src/test/resources/parser/grammar.glue | 1 +
joshua-core/src/test/resources/parser/input | 4 +
.../src/test/resources/parser/output.gold | 4 +
.../src/test/resources/parser/parse.config | 18 +
joshua-core/src/test/resources/parser/test.sh | 29 +
joshua-core/src/test/resources/parser/weights | 4 +
.../src/test/resources/pipeline/.gitignore | 2 +
.../src/test/resources/pipeline/Makefile | 10 +
.../src/test/resources/pipeline/final-bleu.gold | 1 +
.../test/resources/pipeline/input/devtest.en.0 | 100 +
.../test/resources/pipeline/input/devtest.en.1 | 100 +
.../test/resources/pipeline/input/devtest.en.2 | 100 +
.../test/resources/pipeline/input/devtest.en.3 | 100 +
.../test/resources/pipeline/input/devtest.ur | 100 +
.../src/test/resources/pipeline/input/train.en | 1000 +
.../src/test/resources/pipeline/input/train.ur | 1000 +
.../src/test/resources/pipeline/input/tune.en.0 | 100 +
.../src/test/resources/pipeline/input/tune.en.1 | 100 +
.../src/test/resources/pipeline/input/tune.en.2 | 100 +
.../src/test/resources/pipeline/input/tune.en.3 | 100 +
.../src/test/resources/pipeline/input/tune.ur | 100 +
.../src/test/resources/pipeline/test-ghkm.sh | 43 +
joshua-core/src/test/resources/pipeline/test.sh | 39 +
.../resources/prune-equivalent-translations.py | 47 +
joshua-core/src/test/resources/run-all-tests.sh | 55 +
.../src/test/resources/scripts/.gitignore | 1 +
.../test/resources/scripts/merge_lms_test.py | 53 +
.../resources/scripts/normalization/.gitignore | 2 +
.../scripts/normalization/data/train.en | 21 +
.../scripts/normalization/data/train.en.norm | 21 +
.../resources/scripts/normalization/test.sh | 29 +
.../test/resources/scripts/run_bundler_test.py | 378 +
.../scripts/support/moses_grammar/input | 10 +
.../support/moses_grammar/output.expected | 10 +
.../scripts/support/moses_grammar/test.sh | 30 +
.../src/test/resources/server/http/expected | 16 +
.../src/test/resources/server/http/test.sh | 36 +
.../src/test/resources/server/tcp-text/expected | 9 +
.../src/test/resources/server/tcp-text/test.sh | 47 +
joshua-core/src/test/resources/testng.xml | 30 +
joshua-core/src/test/resources/thrax/.gitignore | 5 +
.../resources/thrax/extraction/input/thrax.conf | 71 +
.../resources/thrax/extraction/input/train.a | 100 +
.../resources/thrax/extraction/input/train.en | 100 +
.../resources/thrax/extraction/input/train.ps | 100 +
.../src/test/resources/thrax/extraction/test.sh | 36 +
.../resources/thrax/filtering/dev.hi-en.hi.1 | 1 +
.../test/resources/thrax/filtering/exact.gold | 993 +
.../resources/thrax/filtering/exact.log.gold | 17 +
.../test/resources/thrax/filtering/fast.gold | 1087 +
.../resources/thrax/filtering/fast.log.gold | 17 +
.../test/resources/thrax/filtering/grammar.de | 4 +
.../thrax/filtering/grammar.filtered.gz | Bin 0 -> 134958 bytes
.../src/test/resources/thrax/filtering/input.de | 3 +
.../resources/thrax/filtering/loose.log.gold | 16 +
.../resources/thrax/filtering/test-exact.sh | 34 +
.../test/resources/thrax/filtering/test-fast.sh | 34 +
.../resources/thrax/filtering/test-loose.sh | 34 +
joshua-servlet/.gitignore | 1 +
joshua-servlet/README.md | 5 +
joshua-servlet/pom.xml | 89 +
.../apache/joshua/decoder/DecoderServlet.java | 72 +
.../decoder/DecoderServletContextListener.java | 47 +
.../joshua/decoder/DecoderServletTest.java | 90 +
.../src/test/resources/log4j.properties | 20 +
.../src/test/resources/server/http/expected | 16 +
joshua-servlet/src/test/resources/web.xml | 23 +
pom.xml | 189 +-
resources/berkeley_lm/lm | 16 -
resources/berkeley_lm/lm.berkeleylm | Bin 4294 -> 0 bytes
resources/berkeley_lm/lm.berkeleylm.gz | Bin 1786 -> 0 bytes
resources/berkeley_lm/lm.gz | Bin 162 -> 0 bytes
resources/grammar.glue | 4 -
resources/kbest_extraction/glue-grammar | 3 -
resources/kbest_extraction/grammar | 25 -
resources/kbest_extraction/joshua.config | 27 -
resources/kbest_extraction/lm.gz | Bin 2466496 -> 0 bytes
resources/kbest_extraction/output.gold | 3126 ---
resources/kbest_extraction/output.scores.gold | 3126 ---
resources/phrase_decoder/config | 29 -
resources/phrase_decoder/constrained.config | 28 -
.../phrase_decoder/constrained.output.gold | 5 -
resources/phrase_decoder/lm.1.gz | Bin 2235 -> 0 bytes
resources/phrase_decoder/output.gold | 1 -
resources/phrase_decoder/rules.1.gz | Bin 2998042 -> 0 bytes
resources/wa_grammar | 3 -
resources/wa_grammar.packed/config | 1 -
resources/wa_grammar.packed/encoding | Bin 154 -> 0 bytes
.../wa_grammar.packed/slice_00000.alignments | Bin 45 -> 0 bytes
.../wa_grammar.packed/slice_00000.features | Bin 47 -> 0 bytes
resources/wa_grammar.packed/slice_00000.source | Bin 204 -> 0 bytes
resources/wa_grammar.packed/slice_00000.target | Bin 128 -> 0 bytes
.../wa_grammar.packed/slice_00000.target.lookup | Bin 32 -> 0 bytes
resources/wa_grammar.packed/vocabulary | Bin 238 -> 0 bytes
.../java/org/apache/joshua/adagrad/AdaGrad.java | 160 -
.../org/apache/joshua/adagrad/AdaGradCore.java | 3127 ---
.../org/apache/joshua/adagrad/Optimizer.java | 728 -
.../apache/joshua/corpus/AbstractPhrase.java | 133 -
.../org/apache/joshua/corpus/BasicPhrase.java | 97 -
.../apache/joshua/corpus/ContiguousPhrase.java | 127 -
.../java/org/apache/joshua/corpus/Corpus.java | 160 -
.../java/org/apache/joshua/corpus/Phrase.java | 117 -
.../java/org/apache/joshua/corpus/Span.java | 175 -
.../org/apache/joshua/corpus/SymbolTable.java | 327 -
.../apache/joshua/corpus/TerminalIterator.java | 85 -
.../org/apache/joshua/corpus/Vocabulary.java | 301 -
.../joshua/corpus/syntax/ArraySyntaxTree.java | 411 -
.../apache/joshua/corpus/syntax/SyntaxTree.java | 34 -
.../org/apache/joshua/decoder/ArgsParser.java | 118 -
.../java/org/apache/joshua/decoder/BLEU.java | 562 -
.../java/org/apache/joshua/decoder/Decoder.java | 813 -
.../apache/joshua/decoder/DecoderThread.java | 201 -
.../joshua/decoder/JoshuaConfiguration.java | 729 -
.../apache/joshua/decoder/JoshuaDecoder.java | 148 -
.../joshua/decoder/NbestMinRiskReranker.java | 446 -
.../joshua/decoder/StructuredTranslation.java | 157 -
.../decoder/StructuredTranslationFactory.java | 120 -
.../java/org/apache/joshua/decoder/Support.java | 86 -
.../org/apache/joshua/decoder/Translation.java | 239 -
.../org/apache/joshua/decoder/Translations.java | 158 -
.../joshua/decoder/chart_parser/Cell.java | 294 -
.../joshua/decoder/chart_parser/Chart.java | 746 -
.../decoder/chart_parser/ComputeNodeResult.java | 225 -
.../decoder/chart_parser/CubePruneState.java | 114 -
.../joshua/decoder/chart_parser/DotChart.java | 476 -
.../joshua/decoder/chart_parser/SourcePath.java | 63 -
.../decoder/chart_parser/StateConstraint.java | 75 -
.../joshua/decoder/chart_parser/SuperNode.java | 62 -
.../decoder/chart_parser/package-info.java | 24 -
.../joshua/decoder/ff/ArityPhrasePenalty.java | 73 -
.../joshua/decoder/ff/FeatureFunction.java | 364 -
.../apache/joshua/decoder/ff/FeatureVector.java | 385 -
.../joshua/decoder/ff/LabelCombinationFF.java | 62 -
.../joshua/decoder/ff/LabelSubstitutionFF.java | 131 -
.../joshua/decoder/ff/LexicalFeatures.java | 152 -
.../apache/joshua/decoder/ff/OOVPenalty.java | 108 -
.../apache/joshua/decoder/ff/PhraseModel.java | 134 -
.../apache/joshua/decoder/ff/PhrasePenalty.java | 87 -
.../apache/joshua/decoder/ff/RuleCountBin.java | 77 -
.../org/apache/joshua/decoder/ff/RuleFF.java | 126 -
.../apache/joshua/decoder/ff/RuleLength.java | 52 -
.../decoder/ff/RulePropertiesQuerying.java | 49 -
.../org/apache/joshua/decoder/ff/RuleShape.java | 99 -
.../joshua/decoder/ff/SourceDependentFF.java | 29 -
.../apache/joshua/decoder/ff/SourcePathFF.java | 63 -
.../apache/joshua/decoder/ff/StatefulFF.java | 88 -
.../apache/joshua/decoder/ff/StatelessFF.java | 79 -
.../apache/joshua/decoder/ff/TargetBigram.java | 216 -
.../apache/joshua/decoder/ff/WordPenalty.java | 80 -
.../ff/fragmentlm/ConcatenationIterator.java | 93 -
.../decoder/ff/fragmentlm/FragmentLMFF.java | 368 -
.../ff/fragmentlm/PennTreebankReader.java | 135 -
.../joshua/decoder/ff/fragmentlm/Tree.java | 779 -
.../joshua/decoder/ff/fragmentlm/Trees.java | 270 -
.../apache/joshua/decoder/ff/lm/AbstractLM.java | 122 -
.../apache/joshua/decoder/ff/lm/ArpaFile.java | 328 -
.../apache/joshua/decoder/ff/lm/ArpaNgram.java | 73 -
.../ff/lm/DefaultNGramLanguageModel.java | 134 -
.../org/apache/joshua/decoder/ff/lm/KenLM.java | 233 -
.../joshua/decoder/ff/lm/LanguageModelFF.java | 495 -
.../decoder/ff/lm/NGramLanguageModel.java | 73 -
.../ff/lm/StateMinimizingLanguageModel.java | 193 -
.../joshua/decoder/ff/lm/berkeley_lm/LICENSE | 13 -
.../ff/lm/berkeley_lm/LMGrammarBerkeley.java | 205 -
.../joshua/decoder/ff/lm/berkeley_lm/README | 5 -
.../ff/lm/berkeley_lm/SymbolTableWrapper.java | 102 -
.../ff/lm/bloomfilter_lm/BloomFilter.java | 215 -
.../BloomFilterLanguageModel.java | 560 -
.../ff/lm/bloomfilter_lm/package-info.java | 25 -
.../joshua/decoder/ff/lm/buildin_lm/TrieLM.java | 331 -
.../decoder/ff/lm/buildin_lm/package-info.java | 19 -
.../joshua/decoder/ff/lm/package-info.java | 42 -
.../apache/joshua/decoder/ff/package-info.java | 42 -
.../joshua/decoder/ff/phrase/Distortion.java | 71 -
.../ff/similarity/EdgePhraseSimilarityFF.java | 279 -
.../decoder/ff/state_maintenance/DPState.java | 34 -
.../ff/state_maintenance/KenLMState.java | 56 -
.../ff/state_maintenance/NgramDPState.java | 100 -
.../joshua/decoder/ff/tm/AbstractGrammar.java | 228 -
.../decoder/ff/tm/BasicRuleCollection.java | 101 -
.../joshua/decoder/ff/tm/CreateGlueGrammar.java | 126 -
.../apache/joshua/decoder/ff/tm/Grammar.java | 120 -
.../joshua/decoder/ff/tm/GrammarReader.java | 158 -
.../org/apache/joshua/decoder/ff/tm/Rule.java | 635 -
.../joshua/decoder/ff/tm/RuleCollection.java | 76 -
.../decoder/ff/tm/SentenceFilteredGrammar.java | 366 -
.../org/apache/joshua/decoder/ff/tm/Trie.java | 108 -
.../ff/tm/UnsortedRuleCollectionException.java | 40 -
.../decoder/ff/tm/format/HieroFormatReader.java | 106 -
.../decoder/ff/tm/format/MosesFormatReader.java | 108 -
.../ff/tm/hash_based/ExtensionIterator.java | 73 -
.../tm/hash_based/MemoryBasedBatchGrammar.java | 279 -
.../ff/tm/hash_based/MemoryBasedRuleBin.java | 59 -
.../ff/tm/hash_based/MemoryBasedTrie.java | 88 -
.../decoder/ff/tm/hash_based/package-info.java | 23 -
.../joshua/decoder/ff/tm/package-info.java | 25 -
.../decoder/ff/tm/packed/PackedGrammar.java | 1064 -
.../ff/tm/packed/SliceAggregatingTrie.java | 236 -
.../decoder/hypergraph/AlignedSourceTokens.java | 112 -
.../decoder/hypergraph/AllSpansWalker.java | 63 -
.../hypergraph/DefaultInsideOutside.java | 407 -
.../hypergraph/FeatureVectorExtractor.java | 80 -
.../joshua/decoder/hypergraph/ForestWalker.java | 79 -
.../GrammarBuilderWalkerFunction.java | 179 -
.../joshua/decoder/hypergraph/HGNode.java | 331 -
.../joshua/decoder/hypergraph/HyperEdge.java | 101 -
.../joshua/decoder/hypergraph/HyperGraph.java | 163 -
.../decoder/hypergraph/HyperGraphPruning.java | 176 -
.../decoder/hypergraph/KBestExtractor.java | 1052 -
.../hypergraph/OutputStringExtractor.java | 195 -
.../hypergraph/StringToTreeConverter.java | 74 -
.../hypergraph/TrivialInsideOutside.java | 31 -
.../decoder/hypergraph/ViterbiExtractor.java | 178 -
.../decoder/hypergraph/WalkerFunction.java | 37 -
.../hypergraph/WordAlignmentExtractor.java | 134 -
.../decoder/hypergraph/WordAlignmentState.java | 192 -
.../joshua/decoder/hypergraph/package-info.java | 25 -
.../apache/joshua/decoder/io/DeNormalize.java | 203 -
.../apache/joshua/decoder/io/JSONMessage.java | 159 -
.../decoder/io/TranslationRequestStream.java | 179 -
.../org/apache/joshua/decoder/package-info.java | 26 -
.../apache/joshua/decoder/phrase/Candidate.java | 241 -
.../decoder/phrase/CandidateComparator.java | 28 -
.../apache/joshua/decoder/phrase/Coverage.java | 235 -
.../apache/joshua/decoder/phrase/Future.java | 119 -
.../apache/joshua/decoder/phrase/Header.java | 87 -
.../joshua/decoder/phrase/Hypothesis.java | 154 -
.../org/apache/joshua/decoder/phrase/Note.java | 44 -
.../joshua/decoder/phrase/PhraseChart.java | 197 -
.../joshua/decoder/phrase/PhraseTable.java | 183 -
.../org/apache/joshua/decoder/phrase/Stack.java | 229 -
.../apache/joshua/decoder/phrase/Stacks.java | 271 -
.../joshua/decoder/phrase/TargetPhrases.java | 80 -
.../decoder/segment_file/ConstraintRule.java | 100 -
.../decoder/segment_file/ConstraintSpan.java | 80 -
.../decoder/segment_file/ParseTreeInput.java | 40 -
.../decoder/segment_file/ParsedSentence.java | 56 -
.../joshua/decoder/segment_file/Sentence.java | 450 -
.../joshua/decoder/segment_file/Token.java | 158 -
.../decoder/segment_file/package-info.java | 25 -
.../java/org/apache/joshua/lattice/Arc.java | 117 -
.../java/org/apache/joshua/lattice/Lattice.java | 587 -
.../java/org/apache/joshua/lattice/Node.java | 159 -
.../lattice/NodeIdentifierComparator.java | 41 -
.../org/apache/joshua/lattice/package-info.java | 22 -
.../java/org/apache/joshua/metrics/BLEU.java | 575 -
.../org/apache/joshua/metrics/BLEU_SBP.java | 63 -
.../apache/joshua/metrics/EvaluationMetric.java | 409 -
.../apache/joshua/metrics/GradeLevelBLEU.java | 280 -
.../java/org/apache/joshua/metrics/METEOR.java | 235 -
.../joshua/metrics/MinimumChangeBLEU.java | 223 -
.../joshua/metrics/NewMetric.java.template | 134 -
.../java/org/apache/joshua/metrics/Precis.java | 334 -
.../joshua/metrics/PrecisMinusSourceBLEU.java | 182 -
.../java/org/apache/joshua/metrics/SARI.java | 681 -
.../org/apache/joshua/metrics/SourceBLEU.java | 107 -
.../java/org/apache/joshua/metrics/TER.java | 460 -
.../org/apache/joshua/metrics/TERMinusBLEU.java | 194 -
.../org/apache/joshua/metrics/TercomRunner.java | 115 -
.../org/apache/joshua/metrics/ZeroOneLoss.java | 88 -
src/main/java/org/apache/joshua/mira/MIRA.java | 160 -
.../java/org/apache/joshua/mira/MIRACore.java | 3112 ---
.../java/org/apache/joshua/mira/Optimizer.java | 643 -
.../joshua/oracle/OracleExtractionHG.java | 797 -
.../apache/joshua/oracle/OracleExtractor.java | 58 -
.../java/org/apache/joshua/oracle/SplitHg.java | 300 -
.../org/apache/joshua/oracle/package-info.java | 26 -
.../apache/joshua/pro/ClassifierInterface.java | 41 -
.../org/apache/joshua/pro/ClassifierMegaM.java | 121 -
.../apache/joshua/pro/ClassifierPerceptron.java | 114 -
.../org/apache/joshua/pro/ClassifierSVM.java | 140 -
.../java/org/apache/joshua/pro/Optimizer.java | 454 -
src/main/java/org/apache/joshua/pro/PRO.java | 159 -
.../java/org/apache/joshua/pro/PROCore.java | 3027 ---
.../org/apache/joshua/server/ServerThread.java | 294 -
.../org/apache/joshua/server/TcpServer.java | 65 -
.../joshua/subsample/AlignedSubsampler.java | 103 -
.../org/apache/joshua/subsample/Alignment.java | 92 -
.../org/apache/joshua/subsample/BiCorpus.java | 186 -
.../joshua/subsample/BiCorpusFactory.java | 83 -
.../org/apache/joshua/subsample/PhrasePair.java | 72 -
.../apache/joshua/subsample/PhraseReader.java | 46 -
.../apache/joshua/subsample/PhraseWriter.java | 79 -
.../org/apache/joshua/subsample/Subsampler.java | 246 -
.../apache/joshua/subsample/SubsamplerCLI.java | 141 -
.../apache/joshua/subsample/package-info.java | 25 -
.../org/apache/joshua/tools/GrammarPacker.java | 959 -
.../apache/joshua/tools/GrammarPackerCli.java | 156 -
.../org/apache/joshua/tools/LabelPhrases.java | 111 -
.../org/apache/joshua/tools/TestSetFilter.java | 383 -
.../java/org/apache/joshua/ui/Orientation.java | 23 -
.../org/apache/joshua/ui/StartupWindow.java | 87 -
.../java/org/apache/joshua/ui/package-info.java | 22 -
.../ui/tree_visualizer/DerivationTree.java | 103 -
.../ui/tree_visualizer/DerivationTreeEdge.java | 27 -
.../DerivationTreeTransformer.java | 117 -
.../ui/tree_visualizer/DerivationViewer.java | 128 -
.../tree_visualizer/DerivationViewerApplet.java | 51 -
.../apache/joshua/ui/tree_visualizer/Node.java | 59 -
.../ui/tree_visualizer/browser/Browser.java | 237 -
.../browser/DerivationTreeFrame.java | 253 -
.../browser/TranslationInfo.java | 56 -
.../joshua/ui/tree_visualizer/tree/Tree.java | 283 -
.../java/org/apache/joshua/util/Algorithms.java | 85 -
src/main/java/org/apache/joshua/util/Bits.java | 128 -
.../java/org/apache/joshua/util/BotMap.java | 94 -
src/main/java/org/apache/joshua/util/Cache.java | 176 -
.../java/org/apache/joshua/util/ChartSpan.java | 88 -
.../apache/joshua/util/CommandLineParser.java | 738 -
.../java/org/apache/joshua/util/Constants.java | 36 -
.../java/org/apache/joshua/util/Counted.java | 92 -
.../java/org/apache/joshua/util/Counts.java | 306 -
.../org/apache/joshua/util/ExtractTopCand.java | 188 -
.../org/apache/joshua/util/FileUtility.java | 318 -
.../org/apache/joshua/util/FormatUtils.java | 245 -
.../org/apache/joshua/util/IntegerPair.java | 36 -
.../java/org/apache/joshua/util/JoshuaEval.java | 624 -
.../java/org/apache/joshua/util/ListUtil.java | 95 -
src/main/java/org/apache/joshua/util/Lists.java | 567 -
.../apache/joshua/util/NBestListUtility.java | 74 -
src/main/java/org/apache/joshua/util/Ngram.java | 105 -
.../org/apache/joshua/util/NullIterator.java | 65 -
.../apache/joshua/util/PackedGrammarServer.java | 87 -
src/main/java/org/apache/joshua/util/Pair.java | 130 -
.../java/org/apache/joshua/util/Platform.java | 27 -
.../org/apache/joshua/util/QuietFormatter.java | 36 -
src/main/java/org/apache/joshua/util/Regex.java | 143 -
.../org/apache/joshua/util/ReverseOrder.java | 39 -
.../org/apache/joshua/util/SampledList.java | 69 -
.../org/apache/joshua/util/SocketUtility.java | 144 -
.../org/apache/joshua/util/StreamGobbler.java | 50 -
.../joshua/util/UnicodeCharacterName.java | 22466 -----------------
.../apache/joshua/util/encoding/Analyzer.java | 235 -
.../joshua/util/encoding/EightBitQuantizer.java | 92 -
.../util/encoding/EncoderConfiguration.java | 160 -
.../joshua/util/encoding/EncoderFactory.java | 42 -
.../util/encoding/FeatureTypeAnalyzer.java | 254 -
.../joshua/util/encoding/FloatEncoder.java | 39 -
.../apache/joshua/util/encoding/IntEncoder.java | 39 -
.../util/encoding/PrimitiveFloatEncoder.java | 129 -
.../util/encoding/PrimitiveIntEncoder.java | 111 -
.../joshua/util/encoding/VariableQuantizer.java | 106 -
.../org/apache/joshua/util/io/BinaryIn.java | 91 -
.../org/apache/joshua/util/io/BinaryOut.java | 505 -
.../apache/joshua/util/io/IndexedReader.java | 155 -
.../org/apache/joshua/util/io/LineReader.java | 368 -
.../org/apache/joshua/util/io/NullReader.java | 63 -
.../joshua/util/io/ProgressInputStream.java | 82 -
.../java/org/apache/joshua/util/io/Reader.java | 51 -
.../org/apache/joshua/util/io/package-info.java | 22 -
.../org/apache/joshua/util/package-info.java | 22 -
.../util/quantization/BooleanQuantizer.java | 45 -
.../joshua/util/quantization/Quantizer.java | 45 -
.../quantization/QuantizerConfiguration.java | 119 -
.../util/quantization/QuantizerFactory.java | 50 -
.../util/quantization/StatelessQuantizer.java | 38 -
.../joshua/util/quantization/package-info.java | 19 -
.../joshua/zmert/IntermediateOptimizer.java | 991 -
.../java/org/apache/joshua/zmert/MertCore.java | 3191 ---
.../java/org/apache/joshua/zmert/ZMERT.java | 156 -
.../org/apache/joshua/zmert/package-info.java | 24 -
src/main/resources/log4j.properties | 20 -
src/overview.html | 41 -
.../apache/joshua/corpus/CorpusArrayTest.java | 177 -
.../java/org/apache/joshua/corpus/SpanTest.java | 47 -
.../apache/joshua/corpus/VocabularyTest.java | 135 -
.../joshua/corpus/vocab/VocabularyTest.java | 180 -
.../ArtificialGrammarAndCorpusCreater.java | 130 -
.../joshua/decoder/DecoderThreadTest.java | 172 -
.../joshua/decoder/JoshuaDecoderTest.java | 83 -
.../joshua/decoder/TestConfigFileCreater.java | 184 -
.../apache/joshua/decoder/TranslationsTest.java | 87 -
.../decoder/ff/ArityPhrasePenaltyFFTest.java | 64 -
.../joshua/decoder/ff/lm/ArpaFileTest.java | 226 -
.../decoder/ff/lm/LanguageModelFFTest.java | 95 -
.../LMBerkeleySentenceProbablityTest.java | 47 -
.../lm/berkeley_lm/LMGrammarBerkeleyTest.java | 80 -
.../joshua/decoder/io/DeNormalizeTest.java | 273 -
.../decoder/io/TranslationRequestTest.java | 149 -
.../kbest_extraction/KBestExtractionTest.java | 80 -
.../joshua/decoder/phrase/CoverageTest.java | 140 -
.../ConstrainedPhraseDecodingTest.java | 77 -
.../phrase/decode/PhraseDecodingTest.java | 77 -
.../segment_file/AlmostTooLongSentenceTest.java | 96 -
.../decoder/segment_file/SentenceTest.java | 109 -
.../java/org/apache/joshua/lattice/ArcTest.java | 86 -
.../org/apache/joshua/lattice/LatticeTest.java | 192 -
.../org/apache/joshua/lattice/NodeTest.java | 105 -
.../org/apache/joshua/packed/Benchmark.java | 126 -
.../org/apache/joshua/packed/CountRules.java | 110 -
.../org/apache/joshua/packed/PrintRules.java | 199 -
src/test/java/org/apache/joshua/packed/README | 6 -
.../org/apache/joshua/packed/VocabTest.java | 58 -
.../java/org/apache/joshua/packed/packer.config | 6 -
.../java/org/apache/joshua/packed/small_grammar | 20000 ---------------
src/test/java/org/apache/joshua/packed/test.sh | 20 -
.../apache/joshua/system/AlignmentMapTest.java | 73 -
.../org/apache/joshua/system/KenLmTest.java | 95 -
.../system/MultithreadedTranslationTests.java | 156 -
.../joshua/system/StructuredOutputTest.java | 118 -
.../system/StructuredTranslationTest.java | 274 -
.../ui/tree_visualizer/tree/TreeTest.java | 111 -
.../java/org/apache/joshua/util/BitsTest.java | 187 -
.../java/org/apache/joshua/util/CacheTest.java | 53 -
.../java/org/apache/joshua/util/CountsTest.java | 98 -
.../org/apache/joshua/util/FormatUtilsTest.java | 74 -
.../org/apache/joshua/util/io/BinaryTest.java | 74 -
.../java/org/apache/joshua/zmert/BLEUTest.java | 132 -
src/test/resources/bn-en/hiero/.gitignore | 4 -
src/test/resources/bn-en/hiero/class.map | 5140 ----
.../resources/bn-en/hiero/class_lm_2gram.gz | Bin 18052 -> 0 bytes
.../resources/bn-en/hiero/class_lm_9gram.gz | Bin 12733137 -> 0 bytes
src/test/resources/bn-en/hiero/glue-grammar | 3 -
src/test/resources/bn-en/hiero/grammar.gz | Bin 518164 -> 0 bytes
src/test/resources/bn-en/hiero/input.bn | 100 -
.../bn-en/hiero/joshua-berkeleylm.config | 46 -
.../resources/bn-en/hiero/joshua-classlm.config | 51 -
src/test/resources/bn-en/hiero/joshua.config | 50 -
src/test/resources/bn-en/hiero/lm.gz | Bin 2466496 -> 0 bytes
.../resources/bn-en/hiero/output-classlm.gold | 887 -
src/test/resources/bn-en/hiero/output.gold | 805 -
src/test/resources/bn-en/hiero/output.gold.bleu | 14 -
.../bn-en/hiero/output.scores.berkeleylm.gold | 100 -
.../resources/bn-en/hiero/output.scores.gold | 805 -
src/test/resources/bn-en/hiero/reference.en.0 | 100 -
src/test/resources/bn-en/hiero/reference.en.1 | 100 -
src/test/resources/bn-en/hiero/reference.en.2 | 100 -
src/test/resources/bn-en/hiero/reference.en.3 | 100 -
.../resources/bn-en/hiero/test-berkeleylm.sh | 33 -
src/test/resources/bn-en/hiero/test-classlm.sh | 32 -
src/test/resources/bn-en/hiero/test-filter.sh | 35 -
src/test/resources/bn-en/hiero/test.sh | 35 -
src/test/resources/bn-en/hiero/topN.pl | 18 -
src/test/resources/bn-en/packed/.gitignore | 3 -
src/test/resources/bn-en/packed/grammar.glue | 5673 -----
src/test/resources/bn-en/packed/grammar.gz | Bin 3540984 -> 0 bytes
.../bn-en/packed/grammar.packed/encoding | Bin 767 -> 0 bytes
.../packed/grammar.packed/slice_00000.features | Bin 4631480 -> 0 bytes
.../packed/grammar.packed/slice_00000.source | Bin 4240012 -> 0 bytes
.../packed/grammar.packed/slice_00000.target | Bin 162776 -> 0 bytes
.../grammar.packed/slice_00000.target.lookup | Bin 56 -> 0 bytes
.../bn-en/packed/grammar.packed/vocabulary | Bin 136522 -> 0 bytes
src/test/resources/bn-en/packed/input.bn | 100 -
src/test/resources/bn-en/packed/joshua.config | 47 -
src/test/resources/bn-en/packed/lm.gz | Bin 2466496 -> 0 bytes
src/test/resources/bn-en/packed/output.gold | 862 -
.../resources/bn-en/packed/output.scores.gold | 862 -
src/test/resources/bn-en/packed/reference.en.0 | 100 -
src/test/resources/bn-en/packed/reference.en.1 | 100 -
src/test/resources/bn-en/packed/reference.en.2 | 100 -
src/test/resources/bn-en/packed/reference.en.3 | 100 -
.../resources/bn-en/packed/reference.en.all | 400 -
src/test/resources/bn-en/packed/test.sh | 20 -
src/test/resources/bn-en/samt/grammar.glue | 5673 -----
src/test/resources/bn-en/samt/grammar.gz | Bin 3847934 -> 0 bytes
src/test/resources/bn-en/samt/input.bn | 100 -
src/test/resources/bn-en/samt/joshua.config | 47 -
src/test/resources/bn-en/samt/lm.gz | Bin 2466496 -> 0 bytes
src/test/resources/bn-en/samt/output.gold | 0
src/test/resources/bn-en/samt/output.gold.bleu | 14 -
.../resources/bn-en/samt/output.scores.gold | 862 -
src/test/resources/bn-en/samt/reference.en.0 | 100 -
src/test/resources/bn-en/samt/reference.en.1 | 100 -
src/test/resources/bn-en/samt/reference.en.2 | 100 -
src/test/resources/bn-en/samt/reference.en.3 | 100 -
src/test/resources/bn-en/samt/test.sh | 35 -
src/test/resources/data/tiny.en | 5 -
.../resources/decoder/constrained/.gitignore | 4 -
.../resources/decoder/constrained/glue-grammar | 3 -
.../resources/decoder/constrained/gold.scores | 27 -
.../resources/decoder/constrained/grammar.gz | Bin 518164 -> 0 bytes
src/test/resources/decoder/constrained/input.bn | 8 -
.../resources/decoder/constrained/joshua.config | 45 -
src/test/resources/decoder/constrained/lm.gz | Bin 2466496 -> 0 bytes
.../resources/decoder/constrained/output.bleu | 0
.../resources/decoder/constrained/output.gold | 30 -
src/test/resources/decoder/constrained/test.sh | 30 -
src/test/resources/decoder/constrained/weights | 22 -
.../resources/decoder/denormalization/input.txt | 1 -
.../decoder/denormalization/output.expected | 1 -
.../resources/decoder/denormalization/test.sh | 30 -
src/test/resources/decoder/dont-crash/input | 10 -
src/test/resources/decoder/dont-crash/test.sh | 29 -
.../resources/decoder/empty-test/.gitignore | 3 -
src/test/resources/decoder/empty-test/input | 1 -
.../resources/decoder/empty-test/output.gold | 1 -
src/test/resources/decoder/empty-test/test.sh | 29 -
.../resources/decoder/fragmentlm/fragments.txt | 7 -
src/test/resources/decoder/fragmentlm/glue | 1 -
src/test/resources/decoder/fragmentlm/grammar | 4 -
src/test/resources/decoder/fragmentlm/input | 1 -
.../resources/decoder/fragmentlm/joshua.config | 109 -
.../resources/decoder/fragmentlm/mapping.txt | 4 -
src/test/resources/decoder/fragmentlm/test.sh | 30 -
.../decoder/k-best-extraction/glue-grammar | 3 -
.../resources/decoder/k-best-extraction/grammar | 25 -
.../decoder/k-best-extraction/input.txt | 1 -
.../decoder/k-best-extraction/joshua.config | 27 -
.../resources/decoder/k-best-extraction/lm.gz | Bin 2466496 -> 0 bytes
.../decoder/k-best-extraction/output.gold | 3126 ---
.../k-best-extraction/output.scores.gold | 3126 ---
.../resources/decoder/k-best-extraction/test.sh | 33 -
.../resources/decoder/left-state/glue-grammar | 3 -
.../resources/decoder/left-state/grammar.gz | Bin 518164 -> 0 bytes
src/test/resources/decoder/left-state/input.bn | 2 -
.../resources/decoder/left-state/joshua.config | 44 -
src/test/resources/decoder/left-state/lm.gz | Bin 2466496 -> 0 bytes
.../resources/decoder/left-state/output.gold | 600 -
.../decoder/left-state/output.scores.gold | 600 -
src/test/resources/decoder/left-state/test.sh | 33 -
src/test/resources/decoder/lowercaser/config | 140 -
.../resources/decoder/lowercaser/grammar.glue | 4 -
.../resources/decoder/lowercaser/grammar.test | 1 -
.../resources/decoder/lowercaser/output.gold | 5 -
src/test/resources/decoder/lowercaser/test.sh | 40 -
.../decoder/metadata/add_rule/output.gold | 4 -
.../resources/decoder/metadata/add_rule/test.sh | 32 -
.../resources/decoder/moses-compat/n-best.txt | 0
.../decoder/moses-compat/output.expected | 6 -
src/test/resources/decoder/moses-compat/test.sh | 40 -
src/test/resources/decoder/n-ary/glue-grammar | 3 -
src/test/resources/decoder/n-ary/gold.scores | 2 -
src/test/resources/decoder/n-ary/grammar | 9 -
src/test/resources/decoder/n-ary/input.txt | 2 -
src/test/resources/decoder/n-ary/joshua.config | 22 -
src/test/resources/decoder/n-ary/lm.gz | Bin 2466496 -> 0 bytes
src/test/resources/decoder/n-ary/output.bleu | 0
src/test/resources/decoder/n-ary/output.gold | 2 -
src/test/resources/decoder/n-ary/test.sh | 33 -
src/test/resources/decoder/n-ary/weights | 6 -
.../decoder/num_translation_options/README | 1 -
.../num_translation_options/glue-grammar | 3 -
.../decoder/num_translation_options/grammar.gz | Bin 119 -> 0 bytes
.../grammar.packed/encoding | Bin 32 -> 0 bytes
.../grammar.packed/slice_00000.features | Bin 43 -> 0 bytes
.../grammar.packed/slice_00000.source | Bin 132 -> 0 bytes
.../grammar.packed/slice_00000.target | Bin 120 -> 0 bytes
.../grammar.packed/slice_00000.target.lookup | Bin 32 -> 0 bytes
.../grammar.packed/vocabulary | Bin 144 -> 0 bytes
.../decoder/num_translation_options/input | 1 -
.../num_translation_options/joshua.config | 30 -
.../joshua.config.packed | 30 -
.../decoder/num_translation_options/lm.gz | Bin 2466496 -> 0 bytes
.../decoder/num_translation_options/output.gold | 12 -
.../decoder/num_translation_options/test.sh | 17 -
src/test/resources/decoder/oov-list/config | 29 -
.../resources/decoder/oov-list/glue-grammar | 3 -
src/test/resources/decoder/oov-list/grammar | 11 -
src/test/resources/decoder/oov-list/input.txt | 3 -
src/test/resources/decoder/oov-list/output.gold | 3 -
src/test/resources/decoder/oov-list/test.sh | 30 -
.../resources/decoder/phrase/constrained/config | 29 -
.../decoder/phrase/constrained/corpus.es | 1 -
.../decoder/phrase/constrained/glue.grammar | 3 -
.../decoder/phrase/constrained/output.gold | 5 -
.../decoder/phrase/constrained/test.sh | 32 -
src/test/resources/decoder/phrase/decode/config | 29 -
.../decoder/phrase/decode/config.packed | 29 -
.../resources/decoder/phrase/decode/corpus.es | 1 -
.../resources/decoder/phrase/decode/lm.1.gz | Bin 2235 -> 0 bytes
.../resources/decoder/phrase/decode/output.gold | 1 -
.../resources/decoder/phrase/decode/rules.1.gz | Bin 2998042 -> 0 bytes
.../decoder/phrase/decode/rules.packed/config | 2 -
.../decoder/phrase/decode/rules.packed/encoding | Bin 87 -> 0 bytes
.../decode/rules.packed/slice_00000.features | Bin 4128858 -> 0 bytes
.../decode/rules.packed/slice_00000.source | Bin 1982244 -> 0 bytes
.../decode/rules.packed/slice_00000.target | Bin 2652936 -> 0 bytes
.../rules.packed/slice_00000.target.lookup | Bin 32 -> 0 bytes
.../phrase/decode/rules.packed/vocabulary | Bin 169236 -> 0 bytes
.../decoder/phrase/decode/test-packed.sh | 32 -
.../resources/decoder/phrase/decode/test.sh | 17 -
.../decoder/phrase/include-align-index/README | 2 -
.../decoder/phrase/include-align-index/config | 29 -
.../phrase/include-align-index/corpus.es | 1 -
.../decoder/phrase/include-align-index/lm.1.gz | Bin 2235 -> 0 bytes
.../decoder/phrase/include-align-index/log | 50 -
.../decoder/phrase/include-align-index/output | 1 -
.../phrase/include-align-index/output.gold | 1 -
.../phrase/include-align-index/rules.1.gz | Bin 2998042 -> 0 bytes
.../decoder/phrase/include-align-index/test.sh | 32 -
.../decoder/phrase/unique-hypotheses/README | 1 -
.../decoder/phrase/unique-hypotheses/corpus.es | 1 -
.../phrase/unique-hypotheses/joshua.config | 23 -
.../decoder/phrase/unique-hypotheses/lm.1.gz | 1 -
.../phrase/unique-hypotheses/output.gold | 300 -
.../decoder/phrase/unique-hypotheses/rules.1.gz | 1 -
.../decoder/phrase/unique-hypotheses/test.sh | 32 -
.../resources/decoder/rescoring/glue-grammar | 3 -
src/test/resources/decoder/rescoring/grammar.gz | Bin 177 -> 0 bytes
src/test/resources/decoder/rescoring/input.txt | 2 -
.../resources/decoder/rescoring/joshua.config | 31 -
.../resources/decoder/rescoring/output.gold | 12 -
src/test/resources/decoder/rescoring/test.sh | 30 -
src/test/resources/decoder/segment-oovs/config | 41 -
.../resources/decoder/segment-oovs/input.txt | 1 -
.../decoder/segment-oovs/output.expected | 82 -
src/test/resources/decoder/segment-oovs/test.sh | 31 -
.../decoder/source-annotations/grammar | 5 -
.../decoder/source-annotations/grammar.glue | 3 -
.../decoder/source-annotations/input.txt | 1 -
.../decoder/source-annotations/joshua.config | 140 -
.../decoder/source-annotations/lm.kenlm | Bin 25355958 -> 0 bytes
.../decoder/source-annotations/output.gold | 2 -
.../decoder/source-annotations/test.sh | 36 -
.../resources/decoder/target-bigram/out.gold | 3 -
.../resources/decoder/target-bigram/test.sh | 32 -
src/test/resources/decoder/target-bigram/vocab | 4 -
src/test/resources/decoder/too-long/output.gold | 4 -
src/test/resources/decoder/too-long/test.sh | 36 -
.../decoder/tree-output/fragment-map.txt | 2 -
.../resources/decoder/tree-output/glue-grammar | 6 -
.../resources/decoder/tree-output/grammar.gz | Bin 134 -> 0 bytes
src/test/resources/decoder/tree-output/input | 5 -
.../resources/decoder/tree-output/joshua.config | 45 -
src/test/resources/decoder/tree-output/lm.gz | Bin 2466496 -> 0 bytes
.../resources/decoder/tree-output/output.gold | 5 -
src/test/resources/decoder/tree-output/test.sh | 30 -
.../resources/grammar/sparse-features/grammar | 1 -
.../grammar/sparse-features/grammar.glue | 3 -
.../sparse-features/grammar.packed/encoding | Bin 118 -> 0 bytes
.../grammar.packed/slice_00000.features | Bin 18 -> 0 bytes
.../grammar.packed/slice_00000.source | Bin 52 -> 0 bytes
.../grammar.packed/slice_00000.target | Bin 24 -> 0 bytes
.../grammar.packed/slice_00000.target.lookup | Bin 16 -> 0 bytes
.../sparse-features/grammar.packed/vocabulary | Bin 104 -> 0 bytes
.../sparse-features/joshua-packed.config | 12 -
.../grammar/sparse-features/joshua.config | 12 -
.../grammar/sparse-features/output.gold | 1 -
.../grammar/sparse-features/test-packed.sh | 32 -
.../resources/grammar/sparse-features/test.sh | 32 -
src/test/resources/joshua/README.broken | 1 -
src/test/resources/lattice-short/README | 3 -
src/test/resources/lattice-short/glue-grammar | 3 -
src/test/resources/lattice-short/grammar.test | 3 -
src/test/resources/lattice-short/input | 5 -
src/test/resources/lattice-short/joshua.config | 39 -
.../resources/lattice-short/output.expected | 18 -
src/test/resources/lattice-short/test.lm | 113 -
src/test/resources/lattice-short/test.sh | 31 -
src/test/resources/lattice/.gitignore | 3 -
src/test/resources/lattice/README | 4 -
src/test/resources/lattice/glue-grammar | 3 -
src/test/resources/lattice/grammar.test | 204 -
src/test/resources/lattice/joshua.config | 47 -
src/test/resources/lattice/output.expected | 33 -
src/test/resources/lattice/test-lattice.pdf | Bin 10943 -> 0 bytes
src/test/resources/lattice/test.lm | 113 -
src/test/resources/lattice/test.plf | 4 -
src/test/resources/lattice/test.sh | 37 -
src/test/resources/lm/berkeley/lm | 16 -
src/test/resources/lm/berkeley/lm.berkeleylm | Bin 4294 -> 0 bytes
src/test/resources/lm/berkeley/lm.berkeleylm.gz | Bin 1786 -> 0 bytes
src/test/resources/lm/berkeley/lm.gz | Bin 162 -> 0 bytes
src/test/resources/lm/berkeley/output.gold | 4 -
src/test/resources/lm/berkeley/test.sh | 30 -
src/test/resources/packed-grammar/.gitignore | 8 -
src/test/resources/packed-grammar/README | 2 -
src/test/resources/packed-grammar/grammar.gz | Bin 576901 -> 0 bytes
src/test/resources/packed-grammar/input.bn | 100 -
src/test/resources/packed-grammar/joshua.config | 46 -
src/test/resources/packed-grammar/lm.gz | Bin 2466496 -> 0 bytes
src/test/resources/packed-grammar/output.gold | 100 -
.../resources/packed-grammar/reference.en.0 | 100 -
.../resources/packed-grammar/reference.en.1 | 100 -
.../resources/packed-grammar/reference.en.2 | 100 -
.../resources/packed-grammar/reference.en.3 | 100 -
.../resources/packed-grammar/test-multiple.sh | 31 -
src/test/resources/packed-grammar/test.sh | 38 -
src/test/resources/parser/grammar | 11 -
src/test/resources/parser/grammar.glue | 1 -
src/test/resources/parser/input | 4 -
src/test/resources/parser/output.gold | 4 -
src/test/resources/parser/parse.config | 18 -
src/test/resources/parser/test.sh | 29 -
src/test/resources/parser/weights | 4 -
src/test/resources/pipeline/.gitignore | 2 -
src/test/resources/pipeline/Makefile | 10 -
src/test/resources/pipeline/final-bleu.gold | 1 -
src/test/resources/pipeline/input/devtest.en.0 | 100 -
src/test/resources/pipeline/input/devtest.en.1 | 100 -
src/test/resources/pipeline/input/devtest.en.2 | 100 -
src/test/resources/pipeline/input/devtest.en.3 | 100 -
src/test/resources/pipeline/input/devtest.ur | 100 -
src/test/resources/pipeline/input/train.en | 1000 -
src/test/resources/pipeline/input/train.ur | 1000 -
src/test/resources/pipeline/input/tune.en.0 | 100 -
src/test/resources/pipeline/input/tune.en.1 | 100 -
src/test/resources/pipeline/input/tune.en.2 | 100 -
src/test/resources/pipeline/input/tune.en.3 | 100 -
src/test/resources/pipeline/input/tune.ur | 100 -
src/test/resources/pipeline/test-ghkm.sh | 43 -
src/test/resources/pipeline/test.sh | 39 -
.../resources/prune-equivalent-translations.py | 47 -
src/test/resources/run-all-tests.sh | 55 -
src/test/resources/scripts/.gitignore | 1 -
src/test/resources/scripts/merge_lms_test.py | 53 -
.../resources/scripts/normalization/.gitignore | 2 -
.../scripts/normalization/data/train.en | 21 -
.../scripts/normalization/data/train.en.norm | 21 -
.../resources/scripts/normalization/test.sh | 29 -
src/test/resources/scripts/run_bundler_test.py | 378 -
.../scripts/support/moses_grammar/input | 10 -
.../support/moses_grammar/output.expected | 10 -
.../scripts/support/moses_grammar/test.sh | 30 -
src/test/resources/server/http/expected | 16 -
src/test/resources/server/http/test.sh | 36 -
src/test/resources/server/tcp-text/expected | 9 -
src/test/resources/server/tcp-text/test.sh | 47 -
src/test/resources/testng.xml | 30 -
src/test/resources/thrax/.gitignore | 5 -
.../resources/thrax/extraction/input/thrax.conf | 71 -
.../resources/thrax/extraction/input/train.a | 100 -
.../resources/thrax/extraction/input/train.en | 100 -
.../resources/thrax/extraction/input/train.ps | 100 -
src/test/resources/thrax/extraction/test.sh | 36 -
.../resources/thrax/filtering/dev.hi-en.hi.1 | 1 -
src/test/resources/thrax/filtering/exact.gold | 993 -
.../resources/thrax/filtering/exact.log.gold | 17 -
src/test/resources/thrax/filtering/fast.gold | 1087 -
.../resources/thrax/filtering/fast.log.gold | 17 -
src/test/resources/thrax/filtering/grammar.de | 4 -
.../thrax/filtering/grammar.filtered.gz | Bin 134958 -> 0 bytes
src/test/resources/thrax/filtering/input.de | 3 -
.../resources/thrax/filtering/loose.log.gold | 16 -
.../resources/thrax/filtering/test-exact.sh | 34 -
src/test/resources/thrax/filtering/test-fast.sh | 34 -
.../resources/thrax/filtering/test-loose.sh | 34 -
1330 files changed, 157903 insertions(+), 157486 deletions(-)
----------------------------------------------------------------------
[52/60] incubator-joshua git commit: maven multi-module layout 2nd
commit: adding joshua-servlet
Posted by mj...@apache.org.
maven multi-module layout 2nd commit: adding joshua-servlet
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/ae3a5df2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/ae3a5df2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/ae3a5df2
Branch: refs/heads/maven-multi-module
Commit: ae3a5df2e0e1377f7447cc64c9947b7632dec308
Parents: e273439
Author: Pavel Danchenko <da...@amazon.com>
Authored: Tue Jun 21 17:17:26 2016 +0200
Committer: Pavel Danchenko <da...@amazon.com>
Committed: Tue Jun 21 18:19:51 2016 +0200
----------------------------------------------------------------------
joshua-core/pom.xml | 214 +++++++++----------
joshua-servlet/.gitignore | 1 +
joshua-servlet/README.md | 5 +
joshua-servlet/pom.xml | 89 ++++++++
.../apache/joshua/decoder/DecoderServlet.java | 72 +++++++
.../decoder/DecoderServletContextListener.java | 47 ++++
.../joshua/decoder/DecoderServletTest.java | 90 ++++++++
.../src/test/resources/log4j.properties | 20 ++
.../src/test/resources/server/http/expected | 16 ++
joshua-servlet/src/test/resources/web.xml | 23 ++
pom.xml | 189 +++++-----------
11 files changed, 518 insertions(+), 248 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ae3a5df2/joshua-core/pom.xml
----------------------------------------------------------------------
diff --git a/joshua-core/pom.xml b/joshua-core/pom.xml
index fa9b721..e68a96b 100644
--- a/joshua-core/pom.xml
+++ b/joshua-core/pom.xml
@@ -19,15 +19,14 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>org.apache</groupId>
- <artifactId>apache</artifactId>
- <version>10</version>
+ <groupId>org.apache.joshua</groupId>
+ <artifactId>joshua-parent</artifactId>
+ <version>6.0.6-SNAPSHOT</version>
</parent>
- <groupId>org.apache.joshua</groupId>
- <artifactId>joshua-parent</artifactId>
- <packaging>pom</packaging>
- <version>6.0.6-SNAPSHOT</version>
- <name>Apache Joshua Machine Translation Toolkit - Parent</name>
+
+ <artifactId>joshua-core</artifactId>
+ <packaging>jar</packaging>
+ <name>Apache Joshua Machine Translation Toolkit - Core</name>
<description>Joshua is an open-source statistical machine
translation decoder for phrase-based, hierarchical,
and syntax-based machine translation, written in Java.
@@ -38,108 +37,107 @@
<properties>
<slf4j.version>1.7.21</slf4j.version>
</properties>
- <licenses>
- <license>
- <name>The Apache Software License, Version 2.0</name>
- <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url>
- </license>
- </licenses>
-
- <organization>
- <name>The Apache Software Foundation</name>
- <url>http://www.apache.org/</url>
- </organization>
-
- <developers>
- <developer>
- <id>lewismc</id>
- <name>Lewis John McGibbney</name>
- <email>lewismc [at] apache [dot] org</email>
- <roles>
- <role>Committer</role>
- <role>PMC Member</role>
- </roles>
- </developer>
- <developer>
- <id>mjpost</id>
- <name>Matt Post</name>
- <email>post [at] cs [dot] jhu [dot] edu</email>
- <roles>
- <role>Committer</role>
- <role>PMC Member</role>
- </roles>
- </developer>
- </developers>
-
- <mailingLists>
- <mailingList>
- <name>Dev Mailing List</name>
- <post>dev[at]joshua[dot]incubator[dot]apache[dot]org</post>
- <subscribe>dev-subscribe[at]joshua[dot]incubator[dot]apache[dot]org</subscribe>
- <unsubscribe>dev-unsubscribe[at]joshua[dot]incubator[dot]apache[dot]org</unsubscribe>
- <archive>http://mail-archives.apache.org/mod_mbox/incubator-joshua-dev/</archive>
- </mailingList>
-
- <mailingList>
- <name>User Mailing List</name>
- <post>user[at]joshua[dot]incubator[dot]apache[dot]org</post>
- <subscribe>user-subscribe[at]joshua[dot]incubator[dot]apache[dot]org</subscribe>
- <unsubscribe>user-unsubscribe[at]joshua[dot]incubator[dot]apache[dot]org</unsubscribe>
- <archive>http://mail-archives.apache.org/mod_mbox/incubator-joshua-user/</archive>
- </mailingList>
-
- <mailingList>
- <name>Commits Mailing List</name>
- <post>commits[at]joshua[dot]incubator[dot]apache[dot]org</post>
- <subscribe>commits-subscribe[at]joshua[dot]incubator[dot]apache[dot]org</subscribe>
- <unsubscribe>commits-unsubscribe[at]joshua[dot]incubator[dot]apache[dot]org</unsubscribe>
- <archive>http://mail-archives.apache.org/mod_mbox/incubator-joshua-commits/</archive>
- </mailingList>
- </mailingLists>
-
- <scm>
- <connection>scm:git:http://git-wip-us.apache.org/repos/asf/incubator-joshua.git</connection>
- <developerConnection>scm:git:http://git-wip-us.apache.org/repos/asf/incubator-joshua.git</developerConnection>
- <url>https://git-wip-us.apache.org/repos/asf/incubator-joshua.git</url>
- <tag>HEAD</tag>
- </scm>
- <issueManagement>
- <system>JIRA</system>
- <url>https://issues.apache.org/jira/browse/JOSHUA</url>
- </issueManagement>
- <ciManagement>
- <system>Jenkins</system>
- <url>https://builds.apache.org/job/joshua_master/</url>
- </ciManagement>
-
- <modules>
- <module>joshua-core</module>
- <module>joshua-servlet</module>
- </modules>
-
- <dependencyManagement>
- <dependencies>
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.12</version>
- </dependency>
- </dependencies>
- </dependencyManagement>
<build>
- <defaultGoal>install</defaultGoal>
- <pluginManagement>
- <plugins>
- <plugin>
- <artifactId>maven-compiler-plugin</artifactId>
- <configuration>
- <source>1.8</source>
- <target>1.8</target>
- </configuration>
- </plugin>
- </plugins>
- </pluginManagement>
+ <plugins>
+ <plugin>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <configuration>
+ <archive>
+ <manifest>
+ <mainClass>org.apache.joshua.decoder.JoshuaDecoder</mainClass>
+ </manifest>
+ </archive>
+ <descriptorRefs>
+ <descriptorRef>jar-with-dependencies</descriptorRef>
+ </descriptorRefs>
+ </configuration>
+ </plugin>
+ <plugin>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.19.1</version>
+ </plugin>
+ </plugins>
</build>
+ <dependencies>
+ <dependency>
+ <groupId>edu.berkeley.nlp</groupId>
+ <artifactId>berkeleylm</artifactId>
+ <version>1.1.2</version>
+ </dependency>
+ <dependency>
+ <groupId>commons-cli</groupId>
+ <artifactId>commons-cli</artifactId>
+ <version>1.2</version>
+ </dependency>
+ <dependency>
+ <groupId>net.sf.jung</groupId>
+ <artifactId>jung-algorithms</artifactId>
+ <version>2.0</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>net.sf.jung</groupId>
+ <artifactId>jung-api</artifactId>
+ <version>2.0</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>net.sf.jung</groupId>
+ <artifactId>jung-graph-impl</artifactId>
+ <version>2.0</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>net.sf.jung</groupId>
+ <artifactId>jung-visualization</artifactId>
+ <version>2.0</version>
+ <optional>true</optional>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>19.0</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>2.5</version>
+ </dependency>
+ <dependency>
+ <groupId>args4j</groupId>
+ <artifactId>args4j</artifactId>
+ <version>2.0.29</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+
+ <!-- Test Dependencies -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.testng</groupId>
+ <artifactId>testng</artifactId>
+ <version>6.9.10</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <version>2.0.52-beta</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ae3a5df2/joshua-servlet/.gitignore
----------------------------------------------------------------------
diff --git a/joshua-servlet/.gitignore b/joshua-servlet/.gitignore
new file mode 100644
index 0000000..b83d222
--- /dev/null
+++ b/joshua-servlet/.gitignore
@@ -0,0 +1 @@
+/target/
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ae3a5df2/joshua-servlet/README.md
----------------------------------------------------------------------
diff --git a/joshua-servlet/README.md b/joshua-servlet/README.md
new file mode 100644
index 0000000..05df4f3
--- /dev/null
+++ b/joshua-servlet/README.md
@@ -0,0 +1,5 @@
+# Joshua Servlet module
+This is Servlet 3.1 adapater module for Joshua decoder.
+To simply run the pre-configured jetty server on port 8080 use
+
+ mvn jetty:run
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ae3a5df2/joshua-servlet/pom.xml
----------------------------------------------------------------------
diff --git a/joshua-servlet/pom.xml b/joshua-servlet/pom.xml
new file mode 100644
index 0000000..f8ef0c1
--- /dev/null
+++ b/joshua-servlet/pom.xml
@@ -0,0 +1,89 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.joshua</groupId>
+ <artifactId>joshua-parent</artifactId>
+ <version>6.0.6-SNAPSHOT</version>
+ </parent>
+ <artifactId>joshua-servlet</artifactId>
+ <packaging>jar</packaging>
+ <name>Apache Joshua Machine Translation Toolkit - Servlet</name>
+ <description>Joshua is an open-source statistical machine
+ translation decoder for phrase-based, hierarchical,
+ and syntax-based machine translation, written in Java.
+ </description>
+
+ <properties>
+ <jetty.version>9.3.8.v20160314</jetty.version>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-maven-plugin</artifactId>
+ <version>${jetty.version}</version>
+ <configuration>
+ <webApp>
+ <descriptor>${project.basedir}/src/test/resources/web.xml</descriptor>
+ </webApp>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+ <dependencies>
+
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>joshua-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+
+ <dependency>
+ <groupId>javax.servlet</groupId>
+ <artifactId>javax.servlet-api</artifactId>
+ <version>3.1.0</version>
+ <scope>provided</scope>
+ </dependency>
+
+ <!-- Test Dependencies -->
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.12</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-server</artifactId>
+ <version>${jetty.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.eclipse.jetty</groupId>
+ <artifactId>jetty-annotations</artifactId>
+ <version>${jetty.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ae3a5df2/joshua-servlet/src/main/java/org/apache/joshua/decoder/DecoderServlet.java
----------------------------------------------------------------------
diff --git a/joshua-servlet/src/main/java/org/apache/joshua/decoder/DecoderServlet.java b/joshua-servlet/src/main/java/org/apache/joshua/decoder/DecoderServlet.java
new file mode 100644
index 0000000..93ae10d
--- /dev/null
+++ b/joshua-servlet/src/main/java/org/apache/joshua/decoder/DecoderServlet.java
@@ -0,0 +1,72 @@
+/**
+�* Copyright 2016 �Amazon.com, Inc. or its affiliates. All Rights Reserved.
+�* Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except
+ * in compliance with the License. A copy of the License is located at
+ * http://aws.amazon.com/apache-2-0/
+ * or in the "license" file accompanying this file. This file is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and limitations under the License.
+�*/
+package org.apache.joshua.decoder;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.nio.charset.Charset;
+
+import javax.servlet.ServletException;
+import javax.servlet.annotation.WebServlet;
+import javax.servlet.http.HttpServlet;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+
+import org.apache.joshua.decoder.io.JSONMessage;
+import org.apache.joshua.decoder.io.TranslationRequestStream;
+
+/**
+ * Simple servlet implementation to handle translation request via <code>q</code> parameter.
+ */
+@WebServlet(urlPatterns = "/")
+public class DecoderServlet extends HttpServlet {
+
+ @Override
+ protected void doGet(HttpServletRequest req, HttpServletResponse resp)
+ throws ServletException, IOException {
+ Decoder decoder = (Decoder)getServletContext().getAttribute(DecoderServletContextListener.DECODER_CONTEXT_ATTRIBUTE_NAME);
+
+ String param = req.getParameter("q");
+ try (InputStream in = new ByteArrayInputStream(param.getBytes());
+ OutputStream out = resp.getOutputStream()) {
+ resp.setHeader("Content-Type", "application/json");
+ handleRequest(decoder, in, out);
+ }
+ }
+
+ @Override
+ protected void doPost(HttpServletRequest req, HttpServletResponse resp)
+ throws ServletException, IOException {
+ Decoder decoder = (Decoder)getServletContext().getAttribute(DecoderServletContextListener.DECODER_CONTEXT_ATTRIBUTE_NAME);
+
+ try (InputStream in = req.getInputStream();
+ OutputStream out = resp.getOutputStream()) {
+ resp.setHeader("Content-Type", "application/json");
+ handleRequest(decoder, in, out);
+ }
+ }
+
+ private void handleRequest(Decoder decoder, InputStream in, OutputStream out) throws IOException {
+ BufferedReader reader = new BufferedReader(new InputStreamReader(in, Charset.forName("UTF-8")));
+ TranslationRequestStream request = new TranslationRequestStream(reader, decoder.getJoshuaConfiguration());
+
+ Translations translations = decoder.decodeAll(request);
+
+ JSONMessage message = new JSONMessage();
+ for (Translation translation : translations) {
+ message.addTranslation(translation);
+ }
+ out.write(message.toString().getBytes());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ae3a5df2/joshua-servlet/src/main/java/org/apache/joshua/decoder/DecoderServletContextListener.java
----------------------------------------------------------------------
diff --git a/joshua-servlet/src/main/java/org/apache/joshua/decoder/DecoderServletContextListener.java b/joshua-servlet/src/main/java/org/apache/joshua/decoder/DecoderServletContextListener.java
new file mode 100644
index 0000000..3860f7e
--- /dev/null
+++ b/joshua-servlet/src/main/java/org/apache/joshua/decoder/DecoderServletContextListener.java
@@ -0,0 +1,47 @@
+/**
+�* Copyright 2016 �Amazon.com, Inc. or its affiliates. All Rights Reserved.
+�* Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except
+ * in compliance with the License. A copy of the License is located at
+ * http://aws.amazon.com/apache-2-0/
+ * or in the "license" file accompanying this file. This file is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and limitations under the License.
+�*/
+package org.apache.joshua.decoder;
+
+import javax.servlet.ServletContextEvent;
+import javax.servlet.ServletContextListener;
+import javax.servlet.annotation.WebListener;
+
+import com.google.common.base.Throwables;
+
+/**
+ * Initializes {@link Decoder} via <code>decoderArgsLine</code> init parameter.
+ */
+@WebListener
+public class DecoderServletContextListener implements ServletContextListener {
+
+ /**
+ * Attribute name to locate initialized {@link Decoder} instance in ServletContext.
+ */
+ public static final String DECODER_CONTEXT_ATTRIBUTE_NAME = Decoder.class.getName();
+
+ @Override
+ public void contextDestroyed(ServletContextEvent sce) {
+ }
+
+ @Override
+ public void contextInitialized(ServletContextEvent sce) {
+ String argsLine = sce.getServletContext().getInitParameter("decoderArgsLine");
+ try {
+ JoshuaConfiguration joshuaConfiguration = new JoshuaConfiguration();
+ ArgsParser userArgs = new ArgsParser(argsLine.split(" "), joshuaConfiguration);
+ joshuaConfiguration.use_structured_output = true;
+ joshuaConfiguration.sanityCheck();
+ Decoder decoder = new Decoder(joshuaConfiguration, userArgs.getConfigFile());
+ sce.getServletContext().setAttribute(DECODER_CONTEXT_ATTRIBUTE_NAME, decoder);
+ } catch (Exception ex) {
+ Throwables.propagate(ex);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ae3a5df2/joshua-servlet/src/test/java/org/apache/joshua/decoder/DecoderServletTest.java
----------------------------------------------------------------------
diff --git a/joshua-servlet/src/test/java/org/apache/joshua/decoder/DecoderServletTest.java b/joshua-servlet/src/test/java/org/apache/joshua/decoder/DecoderServletTest.java
new file mode 100644
index 0000000..fcfc539
--- /dev/null
+++ b/joshua-servlet/src/test/java/org/apache/joshua/decoder/DecoderServletTest.java
@@ -0,0 +1,90 @@
+/**
+�* Copyright 2016 �Amazon.com, Inc. or its affiliates. All Rights Reserved.
+�* Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except
+ * in compliance with the License. A copy of the License is located at
+ * http://aws.amazon.com/apache-2-0/
+ * or in the "license" file accompanying this file. This file is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and limitations under the License.
+�*/
+package org.apache.joshua.decoder;
+
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.nio.charset.Charset;
+import java.nio.file.Paths;
+import java.util.Random;
+
+import com.google.common.io.ByteSink;
+import com.google.common.io.CharStreams;
+import com.google.common.io.Resources;
+
+import org.eclipse.jetty.annotations.AnnotationConfiguration;
+import org.eclipse.jetty.server.Server;
+import org.eclipse.jetty.util.resource.PathResource;
+import org.eclipse.jetty.webapp.Configuration;
+import org.eclipse.jetty.webapp.WebAppContext;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class DecoderServletTest {
+
+ private static int port;
+ private static Server server;
+
+ @BeforeClass
+ public static void startServer() throws Exception {
+ port = 8080 + new Random().nextInt(100);
+ System.out.println("Jetty test port is " + port);
+ server = new Server(port);
+ WebAppContext context = new WebAppContext();
+ context.setConfigurations(new Configuration[] {new AnnotationConfiguration()});
+ context.setContextPath("/");
+ context.setInitParameter("decoderArgsLine", "-mark-oovs true");
+ context.getMetaData().addContainerResource(new PathResource(Paths.get("./target/classes")));
+
+ server.setHandler(context);
+ server.start();
+ server.dump(System.err);
+ }
+
+ @AfterClass
+ public static void stopServer() throws Exception {
+ server.stop();
+ }
+
+ @Test
+ public void translateWithQueryParam() throws Exception {
+ String response = Resources.toString(
+ new URL("http://localhost:" + port + "/?q=I%20love%20it%20when%20I%20get%20the%20house%20clean%20before%20the%20weekend"),
+ Charset.forName("UTF-8"));
+ String expected = Resources.toString(Resources.getResource("server/http/expected"), Charset.forName("UTF-8"));
+ Assert.assertEquals(expected, response);
+ }
+
+ @Test
+ public void translateWithRawRequest() throws Exception {
+ String data = "I love it when I get the house clean before the weekend";
+ //Resources.asCharSink(new URL("http://localhost:" + port), Charset.forName("UTF-8")).
+ URL requestUrl = new URL("http://localhost:" + port);
+ HttpURLConnection conn = (HttpURLConnection) requestUrl.openConnection();
+ conn.setDoOutput(true);
+ new ByteSink() {
+ @Override
+ public OutputStream openStream() throws IOException {
+ return conn.getOutputStream();
+ }
+ }.write(data.getBytes("UTF-8"));
+ String response = CharStreams.toString(new InputStreamReader(conn.getInputStream()));
+ conn.disconnect();
+
+ String expected = Resources.toString(Resources.getResource("server/http/expected"), Charset.forName("UTF-8"));
+ Assert.assertEquals(expected, response);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ae3a5df2/joshua-servlet/src/test/resources/log4j.properties
----------------------------------------------------------------------
diff --git a/joshua-servlet/src/test/resources/log4j.properties b/joshua-servlet/src/test/resources/log4j.properties
new file mode 100644
index 0000000..5fb3730
--- /dev/null
+++ b/joshua-servlet/src/test/resources/log4j.properties
@@ -0,0 +1,20 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# log4j settings
+log4j.rootLogger=INFO, stdout
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.Target=System.err
+log4j.appender.stdout.layout=org.apache.log4j.SimpleLayout
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ae3a5df2/joshua-servlet/src/test/resources/server/http/expected
----------------------------------------------------------------------
diff --git a/joshua-servlet/src/test/resources/server/http/expected b/joshua-servlet/src/test/resources/server/http/expected
new file mode 100644
index 0000000..b1607af
--- /dev/null
+++ b/joshua-servlet/src/test/resources/server/http/expected
@@ -0,0 +1,16 @@
+{
+ "data": {
+ "translations": [
+ {
+ "translatedText": "I_OOV love_OOV it_OOV when_OOV I_OOV get_OOV the_OOV house_OOV clean_OOV before_OOV the_OOV weekend_OOV",
+ "raw_nbest": [
+ {
+ "hyp": "0 ||| I_OOV love_OOV it_OOV when_OOV I_OOV get_OOV the_OOV house_OOV clean_OOV before_OOV the_OOV weekend_OOV ||| tm_glue_0\u003d12.000 ||| 0.000",
+ "totalScore": 0.0
+ }
+ ]
+ }
+ ]
+ },
+ "metadata": []
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ae3a5df2/joshua-servlet/src/test/resources/web.xml
----------------------------------------------------------------------
diff --git a/joshua-servlet/src/test/resources/web.xml b/joshua-servlet/src/test/resources/web.xml
new file mode 100644
index 0000000..c7c68b8
--- /dev/null
+++ b/joshua-servlet/src/test/resources/web.xml
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Copyright 2016 �Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except
+ in compliance with the License. A copy of the License is located at
+ http://aws.amazon.com/apache-2-0/
+ or in the "license" file accompanying this file. This file is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and limitations under the License.
+-->
+<web-app
+ xmlns="http://xmlns.jcp.org/xml/ns/javaee"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://xmlns.jcp.org/xml/ns/javaee http://xmlns.jcp.org/xml/ns/javaee/web-app_3_1.xsd"
+ metadata-complete="false"
+ version="3.1">
+
+ <context-param>
+ <param-name>decoderArgsLine</param-name>
+ <param-value>-mark-oovs true</param-value>
+ </context-param>
+
+</web-app>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ae3a5df2/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index bc13483..7dc2c26 100644
--- a/pom.xml
+++ b/pom.xml
@@ -24,10 +24,10 @@
<version>10</version>
</parent>
<groupId>org.apache.joshua</groupId>
- <artifactId>joshua</artifactId>
- <packaging>jar</packaging>
+ <artifactId>joshua-parent</artifactId>
+ <packaging>pom</packaging>
<version>6.0.6-SNAPSHOT</version>
- <name>Apache Joshua Machine Translation Toolkit</name>
+ <name>Apache Joshua Machine Translation Toolkit - Parent</name>
<description>Joshua is an open-source statistical machine
translation decoder for phrase-based, hierarchical,
and syntax-based machine translation, written in Java.
@@ -112,145 +112,54 @@
<url>https://builds.apache.org/job/joshua_master/</url>
</ciManagement>
+ <modules>
+ <module>joshua-core</module>
+ <module>joshua-servlet</module>
+ </modules>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.12</version>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
<build>
<defaultGoal>install</defaultGoal>
- <directory>target</directory>
- <outputDirectory>${basedir}/target/classes</outputDirectory>
- <finalName>${project.artifactId}-${project.version}</finalName>
- <testOutputDirectory>${basedir}/target/test-classes</testOutputDirectory>
- <sourceDirectory>${basedir}/src/main/java</sourceDirectory>
- <testSourceDirectory>${basedir}/src/test/java</testSourceDirectory>
- <plugins>
- <plugin>
- <artifactId>maven-compiler-plugin</artifactId>
- <configuration>
- <source>1.8</source>
- <target>1.8</target>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-assembly-plugin</artifactId>
- <configuration>
- <archive>
- <manifest>
- <mainClass>org.apache.joshua.decoder.JoshuaDecoder</mainClass>
- </manifest>
- </archive>
- <descriptorRefs>
- <descriptorRef>jar-with-dependencies</descriptorRef>
- </descriptorRefs>
- </configuration>
- </plugin>
- <plugin>
- <artifactId>maven-surefire-plugin</artifactId>
- <version>2.19.1</version>
- </plugin>
- <plugin>
- <groupId>org.apache.rat</groupId>
- <artifactId>apache-rat-plugin</artifactId>
- <executions>
- <execution>
- <phase>test</phase>
- <goals>
- <goal>check</goal>
- </goals>
- <configuration>
- <includes>
- <include>src/main/java</include>
- <include>src/main/resources</include>
- <include>src/test/java</include>
- <include>pom.xml</include>
- </includes>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <source>1.8</source>
+ <target>1.8</target>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>test</phase>
+ <goals>
+ <goal>check</goal>
+ </goals>
+ <configuration>
+ <includes>
+ <include>src/main/java</include>
+ <include>src/main/resources</include>
+ <include>src/test/java</include>
+ <include>pom.xml</include>
+ </includes>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </pluginManagement>
</build>
- <dependencies>
- <dependency>
- <groupId>edu.berkeley.nlp</groupId>
- <artifactId>berkeleylm</artifactId>
- <version>1.1.2</version>
- </dependency>
- <dependency>
- <groupId>commons-cli</groupId>
- <artifactId>commons-cli</artifactId>
- <version>1.2</version>
- </dependency>
- <dependency>
- <groupId>net.sf.jung</groupId>
- <artifactId>jung-algorithms</artifactId>
- <version>2.0</version>
- <optional>true</optional>
- </dependency>
- <dependency>
- <groupId>net.sf.jung</groupId>
- <artifactId>jung-api</artifactId>
- <version>2.0</version>
- <optional>true</optional>
- </dependency>
- <dependency>
- <groupId>net.sf.jung</groupId>
- <artifactId>jung-graph-impl</artifactId>
- <version>2.0</version>
- <optional>true</optional>
- </dependency>
- <dependency>
- <groupId>net.sf.jung</groupId>
- <artifactId>jung-visualization</artifactId>
- <version>2.0</version>
- <optional>true</optional>
- </dependency>
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <version>19.0</version>
- </dependency>
- <dependency>
- <groupId>com.google.code.gson</groupId>
- <artifactId>gson</artifactId>
- <version>2.5</version>
- </dependency>
- <dependency>
- <groupId>args4j</groupId>
- <artifactId>args4j</artifactId>
- <version>2.0.29</version>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-api</artifactId>
- <version>${slf4j.version}</version>
- </dependency>
- <dependency>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- <version>${slf4j.version}</version>
- </dependency>
- <dependency>
- <groupId>concurrent</groupId>
- <artifactId>concurrent</artifactId>
- <version>1.3.4</version>
- </dependency>
- <!-- Test Dependencies -->
- <dependency>
- <groupId>junit</groupId>
- <artifactId>junit</artifactId>
- <version>4.12</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.testng</groupId>
- <artifactId>testng</artifactId>
- <version>6.9.10</version>
- <scope>test</scope>
- </dependency>
- <dependency>
- <groupId>org.mockito</groupId>
- <artifactId>mockito-core</artifactId>
- <version>2.0.52-beta</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
</project>
[53/60] incubator-joshua git commit: Merge branch
'maven-multi-module' of https://github.com/logogin/incubator-joshua into
maven-multi-module
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/test/resources/server/http/expected
----------------------------------------------------------------------
diff --cc joshua-core/src/test/resources/server/http/expected
index 0000000,11ea273..d0a254b
mode 000000,100644..100644
--- a/joshua-core/src/test/resources/server/http/expected
+++ b/joshua-core/src/test/resources/server/http/expected
@@@ -1,0 -1,15 +1,16 @@@
+ {
+ "data": {
+ "translations": [
+ {
+ "translatedText": "I_OOV love_OOV it_OOV when_OOV I_OOV get_OOV the_OOV house_OOV clean_OOV before_OOV the_OOV weekend_OOV",
+ "raw_nbest": [
+ {
+ "hyp": "I_OOV love_OOV it_OOV when_OOV I_OOV get_OOV the_OOV house_OOV clean_OOV before_OOV the_OOV weekend_OOV",
+ "totalScore": 0.0
+ }
+ ]
+ }
+ ]
- }
++ },
++ "metadata": []
+ }
[22/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/QuietFormatter.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/QuietFormatter.java b/joshua-core/src/main/java/org/apache/joshua/util/QuietFormatter.java
new file mode 100644
index 0000000..7220080
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/QuietFormatter.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util;
+
+import java.util.logging.Formatter;
+import java.util.logging.LogRecord;
+
+/**
+ * Log formatter that prints just the message, with no time stamp.
+ *
+ * @author Lane Schwartz
+ * @version $LastChangedDate$
+ */
+public class QuietFormatter extends Formatter {
+
+ public String format(LogRecord record) {
+ return "" + formatMessage(record) + "\n";
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/Regex.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/Regex.java b/joshua-core/src/main/java/org/apache/joshua/util/Regex.java
new file mode 100644
index 0000000..e592c11
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/Regex.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util;
+
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
+
+/**
+ * This class provides a repository for common regex patterns so that we don't keep recompiling them
+ * over and over again. Some convenience methods are provided to make the interface more similar to
+ * the convenience functions on String. The convenience methods on String are deprecated except for
+ * one-shot patterns (which, by definition, are not in loops).
+ *
+ * @author wren ng thornton wren@users.sourceforge.net
+ * @version $LastChangedDate: 2009-03-28 07:40:25 -0400 (Sat, 28 Mar 2009) $
+ */
+public class Regex {
+ // Alas, Pattern is final, thus no subclassing and this indirection
+ private final Pattern pattern;
+
+ // ===============================================================
+ // Singletons -- add all common patterns here
+ // ===============================================================
+ /**
+ * A pattern to match if the complete string is empty except for whitespace and end-of-line
+ * comments beginning with an octothorpe (<code>#</code>).
+ */
+ public static final Regex commentOrEmptyLine = new Regex("^\\s*(?:\\#.*)?$");
+
+ // BUG: this should be replaced by a real regex for numbers.
+ // Perhaps "^[\\+\\-]?\\d+(?:\\.\\d+)?$" is enough.
+ // This is only used by JoshuaDecoder.writeConfigFile so far.
+ /**
+ * A pattern to match floating point numbers. (Current implementation is overly permissive.)
+ */
+ public static final Regex floatingNumber = new Regex("^[\\d\\.\\-\\+]+");
+
+ // Common patterns for splitting
+ /**
+ * A pattern for splitting on one or more whitespace.
+ */
+ public static final Regex spaces = new Regex("\\s+");
+
+ /**
+ * A pattern for splitting on one or more whitespace.
+ */
+ public static final Regex tabs = new Regex("\\t+");
+
+ /**
+ * A pattern for splitting on the equals character, with optional whitespace on each side.
+ */
+ public static final Regex equalsWithSpaces = new Regex("\\s*=\\s*");
+
+ /**
+ * A pattern for splitting on three vertical pipes, with one or more whitespace on each side.
+ */
+ public static final Regex threeBarsWithSpace = new Regex("\\s\\|{3}\\s");
+
+
+ // ===============================================================
+ // Constructor
+ // ===============================================================
+
+ public Regex(String regex) throws PatternSyntaxException {
+ this.pattern = Pattern.compile(regex);
+ }
+
+
+ // ===============================================================
+ // Convenience Methods
+ // ===============================================================
+
+ /**
+ * Returns whether the input string matches this <code>Regex</code>.
+ * @param input a String to match against the Regex
+ * @return true if the input string matches this Regex
+ */
+ public final boolean matches(String input) {
+ return this.pattern.matcher(input).matches();
+ }
+
+
+ /**
+ * Split a character sequence, removing instances of this <code>Regex</code>.
+ * @param input an input string to split
+ * @return a String array representing the split character sequences less the regex characters patterns
+ */
+ public final String[] split(CharSequence input) {
+ return this.pattern.split(input);
+ }
+
+
+ /**
+ * Split a character sequence, removing instances of this <code>Regex</code>, up to a limited
+ * number of segments.
+ * @param input an input string to split
+ * @param limit maximum number of splits
+ * @return a String array representing the split character sequences less the regex characters patterns
+ */
+ public final String[] split(CharSequence input, int limit) {
+ return this.pattern.split(input, limit);
+ }
+
+
+ /**
+ * Replace all substrings of the input which match this <code>Regex</code> with the specified
+ * replacement string.
+ * @param input an input string for which to make replacements
+ * @param replacement the replacement string
+ * @return a new replacement string
+ */
+ public final String replaceAll(String input, String replacement) {
+ return this.pattern.matcher(input).replaceAll(replacement);
+ }
+
+
+ /**
+ * Replace the first substring of the input which matches this <code>Regex</code> with the
+ * specified replacement string.
+ * @param input the input string for replacement
+ * @param replacement the first substring of the input to replace
+ * @return the new string
+ */
+ public final String replaceFirst(String input, String replacement) {
+ return this.pattern.matcher(input).replaceFirst(replacement);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/ReverseOrder.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/ReverseOrder.java b/joshua-core/src/main/java/org/apache/joshua/util/ReverseOrder.java
new file mode 100644
index 0000000..0270036
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/ReverseOrder.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util;
+
+import java.util.Comparator;
+
+/**
+ * ReverseOrder is a Comparator that reverses the natural order of Comparable objects.
+ *
+ * @author Chris Callison-Burch
+ * @since 2 June 2008
+ */
+public class ReverseOrder<K extends Comparable<K>> implements Comparator<K> {
+
+ public int compare(K obj1, K obj2) {
+ int comparison = obj1.compareTo(obj2);
+ if (comparison != 0) {
+ comparison = comparison * -1;
+ }
+ return comparison;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/SampledList.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/SampledList.java b/joshua-core/src/main/java/org/apache/joshua/util/SampledList.java
new file mode 100644
index 0000000..60b0ef9
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/SampledList.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util;
+
+import java.util.AbstractList;
+import java.util.List;
+
+/**
+ * List that performs sampling at specified intervals.
+ *
+ * @author Lane Schwartz
+ * @version $LastChangedDate$
+ */
+public class SampledList<E> extends AbstractList<E> implements List<E> {
+
+ private final List<E> list;
+ private final int size;
+ private final int stepSize;
+
+ /**
+ * Constructs a sampled list backed by a provided list.
+ * <p>
+ * The maximum size of this list will be no greater than the provided sample size.
+ *
+ * @param list List from which to sample.
+ * @param sampleSize Maximum number of items to include in the new sampled list.
+ */
+ public SampledList(List<E> list, int sampleSize) {
+ this.list = list;
+
+ int listSize = list.size();
+
+ if (listSize <= sampleSize) {
+ this.size = listSize;
+ this.stepSize = 1;
+ } else {
+ this.size = sampleSize;
+ this.stepSize = listSize / sampleSize;
+ }
+
+ }
+
+ @Override
+ public E get(int index) {
+ return list.get(index * stepSize);
+ }
+
+ @Override
+ public int size() {
+ return size;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/SocketUtility.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/SocketUtility.java b/joshua-core/src/main/java/org/apache/joshua/util/SocketUtility.java
new file mode 100644
index 0000000..134fd35
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/SocketUtility.java
@@ -0,0 +1,144 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util;
+
+import java.io.BufferedReader;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.net.SocketAddress;
+
+/**
+ *
+ * @author Zhifei Li, zhifei.work@gmail.com
+ * @version $LastChangedDate$
+ */
+public class SocketUtility {
+
+ // ############# client side #########
+ // connect to server
+ public static ClientConnection open_connection_client(String hostname, int port) {
+ ClientConnection res = new ClientConnection();
+ // TODO: remove from class
+ // res.hostname = hostname;
+ // res.port = port;
+ try {
+ InetAddress addr = InetAddress.getByName(hostname);
+ SocketAddress sockaddr = new InetSocketAddress(addr, port);
+
+ res.socket = new Socket(); // Create an unbound socket
+ // This method will block no more than timeoutMs If the timeout occurs, SocketTimeoutException
+ // is thrown.
+ int timeoutMs = 3000; // 2 seconds
+ res.socket.connect(sockaddr, timeoutMs);
+ res.socket.setKeepAlive(true);
+ // file
+ res.in = new BufferedReader(new InputStreamReader(res.socket.getInputStream()));
+ res.out = new PrintWriter(new OutputStreamWriter(res.socket.getOutputStream()));
+
+ // TODO: for debugging, but should be removed
+ // res.data_in = new DataInputStream(new BufferedInputStream( res.socket.getInputStream()));
+ // res.data_out = new DataOutputStream(new BufferedOutputStream
+ // (res.socket.getOutputStream()));
+
+ } catch ( IOException e) {
+ throw new RuntimeException(e);
+ }
+ return res;
+ }
+
+
+ public static class ClientConnection {
+ // TODO: These are never read from, so we're planning to remove them
+ // String hostname; // server name
+ // int port; // server port
+ Socket socket;
+ public BufferedReader in;
+ public PrintWriter out;
+
+ // TODO: for debugging, but should be removed
+ // public DataOutputStream data_out;
+ // public DataInputStream data_in;
+
+ public String exe_request(String line_out) {
+ String line_res = null;
+ try {
+ out.println(line_out);
+ out.flush();
+ line_res = in.readLine(); // TODO block function, big bug, the server may close the section
+ // (e.g., the server thread is dead due to out of memory(which is
+ // possible due to cache) )
+ } catch (IOException ioe) {
+ ioe.printStackTrace();
+ }
+ return line_res;
+ }
+
+ public void write_line(String line_out) {
+ out.println(line_out);
+ out.flush();
+ }
+
+ public void write_int(int line_out) {
+ out.println(line_out);
+ out.flush();
+ }
+
+ public String read_line() {
+ String line_res = null;
+ try {
+ line_res = in.readLine(); // TODO block function, big bug, the server may close the section
+ // (e.g., the server thread is dead due to out of memory(which is
+ // possible due to cache) )
+ } catch (IOException ioe) {
+ ioe.printStackTrace();
+ }
+ return line_res;
+ }
+
+
+ public void close() {
+ try {
+ socket.close();
+ } catch (IOException ioe) {
+ ioe.printStackTrace();
+ }
+ }
+
+ public static double readDoubleLittleEndian(DataInputStream d_in) {
+ long accum = 0;
+ try {
+ for (int shiftBy = 0; shiftBy < 64; shiftBy += 8) {
+ // must cast to long or shift done modulo 32
+ accum |= ((long) (d_in.readByte() & 0xff)) << shiftBy;
+ }
+ } catch (IOException ioe) {
+ ioe.printStackTrace();
+ }
+
+ return Double.longBitsToDouble(accum);
+ // there is no such method as Double.reverseBytes(d);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/src/main/java/org/apache/joshua/util/StreamGobbler.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/util/StreamGobbler.java b/joshua-core/src/main/java/org/apache/joshua/util/StreamGobbler.java
new file mode 100644
index 0000000..7bb12ca
--- /dev/null
+++ b/joshua-core/src/main/java/org/apache/joshua/util/StreamGobbler.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.joshua.util;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+
+/**
+ * Based on: http://www.javaworld.com/javaworld/jw-12-2000/jw-1229-traps.html?page=4
+ */
+public class StreamGobbler extends Thread {
+ InputStream istream;
+ boolean verbose;
+
+ public StreamGobbler(InputStream is, int p) {
+ istream = is;
+ verbose = (p != 0);
+ }
+
+ public void run() {
+ try {
+ InputStreamReader isreader = new InputStreamReader(istream);
+ BufferedReader br = new BufferedReader(isreader);
+ String line = null;
+ while ((line = br.readLine()) != null) {
+ if (verbose) System.out.println(line);
+ }
+ } catch (IOException ioe) {
+ ioe.printStackTrace();
+ }
+ }
+}
[57/60] incubator-joshua git commit: Merge branch
'maven-multi-module' of https://github.com/logogin/incubator-joshua into
maven-multi-module
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/Rule.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/Rule.java
index 0000000,0e5a4bc..15fbec1
mode 000000,100644..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/Rule.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/Rule.java
@@@ -1,0 -1,633 +1,635 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.decoder.ff.tm;
+
++import static org.apache.joshua.decoder.ff.tm.OwnerMap.UNKNOWN_OWNER_ID;
++
+ import java.util.ArrayList;
+ import java.util.Arrays;
+ import java.util.Comparator;
+ import java.util.HashMap;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.regex.Pattern;
+
+ import com.google.common.base.Supplier;
+ import com.google.common.base.Suppliers;
+
+ import org.apache.joshua.corpus.Vocabulary;
+ import org.apache.joshua.decoder.ff.FeatureFunction;
+ import org.apache.joshua.decoder.ff.FeatureVector;
+ import org.apache.joshua.decoder.segment_file.Sentence;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
+ /**
+ * This class define the interface for Rule.
+ *
+ * All feature scores are interpreted as negative log probabilities, and are therefore negated.
+ * Note that not all features need to be negative log probs, but you should be aware that they
+ * will be negated, so if you want a positive count, it should come in as negative.
+ *
+ * Normally, the feature score in the rule should be *cost* (i.e., -LogP), so that the feature
+ * weight should be positive
+ *
+ * @author Zhifei Li, zhifei.work@gmail.com
+ * @author Matt Post post@cs.jhu.edu
+ */
+ public class Rule implements Comparator<Rule>, Comparable<Rule> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Rule.class);
+ private int lhs; // tag of this rule
+ private int[] source; // pointer to the RuleCollection, as all the rules under it share the same
+ // Source side
+ protected int arity;
+
+ // And a string containing the sparse ones
+ //protected final String sparseFeatureString;
+ protected final Supplier<String> sparseFeatureStringSupplier;
+ private final Supplier<FeatureVector> featuresSupplier;
+
+ /*
+ * a feature function will be fired for this rule only if the owner of the rule matches the owner
+ * of the feature function
+ */
- private int owner = -1;
++ private OwnerId owner = UNKNOWN_OWNER_ID;
+
+ /**
+ * This is the cost computed only from the features present with the grammar rule. This cost is
+ * needed to sort the rules in the grammar for cube pruning, but isn't the full cost of applying
+ * the rule (which will include contextual features that can't be computed until the rule is
+ * applied).
+ */
+ private float estimatedCost = Float.NEGATIVE_INFINITY;
+
+ private float precomputableCost = Float.NEGATIVE_INFINITY;
+
+ private int[] target;
+
+ // The alignment string, e.g., 0-0 0-1 1-1 2-1
+ private String alignmentString;
+ private final Supplier<byte[]> alignmentSupplier;
+
+ /**
+ * Constructs a new rule using the provided parameters. Rule id for this rule is
+ * undefined. Note that some of the sparse features may be unlabeled, but they cannot be mapped to
+ * their default names ("tm_OWNER_INDEX") until later, when we know the owner of the rule. This is
+ * not known until the rule is actually added to a grammar in Grammar::addRule().
+ *
+ * Constructor used by other constructors below;
+ *
+ * @param lhs Left-hand side of the rule.
+ * @param source Source language right-hand side of the rule.
+ * @param target Target language right-hand side of the rule.
+ * @param sparseFeatures Feature value scores for the rule.
+ * @param arity Number of nonterminals in the source language right-hand side.
+ * @param owner todo
+ */
- public Rule(int lhs, int[] source, int[] target, String sparseFeatures, int arity, int owner) {
++ public Rule(int lhs, int[] source, int[] target, String sparseFeatures, int arity, OwnerId owner) {
+ this.lhs = lhs;
+ this.source = source;
+ this.arity = arity;
+ this.owner = owner;
+ this.target = target;
+ this.sparseFeatureStringSupplier = Suppliers.memoize(() -> { return sparseFeatures; });
+ this.featuresSupplier = initializeFeatureSupplierFromString();
+ this.alignmentSupplier = initializeAlignmentSupplier();
+ }
+
+ /**
+ * Constructor used by PackedGrammar's sortRules()
+ * @param lhs todo
+ * @param sourceRhs todo
+ * @param targetRhs todo
+ * @param features todo
+ * @param arity todo
+ * @param owner todo
+ */
- public Rule(int lhs, int[] sourceRhs, int[] targetRhs, FeatureVector features, int arity, int owner) {
++ public Rule(int lhs, int[] sourceRhs, int[] targetRhs, FeatureVector features, int arity, OwnerId owner) {
+ this.lhs = lhs;
+ this.source = sourceRhs;
+ this.arity = arity;
+ this.owner = owner;
+ this.target = targetRhs;
+ this.featuresSupplier = Suppliers.memoize(() -> { return features; });
+ this.sparseFeatureStringSupplier = initializeSparseFeaturesStringSupplier();
+ this.alignmentSupplier = initializeAlignmentSupplier();
+ }
+
+ /**
+ * Constructor used for SamtFormatReader and GrammarBuilderWalkerFunction's getRuleWithSpans()
- * Owner set to -1
++ * Rule is unowned.
+ * @param lhs todo
+ * @param sourceRhs todo
+ * @param targetRhs todo
+ * @param sparseFeatures todo
+ * @param arity todo
+ */
+ public Rule(int lhs, int[] sourceRhs, int[] targetRhs, String sparseFeatures, int arity) {
- this(lhs, sourceRhs, targetRhs, sparseFeatures, arity, -1);
++ this(lhs, sourceRhs, targetRhs, sparseFeatures, arity, OwnerMap.UNKNOWN_OWNER_ID);
+ }
+
+ /**
+ * Constructor used for addOOVRules(), HieroFormatReader and PhraseRule.
+ * @param lhs todo
+ * @param sourceRhs todo
+ * @param targetRhs todo
+ * @param sparseFeatures todo
+ * @param arity todo
+ * @param alignment todo
+ */
+ public Rule(int lhs, int[] sourceRhs, int[] targetRhs, String sparseFeatures, int arity, String alignment) {
+ this(lhs, sourceRhs, targetRhs, sparseFeatures, arity);
+ this.alignmentString = alignment;
+ }
+
+ /**
+ * Constructor (implicitly) used by PackedRule
+ */
+ public Rule() {
+ this.lhs = -1;
+ this.sparseFeatureStringSupplier = initializeSparseFeaturesStringSupplier();
+ this.featuresSupplier = initializeFeatureSupplierFromString();
+ this.alignmentSupplier = initializeAlignmentSupplier();
+ }
+
+ // ==========================================================================
+ // Lazy loading Suppliers for alignments, feature vector, and feature strings
+ // ==========================================================================
+
+ private Supplier<byte[]> initializeAlignmentSupplier(){
+ return Suppliers.memoize(() ->{
+ byte[] alignment = null;
+ String alignmentString = getAlignmentString();
+ if (alignmentString != null) {
+ String[] tokens = alignmentString.split("[-\\s]+");
+ alignment = new byte[tokens.length];
+ for (int i = 0; i < tokens.length; i++)
+ alignment[i] = (byte) Short.parseShort(tokens[i]);
+ }
+ return alignment;
+ });
+ }
+
+ /**
+ * If Rule was constructed with sparseFeatures String, we lazily populate the
+ * FeatureSupplier.
+ */
+ private Supplier<FeatureVector> initializeFeatureSupplierFromString(){
+ return Suppliers.memoize(() ->{
- if (owner != -1) {
- return new FeatureVector(getFeatureString(), "tm_" + Vocabulary.word(owner) + "_");
++ if (!owner.equals(UNKNOWN_OWNER_ID)) {
++ return new FeatureVector(getFeatureString(), "tm_" + OwnerMap.getOwner(owner) + "_");
+ } else {
+ return new FeatureVector();
+ }
+ });
+ }
+
+ /**
+ * If Rule was constructed with a FeatureVector, we lazily populate the sparseFeaturesStringSupplier.
+ */
+ private Supplier<String> initializeSparseFeaturesStringSupplier() {
+ return Suppliers.memoize(() -> {
+ return getFeatureVector().toString();
+ });
+ }
+
+ // ===============================================================
+ // Attributes
+ // ===============================================================
+
+ public void setEnglish(int[] eng) {
+ this.target = eng;
+ }
+
+ public int[] getEnglish() {
+ return this.target;
+ }
+
+ /**
+ * Two Rules are equal of they have the same LHS, the same source RHS and the same target
+ * RHS.
+ *
+ * @param o the object to check for equality
+ * @return true if o is the same Rule as this rule, false otherwise
+ */
+ public boolean equals(Object o) {
+ if (!(o instanceof Rule)) {
+ return false;
+ }
+ Rule other = (Rule) o;
+ if (getLHS() != other.getLHS()) {
+ return false;
+ }
+ if (!Arrays.equals(getFrench(), other.getFrench())) {
+ return false;
+ }
+ if (!Arrays.equals(target, other.getEnglish())) {
+ return false;
+ }
+ return true;
+ }
+
+ public int hashCode() {
+ // I just made this up. If two rules are equal they'll have the
+ // same hashcode. Maybe someone else can do a better job though?
+ int frHash = Arrays.hashCode(getFrench());
+ int enHash = Arrays.hashCode(target);
+ return frHash ^ enHash ^ getLHS();
+ }
+
+ // ===============================================================
+ // Attributes
+ // ===============================================================
+
+ public void setArity(int arity) {
+ this.arity = arity;
+ }
+
+ public int getArity() {
+ return this.arity;
+ }
+
- public void setOwner(int owner) {
++ public void setOwner(final OwnerId owner) {
+ this.owner = owner;
+ }
+
- public int getOwner() {
++ public OwnerId getOwner() {
+ return this.owner;
+ }
+
+ public void setLHS(int lhs) {
+ this.lhs = lhs;
+ }
+
+ public int getLHS() {
+ return this.lhs;
+ }
+
+ public void setFrench(int[] french) {
+ this.source = french;
+ }
+
+ public int[] getFrench() {
+ return this.source;
+ }
+
+ /**
+ * This function does the work of turning the string version of the sparse features (passed in
+ * when the rule was created) into an actual set of features. This is a bit complicated because we
+ * support intermingled labeled and unlabeled features, where the unlabeled features are mapped to
+ * a default name template of the form "tm_OWNER_INDEX".
+ *
+ * This function returns the dense (phrasal) features discovered when the rule was loaded. Dense
+ * features are the list of unlabeled features that preceded labeled ones. They can also be
+ * specified as labeled features of the form "tm_OWNER_INDEX", but the former format is preferred.
+ *
+ * @return the {@link org.apache.joshua.decoder.ff.FeatureVector} for this rule
+ */
+ public FeatureVector getFeatureVector() {
+ return featuresSupplier.get();
+ }
+
+ /**
+ * This function returns the estimated cost of a rule, which should have been computed when the
+ * grammar was first sorted via a call to Rule::estimateRuleCost(). This function is a getter
+ * only; it will not compute the value if it has not already been set. It is necessary in addition
+ * to estimateRuleCost(models) because sometimes the value needs to be retrieved from contexts
+ * that do not have access to the feature functions.
+ *
+ * This function is called by the rule comparator when sorting the grammar. As such it may be
+ * called many times and any implementation of it should be a cached implementation.
+ *
+ * @return the estimated cost of the rule (a lower bound on the true cost)
+ */
+ public float getEstimatedCost() {
+ return estimatedCost;
+ }
+
+ /**
+ * Precomputable costs is the inner product of the weights found on each grammar rule and the
+ * weight vector. This is slightly different from the estimated rule cost, which can include other
+ * features (such as a language model estimate). This getter and setter should also be cached, and
+ * is basically provided to allow the PhraseModel feature to cache its (expensive) computation for
+ * each rule.
+ *
+ * The weights are passed in as dense weights and sparse weights. This allows the dense weight
+ * precomputation to be even faster (since we don't have to query a hash map.
+ *
+ * @param dense_weights the dense weights from the model
+ * @param weights the sparse weights from the model
+ */
+ public void setPrecomputableCost(float[] dense_weights, FeatureVector weights) {
+ float cost = 0.0f;
+ FeatureVector features = getFeatureVector();
+ for (int i = 0; i < features.getDenseFeatures().size() && i < dense_weights.length; i++) {
+ cost += dense_weights[i] * features.getDense(i);
+ }
+
+ for (String key: features.getSparseFeatures().keySet()) {
+ cost += weights.getSparse(key) * features.getSparse(key);
+ }
+
+ this.precomputableCost = cost;
+ }
+
+ /**
+ * @return the precomputed model cost of each rule
+ */
+ public float getPrecomputableCost() {
+ return precomputableCost;
+ }
+
+ public float getDenseFeature(int k) {
+ return getFeatureVector().getDense(k);
+ }
+
+ /**
+ * This function estimates the cost of a rule, which is used for sorting the rules for cube
+ * pruning. The estimated cost is basically the set of precomputable features (features listed
+ * along with the rule in the grammar file) along with any other estimates that other features
+ * would like to contribute (e.g., a language model estimate). This cost will be a lower bound on
+ * the rule's actual cost.
+ *
+ * The value of this function is used only for sorting the rules. When the rule is later applied
+ * in context to particular hypernodes, the rule's actual cost is computed.
+ *
+ * @param models the list of models available to the decoder
+ * @return estimated cost of the rule
+ */
+ public float estimateRuleCost(List<FeatureFunction> models) {
+ if (null == models)
+ return 0.0f;
+
+ if (this.estimatedCost <= Float.NEGATIVE_INFINITY) {
+ this.estimatedCost = 0.0f; // weights.innerProduct(computeFeatures());
+
+ LOG.debug("estimateCost({} ;; {})", getFrenchWords(), getEnglishWords());
+ for (FeatureFunction ff : models) {
+ float val = ff.estimateCost(this, null);
+ LOG.debug(" FEATURE {} -> {}", ff.getName(), val);
+ this.estimatedCost += val;
+ }
+ }
+
+ return estimatedCost;
+ }
+
+ // ===============================================================
+ // Methods
+ // ===============================================================
+
+ public String toString() {
+ StringBuffer sb = new StringBuffer();
+ sb.append(Vocabulary.word(this.getLHS()));
+ sb.append(" ||| ");
+ sb.append(getFrenchWords());
+ sb.append(" ||| ");
+ sb.append(getEnglishWords());
+ sb.append(" |||");
+ sb.append(" " + getFeatureVector());
+ sb.append(String.format(" ||| est=%.3f", getEstimatedCost()));
+ sb.append(String.format(" pre=%.3f", getPrecomputableCost()));
+ return sb.toString();
+ }
+
+ /**
+ * Returns a version of the rule suitable for reading in from a text file.
+ *
+ * @return string version of the rule
+ */
+ public String textFormat() {
+ StringBuffer sb = new StringBuffer();
+ sb.append(Vocabulary.word(this.getLHS()));
+ sb.append(" |||");
+
+ int nt = 1;
+ for (int i = 0; i < getFrench().length; i++) {
+ if (getFrench()[i] < 0)
+ sb.append(" " + Vocabulary.word(getFrench()[i]).replaceFirst("\\]", String.format(",%d]", nt++)));
+ else
+ sb.append(" " + Vocabulary.word(getFrench()[i]));
+ }
+ sb.append(" |||");
+ nt = 1;
+ for (int i = 0; i < getEnglish().length; i++) {
+ if (getEnglish()[i] < 0)
+ sb.append(" " + Vocabulary.word(getEnglish()[i]).replaceFirst("\\]", String.format(",%d]", nt++)));
+ else
+ sb.append(" " + Vocabulary.word(getEnglish()[i]));
+ }
+ sb.append(" |||");
+ sb.append(" " + getFeatureString());
+ if (getAlignmentString() != null)
+ sb.append(" ||| " + getAlignmentString());
+ return sb.toString();
+ }
+
+ public String getFeatureString() {
+ return sparseFeatureStringSupplier.get();
+ }
+
+ /**
+ * Returns an alignment as a sequence of integers. The integers at positions i and i+1 are paired,
+ * with position i indexing the source and i+1 the target.
+ *
+ * @return a byte[] from the {@link com.google.common.base.Supplier}
+ */
+ public byte[] getAlignment() {
+ return this.alignmentSupplier.get();
+ }
+
+ public String getAlignmentString() {
+ return this.alignmentString;
+ }
+
+ /**
+ * The nonterminals on the English side are pointers to the source side nonterminals (-1 and -2),
+ * rather than being directly encoded. These number indicate the correspondence between the
+ * nonterminals on each side, introducing a level of indirection however when we want to resolve
+ * them. So to get the ID, we need to look up the corresponding source side ID.
+ *
+ * @return The string of English words
+ */
+ public String getEnglishWords() {
+ int[] foreignNTs = getForeignNonTerminals();
+
+ StringBuilder sb = new StringBuilder();
+ for (Integer index : getEnglish()) {
+ if (index >= 0)
+ sb.append(Vocabulary.word(index) + " ");
+ else
+ sb.append(Vocabulary.word(foreignNTs[-index - 1]).replace("]",
+ String.format(",%d] ", Math.abs(index))));
+ }
+
+ return sb.toString().trim();
+ }
+
+ public boolean isTerminal() {
+ for (int i = 0; i < getEnglish().length; i++)
+ if (getEnglish()[i] < 0)
+ return false;
+
+ return true;
+ }
+
+ /**
+ * Return the French (source) nonterminals as list of Strings
+ *
+ * @return a list of strings
+ */
+ public int[] getForeignNonTerminals() {
+ int[] nts = new int[getArity()];
+ int index = 0;
+ for (int id : getFrench())
+ if (id < 0)
+ nts[index++] = -id;
+ return nts;
+ }
+
+ /**
+ * Returns an array of size getArity() containing the source indeces of non terminals.
+ *
+ * @return an array of size getArity() containing the source indeces of non terminals
+ */
+ public int[] getNonTerminalSourcePositions() {
+ int[] nonTerminalPositions = new int[getArity()];
+ int ntPos = 0;
+ for (int sourceIdx = 0; sourceIdx < getFrench().length; sourceIdx++) {
+ if (getFrench()[sourceIdx] < 0)
+ nonTerminalPositions[ntPos++] = sourceIdx;
+ }
+ return nonTerminalPositions;
+ }
+
+ /**
+ * Parses the Alignment byte[] into a Map from target to (possibly a list of) source positions.
+ * Used by the WordAlignmentExtractor.
+ *
+ * @return a {@link java.util.Map} of alignments
+ */
+ public Map<Integer, List<Integer>> getAlignmentMap() {
+ byte[] alignmentArray = getAlignment();
+ Map<Integer, List<Integer>> alignmentMap = new HashMap<Integer, List<Integer>>();
+ if (alignmentArray != null) {
+ for (int alignmentIdx = 0; alignmentIdx < alignmentArray.length; alignmentIdx += 2 ) {
+ int s = alignmentArray[alignmentIdx];
+ int t = alignmentArray[alignmentIdx + 1];
+ List<Integer> values = alignmentMap.get(t);
+ if (values == null)
+ alignmentMap.put(t, values = new ArrayList<Integer>());
+ values.add(s);
+ }
+ }
+ return alignmentMap;
+ }
+
+ /**
+ * Return the English (target) nonterminals as list of Strings
+ *
+ * @return list of strings
+ */
+ public int[] getEnglishNonTerminals() {
+ int[] nts = new int[getArity()];
+ int[] foreignNTs = getForeignNonTerminals();
+ int index = 0;
+
+ for (int i : getEnglish()) {
+ if (i < 0)
+ nts[index++] = foreignNTs[Math.abs(getEnglish()[i]) - 1];
+ }
+
+ return nts;
+ }
+
+ private int[] getNormalizedEnglishNonterminalIndices() {
+ int[] result = new int[getArity()];
+
+ int ntIndex = 0;
+ for (Integer index : getEnglish()) {
+ if (index < 0)
+ result[ntIndex++] = -index - 1;
+ }
+
+ return result;
+ }
+
+ public boolean isInverting() {
+ int[] normalizedEnglishNonTerminalIndices = getNormalizedEnglishNonterminalIndices();
+ if (normalizedEnglishNonTerminalIndices.length == 2) {
+ if (normalizedEnglishNonTerminalIndices[0] == 1) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public String getFrenchWords() {
+ return Vocabulary.getWords(getFrench());
+ }
+
+ public static final String NT_REGEX = "\\[[^\\]]+?\\]";
+
+ private Pattern getPattern() {
+ String source = getFrenchWords();
+ String pattern = Pattern.quote(source);
+ pattern = pattern.replaceAll(NT_REGEX, "\\\\E.+\\\\Q");
+ pattern = pattern.replaceAll("\\\\Q\\\\E", "");
+ pattern = "(?:^|\\s)" + pattern + "(?:$|\\s)";
+ return Pattern.compile(pattern);
+ }
+
+ /**
+ * Matches the string representation of the rule's source side against a sentence
+ *
+ * @param sentence {@link org.apache.joshua.lattice.Lattice} input
+ * @return true if there is a match
+ */
+ public boolean matches(Sentence sentence) {
+ boolean match = getPattern().matcher(sentence.fullSource()).find();
+ // System.err.println(String.format("match(%s,%s) = %s", Pattern.quote(getFrenchWords()),
+ // sentence.annotatedSource(), match));
+ return match;
+ }
+
+ /**
+ * This comparator is used for sorting the rules during cube pruning. An estimate of the cost
+ * of each rule is computed and used to sort.
+ */
+ public static Comparator<Rule> EstimatedCostComparator = new Comparator<Rule>() {
+ public int compare(Rule rule1, Rule rule2) {
+ float cost1 = rule1.getEstimatedCost();
+ float cost2 = rule2.getEstimatedCost();
+ return Float.compare(cost2, cost1);
+ }
+ };
+
+ public int compare(Rule rule1, Rule rule2) {
+ return EstimatedCostComparator.compare(rule1, rule2);
+ }
+
+ public int compareTo(Rule other) {
+ return EstimatedCostComparator.compare(this, other);
+ }
+
+ public String getRuleString() {
+ return String.format("%s -> %s ||| %s", Vocabulary.word(getLHS()), getFrenchWords(), getEnglishWords());
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/SentenceFilteredGrammar.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/SentenceFilteredGrammar.java
index 0000000,c952b05..54f68b2
mode 000000,100644..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/SentenceFilteredGrammar.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/SentenceFilteredGrammar.java
@@@ -1,0 -1,366 +1,366 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.decoder.ff.tm;
+
+ import java.util.Collection;
+ import java.util.HashMap;
+ import java.util.Iterator;
+ import java.util.Map.Entry;
+
+ import org.apache.joshua.decoder.ff.tm.hash_based.ExtensionIterator;
+ import org.apache.joshua.decoder.ff.tm.hash_based.MemoryBasedBatchGrammar;
+ import org.apache.joshua.decoder.segment_file.Sentence;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
+ /**
+ * This class implements dynamic sentence-level filtering. This is accomplished with a parallel
+ * trie, a subset of the original trie, that only contains trie paths that are reachable from
+ * traversals of the current sentence.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ */
+ public class SentenceFilteredGrammar extends MemoryBasedBatchGrammar {
+
+ private static final Logger LOG = LoggerFactory.getLogger(SentenceFilteredGrammar.class);
+
+ private AbstractGrammar baseGrammar;
+ private SentenceFilteredTrie filteredTrie;
+ private int[] tokens;
+ private Sentence sentence;
+
+ /**
+ * Construct a new sentence-filtered grammar. The main work is done in the enclosed trie (obtained
+ * from the base grammar, which contains the complete grammar).
+ *
+ * @param baseGrammar a new {@link org.apache.joshua.decoder.ff.tm.AbstractGrammar} to populate
+ * @param sentence {@link org.apache.joshua.lattice.Lattice} input
+ */
+ SentenceFilteredGrammar(AbstractGrammar baseGrammar, Sentence sentence) {
- super(baseGrammar.joshuaConfiguration);
++ super(OwnerMap.getOwner(baseGrammar.getOwner()), baseGrammar.joshuaConfiguration, baseGrammar.getSpanLimit());
+ this.baseGrammar = baseGrammar;
+ this.sentence = sentence;
+ this.tokens = sentence.getWordIDs();
+
+ int origCount = getNumRules(baseGrammar.getTrieRoot());
+ long startTime = System.currentTimeMillis();
+
+ /* Filter the rules; returns non-null object */
+ this.filteredTrie = filter(baseGrammar.getTrieRoot());
+ int filteredCount = getNumRules();
+
+ float seconds = (System.currentTimeMillis() - startTime) / 1000.0f;
+
+ LOG.debug("Sentence-level filtering of sentence {} ({} -> {} rules) in {} seconds",
+ sentence.id(), origCount, filteredCount, seconds);
+ }
+
+ @Override
+ public Trie getTrieRoot() {
+ return filteredTrie;
+ }
+
+ /**
+ * This function is poorly named: it doesn't mean whether a rule exists in the grammar for the
+ * current span, but whether the grammar is permitted to apply rules to the current span (a
+ * grammar-level parameter). As such we can just chain to the underlying grammar.
+ */
+ @Override
+ public boolean hasRuleForSpan(int startIndex, int endIndex, int pathLength) {
+ return baseGrammar.hasRuleForSpan(startIndex, endIndex, pathLength);
+ }
+
+ @Override
+ public int getNumRules() {
+ return getNumRules(getTrieRoot());
+ }
+
+ /**
+ * A convenience function that counts the number of rules in a grammar's trie.
+ *
+ * @param node the {@link org.apache.joshua.decoder.ff.tm.Trie} implementation for which to count rules
+ * @return the number of rules
+ */
+ public int getNumRules(Trie node) {
+ int numRules = 0;
+ if (node != null) {
+ if (node.getRuleCollection() != null)
+ numRules += node.getRuleCollection().getRules().size();
+
+ if (node.getExtensions() != null)
+ for (Trie child : node.getExtensions())
+ numRules += getNumRules(child);
+ }
+
+ return numRules;
+ }
+
+ /**
+ * What is the algorithm?
+ *
+ * Take the first word of the sentence, and start at the root of the trie. There are two things to
+ * consider: (a) word matches and (b) nonterminal matches.
+ *
+ * For a word match, simply follow that arc along the trie. We create a parallel arc in our
+ * filtered grammar to represent it. Each arc in the filtered trie knows about its
+ * corresponding/underlying node in the unfiltered grammar trie.
+ *
+ * A nonterminal is always permitted to match. The question then is how much of the input sentence
+ * we imagine it consumed. The answer is that it could have been any amount. So the recursive call
+ * has to be a set of calls, one each to the next trie node with different lengths of the sentence
+ * remaining.
+ *
+ * A problem occurs when we have multiple sequential nonterminals. For scope-3 grammars, there can
+ * be four sequential nonterminals (in the case when they are grounded by terminals on both ends
+ * of the nonterminal chain). We'd like to avoid looking at all possible ways to split up the
+ * subsequence, because with respect to filtering rules, they are all the same.
+ *
+ * We accomplish this with the following restriction: for purposes of grammar filtering, only the
+ * first in a sequence of nonterminal traversals can consume more than one word. Each of the
+ * subsequent ones would have to consume just one word. We then just have to record in the
+ * recursive call whether the last traversal was a nonterminal or not.
+ *
+ * @param unfilteredTrieRoot todo
+ * @return the root of the filtered trie
+ */
+ private SentenceFilteredTrie filter(Trie unfilteredTrieRoot) {
+ SentenceFilteredTrie filteredTrieRoot = new SentenceFilteredTrie(unfilteredTrieRoot);
+
+ // System.err.println(String.format("FILTERING TO SENTENCE\n %s\n",
+ // Vocabulary.getWords(tokens)));
+
+ /*
+ * The root of the trie is where rule applications start, so we simply try all possible
+ * positions in the sentence.
+ */
+ for (int i = 0; i < tokens.length; i++) {
+ filter(i, filteredTrieRoot, false);
+ }
+
+ return filteredTrieRoot;
+ }
+
+ /**
+ * Matches rules against the sentence. Intelligently handles chains of sequential nonterminals.
+ * Marks arcs that are traversable for this sentence.
+ *
+ * @param i the position in the sentence to start matching
+ * @param trie the trie node to match against
+ * @param lastWasNT true if the match that brought us here was against a nonterminal
+ */
+ private void filter(int i, SentenceFilteredTrie trieNode, boolean lastWasNT) {
+ if (i >= tokens.length)
+ return;
+
+ /* Make sure the underlying unfiltered node has children. */
+ Trie unfilteredTrieNode = trieNode.unfilteredTrieNode;
+ if (unfilteredTrieNode.getChildren() == null) {
+ // trieNode.path.retreat();
+ return;
+ }
+
+ /* Match a word */
+ Trie trie = unfilteredTrieNode.match(tokens[i]);
+ if (trie != null) {
+ /*
+ * The current filtered node might already have an arc for this label. If so, retrieve it
+ * (since we still need to follow it); if not, create it.
+ */
+ SentenceFilteredTrie nextFilteredTrie = trieNode.match(tokens[i]);
+ if (nextFilteredTrie == null) {
+ nextFilteredTrie = new SentenceFilteredTrie(trie);
+ trieNode.children.put(tokens[i], nextFilteredTrie);
+ }
+
+ /*
+ * Now continue, trying to match the child node against the next position in the sentence. The
+ * third argument records that this match was not against a nonterminal.
+ */
+ filter(i + 1, nextFilteredTrie, false);
+ }
+
+ /*
+ * Now we attempt to match nonterminals. Any nonterminal is permitted to match any region of the
+ * sentence, up to the maximum span for that grammar. So we enumerate all children of the
+ * current (unfiltered) trie grammar node, looking for nonterminals (items whose label value is
+ * less than 0), then recurse.
+ *
+ * There is one subtlely. Adjacent nonterminals in a grammar rule can match a span (i, j) in (j
+ * - i - 1) ways, but for purposes of determining whether a rule fits, this is all wasted
+ * effort. To handle this, we allow the first nonterminal in a sequence to record 1, 2, 3, ...
+ * terminals (up to the grammar's span limit, or the rest of the sentence, whichever is
+ * shorter). Subsequent adjacent nonterminals are permitted to consume only a single terminal.
+ */
+ HashMap<Integer, ? extends Trie> children = unfilteredTrieNode.getChildren();
+ if (children != null) {
+ for (int label : children.keySet()) {
+ if (label < 0) {
+ SentenceFilteredTrie nextFilteredTrie = trieNode.match(label);
+ if (nextFilteredTrie == null) {
+ nextFilteredTrie = new SentenceFilteredTrie(unfilteredTrieNode.match(label));
+ trieNode.children.put(label, nextFilteredTrie);
+ }
+
+ /*
+ * Recurse. If the last match was a nonterminal, we can only consume one more token.
+ *
+ * TODO: This goes too far by looking at the whole sentence; each grammar has a maximum
+ * span limit which should be consulted. What we should be doing is passing the point
+ * where we started matching the current sentence, so we can apply this span limit, which
+ * is easily accessible (baseGrammar.spanLimit).
+ */
+ int maxJ = lastWasNT ? (i + 1) : tokens.length;
+ for (int j = i + 1; j <= maxJ; j++) {
+ filter(j, nextFilteredTrie, true);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Alternate filter that uses regular expressions, walking the grammar trie and matching the
+ * source side of each rule collection against the input sentence. Failed matches are discarded,
+ * and trie nodes extending from that position need not be explored.
+ *
+ * @param unfilteredTrie todo
+ * @return the root of the filtered trie if any rules were retained, otherwise null
+ */
+ @SuppressWarnings("unused")
+ private SentenceFilteredTrie filter_regexp(Trie unfilteredTrie) {
+ SentenceFilteredTrie trie = null;
+
+ /* Case 1: keep the trie node if it has a rule collection that matches the sentence */
+ if (unfilteredTrie.hasRules())
+ if (matchesSentence(unfilteredTrie))
+ trie = new SentenceFilteredTrie(unfilteredTrie);
+ else
+ return null;
+
+ /* Case 2: keep the trie node if it has children who have valid rule collections */
+ if (unfilteredTrie.hasExtensions())
+ for (Entry<Integer, ? extends Trie> arc : unfilteredTrie.getChildren().entrySet()) {
+ Trie unfilteredChildTrie = arc.getValue();
+ SentenceFilteredTrie nextTrie = filter_regexp(unfilteredChildTrie);
+ if (nextTrie != null) {
+ if (trie == null)
+ trie = new SentenceFilteredTrie(unfilteredTrie);
+ trie.children.put(arc.getKey(), nextTrie);
+ }
+ }
+
+ return trie;
+ }
+
+ private boolean matchesSentence(Trie childTrie) {
+ Rule rule = childTrie.getRuleCollection().getRules().get(0);
+ return rule.matches(sentence);
+ }
+
+ /**
+ * Implements a filtered trie, by sitting on top of a base trie and annotating nodes that match
+ * the given input sentence.
+ *
+ * @author Matt Post post@cs.jhu.edu
+ *
+ */
+ public class SentenceFilteredTrie implements Trie {
+
+ /* The underlying unfiltered trie node. */
+ private Trie unfilteredTrieNode;
+
+ /* The child nodes in the filtered trie. */
+ private HashMap<Integer, SentenceFilteredTrie> children = null;
+
+ /**
+ * Constructor.
+ *
+ * @param unfilteredTrieNode todo
+ */
+ public SentenceFilteredTrie(Trie unfilteredTrieNode) {
+ this.unfilteredTrieNode = unfilteredTrieNode;
+ this.children = new HashMap<Integer, SentenceFilteredTrie>();
+ }
+
+ @Override
+ public SentenceFilteredTrie match(int wordID) {
+ if (children != null)
+ return children.get(wordID);
+ return null;
+ }
+
+ @Override
+ public boolean hasExtensions() {
+ return children != null;
+ }
+
+ @Override
+ public Collection<SentenceFilteredTrie> getExtensions() {
+ if (children != null)
+ return children.values();
+
+ return null;
+ }
+
+ @Override
+ public HashMap<Integer, SentenceFilteredTrie> getChildren() {
+ return children;
+ }
+
+ @Override
+ public boolean hasRules() {
+ // Chain to the underlying unfiltered node.
+ return unfilteredTrieNode.hasRules();
+ }
+
+ @Override
+ public RuleCollection getRuleCollection() {
+ // Chain to the underlying unfiltered node, since the rule collection just varies by target
+ // side.
+ return unfilteredTrieNode.getRuleCollection();
+ }
+
+ /**
+ * Counts the number of rules.
+ *
+ * @return the number of rules rooted at this node.
+ */
+ public int getNumRules() {
+ int numRules = 0;
+ if (getTrieRoot() != null)
+ if (getTrieRoot().getRuleCollection() != null)
+ numRules += getTrieRoot().getRuleCollection().getRules().size();
+
+ for (SentenceFilteredTrie node : getExtensions())
+ numRules += node.getNumRules();
+
+ return numRules;
+ }
+
+ @Override
+ public Iterator<Integer> getTerminalExtensionIterator() {
+ return new ExtensionIterator(children, true);
+ }
+
+ @Override
+ public Iterator<Integer> getNonterminalExtensionIterator() {
+ return new ExtensionIterator(children, false);
+ }
+ }
+ }
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/1bb8a203/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/hash_based/MemoryBasedBatchGrammar.java
----------------------------------------------------------------------
diff --cc joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/hash_based/MemoryBasedBatchGrammar.java
index 0000000,f346e7a..365102b
mode 000000,100644..100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/hash_based/MemoryBasedBatchGrammar.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/tm/hash_based/MemoryBasedBatchGrammar.java
@@@ -1,0 -1,292 +1,279 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+ package org.apache.joshua.decoder.ff.tm.hash_based;
+
+ import java.io.IOException;
+ import java.util.ArrayList;
+ import java.util.HashMap;
+ import java.util.List;
+
+ import org.apache.joshua.corpus.Vocabulary;
+ import org.apache.joshua.decoder.JoshuaConfiguration;
+ import org.apache.joshua.decoder.JoshuaConfiguration.OOVItem;
+ import org.apache.joshua.decoder.ff.FeatureFunction;
+ import org.apache.joshua.decoder.ff.tm.AbstractGrammar;
++import org.apache.joshua.decoder.ff.tm.OwnerMap;
+ import org.apache.joshua.decoder.ff.tm.Rule;
+ import org.apache.joshua.decoder.ff.tm.GrammarReader;
+ import org.apache.joshua.decoder.ff.tm.Trie;
+ import org.apache.joshua.decoder.ff.tm.format.HieroFormatReader;
+ import org.apache.joshua.decoder.ff.tm.format.MosesFormatReader;
+ import org.apache.joshua.util.FormatUtils;
+ import org.slf4j.Logger;
+ import org.slf4j.LoggerFactory;
+
+ /**
+ * This class implements a memory-based bilingual BatchGrammar.
+ * <p>
+ * The rules are stored in a trie. Each trie node has: (1) RuleBin: a list of rules matching the
+ * french sides so far (2) A HashMap of next-layer trie nodes, the next french word used as the key
+ * in HashMap
+ *
+ * @author Zhifei Li zhifei.work@gmail.com
+ * @author Matt Post post@cs.jhu.edu
+ */
+ public class MemoryBasedBatchGrammar extends AbstractGrammar {
+
+ private static final Logger LOG = LoggerFactory.getLogger(MemoryBasedBatchGrammar.class);
+
+ // ===============================================================
+ // Instance Fields
+ // ===============================================================
+
+ /* The number of rules read. */
+ private int qtyRulesRead = 0;
+
+ /* The number of distinct source sides. */
+ private int qtyRuleBins = 0;
+
+ private int numDenseFeatures = 0;
+
+ /* The trie root. */
- private MemoryBasedTrie root = null;
++ private MemoryBasedTrie root = new MemoryBasedTrie();
+
+ /* The file containing the grammar. */
+ private String grammarFile;
+
+ private GrammarReader<Rule> modelReader;
+
+ // ===============================================================
+ // Static Fields
+ // ===============================================================
+
+ // ===============================================================
+ // Constructors
+ // ===============================================================
+
- public MemoryBasedBatchGrammar(JoshuaConfiguration joshuaConfiguration) {
- super(joshuaConfiguration);
- this.root = new MemoryBasedTrie();
- this.joshuaConfiguration = joshuaConfiguration;
- setSpanLimit(20);
- }
-
- public MemoryBasedBatchGrammar(String owner, JoshuaConfiguration joshuaConfiguration) {
- this(joshuaConfiguration);
- this.owner = Vocabulary.id(owner);
++ /**
++ * Constructor used by Decoder mostly. Default spanLimit of 20
++ */
++ public MemoryBasedBatchGrammar(String owner, JoshuaConfiguration config, int spanLimit) {
++ super(owner, config, spanLimit);
+ }
+
- public MemoryBasedBatchGrammar(GrammarReader<Rule> gr, JoshuaConfiguration joshuaConfiguration) {
- // this.defaultOwner = Vocabulary.id(defaultOwner);
- // this.defaultLHS = Vocabulary.id(defaultLHSSymbol);
- this(joshuaConfiguration);
- modelReader = gr;
++ /**
++ * Constructor to initialize a GrammarReader (unowned)
++ */
++ public MemoryBasedBatchGrammar(
++ final GrammarReader<Rule> reader, final JoshuaConfiguration config, final int spanLimit) {
++ super(OwnerMap.UNKNOWN_OWNER, config, spanLimit);
++ modelReader = reader;
+ }
+
+ public MemoryBasedBatchGrammar(String formatKeyword, String grammarFile, String owner,
+ String defaultLHSSymbol, int spanLimit, JoshuaConfiguration joshuaConfiguration)
+ throws IOException {
+
- this(joshuaConfiguration);
- this.owner = Vocabulary.id(owner);
++ super(owner, joshuaConfiguration, spanLimit);
+ Vocabulary.id(defaultLHSSymbol);
- this.spanLimit = spanLimit;
+ this.grammarFile = grammarFile;
+
+ // ==== loading grammar
+ this.modelReader = createReader(formatKeyword, grammarFile);
+ if (modelReader != null) {
+ for (Rule rule : modelReader)
+ if (rule != null) {
+ addRule(rule);
+ }
+ } else {
+ LOG.info("Couldn't create a GrammarReader for file {} with format {}",
+ grammarFile, formatKeyword);
+ }
+
+ this.printGrammar();
+ }
+
+ protected GrammarReader<Rule> createReader(String format, String grammarFile) throws IOException {
+
+ if (grammarFile != null) {
+ if ("hiero".equals(format) || "thrax".equals(format)) {
+ return new HieroFormatReader(grammarFile);
+ } else if ("moses".equals(format)) {
+ return new MosesFormatReader(grammarFile);
+ } else {
+ throw new RuntimeException(String.format("* FATAL: unknown grammar format '%s'", format));
+ }
+ }
+ return null;
+ }
+
+ // ===============================================================
+ // Methods
+ // ===============================================================
+
- public void setSpanLimit(int spanLimit) {
- this.spanLimit = spanLimit;
- }
-
+ @Override
+ public int getNumRules() {
+ return this.qtyRulesRead;
+ }
+
+ /**
+ * if the span covered by the chart bin is greater than the limit, then return false
+ */
+ public boolean hasRuleForSpan(int i, int j, int pathLength) {
+ if (this.spanLimit == -1) { // mono-glue grammar
+ return (i == 0);
+ } else {
+ // System.err.println(String.format("%s HASRULEFORSPAN(%d,%d,%d)/%d = %s",
+ // Vocabulary.word(this.owner), i, j, pathLength, spanLimit, pathLength <= this.spanLimit));
+ return (pathLength <= this.spanLimit);
+ }
+ }
+
+ public Trie getTrieRoot() {
+ return this.root;
+ }
+
+ /**
+ * Adds a rule to the grammar.
+ */
+ public void addRule(Rule rule) {
+
- // TODO: Why two increments?
+ this.qtyRulesRead++;
+
- // if (owner == -1) {
- // System.err.println("* FATAL: MemoryBasedBatchGrammar::addRule(): owner not set for grammar");
- // System.exit(1);
- // }
+ rule.setOwner(owner);
+
+ if (numDenseFeatures == 0)
+ numDenseFeatures = rule.getFeatureVector().getDenseFeatures().size();
+
+ // === identify the position, and insert the trie nodes as necessary
+ MemoryBasedTrie pos = root;
+ int[] french = rule.getFrench();
+
+ maxSourcePhraseLength = Math.max(maxSourcePhraseLength, french.length);
+
+ for (int k = 0; k < french.length; k++) {
+ int curSymID = french[k];
+
+ /*
+ * Note that the nonTerminal symbol in the french is not cleaned (i.e., will be sth like
+ * [X,1]), but the symbol in the Trie has to be cleaned, so that the match does not care about
+ * the markup (i.e., [X,1] or [X,2] means the same thing, that is X) if
+ * (Vocabulary.nt(french[k])) { curSymID = modelReader.cleanNonTerminal(french[k]); if
+ * (logger.isLoggable(Level.FINEST)) logger.finest("Amended to: " + curSymID); }
+ */
+
+ MemoryBasedTrie nextLayer = (MemoryBasedTrie) pos.match(curSymID);
+ if (null == nextLayer) {
+ nextLayer = new MemoryBasedTrie();
+ if (pos.hasExtensions() == false) {
+ pos.childrenTbl = new HashMap<Integer, MemoryBasedTrie>();
+ }
+ pos.childrenTbl.put(curSymID, nextLayer);
+ }
+ pos = nextLayer;
+ }
+
+ // === add the rule into the trie node
+ if (!pos.hasRules()) {
+ pos.ruleBin = new MemoryBasedRuleBin(rule.getArity(), rule.getFrench());
+ this.qtyRuleBins++;
+ }
+ pos.ruleBin.addRule(rule);
+ }
+
+ protected void printGrammar() {
+ LOG.info("MemoryBasedBatchGrammar: Read {} rules with {} distinct source sides from '{}'",
+ this.qtyRulesRead, this.qtyRuleBins, grammarFile);
+ }
+
+ /***
+ * Takes an input word and creates an OOV rule in the current grammar for that word.
+ *
+ * @param sourceWord integer representation of word
+ * @param featureFunctions {@link java.util.List} of {@link org.apache.joshua.decoder.ff.FeatureFunction}'s
+ */
+ @Override
+ public void addOOVRules(int sourceWord, List<FeatureFunction> featureFunctions) {
+
+ // TODO: _OOV shouldn't be outright added, since the word might not be OOV for the LM (but now
+ // almost
+ // certainly is)
+ final int targetWord = this.joshuaConfiguration.mark_oovs ? Vocabulary.id(Vocabulary
+ .word(sourceWord) + "_OOV") : sourceWord;
+
+ int[] sourceWords = { sourceWord };
+ int[] targetWords = { targetWord };
+ final String oovAlignment = "0-0";
+
+ if (this.joshuaConfiguration.oovList != null && this.joshuaConfiguration.oovList.size() != 0) {
+ for (OOVItem item : this.joshuaConfiguration.oovList) {
+ Rule oovRule = new Rule(Vocabulary.id(item.label), sourceWords, targetWords, "", 0,
+ oovAlignment);
+ addRule(oovRule);
+ oovRule.estimateRuleCost(featureFunctions);
+ }
+ } else {
+ int nt_i = Vocabulary.id(this.joshuaConfiguration.default_non_terminal);
+ Rule oovRule = new Rule(nt_i, sourceWords, targetWords, "", 0, oovAlignment);
+ addRule(oovRule);
+ oovRule.estimateRuleCost(featureFunctions);
+ }
+ }
+
+ /**
+ * Adds a default set of glue rules.
+ *
+ * @param featureFunctions an {@link java.util.ArrayList} of {@link org.apache.joshua.decoder.ff.FeatureFunction}'s
+ */
+ public void addGlueRules(ArrayList<FeatureFunction> featureFunctions) {
+ HieroFormatReader reader = new HieroFormatReader();
+
+ String goalNT = FormatUtils.cleanNonTerminal(joshuaConfiguration.goal_symbol);
+ String defaultNT = FormatUtils.cleanNonTerminal(joshuaConfiguration.default_non_terminal);
+
+ String[] ruleStrings = new String[] {
+ String.format("[%s] ||| %s ||| %s ||| 0", goalNT, Vocabulary.START_SYM,
+ Vocabulary.START_SYM),
+ String.format("[%s] ||| [%s,1] [%s,2] ||| [%s,1] [%s,2] ||| -1", goalNT, goalNT, defaultNT,
+ goalNT, defaultNT),
+ String.format("[%s] ||| [%s,1] %s ||| [%s,1] %s ||| 0", goalNT, goalNT,
+ Vocabulary.STOP_SYM, goalNT, Vocabulary.STOP_SYM) };
+
+ for (String ruleString : ruleStrings) {
+ Rule rule = reader.parseLine(ruleString);
+ addRule(rule);
+ rule.estimateRuleCost(featureFunctions);
+ }
+ }
+
+ @Override
+ public int getNumDenseFeatures() {
+ return numDenseFeatures;
+ }
+ }
[49/60] [partial] incubator-joshua git commit: maven multi-module
layout 1st commit: moving files into joshua-core
Posted by mj...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/e2734396/joshua-core/resources/kbest_extraction/output.gold
----------------------------------------------------------------------
diff --git a/joshua-core/resources/kbest_extraction/output.gold b/joshua-core/resources/kbest_extraction/output.gold
new file mode 100644
index 0000000..e75bb9e
--- /dev/null
+++ b/joshua-core/resources/kbest_extraction/output.gold
@@ -0,0 +1,3126 @@
+0 ||| A A A A A ||| lm_0=-28.045 tm_pt_0=-172.000 tm_glue_0=5.000 ||| -195.045
+0 ||| B A A A A ||| lm_0=-28.045 tm_pt_0=-173.000 tm_glue_0=5.000 ||| -196.045
+0 ||| C A A A A ||| lm_0=-28.045 tm_pt_0=-175.000 tm_glue_0=5.000 ||| -198.045
+0 ||| A B A A A ||| lm_0=-28.045 tm_pt_0=-176.000 tm_glue_0=5.000 ||| -199.045
+0 ||| B B A A A ||| lm_0=-28.045 tm_pt_0=-177.000 tm_glue_0=5.000 ||| -200.045
+0 ||| D A A A A ||| lm_0=-28.045 tm_pt_0=-177.000 tm_glue_0=5.000 ||| -200.045
+0 ||| A A A A B ||| lm_0=-28.045 tm_pt_0=-178.000 tm_glue_0=5.000 ||| -201.045
+0 ||| A A A B A ||| lm_0=-28.045 tm_pt_0=-178.000 tm_glue_0=5.000 ||| -201.045
+0 ||| A A B A A ||| lm_0=-28.045 tm_pt_0=-178.000 tm_glue_0=5.000 ||| -201.045
+0 ||| A C A A A ||| lm_0=-28.045 tm_pt_0=-178.000 tm_glue_0=5.000 ||| -201.045
+0 ||| B A A A B ||| lm_0=-28.045 tm_pt_0=-179.000 tm_glue_0=5.000 ||| -202.045
+0 ||| B A A B A ||| lm_0=-28.045 tm_pt_0=-179.000 tm_glue_0=5.000 ||| -202.045
+0 ||| B A B A A ||| lm_0=-28.045 tm_pt_0=-179.000 tm_glue_0=5.000 ||| -202.045
+0 ||| B C A A A ||| lm_0=-28.045 tm_pt_0=-179.000 tm_glue_0=5.000 ||| -202.045
+0 ||| C B A A A ||| lm_0=-28.045 tm_pt_0=-179.000 tm_glue_0=5.000 ||| -202.045
+0 ||| A A A C A ||| lm_0=-28.045 tm_pt_0=-180.000 tm_glue_0=5.000 ||| -203.045
+0 ||| C A A A B ||| lm_0=-28.045 tm_pt_0=-181.000 tm_glue_0=5.000 ||| -204.045
+0 ||| C A A B A ||| lm_0=-28.045 tm_pt_0=-181.000 tm_glue_0=5.000 ||| -204.045
+0 ||| B A A C A ||| lm_0=-28.045 tm_pt_0=-181.000 tm_glue_0=5.000 ||| -204.045
+0 ||| C A B A A ||| lm_0=-28.045 tm_pt_0=-181.000 tm_glue_0=5.000 ||| -204.045
+0 ||| E A A A A ||| lm_0=-28.045 tm_pt_0=-181.000 tm_glue_0=5.000 ||| -204.045
+0 ||| C C A A A ||| lm_0=-28.045 tm_pt_0=-181.000 tm_glue_0=5.000 ||| -204.045
+0 ||| D B A A A ||| lm_0=-28.045 tm_pt_0=-181.000 tm_glue_0=5.000 ||| -204.045
+0 ||| A A A A C ||| lm_0=-28.045 tm_pt_0=-182.000 tm_glue_0=5.000 ||| -205.045
+0 ||| A B A B A ||| lm_0=-28.045 tm_pt_0=-182.000 tm_glue_0=5.000 ||| -205.045
+0 ||| A B A A B ||| lm_0=-28.045 tm_pt_0=-182.000 tm_glue_0=5.000 ||| -205.045
+0 ||| A A C A A ||| lm_0=-28.045 tm_pt_0=-182.000 tm_glue_0=5.000 ||| -205.045
+0 ||| A B B A A ||| lm_0=-28.045 tm_pt_0=-182.000 tm_glue_0=5.000 ||| -205.045
+0 ||| A D A A A ||| lm_0=-28.045 tm_pt_0=-182.000 tm_glue_0=5.000 ||| -205.045
+0 ||| B A A A C ||| lm_0=-28.045 tm_pt_0=-183.000 tm_glue_0=5.000 ||| -206.045
+0 ||| B B A A B ||| lm_0=-28.045 tm_pt_0=-183.000 tm_glue_0=5.000 ||| -206.045
+0 ||| B B A B A ||| lm_0=-28.045 tm_pt_0=-183.000 tm_glue_0=5.000 ||| -206.045
+0 ||| D A A A B ||| lm_0=-28.045 tm_pt_0=-183.000 tm_glue_0=5.000 ||| -206.045
+0 ||| B A C A A ||| lm_0=-28.045 tm_pt_0=-183.000 tm_glue_0=5.000 ||| -206.045
+0 ||| D A A B A ||| lm_0=-28.045 tm_pt_0=-183.000 tm_glue_0=5.000 ||| -206.045
+0 ||| B B B A A ||| lm_0=-28.045 tm_pt_0=-183.000 tm_glue_0=5.000 ||| -206.045
+0 ||| C A A C A ||| lm_0=-28.045 tm_pt_0=-183.000 tm_glue_0=5.000 ||| -206.045
+0 ||| D A B A A ||| lm_0=-28.045 tm_pt_0=-183.000 tm_glue_0=5.000 ||| -206.045
+0 ||| B D A A A ||| lm_0=-28.045 tm_pt_0=-183.000 tm_glue_0=5.000 ||| -206.045
+0 ||| D C A A A ||| lm_0=-28.045 tm_pt_0=-183.000 tm_glue_0=5.000 ||| -206.045
+0 ||| A A B A B ||| lm_0=-28.045 tm_pt_0=-184.000 tm_glue_0=5.000 ||| -207.045
+0 ||| A A B B A ||| lm_0=-28.045 tm_pt_0=-184.000 tm_glue_0=5.000 ||| -207.045
+0 ||| A A A B B ||| lm_0=-28.045 tm_pt_0=-184.000 tm_glue_0=5.000 ||| -207.045
+0 ||| A B A C A ||| lm_0=-28.045 tm_pt_0=-184.000 tm_glue_0=5.000 ||| -207.045
+0 ||| A C A A B ||| lm_0=-28.045 tm_pt_0=-184.000 tm_glue_0=5.000 ||| -207.045
+0 ||| A C A B A ||| lm_0=-28.045 tm_pt_0=-184.000 tm_glue_0=5.000 ||| -207.045
+0 ||| A C B A A ||| lm_0=-28.045 tm_pt_0=-184.000 tm_glue_0=5.000 ||| -207.045
+0 ||| A A D A A ||| lm_0=-28.045 tm_pt_0=-184.000 tm_glue_0=5.000 ||| -207.045
+0 ||| C A A A C ||| lm_0=-28.045 tm_pt_0=-185.000 tm_glue_0=5.000 ||| -208.045
+0 ||| B A A B B ||| lm_0=-28.045 tm_pt_0=-185.000 tm_glue_0=5.000 ||| -208.045
+0 ||| C B A B A ||| lm_0=-28.045 tm_pt_0=-185.000 tm_glue_0=5.000 ||| -208.045
+0 ||| B A B A B ||| lm_0=-28.045 tm_pt_0=-185.000 tm_glue_0=5.000 ||| -208.045
+0 ||| B C A B A ||| lm_0=-28.045 tm_pt_0=-185.000 tm_glue_0=5.000 ||| -208.045
+0 ||| B C A A B ||| lm_0=-28.045 tm_pt_0=-185.000 tm_glue_0=5.000 ||| -208.045
+0 ||| B A B B A ||| lm_0=-28.045 tm_pt_0=-185.000 tm_glue_0=5.000 ||| -208.045
+0 ||| C B A A B ||| lm_0=-28.045 tm_pt_0=-185.000 tm_glue_0=5.000 ||| -208.045
+0 ||| D A A C A ||| lm_0=-28.045 tm_pt_0=-185.000 tm_glue_0=5.000 ||| -208.045
+0 ||| C A C A A ||| lm_0=-28.045 tm_pt_0=-185.000 tm_glue_0=5.000 ||| -208.045
+0 ||| B B A C A ||| lm_0=-28.045 tm_pt_0=-185.000 tm_glue_0=5.000 ||| -208.045
+0 ||| E B A A A ||| lm_0=-28.045 tm_pt_0=-185.000 tm_glue_0=5.000 ||| -208.045
+0 ||| B C B A A ||| lm_0=-28.045 tm_pt_0=-185.000 tm_glue_0=5.000 ||| -208.045
+0 ||| C B B A A ||| lm_0=-28.045 tm_pt_0=-185.000 tm_glue_0=5.000 ||| -208.045
+0 ||| C D A A A ||| lm_0=-28.045 tm_pt_0=-185.000 tm_glue_0=5.000 ||| -208.045
+0 ||| B A D A A ||| lm_0=-28.045 tm_pt_0=-185.000 tm_glue_0=5.000 ||| -208.045
+0 ||| A B A A C ||| lm_0=-28.045 tm_pt_0=-186.000 tm_glue_0=5.000 ||| -209.045
+0 ||| A A A C B ||| lm_0=-28.045 tm_pt_0=-186.000 tm_glue_0=5.000 ||| -209.045
+0 ||| A A B C A ||| lm_0=-28.045 tm_pt_0=-186.000 tm_glue_0=5.000 ||| -209.045
+0 ||| A B C A A ||| lm_0=-28.045 tm_pt_0=-186.000 tm_glue_0=5.000 ||| -209.045
+0 ||| A C A C A ||| lm_0=-28.045 tm_pt_0=-186.000 tm_glue_0=5.000 ||| -209.045
+0 ||| A A A D A ||| lm_0=-28.045 tm_pt_0=-186.000 tm_glue_0=5.000 ||| -209.045
+0 ||| C A A B B ||| lm_0=-28.045 tm_pt_0=-187.000 tm_glue_0=5.000 ||| -210.045
+0 ||| D A A A C ||| lm_0=-28.045 tm_pt_0=-187.000 tm_glue_0=5.000 ||| -210.045
+0 ||| C A B A B ||| lm_0=-28.045 tm_pt_0=-187.000 tm_glue_0=5.000 ||| -210.045
+0 ||| B A A C B ||| lm_0=-28.045 tm_pt_0=-187.000 tm_glue_0=5.000 ||| -210.045
+0 ||| C C A A B ||| lm_0=-28.045 tm_pt_0=-187.000 tm_glue_0=5.000 ||| -210.045
+0 ||| D B A A B ||| lm_0=-28.045 tm_pt_0=-187.000 tm_glue_0=5.000 ||| -210.045
+0 ||| B B A A C ||| lm_0=-28.045 tm_pt_0=-187.000 tm_glue_0=5.000 ||| -210.045
+0 ||| E A A A B ||| lm_0=-28.045 tm_pt_0=-187.000 tm_glue_0=5.000 ||| -210.045
+0 ||| D B A B A ||| lm_0=-28.045 tm_pt_0=-187.000 tm_glue_0=5.000 ||| -210.045
+0 ||| D A C A A ||| lm_0=-28.045 tm_pt_0=-187.000 tm_glue_0=5.000 ||| -210.045
+0 ||| E A A B A ||| lm_0=-28.045 tm_pt_0=-187.000 tm_glue_0=5.000 ||| -210.045
+0 ||| B A A D A ||| lm_0=-28.045 tm_pt_0=-187.000 tm_glue_0=5.000 ||| -210.045
+0 ||| D D A A A ||| lm_0=-28.045 tm_pt_0=-187.000 tm_glue_0=5.000 ||| -210.045
+0 ||| B C A C A ||| lm_0=-28.045 tm_pt_0=-187.000 tm_glue_0=5.000 ||| -210.045
+0 ||| C A D A A ||| lm_0=-28.045 tm_pt_0=-187.000 tm_glue_0=5.000 ||| -210.045
+0 ||| C A B B A ||| lm_0=-28.045 tm_pt_0=-187.000 tm_glue_0=5.000 ||| -210.045
+0 ||| E A B A A ||| lm_0=-28.045 tm_pt_0=-187.000 tm_glue_0=5.000 ||| -210.045
+0 ||| C B A C A ||| lm_0=-28.045 tm_pt_0=-187.000 tm_glue_0=5.000 ||| -210.045
+0 ||| E C A A A ||| lm_0=-28.045 tm_pt_0=-187.000 tm_glue_0=5.000 ||| -210.045
+0 ||| C C A B A ||| lm_0=-28.045 tm_pt_0=-187.000 tm_glue_0=5.000 ||| -210.045
+0 ||| B B C A A ||| lm_0=-28.045 tm_pt_0=-187.000 tm_glue_0=5.000 ||| -210.045
+0 ||| B A B C A ||| lm_0=-28.045 tm_pt_0=-187.000 tm_glue_0=5.000 ||| -210.045
+0 ||| C C B A A ||| lm_0=-28.045 tm_pt_0=-187.000 tm_glue_0=5.000 ||| -210.045
+0 ||| D B B A A ||| lm_0=-28.045 tm_pt_0=-187.000 tm_glue_0=5.000 ||| -210.045
+0 ||| A B A B B ||| lm_0=-28.045 tm_pt_0=-188.000 tm_glue_0=5.000 ||| -211.045
+0 ||| A B B B A ||| lm_0=-28.045 tm_pt_0=-188.000 tm_glue_0=5.000 ||| -211.045
+0 ||| A A A B C ||| lm_0=-28.045 tm_pt_0=-188.000 tm_glue_0=5.000 ||| -211.045
+0 ||| A D A B A ||| lm_0=-28.045 tm_pt_0=-188.000 tm_glue_0=5.000 ||| -211.045
+0 ||| A C A A C ||| lm_0=-28.045 tm_pt_0=-188.000 tm_glue_0=5.000 ||| -211.045
+0 ||| A A C B A ||| lm_0=-28.045 tm_pt_0=-188.000 tm_glue_0=5.000 ||| -211.045
+0 ||| A A C A B ||| lm_0=-28.045 tm_pt_0=-188.000 tm_glue_0=5.000 ||| -211.045
+0 ||| A C C A A ||| lm_0=-28.045 tm_pt_0=-188.000 tm_glue_0=5.000 ||| -211.045
+0 ||| A A B A C ||| lm_0=-28.045 tm_pt_0=-188.000 tm_glue_0=5.000 ||| -211.045
+0 ||| A B D A A ||| lm_0=-28.045 tm_pt_0=-188.000 tm_glue_0=5.000 ||| -211.045
+0 ||| A A A A D ||| lm_0=-28.045 tm_pt_0=-188.000 tm_glue_0=5.000 ||| -211.045
+0 ||| A A E A A ||| lm_0=-28.045 tm_pt_0=-188.000 tm_glue_0=5.000 ||| -211.045
+0 ||| A B B A B ||| lm_0=-28.045 tm_pt_0=-188.000 tm_glue_0=5.000 ||| -211.045
+0 ||| A E A A A ||| lm_0=-28.045 tm_pt_0=-188.000 tm_glue_0=5.000 ||| -211.045
+0 ||| A D A A B ||| lm_0=-28.045 tm_pt_0=-188.000 tm_glue_0=5.000 ||| -211.045
+0 ||| A D B A A ||| lm_0=-28.045 tm_pt_0=-188.000 tm_glue_0=5.000 ||| -211.045
+0 ||| B D A A B ||| lm_0=-28.045 tm_pt_0=-189.000 tm_glue_0=5.000 ||| -212.045
+0 ||| C B A A C ||| lm_0=-28.045 tm_pt_0=-189.000 tm_glue_0=5.000 ||| -212.045
+0 ||| B B B B A ||| lm_0=-28.045 tm_pt_0=-189.000 tm_glue_0=5.000 ||| -212.045
+0 ||| B A C A B ||| lm_0=-28.045 tm_pt_0=-189.000 tm_glue_0=5.000 ||| -212.045
+0 ||| D A A B B ||| lm_0=-28.045 tm_pt_0=-189.000 tm_glue_0=5.000 ||| -212.045
+0 ||| E A A C A ||| lm_0=-28.045 tm_pt_0=-189.000 tm_glue_0=5.000 ||| -212.045
+0 ||| B B A B B ||| lm_0=-28.045 tm_pt_0=-189.000 tm_glue_0=5.000 ||| -212.045
+0 ||| C A A D A ||| lm_0=-28.045 tm_pt_0=-189.000 tm_glue_0=5.000 ||| -212.045
+0 ||| B A A B C ||| lm_0=-28.045 tm_pt_0=-189.000 tm_glue_0=5.000 ||| -212.045
+0 ||| D C A B A ||| lm_0=-28.045 tm_pt_0=-189.000 tm_glue_0=5.000 ||| -212.045
+0 ||| B C A A C ||| lm_0=-28.045 tm_pt_0=-189.000 tm_glue_0=5.000 ||| -212.045
+0 ||| D B A C A ||| lm_0=-28.045 tm_pt_0=-189.000 tm_glue_0=5.000 ||| -212.045
+0 ||| B A A A D ||| lm_0=-28.045 tm_pt_0=-189.000 tm_glue_0=5.000 ||| -212.045
+0 ||| B A C B A ||| lm_0=-28.045 tm_pt_0=-189.000 tm_glue_0=5.000 ||| -212.045
+0 ||| D C A A B ||| lm_0=-28.045 tm_pt_0=-189.000 tm_glue_0=5.000 ||| -212.045
+0 ||| D A B B A ||| lm_0=-28.045 tm_pt_0=-189.000 tm_glue_0=5.000 ||| -212.045
+0 ||| B B B A B ||| lm_0=-28.045 tm_pt_0=-189.000 tm_glue_0=5.000 ||| -212.045
+0 ||| B D A B A ||| lm_0=-28.045 tm_pt_0=-189.000 tm_glue_0=5.000 ||| -212.045
+0 ||| C A A C B ||| lm_0=-28.045 tm_pt_0=-189.000 tm_glue_0=5.000 ||| -212.045
+0 ||| B A E A A ||| lm_0=-28.045 tm_pt_0=-189.000 tm_glue_0=5.000 ||| -212.045
+0 ||| B A B A C ||| lm_0=-28.045 tm_pt_0=-189.000 tm_glue_0=5.000 ||| -212.045
+0 ||| C C A C A ||| lm_0=-28.045 tm_pt_0=-189.000 tm_glue_0=5.000 ||| -212.045
+0 ||| D A B A B ||| lm_0=-28.045 tm_pt_0=-189.000 tm_glue_0=5.000 ||| -212.045
+0 ||| B C C A A ||| lm_0=-28.045 tm_pt_0=-189.000 tm_glue_0=5.000 ||| -212.045
+0 ||| C A B C A ||| lm_0=-28.045 tm_pt_0=-189.000 tm_glue_0=5.000 ||| -212.045
+0 ||| B E A A A ||| lm_0=-28.045 tm_pt_0=-189.000 tm_glue_0=5.000 ||| -212.045
+0 ||| D A D A A ||| lm_0=-28.045 tm_pt_0=-189.000 tm_glue_0=5.000 ||| -212.045
+0 ||| C B C A A ||| lm_0=-28.045 tm_pt_0=-189.000 tm_glue_0=5.000 ||| -212.045
+0 ||| B B D A A ||| lm_0=-28.045 tm_pt_0=-189.000 tm_glue_0=5.000 ||| -212.045
+0 ||| D C B A A ||| lm_0=-28.045 tm_pt_0=-189.000 tm_glue_0=5.000 ||| -212.045
+0 ||| B D B A A ||| lm_0=-28.045 tm_pt_0=-189.000 tm_glue_0=5.000 ||| -212.045
+0 ||| A C B A B ||| lm_0=-28.045 tm_pt_0=-190.000 tm_glue_0=5.000 ||| -213.045
+0 ||| A A D A B ||| lm_0=-28.045 tm_pt_0=-190.000 tm_glue_0=5.000 ||| -213.045
+0 ||| A A B B B ||| lm_0=-28.045 tm_pt_0=-190.000 tm_glue_0=5.000 ||| -213.045
+0 ||| A A A C C ||| lm_0=-28.045 tm_pt_0=-190.000 tm_glue_0=5.000 ||| -213.045
+0 ||| A B A C B ||| lm_0=-28.045 tm_pt_0=-190.000 tm_glue_0=5.000 ||| -213.045
+0 ||| A C A B B ||| lm_0=-28.045 tm_pt_0=-190.000 tm_glue_0=5.000 ||| -213.045
+0 ||| A B B C A ||| lm_0=-28.045 tm_pt_0=-190.000 tm_glue_0=5.000 ||| -213.045
+0 ||| A A A E A ||| lm_0=-28.045 tm_pt_0=-190.000 tm_glue_0=5.000 ||| -213.045
+0 ||| A C B B A ||| lm_0=-28.045 tm_pt_0=-190.000 tm_glue_0=5.000 ||| -213.045
+0 ||| A A D B A ||| lm_0=-28.045 tm_pt_0=-190.000 tm_glue_0=5.000 ||| -213.045
+0 ||| A B A D A ||| lm_0=-28.045 tm_pt_0=-190.000 tm_glue_0=5.000 ||| -213.045
+0 ||| A D A C A ||| lm_0=-28.045 tm_pt_0=-190.000 tm_glue_0=5.000 ||| -213.045
+0 ||| A A C C A ||| lm_0=-28.045 tm_pt_0=-190.000 tm_glue_0=5.000 ||| -213.045
+0 ||| A C D A A ||| lm_0=-28.045 tm_pt_0=-190.000 tm_glue_0=5.000 ||| -213.045
+0 ||| B C A B B ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| B A B B B ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| B C B A B ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| C B B A B ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| E B A A B ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| C A A B C ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| D A A C B ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| C A C A B ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| B B A C B ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| C A B A C ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| C D A A B ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| B A D A B ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| C B A B B ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| E A A A C ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| D B A A C ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| C A A A D ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| D A B C A ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| C C A A C ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| B A A C C ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| B A D B A ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| B A C C A ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| B B B C A ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| D C A C A ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| D A A D A ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| B B A D A ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| C A C B A ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| B A A E A ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| C C C A A ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| B D A C A ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| E B A B A ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| E B B A A ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| B C B B A ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| C B B B A ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| C A E A A ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| C D A B A ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| C D B A A ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| B C D A A ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| E D A A A ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| D B C A A ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| C E A A A ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| E A C A A ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| C B D A A ||| lm_0=-28.045 tm_pt_0=-191.000 tm_glue_0=5.000 ||| -214.045
+0 ||| A B C A B ||| lm_0=-28.045 tm_pt_0=-192.000 tm_glue_0=5.000 ||| -215.045
+0 ||| A B B A C ||| lm_0=-28.045 tm_pt_0=-192.000 tm_glue_0=5.000 ||| -215.045
+0 ||| A D A A C ||| lm_0=-28.045 tm_pt_0=-192.000 tm_glue_0=5.000 ||| -215.045
+0 ||| A A C A C ||| lm_0=-28.045 tm_pt_0=-192.000 tm_glue_0=5.000 ||| -215.045
+0 ||| A B A B C ||| lm_0=-28.045 tm_pt_0=-192.000 tm_glue_0=5.000 ||| -215.045
+0 ||| A A B C B ||| lm_0=-28.045 tm_pt_0=-192.000 tm_glue_0=5.000 ||| -215.045
+0 ||| A C A C B ||| lm_0=-28.045 tm_pt_0=-192.000 tm_glue_0=5.000 ||| -215.045
+0 ||| A B A A D ||| lm_0=-28.045 tm_pt_0=-192.000 tm_glue_0=5.000 ||| -215.045
+0 ||| A A A D B ||| lm_0=-28.045 tm_pt_0=-192.000 tm_glue_0=5.000 ||| -215.045
+0 ||| A B C B A ||| lm_0=-28.045 tm_pt_0=-192.000 tm_glue_0=5.000 ||| -215.045
+0 ||| A C A D A ||| lm_0=-28.045 tm_pt_0=-192.000 tm_glue_0=5.000 ||| -215.045
+0 ||| A A D C A ||| lm_0=-28.045 tm_pt_0=-192.000 tm_glue_0=5.000 ||| -215.045
+0 ||| A C B C A ||| lm_0=-28.045 tm_pt_0=-192.000 tm_glue_0=5.000 ||| -215.045
+0 ||| A A B D A ||| lm_0=-28.045 tm_pt_0=-192.000 tm_glue_0=5.000 ||| -215.045
+0 ||| A B E A A ||| lm_0=-28.045 tm_pt_0=-192.000 tm_glue_0=5.000 ||| -215.045
+0 ||| A D C A A ||| lm_0=-28.045 tm_pt_0=-192.000 tm_glue_0=5.000 ||| -215.045
+0 ||| B B C A B ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| E C A A B ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| C C A B B ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| B A B C B ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| C C B B A ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| C C B A B ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| D B B A B ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| E A A B B ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| D A C B A ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| B A C A C ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| D A C A B ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| D D A B A ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| D A B A C ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| B D A A C ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| C C D A A ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| D B A B B ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| B A B D A ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| B A A D B ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| C A D B A ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| B B A A D ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| E A B B A ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| D A A A D ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| D C C A A ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| D A A B C ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| E C A B A ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| C A D A B ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| B B C B A ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| B C A C B ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| B D C A A ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| C A B B B ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| C D A C A ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| D D A A B ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| D B B B A ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| C A A C C ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| D D B A A ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| B B A B C ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| B A D C A ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| C B A C B ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| C A A E A ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| E A B A B ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| D B D A A ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| D C A A C ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| B C A D A ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| B B B A C ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| D E A A A ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| C A C C A ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| E A D A A ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| C B B C A ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| E C B A A ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| B C B C A ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| B B E A A ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| C B A D A ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| D A E A A ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| E B A C A ||| lm_0=-28.045 tm_pt_0=-193.000 tm_glue_0=5.000 ||| -216.045
+0 ||| A D B A B ||| lm_0=-28.045 tm_pt_0=-194.000 tm_glue_0=5.000 ||| -217.045
+0 ||| A A C B B ||| lm_0=-28.045 tm_pt_0=-194.000 tm_glue_0=5.000 ||| -217.045
+0 ||| A C A B C ||| lm_0=-28.045 tm_pt_0=-194.000 tm_glue_0=5.000 ||| -217.045
+0 ||| A B B B B ||| lm_0=-28.045 tm_pt_0=-194.000 tm_glue_0=5.000 ||| -217.045
+0 ||| A B A C C ||| lm_0=-28.045 tm_pt_0=-194.000 tm_glue_0=5.000 ||| -217.045
+0 ||| A B D A B ||| lm_0=-28.045 tm_pt_0=-194.000 tm_glue_0=5.000 ||| -217.045
+0 ||| A A E A B ||| lm_0=-28.045 tm_pt_0=-194.000 tm_glue_0=5.000 ||| -217.045
+0 ||| A C C A B ||| lm_0=-28.045 tm_pt_0=-194.000 tm_glue_0=5.000 ||| -217.045
+0 ||| A E A A B ||| lm_0=-28.045 tm_pt_0=-194.000 tm_glue_0=5.000 ||| -217.045
+0 ||| A D A B B ||| lm_0=-28.045 tm_pt_0=-194.000 tm_glue_0=5.000 ||| -217.045
+0 ||| A A B B C ||| lm_0=-28.045 tm_pt_0=-194.000 tm_glue_0=5.000 ||| -217.045
+0 ||| A B D B A ||| lm_0=-28.045 tm_pt_0=-194.000 tm_glue_0=5.000 ||| -217.045
+0 ||| A A D A C ||| lm_0=-28.045 tm_pt_0=-194.000 tm_glue_0=5.000 ||| -217.045
+0 ||| A C B A C ||| lm_0=-28.045 tm_pt_0=-194.000 tm_glue_0=5.000 ||| -217.045
+0 ||| A B C C A ||| lm_0=-28.045 tm_pt_0=-194.000 tm_glue_0=5.000 ||| -217.045
+0 ||| A A A B D ||| lm_0=-28.045 tm_pt_0=-194.000 tm_glue_0=5.000 ||| -217.045
+0 ||| A C C B A ||| lm_0=-28.045 tm_pt_0=-194.000 tm_glue_0=5.000 ||| -217.045
+0 ||| A A B A D ||| lm_0=-28.045 tm_pt_0=-194.000 tm_glue_0=5.000 ||| -217.045
+0 ||| A D D A A ||| lm_0=-28.045 tm_pt_0=-194.000 tm_glue_0=5.000 ||| -217.045
+0 ||| A C A A D ||| lm_0=-28.045 tm_pt_0=-194.000 tm_glue_0=5.000 ||| -217.045
+0 ||| A D B B A ||| lm_0=-28.045 tm_pt_0=-194.000 tm_glue_0=5.000 ||| -217.045
+0 ||| A E A B A ||| lm_0=-28.045 tm_pt_0=-194.000 tm_glue_0=5.000 ||| -217.045
+0 ||| A C E A A ||| lm_0=-28.045 tm_pt_0=-194.000 tm_glue_0=5.000 ||| -217.045
+0 ||| A A E B A ||| lm_0=-28.045 tm_pt_0=-194.000 tm_glue_0=5.000 ||| -217.045
+0 ||| A E B A A ||| lm_0=-28.045 tm_pt_0=-194.000 tm_glue_0=5.000 ||| -217.045
+0 ||| A B A E A ||| lm_0=-28.045 tm_pt_0=-194.000 tm_glue_0=5.000 ||| -217.045
+0 ||| B B B B B ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| E A A C B ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| D C A B B ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| B C A B C ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| B D A B B ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| B E A A B ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| D A B B B ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| B A E A B ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| B A C B B ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| B C B A C ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| C C A C B ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| C A B C B ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| C A A D B ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| D B A C B ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| B C C A B ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| B B C C A ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| B A B B C ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| B B D A B ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| C B C A B ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| C B E A A ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| C B A A D ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| D A D A B ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| D C B A B ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| E A B C A ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| B D B A B ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| E C A C A ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| C B A B C ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| B A E B A ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| D A A C C ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| D A C C A ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| C B B A C ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| C C B C A ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| E B A A C ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| C A B D A ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| B A D A C ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| C A D C A ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| B B A C C ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| C D C A A ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| C A C A C ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| D D A C A ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| C D A A C ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| B B D B A ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| B A B A D ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| E B C A A ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| B C A A D ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| D A D B A ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| B A A B D ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| B E A B A ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| D C D A A ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| D B B C A ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| D B A D A ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| E A A D A ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| B D D A A ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| C B C B A ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| B C E A A ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| B C C B A ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| B E B A A ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| C C A D A ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| B D B B A ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| D A A E A ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| D C B B A ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| B B A E A ||| lm_0=-28.045 tm_pt_0=-195.000 tm_glue_0=5.000 ||| -218.045
+0 ||| A B B C B ||| lm_0=-28.045 tm_pt_0=-196.000 tm_glue_0=5.000 ||| -219.045
+0 ||| A A A C D ||| lm_0=-28.045 tm_pt_0=-196.000 tm_glue_0=5.000 ||| -219.045
+0 ||| A B A D B ||| lm_0=-28.045 tm_pt_0=-196.000 tm_glue_0=5.000 ||| -219.045
+0 ||| A A A D C ||| lm_0=-28.045 tm_pt_0=-196.000 tm_glue_0=5.000 ||| -219.045
+0 ||| A C D A B ||| lm_0=-28.045 tm_pt_0=-196.000 tm_glue_0=5.000 ||| -219.045
+0 ||| A C A C C ||| lm_0=-28.045 tm_pt_0=-196.000 tm_glue_0=5.000 ||| -219.045
+0 ||| A A A E B ||| lm_0=-28.045 tm_pt_0=-196.000 tm_glue_0=5.000 ||| -219.045
+0 ||| A C B B B ||| lm_0=-28.045 tm_pt_0=-196.000 tm_glue_0=5.000 ||| -219.045
+0 ||| A A D B B ||| lm_0=-28.045 tm_pt_0=-196.000 tm_glue_0=5.000 ||| -219.045
+0 ||| A A C C B ||| lm_0=-28.045 tm_pt_0=-196.000 tm_glue_0=5.000 ||| -219.045
+0 ||| A D A C B ||| lm_0=-28.045 tm_pt_0=-196.000 tm_glue_0=5.000 ||| -219.045
+0 ||| A A B E A ||| lm_0=-28.045 tm_pt_0=-196.000 tm_glue_0=5.000 ||| -219.045
+0 ||| A A A A E ||| lm_0=-28.045 tm_pt_0=-196.000 tm_glue_0=5.000 ||| -219.045
+0 ||| A B C A C ||| lm_0=-28.045 tm_pt_0=-196.000 tm_glue_0=5.000 ||| -219.045
+0 ||| A D B C A ||| lm_0=-28.045 tm_pt_0=-196.000 tm_glue_0=5.000 ||| -219.045
+0 ||| A A B C C ||| lm_0=-28.045 tm_pt_0=-196.000 tm_glue_0=5.000 ||| -219.045
+0 ||| A E A C A ||| lm_0=-28.045 tm_pt_0=-196.000 tm_glue_0=5.000 ||| -219.045
+0 ||| A B D C A ||| lm_0=-28.045 tm_pt_0=-196.000 tm_glue_0=5.000 ||| -219.045
+0 ||| A C C C A ||| lm_0=-28.045 tm_pt_0=-196.000 tm_glue_0=5.000 ||| -219.045
+0 ||| A C D B A ||| lm_0=-28.045 tm_pt_0=-196.000 tm_glue_0=5.000 ||| -219.045
+0 ||| A A C D A ||| lm_0=-28.045 tm_pt_0=-196.000 tm_glue_0=5.000 ||| -219.045
+0 ||| A D A D A ||| lm_0=-28.045 tm_pt_0=-196.000 tm_glue_0=5.000 ||| -219.045
+0 ||| A B B D A ||| lm_0=-28.045 tm_pt_0=-196.000 tm_glue_0=5.000 ||| -219.045
+0 ||| A A E C A ||| lm_0=-28.045 tm_pt_0=-196.000 tm_glue_0=5.000 ||| -219.045
+0 ||| A C A E A ||| lm_0=-28.045 tm_pt_0=-196.000 tm_glue_0=5.000 ||| -219.045
+0 ||| E A A B C ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| D C A C B ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| D A C A C ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| B B B C B ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| B A C C B ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| B A D B B ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| B A B C C ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| C C B A C ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| B A A C D ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| D A A D B ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| E B B A B ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| C C A B C ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| B C A C C ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| C A B A D ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| C A A B D ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| C C A A D ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| D B B A C ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| E C A A C ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| B B C A C ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| B A A D C ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| E A A A D ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| B B A D B ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| C A D A C ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| D A B C B ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| D C B C A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| D B A B C ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| D B A A D ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| E B A B B ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| C A C B B ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| B A C D A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| B D A C B ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| C C C A B ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| B A A E B ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| B A E C A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| D B C A B ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| C A E A B ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| B A B E A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| C B B B B ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| B C B B B ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| C D B B A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| D D A A C ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| E D A A B ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| B B D C A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| C D A B B ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| C A B B C ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| B D B C A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| B C D A B ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| C D B A B ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| D A D C A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| E A B A C ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| B A A A E ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| D A B D A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| C E A A B ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| C B A C C ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| B E A C A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| C B D A B ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| E A C A B ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| C B C C A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| E B D A A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| C A E B A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| E D B A A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| C B A E A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| E E A A A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| B C C C A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| D B E A A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| B B B D A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| C D D A A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| D C A D A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| D D C A A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| C C C B A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| C C E A A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| E B B B A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| C E B A A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| B C D B A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| E C C A A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| C E A B A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| E A E A A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| B C A E A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| D B C B A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| E D A B A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| E A C B A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| C B D B A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| B D A D A ||| lm_0=-28.045 tm_pt_0=-197.000 tm_glue_0=5.000 ||| -220.045
+0 ||| A E A A C ||| lm_0=-28.045 tm_pt_0=-198.000 tm_glue_0=5.000 ||| -221.045
+0 ||| A B B A D ||| lm_0=-28.045 tm_pt_0=-198.000 tm_glue_0=5.000 ||| -221.045
+0 ||| A A B D B ||| lm_0=-28.045 tm_pt_0=-198.000 tm_glue_0=5.000 ||| -221.045
+0 ||| A B B B C ||| lm_0=-28.045 tm_pt_0=-198.000 tm_glue_0=5.000 ||| -221.045
+0 ||| A A E A C ||| lm_0=-28.045 tm_pt_0=-198.000 tm_glue_0=5.000 ||| -221.045
+0 ||| A D B A C ||| lm_0=-28.045 tm_pt_0=-198.000 tm_glue_0=5.000 ||| -221.045
+0 ||| A A C B C ||| lm_0=-28.045 tm_pt_0=-198.000 tm_glue_0=5.000 ||| -221.045
+0 ||| A C B C B ||| lm_0=-28.045 tm_pt_0=-198.000 tm_glue_0=5.000 ||| -221.045
+0 ||| A B D A C ||| lm_0=-28.045 tm_pt_0=-198.000 tm_glue_0=5.000 ||| -221.045
+0 ||| A D A B C ||| lm_0=-28.045 tm_pt_0=-198.000 tm_glue_0=5.000 ||| -221.045
+0 ||| A C C A C ||| lm_0=-28.045 tm_pt_0=-198.000 tm_glue_0=5.000 ||| -221.045
+0 ||| A B E A B ||| lm_0=-28.045 tm_pt_0=-198.000 tm_glue_0=5.000 ||| -221.045
+0 ||| A A D C B ||| lm_0=-28.045 tm_pt_0=-198.000 tm_glue_0=5.000 ||| -221.045
+0 ||| A C A D B ||| lm_0=-28.045 tm_pt_0=-198.000 tm_glue_0=5.000 ||| -221.045
+0 ||| A B C B B ||| lm_0=-28.045 tm_pt_0=-198.000 tm_glue_0=5.000 ||| -221.045
+0 ||| A A C A D ||| lm_0=-28.045 tm_pt_0=-198.000 tm_glue_0=5.000 ||| -221.045
+0 ||| A D C A B ||| lm_0=-28.045 tm_pt_0=-198.000 tm_glue_0=5.000 ||| -221.045
+0 ||| A B A B D ||| lm_0=-28.045 tm_pt_0=-198.000 tm_glue_0=5.000 ||| -221.045
+0 ||| A D C B A ||| lm_0=-28.045 tm_pt_0=-198.000 tm_glue_0=5.000 ||| -221.045
+0 ||| A D A A D ||| lm_0=-28.045 tm_pt_0=-198.000 tm_glue_0=5.000 ||| -221.045
+0 ||| A B E B A ||| lm_0=-28.045 tm_pt_0=-198.000 tm_glue_0=5.000 ||| -221.045
+0 ||| A C D C A ||| lm_0=-28.045 tm_pt_0=-198.000 tm_glue_0=5.000 ||| -221.045
+0 ||| A E C A A ||| lm_0=-28.045 tm_pt_0=-198.000 tm_glue_0=5.000 ||| -221.045
+0 ||| A C B D A ||| lm_0=-28.045 tm_pt_0=-198.000 tm_glue_0=5.000 ||| -221.045
+0 ||| A D E A A ||| lm_0=-28.045 tm_pt_0=-198.000 tm_glue_0=5.000 ||| -221.045
+0 ||| A A D D A ||| lm_0=-28.045 tm_pt_0=-198.000 tm_glue_0=5.000 ||| -221.045
+0 ||| D C B A C ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| C C D A B ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| B A B D B ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| B B C B B ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| C A D B B ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| E A B B B ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| E C A B B ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| B D A A D ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| D C C A B ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| D B D A B ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| D D B A B ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| D B B B B ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| E C B A B ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| C A A E B ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| B A D C B ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| C B B C B ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| B B B A D ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| B C A D B ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| E A D A B ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| D E A A B ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| C A C C B ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| C B A D B ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| B B E A B ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| B C B C B ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| D A E A B ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| E B A C B ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| D C A B C ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| C A B C C ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| E A A C C ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| B B B B C ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| B A E A C ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| B D A B C ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| C C A C C ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| D A B B C ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| B E A A C ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| B A C B C ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| D A D A C ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| B D B A C ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| C A A D C ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| B C C A C ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| D B A C C ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| C B C A C ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| B A C A D ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| B B D A C ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| D A B A D ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| D D A B B ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| D A C B B ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| B D C A B ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| D C A A D ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| C C B B B ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| D A A B D ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| B B A B D ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| C A A A E ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| C A A C D ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| C D A C B ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| B A D D A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| C C D B A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| D C C B A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| E A D B A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| B D C B A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| D E A B A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| E C B B A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| B B E B A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| E B A D A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| D A E B A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| C A B E A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| E A A E A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| D B D B A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| C A E C A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| C C A E A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| C B D C A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| B C D C A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| E B B C A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| C D A D A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| D D D A A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| C D B C A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| E C D A A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| E A C C A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| B E C A A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| D B A E A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| D E B A A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| D D B B A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| B D E A A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| E D A C A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| D C E A A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| C E A C A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| B C B D A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| D B C C A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| C B B D A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| C A C D A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| C C C C A ||| lm_0=-28.045 tm_pt_0=-199.000 tm_glue_0=5.000 ||| -222.045
+0 ||| A C A B D ||| lm_0=-28.045 tm_pt_0=-200.000 tm_glue_0=5.000 ||| -223.045
+0 ||| A D B B B ||| lm_0=-28.045 tm_pt_0=-200.000 tm_glue_0=5.000 ||| -223.045
+0 ||| A D D A B ||| lm_0=-28.045 tm_pt_0=-200.000 tm_glue_0=5.000 ||| -223.045
+0 ||| A B C C B ||| lm_0=-28.045 tm_pt_0=-200.000 tm_glue_0=5.000 ||| -223.045
+0 ||| A E B B A ||| lm_0=-28.045 tm_pt_0=-200.000 tm_glue_0=5.000 ||| -223.045
+0 ||| A C C B B ||| lm_0=-28.045 tm_pt_0=-200.000 tm_glue_0=5.000 ||| -223.045
+0 ||| A A B B D ||| lm_0=-28.045 tm_pt_0=-200.000 tm_glue_0=5.000 ||| -223.045
+0 ||| A D C C A ||| lm_0=-28.045 tm_pt_0=-200.000 tm_glue_0=5.000 ||| -223.045
+0 ||| A B D B B ||| lm_0=-28.045 tm_pt_0=-200.000 tm_glue_0=5.000 ||| -223.045
+0 ||| A A E B B ||| lm_0=-28.045 tm_pt_0=-200.000 tm_glue_0=5.000 ||| -223.045
+0 ||| A C E B A ||| lm_0=-28.045 tm_pt_0=-200.000 tm_glue_0=5.000 ||| -223.045
+0 ||| A C E A B ||| lm_0=-28.045 tm_pt_0=-200.000 tm_glue_0=5.000 ||| -223.045
+0 ||| A E B A B ||| lm_0=-28.045 tm_pt_0=-200.000 tm_glue_0=5.000 ||| -223.045
+0 ||| A D D B A ||| lm_0=-28.045 tm_pt_0=-200.000 tm_glue_0=5.000 ||| -223.045
+0 ||| A B A E B ||| lm_0=-28.045 tm_pt_0=-200.000 tm_glue_0=5.000 ||| -223.045
+0 ||| A B E C A ||| lm_0=-28.045 tm_pt_0=-200.000 tm_glue_0=5.000 ||| -223.045
+0 ||| A A D A D ||| lm_0=-28.045 tm_pt_0=-200.000 tm_glue_0=5.000 ||| -223.045
+0 ||| A E D A A ||| lm_0=-28.045 tm_pt_0=-200.000 tm_glue_0=5.000 ||| -223.045
+0 ||| A B A A E ||| lm_0=-28.045 tm_pt_0=-200.000 tm_glue_0=5.000 ||| -223.045
+0 ||| A A C E A ||| lm_0=-28.045 tm_pt_0=-200.000 tm_glue_0=5.000 ||| -223.045
+0 ||| A C B B C ||| lm_0=-28.045 tm_pt_0=-200.000 tm_glue_0=5.000 ||| -223.045
+0 ||| A B B E A ||| lm_0=-28.045 tm_pt_0=-200.000 tm_glue_0=5.000 ||| -223.045
+0 ||| A C D A C ||| lm_0=-28.045 tm_pt_0=-200.000 tm_glue_0=5.000 ||| -223.045
+0 ||| A D A E A ||| lm_0=-28.045 tm_pt_0=-200.000 tm_glue_0=5.000 ||| -223.045
+0 ||| A B A D C ||| lm_0=-28.045 tm_pt_0=-200.000 tm_glue_0=5.000 ||| -223.045
+0 ||| A B C D A ||| lm_0=-28.045 tm_pt_0=-200.000 tm_glue_0=5.000 ||| -223.045
+0 ||| A A A E C ||| lm_0=-28.045 tm_pt_0=-200.000 tm_glue_0=5.000 ||| -223.045
+0 ||| A B A C D ||| lm_0=-28.045 tm_pt_0=-200.000 tm_glue_0=5.000 ||| -223.045
+0 ||| A E A B B ||| lm_0=-28.045 tm_pt_0=-200.000 tm_glue_0=5.000 ||| -223.045
+0 ||| A C B A D ||| lm_0=-28.045 tm_pt_0=-200.000 tm_glue_0=5.000 ||| -223.045
+0 ||| A A D B C ||| lm_0=-28.045 tm_pt_0=-200.000 tm_glue_0=5.000 ||| -223.045
+0 ||| A B B C C ||| lm_0=-28.045 tm_pt_0=-200.000 tm_glue_0=5.000 ||| -223.045
+0 ||| A D A C C ||| lm_0=-28.045 tm_pt_0=-200.000 tm_glue_0=5.000 ||| -223.045
+0 ||| A A C C C ||| lm_0=-28.045 tm_pt_0=-200.000 tm_glue_0=5.000 ||| -223.045
+0 ||| B B B C C ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| D A C C B ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| D C B B B ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| B A E B B ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| E A C A C ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| D C D B A ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| E C A C B ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| D D A C B ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| C C D C A ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| B B C C B ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| D A A E B ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| E B C B A ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| C D C A B ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| B C A B D ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| C C B D A ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| C A D C B ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| B C B A D ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| C D C B A ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| C B E A B ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| C A B D B ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| B B E C A ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| C C B C B ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| D C A C C ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| C B E B A ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| B D B B B ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| E B C A B ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| E A D C A ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| D B B C B ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| B A B B D ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| B B C D A ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| B B D B B ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| D A D B B ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| D C C C A ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| D C D A B ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| B E A B B ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| E C B C A ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| C C A D B ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| B D D A B ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| D A C D A ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| E A A D B ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| D B A D B ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| E C A D A ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| B E B A B ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| C B C B B ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| E A B D A ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| B C E A B ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| D E A C A ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| E A B C B ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| B D D B A ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| B B A E B ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| B D C C A ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| B B A C D ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| B B B E A ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| B C C B B ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| B D A E A ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| C B A B D ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| D A E C A ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| C A C A D ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| D B D C A ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| E B B A C ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| B C E B A ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| D A A C D ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| C D E A A ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| B A D A D ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| D B B D A ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| B A C C C ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| E D C A A ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| D A A D C ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| D D A D A ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| C B B A D ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| C E C A A ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| B A D B C ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| D A B E A ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| E B A A D ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| E B E A A ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| B B A D C ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| D C A E A ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| E B A B C ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| B E D A A ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| B D A C C ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| B E B B A ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| D A B C C ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| B A C E A ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| C D A B C ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| D D B C A ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| C A C B C ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| C A D D A ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| C E A A C ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| D A A A E ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| C B B B C ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| B A A E C ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| C D A A D ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| C C C A C ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| E D A A C ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| C A E A C ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| D B C A C ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| B C B B C ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| B B A A E ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| B C D A C ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| C D B A C ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| C B D A C ||| lm_0=-28.045 tm_pt_0=-201.000 tm_glue_0=5.000 ||| -224.045
+0 ||| A C D B B ||| lm_0=-28.045 tm_pt_0=-202.000 tm_glue_0=5.000 ||| -225.045
+0 ||| A A B C D ||| lm_0=-28.045 tm_pt_0=-202.000 tm_glue_0=5.000 ||| -225.045
+0 ||| A C C C B ||| lm_0=-28.045 tm_pt_0=-202.000 tm_glue_0=5.000 ||| -225.045
+0 ||| A A C D B ||| lm_0=-28.045 tm_pt_0=-202.000 tm_glue_0=5.000 ||| -225.045
+0 ||| A D A D B ||| lm_0=-28.045 tm_pt_0=-202.000 tm_glue_0=5.000 ||| -225.045
+0 ||| A E A C B ||| lm_0=-28.045 tm_pt_0=-202.000 tm_glue_0=5.000 ||| -225.045
+0 ||| A B D C B ||| lm_0=-28.045 tm_pt_0=-202.000 tm_glue_0=5.000 ||| -225.045
+0 ||| A B B D B ||| lm_0=-28.045 tm_pt_0=-202.000 tm_glue_0=5.000 ||| -225.045
+0 ||| A A E D A ||| lm_0=-28.045 tm_pt_0=-202.000 tm_glue_0=5.000 ||| -225.045
+0 ||| A A E C B ||| lm_0=-28.045 tm_pt_0=-202.000 tm_glue_0=5.000 ||| -225.045
+0 ||| A C A E B ||| lm_0=-28.045 tm_pt_0=-202.000 tm_glue_0=5.000 ||| -225.045
+0 ||| A A B A E ||| lm_0=-28.045 tm_pt_0=-202.000 tm_glue_0=5.000 ||| -225.045
+0 ||| A C E C A ||| lm_0=-28.045 tm_pt_0=-202.000 tm_glue_0=5.000 ||| -225.045
+0 ||| A A A D D ||| lm_0=-28.045 tm_pt_0=-202.000 tm_glue_0=5.000 ||| -225.045
+0 ||| A D D C A ||| lm_0=-28.045 tm_pt_0=-202.000 tm_glue_0=5.000 ||| -225.045
+0 ||| A A B E B ||| lm_0=-28.045 tm_pt_0=-202.000 tm_glue_0=5.000 ||| -225.045
+0 ||| A A D E A ||| lm_0=-28.045 tm_pt_0=-202.000 tm_glue_0=5.000 ||| -225.045
+0 ||| A C A C D ||| lm_0=-28.045 tm_pt_0=-202.000 tm_glue_0=5.000 ||| -225.045
+0 ||| A C C D A ||| lm_0=-28.045 tm_pt_0=-202.000 tm_glue_0=5.000 ||| -225.045
+0 ||| A B C A D ||| lm_0=-28.045 tm_pt_0=-202.000 tm_glue_0=5.000 ||| -225.045
+0 ||| A D B D A ||| lm_0=-28.045 tm_pt_0=-202.000 tm_glue_0=5.000 ||| -225.045
+0 ||| A D B C B ||| lm_0=-28.045 tm_pt_0=-202.000 tm_glue_0=5.000 ||| -225.045
+0 ||| A E A D A ||| lm_0=-28.045 tm_pt_0=-202.000 tm_glue_0=5.000 ||| -225.045
+0 ||| A C A A E ||| lm_0=-28.045 tm_pt_0=-202.000 tm_glue_0=5.000 ||| -225.045
+0 ||| A C B E A ||| lm_0=-28.045 tm_pt_0=-202.000 tm_glue_0=5.000 ||| -225.045
+0 ||| A D C A C ||| lm_0=-28.045 tm_pt_0=-202.000 tm_glue_0=5.000 ||| -225.045
+0 ||| A B D D A ||| lm_0=-28.045 tm_pt_0=-202.000 tm_glue_0=5.000 ||| -225.045
+0 ||| A A A B E ||| lm_0=-28.045 tm_pt_0=-202.000 tm_glue_0=5.000 ||| -225.045
+0 ||| A E B C A ||| lm_0=-28.045 tm_pt_0=-202.000 tm_glue_0=5.000 ||| -225.045
+0 ||| A A B D C ||| lm_0=-28.045 tm_pt_0=-202.000 tm_glue_0=5.000 ||| -225.045
+0 ||| A C A D C ||| lm_0=-28.045 tm_pt_0=-202.000 tm_glue_0=5.000 ||| -225.045
+0 ||| A C B C C ||| lm_0=-28.045 tm_pt_0=-202.000 tm_glue_0=5.000 ||| -225.045
+0 ||| A A D C C ||| lm_0=-28.045 tm_pt_0=-202.000 tm_glue_0=5.000 ||| -225.045
+0 ||| A B E A C ||| lm_0=-28.045 tm_pt_0=-202.000 tm_glue_0=5.000 ||| -225.045
+0 ||| A B C B C ||| lm_0=-28.045 tm_pt_0=-202.000 tm_glue_0=5.000 ||| -225.045
+0 ||| C C E A B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| B B C B C ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| B A B D C ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| C C D A C ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| D C C A C ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| C B B C C ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| E A B B C ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| E C A B C ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| B C C C B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| B A D C C ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| C B A D C ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| D B D A C ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| D D B A C ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| D B B B C ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| C A A E C ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| E C B A C ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| C D A C C ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| C A C C C ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| B C A D C ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| E E A A B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| C B A E B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| D E A A C ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| E A D A C ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| C B C C B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| E B D A B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| B B E A C ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| D B E A B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| C D D A B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| B C B C C ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| C A E B B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| B B B D B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| C C C B B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| D A C A D ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| D A E A C ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| C B A A E ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| C A D B C ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| E D B A B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| E B A C C ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| D C D C A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| D D C A B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| B A A B E ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| E A A B D ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| E B A E A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| D C A D B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| D A D D A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| C C A B D ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| E D B B A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| B A C D B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| B C E C A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| B A B C D ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| C D A E A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| C C B A D ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| B A D E A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| B B C A D ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| C E B B A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| B C A C D ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| C A C E A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| E C A A D ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| C D C C A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| D B B A D ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| E B D B A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| B A E C B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| E B C C A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| C A D A D ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| C B E C A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| B A B E B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| E E A B A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| B C D B B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| D B E B A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| B D B C B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| D D C B A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| C D B B B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| C D D B A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| B B D C B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| C C E B A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| B E A C B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| E C C B A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| D A B D B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| B E B C A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| C E B A B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| B D D C A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| D B A B D ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| D E C A A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| E B B B B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| B E A D A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| B C A A E ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| D D E A A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| E A C B B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| C B B E A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| C E A B B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| E C E A A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| B A B A E ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| B B D D A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| E C C A B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| C E D A A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| E D A B B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| C B C D A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| B A A D D ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| E E B A A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| E A E A B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| B A E D A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| D B C B B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| E D D A A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| B C A E B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| B C C D A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| B D A D B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| D C B D A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| C B D B B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| B D B D A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| D C B C B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| B C B E A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| D D A A D ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| E A E B A ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| C A B B D ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| E A B A D ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| B D C A C ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| D A D C B ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| D A C B C ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| D D A B C ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| C C B B C ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| C B A C D ||| lm_0=-28.045 tm_pt_0=-203.000 tm_glue_0=5.000 ||| -226.045
+0 ||| A C E A C ||| lm_0=-28.045 tm_pt_0=-204.000 tm_glue_0=5.000 ||| -227.045
+0 ||| A B D B C ||| lm_0=-28.045 tm_pt_0=-204.000 tm_glue_0=5.000 ||| -227.045
+0 ||| A E B A C ||| lm_0=-28.045 tm_pt_0=-204.000 tm_glue_0=5.000 ||| -227.045
+0 ||| A A E B C ||| lm_0=-28.045 tm_pt_0=-204.000 tm_glue_0=5.000 ||| -227.045
+0 ||| A D D A C ||| lm_0=-28.045 tm_pt_0=-204.000 tm_glue_0=5.000 ||| -227.045
+0 ||| A B C C C ||| lm_0=-28.045 tm_pt_0=-204.000 tm_glue_0=5.000 ||| -227.045
+0 ||| A E C A B ||| lm_0=-28.045 tm_pt_0=-204.000 tm_glue_0=5.000 ||| -227.045
+0 ||| A A D D B ||| lm_0=-28.045 tm_pt_0=-204.000 tm_glue_0=5.000 ||| -227.045
+0 ||| A D E A B ||| lm_0=-28.045 tm_pt_0=-204.000 tm_glue_0=5.000 ||| -227.045
+0 ||| A D B B C ||| lm_0=-28.045 tm_pt_0=-204.000 tm_glue_0=5.000 ||| -227.045
+0 ||| A C D C B ||| lm_0=-28.045 tm_pt_0=-204.000 tm_glue_0=5.000 ||| -227.045
+0 ||| A B E B B ||| lm_0=-28.045 tm_pt_0=-204.000 tm_glue_0=5.000 ||| -227.045
+0 ||| A D E B A ||| lm_0=-28.045 tm_pt_0=-204.000 tm_glue_0=5.000 ||| -227.045
+0 ||| A D C B B ||| lm_0=-28.045 tm_pt_0=-204.000 tm_glue_0=5.000 ||| -227.045
+0 ||| A B C E A ||| lm_0=-28.045 tm_pt_0=-204.000 tm_glue_0=5.000 ||| -227.045
+0 ||| A B A E C ||| lm_0=-28.045 tm_pt_0=-204.000 tm_glue_0=5.000 ||| -227.045
+0 ||| A E C B A ||| lm_0=-28.045 tm_pt_0=-204.000 tm_glue_0=5.000 ||| -227.045
+0 ||| A A A C E ||| lm_0=-28.045 tm_pt_0=-204.000 tm_glue_0=5.000 ||| -227.045
+0 ||| A E E A A ||| lm_0=-28.045 tm_pt_0=-204.000 tm_glue_0=5.000 ||| -227.045
+0 ||| A A C B D ||| lm_0=-28.045 tm_pt_0=-204.000 tm_glue_0=5.000 ||| -227.045
+0 ||| A C D D A ||| lm_0=-28.045 tm_pt_0=-204.000 tm_glue_0=5.000 ||| -227.045
+0 ||| A D B A D ||| lm_0=-28.045 tm_pt_0=-204.000 tm_glue_0=5.000 ||| -227.045
+0 ||| A E A A D ||| lm_0=-28.045 tm_pt_0=-204.000 tm_glue_0=5.000 ||| -227.045
+0 ||| A C C B C ||| lm_0=-28.045 tm_pt_0=-204.000 tm_glue_0=5.000 ||| -227.045
+0 ||| A A E A D ||| lm_0=-28.045 tm_pt_0=-204.000 tm_glue_0=5.000 ||| -227.045
+0 ||| A B B B D ||| lm_0=-28.045 tm_pt_0=-204.000 tm_glue_0=5.000 ||| -227.045
+0 ||| A D A B D ||| lm_0=-28.045 tm_pt_0=-204.000 tm_glue_0=5.000 ||| -227.045
+0 ||| A B D A D ||| lm_0=-28.045 tm_pt_0=-204.000 tm_glue_0=5.000 ||| -227.045
+0 ||| A C C A D ||| lm_0=-28.045 tm_pt_0=-204.000 tm_glue_0=5.000 ||| -227.045
+0 ||| A E A B C ||| lm_0=-28.045 tm_pt_0=-204.000 tm_glue_0=5.000 ||| -227.045
+0 ||| A C B D B ||| lm_0=-28.045 tm_pt_0=-204.000 tm_glue_0=5.000 ||| -227.045
+0 ||| E C B B B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| E C D A B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| B B E B B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| C A B E B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| E B A D B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| D A E B B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| C B D C B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| E A A E B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| D B D B B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| C D B C B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| C A E C B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| C C A E B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| B C D C B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| D D D A B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| E B B C B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| C D A D B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| B E C A B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| B D E A B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| E A C C B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| D B A E B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| D E B A B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| D D B B B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| B C B D B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| C E A C B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| E D A C B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| D C E A B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| D B C C B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| C A C D B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| D C B A D ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| C B B D B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| C C C C B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| D C B B C ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| D A C C C ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| D C D A C ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| C B C B C ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| E A A D C ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| B A D D B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| D C C B B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| C C D B B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| B D C B B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| B E B A C ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| C B C A D ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| B D A B D ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| E A A C D ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| C A B C D ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| E C A C C ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| D C A B D ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| B B C C C ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| B A E A D ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| D D A C C ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| B B B B D ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| C D C A C ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| D A A E C ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| E A D B B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| D B A C D ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| B A C B D ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| C B E A C ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| B D B B C ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| D B B C C ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| B E A A D ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| D A B B D ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| D B A D C ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| E B C A C ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| D E A B B ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| B A E B C ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| B C C A D ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| C C A C D ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| C C B C C ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| D A D A D ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| B B D B C ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| E D B C A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| B D D A C ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| B E A B C ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| C A A D D ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| C C A D C ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| B D B A D ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| B E C B A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| E A B C C ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| D D D B A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| B C E A C ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| C C E C A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| B B A E C ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| D E B B A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| B C C B C ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| E A B E A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| B A A C E ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| E B B D A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| C A A B E ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| D E D A A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| C A B A E ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| E C A E A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| C C A A E ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| E C D B A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| E A A A E ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| B E E A A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| D B A A E ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| C C B E A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| C A B D C ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| C E B C A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| D A D B C ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| D A C E A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| B B D A D ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| B B C E A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| C A D C C ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| E A E C A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| D B E C A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| B D E B A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| E B D C A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| C A E D A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| C D D C A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| E E A C A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| D D C C A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| D B B E A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| C A D E A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| C B D D A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| E C C C A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| C C C D A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| E A C D A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| C D B D A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| D D A E A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| B C D D A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| D B C D A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| D C E B A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| E D A D A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| C E A D A ||| lm_0=-28.045 tm_pt_0=-205.000 tm_glue_0=5.000 ||| -228.045
+0 ||| A B E C B ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A A C E B ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A D A E B ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A C B B D ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A C D B C ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A D C C B ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A C C C C ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A A C C D ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A D D B B ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A D A C D ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A B B C D ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A C E B B ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A A C D C ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A B B E B ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A D A D C ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A E A C C ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A B D C C ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A A B E C ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A C D A D ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A A E C C ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A E B B B ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A E D A B ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A B D E A ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A C A E C ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A B A B E ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A D E C A ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A A C A E ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A D B C C ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A C C E A ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A A D B D ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A B B D C ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A A E E A ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A B B A E ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A D A A E ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A D B E A ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A B A D D ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A A A E D ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A B E D A ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A B C D B ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A E C C A ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A E A E A ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A D C D A ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| A E D B A ||| lm_0=-28.045 tm_pt_0=-206.000 tm_glue_0=5.000 ||| -229.045
+0 ||| B A C A E ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| E E A A C ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| D C D B B ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| E B C B B ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| C C D C B ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| D C C C B ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| B B B C D ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| C C E A C ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| B B C D B ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| B C C C C ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| C D C B B ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| E A D C B ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| C B E B B ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| B B E C B ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| B D C C B ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| E C A D B ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| C E C A B ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| E C B C B ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| B D D B B ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| D E A C B ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| E A B D B ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| D B D C B ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| B B B E B ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| D D A D B ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| D A E C B ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| B D A E B ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| E D C A B ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| B C E B B ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| D B B D B ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| C D E A B ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| C A E B C ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| D C A E B ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| C D B A D ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| B B B D C ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| D D B C B ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| B E D A B ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| B A C E B ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| D A B E B ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| C B D A D ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| B E B B B ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| C B A E C ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| C D D A C ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||| C B C C C ||| lm_0=-28.045 tm_pt_0=-207.000 tm_glue_0=5.000 ||| -230.045
+0 ||
<TRUNCATED>