You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by mj...@apache.org on 2016/06/01 02:51:50 UTC
[55/94] [abbrv] incubator-joshua git commit: Rebased with JOSHUA-252
and resolved merge conflicts
Rebased with JOSHUA-252 and resolved merge conflicts
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/9d6f84d3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/9d6f84d3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/9d6f84d3
Branch: refs/heads/master
Commit: 9d6f84d35754a099123c256b9932a89a2bd316aa
Parents: 1586853 02f3ef1
Author: Thamme Gowda <tg...@gmail.com>
Authored: Wed May 25 22:34:57 2016 -0700
Committer: Thamme Gowda <tg...@gmail.com>
Committed: Wed May 25 22:34:57 2016 -0700
----------------------------------------------------------------------
.gitmodules | 7 -
Dockerfile | 39 -
bin/bleu | 15 +
bin/extract-1best | 15 +
bin/joshua-decoder | 16 +
bin/meteor | 15 +
docker/Dockerfile | 39 +
docker/ar-en-phrase/Dockerfile | 11 +
docker/zh-en-hiero/Dockerfile | 11 +
examples/README.md | 39 +-
examples/README.sp_to_en | 32 -
examples/docker/ar-en-phrase/Dockerfile | 11 -
examples/docker/zh-en-hiero/Dockerfile | 11 -
ext/berkeleylm | 1 -
ext/giza-pp/GIZA++-v2/ATables.cpp | 119 -
ext/giza-pp/GIZA++-v2/ATables.h | 172 -
ext/giza-pp/GIZA++-v2/AlignTables.cpp | 44 -
ext/giza-pp/GIZA++-v2/AlignTables.h | 118 -
ext/giza-pp/GIZA++-v2/Array.h | 5 -
ext/giza-pp/GIZA++-v2/Array2.h | 107 -
ext/giza-pp/GIZA++-v2/Array4.h | 78 -
ext/giza-pp/GIZA++-v2/D4Tables.h | 460 --
ext/giza-pp/GIZA++-v2/D5Tables.h | 235 -
ext/giza-pp/GIZA++-v2/Dictionary.cpp | 94 -
ext/giza-pp/GIZA++-v2/Dictionary.h | 48 -
ext/giza-pp/GIZA++-v2/FlexArray.h | 58 -
ext/giza-pp/GIZA++-v2/ForwardBackward.cpp | 242 -
ext/giza-pp/GIZA++-v2/ForwardBackward.h | 62 -
ext/giza-pp/GIZA++-v2/GNU.GPL | 282 -
ext/giza-pp/GIZA++-v2/Globals.h | 73 -
ext/giza-pp/GIZA++-v2/HMMTables.cpp | 177 -
ext/giza-pp/GIZA++-v2/HMMTables.h | 172 -
ext/giza-pp/GIZA++-v2/LICENSE | 282 -
ext/giza-pp/GIZA++-v2/Makefile | 140 -
ext/giza-pp/GIZA++-v2/Makefile.definitions | 0
ext/giza-pp/GIZA++-v2/Makefile.src | 2 -
ext/giza-pp/GIZA++-v2/MoveSwapMatrix.cpp | 231 -
ext/giza-pp/GIZA++-v2/MoveSwapMatrix.h | 116 -
ext/giza-pp/GIZA++-v2/NTables.cpp | 93 -
ext/giza-pp/GIZA++-v2/NTables.h | 145 -
ext/giza-pp/GIZA++-v2/Parameter.cpp | 144 -
ext/giza-pp/GIZA++-v2/Parameter.h | 200 -
ext/giza-pp/GIZA++-v2/Perplexity.cpp | 40 -
ext/giza-pp/GIZA++-v2/Perplexity.h | 108 -
ext/giza-pp/GIZA++-v2/Pointer.h | 175 -
ext/giza-pp/GIZA++-v2/README | 508 --
ext/giza-pp/GIZA++-v2/TTables.cpp | 323 -
ext/giza-pp/GIZA++-v2/TTables.h | 417 --
ext/giza-pp/GIZA++-v2/Vector.h | 427 --
ext/giza-pp/GIZA++-v2/WordClasses.h | 96 -
ext/giza-pp/GIZA++-v2/alignment.cpp | 38 -
ext/giza-pp/GIZA++-v2/alignment.h | 227 -
ext/giza-pp/GIZA++-v2/collCounts.cpp | 293 -
ext/giza-pp/GIZA++-v2/collCounts.h | 80 -
ext/giza-pp/GIZA++-v2/defs.h | 78 -
ext/giza-pp/GIZA++-v2/dependencies | 635 --
ext/giza-pp/GIZA++-v2/file_spec.h | 59 -
ext/giza-pp/GIZA++-v2/getSentence.cpp | 340 --
ext/giza-pp/GIZA++-v2/getSentence.h | 123 -
ext/giza-pp/GIZA++-v2/hmm.cpp | 405 --
ext/giza-pp/GIZA++-v2/hmm.h | 82 -
ext/giza-pp/GIZA++-v2/logprob.cpp | 154 -
ext/giza-pp/GIZA++-v2/logprob.h | 217 -
ext/giza-pp/GIZA++-v2/main.cpp | 719 ---
ext/giza-pp/GIZA++-v2/model1.cpp | 283 -
ext/giza-pp/GIZA++-v2/model1.h | 158 -
ext/giza-pp/GIZA++-v2/model2.cpp | 232 -
ext/giza-pp/GIZA++-v2/model2.h | 63 -
ext/giza-pp/GIZA++-v2/model2to3.cpp | 398 --
ext/giza-pp/GIZA++-v2/model3.cpp | 511 --
ext/giza-pp/GIZA++-v2/model3.h | 132 -
ext/giza-pp/GIZA++-v2/model345-peg.cpp | 191 -
ext/giza-pp/GIZA++-v2/model3_viterbi.cpp | 656 --
.../GIZA++-v2/model3_viterbi_with_tricks.cpp | 690 ---
ext/giza-pp/GIZA++-v2/myassert.cpp | 20 -
ext/giza-pp/GIZA++-v2/myassert.h | 20 -
ext/giza-pp/GIZA++-v2/mymath.h | 9 -
ext/giza-pp/GIZA++-v2/mystl.h | 321 -
ext/giza-pp/GIZA++-v2/parse.cpp | 151 -
ext/giza-pp/GIZA++-v2/plain2snt.cpp | 116 -
ext/giza-pp/GIZA++-v2/reports.cpp | 211 -
ext/giza-pp/GIZA++-v2/small_snt2cooc.cpp | 91 -
ext/giza-pp/GIZA++-v2/snt2cooc.cpp | 107 -
ext/giza-pp/GIZA++-v2/snt2plain.cpp | 91 -
ext/giza-pp/GIZA++-v2/trainGIZA++.sh | 34 -
ext/giza-pp/GIZA++-v2/transpair_model1.h | 108 -
ext/giza-pp/GIZA++-v2/transpair_model2.h | 52 -
ext/giza-pp/GIZA++-v2/transpair_model3.cpp | 197 -
ext/giza-pp/GIZA++-v2/transpair_model3.h | 84 -
ext/giza-pp/GIZA++-v2/transpair_model4.cpp | 179 -
ext/giza-pp/GIZA++-v2/transpair_model4.h | 79 -
ext/giza-pp/GIZA++-v2/transpair_model5.cpp | 243 -
ext/giza-pp/GIZA++-v2/transpair_model5.h | 74 -
ext/giza-pp/GIZA++-v2/transpair_modelhmm.h | 223 -
ext/giza-pp/GIZA++-v2/utility.cpp | 30 -
ext/giza-pp/GIZA++-v2/utility.h | 54 -
ext/giza-pp/GIZA++-v2/vocab.cpp | 90 -
ext/giza-pp/GIZA++-v2/vocab.h | 104 -
ext/giza-pp/Makefile | 20 -
ext/giza-pp/README | 8 -
ext/giza-pp/mkcls-v2/Array.h | 370 --
ext/giza-pp/mkcls-v2/FixedArray.h | 287 -
ext/giza-pp/mkcls-v2/FlexArray.h | 48 -
ext/giza-pp/mkcls-v2/GDAOptimization.cpp | 159 -
ext/giza-pp/mkcls-v2/GDAOptimization.h | 80 -
ext/giza-pp/mkcls-v2/GNU.GPL | 282 -
ext/giza-pp/mkcls-v2/HCOptimization.cpp | 57 -
ext/giza-pp/mkcls-v2/HCOptimization.h | 54 -
ext/giza-pp/mkcls-v2/IterOptimization.cpp | 199 -
ext/giza-pp/mkcls-v2/IterOptimization.h | 123 -
ext/giza-pp/mkcls-v2/KategProblem.cpp | 1001 ---
ext/giza-pp/mkcls-v2/KategProblem.h | 439 --
ext/giza-pp/mkcls-v2/KategProblemKBC.cpp | 243 -
ext/giza-pp/mkcls-v2/KategProblemKBC.h | 157 -
ext/giza-pp/mkcls-v2/KategProblemTest.cpp | 695 ---
ext/giza-pp/mkcls-v2/KategProblemTest.h | 60 -
ext/giza-pp/mkcls-v2/KategProblemWBC.cpp | 344 --
ext/giza-pp/mkcls-v2/KategProblemWBC.h | 131 -
ext/giza-pp/mkcls-v2/LICENSE | 282 -
ext/giza-pp/mkcls-v2/MSBOptimization.cpp | 229 -
ext/giza-pp/mkcls-v2/MSBOptimization.h | 77 -
ext/giza-pp/mkcls-v2/MYOptimization.cpp | 85 -
ext/giza-pp/mkcls-v2/MYOptimization.h | 61 -
ext/giza-pp/mkcls-v2/Makefile | 23 -
ext/giza-pp/mkcls-v2/Optimization.cpp | 30 -
ext/giza-pp/mkcls-v2/Optimization.h | 49 -
ext/giza-pp/mkcls-v2/PopOptimization.cpp | 105 -
ext/giza-pp/mkcls-v2/PopOptimization.h | 89 -
ext/giza-pp/mkcls-v2/Problem.cpp | 165 -
ext/giza-pp/mkcls-v2/Problem.h | 159 -
ext/giza-pp/mkcls-v2/ProblemTest.cpp | 263 -
ext/giza-pp/mkcls-v2/ProblemTest.h | 65 -
ext/giza-pp/mkcls-v2/README | 10 -
ext/giza-pp/mkcls-v2/RRTOptimization.cpp | 217 -
ext/giza-pp/mkcls-v2/RRTOptimization.h | 79 -
ext/giza-pp/mkcls-v2/SAOptimization.cpp | 280 -
ext/giza-pp/mkcls-v2/SAOptimization.h | 86 -
ext/giza-pp/mkcls-v2/StatVar.cpp | 140 -
ext/giza-pp/mkcls-v2/StatVar.h | 134 -
ext/giza-pp/mkcls-v2/TAOptimization.cpp | 208 -
ext/giza-pp/mkcls-v2/TAOptimization.h | 78 -
ext/giza-pp/mkcls-v2/general.cpp | 120 -
ext/giza-pp/mkcls-v2/general.h | 89 -
ext/giza-pp/mkcls-v2/makePackage.sh | 43 -
ext/giza-pp/mkcls-v2/mkcls.cpp | 618 --
ext/giza-pp/mkcls-v2/my.h | 54 -
ext/giza-pp/mkcls-v2/myassert.h | 44 -
ext/giza-pp/mkcls-v2/myleda.h | 232 -
ext/giza-pp/mkcls-v2/mystl.h | 116 -
ext/kenlm | 1 -
ext/symal/Makefile | 11 -
ext/symal/cmd.c | 649 --
ext/symal/cmd.h | 51 -
ext/symal/giza2bal.pl | 112 -
ext/symal/symal.cpp | 503 --
jni/kenlm_wrap.cc | 17 +
scripts/training/pipeline.pl | 15 +
.../org/apache/joshua/corpus/BasicPhrase.java | 2 +-
.../apache/joshua/corpus/ContiguousPhrase.java | 60 +-
.../java/org/apache/joshua/corpus/Corpus.java | 21 +-
.../java/org/apache/joshua/corpus/Phrase.java | 3 +
.../java/org/apache/joshua/corpus/Span.java | 4 +-
.../org/apache/joshua/corpus/SymbolTable.java | 597 +-
.../apache/joshua/corpus/TerminalIterator.java | 5 +-
.../org/apache/joshua/corpus/Vocabulary.java | 42 +-
.../java/org/apache/joshua/corpus/package.html | 19 -
.../joshua/corpus/syntax/ArraySyntaxTree.java | 15 +-
.../org/apache/joshua/decoder/ArgsParser.java | 9 +-
.../java/org/apache/joshua/decoder/BLEU.java | 35 +-
.../java/org/apache/joshua/decoder/Decoder.java | 28 +-
.../apache/joshua/decoder/DecoderThread.java | 19 +-
.../joshua/decoder/JoshuaConfiguration.java | 13 +-
.../apache/joshua/decoder/JoshuaDecoder.java | 6 +-
.../joshua/decoder/NbestMinRiskReranker.java | 7 +-
.../joshua/decoder/StructuredTranslation.java | 7 +-
.../java/org/apache/joshua/decoder/Support.java | 5 +-
.../org/apache/joshua/decoder/Translation.java | 74 +-
.../org/apache/joshua/decoder/Translations.java | 6 +-
.../joshua/decoder/chart_parser/Chart.java | 6 +-
.../decoder/chart_parser/ComputeNodeResult.java | 43 +-
.../chart_parser/ManualConstraintsHandler.java | 13 +-
.../joshua/decoder/chart_parser/SourcePath.java | 2 +-
.../decoder/chart_parser/StateConstraint.java | 4 +-
.../decoder/chart_parser/package-info.java | 24 +
.../joshua/decoder/chart_parser/package.html | 23 -
.../joshua/decoder/ff/ArityPhrasePenalty.java | 4 +-
.../joshua/decoder/ff/FeatureFunction.java | 103 +-
.../apache/joshua/decoder/ff/FeatureVector.java | 32 +-
.../apache/joshua/decoder/ff/OOVPenalty.java | 2 +-
.../apache/joshua/decoder/ff/PhraseModel.java | 4 +-
.../apache/joshua/decoder/ff/SourcePathFF.java | 4 +-
.../apache/joshua/decoder/ff/StatefulFF.java | 4 +-
.../apache/joshua/decoder/ff/StatelessFF.java | 4 +-
.../apache/joshua/decoder/ff/TargetBigram.java | 2 +-
.../apache/joshua/decoder/ff/WordPenalty.java | 4 +-
.../decoder/ff/fragmentlm/FragmentLMFF.java | 49 +-
.../joshua/decoder/ff/fragmentlm/Tree.java | 54 +-
.../joshua/decoder/ff/fragmentlm/Trees.java | 3 +
.../apache/joshua/decoder/ff/lm/AbstractLM.java | 15 +-
.../ff/lm/DefaultNGramLanguageModel.java | 10 +-
.../org/apache/joshua/decoder/ff/lm/KenLM.java | 17 +-
.../joshua/decoder/ff/lm/LanguageModelFF.java | 86 +-
.../decoder/ff/lm/NGramLanguageModel.java | 20 +-
.../ff/lm/StateMinimizingLanguageModel.java | 12 +-
.../BloomFilterLanguageModel.java | 1 +
.../ff/lm/bloomfilter_lm/package-info.java | 25 +
.../decoder/ff/lm/bloomfilter_lm/package.html | 19 -
.../joshua/decoder/ff/lm/buildin_lm/TrieLM.java | 4 +-
.../joshua/decoder/ff/lm/package-info.java | 42 +
.../apache/joshua/decoder/ff/lm/package.html | 35 -
.../apache/joshua/decoder/ff/package-info.java | 42 +
.../org/apache/joshua/decoder/ff/package.html | 37 -
.../decoder/ff/state_maintenance/DPState.java | 4 +-
.../ff/state_maintenance/KenLMState.java | 4 +-
.../ff/state_maintenance/NgramDPState.java | 4 +-
.../joshua/decoder/ff/tm/AbstractGrammar.java | 16 +-
.../joshua/decoder/ff/tm/BilingualRule.java | 8 +-
.../apache/joshua/decoder/ff/tm/Grammar.java | 25 +-
.../joshua/decoder/ff/tm/GrammarReader.java | 4 +-
.../joshua/decoder/ff/tm/MonolingualRule.java | 8 +-
.../apache/joshua/decoder/ff/tm/PhraseRule.java | 10 +-
.../org/apache/joshua/decoder/ff/tm/Rule.java | 48 +-
.../joshua/decoder/ff/tm/RuleCollection.java | 10 +-
.../decoder/ff/tm/SentenceFilteredGrammar.java | 17 +-
.../org/apache/joshua/decoder/ff/tm/Trie.java | 18 +-
.../decoder/ff/tm/format/HieroFormatReader.java | 3 +-
.../ff/tm/format/PhraseFormatReader.java | 12 +-
.../tm/hash_based/MemoryBasedBatchGrammar.java | 10 +-
.../ff/tm/hash_based/MemoryBasedRuleBin.java | 2 +-
.../ff/tm/hash_based/MemoryBasedTrie.java | 2 +-
.../decoder/ff/tm/hash_based/package-info.java | 23 +
.../decoder/ff/tm/hash_based/package.html | 17 -
.../joshua/decoder/ff/tm/package-info.java | 25 +
.../apache/joshua/decoder/ff/tm/package.html | 17 -
.../decoder/ff/tm/packed/PackedGrammar.java | 9 +-
.../ff/tm/packed/SliceAggregatingTrie.java | 25 +-
.../decoder/hypergraph/AlignedSourceTokens.java | 10 +-
.../decoder/hypergraph/AllSpansWalker.java | 7 +-
.../hypergraph/DefaultInsideOutside.java | 2 +-
.../joshua/decoder/hypergraph/HGNode.java | 9 +-
.../joshua/decoder/hypergraph/HyperEdge.java | 4 +-
.../joshua/decoder/hypergraph/HyperGraph.java | 5 +-
.../decoder/hypergraph/HyperGraphPruning.java | 2 +-
.../decoder/hypergraph/KBestExtractor.java | 88 +-
.../hypergraph/TrivialInsideOutside.java | 2 +-
.../decoder/hypergraph/ViterbiExtractor.java | 48 +-
.../decoder/hypergraph/WalkerFunction.java | 3 +
.../hypergraph/WordAlignmentExtractor.java | 1 +
.../decoder/hypergraph/WordAlignmentState.java | 3 +
.../joshua/decoder/hypergraph/package-info.java | 25 +
.../joshua/decoder/hypergraph/package.html | 18 -
.../apache/joshua/decoder/io/DeNormalize.java | 10 +-
.../decoder/io/TranslationRequestStream.java | 6 +-
.../org/apache/joshua/decoder/package-info.java | 26 +
.../java/org/apache/joshua/decoder/package.html | 21 -
.../apache/joshua/decoder/phrase/Candidate.java | 10 +-
.../apache/joshua/decoder/phrase/Coverage.java | 42 +-
.../apache/joshua/decoder/phrase/Future.java | 36 +-
.../joshua/decoder/phrase/Hypothesis.java | 8 +-
.../joshua/decoder/phrase/PhraseChart.java | 22 +-
.../joshua/decoder/phrase/PhraseTable.java | 13 +-
.../org/apache/joshua/decoder/phrase/Stack.java | 22 +-
.../apache/joshua/decoder/phrase/Stacks.java | 12 +-
.../joshua/decoder/phrase/TargetPhrases.java | 5 +-
.../decoder/segment_file/ConstraintRule.java | 27 +-
.../decoder/segment_file/ConstraintSpan.java | 8 +-
.../joshua/decoder/segment_file/Sentence.java | 26 +-
.../joshua/decoder/segment_file/Token.java | 29 +-
.../decoder/segment_file/package-info.java | 25 +
.../joshua/decoder/segment_file/package.html | 17 -
.../java/org/apache/joshua/lattice/Lattice.java | 22 +-
.../java/org/apache/joshua/lattice/Node.java | 1 +
.../org/apache/joshua/lattice/package-info.java | 22 +
.../java/org/apache/joshua/lattice/package.html | 18 -
.../java/org/apache/joshua/metrics/BLEU.java | 9 +-
.../apache/joshua/metrics/EvaluationMetric.java | 6 +-
.../joshua/oracle/OracleExtractionHG.java | 47 +-
.../java/org/apache/joshua/oracle/SplitHg.java | 2 +-
.../org/apache/joshua/oracle/package-info.java | 26 +
.../java/org/apache/joshua/oracle/package.html | 24 -
.../org/apache/joshua/server/ServerThread.java | 1 +
.../org/apache/joshua/server/TcpServer.java | 3 -
.../joshua/subsample/AlignedSubsampler.java | 5 +-
.../org/apache/joshua/subsample/Alignment.java | 4 +-
.../org/apache/joshua/subsample/BiCorpus.java | 16 +-
.../joshua/subsample/BiCorpusFactory.java | 23 +-
.../org/apache/joshua/subsample/PhrasePair.java | 2 +-
.../apache/joshua/subsample/PhraseReader.java | 2 +-
.../apache/joshua/subsample/PhraseWriter.java | 2 +-
.../org/apache/joshua/subsample/Subsampler.java | 11 +-
.../apache/joshua/subsample/SubsamplerCLI.java | 13 +-
.../apache/joshua/subsample/package-info.java | 25 +
.../org/apache/joshua/subsample/package.html | 25 -
.../org/apache/joshua/tools/GrammarPacker.java | 2 +-
.../org/apache/joshua/tools/LabelPhrases.java | 5 +-
.../org/apache/joshua/tools/TestSetFilter.java | 2 +
.../java/org/apache/joshua/ui/package-info.java | 22 +
src/main/java/org/apache/joshua/ui/package.html | 25 -
.../ui/tree_visualizer/browser/Browser.java | 3 +-
.../joshua/ui/tree_visualizer/tree/Tree.java | 410 +-
.../java/org/apache/joshua/util/Algorithms.java | 4 +-
src/main/java/org/apache/joshua/util/Bits.java | 12 +-
.../java/org/apache/joshua/util/ChartSpan.java | 19 +-
.../org/apache/joshua/util/CompareGrammars.java | 4 +-
.../java/org/apache/joshua/util/Counted.java | 3 +-
.../java/org/apache/joshua/util/Counts.java | 18 +-
.../org/apache/joshua/util/ExtractTopCand.java | 16 +-
.../org/apache/joshua/util/FileUtility.java | 38 +-
.../org/apache/joshua/util/FormatUtils.java | 18 +-
.../java/org/apache/joshua/util/ListUtil.java | 2 +-
src/main/java/org/apache/joshua/util/Ngram.java | 16 +-
.../org/apache/joshua/util/NullIterator.java | 2 +-
src/main/java/org/apache/joshua/util/Regex.java | 15 +-
.../org/apache/joshua/util/SocketUtility.java | 2 +-
.../util/encoding/EncoderConfiguration.java | 2 +-
.../org/apache/joshua/util/io/BinaryIn.java | 15 +-
.../org/apache/joshua/util/io/BinaryOut.java | 4 +-
.../apache/joshua/util/io/IndexedReader.java | 21 +-
.../org/apache/joshua/util/io/LineReader.java | 17 +-
.../org/apache/joshua/util/io/NullReader.java | 2 +-
.../joshua/util/io/ProgressInputStream.java | 2 +-
.../java/org/apache/joshua/util/io/Reader.java | 19 +-
.../org/apache/joshua/util/io/package-info.java | 22 +
.../java/org/apache/joshua/util/io/package.html | 18 -
.../org/apache/joshua/util/package-info.java | 22 +
.../java/org/apache/joshua/util/package.html | 18 -
.../org/apache/joshua/zmert/package-info.java | 24 +
.../java/org/apache/joshua/zmert/package.html | 24 -
.../joshua/corpus/vocab/VocabularyTest.java | 24 +-
.../decoder/segment_file/SentenceTest.java | 11 +-
.../java/org/apache/joshua/lattice/ArcTest.java | 4 +-
.../org/apache/joshua/lattice/LatticeTest.java | 73 +-
.../org/apache/joshua/lattice/NodeTest.java | 23 +-
.../org/apache/joshua/util/io/BinaryTest.java | 10 +-
.../java/org/apache/joshua/zmert/BLEUTest.java | 2 +-
src/test/resources/bn-en/hiero/.gitignore | 4 +
src/test/resources/bn-en/hiero/class.map | 5140 ++++++++++++++++
.../resources/bn-en/hiero/class_lm_2gram.gz | Bin 0 -> 18052 bytes
.../resources/bn-en/hiero/class_lm_9gram.gz | Bin 0 -> 12733137 bytes
src/test/resources/bn-en/hiero/glue-grammar | 3 +
src/test/resources/bn-en/hiero/grammar.gz | Bin 0 -> 518164 bytes
src/test/resources/bn-en/hiero/input.bn | 100 +
.../bn-en/hiero/joshua-berkeleylm.config | 46 +
.../resources/bn-en/hiero/joshua-classlm.config | 51 +
src/test/resources/bn-en/hiero/joshua.config | 50 +
src/test/resources/bn-en/hiero/lm.gz | Bin 0 -> 2466496 bytes
.../resources/bn-en/hiero/output-classlm.gold | 678 +++
src/test/resources/bn-en/hiero/output.gold | 805 +++
src/test/resources/bn-en/hiero/output.gold.bleu | 14 +
.../bn-en/hiero/output.scores.berkeleylm.gold | 100 +
.../resources/bn-en/hiero/output.scores.gold | 805 +++
src/test/resources/bn-en/hiero/reference.en.0 | 100 +
src/test/resources/bn-en/hiero/reference.en.1 | 100 +
src/test/resources/bn-en/hiero/reference.en.2 | 100 +
src/test/resources/bn-en/hiero/reference.en.3 | 100 +
.../resources/bn-en/hiero/test-berkeleylm.sh | 33 +
src/test/resources/bn-en/hiero/test-classlm.sh | 32 +
src/test/resources/bn-en/hiero/test-filter.sh | 35 +
src/test/resources/bn-en/hiero/test.sh | 35 +
src/test/resources/bn-en/hiero/topN.pl | 18 +
src/test/resources/bn-en/packed/.gitignore | 3 +
src/test/resources/bn-en/packed/grammar.glue | 5673 ++++++++++++++++++
src/test/resources/bn-en/packed/grammar.gz | Bin 0 -> 3540984 bytes
.../bn-en/packed/grammar.packed/encoding | Bin 0 -> 767 bytes
.../packed/grammar.packed/slice_00000.features | Bin 0 -> 4631480 bytes
.../packed/grammar.packed/slice_00000.source | Bin 0 -> 4240012 bytes
.../packed/grammar.packed/slice_00000.target | Bin 0 -> 162776 bytes
.../grammar.packed/slice_00000.target.lookup | Bin 0 -> 56 bytes
.../bn-en/packed/grammar.packed/vocabulary | Bin 0 -> 136522 bytes
src/test/resources/bn-en/packed/input.bn | 100 +
src/test/resources/bn-en/packed/joshua.config | 47 +
src/test/resources/bn-en/packed/lm.gz | Bin 0 -> 2466496 bytes
src/test/resources/bn-en/packed/output.gold | 862 +++
.../resources/bn-en/packed/output.scores.gold | 862 +++
src/test/resources/bn-en/packed/reference.en.0 | 100 +
src/test/resources/bn-en/packed/reference.en.1 | 100 +
src/test/resources/bn-en/packed/reference.en.2 | 100 +
src/test/resources/bn-en/packed/reference.en.3 | 100 +
.../resources/bn-en/packed/reference.en.all | 400 ++
src/test/resources/bn-en/packed/test.sh | 20 +
src/test/resources/bn-en/samt/grammar.glue | 5673 ++++++++++++++++++
src/test/resources/bn-en/samt/grammar.gz | Bin 0 -> 3847934 bytes
src/test/resources/bn-en/samt/input.bn | 100 +
src/test/resources/bn-en/samt/joshua.config | 47 +
src/test/resources/bn-en/samt/lm.gz | Bin 0 -> 2466496 bytes
src/test/resources/bn-en/samt/output.gold | 0
src/test/resources/bn-en/samt/output.gold.bleu | 14 +
.../resources/bn-en/samt/output.scores.gold | 862 +++
src/test/resources/bn-en/samt/reference.en.0 | 100 +
src/test/resources/bn-en/samt/reference.en.1 | 100 +
src/test/resources/bn-en/samt/reference.en.2 | 100 +
src/test/resources/bn-en/samt/reference.en.3 | 100 +
src/test/resources/bn-en/samt/test.sh | 35 +
src/test/resources/data/tiny.en | 5 +
.../resources/decoder/constrained/.gitignore | 4 +
.../resources/decoder/constrained/glue-grammar | 3 +
.../resources/decoder/constrained/gold.scores | 27 +
.../resources/decoder/constrained/grammar.gz | Bin 0 -> 518164 bytes
src/test/resources/decoder/constrained/input.bn | 8 +
.../resources/decoder/constrained/joshua.config | 45 +
src/test/resources/decoder/constrained/lm.gz | Bin 0 -> 2466496 bytes
.../resources/decoder/constrained/output.bleu | 0
.../resources/decoder/constrained/output.gold | 30 +
src/test/resources/decoder/constrained/test.sh | 30 +
src/test/resources/decoder/constrained/weights | 22 +
.../resources/decoder/denormalization/input.txt | 1 +
.../decoder/denormalization/output.expected | 1 +
.../resources/decoder/denormalization/test.sh | 30 +
src/test/resources/decoder/dont-crash/input | 5 +
.../resources/decoder/dont-crash/output.gold | 1 +
src/test/resources/decoder/dont-crash/test.sh | 29 +
.../resources/decoder/empty-test/.gitignore | 3 +
src/test/resources/decoder/empty-test/input | 1 +
.../resources/decoder/empty-test/output.gold | 1 +
src/test/resources/decoder/empty-test/test.sh | 29 +
.../resources/decoder/fragmentlm/fragments.txt | 7 +
src/test/resources/decoder/fragmentlm/glue | 1 +
src/test/resources/decoder/fragmentlm/grammar | 4 +
src/test/resources/decoder/fragmentlm/input | 1 +
.../resources/decoder/fragmentlm/joshua.config | 109 +
.../resources/decoder/fragmentlm/mapping.txt | 4 +
src/test/resources/decoder/fragmentlm/test.sh | 30 +
.../decoder/k-best-extraction/glue-grammar | 3 +
.../resources/decoder/k-best-extraction/grammar | 25 +
.../decoder/k-best-extraction/input.txt | 1 +
.../decoder/k-best-extraction/joshua.config | 27 +
.../resources/decoder/k-best-extraction/lm.gz | Bin 0 -> 2466496 bytes
.../decoder/k-best-extraction/output.gold | 3126 ++++++++++
.../k-best-extraction/output.scores.gold | 3126 ++++++++++
.../resources/decoder/k-best-extraction/test.sh | 33 +
.../resources/decoder/left-state/glue-grammar | 3 +
.../resources/decoder/left-state/grammar.gz | Bin 0 -> 518164 bytes
src/test/resources/decoder/left-state/input.bn | 2 +
.../resources/decoder/left-state/joshua.config | 44 +
src/test/resources/decoder/left-state/lm.gz | Bin 0 -> 2466496 bytes
.../resources/decoder/left-state/output.gold | 600 ++
.../decoder/left-state/output.scores.gold | 600 ++
src/test/resources/decoder/left-state/test.sh | 33 +
src/test/resources/decoder/lowercaser/config | 140 +
.../resources/decoder/lowercaser/grammar.glue | 4 +
.../resources/decoder/lowercaser/grammar.test | 1 +
.../resources/decoder/lowercaser/output.gold | 5 +
src/test/resources/decoder/lowercaser/test.sh | 40 +
.../resources/decoder/moses-compat/n-best.txt | 0
.../decoder/moses-compat/output.expected | 6 +
src/test/resources/decoder/moses-compat/test.sh | 40 +
src/test/resources/decoder/n-ary/glue-grammar | 3 +
src/test/resources/decoder/n-ary/gold.scores | 2 +
src/test/resources/decoder/n-ary/grammar | 9 +
src/test/resources/decoder/n-ary/input.txt | 2 +
src/test/resources/decoder/n-ary/joshua.config | 22 +
src/test/resources/decoder/n-ary/lm.gz | Bin 0 -> 2466496 bytes
src/test/resources/decoder/n-ary/output.bleu | 0
src/test/resources/decoder/n-ary/output.gold | 2 +
src/test/resources/decoder/n-ary/test.sh | 33 +
src/test/resources/decoder/n-ary/weights | 6 +
.../decoder/num_translation_options/README | 1 +
.../num_translation_options/glue-grammar | 3 +
.../decoder/num_translation_options/grammar.gz | Bin 0 -> 119 bytes
.../grammar.packed/encoding | Bin 0 -> 32 bytes
.../grammar.packed/slice_00000.features | Bin 0 -> 43 bytes
.../grammar.packed/slice_00000.source | Bin 0 -> 132 bytes
.../grammar.packed/slice_00000.target | Bin 0 -> 120 bytes
.../grammar.packed/slice_00000.target.lookup | Bin 0 -> 32 bytes
.../grammar.packed/vocabulary | Bin 0 -> 144 bytes
.../decoder/num_translation_options/input | 1 +
.../num_translation_options/joshua.config | 30 +
.../joshua.config.packed | 30 +
.../decoder/num_translation_options/lm.gz | Bin 0 -> 2466496 bytes
.../decoder/num_translation_options/output.gold | 12 +
.../decoder/num_translation_options/test.sh | 17 +
src/test/resources/decoder/oov-list/config | 29 +
.../resources/decoder/oov-list/glue-grammar | 3 +
src/test/resources/decoder/oov-list/grammar | 11 +
src/test/resources/decoder/oov-list/input.txt | 3 +
src/test/resources/decoder/oov-list/output.gold | 3 +
src/test/resources/decoder/oov-list/test.sh | 30 +
.../resources/decoder/phrase/constrained/config | 29 +
.../decoder/phrase/constrained/corpus.es | 1 +
.../decoder/phrase/constrained/glue.grammar | 3 +
.../decoder/phrase/constrained/output.gold | 5 +
.../decoder/phrase/constrained/test.sh | 32 +
src/test/resources/decoder/phrase/decode/config | 29 +
.../decoder/phrase/decode/config.packed | 29 +
.../resources/decoder/phrase/decode/corpus.es | 1 +
.../resources/decoder/phrase/decode/lm.1.gz | Bin 0 -> 2235 bytes
.../resources/decoder/phrase/decode/output.gold | 1 +
.../resources/decoder/phrase/decode/rules.1.gz | Bin 0 -> 2998042 bytes
.../decoder/phrase/decode/rules.packed/config | 1 +
.../decoder/phrase/decode/rules.packed/encoding | Bin 0 -> 87 bytes
.../decode/rules.packed/slice_00000.features | Bin 0 -> 4128858 bytes
.../decode/rules.packed/slice_00000.source | Bin 0 -> 1982228 bytes
.../decode/rules.packed/slice_00000.target | Bin 0 -> 1463856 bytes
.../rules.packed/slice_00000.target.lookup | Bin 0 -> 28 bytes
.../phrase/decode/rules.packed/vocabulary | Bin 0 -> 169225 bytes
.../decoder/phrase/decode/test-packed.sh | 32 +
.../resources/decoder/phrase/decode/test.sh | 17 +
.../decoder/phrase/include-align-index/README | 2 +
.../decoder/phrase/include-align-index/config | 29 +
.../phrase/include-align-index/corpus.es | 1 +
.../decoder/phrase/include-align-index/lm.1.gz | Bin 0 -> 2235 bytes
.../decoder/phrase/include-align-index/log | 50 +
.../decoder/phrase/include-align-index/output | 1 +
.../phrase/include-align-index/output.gold | 1 +
.../phrase/include-align-index/rules.1.gz | Bin 0 -> 2998042 bytes
.../decoder/phrase/include-align-index/test.sh | 17 +
.../decoder/phrase/unique-hypotheses/README | 1 +
.../decoder/phrase/unique-hypotheses/corpus.es | 1 +
.../phrase/unique-hypotheses/joshua.config | 23 +
.../decoder/phrase/unique-hypotheses/lm.1.gz | 1 +
.../phrase/unique-hypotheses/output.gold | 300 +
.../decoder/phrase/unique-hypotheses/rules.1.gz | 1 +
.../decoder/phrase/unique-hypotheses/test.sh | 32 +
.../regexp-grammar-both-rule-types/.gitignore | 2 +
.../regexp-grammar-both-rule-types/README | 16 +
.../regexp-grammar-both-rule-types/config | 9 +
.../regexp-grammar-both-rule-types/glue-grammar | 3 +
.../regexp-grammar-both-rule-types/input | 5 +
.../regexp-grammar-both-rule-types/output.gold | 12 +
.../regexp-grammar | 12 +
.../regexp-grammar-both-rule-types/test.sh | 29 +
.../regexp-grammar-both-rule-types/weights | 4 +
.../resources/decoder/regexp-grammar/.gitignore | 2 +
.../resources/decoder/regexp-grammar/README | 10 +
.../resources/decoder/regexp-grammar/config | 11 +
.../decoder/regexp-grammar/glue-grammar | 3 +
src/test/resources/decoder/regexp-grammar/input | 4 +
.../decoder/regexp-grammar/output.gold | 4 +
.../decoder/regexp-grammar/regexp-grammar | 6 +
.../resources/decoder/regexp-grammar/test.sh | 29 +
.../resources/decoder/regexp-grammar/weights | 5 +
.../resources/decoder/rescoring/glue-grammar | 3 +
src/test/resources/decoder/rescoring/grammar.gz | Bin 0 -> 177 bytes
src/test/resources/decoder/rescoring/input.txt | 2 +
.../resources/decoder/rescoring/joshua.config | 31 +
.../resources/decoder/rescoring/output.gold | 12 +
src/test/resources/decoder/rescoring/test.sh | 30 +
src/test/resources/decoder/segment-oovs/config | 41 +
.../resources/decoder/segment-oovs/input.txt | 1 +
.../decoder/segment-oovs/output.expected | 82 +
src/test/resources/decoder/segment-oovs/test.sh | 31 +
.../decoder/source-annotations/grammar | 5 +
.../decoder/source-annotations/grammar.glue | 3 +
.../decoder/source-annotations/input.txt | 1 +
.../decoder/source-annotations/joshua.config | 140 +
.../decoder/source-annotations/lm.kenlm | Bin 0 -> 25355958 bytes
.../decoder/source-annotations/output.gold | 2 +
.../decoder/source-annotations/test.sh | 36 +
.../resources/decoder/target-bigram/out.gold | 3 +
.../resources/decoder/target-bigram/test.sh | 32 +
src/test/resources/decoder/target-bigram/vocab | 4 +
src/test/resources/decoder/too-long/output.gold | 4 +
src/test/resources/decoder/too-long/test.sh | 36 +
.../decoder/tree-output/fragment-map.txt | 2 +
.../resources/decoder/tree-output/glue-grammar | 6 +
.../resources/decoder/tree-output/grammar.gz | Bin 0 -> 134 bytes
src/test/resources/decoder/tree-output/input | 5 +
.../resources/decoder/tree-output/joshua.config | 45 +
src/test/resources/decoder/tree-output/lm.gz | Bin 0 -> 2466496 bytes
.../resources/decoder/tree-output/output.gold | 5 +
src/test/resources/decoder/tree-output/test.sh | 30 +
.../resources/grammar/sparse-features/grammar | 1 +
.../grammar/sparse-features/grammar.glue | 3 +
.../sparse-features/grammar.packed/encoding | Bin 0 -> 118 bytes
.../grammar.packed/slice_00000.features | Bin 0 -> 18 bytes
.../grammar.packed/slice_00000.source | Bin 0 -> 52 bytes
.../grammar.packed/slice_00000.target | Bin 0 -> 24 bytes
.../grammar.packed/slice_00000.target.lookup | Bin 0 -> 16 bytes
.../sparse-features/grammar.packed/vocabulary | Bin 0 -> 104 bytes
.../sparse-features/joshua-packed.config | 12 +
.../grammar/sparse-features/joshua.config | 12 +
.../grammar/sparse-features/output.gold | 1 +
.../grammar/sparse-features/test-packed.sh | 32 +
.../resources/grammar/sparse-features/test.sh | 32 +
src/test/resources/joshua/README.broken | 1 +
src/test/resources/lattice-short/README | 3 +
src/test/resources/lattice-short/glue-grammar | 3 +
src/test/resources/lattice-short/grammar.test | 3 +
src/test/resources/lattice-short/input | 5 +
src/test/resources/lattice-short/joshua.config | 39 +
.../resources/lattice-short/output.expected | 18 +
src/test/resources/lattice-short/test.lm | 113 +
src/test/resources/lattice-short/test.sh | 31 +
src/test/resources/lattice/.gitignore | 3 +
src/test/resources/lattice/README | 4 +
src/test/resources/lattice/glue-grammar | 3 +
src/test/resources/lattice/grammar.test | 204 +
src/test/resources/lattice/joshua.config | 47 +
src/test/resources/lattice/output.expected | 33 +
src/test/resources/lattice/test-lattice.pdf | Bin 0 -> 10943 bytes
src/test/resources/lattice/test.lm | 113 +
src/test/resources/lattice/test.plf | 4 +
src/test/resources/lattice/test.sh | 37 +
src/test/resources/lm/berkeley/lm | 16 +
src/test/resources/lm/berkeley/lm.berkeleylm | Bin 0 -> 4294 bytes
src/test/resources/lm/berkeley/lm.berkeleylm.gz | Bin 0 -> 1786 bytes
src/test/resources/lm/berkeley/lm.gz | Bin 0 -> 162 bytes
src/test/resources/lm/berkeley/output.gold | 4 +
src/test/resources/lm/berkeley/test.sh | 30 +
src/test/resources/packed-grammar/.gitignore | 8 +
src/test/resources/packed-grammar/README | 2 +
src/test/resources/packed-grammar/grammar.gz | Bin 0 -> 576901 bytes
src/test/resources/packed-grammar/input.bn | 100 +
src/test/resources/packed-grammar/joshua.config | 46 +
src/test/resources/packed-grammar/lm.gz | Bin 0 -> 2466496 bytes
src/test/resources/packed-grammar/output.gold | 100 +
.../resources/packed-grammar/reference.en.0 | 100 +
.../resources/packed-grammar/reference.en.1 | 100 +
.../resources/packed-grammar/reference.en.2 | 100 +
.../resources/packed-grammar/reference.en.3 | 100 +
.../resources/packed-grammar/test-multiple.sh | 31 +
src/test/resources/packed-grammar/test.sh | 38 +
src/test/resources/parser/grammar | 11 +
src/test/resources/parser/grammar.glue | 1 +
src/test/resources/parser/input | 4 +
src/test/resources/parser/output.gold | 4 +
src/test/resources/parser/parse.config | 18 +
src/test/resources/parser/test.sh | 29 +
src/test/resources/parser/weights | 4 +
src/test/resources/pipeline/.gitignore | 2 +
src/test/resources/pipeline/Makefile | 10 +
src/test/resources/pipeline/final-bleu.gold | 1 +
src/test/resources/pipeline/input/devtest.en.0 | 100 +
src/test/resources/pipeline/input/devtest.en.1 | 100 +
src/test/resources/pipeline/input/devtest.en.2 | 100 +
src/test/resources/pipeline/input/devtest.en.3 | 100 +
src/test/resources/pipeline/input/devtest.ur | 100 +
src/test/resources/pipeline/input/train.en | 1000 +++
src/test/resources/pipeline/input/train.ur | 1000 +++
src/test/resources/pipeline/input/tune.en.0 | 100 +
src/test/resources/pipeline/input/tune.en.1 | 100 +
src/test/resources/pipeline/input/tune.en.2 | 100 +
src/test/resources/pipeline/input/tune.en.3 | 100 +
src/test/resources/pipeline/input/tune.ur | 100 +
src/test/resources/pipeline/test-ghkm.sh | 43 +
src/test/resources/pipeline/test.sh | 39 +
.../resources/prune-equivalent-translations.py | 47 +
src/test/resources/run-all-tests.sh | 55 +
src/test/resources/scripts/.gitignore | 1 +
src/test/resources/scripts/merge_lms_test.py | 53 +
.../resources/scripts/normalization/.gitignore | 2 +
.../scripts/normalization/data/train.en | 21 +
.../scripts/normalization/data/train.en.norm | 21 +
.../resources/scripts/normalization/test.sh | 29 +
src/test/resources/scripts/run_bundler_test.py | 378 ++
.../scripts/support/moses_grammar/input | 10 +
.../support/moses_grammar/output.expected | 10 +
.../scripts/support/moses_grammar/test.sh | 30 +
src/test/resources/server/http/expected | 15 +
src/test/resources/server/http/test.sh | 36 +
src/test/resources/server/tcp-text/expected | 9 +
src/test/resources/server/tcp-text/test.sh | 45 +
src/test/resources/testng.xml | 30 +
src/test/resources/thrax/.gitignore | 5 +
.../resources/thrax/extraction/input/thrax.conf | 71 +
.../resources/thrax/extraction/input/train.a | 100 +
.../resources/thrax/extraction/input/train.en | 100 +
.../resources/thrax/extraction/input/train.ps | 100 +
src/test/resources/thrax/extraction/test.sh | 36 +
.../resources/thrax/filtering/dev.hi-en.hi.1 | 1 +
src/test/resources/thrax/filtering/exact.gold | 993 +++
.../resources/thrax/filtering/exact.log.gold | 17 +
src/test/resources/thrax/filtering/fast.gold | 1087 ++++
.../resources/thrax/filtering/fast.log.gold | 17 +
src/test/resources/thrax/filtering/grammar.de | 4 +
.../thrax/filtering/grammar.filtered.gz | Bin 0 -> 134958 bytes
src/test/resources/thrax/filtering/input.de | 3 +
.../resources/thrax/filtering/loose.log.gold | 16 +
.../resources/thrax/filtering/test-exact.sh | 34 +
src/test/resources/thrax/filtering/test-fast.sh | 34 +
.../resources/thrax/filtering/test-loose.sh | 34 +
test/bn-en/hiero/.gitignore | 4 -
test/bn-en/hiero/class.map | 5140 ----------------
test/bn-en/hiero/class_lm_2gram.gz | Bin 18052 -> 0 bytes
test/bn-en/hiero/class_lm_9gram.gz | Bin 12733137 -> 0 bytes
test/bn-en/hiero/glue-grammar | 3 -
test/bn-en/hiero/grammar.gz | Bin 518164 -> 0 bytes
test/bn-en/hiero/input.bn | 100 -
test/bn-en/hiero/joshua-berkeleylm.config | 46 -
test/bn-en/hiero/joshua-classlm.config | 51 -
test/bn-en/hiero/joshua.config | 50 -
test/bn-en/hiero/lm.gz | Bin 2466496 -> 0 bytes
test/bn-en/hiero/output-classlm.gold | 678 ---
test/bn-en/hiero/output.gold | 805 ---
test/bn-en/hiero/output.gold.bleu | 14 -
test/bn-en/hiero/output.scores.berkeleylm.gold | 100 -
test/bn-en/hiero/output.scores.gold | 805 ---
test/bn-en/hiero/reference.en.0 | 100 -
test/bn-en/hiero/reference.en.1 | 100 -
test/bn-en/hiero/reference.en.2 | 100 -
test/bn-en/hiero/reference.en.3 | 100 -
test/bn-en/hiero/test-berkeleylm.sh | 33 -
test/bn-en/hiero/test-classlm.sh | 32 -
test/bn-en/hiero/test-filter.sh | 35 -
test/bn-en/hiero/test.sh | 35 -
test/bn-en/hiero/topN.pl | 18 -
test/bn-en/packed/.gitignore | 3 -
test/bn-en/packed/grammar.glue | 5673 ------------------
test/bn-en/packed/grammar.gz | Bin 3540984 -> 0 bytes
test/bn-en/packed/grammar.packed/encoding | Bin 767 -> 0 bytes
.../packed/grammar.packed/slice_00000.features | Bin 4631480 -> 0 bytes
.../packed/grammar.packed/slice_00000.source | Bin 4240012 -> 0 bytes
.../packed/grammar.packed/slice_00000.target | Bin 162776 -> 0 bytes
.../grammar.packed/slice_00000.target.lookup | Bin 56 -> 0 bytes
test/bn-en/packed/grammar.packed/vocabulary | Bin 136522 -> 0 bytes
test/bn-en/packed/input.bn | 100 -
test/bn-en/packed/joshua.config | 47 -
test/bn-en/packed/lm.gz | Bin 2466496 -> 0 bytes
test/bn-en/packed/output.gold | 862 ---
test/bn-en/packed/output.scores.gold | 862 ---
test/bn-en/packed/reference.en.0 | 100 -
test/bn-en/packed/reference.en.1 | 100 -
test/bn-en/packed/reference.en.2 | 100 -
test/bn-en/packed/reference.en.3 | 100 -
test/bn-en/packed/reference.en.all | 400 --
test/bn-en/packed/test.sh | 20 -
test/bn-en/samt/grammar.glue | 5673 ------------------
test/bn-en/samt/grammar.gz | Bin 3847934 -> 0 bytes
test/bn-en/samt/input.bn | 100 -
test/bn-en/samt/joshua.config | 47 -
test/bn-en/samt/lm.gz | Bin 2466496 -> 0 bytes
test/bn-en/samt/output.gold | 0
test/bn-en/samt/output.gold.bleu | 14 -
test/bn-en/samt/output.scores.gold | 862 ---
test/bn-en/samt/reference.en.0 | 100 -
test/bn-en/samt/reference.en.1 | 100 -
test/bn-en/samt/reference.en.2 | 100 -
test/bn-en/samt/reference.en.3 | 100 -
test/bn-en/samt/test.sh | 35 -
test/decoder/constrained/.gitignore | 4 -
test/decoder/constrained/glue-grammar | 3 -
test/decoder/constrained/gold.scores | 27 -
test/decoder/constrained/grammar.gz | Bin 518164 -> 0 bytes
test/decoder/constrained/input.bn | 8 -
test/decoder/constrained/joshua.config | 45 -
test/decoder/constrained/lm.gz | Bin 2466496 -> 0 bytes
test/decoder/constrained/output.bleu | 0
test/decoder/constrained/output.gold | 30 -
test/decoder/constrained/test.sh | 30 -
test/decoder/constrained/weights | 22 -
test/decoder/denormalization/input.txt | 1 -
test/decoder/denormalization/output.expected | 1 -
test/decoder/denormalization/test.sh | 30 -
test/decoder/dont-crash/input | 5 -
test/decoder/dont-crash/output.gold | 1 -
test/decoder/dont-crash/test.sh | 29 -
test/decoder/empty-test/.gitignore | 3 -
test/decoder/empty-test/input | 1 -
test/decoder/empty-test/output.gold | 1 -
test/decoder/empty-test/test.sh | 29 -
test/decoder/fragmentlm/fragments.txt | 7 -
test/decoder/fragmentlm/glue | 1 -
test/decoder/fragmentlm/grammar | 4 -
test/decoder/fragmentlm/input | 1 -
test/decoder/fragmentlm/joshua.config | 109 -
test/decoder/fragmentlm/mapping.txt | 4 -
test/decoder/fragmentlm/test.sh | 30 -
test/decoder/k-best-extraction/glue-grammar | 3 -
test/decoder/k-best-extraction/grammar | 25 -
test/decoder/k-best-extraction/input.txt | 1 -
test/decoder/k-best-extraction/joshua.config | 27 -
test/decoder/k-best-extraction/lm.gz | Bin 2466496 -> 0 bytes
test/decoder/k-best-extraction/output.gold | 3126 ----------
.../k-best-extraction/output.scores.gold | 3126 ----------
test/decoder/k-best-extraction/test.sh | 33 -
test/decoder/left-state/glue-grammar | 3 -
test/decoder/left-state/grammar.gz | Bin 518164 -> 0 bytes
test/decoder/left-state/input.bn | 2 -
test/decoder/left-state/joshua.config | 44 -
test/decoder/left-state/lm.gz | Bin 2466496 -> 0 bytes
test/decoder/left-state/output.gold | 600 --
test/decoder/left-state/output.scores.gold | 600 --
test/decoder/left-state/test.sh | 33 -
test/decoder/lowercaser/config | 140 -
test/decoder/lowercaser/grammar.glue | 4 -
test/decoder/lowercaser/grammar.test | 1 -
test/decoder/lowercaser/output.gold | 5 -
test/decoder/lowercaser/test.sh | 40 -
test/decoder/moses-compat/n-best.txt | 0
test/decoder/moses-compat/output.expected | 6 -
test/decoder/moses-compat/test.sh | 40 -
test/decoder/n-ary/glue-grammar | 3 -
test/decoder/n-ary/gold.scores | 2 -
test/decoder/n-ary/grammar | 9 -
test/decoder/n-ary/input.txt | 2 -
test/decoder/n-ary/joshua.config | 22 -
test/decoder/n-ary/lm.gz | Bin 2466496 -> 0 bytes
test/decoder/n-ary/output.bleu | 0
test/decoder/n-ary/output.gold | 2 -
test/decoder/n-ary/test.sh | 33 -
test/decoder/n-ary/weights | 6 -
test/decoder/num_translation_options/README | 1 -
.../num_translation_options/glue-grammar | 3 -
test/decoder/num_translation_options/grammar.gz | Bin 119 -> 0 bytes
.../grammar.packed/encoding | Bin 32 -> 0 bytes
.../grammar.packed/slice_00000.features | Bin 43 -> 0 bytes
.../grammar.packed/slice_00000.source | Bin 132 -> 0 bytes
.../grammar.packed/slice_00000.target | Bin 120 -> 0 bytes
.../grammar.packed/slice_00000.target.lookup | Bin 32 -> 0 bytes
.../grammar.packed/vocabulary | Bin 144 -> 0 bytes
test/decoder/num_translation_options/input | 1 -
.../num_translation_options/joshua.config | 30 -
.../joshua.config.packed | 30 -
test/decoder/num_translation_options/lm.gz | Bin 2466496 -> 0 bytes
.../decoder/num_translation_options/output.gold | 12 -
test/decoder/num_translation_options/test.sh | 17 -
test/decoder/oov-list/config | 29 -
test/decoder/oov-list/glue-grammar | 3 -
test/decoder/oov-list/grammar | 11 -
test/decoder/oov-list/input.txt | 3 -
test/decoder/oov-list/output.gold | 3 -
test/decoder/oov-list/test.sh | 30 -
test/decoder/phrase/constrained/config | 29 -
test/decoder/phrase/constrained/corpus.es | 1 -
test/decoder/phrase/constrained/glue.grammar | 3 -
test/decoder/phrase/constrained/output.gold | 5 -
test/decoder/phrase/constrained/test.sh | 32 -
test/decoder/phrase/decode/config | 29 -
test/decoder/phrase/decode/config.packed | 29 -
test/decoder/phrase/decode/corpus.es | 1 -
test/decoder/phrase/decode/lm.1.gz | Bin 2235 -> 0 bytes
test/decoder/phrase/decode/output.gold | 1 -
test/decoder/phrase/decode/rules.1.gz | Bin 2998042 -> 0 bytes
test/decoder/phrase/decode/rules.packed/config | 1 -
.../decoder/phrase/decode/rules.packed/encoding | Bin 87 -> 0 bytes
.../decode/rules.packed/slice_00000.features | Bin 4128858 -> 0 bytes
.../decode/rules.packed/slice_00000.source | Bin 1982228 -> 0 bytes
.../decode/rules.packed/slice_00000.target | Bin 1463856 -> 0 bytes
.../rules.packed/slice_00000.target.lookup | Bin 28 -> 0 bytes
.../phrase/decode/rules.packed/vocabulary | Bin 169225 -> 0 bytes
test/decoder/phrase/decode/test-packed.sh | 32 -
test/decoder/phrase/decode/test.sh | 17 -
test/decoder/phrase/include-align-index/README | 2 -
test/decoder/phrase/include-align-index/config | 29 -
.../phrase/include-align-index/corpus.es | 1 -
test/decoder/phrase/include-align-index/lm.1.gz | Bin 2235 -> 0 bytes
test/decoder/phrase/include-align-index/log | 50 -
test/decoder/phrase/include-align-index/output | 1 -
.../phrase/include-align-index/output.gold | 1 -
.../phrase/include-align-index/rules.1.gz | Bin 2998042 -> 0 bytes
test/decoder/phrase/include-align-index/test.sh | 17 -
test/decoder/phrase/unique-hypotheses/README | 1 -
test/decoder/phrase/unique-hypotheses/corpus.es | 1 -
.../phrase/unique-hypotheses/joshua.config | 23 -
test/decoder/phrase/unique-hypotheses/lm.1.gz | 1 -
.../phrase/unique-hypotheses/output.gold | 300 -
.../decoder/phrase/unique-hypotheses/rules.1.gz | 1 -
test/decoder/phrase/unique-hypotheses/test.sh | 32 -
.../regexp-grammar-both-rule-types/.gitignore | 2 -
.../regexp-grammar-both-rule-types/README | 16 -
.../regexp-grammar-both-rule-types/config | 9 -
.../regexp-grammar-both-rule-types/glue-grammar | 3 -
.../regexp-grammar-both-rule-types/input | 5 -
.../regexp-grammar-both-rule-types/output.gold | 12 -
.../regexp-grammar | 12 -
.../regexp-grammar-both-rule-types/test.sh | 29 -
.../regexp-grammar-both-rule-types/weights | 4 -
test/decoder/regexp-grammar/.gitignore | 2 -
test/decoder/regexp-grammar/README | 10 -
test/decoder/regexp-grammar/config | 11 -
test/decoder/regexp-grammar/glue-grammar | 3 -
test/decoder/regexp-grammar/input | 4 -
test/decoder/regexp-grammar/output.gold | 4 -
test/decoder/regexp-grammar/regexp-grammar | 6 -
test/decoder/regexp-grammar/test.sh | 29 -
test/decoder/regexp-grammar/weights | 5 -
test/decoder/rescoring/glue-grammar | 3 -
test/decoder/rescoring/grammar.gz | Bin 177 -> 0 bytes
test/decoder/rescoring/input.txt | 2 -
test/decoder/rescoring/joshua.config | 31 -
test/decoder/rescoring/output.gold | 12 -
test/decoder/rescoring/test.sh | 30 -
test/decoder/segment-oovs/config | 41 -
test/decoder/segment-oovs/input.txt | 1 -
test/decoder/segment-oovs/output.expected | 82 -
test/decoder/segment-oovs/test.sh | 31 -
test/decoder/source-annotations/grammar | 5 -
test/decoder/source-annotations/grammar.glue | 3 -
test/decoder/source-annotations/input.txt | 1 -
test/decoder/source-annotations/joshua.config | 140 -
test/decoder/source-annotations/lm.kenlm | Bin 25355958 -> 0 bytes
test/decoder/source-annotations/output.gold | 2 -
test/decoder/source-annotations/test.sh | 36 -
test/decoder/target-bigram/out.gold | 3 -
test/decoder/target-bigram/test.sh | 32 -
test/decoder/target-bigram/vocab | 4 -
test/decoder/too-long/output.gold | 4 -
test/decoder/too-long/test.sh | 36 -
test/decoder/tree-output/fragment-map.txt | 2 -
test/decoder/tree-output/glue-grammar | 6 -
test/decoder/tree-output/grammar.gz | Bin 134 -> 0 bytes
test/decoder/tree-output/input | 5 -
test/decoder/tree-output/joshua.config | 45 -
test/decoder/tree-output/lm.gz | Bin 2466496 -> 0 bytes
test/decoder/tree-output/output.gold | 5 -
test/decoder/tree-output/test.sh | 30 -
test/grammar/sparse-features/grammar | 1 -
test/grammar/sparse-features/grammar.glue | 3 -
.../sparse-features/grammar.packed/encoding | Bin 118 -> 0 bytes
.../grammar.packed/slice_00000.features | Bin 18 -> 0 bytes
.../grammar.packed/slice_00000.source | Bin 52 -> 0 bytes
.../grammar.packed/slice_00000.target | Bin 24 -> 0 bytes
.../grammar.packed/slice_00000.target.lookup | Bin 16 -> 0 bytes
.../sparse-features/grammar.packed/vocabulary | Bin 104 -> 0 bytes
.../sparse-features/joshua-packed.config | 12 -
test/grammar/sparse-features/joshua.config | 12 -
test/grammar/sparse-features/output.gold | 1 -
test/grammar/sparse-features/test-packed.sh | 32 -
test/grammar/sparse-features/test.sh | 32 -
test/joshua/README.broken | 1 -
test/lattice-short/README | 3 -
test/lattice-short/glue-grammar | 3 -
test/lattice-short/grammar.test | 3 -
test/lattice-short/input | 5 -
test/lattice-short/joshua.config | 39 -
test/lattice-short/output.expected | 18 -
test/lattice-short/test.lm | 113 -
test/lattice-short/test.sh | 31 -
test/lattice/.gitignore | 3 -
test/lattice/README | 4 -
test/lattice/glue-grammar | 3 -
test/lattice/grammar.test | 204 -
test/lattice/joshua.config | 47 -
test/lattice/output.expected | 33 -
test/lattice/test-lattice.pdf | Bin 10943 -> 0 bytes
test/lattice/test.lm | 113 -
test/lattice/test.plf | 4 -
test/lattice/test.sh | 37 -
test/lm/berkeley/lm | 16 -
test/lm/berkeley/lm.berkeleylm | Bin 4294 -> 0 bytes
test/lm/berkeley/lm.berkeleylm.gz | Bin 1786 -> 0 bytes
test/lm/berkeley/lm.gz | Bin 162 -> 0 bytes
test/lm/berkeley/output.gold | 4 -
test/lm/berkeley/test.sh | 30 -
test/packed-grammar/.gitignore | 8 -
test/packed-grammar/README | 2 -
test/packed-grammar/grammar.gz | Bin 576901 -> 0 bytes
test/packed-grammar/input.bn | 100 -
test/packed-grammar/joshua.config | 46 -
test/packed-grammar/lm.gz | Bin 2466496 -> 0 bytes
test/packed-grammar/output.gold | 100 -
test/packed-grammar/reference.en.0 | 100 -
test/packed-grammar/reference.en.1 | 100 -
test/packed-grammar/reference.en.2 | 100 -
test/packed-grammar/reference.en.3 | 100 -
test/packed-grammar/test-multiple.sh | 31 -
test/packed-grammar/test.sh | 38 -
test/parser/grammar | 11 -
test/parser/grammar.glue | 1 -
test/parser/input | 4 -
test/parser/output.gold | 4 -
test/parser/parse.config | 18 -
test/parser/test.sh | 29 -
test/parser/weights | 4 -
test/pipeline/.gitignore | 2 -
test/pipeline/Makefile | 10 -
test/pipeline/final-bleu.gold | 1 -
test/pipeline/input/devtest.en.0 | 100 -
test/pipeline/input/devtest.en.1 | 100 -
test/pipeline/input/devtest.en.2 | 100 -
test/pipeline/input/devtest.en.3 | 100 -
test/pipeline/input/devtest.ur | 100 -
test/pipeline/input/train.en | 1000 ---
test/pipeline/input/train.ur | 1000 ---
test/pipeline/input/tune.en.0 | 100 -
test/pipeline/input/tune.en.1 | 100 -
test/pipeline/input/tune.en.2 | 100 -
test/pipeline/input/tune.en.3 | 100 -
test/pipeline/input/tune.ur | 100 -
test/pipeline/test-ghkm.sh | 43 -
test/pipeline/test.sh | 39 -
test/prune-equivalent-translations.py | 47 -
test/run-all-tests.sh | 55 -
test/scripts/.gitignore | 1 -
test/scripts/merge_lms_test.py | 53 -
test/scripts/normalization/.gitignore | 2 -
test/scripts/normalization/data/train.en | 21 -
test/scripts/normalization/data/train.en.norm | 21 -
test/scripts/normalization/test.sh | 29 -
test/scripts/run_bundler_test.py | 378 --
test/scripts/support/moses_grammar/input | 10 -
.../support/moses_grammar/output.expected | 10 -
test/scripts/support/moses_grammar/test.sh | 30 -
test/server/http/expected | 15 -
test/server/http/test.sh | 36 -
test/server/tcp-text/expected | 9 -
test/server/tcp-text/test.sh | 45 -
test/testng.xml | 13 -
test/thrax/.gitignore | 5 -
test/thrax/extraction/input/thrax.conf | 71 -
test/thrax/extraction/input/train.a | 100 -
test/thrax/extraction/input/train.en | 100 -
test/thrax/extraction/input/train.ps | 100 -
test/thrax/extraction/test.sh | 36 -
test/thrax/filtering/dev.hi-en.hi.1 | 1 -
test/thrax/filtering/exact.gold | 993 ---
test/thrax/filtering/exact.log.gold | 17 -
test/thrax/filtering/fast.gold | 1087 ----
test/thrax/filtering/fast.log.gold | 17 -
test/thrax/filtering/grammar.de | 4 -
test/thrax/filtering/grammar.filtered.gz | Bin 134958 -> 0 bytes
test/thrax/filtering/input.de | 3 -
test/thrax/filtering/loose.log.gold | 16 -
test/thrax/filtering/test-exact.sh | 34 -
test/thrax/filtering/test-fast.sh | 34 -
test/thrax/filtering/test-loose.sh | 34 -
1006 files changed, 44235 insertions(+), 69597 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/scripts/training/pipeline.pl
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/corpus/Vocabulary.java
----------------------------------------------------------------------
diff --cc src/main/java/org/apache/joshua/corpus/Vocabulary.java
index 42ce482,582efc6..2bcc447
--- a/src/main/java/org/apache/joshua/corpus/Vocabulary.java
+++ b/src/main/java/org/apache/joshua/corpus/Vocabulary.java
@@@ -45,9 -46,8 +48,9 @@@ import org.slf4j.LoggerFactory
* @author Juri Ganitkevitch
*/
- public class Vocabulary {
+ public class Vocabulary implements Externalizable {
+ private static final Logger LOG = LoggerFactory.getLogger(Vocabulary.class);
private final static ArrayList<NGramLanguageModel> LMs = new ArrayList<>();
private static List<String> idToString;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/decoder/ArgsParser.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/decoder/Decoder.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/decoder/DecoderThread.java
----------------------------------------------------------------------
diff --cc src/main/java/org/apache/joshua/decoder/DecoderThread.java
index 486df4b,0f80335..d6f5233
--- a/src/main/java/org/apache/joshua/decoder/DecoderThread.java
+++ b/src/main/java/org/apache/joshua/decoder/DecoderThread.java
@@@ -104,10 -103,10 +105,10 @@@ public class DecoderThread extends Thre
// skip blank sentences
if (sentence.isEmpty()) {
- Decoder.LOG(1, String.format("Translation %d: Translation took 0 seconds", sentence.id()));
+ LOG.info("Translation {}: Translation took 0 seconds", sentence.id());
return new Translation(sentence, null, featureFunctions, joshuaConfiguration);
}
-
+
long startTime = System.currentTimeMillis();
int numGrammars = allGrammars.size();
@@@ -137,12 -136,12 +138,12 @@@
joshuaConfiguration.goal_symbol, joshuaConfiguration);
hypergraph = (joshuaConfiguration.use_dot_chart)
- ? chart.expand()
- : chart.expandSansDotChart();
+ ? chart.expand()
+ : chart.expandSansDotChart();
}
-
+
} catch (java.lang.OutOfMemoryError e) {
- Decoder.LOG(1, String.format("Input %d: out of memory", sentence.id()));
+ LOG.error("Input {}: out of memory", sentence.id());
hypergraph = null;
}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/decoder/JoshuaConfiguration.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/decoder/JoshuaDecoder.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/decoder/NbestMinRiskReranker.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/decoder/Translation.java
----------------------------------------------------------------------
diff --cc src/main/java/org/apache/joshua/decoder/Translation.java
index 9ae1aaa,0ee1f08..ab37814
--- a/src/main/java/org/apache/joshua/decoder/Translation.java
+++ b/src/main/java/org/apache/joshua/decoder/Translation.java
@@@ -84,14 -81,15 +84,14 @@@ public class Translation
// We must put this weight as zero, otherwise we get an error when we try to retrieve it
// without checking
Decoder.weights.increment("BLEU", 0);
-
+
if (joshuaConfiguration.topN == 0) {
-
+
/* construct Viterbi output */
final String best = getViterbiString(hypergraph);
-
+
- Decoder.LOG(1, String.format("Translation %d: %.3f %s", source.id(), hypergraph.goalNode.getScore(),
- best));
+ LOG.info("Translation {}: {} {}", source.id(), hypergraph.goalNode.getScore(), best);
-
+
/*
* Setting topN to 0 turns off k-best extraction, in which case we need to parse through
* the output-string, with the understanding that we can only substitute variables for the
@@@ -131,26 -129,26 +131,26 @@@
}
float seconds = (float) (System.currentTimeMillis() - startTime) / 1000.0f;
- Decoder.LOG(1, String.format("Input %d: %d-best extraction took %.3f seconds", id(),
- joshuaConfiguration.topN, seconds));
+ LOG.info("Input {}: {}-best extraction took {} seconds", id(),
+ joshuaConfiguration.topN, seconds);
- } else {
-
- // Failed translations and blank lines get empty formatted outputs
- // @formatter:off
- String outputString = joshuaConfiguration.outputFormat
- .replace("%s", source.source())
- .replace("%e", "")
- .replace("%S", "")
- .replace("%t", "()")
- .replace("%i", Integer.toString(source.id()))
- .replace("%f", "")
- .replace("%c", "0.000");
- // @formatter:on
-
- out.write(outputString);
- out.newLine();
- }
+ } else {
+
+ // Failed translations and blank lines get empty formatted outputs
+ // @formatter:off
+ String outputString = joshuaConfiguration.outputFormat
+ .replace("%s", source.source())
+ .replace("%e", "")
+ .replace("%S", "")
+ .replace("%t", "()")
+ .replace("%i", Integer.toString(source.id()))
+ .replace("%f", "")
+ .replace("%c", "0.000");
+ // @formatter:on
+
+ out.write(outputString);
+ out.newLine();
+ }
out.flush();
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/decoder/chart_parser/Chart.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/decoder/chart_parser/ComputeNodeResult.java
----------------------------------------------------------------------
diff --cc src/main/java/org/apache/joshua/decoder/chart_parser/ComputeNodeResult.java
index 105565d,73c3cd0..e691da6
--- a/src/main/java/org/apache/joshua/decoder/chart_parser/ComputeNodeResult.java
+++ b/src/main/java/org/apache/joshua/decoder/chart_parser/ComputeNodeResult.java
@@@ -70,9 -73,11 +77,8 @@@ public class ComputeNodeResult
// The total Viterbi cost of this edge. This is the Viterbi cost of the tail nodes, plus
// whatever costs we incur applying this rule to create a new hyperedge.
float viterbiCost = 0.0f;
-
-
- if (Decoder.VERBOSE >= 4) {
- System.err.println("ComputeNodeResult():");
- System.err.println("-> RULE " + rule);
- }
+ LOG.debug("ComputeNodeResult():");
+ LOG.info("-> RULE {}", rule);
/*
* Here we sum the accumulated cost of each of the tail nodes. The total cost of the new
@@@ -106,19 -113,22 +112,18 @@@
DPState newState = feature.compute(rule, tailNodes, i, j, sourcePath, sentence, acc);
transitionCost += acc.getScore();
- if (Decoder.VERBOSE >= 4)
- System.err.println(String.format("-> FEATURE %s = %.3f * %.3f = %.3f",
- feature.getName(), acc.getScore() / Decoder.weights.getSparse(feature.getName()),
- Decoder.weights.getSparse(feature.getName()), acc.getScore()));
++
+ LOG.debug("FEATURE {} = {} * {} = {}", feature.getName(),
+ acc.getScore() / Decoder.weights.getSparse(feature.getName()),
+ Decoder.weights.getSparse(feature.getName()), acc.getScore());
if (feature.isStateful()) {
futureCostEstimate += feature.estimateFutureCost(rule, newState, sentence);
allDPStates.add(((StatefulFF)feature).getStateIndex(), newState);
}
}
-
-
viterbiCost += transitionCost;
-
- if (Decoder.VERBOSE >= 4)
- System.err.println(String.format("-> COST = %.3f", transitionCost));
-
+ LOG.debug("-> COST = {}", transitionCost);
-
// Set the final results.
this.pruningCostEstimate = viterbiCost + futureCostEstimate;
this.viterbiCost = viterbiCost;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/decoder/chart_parser/ManualConstraintsHandler.java
----------------------------------------------------------------------
diff --cc src/main/java/org/apache/joshua/decoder/chart_parser/ManualConstraintsHandler.java
index 99cbf7f,06a14ee..3b7c644
--- a/src/main/java/org/apache/joshua/decoder/chart_parser/ManualConstraintsHandler.java
+++ b/src/main/java/org/apache/joshua/decoder/chart_parser/ManualConstraintsHandler.java
@@@ -27,11 -29,9 +27,11 @@@ import org.apache.joshua.decoder.ff.tm.
import org.apache.joshua.decoder.ff.tm.Rule;
import org.apache.joshua.decoder.segment_file.ConstraintRule;
import org.apache.joshua.decoder.segment_file.ConstraintSpan;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
- * @author Zhifei Li, <zh...@gmail.com>
+ * @author Zhifei Li, zhifei.work@gmail.com
*/
public class ManualConstraintsHandler {
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/decoder/ff/StatefulFF.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/decoder/ff/fragmentlm/FragmentLMFF.java
----------------------------------------------------------------------
diff --cc src/main/java/org/apache/joshua/decoder/ff/fragmentlm/FragmentLMFF.java
index 232b2b9,f969396..b191c2f
--- a/src/main/java/org/apache/joshua/decoder/ff/fragmentlm/FragmentLMFF.java
+++ b/src/main/java/org/apache/joshua/decoder/ff/fragmentlm/FragmentLMFF.java
@@@ -36,37 -36,35 +36,37 @@@ import org.apache.joshua.decoder.ff.tm.
import org.apache.joshua.decoder.hypergraph.HGNode;
import org.apache.joshua.decoder.hypergraph.HyperEdge;
import org.apache.joshua.decoder.segment_file.Sentence;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
- * Feature function that reads in a list of language model fragments and matches them against the
+ * <p>Feature function that reads in a list of language model fragments and matches them against the
* hypergraph. This allows for language model fragment "glue" features, which fire when LM fragments
* (supplied as input) are assembled. These LM fragments are presumably useful in ensuring
- * grammaticality and can be independent of the translation model fragments.
+ * grammaticality and can be independent of the translation model fragments.</p>
*
- * Usage: in the Joshua Configuration file, put
+ * <p>Usage: in the Joshua Configuration file, put</p>
*
- * feature-function = FragmentLM -lm LM_FRAGMENTS_FILE -map RULE_FRAGMENTS_MAP_FILE
+ * <code>feature-function = FragmentLM -lm LM_FRAGMENTS_FILE -map RULE_FRAGMENTS_MAP_FILE</code>
*
- * LM_FRAGMENTS_FILE is a pointer to a file containing a list of fragments that it should look for.
- * The format of the file is one fragment per line in PTB format, e.g.:
+ * <p>LM_FRAGMENTS_FILE is a pointer to a file containing a list of fragments that it should look for.
+ * The format of the file is one fragment per line in PTB format, e.g.:</p>
*
- * (S NP (VP (VBD said) SBAR) (. .))
+ * <code>(S NP (VP (VBD said) SBAR) (. .))</code>
*
- * RULE_FRAGMENTS_MAP_FILE points to a file that maps fragments to the flattened SCFG rule format
+ * <p>RULE_FRAGMENTS_MAP_FILE points to a file that maps fragments to the flattened SCFG rule format
* that Joshua uses. This mapping is necessary because Joshua's rules have been flattened, meaning
* that their internal structure has been removed, yet this structure is needed for matching LM
- * fragments. The format of the file is
+ * fragments. The format of the file is</p>
*
- * FRAGMENT ||| RULE-TARGET-SIDE
+ * <code>FRAGMENT ||| RULE-TARGET-SIDE</code>
*
- * for example,
+ * <p>for example,</p>
*
- * (S (NP (DT the) (NN man)) VP .) ||| the man [VP,1] [.,2] (SBAR (IN that) (S (NP (PRP he)) (VP
- * (VBD was) (VB done)))) ||| that he was done (VP (VBD said) SBAR) ||| said SBAR
+ * <code>(S (NP (DT the) (NN man)) VP .) ||| the man [VP,1] [.,2] (SBAR (IN that) (S (NP (PRP he)) (VP
+ * (VBD was) (VB done)))) ||| that he was done (VP (VBD said) SBAR) ||| said SBAR</code>
*
- * @author Matt Post <po...@cs.jhu.edu>
+ * @author Matt Post post@cs.jhu.edu
*/
public class FragmentLMFF extends StatefulFF {
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/decoder/ff/fragmentlm/Tree.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/decoder/ff/lm/DefaultNGramLanguageModel.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/decoder/ff/lm/KenLM.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/decoder/ff/lm/LanguageModelFF.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/decoder/ff/lm/bloomfilter_lm/BloomFilterLanguageModel.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/decoder/ff/lm/buildin_lm/TrieLM.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/decoder/ff/tm/AbstractGrammar.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/decoder/ff/tm/GrammarReader.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/decoder/ff/tm/MonolingualRule.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/decoder/ff/tm/Rule.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/decoder/ff/tm/SentenceFilteredGrammar.java
----------------------------------------------------------------------
diff --cc src/main/java/org/apache/joshua/decoder/ff/tm/SentenceFilteredGrammar.java
index 42652ce,ce04e15..2362cfd
--- a/src/main/java/org/apache/joshua/decoder/ff/tm/SentenceFilteredGrammar.java
+++ b/src/main/java/org/apache/joshua/decoder/ff/tm/SentenceFilteredGrammar.java
@@@ -34,12 -32,9 +34,12 @@@ import org.slf4j.LoggerFactory
* trie, a subset of the original trie, that only contains trie paths that are reachable from
* traversals of the current sentence.
*
- * @author Matt Post <po...@cs.jhu.edu>
+ * @author Matt Post post@cs.jhu.edu
*/
public class SentenceFilteredGrammar extends MemoryBasedBatchGrammar {
+
+ private static final Logger LOG = LoggerFactory.getLogger(SentenceFilteredGrammar.class);
+
private AbstractGrammar baseGrammar;
private SentenceFilteredTrie filteredTrie;
private int[] tokens;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/decoder/ff/tm/hash_based/MemoryBasedBatchGrammar.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/decoder/ff/tm/packed/PackedGrammar.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/decoder/hypergraph/HyperGraph.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/decoder/phrase/Future.java
----------------------------------------------------------------------
diff --cc src/main/java/org/apache/joshua/decoder/phrase/Future.java
index 3cceb76,aa24390..0ece4a3
--- a/src/main/java/org/apache/joshua/decoder/phrase/Future.java
+++ b/src/main/java/org/apache/joshua/decoder/phrase/Future.java
@@@ -18,10 -18,8 +18,9 @@@
*/
package org.apache.joshua.decoder.phrase;
--import org.apache.joshua.decoder.Decoder;
import org.apache.joshua.util.ChartSpan;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class Future {
@@@ -74,18 -71,16 +74,18 @@@
}
}
}
-
+
- if (Decoder.VERBOSE >= 3) {
- for (int i = 1; i < chart.SentenceLength(); i++)
- for (int j = i + 1; j < chart.SentenceLength(); j++)
- System.err.println(String.format("future cost from %d to %d is %.3f", i-1, j-2, getEntry(i, j)));
+ if (LOG.isDebugEnabled()) {
+ for (int i = 1; i < chart.SentenceLength(); i++) {
+ for (int j = i + 1; j < chart.SentenceLength(); j++) {
+ LOG.debug("future cost from {} to {} is {}", i - 1, j - 2, getEntry(i, j));
+ }
+ }
}
}
-
+
public float Full() {
- // System.err.println("Future::Full(): " + Entry(1, sentlen));
+ // System.err.println("Future::Full(): " + Entry(1, sentlen));
return getEntry(1, sentlen);
}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/decoder/phrase/PhraseChart.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/decoder/phrase/Stack.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/decoder/phrase/Stacks.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/decoder/segment_file/Sentence.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/decoder/segment_file/Token.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/lattice/Lattice.java
----------------------------------------------------------------------
diff --cc src/main/java/org/apache/joshua/lattice/Lattice.java
index 7679cfd,d89c948..2332159
--- a/src/main/java/org/apache/joshua/lattice/Lattice.java
+++ b/src/main/java/org/apache/joshua/lattice/Lattice.java
@@@ -37,12 -37,11 +37,11 @@@ import org.slf4j.LoggerFactory
/**
* A lattice representation of a directed graph.
- *
+ *
* @author Lane Schwartz
- * @author Matt Post <po...@cs.jhu.edu>
+ * @author Matt Post post@cs.jhu.edu
* @since 2008-07-08
- *
+ *
- * @param Label Type of label associated with an arc.
*/
public class Lattice<Value> implements Iterable<Node<Value>> {
@@@ -71,8 -70,9 +70,9 @@@
* <p>
* The list of nodes must already be in topological order. If the list is not in topological
* order, the behavior of the lattice is not defined.
- *
+ *
* @param nodes A list of nodes which must be in topological order.
+ * @param config a populated {@link org.apache.joshua.decoder.JoshuaConfiguration}
*/
public Lattice(List<Node<Value>> nodes, JoshuaConfiguration config) {
this.nodes = nodes;
@@@ -90,8 -90,9 +90,9 @@@
/**
* Instantiates a lattice from a linear chain of values, i.e., a sentence.
- *
+ *
* @param linearChain a sequence of Value objects
+ * @param config a populated {@link org.apache.joshua.decoder.JoshuaConfiguration}
*/
public Lattice(Value[] linearChain, JoshuaConfiguration config) {
this.latticeHasAmbiguity = false;
@@@ -125,10 -126,9 +126,9 @@@
/**
* Computes the shortest distance between two nodes, which is used (perhaps among other places) in
* computing which rules can apply over which spans of the input
- *
+ *
- * @param tail
- * @param head
- * @return the distance, a positive number, or -1 if there is no path between the nodes
+ * @param arc an {@link org.apache.joshua.lattice.Arc} of values
+ * @return the shortest distance between two nodes
*/
public int distance(Arc<Value> arc) {
return this.getShortestPath(arc.getTail().getNumber(), arc.getHead().getNumber());
@@@ -140,8 -140,9 +140,9 @@@
/**
* Convenience method to get a lattice from a linear sequence of {@link Token} objects.
- *
+ *
- * @param linearChain
+ * @param source input string from which to create a {@link org.apache.joshua.lattice.Lattice}
+ * @param config a populated {@link org.apache.joshua.decoder.JoshuaConfiguration}
* @return Lattice representation of the linear chain.
*/
public static Lattice<Token> createTokenLatticeFromString(String source, JoshuaConfiguration config) {
@@@ -243,8 -244,9 +244,9 @@@
/**
* Constructs a lattice from a given string representation.
- *
+ *
* @param data String representation of a lattice.
+ * @param config a populated {@link org.apache.joshua.decoder.JoshuaConfiguration}
* @return A lattice that corresponds to the given string.
*/
public static Lattice<String> createStringLatticeFromString(String data, JoshuaConfiguration config) {
@@@ -443,10 -445,10 +445,10 @@@
/**
* Replaced the arc from node i to j with the supplied lattice. This is used to do OOV
* segmentation of words in a lattice.
- *
+ *
- * @param i
- * @param j
- * @param lattice
+ * @param i start node of arc
+ * @param j end node of arc
+ * @param newNodes new nodes used within the replacement operation
*/
public void insert(int i, int j, List<Node<Value>> newNodes) {
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/metrics/BLEU.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/server/ServerThread.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/server/TcpServer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/subsample/Subsampler.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/tools/GrammarPacker.java
----------------------------------------------------------------------
diff --cc src/main/java/org/apache/joshua/tools/GrammarPacker.java
index 93cb610,517b744..8bdeb3b
--- a/src/main/java/org/apache/joshua/tools/GrammarPacker.java
+++ b/src/main/java/org/apache/joshua/tools/GrammarPacker.java
@@@ -148,10 -147,10 +148,10 @@@ public class GrammarPacker
/**
* Executes the packing.
*
- * @throws IOException
+ * @throws IOException if there is an error reading the grammar
*/
public void pack() throws IOException {
- logger.info("Beginning exploration pass.");
+ LOG.info("Beginning exploration pass.");
LineReader grammar_reader = null;
LineReader alignment_reader = null;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/tools/LabelPhrases.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/tools/TestSetFilter.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/util/CompareGrammars.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/util/ExtractTopCand.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/util/FormatUtils.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9d6f84d3/src/main/java/org/apache/joshua/util/io/BinaryOut.java
----------------------------------------------------------------------