You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by mj...@apache.org on 2016/09/08 19:32:47 UTC
[3/3] incubator-joshua git commit: Minor changes (document,
file names)
Minor changes (document, file names)
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/ffd9562e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/ffd9562e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/ffd9562e
Branch: refs/heads/master
Commit: ffd9562ea8b0ca038e83ed2d135312509989c622
Parents: 938ab94
Author: Matt Post <po...@cs.jhu.edu>
Authored: Thu Sep 8 12:55:51 2016 -0400
Committer: Matt Post <po...@cs.jhu.edu>
Committed: Thu Sep 8 12:55:51 2016 -0400
----------------------------------------------------------------------
.../hypergraph/OutputStringExtractor.java | 22 +++++++--
.../apache/joshua/decoder/phrase/Stacks.java | 3 +-
.../decoder/phrase/include-align-index/log | 50 --------------------
.../decoder/phrase/include-align-index/output | 1 -
4 files changed, 19 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ffd9562e/src/main/java/org/apache/joshua/decoder/hypergraph/OutputStringExtractor.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/hypergraph/OutputStringExtractor.java b/src/main/java/org/apache/joshua/decoder/hypergraph/OutputStringExtractor.java
index 77b76a4..0c00023 100644
--- a/src/main/java/org/apache/joshua/decoder/hypergraph/OutputStringExtractor.java
+++ b/src/main/java/org/apache/joshua/decoder/hypergraph/OutputStringExtractor.java
@@ -37,6 +37,11 @@ public class OutputStringExtractor implements WalkerFunction, DerivationVisitor
private final Stack<OutputString> outputStringStack = new Stack<>();
private final boolean extractSource;
+ /* This comes from the WalkerFunction interface. It is applied at every HGNode in the
+ * hypergraph.
+ *
+ * @see org.apache.joshua.decoder.hypergraph.WalkerFunction#apply(org.apache.joshua.decoder.hypergraph.HGNode, int)
+ */
@Override
public void apply(HGNode node, int nodeIndex) {
apply(node.bestHyperedge.getRule(), nodeIndex);
@@ -47,11 +52,24 @@ public class OutputStringExtractor implements WalkerFunction, DerivationVisitor
* apply() for Viterbi extraction but using the edge from
* the Derivation state.
*/
+
+ /*
+ * (non-Javadoc)
+ * @see org.apache.joshua.decoder.hypergraph.KBestExtractor.DerivationVisitor#before(org.apache.joshua.decoder.hypergraph.KBestExtractor.DerivationState, int, int)
+ */
@Override
public void before(final DerivationState state, int level, int tailNodeIndex) {
apply(state.edge.getRule(), tailNodeIndex);
}
+ /* Nothing to do after the visit.
+ *
+ * (non-Javadoc)
+ * @see org.apache.joshua.decoder.hypergraph.KBestExtractor.DerivationVisitor#after(org.apache.joshua.decoder.hypergraph.KBestExtractor.DerivationState, int, int)
+ */
+ @Override
+ public void after(DerivationState state, int level, int tailNodeIndex) {}
+
private void apply(Rule rule, int nodeIndex) {
if (rule != null) {
final int[] words = extractSource ? rule.getFrench() : rule.getEnglish();
@@ -59,10 +77,6 @@ public class OutputStringExtractor implements WalkerFunction, DerivationVisitor
}
}
- /** Nothing to do */
- @Override
- public void after(DerivationState state, int level, int tailNodeIndex) {}
-
private static int getSourceNonTerminalPosition(final int[] words, int nonTerminalIndex) {
int nonTerminalsSeen = 0;
for (int i = 0; i < words.length; i++) {
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ffd9562e/src/main/java/org/apache/joshua/decoder/phrase/Stacks.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/phrase/Stacks.java b/src/main/java/org/apache/joshua/decoder/phrase/Stacks.java
index cfeaea2..d02ad1a 100644
--- a/src/main/java/org/apache/joshua/decoder/phrase/Stacks.java
+++ b/src/main/java/org/apache/joshua/decoder/phrase/Stacks.java
@@ -32,7 +32,7 @@ package org.apache.joshua.decoder.phrase;
* ensures that the coverage vector is consistent but the resulting hypergraph may not be projective,
* which is different from the CKY algorithm, which does produce projective derivations.
*
- * TODO Lattice decoding is not yet supported (March 2015).
+ * TODO Lattice decoding is not yet supported.
*/
import static org.apache.joshua.decoder.ff.tm.OwnerMap.UNKNOWN_OWNER;
@@ -40,7 +40,6 @@ import static org.apache.joshua.decoder.ff.tm.OwnerMap.UNKNOWN_OWNER;
import java.util.ArrayList;
import java.util.List;
-import org.apache.joshua.corpus.Span;
import org.apache.joshua.decoder.JoshuaConfiguration;
import org.apache.joshua.decoder.chart_parser.ComputeNodeResult;
import org.apache.joshua.decoder.ff.FeatureFunction;
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ffd9562e/src/test/resources/decoder/phrase/include-align-index/log
----------------------------------------------------------------------
diff --git a/src/test/resources/decoder/phrase/include-align-index/log b/src/test/resources/decoder/phrase/include-align-index/log
deleted file mode 100644
index 05cd80f..0000000
--- a/src/test/resources/decoder/phrase/include-align-index/log
+++ /dev/null
@@ -1,50 +0,0 @@
-Parameters read from configuration file:
- tm = 'moses -owner pt -maxspan 0 -path rules.1.gz -max-source-len 5'
- featurefunction = 'StateMinimizingLanguageModel -lm_order 5 -lm_file lm.1.gz'
- search = 'stack'
- markoovs = 'false'
- poplimit = '10'
- topn = '1'
- outputformat = '%i ||| %s ||| %f ||| %c'
- includealignindex = 'true'
- reorderinglimit = '6'
- featurefunction = 'OOVPenalty'
- featurefunction = 'WordPenalty'
- featurefunction = 'Distortion'
- featurefunction = 'PhrasePenalty -owner pt'
-Parameters overridden from the command line:
- threads = '1'
- c = 'config'
-Read 9 weights (0 of them dense)
-Reading grammar from file rules.1.gz...
-........10........20........30........40........50........60........70........80........90.....100%
-MemoryBasedBatchGrammar: Read 165161 rules with 18 distinct source sides from 'rules.1.gz'
-Couldn't create a GrammarReader for file null with format phrase
-MemoryBasedBatchGrammar: Read 0 rules with 0 distinct source sides from 'null'
-Memory used 219.6 MB
-Grammar loading took: 0 seconds.
-Stateful object with state index 0
-Loading the LM will be faster if you build a binary file.
-Reading lm.1.gz
-----5---10---15---20---25---30---35---40---45---50---55---60---65---70---75---80---85---90---95--100
-****************************************************************************************************
-FEATURE: tm_pt (weight 0.000)
-FEATURE: tm_custom (weight 0.000)
-FEATURE: lm_0, order 5 (weight 0.204)
-FEATURE: OOVPenalty (weight 1.000)
-FEATURE: WordPenalty (weight -0.202)
-FEATURE: Distortion (weight 0.115)
-FEATURE: PhrasePenalty (weight -0.237)
-Grammar sorting happening lazily on-demand.
-Model loading took 0 seconds
-Memory used 219.6 MB
-Input 0: <s> una estrategia republicana para obstaculizar la reelecci�n de Obama </s>
-Input 0: Collecting options took 0.000 seconds
-Input 0: Search took 0.013 seconds
-Input 0: Translation took 1.532 seconds
-Input 0: Memory used is 392.5 MB
-Translation 0: -7.496 a strategy republican to hinder reelection Obama
-Input 0: 1-best extraction took 0.026 seconds
-Decoding completed.
-Memory used 401.6 MB
-Total running time: 2 seconds
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/ffd9562e/src/test/resources/decoder/phrase/include-align-index/output
----------------------------------------------------------------------
diff --git a/src/test/resources/decoder/phrase/include-align-index/output b/src/test/resources/decoder/phrase/include-align-index/output
deleted file mode 100644
index 509a3de..0000000
--- a/src/test/resources/decoder/phrase/include-align-index/output
+++ /dev/null
@@ -1 +0,0 @@
-0 ||| a strategy |0-1| republican |2-2| to hinder |3-4| reelection |5-6| Obama |7-8| ||| tm_pt_0=-9.702 tm_pt_1=-10.800 tm_pt_2=-7.543 tm_pt_3=-8.555 lm_0=-19.117 OOVPenalty=0.000 WordPenalty=-3.040 Distortion=0.000 PhrasePenalty=5.000 ||| -7.496