You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by mj...@apache.org on 2016/09/17 12:27:51 UTC
[02/14] incubator-joshua git commit: Removed global state index for
Stateful feature functions
Removed global state index for Stateful feature functions
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/dc149e62
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/dc149e62
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/dc149e62
Branch: refs/heads/7_confsystem
Commit: dc149e628ebe67ba29da39cf6db17e76733c2298
Parents: f2edda0
Author: Hieber, Felix <fh...@amazon.de>
Authored: Thu Sep 15 10:16:09 2016 +0200
Committer: Hieber, Felix <fh...@amazon.de>
Committed: Thu Sep 15 17:34:00 2016 +0200
----------------------------------------------------------------------
.../java/org/apache/joshua/decoder/Decoder.java | 14 +++++++-------
.../org/apache/joshua/decoder/ff/StatefulFF.java | 16 ++++++----------
.../joshua/decoder/ff/lm/LanguageModelFF.java | 2 +-
3 files changed, 14 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/dc149e62/joshua-core/src/main/java/org/apache/joshua/decoder/Decoder.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/Decoder.java b/joshua-core/src/main/java/org/apache/joshua/decoder/Decoder.java
index e448ec8..a5e1e3c 100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/Decoder.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/Decoder.java
@@ -18,6 +18,7 @@
*/
package org.apache.joshua.decoder;
+import static com.typesafe.config.ConfigValueFactory.fromAnyRef;
import static org.apache.joshua.decoder.ff.FeatureMap.hashFeature;
import static org.apache.joshua.decoder.ff.tm.hash_based.TextGrammarFactory.createCustomGrammar;
import static org.apache.joshua.decoder.ff.tm.hash_based.TextGrammarFactory.createGlueTextGrammar;
@@ -245,7 +246,6 @@ public class Decoder {
Vocabulary.clear();
Vocabulary.unregisterLanguageModels();
LanguageModelFF.resetLmIndex();
- StatefulFF.resetGlobalStateIndex();
}
/**
@@ -448,23 +448,23 @@ public class Decoder {
}
// (2) instantiate other feature functions by class name
+ int statefulStateIndex = 0;
for (Config featureConfig : config.getConfigList("feature_functions")) {
final Class<?> clazz = getClassFromPackages(featureConfig.getString("class"), FEATURE_PACKAGES);
+ if (StatefulFF.class.isAssignableFrom(clazz)) {
+ featureConfig = featureConfig.withValue("state_index", fromAnyRef(statefulStateIndex++));
+ }
try {
final Constructor<?> constructor = clazz.getConstructor(Config.class, FeatureVector.class);
final FeatureFunction feature = (FeatureFunction) constructor.newInstance(featureConfig, weights);
+ LOG.info("FEATURE: {}", feature.logString());
result.add(feature);
} catch (Exception e) {
LOG.error("Unable to instantiate feature '{}'", clazz.getName());
Throwables.propagate(e);
}
}
-
- final ImmutableList<FeatureFunction> features = result.build();
- for (final FeatureFunction feature : features) {
- LOG.info("FEATURE: {}", feature.logString());
- }
- return features;
+ return result.build();
}
/**
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/dc149e62/joshua-core/src/main/java/org/apache/joshua/decoder/ff/StatefulFF.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/StatefulFF.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/StatefulFF.java
index 8071763..202dc42 100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/StatefulFF.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/StatefulFF.java
@@ -43,21 +43,17 @@ import com.typesafe.config.Config;
public abstract class StatefulFF extends FeatureFunction {
private static final Logger LOG = LoggerFactory.getLogger(StatefulFF.class);
- /* Every stateful FF takes a unique index value and increments this. */
- static int GLOBAL_STATE_INDEX = 0;
/* This records the state index for each instantiated stateful feature function. */
- protected int stateIndex = 0;
+ protected final int stateIndex;
public StatefulFF(final String name, Config featureConfig, FeatureVector weights) {
super(name, featureConfig, weights);
-
- LOG.info("Stateful object with state index {}", GLOBAL_STATE_INDEX);
- stateIndex = GLOBAL_STATE_INDEX++;
- }
-
- public static void resetGlobalStateIndex() {
- GLOBAL_STATE_INDEX = 0;
+ if (!featureConfig.hasPath("state_index")) {
+ throw new RuntimeException("StatefulFF must configure a state_index");
+ }
+ stateIndex = featureConfig.getInt("state_index");
+ LOG.info("StatefulFF with state index {}", stateIndex);
}
public final boolean isStateful() {
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/dc149e62/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/LanguageModelFF.java
----------------------------------------------------------------------
diff --git a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/LanguageModelFF.java b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/LanguageModelFF.java
index e30242e..9cc67ec 100644
--- a/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/LanguageModelFF.java
+++ b/joshua-core/src/main/java/org/apache/joshua/decoder/ff/lm/LanguageModelFF.java
@@ -167,7 +167,7 @@ public class LanguageModelFF extends StatefulFF {
}
public String logString() {
- return String.format("%s, order %d (weight %.3f), classLm=%s", name, languageModel.getOrder(), weights.getOrDefault(featureId), isClassLM);
+ return String.format("%s, order %d (weight %.3f), stateIndex=%d classLm=%s", name, languageModel.getOrder(), weights.getOrDefault(featureId), stateIndex, isClassLM);
}
/**