You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by mj...@apache.org on 2016/06/11 16:22:11 UTC
[1/2] incubator-joshua git commit: fixed test case
Repository: incubator-joshua
Updated Branches:
refs/heads/master 9e606d85b -> 300f0bb40
fixed test case
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/a1e9ccd7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/a1e9ccd7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/a1e9ccd7
Branch: refs/heads/master
Commit: a1e9ccd75a89825b064d07bfb01bfb9dabce7142
Parents: 9e606d8
Author: Matt Post <po...@cs.jhu.edu>
Authored: Sat Jun 11 12:18:29 2016 -0400
Committer: Matt Post <po...@cs.jhu.edu>
Committed: Sat Jun 11 12:18:29 2016 -0400
----------------------------------------------------------------------
src/test/resources/server/tcp-text/test.sh | 6 ++++--
1 file changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/a1e9ccd7/src/test/resources/server/tcp-text/test.sh
----------------------------------------------------------------------
diff --git a/src/test/resources/server/tcp-text/test.sh b/src/test/resources/server/tcp-text/test.sh
index f37edf2..66628ca 100755
--- a/src/test/resources/server/tcp-text/test.sh
+++ b/src/test/resources/server/tcp-text/test.sh
@@ -18,12 +18,14 @@
# This test case starts a server and then throws 10 threads at it to make sure threading is working.
-$JOSHUA/bin/decoder -threads 4 -server-port 9010 -output-format "%i ||| %s" -mark-oovs true > server.log 2>&1 &
+port=9011
+
+$JOSHUA/bin/decoder -threads 4 -server-port $port -output-format "%i ||| %s" -mark-oovs true -v 1 > server.log 2>&1 &
serverpid=$!
sleep 2
for num in $(seq 0 9); do
- echo -e "this\nthat\nthese\n\nthose\nmine\nhis\nyours\nhers" | nc localhost 9010 > output.$num 2> log.$num &
+ echo -e "this\nthat\nthese\n\nthose\nmine\nhis\nyours\nhers" | nc localhost $port > output.$num 2> log.$num &
pids[$num]=$!
done
[2/2] incubator-joshua git commit: Got rid of decoder-aware metadata
handling
Posted by mj...@apache.org.
Got rid of decoder-aware metadata handling
Sorry for the churn on this, but it became obvious that encoding metadata in sentences is the wrong way to go, and leaves open the potential to cause other problems. This commit reverts the changes of commit aa10be5b63c6c2537d6b761a9d3315ee9a8bf25f, essentially: metadata can no longer be passed in to the decoder be encoding it between bars that start a sentence.
Metadata for changing operational parameters, adding rules to the custom grammar, and so on, are now only available to the HTTP server, when passed in on a URL string with the parameters "meta=...". The commands currently supported are
- changing a decoder weight
- getting a list of weights
- adding a custom grammar rule
- removing a custom grammar rule
- listing all the custom grammar rules
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/300f0bb4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/300f0bb4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/300f0bb4
Branch: refs/heads/master
Commit: 300f0bb40d6c104d5b34df2fec6203ccbd1dfd73
Parents: a1e9ccd
Author: Matt Post <po...@cs.jhu.edu>
Authored: Sat Jun 11 12:22:07 2016 -0400
Committer: Matt Post <po...@cs.jhu.edu>
Committed: Sat Jun 11 12:22:07 2016 -0400
----------------------------------------------------------------------
.../java/org/apache/joshua/decoder/Decoder.java | 135 ++---------------
.../apache/joshua/decoder/JoshuaDecoder.java | 10 --
.../org/apache/joshua/decoder/MetaData.java | 85 -----------
.../org/apache/joshua/decoder/Translation.java | 16 --
.../apache/joshua/decoder/io/JSONMessage.java | 14 +-
.../joshua/decoder/segment_file/Sentence.java | 30 ----
.../org/apache/joshua/server/ServerThread.java | 147 ++++++++++++++++++-
.../resources/decoder/metadata/add_rule/test.sh | 0
8 files changed, 160 insertions(+), 277 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/300f0bb4/src/main/java/org/apache/joshua/decoder/Decoder.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/Decoder.java b/src/main/java/org/apache/joshua/decoder/Decoder.java
index fd7bf4b..7164134 100644
--- a/src/main/java/org/apache/joshua/decoder/Decoder.java
+++ b/src/main/java/org/apache/joshua/decoder/Decoder.java
@@ -26,7 +26,6 @@ import java.io.IOException;
import java.io.FileNotFoundException;
import java.lang.reflect.Constructor;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -43,11 +42,9 @@ import org.apache.joshua.decoder.ff.StatefulFF;
import org.apache.joshua.decoder.ff.lm.LanguageModelFF;
import org.apache.joshua.decoder.ff.tm.Grammar;
import org.apache.joshua.decoder.ff.tm.Rule;
-import org.apache.joshua.decoder.ff.tm.Trie;
import org.apache.joshua.decoder.ff.tm.format.HieroFormatReader;
import org.apache.joshua.decoder.ff.tm.hash_based.MemoryBasedBatchGrammar;
import org.apache.joshua.decoder.ff.tm.packed.PackedGrammar;
-import org.apache.joshua.decoder.io.JSONMessage;
import org.apache.joshua.decoder.io.TranslationRequestStream;
import org.apache.joshua.decoder.phrase.PhraseTable;
import org.apache.joshua.decoder.segment_file.Sentence;
@@ -255,118 +252,6 @@ public class Decoder {
}
/**
- * When metadata is found on the input, it needs to be processed. That is done here. Sometimes
- * this involves returning data to the client.
- *
- * @param meta
- * @throws IOException
- */
- private void handleMetadata(MetaData meta) {
- if (meta.type().equals("set_weights")) {
- // Change a decoder weight
- String[] args = meta.tokens();
- System.err.println(Arrays.toString(args));
- for (int i = 0; i < args.length; i += 2) {
- String feature = args[i];
- String newValue = args[i+1];
- float old_weight = Decoder.weights.getWeight(feature);
- Decoder.weights.set(feature, Float.parseFloat(newValue));
- LOG.info("set_weights: {} {} -> {}", feature, old_weight, Decoder.weights.getWeight(args[0]));
- }
-
- } else if (meta.type().equals("get_weights")) {
- meta.setResponse("weights " + Decoder.weights.toString());
-
- } else if (meta.type().equals("add_rule")) {
- String args[] = meta.tokens(" ,,, ");
-
- if (args.length != 2) {
- LOG.error("* INVALID RULE '{}'", meta);
- return;
- }
-
- String source = args[0];
- String target = args[1];
- String featureStr = "";
- if (args.length > 2)
- featureStr = args[2];
-
-
- /* Prepend source and target side nonterminals for phrase-based decoding. Probably better
- * handled in each grammar type's addRule() function.
- */
- String ruleString = (joshuaConfiguration.search_algorithm.equals("stack"))
- ? String.format("[X] ||| [X,1] %s ||| [X,1] %s ||| custom=1 %s", source, target, featureStr)
- : String.format("[X] ||| %s ||| %s ||| custom=1 %s", source, target, featureStr);
-
- Rule rule = new HieroFormatReader().parseLine(ruleString);
- Decoder.this.customPhraseTable.addRule(rule);
- rule.estimateRuleCost(featureFunctions);
- LOG.info("Added custom rule {}", rule.toString());
-
- } else if (meta.type().equals("list_rules")) {
-
- LOG.info("list_rules");
-
- JSONMessage message = new JSONMessage();
-
- // Walk the the grammar trie
- ArrayList<Trie> nodes = new ArrayList<Trie>();
- nodes.add(customPhraseTable.getTrieRoot());
-
- while (nodes.size() > 0) {
- Trie trie = nodes.remove(0);
-
- if (trie == null)
- continue;
-
- if (trie.hasRules()) {
- for (Rule rule: trie.getRuleCollection().getRules()) {
- message.addRule(rule.toString());
- LOG.info("Found rule: " + rule);
- }
- }
-
- if (trie.getExtensions() != null)
- nodes.addAll(trie.getExtensions());
- }
-
- } else if (meta.type().equals("remove_rule")) {
- // Remove a rule from a custom grammar, if present
- String[] args = meta.tokenString().split(" ,,, ");
- if (args.length != 2) {
- return;
- }
-
- // Search for the rule in the trie
- int nt_i = Vocabulary.id(joshuaConfiguration.default_non_terminal);
- Trie trie = customPhraseTable.getTrieRoot().match(nt_i);
-
- for (String word: args[0].split("\\s+")) {
- int id = Vocabulary.id(word);
- Trie nextTrie = trie.match(id);
- if (nextTrie != null)
- trie = nextTrie;
- }
-
- if (trie.hasRules()) {
- Rule matched = null;
- for (Rule rule: trie.getRuleCollection().getRules()) {
- String target = rule.getEnglishWords();
- target = target.substring(target.indexOf(' ') + 1);
-
- if (args[1].equals(target)) {
- matched = rule;
- break;
- }
- }
- trie.getRuleCollection().getRules().remove(matched);
- return;
- }
- }
- }
-
- /**
* This class handles running a DecoderThread (which takes care of the actual translation of an
* input Sentence, returning a Translation object when its done). This is done in a thread so as
* not to tie up the RequestHandler that launched it, freeing it to go on to the next sentence in
@@ -396,10 +281,6 @@ public class Decoder {
* Process any found metadata.
*/
- if (sentence.hasMetaData()) {
- handleMetadata(sentence.getMetaData());
- }
-
/*
* Use the thread to translate the sentence. Then record the translation with the
* corresponding Translations object, and return the thread to the pool.
@@ -870,7 +751,7 @@ public class Decoder {
try {
- Class<?> clas = getClass(featureName);
+ Class<?> clas = getFeatureFunctionClass(featureName);
Constructor<?> constructor = clas.getConstructor(FeatureVector.class,
String[].class, JoshuaConfiguration.class);
FeatureFunction feature = (FeatureFunction) constructor.newInstance(weights, fields, joshuaConfiguration);
@@ -896,7 +777,7 @@ public class Decoder {
* @return the class, found in one of the search paths
* @throws ClassNotFoundException
*/
- private Class<?> getClass(String featureName) {
+ private Class<?> getFeatureFunctionClass(String featureName) {
Class<?> clas = null;
String[] packages = { "org.apache.joshua.decoder.ff", "org.apache.joshua.decoder.ff.lm", "org.apache.joshua.decoder.ff.phrase" };
@@ -915,4 +796,16 @@ public class Decoder {
}
return clas;
}
+
+ /**
+ *
+ * @param rule
+ */
+ public void addCustomRule(Rule rule) {
+ rule.estimateRuleCost(featureFunctions);
+ }
+
+ public Grammar getCustomPhraseTable() {
+ return customPhraseTable;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/300f0bb4/src/main/java/org/apache/joshua/decoder/JoshuaDecoder.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/JoshuaDecoder.java b/src/main/java/org/apache/joshua/decoder/JoshuaDecoder.java
index 13bbb25..4c31655 100644
--- a/src/main/java/org/apache/joshua/decoder/JoshuaDecoder.java
+++ b/src/main/java/org/apache/joshua/decoder/JoshuaDecoder.java
@@ -109,16 +109,6 @@ public class JoshuaDecoder {
nbest_out = new FileWriter(joshuaConfiguration.n_best_file);
for (Translation translation: translations) {
-
- /* Process metadata */
- if (translation.hasMetaData()) {
- MetaData md = translation.getMetaData();
- if (md.type().equals("get_weight")) {
- String weight = md.tokens()[0];
- System.err.println(String.format("You want %s? You got it. It's %.3f", weight,
- Decoder.weights.getWeight(weight)));
- }
- }
/**
* We need to munge the feature value outputs in order to be compatible with Moses tuners.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/300f0bb4/src/main/java/org/apache/joshua/decoder/MetaData.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/MetaData.java b/src/main/java/org/apache/joshua/decoder/MetaData.java
deleted file mode 100644
index 5595296..0000000
--- a/src/main/java/org/apache/joshua/decoder/MetaData.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.joshua.decoder;
-
-/*
- * This class is used to capture metadata command to Joshua on input and pass them to the
- * decoder. A metadata object comprises a type and a list of arguments. The type is one of the
- * following commands:
- *
- * - add_weight NAME VALUE
- * - get_weights -> weights NAME VALUE NAME VALUE...
- * - add_rule SOURCE ,,, TARGET
- *
- * In the future, this should be done through the API.
- */
-
-public class MetaData {
-
- /* The metadata request that came in */
- String type;
-
- /* The list of arguments */
- String tokenString;
-
- /* The response (if any) */
- String response;
-
- public MetaData(String message) {
- message = message.substring(1, message.length() - 1).trim();
-
- int firstSpace = message.indexOf(' ');
- if (firstSpace != -1) {
- this.type = message.substring(0, firstSpace);
- this.tokenString = message.substring(firstSpace + 1);
- } else if (message.length() > 0) {
- this.type = message.substring(0);
- this.tokenString = "";
- } else {
- type = "";
- tokenString = "";
- }
-
- response = "";
- }
-
- public String type() {
- return this.type;
- }
-
- public String tokenString() {
- return this.tokenString;
- }
-
- public String[] tokens(String regex) {
- return this.tokenString.split(regex);
- }
-
- public String[] tokens() {
- return this.tokens("\\s+");
- }
-
- public void setResponse(String response) {
- this.response = response;
- }
-
- public String response() {
- return this.response;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/300f0bb4/src/main/java/org/apache/joshua/decoder/Translation.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/Translation.java b/src/main/java/org/apache/joshua/decoder/Translation.java
index 2c040f0..46f3061 100644
--- a/src/main/java/org/apache/joshua/decoder/Translation.java
+++ b/src/main/java/org/apache/joshua/decoder/Translation.java
@@ -236,20 +236,4 @@ public class Translation {
}
}
}
-
- /**
- * Returns metadata found on the source sentence.
- *
- * (This just goes to demonstrate that a Translation object should just be an additional
- * set of annotations on an input sentence)
- *
- * @return metadata annotations from the source sentence
- */
- public MetaData getMetaData() {
- return source.getMetaData();
- }
-
- public boolean hasMetaData() {
- return source.hasMetaData();
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/300f0bb4/src/main/java/org/apache/joshua/decoder/io/JSONMessage.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/io/JSONMessage.java b/src/main/java/org/apache/joshua/decoder/io/JSONMessage.java
index cba6ec7..3932277 100644
--- a/src/main/java/org/apache/joshua/decoder/io/JSONMessage.java
+++ b/src/main/java/org/apache/joshua/decoder/io/JSONMessage.java
@@ -49,6 +49,7 @@ import org.apache.joshua.decoder.Translation;
public class JSONMessage {
public Data data = null;
+ public String metaData = null;
public List<String> rules = null;
public JSONMessage() {
@@ -74,10 +75,6 @@ public class JSONMessage {
item.addHypothesis(text, score);
}
- if (translation.hasMetaData()) {
- item.setMetaData(translation.getMetaData().response());
- }
-
// old string-based k-best output
// String[] results = translation.toString().split("\\n");
// if (results.length > 0) {
@@ -109,9 +106,12 @@ public class JSONMessage {
return newItem;
}
+ public void setMetaData(String msg) {
+ this.metaData = msg;
+ }
+
public class TranslationItem {
public String translatedText;
- public String metaData = null;
public List<NBestItem> raw_nbest;
public TranslationItem(String value) {
@@ -128,10 +128,6 @@ public class JSONMessage {
public void addHypothesis(String hyp, float score) {
this.raw_nbest.add(new NBestItem(hyp, score));
}
-
- public void setMetaData(String msg) {
- this.metaData = msg;
- }
}
public class NBestItem {
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/300f0bb4/src/main/java/org/apache/joshua/decoder/segment_file/Sentence.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/segment_file/Sentence.java b/src/main/java/org/apache/joshua/decoder/segment_file/Sentence.java
index 789cb0a..d57ecc2 100644
--- a/src/main/java/org/apache/joshua/decoder/segment_file/Sentence.java
+++ b/src/main/java/org/apache/joshua/decoder/segment_file/Sentence.java
@@ -31,7 +31,6 @@ import java.util.regex.Pattern;
import org.apache.joshua.corpus.Vocabulary;
import org.apache.joshua.decoder.JoshuaConfiguration;
-import org.apache.joshua.decoder.MetaData;
import org.apache.joshua.decoder.ff.tm.Grammar;
import org.apache.joshua.lattice.Arc;
import org.apache.joshua.lattice.Lattice;
@@ -78,8 +77,6 @@ public class Sentence {
private JoshuaConfiguration config = null;
- private MetaData metaData;
-
/**
* Constructor. Receives a string representing the input sentence. This string may be a
* string-encoded lattice or a plain text string for decoding.
@@ -94,7 +91,6 @@ public class Sentence {
config = joshuaConfiguration;
- this.metaData = null;
this.constraints = new LinkedList<ConstraintSpan>();
// Check if the sentence has SGML markings denoting the
@@ -107,13 +103,6 @@ public class Sentence {
this.id = Integer.parseInt(idstr);
} else {
- if (hasRawMetaData(inputString)) {
- /* Found some metadata */
- metaData = new MetaData(inputString.substring(0, inputString.indexOf('|', 1)));
-
- inputString = inputString.substring(inputString.indexOf('|', 1) + 1).trim();
- }
-
if (inputString.indexOf(" ||| ") != -1) {
/* Target-side given; used for parsing and forced decoding */
String[] pieces = inputString.split("\\s?\\|{3}\\s?");
@@ -138,25 +127,6 @@ public class Sentence {
if (! (joshuaConfiguration.lattice_decoding && source.startsWith("(((")))
adjustForLength(joshuaConfiguration.maxlen);
}
-
- /**
- * Look for metadata in the input sentence. Metadata is any line starting with a literal '|',
- * up to the next occurrence of a '|'
- *
- * @param inputString
- * @return whether metadata was found
- */
- private boolean hasRawMetaData(String inputString) {
- return inputString.startsWith("| ") && inputString.indexOf(" |") > 0;
- }
-
- public boolean hasMetaData() {
- return this.metaData != null;
- }
-
- public MetaData getMetaData() {
- return this.metaData;
- }
/**
* Indicates whether the underlying lattice is a linear chain, i.e., a sentence.
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/300f0bb4/src/main/java/org/apache/joshua/server/ServerThread.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/server/ServerThread.java b/src/main/java/org/apache/joshua/server/ServerThread.java
index 5913029..0187ad1 100644
--- a/src/main/java/org/apache/joshua/server/ServerThread.java
+++ b/src/main/java/org/apache/joshua/server/ServerThread.java
@@ -27,24 +27,29 @@ import java.net.Socket;
import java.net.SocketException;
import java.net.URLDecoder;
import java.nio.charset.Charset;
+import java.util.ArrayList;
import java.util.HashMap;
import com.sun.net.httpserver.HttpExchange;
import com.sun.net.httpserver.HttpHandler;
+import org.apache.joshua.corpus.Vocabulary;
import org.apache.joshua.decoder.Decoder;
import org.apache.joshua.decoder.JoshuaConfiguration;
import org.apache.joshua.decoder.Translation;
import org.apache.joshua.decoder.Translations;
-import org.apache.joshua.decoder.JoshuaConfiguration.INPUT_TYPE;
-import org.apache.joshua.decoder.JoshuaConfiguration.SERVER_TYPE;
+import org.apache.joshua.decoder.ff.tm.Rule;
+import org.apache.joshua.decoder.ff.tm.Trie;
+import org.apache.joshua.decoder.ff.tm.format.HieroFormatReader;
import org.apache.joshua.decoder.io.JSONMessage;
import org.apache.joshua.decoder.io.TranslationRequestStream;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * This class handles a concurrent request for translations from a newly opened socket.
+ * This class handles a concurrent request for translations from a newly opened socket, for
+ * both raw TCP/IP connections and for HTTP connections.
+ *
*/
public class ServerThread extends Thread implements HttpHandler {
@@ -138,19 +143,28 @@ public class ServerThread extends Thread implements HttpHandler {
out.write(b);
}
}
-
-
+
+ /**
+ * Called to handle an HTTP connection. This looks for metadata in the URL string, which is processed
+ * if present. It also then handles returning a JSON-formatted object to the caller.
+ *
+ * @param client the client connection
+ */
@Override
public void handle(HttpExchange client) throws IOException {
HashMap<String, String> params = queryToMap(URLDecoder.decode(client.getRequestURI().getQuery(), "UTF-8"));
String query = params.get("q");
-
+ String meta = params.get("meta");
+
BufferedReader reader = new BufferedReader(new StringReader(query));
TranslationRequestStream request = new TranslationRequestStream(reader, joshuaConfiguration);
Translations translations = decoder.decodeAll(request);
JSONMessage message = new JSONMessage();
+ if (meta != null && ! meta.isEmpty())
+ message.setMetaData(handleMetadata(meta));
+
for (Translation translation: translations) {
LOG.info("TRANSLATION: '{}' with {} k-best items", translation, translation.getStructuredTranslations().size());
message.addTranslation(translation);
@@ -164,4 +178,125 @@ public class ServerThread extends Thread implements HttpHandler {
reader.close();
}
+
+ /**
+ * Processes metadata commands received in the HTTP request. Some commands result in sending data back.
+ *
+ * @param meta the metadata request
+ * @return result string (for some commands)
+ */
+ private String handleMetadata(String meta) {
+ String[] tokens = meta.split("\\s+", 2);
+ String type = tokens[0];
+ String args = tokens[1];
+ String response = "";
+
+ if (type.equals("get_weight")) {
+ String weight = tokens[1];
+ LOG.info("WEIGHT: %s = %.3f", weight, Decoder.weights.getWeight(weight));
+
+ } else if (type.equals("set_weights")) {
+ // Change a decoder weight
+ String[] argTokens = args.split("\\s+");
+ for (int i = 0; i < argTokens.length; i += 2) {
+ String feature = argTokens[i];
+ String newValue = argTokens[i+1];
+ float old_weight = Decoder.weights.getWeight(feature);
+ Decoder.weights.set(feature, Float.parseFloat(newValue));
+ LOG.info("set_weights: {} {} -> {}", feature, old_weight, Decoder.weights.getWeight(feature));
+ }
+
+ response = "weights " + Decoder.weights.toString();
+
+ } else if (type.equals("get_weights")) {
+ response = "weights " + Decoder.weights.toString();
+
+ } else if (type.equals("add_rule")) {
+ String argTokens[] = args.split(" ,,, ");
+
+ if (argTokens.length != 2) {
+ LOG.error("* INVALID RULE '{}'", meta);
+ return "";
+ }
+
+ String source = argTokens[0];
+ String target = argTokens[1];
+ String featureStr = "";
+ if (argTokens.length > 2)
+ featureStr = argTokens[2];
+
+ /* Prepend source and target side nonterminals for phrase-based decoding. Probably better
+ * handled in each grammar type's addRule() function.
+ */
+ String ruleString = (joshuaConfiguration.search_algorithm.equals("stack"))
+ ? String.format("[X] ||| [X,1] %s ||| [X,1] %s ||| custom=1 %s", source, target, featureStr)
+ : String.format("[X] ||| %s ||| %s ||| custom=1 %s", source, target, featureStr);
+
+ Rule rule = new HieroFormatReader().parseLine(ruleString);
+ decoder.addCustomRule(rule);
+
+ LOG.info("Added custom rule {}", rule.toString());
+
+ } else if (type.equals("list_rules")) {
+
+ LOG.info("list_rules");
+
+ JSONMessage message = new JSONMessage();
+
+ // Walk the the grammar trie
+ ArrayList<Trie> nodes = new ArrayList<Trie>();
+ nodes.add(decoder.getCustomPhraseTable().getTrieRoot());
+
+ while (nodes.size() > 0) {
+ Trie trie = nodes.remove(0);
+
+ if (trie == null)
+ continue;
+
+ if (trie.hasRules()) {
+ for (Rule rule: trie.getRuleCollection().getRules()) {
+ message.addRule(rule.toString());
+ LOG.info("Found rule: " + rule);
+ }
+ }
+
+ if (trie.getExtensions() != null)
+ nodes.addAll(trie.getExtensions());
+ }
+
+ } else if (type.equals("remove_rule")) {
+ // Remove a rule from a custom grammar, if present
+ String[] argTokens = args.split(" ,,, ");
+ if (argTokens.length != 2) {
+ return "";
+ }
+
+ // Search for the rule in the trie
+ int nt_i = Vocabulary.id(joshuaConfiguration.default_non_terminal);
+ Trie trie = decoder.getCustomPhraseTable().getTrieRoot().match(nt_i);
+
+ for (String word: argTokens[0].split("\\s+")) {
+ int id = Vocabulary.id(word);
+ Trie nextTrie = trie.match(id);
+ if (nextTrie != null)
+ trie = nextTrie;
+ }
+
+ if (trie.hasRules()) {
+ Rule matched = null;
+ for (Rule rule: trie.getRuleCollection().getRules()) {
+ String target = rule.getEnglishWords();
+ target = target.substring(target.indexOf(' ') + 1);
+
+ if (argTokens[1].equals(target)) {
+ matched = rule;
+ break;
+ }
+ }
+ trie.getRuleCollection().getRules().remove(matched);
+ return "";
+ }
+ }
+ return response;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/300f0bb4/src/test/resources/decoder/metadata/add_rule/test.sh
----------------------------------------------------------------------
diff --git a/src/test/resources/decoder/metadata/add_rule/test.sh b/src/test/resources/decoder/metadata/add_rule/test.sh
old mode 100755
new mode 100644