You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by mj...@apache.org on 2016/06/10 03:22:22 UTC
incubator-joshua git commit: Added ability to pass in metadata
through the server
Repository: incubator-joshua
Updated Branches:
refs/heads/master e360b56a5 -> 9e606d85b
Added ability to pass in metadata through the server
This is not the right way to do things; really, this should be done via contact with a server, which would then use an API. But in the meantime I need this functionality for demos and tutorials.
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/9e606d85
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/9e606d85
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/9e606d85
Branch: refs/heads/master
Commit: 9e606d85bc284c0e38d91b6526731612377e9658
Parents: e360b56
Author: Matt Post <po...@cs.jhu.edu>
Authored: Thu Jun 9 23:22:22 2016 -0400
Committer: Matt Post <po...@cs.jhu.edu>
Committed: Thu Jun 9 23:22:22 2016 -0400
----------------------------------------------------------------------
.../java/org/apache/joshua/decoder/Decoder.java | 21 +++--
.../apache/joshua/decoder/JoshuaDecoder.java | 2 +
.../org/apache/joshua/decoder/MetaData.java | 26 +++++-
.../tm/hash_based/MemoryBasedBatchGrammar.java | 1 +
.../apache/joshua/decoder/io/JSONMessage.java | 94 +++++++++++++++-----
.../org/apache/joshua/server/ServerThread.java | 17 ++--
6 files changed, 123 insertions(+), 38 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9e606d85/src/main/java/org/apache/joshua/decoder/Decoder.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/Decoder.java b/src/main/java/org/apache/joshua/decoder/Decoder.java
index 088f678..fd7bf4b 100644
--- a/src/main/java/org/apache/joshua/decoder/Decoder.java
+++ b/src/main/java/org/apache/joshua/decoder/Decoder.java
@@ -26,6 +26,7 @@ import java.io.IOException;
import java.io.FileNotFoundException;
import java.lang.reflect.Constructor;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -264,13 +265,18 @@ public class Decoder {
if (meta.type().equals("set_weights")) {
// Change a decoder weight
String[] args = meta.tokens();
+ System.err.println(Arrays.toString(args));
for (int i = 0; i < args.length; i += 2) {
- float old_weight = Decoder.weights.getWeight(args[i]);
- Decoder.weights.set(args[1], Float.parseFloat(args[i+1]));
- LOG.error("@set_weights: {} {} -> {}", args[1], old_weight,
- Decoder.weights.getWeight(args[0]));
+ String feature = args[i];
+ String newValue = args[i+1];
+ float old_weight = Decoder.weights.getWeight(feature);
+ Decoder.weights.set(feature, Float.parseFloat(newValue));
+ LOG.info("set_weights: {} {} -> {}", feature, old_weight, Decoder.weights.getWeight(args[0]));
}
-
+
+ } else if (meta.type().equals("get_weights")) {
+ meta.setResponse("weights " + Decoder.weights.toString());
+
} else if (meta.type().equals("add_rule")) {
String args[] = meta.tokens(" ,,, ");
@@ -300,6 +306,8 @@ public class Decoder {
} else if (meta.type().equals("list_rules")) {
+ LOG.info("list_rules");
+
JSONMessage message = new JSONMessage();
// Walk the the grammar trie
@@ -315,6 +323,7 @@ public class Decoder {
if (trie.hasRules()) {
for (Rule rule: trie.getRuleCollection().getRules()) {
message.addRule(rule.toString());
+ LOG.info("Found rule: " + rule);
}
}
@@ -694,7 +703,7 @@ public class Decoder {
} catch (FileNotFoundException e) {
String msg = String.format("Couldn't load packed grammar from '%s'", path)
+ "Perhaps it doesn't exist, or it may be an old packed file format.";
- throw new RuntimeException(e);
+ throw new RuntimeException(msg);
}
} else {
// thrax, hiero, samt
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9e606d85/src/main/java/org/apache/joshua/decoder/JoshuaDecoder.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/JoshuaDecoder.java b/src/main/java/org/apache/joshua/decoder/JoshuaDecoder.java
index e60cdc9..13bbb25 100644
--- a/src/main/java/org/apache/joshua/decoder/JoshuaDecoder.java
+++ b/src/main/java/org/apache/joshua/decoder/JoshuaDecoder.java
@@ -80,6 +80,8 @@ public class JoshuaDecoder {
new TcpServer(decoder, port, joshuaConfiguration).start();
} else if (joshuaConfiguration.server_type == SERVER_TYPE.HTTP) {
+ joshuaConfiguration.use_structured_output = true;
+
HttpServer server = HttpServer.create(new InetSocketAddress(port), 0);
LOG.info("HTTP Server running and listening on port {}.", port);
server.createContext("/", new ServerThread(null, decoder, joshuaConfiguration));
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9e606d85/src/main/java/org/apache/joshua/decoder/MetaData.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/MetaData.java b/src/main/java/org/apache/joshua/decoder/MetaData.java
index 092dd7a..5595296 100644
--- a/src/main/java/org/apache/joshua/decoder/MetaData.java
+++ b/src/main/java/org/apache/joshua/decoder/MetaData.java
@@ -20,13 +20,27 @@ package org.apache.joshua.decoder;
/*
* This class is used to capture metadata command to Joshua on input and pass them to the
- * decoder.
+ * decoder. A metadata object comprises a type and a list of arguments. The type is one of the
+ * following commands:
+ *
+ * - add_weight NAME VALUE
+ * - get_weights -> weights NAME VALUE NAME VALUE...
+ * - add_rule SOURCE ,,, TARGET
+ *
+ * In the future, this should be done through the API.
*/
public class MetaData {
+
+ /* The metadata request that came in */
String type;
+
+ /* The list of arguments */
String tokenString;
+ /* The response (if any) */
+ String response;
+
public MetaData(String message) {
message = message.substring(1, message.length() - 1).trim();
@@ -41,6 +55,8 @@ public class MetaData {
type = "";
tokenString = "";
}
+
+ response = "";
}
public String type() {
@@ -58,4 +74,12 @@ public class MetaData {
public String[] tokens() {
return this.tokens("\\s+");
}
+
+ public void setResponse(String response) {
+ this.response = response;
+ }
+
+ public String response() {
+ return this.response;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9e606d85/src/main/java/org/apache/joshua/decoder/ff/tm/hash_based/MemoryBasedBatchGrammar.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/ff/tm/hash_based/MemoryBasedBatchGrammar.java b/src/main/java/org/apache/joshua/decoder/ff/tm/hash_based/MemoryBasedBatchGrammar.java
index 9295fd0..f346e7a 100644
--- a/src/main/java/org/apache/joshua/decoder/ff/tm/hash_based/MemoryBasedBatchGrammar.java
+++ b/src/main/java/org/apache/joshua/decoder/ff/tm/hash_based/MemoryBasedBatchGrammar.java
@@ -83,6 +83,7 @@ public class MemoryBasedBatchGrammar extends AbstractGrammar {
super(joshuaConfiguration);
this.root = new MemoryBasedTrie();
this.joshuaConfiguration = joshuaConfiguration;
+ setSpanLimit(20);
}
public MemoryBasedBatchGrammar(String owner, JoshuaConfiguration joshuaConfiguration) {
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9e606d85/src/main/java/org/apache/joshua/decoder/io/JSONMessage.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/io/JSONMessage.java b/src/main/java/org/apache/joshua/decoder/io/JSONMessage.java
index 890de4f..cba6ec7 100644
--- a/src/main/java/org/apache/joshua/decoder/io/JSONMessage.java
+++ b/src/main/java/org/apache/joshua/decoder/io/JSONMessage.java
@@ -24,8 +24,29 @@ import java.util.List;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
+import org.apache.joshua.decoder.StructuredTranslation;
import org.apache.joshua.decoder.Translation;
+/**
+ * Represents a JSON object returned by the server. The object has the format
+ *
+ * { data: {
+ * translations: [
+ * { annotatedSource: "",
+ * translatedText: "",
+ * raw_nbest: [
+ * { hyp: "",
+ * totalScore: 0.0, } ]
+ * tokenization: { ... }
+ * translatedTextRaw: "",
+ * nbest: [
+ * { translatedText: "",
+ * translatedTextRaw: "",
+ * tokenization: { ... } } ] } ] } }
+ *
+ * @author post
+ */
+
public class JSONMessage {
public Data data = null;
public List<String> rules = null;
@@ -40,7 +61,45 @@ public class JSONMessage {
translations = new ArrayList<TranslationItem>();
}
}
+
+ public void addTranslation(Translation translation) {
+ String viterbi = translation.getStructuredTranslations().get(0).getTranslationString();
+
+ TranslationItem item = addTranslation(viterbi);
+
+ for (StructuredTranslation hyp: translation.getStructuredTranslations()) {
+ String text = hyp.getTranslationString();
+ float score = hyp.getTranslationScore();
+
+ item.addHypothesis(text, score);
+ }
+
+ if (translation.hasMetaData()) {
+ item.setMetaData(translation.getMetaData().response());
+ }
+ // old string-based k-best output
+ // String[] results = translation.toString().split("\\n");
+ // if (results.length > 0) {
+ // String rawTranslation = results[0].split(" \\|\\|\\| ")[1];
+ // JSONMessage.TranslationItem item = message.addTranslation(rawTranslation);
+ //
+ // for (String result: results) {
+ // String[] tokens = result.split(" \\|\\|\\| ");
+ // String rawResult = tokens[1];
+ // float score = Float.parseFloat(tokens[3]);
+ // item.addHypothesis(rawResult, score);
+ // }
+ // }
+ }
+
+ /**
+ * Adds a new Translation to the JSON object. A Translation represents one or more hypotheses
+ * (or k-best items)
+ *
+ * @param text
+ * @return the new TranslationItem object
+ */
public TranslationItem addTranslation(String text) {
if (data == null)
data = new Data();
@@ -52,6 +111,7 @@ public class JSONMessage {
public class TranslationItem {
public String translatedText;
+ public String metaData = null;
public List<NBestItem> raw_nbest;
public TranslationItem(String value) {
@@ -59,9 +119,19 @@ public class JSONMessage {
this.raw_nbest = new ArrayList<NBestItem>();
}
+ /**
+ * Adds a new item to the translation's list of k-best items
+ *
+ * @param hyp the hypothesis
+ * @param score its score
+ */
public void addHypothesis(String hyp, float score) {
this.raw_nbest.add(new NBestItem(hyp, score));
}
+
+ public void setMetaData(String msg) {
+ this.metaData = msg;
+ }
}
public class NBestItem {
@@ -80,30 +150,6 @@ public class JSONMessage {
rules.add(rule);
}
- public class MetaData {
-
- public MetaData() {
- }
- }
-
- public static JSONMessage buildMessage(Translation translation) {
- JSONMessage message = new JSONMessage();
- String[] results = translation.toString().split("\\n");
-
- if (results.length > 0) {
- String rawTranslation = results[0].split(" \\|\\|\\| ")[1];
- JSONMessage.TranslationItem item = message.addTranslation(rawTranslation);
-
- for (String result: results) {
- String[] tokens = result.split(" \\|\\|\\| ");
- String rawResult = tokens[1];
- float score = Float.parseFloat(tokens[3]);
- item.addHypothesis(rawResult, score);
- }
- }
- return message;
- }
-
public String toString() {
Gson gson = new GsonBuilder().setPrettyPrinting().create();
return gson.toJson(this) + "\n";
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9e606d85/src/main/java/org/apache/joshua/server/ServerThread.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/server/ServerThread.java b/src/main/java/org/apache/joshua/server/ServerThread.java
index d4dcc65..5913029 100644
--- a/src/main/java/org/apache/joshua/server/ServerThread.java
+++ b/src/main/java/org/apache/joshua/server/ServerThread.java
@@ -145,20 +145,23 @@ public class ServerThread extends Thread implements HttpHandler {
HashMap<String, String> params = queryToMap(URLDecoder.decode(client.getRequestURI().getQuery(), "UTF-8"));
String query = params.get("q");
-
+
BufferedReader reader = new BufferedReader(new StringReader(query));
TranslationRequestStream request = new TranslationRequestStream(reader, joshuaConfiguration);
Translations translations = decoder.decodeAll(request);
- OutputStream out = new HttpWriter(client);
-
+ JSONMessage message = new JSONMessage();
for (Translation translation: translations) {
- if (joshuaConfiguration.input_type == INPUT_TYPE.json || joshuaConfiguration.server_type == SERVER_TYPE.HTTP) {
- JSONMessage message = JSONMessage.buildMessage(translation);
- out.write(message.toString().getBytes());
- }
+ LOG.info("TRANSLATION: '{}' with {} k-best items", translation, translation.getStructuredTranslations().size());
+ message.addTranslation(translation);
}
+
+ OutputStream out = new HttpWriter(client);
+ out.write(message.toString().getBytes());
+ if (LOG.isDebugEnabled())
+ LOG.debug(message.toString());
out.close();
+
reader.close();
}
}