You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by mj...@apache.org on 2017/02/23 21:57:17 UTC
incubator-joshua git commit: now supports relative paths
Repository: incubator-joshua
Updated Branches:
refs/heads/master d0ad680c7 -> 9e43cf6e5
now supports relative paths
Joshua now computes all paths read from config files relative to location
of joshua.config. This eases building language packs and loading models from a library
because you no longer have to worry about cd()ing to the correct place first.
Any relative paths (e.g., those provided in language packs) will be canonicalized by
prefixing the full path to the parent directory containing joshua.config. Absolute
paths are not affected.
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/9e43cf6e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/9e43cf6e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/9e43cf6e
Branch: refs/heads/master
Commit: 9e43cf6e5a9c7dbcc48f0b97b3d6e6f23205f794
Parents: d0ad680
Author: Matt Post <po...@cs.jhu.edu>
Authored: Thu Feb 23 16:54:42 2017 -0500
Committer: Matt Post <po...@cs.jhu.edu>
Committed: Thu Feb 23 16:54:42 2017 -0500
----------------------------------------------------------------------
scripts/language-pack/copy_model.py | 11 ++++---
.../org/apache/joshua/decoder/ArgsParser.java | 4 +++
.../java/org/apache/joshua/decoder/Decoder.java | 2 +-
.../joshua/decoder/JoshuaConfiguration.java | 30 ++++++++++++++++++++
.../joshua/decoder/ff/lm/LanguageModelFF.java | 2 +-
5 files changed, 41 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9e43cf6e/scripts/language-pack/copy_model.py
----------------------------------------------------------------------
diff --git a/scripts/language-pack/copy_model.py b/scripts/language-pack/copy_model.py
index 68ab8ef..4b902b5 100755
--- a/scripts/language-pack/copy_model.py
+++ b/scripts/language-pack/copy_model.py
@@ -97,13 +97,12 @@ fi
set -u
bundledir=$(dirname $0)
-cd $bundledir # relative paths are now safe....
-exec java -mx${mem} \
- -Dfile.encoding=utf8 \
- -Djava.library.path=./lib \
- -cp ./target/joshua-*-jar-with-dependencies.jar \
- org.apache.joshua.decoder.JoshuaDecoder -c joshua.config -v 0 "$@"
+exec java -mx${mem} \\
+ -Dfile.encoding=utf8 \\
+ -Djava.library.path=$bundledir/lib \\
+ -cp $bundledir/target/joshua-*-jar-with-dependencies.jar \\
+ org.apache.joshua.decoder.JoshuaDecoder -c $bundledir/joshua.config -v 0 "$@"
""" % mem
return text
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9e43cf6e/src/main/java/org/apache/joshua/decoder/ArgsParser.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/ArgsParser.java b/src/main/java/org/apache/joshua/decoder/ArgsParser.java
index 33b808c..44cd57b 100644
--- a/src/main/java/org/apache/joshua/decoder/ArgsParser.java
+++ b/src/main/java/org/apache/joshua/decoder/ArgsParser.java
@@ -18,6 +18,7 @@
*/
package org.apache.joshua.decoder;
+import java.io.File;
import java.io.IOException;
import java.nio.charset.Charset;
import java.nio.file.Files;
@@ -85,6 +86,9 @@ public class ArgsParser {
try {
LOG.info("Parameters read from configuration file: {}", getConfigFile());
config.readConfigFile(getConfigFile());
+
+ // Now set the parent directory
+ config.setConfigFilePath(new File(getConfigFile()).getCanonicalFile().getParent());
} catch (IOException e) {
throw new RuntimeException(e);
}
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9e43cf6e/src/main/java/org/apache/joshua/decoder/Decoder.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/Decoder.java b/src/main/java/org/apache/joshua/decoder/Decoder.java
index e105fd2..40bcd32 100644
--- a/src/main/java/org/apache/joshua/decoder/Decoder.java
+++ b/src/main/java/org/apache/joshua/decoder/Decoder.java
@@ -450,7 +450,7 @@ public class Decoder {
String owner = parsedArgs.get("owner");
int span_limit = Integer.parseInt(parsedArgs.get("maxspan"));
- String path = parsedArgs.get("path");
+ String path = joshuaConfiguration.getFilePath(parsedArgs.get("path"));
Grammar grammar;
if (type.equals("moses") || type.equals("phrase")) {
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9e43cf6e/src/main/java/org/apache/joshua/decoder/JoshuaConfiguration.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/JoshuaConfiguration.java b/src/main/java/org/apache/joshua/decoder/JoshuaConfiguration.java
index 6fd73ba..3532241 100644
--- a/src/main/java/org/apache/joshua/decoder/JoshuaConfiguration.java
+++ b/src/main/java/org/apache/joshua/decoder/JoshuaConfiguration.java
@@ -22,6 +22,7 @@ import static org.apache.joshua.util.FormatUtils.cleanNonTerminal;
import static org.apache.joshua.util.FormatUtils.ensureNonTerminalBrackets;
import java.io.File;
+import java.nio.file.Paths;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
@@ -53,6 +54,10 @@ public class JoshuaConfiguration {
private static final Logger LOG = LoggerFactory.getLogger(JoshuaConfiguration.class);
+ // This records the directory from which relative file references are normalized.
+ // If a config file is loaded, that will override the current directory.
+ public String modelRootPath = Paths.get(".").toAbsolutePath().normalize().toString();
+
// whether to construct a StructuredTranslation object for each request instead of
// printing to stdout. Used when the Decoder is used from Java directly.
public Boolean use_structured_output = false;
@@ -365,6 +370,31 @@ public class JoshuaConfiguration {
}
}
+ /**
+ * This records the path of the config file passed as a command-line argument
+ * (-c or -config). It is used to produce absolute path names when relative ones are found
+ * in the config file.
+ */
+ public void setConfigFilePath(String path) {
+ this.modelRootPath = path;
+ }
+
+ /**
+ * Returns the absolute file path of the argument. Files that are already absolute path names
+ * are returned unmodified, but relative file names have the `modelRootPath` prepended (if
+ * it was set).
+ */
+ public String getFilePath(String filename) {
+ if (filename.startsWith("/"))
+ return filename;
+ else if (this.modelRootPath != null)
+ return this.modelRootPath + "/" + filename;
+ else {
+ LOG.warn("File '{}' is a relative path but no config file path was set", filename);
+ return filename;
+ }
+ }
+
public void readConfigFile(String configFile) throws IOException {
LineReader configReader = new LineReader(configFile, false);
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/9e43cf6e/src/main/java/org/apache/joshua/decoder/ff/lm/LanguageModelFF.java
----------------------------------------------------------------------
diff --git a/src/main/java/org/apache/joshua/decoder/ff/lm/LanguageModelFF.java b/src/main/java/org/apache/joshua/decoder/ff/lm/LanguageModelFF.java
index a29c754..959e48e 100644
--- a/src/main/java/org/apache/joshua/decoder/ff/lm/LanguageModelFF.java
+++ b/src/main/java/org/apache/joshua/decoder/ff/lm/LanguageModelFF.java
@@ -114,7 +114,7 @@ public class LanguageModelFF extends StatefulFF {
this.type = parsedArgs.get("lm_type");
this.ngramOrder = Integer.parseInt(parsedArgs.get("lm_order"));
- this.path = parsedArgs.get("lm_file");
+ this.path = config.getFilePath(parsedArgs.get("lm_file"));
if (parsedArgs.containsKey("class_map")) {
this.isClassLM = true;