You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@joshua.apache.org by mj...@apache.org on 2016/09/14 09:11:25 UTC
[03/43] incubator-joshua git commit: only cleanup Hadoop if grammar
was created
only cleanup Hadoop if grammar was created
Project: http://git-wip-us.apache.org/repos/asf/incubator-joshua/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-joshua/commit/d61eb538
Tree: http://git-wip-us.apache.org/repos/asf/incubator-joshua/tree/d61eb538
Diff: http://git-wip-us.apache.org/repos/asf/incubator-joshua/diff/d61eb538
Branch: refs/heads/7
Commit: d61eb538a23e6b1c75fb5eabdba9562c0a1d06e6
Parents: b4e7c00
Author: Matt Post <po...@cs.jhu.edu>
Authored: Wed Sep 7 16:39:41 2016 -0400
Committer: Matt Post <po...@cs.jhu.edu>
Committed: Wed Sep 7 16:39:41 2016 -0400
----------------------------------------------------------------------
scripts/training/pipeline.pl | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-joshua/blob/d61eb538/scripts/training/pipeline.pl
----------------------------------------------------------------------
diff --git a/scripts/training/pipeline.pl b/scripts/training/pipeline.pl
index c0e33d3..41a0cbb 100755
--- a/scripts/training/pipeline.pl
+++ b/scripts/training/pipeline.pl
@@ -1154,7 +1154,7 @@ if (! defined $GRAMMAR_FILE) {
system("mv $thrax_file.tmp $thrax_file");
$cachepipe->cmd("thrax-run",
- "hadoop jar $THRAX/bin/thrax.jar -D mapreduce.task.timeout=0 -D mapreduce.map.java.opts='-Xmx$HADOOP_MEM' -D mapreduce.reduce.java.opts='-Xmx$HADOOP_MEM' -D hadoop.tmp.dir=$TMPDIR $thrax_file $THRAXDIR > thrax.log 2>&1; rm -f grammar grammar.gz; hadoop fs -getmerge $THRAXDIR/final/ grammar.gz; hadoop fs -rm -r $THRAXDIR",
+ "hadoop jar $THRAX/bin/thrax.jar -D mapreduce.task.timeout=0 -D mapreduce.map.java.opts='-Xmx$HADOOP_MEM' -D mapreduce.reduce.java.opts='-Xmx$HADOOP_MEM' -D hadoop.tmp.dir=$TMPDIR $thrax_file $THRAXDIR > thrax.log 2>&1; rm -f grammar grammar.gz; hadoop fs -getmerge $THRAXDIR/final/ grammar.gz",
"$DATA_DIRS{train}/thrax-input-file",
$thrax_file,
"grammar.gz");
@@ -1162,6 +1162,11 @@ if (! defined $GRAMMAR_FILE) {
$GRAMMAR_FILE = "grammar.gz";
+ # cleanup if successful
+ if (-s $GRAMMAR_FILE) {
+ system("hadoop fs -rm -r $THRAXDIR");
+ }
+
} else {
print STDERR "* FATAL: There was no way to build a grammar, and none was passed in\n";