You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2015/06/07 17:52:09 UTC
svn commit: r1684041 - in /lucene/dev/branches/branch_5x: ./ lucene/
lucene/core/ lucene/core/src/test/org/apache/lucene/index/ solr/
solr/contrib/ solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/
solr/test-framework/ solr/test-framework/src/ja...
Author: uschindler
Date: Sun Jun 7 15:52:08 2015
New Revision: 1684041
URL: http://svn.apache.org/r1684041
Log:
Merged revision(s) 1684040 from lucene/dev/trunk:
LUCENE-6530: Refactoring: Simplify ProcessBuilder code at various places
Modified:
lucene/dev/branches/branch_5x/ (props changed)
lucene/dev/branches/branch_5x/lucene/ (props changed)
lucene/dev/branches/branch_5x/lucene/core/ (props changed)
lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java
lucene/dev/branches/branch_5x/solr/ (props changed)
lucene/dev/branches/branch_5x/solr/contrib/ (props changed)
lucene/dev/branches/branch_5x/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/SolrRecordWriter.java
lucene/dev/branches/branch_5x/solr/test-framework/ (props changed)
lucene/dev/branches/branch_5x/solr/test-framework/src/java/org/apache/solr/cloud/IpTables.java
Modified: lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java?rev=1684041&r1=1684040&r2=1684041&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java (original)
+++ lucene/dev/branches/branch_5x/lucene/core/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java Sun Jun 7 15:52:08 2015
@@ -22,11 +22,13 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.PrintStream;
+import java.lang.ProcessBuilder.Redirect;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.nio.file.Paths;
import java.nio.file.SimpleFileVisitor;
import java.util.ArrayList;
import java.util.List;
@@ -96,11 +98,7 @@ public class TestIndexWriterOnJRECrash e
@SuppressForbidden(reason = "ProcessBuilder requires java.io.File for CWD")
public void forkTest() throws Exception {
List<String> cmd = new ArrayList<>();
- cmd.add(System.getProperty("java.home")
- + System.getProperty("file.separator")
- + "bin"
- + System.getProperty("file.separator")
- + "java");
+ cmd.add(Paths.get(System.getProperty("java.home"), "bin", "java").toString());
cmd.add("-Xmx512m");
cmd.add("-Dtests.crashmode=true");
// passing NIGHTLY to this test makes it run for much longer, easier to catch it in the act...
@@ -112,19 +110,18 @@ public class TestIndexWriterOnJRECrash e
cmd.add(System.getProperty("java.class.path"));
cmd.add("org.junit.runner.JUnitCore");
cmd.add(getClass().getName());
- ProcessBuilder pb = new ProcessBuilder(cmd);
- pb.directory(tempDir.toFile());
- pb.redirectErrorStream(true);
+ ProcessBuilder pb = new ProcessBuilder(cmd)
+ .directory(tempDir.toFile())
+ .redirectInput(Redirect.INHERIT)
+ .redirectErrorStream(true);
Process p = pb.start();
// We pump everything to stderr.
PrintStream childOut = System.err;
Thread stdoutPumper = ThreadPumper.start(p.getInputStream(), childOut);
- Thread stderrPumper = ThreadPumper.start(p.getErrorStream(), childOut);
if (VERBOSE) childOut.println(">>> Begin subprocess output");
p.waitFor();
stdoutPumper.join();
- stderrPumper.join();
if (VERBOSE) childOut.println("<<< End subprocess output");
}
@@ -135,7 +132,7 @@ public class TestIndexWriterOnJRECrash e
@Override
public void run() {
try {
- byte [] buffer = new byte [1024];
+ byte[] buffer = new byte [1024];
int len;
while ((len = from.read(buffer)) != -1) {
if (VERBOSE) {
Modified: lucene/dev/branches/branch_5x/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/SolrRecordWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/SolrRecordWriter.java?rev=1684041&r1=1684040&r2=1684041&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/SolrRecordWriter.java (original)
+++ lucene/dev/branches/branch_5x/solr/contrib/map-reduce/src/java/org/apache/solr/hadoop/SolrRecordWriter.java Sun Jun 7 15:52:08 2015
@@ -36,7 +36,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
-import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -104,7 +103,7 @@ class SolrRecordWriter<K, V> extends Rec
public SolrRecordWriter(TaskAttemptContext context, Path outputShardDir, int batchSize) {
this.batchSize = batchSize;
- this.batch = new ArrayList(batchSize);
+ this.batch = new ArrayList<>(batchSize);
Configuration conf = context.getConfiguration();
// setLogLevel("org.apache.solr.core", "WARN");
@@ -134,9 +133,6 @@ class SolrRecordWriter<K, V> extends Rec
public static EmbeddedSolrServer createEmbeddedSolrServer(Path solrHomeDir, FileSystem fs, Path outputShardDir)
throws IOException {
- if (solrHomeDir == null) {
- throw new IOException("Unable to find solr home setting");
- }
LOG.info("Creating embedded Solr server with solrHomeDir: " + solrHomeDir + ", fs: " + fs + ", outputShardDir: " + outputShardDir);
Path solrDataDir = new Path(outputShardDir, "data");
@@ -186,7 +182,7 @@ class SolrRecordWriter<K, V> extends Rec
}
}
- public static void incrementCounter(TaskID taskId, Enum counterName, long incr) {
+ public static void incrementCounter(TaskID taskId, Enum<?> counterName, long incr) {
Reducer<?,?,?,?>.Context context = contextMap.get(taskId);
if (context != null) {
context.getCounter(counterName).increment(incr);
@@ -199,52 +195,18 @@ class SolrRecordWriter<K, V> extends Rec
}
public static Path findSolrConfig(Configuration conf) throws IOException {
- Path solrHome = null;
// FIXME when mrunit supports the new cache apis
//URI[] localArchives = context.getCacheArchives();
Path[] localArchives = DistributedCache.getLocalCacheArchives(conf);
- if (localArchives.length == 0) {
- throw new IOException(String.format(Locale.ENGLISH,
- "No local cache archives, where is %s:%s", SolrOutputFormat
- .getSetupOk(), SolrOutputFormat.getZipName(conf)));
- }
for (Path unpackedDir : localArchives) {
- // Only logged if debugging
- if (LOG.isDebugEnabled()) {
- LOG.debug(String.format(Locale.ENGLISH, "Examining unpack directory %s for %s",
- unpackedDir, SolrOutputFormat.getZipName(conf)));
-
- ProcessBuilder lsCmd = new ProcessBuilder(new String[] { "/bin/ls",
- "-lR", unpackedDir.toString() });
- lsCmd.redirectErrorStream();
- Process ls = lsCmd.start();
- byte[] buf = new byte[16 * 1024];
- InputStream all = ls.getInputStream();
- try {
- int count;
- while ((count = all.read(buf)) >= 0) {
- System.err.write(buf, 0, count);
- }
- } catch (IOException ignore) {
- } finally {
- all.close();
- }
- String exitValue;
- try {
- exitValue = String.valueOf(ls.waitFor());
- } catch (InterruptedException e) {
- exitValue = "interrupted";
- }
- System.err.format(Locale.ENGLISH, "Exit value of 'ls -lR' is %s%n", exitValue);
- }
if (unpackedDir.getName().equals(SolrOutputFormat.getZipName(conf))) {
LOG.info("Using this unpacked directory as solr home: {}", unpackedDir);
- solrHome = unpackedDir;
- break;
+ return unpackedDir;
}
}
-
- return solrHome;
+ throw new IOException(String.format(Locale.ENGLISH,
+ "No local cache archives, where is %s:%s", SolrOutputFormat
+ .getSetupOk(), SolrOutputFormat.getZipName(conf)));
}
/**
Modified: lucene/dev/branches/branch_5x/solr/test-framework/src/java/org/apache/solr/cloud/IpTables.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/solr/test-framework/src/java/org/apache/solr/cloud/IpTables.java?rev=1684041&r1=1684040&r2=1684041&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/solr/test-framework/src/java/org/apache/solr/cloud/IpTables.java (original)
+++ lucene/dev/branches/branch_5x/solr/test-framework/src/java/org/apache/solr/cloud/IpTables.java Sun Jun 7 15:52:08 2015
@@ -19,9 +19,6 @@ package org.apache.solr.cloud;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.io.PrintStream;
import java.util.Collections;
import java.util.HashSet;
import java.util.Set;
@@ -37,31 +34,6 @@ public class IpTables {
.getLogger(IpTables.class);
private static boolean ENABLED = Boolean.getBoolean("solr.tests.use.iptables");
- static class ThreadPumper {
-
- public ThreadPumper() {}
-
- public static Thread start(final InputStream from, final OutputStream to, final boolean verbose) {
- Thread t = new Thread() {
- @Override
- public void run() {
- try {
- byte [] buffer = new byte [1024];
- int len;
- while ((len = from.read(buffer)) != -1) {
- if (verbose) {
- to.write(buffer, 0, len);
- }
- }
- } catch (IOException e) {
- System.err.println("Couldn't pipe from the forked process: " + e.toString());
- }
- }
- };
- t.start();
- return t;
- }
- }
private static Set<Integer> BLOCK_PORTS = Collections.synchronizedSet(new HashSet<Integer>());
@@ -97,20 +69,10 @@ public class IpTables {
}
}
- private static void runCmd(String[] cmd) throws IOException, InterruptedException {
- ProcessBuilder pb = new ProcessBuilder(cmd);
-
- pb.redirectErrorStream(true);
- Process p = pb.start();
-
- // We pump everything to stderr.
- PrintStream childOut = System.err;
- Thread stdoutPumper = ThreadPumper.start(p.getInputStream(), childOut, true);
- Thread stderrPumper = ThreadPumper.start(p.getErrorStream(), childOut, true);
- if (true) childOut.println(">>> Begin subprocess output");
- p.waitFor();
- stdoutPumper.join();
- stderrPumper.join();
- if (true) childOut.println("<<< End subprocess output");
+ private static void runCmd(String... cmd) throws IOException, InterruptedException {
+ final int exitCode = new ProcessBuilder(cmd).inheritIO().start().waitFor();
+ if (exitCode != 0) {
+ throw new IOException("iptables process did not exit successfully, exit code was: " + exitCode);
+ }
}
}