You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@giraph.apache.org by ac...@apache.org on 2012/09/25 19:40:21 UTC
svn commit: r1390014 [4/4] - in /giraph/trunk: ./
src/main/java/org/apache/giraph/ src/main/java/org/apache/giraph/benchmark/
src/main/java/org/apache/giraph/bsp/ src/main/java/org/apache/giraph/comm/
src/main/java/org/apache/giraph/comm/messages/ src/...
Modified: giraph/trunk/src/main/java/org/apache/giraph/utils/InternalVertexRunner.java
URL: http://svn.apache.org/viewvc/giraph/trunk/src/main/java/org/apache/giraph/utils/InternalVertexRunner.java?rev=1390014&r1=1390013&r2=1390014&view=diff
==============================================================================
--- giraph/trunk/src/main/java/org/apache/giraph/utils/InternalVertexRunner.java (original)
+++ giraph/trunk/src/main/java/org/apache/giraph/utils/InternalVertexRunner.java Tue Sep 25 17:40:18 2012
@@ -25,6 +25,7 @@ import java.util.Properties;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
+import org.apache.giraph.GiraphConfiguration;
import org.apache.giraph.graph.GiraphJob;
import org.apache.giraph.graph.MasterCompute;
import org.apache.giraph.graph.Vertex;
@@ -151,30 +152,34 @@ public class InternalVertexRunner {
// Create and configure the job to run the vertex
GiraphJob job = new GiraphJob(vertexClass.getName());
- job.setVertexClass(vertexClass);
- job.setVertexInputFormatClass(vertexInputFormatClass);
- job.setVertexOutputFormatClass(vertexOutputFormatClass);
+ job.getConfiguration().setVertexClass(vertexClass);
+ job.getConfiguration().setVertexInputFormatClass(
+ vertexInputFormatClass);
+ job.getConfiguration().setVertexOutputFormatClass(
+ vertexOutputFormatClass);
if (workerContextClass != null) {
- job.setWorkerContextClass(workerContextClass);
+ job.getConfiguration().setWorkerContextClass(workerContextClass);
}
if (vertexCombinerClass != null) {
- job.setVertexCombinerClass(vertexCombinerClass);
+ job.getConfiguration().setVertexCombinerClass(vertexCombinerClass);
}
if (masterComputeClass != null) {
- job.setMasterComputeClass(masterComputeClass);
+ job.getConfiguration().setMasterComputeClass(masterComputeClass);
}
- job.setWorkerConfiguration(1, 1, 100.0f);
+ job.getConfiguration().setWorkerConfiguration(1, 1, 100.0f);
Configuration conf = job.getConfiguration();
- conf.setBoolean(GiraphJob.SPLIT_MASTER_WORKER, false);
- conf.setBoolean(GiraphJob.LOCAL_TEST_MODE, true);
- conf.set(GiraphJob.ZOOKEEPER_LIST, "localhost:" +
+ conf.setBoolean(GiraphConfiguration.SPLIT_MASTER_WORKER, false);
+ conf.setBoolean(GiraphConfiguration.LOCAL_TEST_MODE, true);
+ conf.set(GiraphConfiguration.ZOOKEEPER_LIST, "localhost:" +
String.valueOf(LOCAL_ZOOKEEPER_PORT));
- conf.set(GiraphJob.ZOOKEEPER_DIR, zkDir.toString());
- conf.set(GiraphJob.ZOOKEEPER_MANAGER_DIRECTORY,
+ conf.set(GiraphConfiguration.ZOOKEEPER_DIR, zkDir.toString());
+ conf.set(GiraphConfiguration.ZOOKEEPER_MANAGER_DIRECTORY,
zkMgrDir.toString());
- conf.set(GiraphJob.CHECKPOINT_DIRECTORY, checkpointsDir.toString());
+ conf.set(
+ GiraphConfiguration.CHECKPOINT_DIRECTORY,
+ checkpointsDir.toString());
for (Map.Entry<String, String> param : params.entrySet()) {
conf.set(param.getKey(), param.getValue());
Modified: giraph/trunk/src/main/java/org/apache/giraph/utils/ReflectionUtils.java
URL: http://svn.apache.org/viewvc/giraph/trunk/src/main/java/org/apache/giraph/utils/ReflectionUtils.java?rev=1390014&r1=1390013&r2=1390014&view=diff
==============================================================================
--- giraph/trunk/src/main/java/org/apache/giraph/utils/ReflectionUtils.java (original)
+++ giraph/trunk/src/main/java/org/apache/giraph/utils/ReflectionUtils.java Tue Sep 25 17:40:18 2012
@@ -29,6 +29,8 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.giraph.ImmutableClassesGiraphConfigurable;
+import org.apache.giraph.ImmutableClassesGiraphConfiguration;
/**
* Helper methods to get type arguments to generic classes. Courtesy of
@@ -159,4 +161,34 @@ public class ReflectionUtils {
}
throw new NoSuchFieldException();
}
+
+ /**
+ * Instantiate classes that are ImmmutableClasssesGiraphConfigurable
+ *
+ * @param theClass Class to instantiate
+ * @param configuration Giraph configuration, may be null
+ * @param <T> Type to instantiate
+ * @return Newly instantiated object with configuration set if possible
+ */
+ @SuppressWarnings("unchecked")
+ public static <T> T newInstance(
+ Class<T> theClass,
+ ImmutableClassesGiraphConfiguration configuration) {
+ T result = null;
+ try {
+ result = theClass.newInstance();
+ } catch (InstantiationException e) {
+ throw new IllegalStateException(
+ "newInstance: Couldn't instantiate " + theClass.getName(), e);
+ } catch (IllegalAccessException e) {
+ throw new IllegalStateException(
+ "newInstance: Illegal access " + theClass.getName(), e);
+ }
+ if (configuration != null) {
+ if (result instanceof ImmutableClassesGiraphConfigurable) {
+ ((ImmutableClassesGiraphConfigurable) result).setConf(configuration);
+ }
+ }
+ return result;
+ }
}
Modified: giraph/trunk/src/main/java/org/apache/giraph/utils/WritableUtils.java
URL: http://svn.apache.org/viewvc/giraph/trunk/src/main/java/org/apache/giraph/utils/WritableUtils.java?rev=1390014&r1=1390013&r2=1390014&view=diff
==============================================================================
--- giraph/trunk/src/main/java/org/apache/giraph/utils/WritableUtils.java (original)
+++ giraph/trunk/src/main/java/org/apache/giraph/utils/WritableUtils.java Tue Sep 25 17:40:18 2012
@@ -193,7 +193,7 @@ public class WritableUtils {
}
/**
- * Read fields from byteArray to a list of Writeable objects.
+ * Read fields from byteArray to a list of Writable objects.
*
* @param byteArray Byte array to find the fields in.
* @param writableClass Class of the objects to instantiate.
Modified: giraph/trunk/src/main/java/org/apache/giraph/zk/ZooKeeperManager.java
URL: http://svn.apache.org/viewvc/giraph/trunk/src/main/java/org/apache/giraph/zk/ZooKeeperManager.java?rev=1390014&r1=1390013&r2=1390014&view=diff
==============================================================================
--- giraph/trunk/src/main/java/org/apache/giraph/zk/ZooKeeperManager.java (original)
+++ giraph/trunk/src/main/java/org/apache/giraph/zk/ZooKeeperManager.java Tue Sep 25 17:40:18 2012
@@ -22,7 +22,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.io.Closeables;
import org.apache.commons.io.FileUtils;
-import org.apache.giraph.graph.GiraphJob;
+import org.apache.giraph.GiraphConfiguration;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -48,7 +48,7 @@ import java.util.Arrays;
import java.util.List;
import java.util.Map;
-import static org.apache.giraph.graph.GiraphJob.BASE_ZNODE_KEY;
+import static org.apache.giraph.GiraphConfiguration.BASE_ZNODE_KEY;
/**
@@ -136,7 +136,7 @@ public class ZooKeeperManager {
taskPartition = conf.getInt("mapred.task.partition", -1);
jobId = conf.get("mapred.job.id", "Unknown Job");
baseDirectory =
- new Path(conf.get(GiraphJob.ZOOKEEPER_MANAGER_DIRECTORY,
+ new Path(conf.get(GiraphConfiguration.ZOOKEEPER_MANAGER_DIRECTORY,
getFinalZooKeeperPath()));
taskDirectory = new Path(baseDirectory,
"_task");
@@ -146,24 +146,24 @@ public class ZooKeeperManager {
Integer.toString(taskPartition) +
COMPUTATION_DONE_SUFFIX);
pollMsecs = conf.getInt(
- GiraphJob.ZOOKEEPER_SERVERLIST_POLL_MSECS,
- GiraphJob.ZOOKEEPER_SERVERLIST_POLL_MSECS_DEFAULT);
+ GiraphConfiguration.ZOOKEEPER_SERVERLIST_POLL_MSECS,
+ GiraphConfiguration.ZOOKEEPER_SERVERLIST_POLL_MSECS_DEFAULT);
serverCount = conf.getInt(
- GiraphJob.ZOOKEEPER_SERVER_COUNT,
- GiraphJob.ZOOKEEPER_SERVER_COUNT_DEFAULT);
+ GiraphConfiguration.ZOOKEEPER_SERVER_COUNT,
+ GiraphConfiguration.ZOOKEEPER_SERVER_COUNT_DEFAULT);
String jobLocalDir = conf.get("job.local.dir");
if (jobLocalDir != null) { // for non-local jobs
zkDirDefault = jobLocalDir +
"/_bspZooKeeper";
} else {
zkDirDefault = System.getProperty("user.dir") + "/" +
- GiraphJob.ZOOKEEPER_MANAGER_DIR_DEFAULT;
+ GiraphConfiguration.ZOOKEEPER_MANAGER_DIR_DEFAULT;
}
- zkDir = conf.get(GiraphJob.ZOOKEEPER_DIR, zkDirDefault);
+ zkDir = conf.get(GiraphConfiguration.ZOOKEEPER_DIR, zkDirDefault);
configFilePath = zkDir + "/zoo.cfg";
zkBasePort = conf.getInt(
- GiraphJob.ZOOKEEPER_SERVER_PORT,
- GiraphJob.ZOOKEEPER_SERVER_PORT_DEFAULT);
+ GiraphConfiguration.ZOOKEEPER_SERVER_PORT,
+ GiraphConfiguration.ZOOKEEPER_SERVER_PORT_DEFAULT);
myHostname = InetAddress.getLocalHost().getCanonicalHostName();
@@ -176,7 +176,7 @@ public class ZooKeeperManager {
* @return directory path with job id
*/
private String getFinalZooKeeperPath() {
- return GiraphJob.ZOOKEEPER_MANAGER_DIR_DEFAULT + "/" + jobId;
+ return GiraphConfiguration.ZOOKEEPER_MANAGER_DIR_DEFAULT + "/" + jobId;
}
/**
@@ -271,7 +271,7 @@ public class ZooKeeperManager {
"for base directory " + baseDirectory + ". If there is an " +
"issue with this directory, please set an accesible " +
"base directory with the Hadoop configuration option " +
- GiraphJob.ZOOKEEPER_MANAGER_DIRECTORY);
+ GiraphConfiguration.ZOOKEEPER_MANAGER_DIRECTORY);
}
Path myCandidacyPath = new Path(
@@ -521,25 +521,25 @@ public class ZooKeeperManager {
try {
writer = new FileWriter(configFilePath);
writer.write("tickTime=" +
- GiraphJob.DEFAULT_ZOOKEEPER_TICK_TIME + "\n");
+ GiraphConfiguration.DEFAULT_ZOOKEEPER_TICK_TIME + "\n");
writer.write("dataDir=" + this.zkDir + "\n");
writer.write("clientPort=" + zkBasePort + "\n");
writer.write("maxClientCnxns=" +
- GiraphJob.DEFAULT_ZOOKEEPER_MAX_CLIENT_CNXNS +
+ GiraphConfiguration.DEFAULT_ZOOKEEPER_MAX_CLIENT_CNXNS +
"\n");
int minSessionTimeout = conf.getInt(
- GiraphJob.ZOOKEEPER_MIN_SESSION_TIMEOUT,
- GiraphJob.DEFAULT_ZOOKEEPER_MIN_SESSION_TIMEOUT);
+ GiraphConfiguration.ZOOKEEPER_MIN_SESSION_TIMEOUT,
+ GiraphConfiguration.DEFAULT_ZOOKEEPER_MIN_SESSION_TIMEOUT);
writer.write("minSessionTimeout=" + minSessionTimeout + "\n");
writer.write("maxSessionTimeout=" +
- GiraphJob.DEFAULT_ZOOKEEPER_MAX_SESSION_TIMEOUT +
+ GiraphConfiguration.DEFAULT_ZOOKEEPER_MAX_SESSION_TIMEOUT +
"\n");
writer.write("initLimit=" +
- GiraphJob.DEFAULT_ZOOKEEPER_INIT_LIMIT + "\n");
+ GiraphConfiguration.DEFAULT_ZOOKEEPER_INIT_LIMIT + "\n");
writer.write("syncLimit=" +
- GiraphJob.DEFAULT_ZOOKEEPER_SYNC_LIMIT + "\n");
+ GiraphConfiguration.DEFAULT_ZOOKEEPER_SYNC_LIMIT + "\n");
writer.write("snapCount=" +
- GiraphJob.DEFAULT_ZOOKEEPER_SNAP_COUNT + "\n");
+ GiraphConfiguration.DEFAULT_ZOOKEEPER_SNAP_COUNT + "\n");
if (serverList.size() != 1) {
writer.write("electionAlg=0\n");
for (int i = 0; i < serverList.size(); ++i) {
@@ -595,14 +595,14 @@ public class ZooKeeperManager {
}
commandList.add(javaHome + "/bin/java");
String zkJavaOptsString =
- conf.get(GiraphJob.ZOOKEEPER_JAVA_OPTS,
- GiraphJob.ZOOKEEPER_JAVA_OPTS_DEFAULT);
+ conf.get(GiraphConfiguration.ZOOKEEPER_JAVA_OPTS,
+ GiraphConfiguration.ZOOKEEPER_JAVA_OPTS_DEFAULT);
String[] zkJavaOptsArray = zkJavaOptsString.split(" ");
if (zkJavaOptsArray != null) {
commandList.addAll(Arrays.asList(zkJavaOptsArray));
}
commandList.add("-cp");
- Path fullJarPath = new Path(conf.get(GiraphJob.ZOOKEEPER_JAR));
+ Path fullJarPath = new Path(conf.get(GiraphConfiguration.ZOOKEEPER_JAR));
commandList.add(fullJarPath.toString());
commandList.add(QuorumPeerMain.class.getName());
commandList.add(configFilePath);
Modified: giraph/trunk/src/test/java/org/apache/giraph/BspCase.java
URL: http://svn.apache.org/viewvc/giraph/trunk/src/test/java/org/apache/giraph/BspCase.java?rev=1390014&r1=1390013&r2=1390014&view=diff
==============================================================================
--- giraph/trunk/src/test/java/org/apache/giraph/BspCase.java (original)
+++ giraph/trunk/src/test/java/org/apache/giraph/BspCase.java Tue Sep 25 17:40:18 2012
@@ -28,6 +28,11 @@ import com.google.common.base.Preconditi
import com.google.common.io.Closeables;
import org.apache.giraph.examples.GeneratedVertexReader;
import org.apache.giraph.graph.GiraphJob;
+import org.apache.giraph.graph.MasterCompute;
+import org.apache.giraph.graph.Vertex;
+import org.apache.giraph.graph.VertexInputFormat;
+import org.apache.giraph.graph.VertexOutputFormat;
+import org.apache.giraph.graph.WorkerContext;
import org.apache.giraph.utils.FileUtils;
import org.apache.giraph.zk.ZooKeeperExt;
import org.apache.hadoop.conf.Configuration;
@@ -71,7 +76,7 @@ public class BspCase implements Watcher
*/
public final Configuration setupConfiguration(GiraphJob job)
throws IOException {
- Configuration conf = job.getConfiguration();
+ GiraphConfiguration conf = job.getConfiguration();
conf.set("mapred.jar", getJarLocation());
// Allow this test to be run on a real Hadoop setup
@@ -80,20 +85,20 @@ public class BspCase implements Watcher
jobTracker + " with jar path " + getJarLocation()
+ " for " + getName());
conf.set("mapred.job.tracker", jobTracker);
- job.setWorkerConfiguration(getNumWorkers(), getNumWorkers(), 100.0f);
+ conf.setWorkerConfiguration(getNumWorkers(), getNumWorkers(), 100.0f);
}
else {
System.out.println("setup: Using local job runner with " +
"location " + getJarLocation() + " for " + getName());
- job.setWorkerConfiguration(1, 1, 100.0f);
+ conf.setWorkerConfiguration(1, 1, 100.0f);
// Single node testing
- conf.setBoolean(GiraphJob.SPLIT_MASTER_WORKER, false);
+ conf.setBoolean(GiraphConfiguration.SPLIT_MASTER_WORKER, false);
}
- conf.setInt(GiraphJob.POLL_ATTEMPTS, 10);
- conf.setInt(GiraphJob.POLL_MSECS, 3 * 1000);
- conf.setInt(GiraphJob.ZOOKEEPER_SERVERLIST_POLL_MSECS, 500);
+ conf.setInt(GiraphConfiguration.POLL_ATTEMPTS, 10);
+ conf.setInt(GiraphConfiguration.POLL_MSECS, 3 * 1000);
+ conf.setInt(GiraphConfiguration.ZOOKEEPER_SERVERLIST_POLL_MSECS, 500);
if (getZooKeeperList() != null) {
- job.setZooKeeperConfiguration(getZooKeeperList());
+ conf.setZooKeeperConfiguration(getZooKeeperList());
}
// GeneratedInputSplit will generate 5 vertices
conf.setLong(GeneratedVertexReader.READER_VERTICES, 5);
@@ -108,10 +113,10 @@ public class BspCase implements Watcher
FileUtils.deletePath(conf, zkManagerDir);
FileUtils.deletePath(conf, checkPointDir);
- conf.set(GiraphJob.ZOOKEEPER_DIR, zookeeperDir.toString());
- conf.set(GiraphJob.ZOOKEEPER_MANAGER_DIRECTORY,
+ conf.set(GiraphConfiguration.ZOOKEEPER_DIR, zookeeperDir.toString());
+ conf.set(GiraphConfiguration.ZOOKEEPER_MANAGER_DIRECTORY,
zkManagerDir.toString());
- conf.set(GiraphJob.CHECKPOINT_DIRECTORY, checkPointDir.toString());
+ conf.set(GiraphConfiguration.CHECKPOINT_DIRECTORY, checkPointDir.toString());
return conf;
}
@@ -135,8 +140,10 @@ public class BspCase implements Watcher
* @return fully configured job instance
* @throws IOException
*/
- protected GiraphJob prepareJob(String name, Class<?> vertexClass,
- Class<?> vertexInputFormatClass) throws IOException {
+ protected GiraphJob prepareJob(String name,
+ Class<? extends Vertex> vertexClass,
+ Class<? extends VertexInputFormat> vertexInputFormatClass)
+ throws IOException {
return prepareJob(name, vertexClass, vertexInputFormatClass, null,
null);
}
@@ -152,8 +159,10 @@ public class BspCase implements Watcher
* @return fully configured job instance
* @throws IOException
*/
- protected GiraphJob prepareJob(String name, Class<?> vertexClass,
- Class<?> vertexInputFormatClass, Class<?> vertexOutputFormatClass,
+ protected GiraphJob prepareJob(String name,
+ Class<? extends Vertex> vertexClass,
+ Class<? extends VertexInputFormat> vertexInputFormatClass,
+ Class<? extends VertexOutputFormat> vertexOutputFormatClass,
Path outputPath) throws IOException {
return prepareJob(name, vertexClass, null, vertexInputFormatClass,
vertexOutputFormatClass, outputPath);
@@ -171,9 +180,13 @@ public class BspCase implements Watcher
* @return fully configured job instance
* @throws IOException
*/
- protected GiraphJob prepareJob(String name, Class<?> vertexClass,
- Class<?> workerContextClass, Class<?> vertexInputFormatClass,
- Class<?> vertexOutputFormatClass, Path outputPath) throws IOException {
+ protected GiraphJob prepareJob(String name,
+ Class<? extends Vertex> vertexClass,
+ Class<? extends WorkerContext> workerContextClass,
+ Class<? extends VertexInputFormat> vertexInputFormatClass,
+ Class<? extends VertexOutputFormat> vertexOutputFormatClass,
+ Path outputPath) throws
+ IOException {
return prepareJob(name, vertexClass, workerContextClass, null,
vertexInputFormatClass, vertexOutputFormatClass, outputPath);
}
@@ -191,23 +204,27 @@ public class BspCase implements Watcher
* @return fully configured job instance
* @throws IOException
*/
- protected GiraphJob prepareJob(String name, Class<?> vertexClass,
- Class<?> workerContextClass, Class<?> masterComputeClass,
- Class<?> vertexInputFormatClass, Class<?> vertexOutputFormatClass,
+ protected GiraphJob prepareJob(String name,
+ Class<? extends Vertex> vertexClass,
+ Class<? extends WorkerContext> workerContextClass,
+ Class<? extends MasterCompute> masterComputeClass,
+ Class<? extends VertexInputFormat> vertexInputFormatClass,
+ Class<? extends VertexOutputFormat> vertexOutputFormatClass,
Path outputPath) throws IOException {
GiraphJob job = new GiraphJob(name);
setupConfiguration(job);
- job.setVertexClass(vertexClass);
- job.setVertexInputFormatClass(vertexInputFormatClass);
+ job.getConfiguration().setVertexClass(vertexClass);
+ job.getConfiguration().setVertexInputFormatClass(vertexInputFormatClass);
if (workerContextClass != null) {
- job.setWorkerContextClass(workerContextClass);
+ job.getConfiguration().setWorkerContextClass(workerContextClass);
}
if (masterComputeClass != null) {
- job.setMasterComputeClass(masterComputeClass);
+ job.getConfiguration().setMasterComputeClass(masterComputeClass);
}
if (vertexOutputFormatClass != null) {
- job.setVertexOutputFormatClass(vertexOutputFormatClass);
+ job.getConfiguration().setVertexOutputFormatClass(
+ vertexOutputFormatClass);
}
if (outputPath != null) {
removeAndSetOutput(job, outputPath);
Modified: giraph/trunk/src/test/java/org/apache/giraph/TestAggregatorsHandling.java
URL: http://svn.apache.org/viewvc/giraph/trunk/src/test/java/org/apache/giraph/TestAggregatorsHandling.java?rev=1390014&r1=1390013&r2=1390014&view=diff
==============================================================================
--- giraph/trunk/src/test/java/org/apache/giraph/TestAggregatorsHandling.java (original)
+++ giraph/trunk/src/test/java/org/apache/giraph/TestAggregatorsHandling.java Tue Sep 25 17:40:18 2012
@@ -40,9 +40,9 @@ public class TestAggregatorsHandling ext
GiraphJob job = prepareJob(getCallingMethodName(),
AggregatorsTestVertex.class,
SimplePageRankVertex.SimplePageRankVertexInputFormat.class);
- job.setMasterComputeClass(
+ job.getConfiguration().setMasterComputeClass(
AggregatorsTestVertex.AggregatorsTestMasterCompute.class);
- job.getConfiguration().setBoolean(GiraphJob.USE_NETTY, true);
+ job.getConfiguration().setBoolean(GiraphConfiguration.USE_NETTY, true);
assertTrue(job.run(true));
}
}
Modified: giraph/trunk/src/test/java/org/apache/giraph/TestAutoCheckpoint.java
URL: http://svn.apache.org/viewvc/giraph/trunk/src/test/java/org/apache/giraph/TestAutoCheckpoint.java?rev=1390014&r1=1390013&r2=1390014&view=diff
==============================================================================
--- giraph/trunk/src/test/java/org/apache/giraph/TestAutoCheckpoint.java (original)
+++ giraph/trunk/src/test/java/org/apache/giraph/TestAutoCheckpoint.java Tue Sep 25 17:40:18 2012
@@ -57,7 +57,7 @@ public class TestAutoCheckpoint extends
}
Path outputPath = getTempPath(getCallingMethodName());
GiraphJob job = prepareJob(getCallingMethodName(),
- SimpleCheckpointVertex.class,
+ SimpleCheckpointVertex.SimpleCheckpointComputation.class,
SimpleCheckpointVertex.SimpleCheckpointVertexWorkerContext.class,
SimpleCheckpointVertex.SimpleCheckpointVertexMasterCompute.class,
SimpleSuperstepVertexInputFormat.class,
@@ -69,13 +69,13 @@ public class TestAutoCheckpoint extends
conf.setInt("mapred.map.max.attempts", 4);
// Trigger failure faster
conf.setInt("mapred.task.timeout", 30000);
- conf.setInt(GiraphJob.POLL_MSECS, 5000);
- conf.setInt(GiraphJob.CHECKPOINT_FREQUENCY, 2);
- conf.set(GiraphJob.CHECKPOINT_DIRECTORY,
+ conf.setInt(GiraphConfiguration.POLL_MSECS, 5000);
+ conf.setInt(GiraphConfiguration.CHECKPOINT_FREQUENCY, 2);
+ conf.set(GiraphConfiguration.CHECKPOINT_DIRECTORY,
getTempPath("_singleFaultCheckpoints").toString());
- conf.setBoolean(GiraphJob.CLEANUP_CHECKPOINTS_AFTER_SUCCESS, false);
- conf.setInt(GiraphJob.ZOOKEEPER_SESSION_TIMEOUT, 10000);
- conf.setInt(GiraphJob.ZOOKEEPER_MIN_SESSION_TIMEOUT, 10000);
+ conf.setBoolean(GiraphConfiguration.CLEANUP_CHECKPOINTS_AFTER_SUCCESS, false);
+ conf.setInt(GiraphConfiguration.ZOOKEEPER_SESSION_TIMEOUT, 10000);
+ conf.setInt(GiraphConfiguration.ZOOKEEPER_MIN_SESSION_TIMEOUT, 10000);
assertTrue(job.run(true));
}
Modified: giraph/trunk/src/test/java/org/apache/giraph/TestBspBasic.java
URL: http://svn.apache.org/viewvc/giraph/trunk/src/test/java/org/apache/giraph/TestBspBasic.java?rev=1390014&r1=1390013&r2=1390014&view=diff
==============================================================================
--- giraph/trunk/src/test/java/org/apache/giraph/TestBspBasic.java (original)
+++ giraph/trunk/src/test/java/org/apache/giraph/TestBspBasic.java Tue Sep 25 17:40:18 2012
@@ -30,7 +30,7 @@ import org.apache.giraph.examples.Simple
import org.apache.giraph.examples.SimpleSuperstepVertex;
import org.apache.giraph.examples.SimpleSuperstepVertex.SimpleSuperstepVertexInputFormat;
import org.apache.giraph.examples.SimpleSuperstepVertex.SimpleSuperstepVertexOutputFormat;
-import org.apache.giraph.graph.BspUtils;
+import org.apache.giraph.graph.EdgeListVertex;
import org.apache.giraph.graph.GiraphJob;
import org.apache.giraph.graph.LocalityInfoSorter;
import org.apache.giraph.graph.GraphState;
@@ -118,13 +118,14 @@ public class TestBspBasic extends BspCas
GraphState<LongWritable, IntWritable, FloatWritable, IntWritable> gs =
new GraphState<LongWritable, IntWritable,
FloatWritable, IntWritable>();
+ ImmutableClassesGiraphConfiguration configuration =
+ new ImmutableClassesGiraphConfiguration(job.getConfiguration());
Vertex<LongWritable, IntWritable, FloatWritable, IntWritable> vertex =
- BspUtils.createVertex(job.getConfiguration());
- vertex.initialize(null, null, null, null);
+ configuration.createVertex();
System.out.println("testInstantiateVertex: Got vertex " + vertex +
", graphState" + gs);
VertexInputFormat<LongWritable, IntWritable, FloatWritable, IntWritable>
- inputFormat = BspUtils.createVertexInputFormat(job.getConfiguration());
+ inputFormat = configuration.createVertexInputFormat();
/*if[HADOOP_NON_SASL_RPC]
List<InputSplit> splitArray =
inputFormat.getSplits(
@@ -142,19 +143,33 @@ public class TestBspBasic extends BspCas
byteArrayOutputStream.toString());
}
+ private static class NullVertex extends EdgeListVertex<
+ NullWritable, NullWritable, NullWritable, NullWritable> {
+ @Override
+ public void compute(Iterable<NullWritable> messages) throws IOException { }
+ }
+
/**
* Test whether vertices with NullWritable for vertex value type, edge value
* type and message value type can be instantiated.
*/
@Test
public void testInstantiateNullVertex() throws IOException {
- Configuration nullConf = new Configuration();
- nullConf.setClass(GiraphJob.VERTEX_VALUE_CLASS, NullWritable.class, Writable.class);
- nullConf.setClass(GiraphJob.EDGE_VALUE_CLASS, NullWritable.class, Writable.class);
- nullConf.setClass(GiraphJob.MESSAGE_VALUE_CLASS, NullWritable.class, Writable.class);
- NullWritable vertexValue = BspUtils.createVertexValue(nullConf);
- NullWritable edgeValue = BspUtils.createEdgeValue(nullConf);
- NullWritable messageValue = BspUtils.createMessageValue(nullConf);
+ GiraphConfiguration nullConf = new GiraphConfiguration();
+ nullConf.setVertexClass(NullVertex.class);
+ ImmutableClassesGiraphConfiguration<
+ NullWritable, NullWritable, NullWritable,
+ NullWritable> immutableClassesGiraphConfiguration =
+ new ImmutableClassesGiraphConfiguration<
+ NullWritable, NullWritable, NullWritable, NullWritable>(
+ nullConf);
+ NullWritable vertexValue =
+ immutableClassesGiraphConfiguration.createVertexValue();
+ NullWritable edgeValue =
+ immutableClassesGiraphConfiguration.createEdgeValue();
+ NullWritable messageValue =
+ immutableClassesGiraphConfiguration.createMessageValue();
+ assertSame(vertexValue.getClass(), NullWritable.class);
assertSame(vertexValue, edgeValue);
assertSame(edgeValue, messageValue);
}
@@ -176,8 +191,9 @@ public class TestBspBasic extends BspCas
}
GiraphJob job = prepareJob(getCallingMethodName(),
SimpleSuperstepVertex.class, SimpleSuperstepVertexInputFormat.class);
- job.setWorkerConfiguration(5, 5, 100.0f);
- job.getConfiguration().setBoolean(GiraphJob.SPLIT_MASTER_WORKER, true);
+ job.getConfiguration().setWorkerConfiguration(5, 5, 100.0f);
+ job.getConfiguration().setBoolean(GiraphConfiguration.SPLIT_MASTER_WORKER,
+ true);
try {
job.run(true);
@@ -185,13 +201,13 @@ public class TestBspBasic extends BspCas
} catch (IllegalArgumentException e) {
}
- job.getConfiguration().setBoolean(GiraphJob.SPLIT_MASTER_WORKER, false);
+ job.getConfiguration().setBoolean(GiraphConfiguration.SPLIT_MASTER_WORKER, false);
try {
job.run(true);
fail();
} catch (IllegalArgumentException e) {
}
- job.setWorkerConfiguration(1, 1, 100.0f);
+ job.getConfiguration().setWorkerConfiguration(1, 1, 100.0f);
job.run(true);
}
@@ -234,7 +250,7 @@ public class TestBspBasic extends BspCas
SimpleSuperstepVertex.class, SimpleSuperstepVertexInputFormat.class,
SimpleSuperstepVertexOutputFormat.class, outputPath);
Configuration conf = job.getConfiguration();
- conf.setFloat(GiraphJob.TOTAL_INPUT_SPLIT_MULTIPLIER, 2.0f);
+ conf.setFloat(GiraphConfiguration.TOTAL_INPUT_SPLIT_MULTIPLIER, 2.0f);
// GeneratedInputSplit will generate 10 vertices
conf.setLong(GeneratedVertexReader.READER_VERTICES, 10);
assertTrue(job.run(true));
@@ -289,7 +305,7 @@ public class TestBspBasic extends BspCas
throws IOException, InterruptedException, ClassNotFoundException {
GiraphJob job = prepareJob(getCallingMethodName(),
SimpleCombinerVertex.class, SimpleSuperstepVertexInputFormat.class);
- job.setVertexCombinerClass(SimpleSumCombiner.class);
+ job.getConfiguration().setVertexCombinerClass(SimpleSumCombiner.class);
assertTrue(job.run(true));
}
@@ -349,9 +365,9 @@ public class TestBspBasic extends BspCas
throws IOException, InterruptedException, ClassNotFoundException {
GiraphJob job = prepareJob(getCallingMethodName(),
SimplePageRankVertex.class, SimplePageRankVertexInputFormat.class);
- job.setWorkerContextClass(
+ job.getConfiguration().setWorkerContextClass(
SimplePageRankVertex.SimplePageRankVertexWorkerContext.class);
- job.setMasterComputeClass(
+ job.getConfiguration().setMasterComputeClass(
SimplePageRankVertex.SimplePageRankVertexMasterCompute.class);
assertTrue(job.run(true));
if (!runningInDistributedMode()) {
@@ -407,26 +423,27 @@ public class TestBspBasic extends BspCas
public void testBspPageRankWithAggregatorWriter()
throws IOException, InterruptedException, ClassNotFoundException {
Path outputPath = getTempPath(getCallingMethodName());
+
GiraphJob job = prepareJob(getCallingMethodName(),
SimplePageRankVertex.class,
SimplePageRankVertex.SimplePageRankVertexInputFormat.class,
SimplePageRankVertex.SimplePageRankVertexOutputFormat.class,
outputPath);
- job.setWorkerContextClass(
+ GiraphConfiguration configuration = job.getConfiguration();
+ configuration.setWorkerContextClass(
SimplePageRankVertex.SimplePageRankVertexWorkerContext.class);
- job.setMasterComputeClass(
+ configuration.setMasterComputeClass(
SimplePageRankVertex.SimplePageRankVertexMasterCompute.class);
-
- Configuration conf = job.getConfiguration();
-
- job.setAggregatorWriterClass(TextAggregatorWriter.class);
+ configuration.setAggregatorWriterClass(TextAggregatorWriter.class);
Path aggregatorValues = getTempPath("aggregatorValues");
- conf.setInt(TextAggregatorWriter.FREQUENCY, TextAggregatorWriter.ALWAYS);
- conf.set(TextAggregatorWriter.FILENAME, aggregatorValues.toString());
+ configuration.setInt(TextAggregatorWriter.FREQUENCY,
+ TextAggregatorWriter.ALWAYS);
+ configuration.set(TextAggregatorWriter.FILENAME,
+ aggregatorValues.toString());
assertTrue(job.run(true));
- FileSystem fs = FileSystem.get(conf);
+ FileSystem fs = FileSystem.get(configuration);
Path valuesFile = new Path(aggregatorValues.toString() + "_0");
try {
@@ -499,9 +516,10 @@ public class TestBspBasic extends BspCas
throws IOException, InterruptedException, ClassNotFoundException {
GiraphJob job = prepareJob(getCallingMethodName(),
SimpleMasterComputeVertex.class, SimplePageRankVertexInputFormat.class);
- job.setWorkerContextClass(
+ job.getConfiguration().setWorkerContextClass(
SimpleMasterComputeVertex.SimpleMasterComputeWorkerContext.class);
- job.setMasterComputeClass(SimpleMasterComputeVertex.SimpleMasterCompute.class);
+ job.getConfiguration().setMasterComputeClass(
+ SimpleMasterComputeVertex.SimpleMasterCompute.class);
assertTrue(job.run(true));
if (!runningInDistributedMode()) {
double finalSum =
Modified: giraph/trunk/src/test/java/org/apache/giraph/TestGraphPartitioner.java
URL: http://svn.apache.org/viewvc/giraph/trunk/src/test/java/org/apache/giraph/TestGraphPartitioner.java?rev=1390014&r1=1390013&r2=1390014&view=diff
==============================================================================
--- giraph/trunk/src/test/java/org/apache/giraph/TestGraphPartitioner.java (original)
+++ giraph/trunk/src/test/java/org/apache/giraph/TestGraphPartitioner.java Tue Sep 25 17:40:18 2012
@@ -75,7 +75,7 @@ public class TestGraphPartitioner extend
Path outputPath = getTempPath("testVertexBalancer");
GiraphJob job = prepareJob("testVertexBalancer",
- SimpleCheckpointVertex.class,
+ SimpleCheckpointVertex.SimpleCheckpointComputation.class,
SimpleCheckpointVertex.SimpleCheckpointVertexWorkerContext.class,
SimpleCheckpointVertex.SimpleCheckpointVertexMasterCompute.class,
SimpleSuperstepVertexInputFormat.class,
@@ -90,7 +90,8 @@ public class TestGraphPartitioner extend
outputPath = getTempPath("testHashPartitioner");
- job = prepareJob("testHashPartitioner", SimpleCheckpointVertex.class,
+ job = prepareJob("testHashPartitioner",
+ SimpleCheckpointVertex.SimpleCheckpointComputation.class,
SimpleCheckpointVertex.SimpleCheckpointVertexWorkerContext.class,
SimpleCheckpointVertex.SimpleCheckpointVertexMasterCompute.class,
SimpleSuperstepVertexInputFormat.class,
@@ -100,14 +101,14 @@ public class TestGraphPartitioner extend
outputPath = getTempPath("testSuperstepHashPartitioner");
job = prepareJob("testSuperstepHashPartitioner",
- SimpleCheckpointVertex.class,
+ SimpleCheckpointVertex.SimpleCheckpointComputation.class,
SimpleCheckpointVertex.SimpleCheckpointVertexWorkerContext.class,
SimpleCheckpointVertex.SimpleCheckpointVertexMasterCompute.class,
SimpleSuperstepVertexInputFormat.class,
SimpleSuperstepVertexOutputFormat.class,
outputPath);
- job.setGraphPartitionerFactoryClass(
+ job.getConfiguration().setGraphPartitionerFactoryClass(
SuperstepHashPartitionerFactory.class);
assertTrue(job.run(true));
@@ -115,14 +116,15 @@ public class TestGraphPartitioner extend
job = new GiraphJob("testHashRangePartitioner");
setupConfiguration(job);
- job.setVertexClass(SimpleCheckpointVertex.class);
- job.setWorkerContextClass(
+ job.getConfiguration().setVertexClass(
+ SimpleCheckpointVertex.SimpleCheckpointComputation.class);
+ job.getConfiguration().setWorkerContextClass(
SimpleCheckpointVertex.SimpleCheckpointVertexWorkerContext.class);
- job.setMasterComputeClass(
+ job.getConfiguration().setMasterComputeClass(
SimpleCheckpointVertex.SimpleCheckpointVertexMasterCompute.class);
- job.setVertexInputFormatClass(SimpleSuperstepVertexInputFormat.class);
- job.setVertexOutputFormatClass(SimpleSuperstepVertexOutputFormat.class);
- job.setGraphPartitionerFactoryClass(
+ job.getConfiguration().setVertexInputFormatClass(SimpleSuperstepVertexInputFormat.class);
+ job.getConfiguration().setVertexOutputFormatClass(SimpleSuperstepVertexOutputFormat.class);
+ job.getConfiguration().setGraphPartitionerFactoryClass(
HashRangePartitionerFactory.class);
outputPath = new Path("/tmp/testHashRangePartitioner");
removeAndSetOutput(job, outputPath);
@@ -131,12 +133,12 @@ public class TestGraphPartitioner extend
outputPath = getTempPath("testReverseIdSuperstepHashPartitioner");
job = prepareJob("testReverseIdSuperstepHashPartitioner",
- SimpleCheckpointVertex.class,
+ SimpleCheckpointVertex.SimpleCheckpointComputation.class,
SimpleCheckpointVertex.SimpleCheckpointVertexWorkerContext.class,
SimpleCheckpointVertex.SimpleCheckpointVertexMasterCompute.class,
SimpleSuperstepVertexInputFormat.class,
SimpleSuperstepVertexOutputFormat.class, outputPath);
- job.setGraphPartitionerFactoryClass(
+ job.getConfiguration().setGraphPartitionerFactoryClass(
SuperstepHashPartitionerFactory.class);
job.getConfiguration().setBoolean(
GeneratedVertexReader.REVERSE_ID_ORDER, true);
Modified: giraph/trunk/src/test/java/org/apache/giraph/TestJsonBase64Format.java
URL: http://svn.apache.org/viewvc/giraph/trunk/src/test/java/org/apache/giraph/TestJsonBase64Format.java?rev=1390014&r1=1390013&r2=1390014&view=diff
==============================================================================
--- giraph/trunk/src/test/java/org/apache/giraph/TestJsonBase64Format.java (original)
+++ giraph/trunk/src/test/java/org/apache/giraph/TestJsonBase64Format.java Tue Sep 25 17:40:18 2012
@@ -22,7 +22,7 @@ import static org.junit.Assert.assertTru
import java.io.IOException;
-import org.apache.giraph.benchmark.PageRankBenchmark;
+import org.apache.giraph.benchmark.EdgeListVertexPageRankBenchmark;
import org.apache.giraph.benchmark.PageRankComputation;
import org.apache.giraph.io.PseudoRandomVertexInputFormat;
import org.apache.giraph.graph.GiraphJob;
@@ -38,14 +38,8 @@ import org.junit.Test;
*/
public class TestJsonBase64Format extends BspCase {
/**
- * Create the test case
- *
- * @param testName name of the test case
+ * Constructor.
*/
- public TestJsonBase64Format(String testName) {
- super(testName);
- }
-
public TestJsonBase64Format() {
super(TestJsonBase64Format.class.getName());
}
@@ -64,9 +58,11 @@ public class TestJsonBase64Format extend
throws IOException, InterruptedException, ClassNotFoundException {
Path outputPath = getTempPath(getCallingMethodName());
- GiraphJob job = prepareJob(getCallingMethodName(), PageRankBenchmark.class,
+ GiraphJob job = prepareJob(getCallingMethodName(),
+ EdgeListVertexPageRankBenchmark.class,
PseudoRandomVertexInputFormat.class,
- JsonBase64VertexOutputFormat.class, outputPath);
+ JsonBase64VertexOutputFormat.class,
+ outputPath);
job.getConfiguration().setLong(
PseudoRandomVertexInputFormat.AGGREGATE_VERTICES, 101);
job.getConfiguration().setLong(
@@ -76,15 +72,18 @@ public class TestJsonBase64Format extend
assertTrue(job.run(true));
Path outputPath2 = getTempPath(getCallingMethodName() + "2");
- job = prepareJob(getCallingMethodName(), PageRankBenchmark.class,
- JsonBase64VertexInputFormat.class, JsonBase64VertexOutputFormat.class,
+ job = prepareJob(getCallingMethodName(),
+ EdgeListVertexPageRankBenchmark.class,
+ JsonBase64VertexInputFormat.class,
+ JsonBase64VertexOutputFormat.class,
outputPath2);
job.getConfiguration().setInt(PageRankComputation.SUPERSTEP_COUNT, 3);
FileInputFormat.setInputPaths(job.getInternalJob(), outputPath);
assertTrue(job.run(true));
Path outputPath3 = getTempPath(getCallingMethodName() + "3");
- job = prepareJob(getCallingMethodName(), PageRankBenchmark.class,
+ job = prepareJob(getCallingMethodName(),
+ EdgeListVertexPageRankBenchmark.class,
PseudoRandomVertexInputFormat.class,
JsonBase64VertexOutputFormat.class, outputPath3);
job.getConfiguration().setLong(
Modified: giraph/trunk/src/test/java/org/apache/giraph/TestManualCheckpoint.java
URL: http://svn.apache.org/viewvc/giraph/trunk/src/test/java/org/apache/giraph/TestManualCheckpoint.java?rev=1390014&r1=1390013&r2=1390014&view=diff
==============================================================================
--- giraph/trunk/src/test/java/org/apache/giraph/TestManualCheckpoint.java (original)
+++ giraph/trunk/src/test/java/org/apache/giraph/TestManualCheckpoint.java Tue Sep 25 17:40:18 2012
@@ -51,17 +51,17 @@ public class TestManualCheckpoint extend
Path checkpointsDir = getTempPath("checkPointsForTesting");
Path outputPath = getTempPath(getCallingMethodName());
GiraphJob job = prepareJob(getCallingMethodName(),
- SimpleCheckpointVertex.class,
+ SimpleCheckpointVertex.SimpleCheckpointComputation.class,
SimpleCheckpointVertex.SimpleCheckpointVertexWorkerContext.class,
SimpleCheckpointVertex.SimpleCheckpointVertexMasterCompute.class,
SimpleSuperstepVertexInputFormat.class,
SimpleSuperstepVertexOutputFormat.class, outputPath);
- job.getConfiguration().set(GiraphJob.CHECKPOINT_DIRECTORY,
+ job.getConfiguration().set(GiraphConfiguration.CHECKPOINT_DIRECTORY,
checkpointsDir.toString());
job.getConfiguration().setBoolean(
- GiraphJob.CLEANUP_CHECKPOINTS_AFTER_SUCCESS, false);
- job.getConfiguration().setInt(GiraphJob.CHECKPOINT_FREQUENCY, 2);
+ GiraphConfiguration.CLEANUP_CHECKPOINTS_AFTER_SUCCESS, false);
+ job.getConfiguration().setInt(GiraphConfiguration.CHECKPOINT_FREQUENCY, 2);
assertTrue(job.run(true));
@@ -80,14 +80,14 @@ public class TestManualCheckpoint extend
" with checkpoint path = " + checkpointsDir);
outputPath = getTempPath(getCallingMethodName() + "Restarted");
GiraphJob restartedJob = prepareJob(getCallingMethodName() + "Restarted",
- SimpleCheckpointVertex.class,
+ SimpleCheckpointVertex.SimpleCheckpointComputation.class,
SimpleCheckpointVertex.SimpleCheckpointVertexWorkerContext.class,
SimpleCheckpointVertex.SimpleCheckpointVertexMasterCompute.class,
SimpleSuperstepVertexInputFormat.class,
SimpleSuperstepVertexOutputFormat.class, outputPath);
- job.setMasterComputeClass(
+ job.getConfiguration().setMasterComputeClass(
SimpleCheckpointVertex.SimpleCheckpointVertexMasterCompute.class);
- restartedJob.getConfiguration().set(GiraphJob.CHECKPOINT_DIRECTORY,
+ restartedJob.getConfiguration().set(GiraphConfiguration.CHECKPOINT_DIRECTORY,
checkpointsDir.toString());
assertTrue(restartedJob.run(true));
Modified: giraph/trunk/src/test/java/org/apache/giraph/TestNotEnoughMapTasks.java
URL: http://svn.apache.org/viewvc/giraph/trunk/src/test/java/org/apache/giraph/TestNotEnoughMapTasks.java?rev=1390014&r1=1390013&r2=1390014&view=diff
==============================================================================
--- giraph/trunk/src/test/java/org/apache/giraph/TestNotEnoughMapTasks.java (original)
+++ giraph/trunk/src/test/java/org/apache/giraph/TestNotEnoughMapTasks.java Tue Sep 25 17:40:18 2012
@@ -55,16 +55,18 @@ public class TestNotEnoughMapTasks exten
}
Path outputPath = getTempPath(getCallingMethodName());
GiraphJob job = prepareJob(getCallingMethodName(),
- SimpleCheckpointVertex.class,
+ SimpleCheckpointVertex.SimpleCheckpointComputation.class,
SimpleSuperstepVertexInputFormat.class,
SimpleSuperstepVertexOutputFormat.class, outputPath);
// An unlikely impossible number of workers to achieve
final int unlikelyWorkers = Short.MAX_VALUE;
- job.setWorkerConfiguration(unlikelyWorkers, unlikelyWorkers, 100.0f);
+ job.getConfiguration().setWorkerConfiguration(unlikelyWorkers,
+ unlikelyWorkers,
+ 100.0f);
// Only one poll attempt of one second to make failure faster
- job.getConfiguration().setInt(GiraphJob.POLL_ATTEMPTS, 1);
- job.getConfiguration().setInt(GiraphJob.POLL_MSECS, 1);
+ job.getConfiguration().setInt(GiraphConfiguration.POLL_ATTEMPTS, 1);
+ job.getConfiguration().setInt(GiraphConfiguration.POLL_MSECS, 1);
assertFalse(job.run(false));
}
}
Modified: giraph/trunk/src/test/java/org/apache/giraph/TestVertexTypes.java
URL: http://svn.apache.org/viewvc/giraph/trunk/src/test/java/org/apache/giraph/TestVertexTypes.java?rev=1390014&r1=1390013&r2=1390014&view=diff
==============================================================================
--- giraph/trunk/src/test/java/org/apache/giraph/TestVertexTypes.java (original)
+++ giraph/trunk/src/test/java/org/apache/giraph/TestVertexTypes.java Tue Sep 25 17:40:18 2012
@@ -101,13 +101,13 @@ public class TestVertexTypes {
public void testMatchingType() throws SecurityException,
NoSuchMethodException, NoSuchFieldException {
Configuration conf = new Configuration();
- conf.setClass(GiraphJob.VERTEX_CLASS,
+ conf.setClass(GiraphConfiguration.VERTEX_CLASS,
GeneratedVertexMatch.class,
Vertex.class);
- conf.setClass(GiraphJob.VERTEX_INPUT_FORMAT_CLASS,
+ conf.setClass(GiraphConfiguration.VERTEX_INPUT_FORMAT_CLASS,
SimpleSuperstepVertexInputFormat.class,
VertexInputFormat.class);
- conf.setClass(GiraphJob.VERTEX_COMBINER_CLASS,
+ conf.setClass(GiraphConfiguration.VERTEX_COMBINER_CLASS,
GeneratedVertexMatchCombiner.class,
VertexCombiner.class);
@SuppressWarnings("rawtypes")
@@ -120,10 +120,10 @@ public class TestVertexTypes {
public void testDerivedMatchingType() throws SecurityException,
NoSuchMethodException, NoSuchFieldException {
Configuration conf = new Configuration();
- conf.setClass(GiraphJob.VERTEX_CLASS,
+ conf.setClass(GiraphConfiguration.VERTEX_CLASS,
DerivedVertexMatch.class,
Vertex.class);
- conf.setClass(GiraphJob.VERTEX_INPUT_FORMAT_CLASS,
+ conf.setClass(GiraphConfiguration.VERTEX_INPUT_FORMAT_CLASS,
SimpleSuperstepVertexInputFormat.class,
VertexInputFormat.class);
@SuppressWarnings("rawtypes")
@@ -136,10 +136,10 @@ public class TestVertexTypes {
public void testDerivedInputFormatType() throws SecurityException,
NoSuchMethodException, NoSuchFieldException {
Configuration conf = new Configuration();
- conf.setClass(GiraphJob.VERTEX_CLASS,
+ conf.setClass(GiraphConfiguration.VERTEX_CLASS,
DerivedVertexMatch.class,
Vertex.class);
- conf.setClass(GiraphJob.VERTEX_INPUT_FORMAT_CLASS,
+ conf.setClass(GiraphConfiguration.VERTEX_INPUT_FORMAT_CLASS,
SimpleSuperstepVertexInputFormat.class,
VertexInputFormat.class);
@SuppressWarnings("rawtypes")
@@ -152,10 +152,10 @@ public class TestVertexTypes {
public void testMismatchingVertex() throws SecurityException,
NoSuchMethodException, NoSuchFieldException {
Configuration conf = new Configuration();
- conf.setClass(GiraphJob.VERTEX_CLASS,
+ conf.setClass(GiraphConfiguration.VERTEX_CLASS,
GeneratedVertexMismatch.class,
Vertex.class);
- conf.setClass(GiraphJob.VERTEX_INPUT_FORMAT_CLASS,
+ conf.setClass(GiraphConfiguration.VERTEX_INPUT_FORMAT_CLASS,
SimpleSuperstepVertexInputFormat.class,
VertexInputFormat.class);
@SuppressWarnings("rawtypes")
@@ -168,12 +168,12 @@ public class TestVertexTypes {
public void testMismatchingCombiner() throws SecurityException,
NoSuchMethodException, NoSuchFieldException {
Configuration conf = new Configuration();
- conf.setClass(GiraphJob.VERTEX_CLASS,
+ conf.setClass(GiraphConfiguration.VERTEX_CLASS,
GeneratedVertexMatch.class, Vertex.class);
- conf.setClass(GiraphJob.VERTEX_INPUT_FORMAT_CLASS,
+ conf.setClass(GiraphConfiguration.VERTEX_INPUT_FORMAT_CLASS,
SimpleSuperstepVertexInputFormat.class,
VertexInputFormat.class);
- conf.setClass(GiraphJob.VERTEX_COMBINER_CLASS,
+ conf.setClass(GiraphConfiguration.VERTEX_COMBINER_CLASS,
GeneratedVertexMismatchCombiner.class,
VertexCombiner.class);
@SuppressWarnings("rawtypes")
@@ -186,13 +186,13 @@ public class TestVertexTypes {
public void testJsonBase64FormatType() throws SecurityException,
NoSuchMethodException, NoSuchFieldException {
Configuration conf = new Configuration();
- conf.setClass(GiraphJob.VERTEX_CLASS,
+ conf.setClass(GiraphConfiguration.VERTEX_CLASS,
GeneratedVertexMatch.class,
Vertex.class);
- conf.setClass(GiraphJob.VERTEX_INPUT_FORMAT_CLASS,
+ conf.setClass(GiraphConfiguration.VERTEX_INPUT_FORMAT_CLASS,
JsonBase64VertexInputFormat.class,
VertexInputFormat.class);
- conf.setClass(GiraphJob.VERTEX_OUTPUT_FORMAT_CLASS,
+ conf.setClass(GiraphConfiguration.VERTEX_OUTPUT_FORMAT_CLASS,
JsonBase64VertexOutputFormat.class,
VertexOutputFormat.class);
@SuppressWarnings("rawtypes")
Modified: giraph/trunk/src/test/java/org/apache/giraph/comm/ConnectionTest.java
URL: http://svn.apache.org/viewvc/giraph/trunk/src/test/java/org/apache/giraph/comm/ConnectionTest.java?rev=1390014&r1=1390013&r2=1390014&view=diff
==============================================================================
--- giraph/trunk/src/test/java/org/apache/giraph/comm/ConnectionTest.java (original)
+++ giraph/trunk/src/test/java/org/apache/giraph/comm/ConnectionTest.java Tue Sep 25 17:40:18 2012
@@ -19,16 +19,25 @@
package org.apache.giraph.comm;
import com.google.common.collect.Sets;
+import java.util.Iterator;
import java.util.Set;
+import org.apache.giraph.GiraphConfiguration;
+import org.apache.giraph.ImmutableClassesGiraphConfiguration;
+import org.apache.giraph.benchmark.EdgeListVertexPageRankBenchmark;
+import org.apache.giraph.benchmark.PageRankBenchmark;
import org.apache.giraph.comm.messages.SimpleMessageStore;
import org.apache.giraph.comm.netty.handler.RequestServerHandler;
import org.apache.giraph.comm.netty.NettyClient;
import org.apache.giraph.comm.netty.NettyServer;
import org.apache.giraph.comm.netty.handler.WorkerRequestServerHandler;
+import org.apache.giraph.graph.EdgeListVertex;
+import org.apache.giraph.graph.MutableVertex;
+import org.apache.giraph.graph.Vertex;
import org.apache.giraph.utils.MockUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.mapreduce.Mapper.Context;
+import org.junit.Before;
import org.junit.Test;
import static org.mockito.Mockito.mock;
@@ -42,6 +51,23 @@ import java.util.Collections;
* Test the netty connections
*/
public class ConnectionTest {
+ /** Class configuration */
+ private ImmutableClassesGiraphConfiguration conf;
+
+ public static class IntVertex extends EdgeListVertex<IntWritable,
+ IntWritable, IntWritable, IntWritable> {
+ @Override
+ public void compute(Iterable<IntWritable> messages) throws IOException {
+ }
+ }
+
+ @Before
+ public void setUp() {
+ GiraphConfiguration tmpConfig = new GiraphConfiguration();
+ tmpConfig.setVertexClass(IntVertex.class);
+ conf = new ImmutableClassesGiraphConfiguration(tmpConfig);
+ }
+
/**
* Test connecting a single client to a single server.
*
@@ -49,7 +75,6 @@ public class ConnectionTest {
*/
@Test
public void connectSingleClientServer() throws IOException {
- Configuration conf = new Configuration();
@SuppressWarnings("rawtypes")
Context context = mock(Context.class);
when(context.getConfiguration()).thenReturn(conf);
@@ -63,7 +88,7 @@ public class ConnectionTest {
new WorkerRequestServerHandler.Factory(serverData));
server.start();
- NettyClient client = new NettyClient(context);
+ NettyClient client = new NettyClient(context, conf);
client.connectAllAddresses(Collections.singleton(server.getMyAddress()));
client.stop();
@@ -77,7 +102,6 @@ public class ConnectionTest {
*/
@Test
public void connectOneClientToThreeServers() throws IOException {
- Configuration conf = new Configuration();
@SuppressWarnings("rawtypes")
Context context = mock(Context.class);
when(context.getConfiguration()).thenReturn(conf);
@@ -96,7 +120,7 @@ public class ConnectionTest {
NettyServer server3 = new NettyServer(conf, requestServerHandlerFactory);
server3.start();
- NettyClient client = new NettyClient(context);
+ NettyClient client = new NettyClient(context, conf);
Set<InetSocketAddress> serverAddresses = Sets.newHashSet();
serverAddresses.add(server1.getMyAddress());
serverAddresses.add(server2.getMyAddress());
@@ -116,7 +140,6 @@ public class ConnectionTest {
*/
@Test
public void connectThreeClientsToOneServer() throws IOException {
- Configuration conf = new Configuration();
@SuppressWarnings("rawtypes")
Context context = mock(Context.class);
when(context.getConfiguration()).thenReturn(conf);
@@ -129,11 +152,11 @@ public class ConnectionTest {
new WorkerRequestServerHandler.Factory(serverData));
server.start();
- NettyClient client1 = new NettyClient(context);
+ NettyClient client1 = new NettyClient(context, conf);
client1.connectAllAddresses(Collections.singleton(server.getMyAddress()));
- NettyClient client2 = new NettyClient(context);
+ NettyClient client2 = new NettyClient(context, conf);
client2.connectAllAddresses(Collections.singleton(server.getMyAddress()));
- NettyClient client3 = new NettyClient(context);
+ NettyClient client3 = new NettyClient(context, conf);
client3.connectAllAddresses(Collections.singleton(server.getMyAddress()));
client1.stop();
Modified: giraph/trunk/src/test/java/org/apache/giraph/comm/RPCCommunicationsTest.java
URL: http://svn.apache.org/viewvc/giraph/trunk/src/test/java/org/apache/giraph/comm/RPCCommunicationsTest.java?rev=1390014&r1=1390013&r2=1390014&view=diff
==============================================================================
--- giraph/trunk/src/test/java/org/apache/giraph/comm/RPCCommunicationsTest.java (original)
+++ giraph/trunk/src/test/java/org/apache/giraph/comm/RPCCommunicationsTest.java Tue Sep 25 17:40:18 2012
@@ -22,6 +22,10 @@ import static org.junit.Assert.assertEqu
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+import java.io.IOException;
+import org.apache.giraph.GiraphConfiguration;
+import org.apache.giraph.ImmutableClassesGiraphConfiguration;
+import org.apache.giraph.graph.EdgeListVertex;
import org.apache.giraph.graph.GiraphJob;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.IntWritable;
@@ -31,33 +35,46 @@ import org.junit.Test;
public class RPCCommunicationsTest {
+ private static class IntVertex extends EdgeListVertex<IntWritable,
+ IntWritable, IntWritable, IntWritable> {
+ @Override
+ public void compute(Iterable<IntWritable> messages) throws IOException {
+ }
+ }
+
@Test
public void testDuplicateRpcPort() throws Exception {
- @SuppressWarnings("rawtypes")
- Context context = mock(Context.class);
- Configuration conf = new Configuration();
- conf.setInt("mapred.task.partition", 9);
- conf.setInt(GiraphJob.MAX_WORKERS, 13);
- when(context.getConfiguration()).thenReturn(conf);
- when(context.getJobID()).thenReturn(new JobID());
+ @SuppressWarnings("rawtypes")
+ Context context = mock(Context.class);
+ GiraphConfiguration conf = new GiraphConfiguration();
+ conf.setInt("mapred.task.partition", 9);
+ conf.setInt(GiraphConfiguration.MAX_WORKERS, 13);
+ conf.setVertexClass(IntVertex.class);
+ ImmutableClassesGiraphConfiguration immutableClassesGiraphConfiguration =
+ new ImmutableClassesGiraphConfiguration(conf);
+ when(context.getConfiguration()).thenReturn(conf);
+ when(context.getJobID()).thenReturn(new JobID());
- RPCCommunications<IntWritable, IntWritable, IntWritable, IntWritable>
- comm1 =
- new RPCCommunications<
- IntWritable, IntWritable,
- IntWritable, IntWritable>(context, null, null);
- RPCCommunications<IntWritable, IntWritable, IntWritable, IntWritable>
- comm2 =
- new RPCCommunications<
- IntWritable, IntWritable,
- IntWritable, IntWritable>(context, null, null);
- RPCCommunications<IntWritable, IntWritable, IntWritable, IntWritable>
- comm3 =
- new RPCCommunications<
- IntWritable, IntWritable,
- IntWritable, IntWritable>(context, null, null);
- assertEquals(comm1.getPort(), 30009);
- assertEquals(comm2.getPort(), 30109);
- assertEquals(comm3.getPort(), 30209);
+ RPCCommunications<IntWritable, IntWritable, IntWritable, IntWritable>
+ comm1 =
+ new RPCCommunications<
+ IntWritable, IntWritable,
+ IntWritable, IntWritable>(
+ context, null, immutableClassesGiraphConfiguration, null);
+ RPCCommunications<IntWritable, IntWritable, IntWritable, IntWritable>
+ comm2 =
+ new RPCCommunications<
+ IntWritable, IntWritable,
+ IntWritable, IntWritable>(
+ context, null, immutableClassesGiraphConfiguration, null);
+ RPCCommunications<IntWritable, IntWritable, IntWritable, IntWritable>
+ comm3 =
+ new RPCCommunications<
+ IntWritable, IntWritable,
+ IntWritable, IntWritable>(
+ context, null, immutableClassesGiraphConfiguration, null);
+ assertEquals(comm1.getPort(), 30009);
+ assertEquals(comm2.getPort(), 30109);
+ assertEquals(comm3.getPort(), 30209);
}
}
Modified: giraph/trunk/src/test/java/org/apache/giraph/comm/RequestFailureTest.java
URL: http://svn.apache.org/viewvc/giraph/trunk/src/test/java/org/apache/giraph/comm/RequestFailureTest.java?rev=1390014&r1=1390013&r2=1390014&view=diff
==============================================================================
--- giraph/trunk/src/test/java/org/apache/giraph/comm/RequestFailureTest.java (original)
+++ giraph/trunk/src/test/java/org/apache/giraph/comm/RequestFailureTest.java Tue Sep 25 17:40:18 2012
@@ -18,6 +18,8 @@
package org.apache.giraph.comm;
+import org.apache.giraph.GiraphConfiguration;
+import org.apache.giraph.ImmutableClassesGiraphConfiguration;
import org.apache.giraph.comm.messages.SimpleMessageStore;
import org.apache.giraph.comm.netty.NettyClient;
import org.apache.giraph.comm.netty.NettyServer;
@@ -25,7 +27,6 @@ import org.apache.giraph.comm.netty.hand
import org.apache.giraph.comm.requests.SendPartitionMessagesRequest;
import org.apache.giraph.comm.requests.WritableRequest;
import org.apache.giraph.graph.EdgeListVertex;
-import org.apache.giraph.graph.GiraphJob;
import org.apache.giraph.graph.Vertex;
import org.apache.giraph.utils.MockUtils;
import org.apache.hadoop.conf.Configuration;
@@ -53,7 +54,7 @@ import java.util.Map;
*/
public class RequestFailureTest {
/** Configuration */
- private Configuration conf;
+ private ImmutableClassesGiraphConfiguration conf;
/** Server data */
private ServerData<IntWritable, IntWritable, IntWritable, IntWritable>
serverData;
@@ -77,16 +78,9 @@ public class RequestFailureTest {
@Before
public void setUp() throws IOException {
// Setup the conf
- conf = new Configuration();
- conf.setClass(GiraphJob.VERTEX_CLASS, TestVertex.class, Vertex.class);
- conf.setClass(GiraphJob.VERTEX_ID_CLASS,
- IntWritable.class, WritableComparable.class);
- conf.setClass(GiraphJob.VERTEX_VALUE_CLASS,
- IntWritable.class, Writable.class);
- conf.setClass(GiraphJob.EDGE_VALUE_CLASS,
- IntWritable.class, Writable.class);
- conf.setClass(GiraphJob.MESSAGE_VALUE_CLASS,
- IntWritable.class, Writable.class);
+ GiraphConfiguration tmpConf = new GiraphConfiguration();
+ tmpConf.setVertexClass(TestVertex.class);
+ conf = new ImmutableClassesGiraphConfiguration(tmpConf);
context = mock(Context.class);
when(context.getConfiguration()).thenReturn(conf);
@@ -94,7 +88,7 @@ public class RequestFailureTest {
private WritableRequest getRequest() {
// Data to send
- int partitionId = 17;
+ int partitionId = 0;
Map<IntWritable, Collection<IntWritable>> vertexIdMessages =
Maps.newHashMap();
for (int i = 1; i < 7; ++i) {
@@ -144,7 +138,7 @@ public class RequestFailureTest {
server = new NettyServer(conf,
new WorkerRequestServerHandler.Factory(serverData));
server.start();
- client = new NettyClient(context);
+ client = new NettyClient(context, conf);
client.connectAllAddresses(Collections.singleton(server.getMyAddress()));
// Send the request 2x
@@ -165,11 +159,11 @@ public class RequestFailureTest {
@Test
public void alreadyProcessedRequest() throws IOException {
// Force a drop of the first request
- conf.setBoolean(GiraphJob.NETTY_SIMULATE_FIRST_RESPONSE_FAILED, true);
+ conf.setBoolean(GiraphConfiguration.NETTY_SIMULATE_FIRST_RESPONSE_FAILED, true);
// One second to finish a request
- conf.setInt(GiraphJob.MAX_REQUEST_MILLISECONDS, 1000);
+ conf.setInt(GiraphConfiguration.MAX_REQUEST_MILLISECONDS, 1000);
// Loop every 2 seconds
- conf.setInt(GiraphJob.WAITING_REQUEST_MSECS, 2000);
+ conf.setInt(GiraphConfiguration.WAITING_REQUEST_MSECS, 2000);
// Start the service
serverData =
@@ -179,7 +173,7 @@ public class RequestFailureTest {
server = new NettyServer(conf,
new WorkerRequestServerHandler.Factory(serverData));
server.start();
- client = new NettyClient(context);
+ client = new NettyClient(context, conf);
client.connectAllAddresses(Collections.singleton(server.getMyAddress()));
// Send the request 2x, but should only be processed once
@@ -200,11 +194,11 @@ public class RequestFailureTest {
@Test
public void resendRequest() throws IOException {
// Force a drop of the first request
- conf.setBoolean(GiraphJob.NETTY_SIMULATE_FIRST_REQUEST_CLOSED, true);
+ conf.setBoolean(GiraphConfiguration.NETTY_SIMULATE_FIRST_REQUEST_CLOSED, true);
// One second to finish a request
- conf.setInt(GiraphJob.MAX_REQUEST_MILLISECONDS, 1000);
+ conf.setInt(GiraphConfiguration.MAX_REQUEST_MILLISECONDS, 1000);
// Loop every 2 seconds
- conf.setInt(GiraphJob.WAITING_REQUEST_MSECS, 2000);
+ conf.setInt(GiraphConfiguration.WAITING_REQUEST_MSECS, 2000);
// Start the service
serverData =
@@ -214,7 +208,7 @@ public class RequestFailureTest {
server = new NettyServer(conf,
new WorkerRequestServerHandler.Factory(serverData));
server.start();
- client = new NettyClient(context);
+ client = new NettyClient(context, conf);
client.connectAllAddresses(Collections.singleton(server.getMyAddress()));
// Send the request 2x, but should only be processed once
Modified: giraph/trunk/src/test/java/org/apache/giraph/comm/RequestTest.java
URL: http://svn.apache.org/viewvc/giraph/trunk/src/test/java/org/apache/giraph/comm/RequestTest.java?rev=1390014&r1=1390013&r2=1390014&view=diff
==============================================================================
--- giraph/trunk/src/test/java/org/apache/giraph/comm/RequestTest.java (original)
+++ giraph/trunk/src/test/java/org/apache/giraph/comm/RequestTest.java Tue Sep 25 17:40:18 2012
@@ -18,6 +18,8 @@
package org.apache.giraph.comm;
+import org.apache.giraph.GiraphConfiguration;
+import org.apache.giraph.ImmutableClassesGiraphConfiguration;
import org.apache.giraph.comm.messages.SimpleMessageStore;
import org.apache.giraph.comm.netty.NettyClient;
import org.apache.giraph.comm.netty.NettyServer;
@@ -61,7 +63,7 @@ import java.util.concurrent.ConcurrentHa
*/
public class RequestTest {
/** Configuration */
- private Configuration conf;
+ private ImmutableClassesGiraphConfiguration conf;
/** Server data */
private ServerData<IntWritable, IntWritable, IntWritable, IntWritable>
serverData;
@@ -83,16 +85,10 @@ public class RequestTest {
@Before
public void setUp() throws IOException {
// Setup the conf
- conf = new Configuration();
- conf.setClass(GiraphJob.VERTEX_CLASS, TestVertex.class, Vertex.class);
- conf.setClass(GiraphJob.VERTEX_ID_CLASS,
- IntWritable.class, WritableComparable.class);
- conf.setClass(GiraphJob.VERTEX_VALUE_CLASS,
- IntWritable.class, Writable.class);
- conf.setClass(GiraphJob.EDGE_VALUE_CLASS,
- IntWritable.class, Writable.class);
- conf.setClass(GiraphJob.MESSAGE_VALUE_CLASS,
- IntWritable.class, Writable.class);
+ GiraphConfiguration tmpConf = new GiraphConfiguration();
+ tmpConf.setClass(GiraphConfiguration.VERTEX_CLASS, TestVertex.class,
+ Vertex.class);
+ conf = new ImmutableClassesGiraphConfiguration(tmpConf);
@SuppressWarnings("rawtypes")
Context context = mock(Context.class);
@@ -106,7 +102,7 @@ public class RequestTest {
server = new NettyServer(conf,
new WorkerRequestServerHandler.Factory(serverData));
server.start();
- client = new NettyClient(context);
+ client = new NettyClient(context, conf);
client.connectAllAddresses(Collections.singleton(server.getMyAddress()));
}
@@ -153,7 +149,7 @@ public class RequestTest {
@Test
public void sendPartitionMessagesRequest() throws IOException {
// Data to send
- int partitionId = 17;
+ int partitionId = 0;
Map<IntWritable, Collection<IntWritable>> vertexIdMessages =
Maps.newHashMap();
for (int i = 1; i < 7; ++i) {
Modified: giraph/trunk/src/test/java/org/apache/giraph/comm/TestMessageStores.java
URL: http://svn.apache.org/viewvc/giraph/trunk/src/test/java/org/apache/giraph/comm/TestMessageStores.java?rev=1390014&r1=1390013&r2=1390014&view=diff
==============================================================================
--- giraph/trunk/src/test/java/org/apache/giraph/comm/TestMessageStores.java (original)
+++ giraph/trunk/src/test/java/org/apache/giraph/comm/TestMessageStores.java Tue Sep 25 17:40:18 2012
@@ -19,6 +19,8 @@
package org.apache.giraph.comm;
import org.apache.commons.collections.CollectionUtils;
+import org.apache.giraph.GiraphConfiguration;
+import org.apache.giraph.ImmutableClassesGiraphConfiguration;
import org.apache.giraph.bsp.CentralizedServiceWorker;
import org.apache.giraph.comm.messages.BasicMessageStore;
import org.apache.giraph.comm.messages.DiskBackedMessageStoreByPartition;
@@ -28,6 +30,7 @@ import org.apache.giraph.comm.messages.M
import org.apache.giraph.comm.messages.MessageStoreFactory;
import org.apache.giraph.comm.messages.SequentialFileMessageStore;
import org.apache.giraph.comm.messages.SimpleMessageStore;
+import org.apache.giraph.graph.EdgeListVertex;
import org.apache.giraph.graph.GiraphJob;
import org.apache.giraph.utils.MockUtils;
import org.apache.hadoop.conf.Configuration;
@@ -62,22 +65,28 @@ import java.util.TreeSet;
/** Test for different types of message stores */
public class TestMessageStores {
private static String directory;
- private static Configuration config;
+ private static ImmutableClassesGiraphConfiguration config;
private static TestData testData;
private static
CentralizedServiceWorker<IntWritable, IntWritable, IntWritable, IntWritable>
service;
+ private static class IntVertex extends EdgeListVertex<IntWritable,
+ IntWritable, IntWritable, IntWritable> {
+
+ @Override
+ public void compute(Iterable<IntWritable> messages) throws IOException {
+ }
+ }
+
@Before
public void prepare() {
directory = "test/";
Configuration.addDefaultResource("giraph-site.xml");
- config = new Configuration();
- config.setClass(GiraphJob.VERTEX_ID_CLASS, IntWritable.class,
- WritableComparable.class);
- config.setClass(GiraphJob.MESSAGE_VALUE_CLASS, IntWritable.class,
- Writable.class);
+ GiraphConfiguration initConfig = new GiraphConfiguration();
+ initConfig.setVertexClass(IntVertex.class);
+ config = new ImmutableClassesGiraphConfiguration(initConfig);
testData = new TestData();
testData.maxId = 1000000;
Modified: giraph/trunk/src/test/java/org/apache/giraph/examples/TryMultiRpcBindingPortsTest.java
URL: http://svn.apache.org/viewvc/giraph/trunk/src/test/java/org/apache/giraph/examples/TryMultiRpcBindingPortsTest.java?rev=1390014&r1=1390013&r2=1390014&view=diff
==============================================================================
--- giraph/trunk/src/test/java/org/apache/giraph/examples/TryMultiRpcBindingPortsTest.java (original)
+++ giraph/trunk/src/test/java/org/apache/giraph/examples/TryMultiRpcBindingPortsTest.java Tue Sep 25 17:40:18 2012
@@ -24,6 +24,7 @@ import static org.junit.Assert.assertTru
import java.util.Map;
import java.util.Set;
+import org.apache.giraph.GiraphConfiguration;
import org.apache.giraph.graph.GiraphJob;
import org.apache.giraph.io.IntIntNullIntTextInputFormat;
import org.apache.giraph.io.IdWithValueTextOutputFormat;
@@ -68,7 +69,7 @@ public class TryMultiRpcBindingPortsTest
// run internally
// fail the first port binding attempt
Map<String, String> params = Maps.<String, String>newHashMap();
- params.put(GiraphJob.FAIL_FIRST_RPC_PORT_BIND_ATTEMPT, "true");
+ params.put(GiraphConfiguration.FAIL_FIRST_RPC_PORT_BIND_ATTEMPT, "true");
Iterable<String> results = InternalVertexRunner.run(
ConnectedComponentsVertex.class,
MinimumIntCombiner.class,
Modified: giraph/trunk/src/test/java/org/apache/giraph/graph/TestEdgeListVertex.java
URL: http://svn.apache.org/viewvc/giraph/trunk/src/test/java/org/apache/giraph/graph/TestEdgeListVertex.java?rev=1390014&r1=1390013&r2=1390014&view=diff
==============================================================================
--- giraph/trunk/src/test/java/org/apache/giraph/graph/TestEdgeListVertex.java (original)
+++ giraph/trunk/src/test/java/org/apache/giraph/graph/TestEdgeListVertex.java Tue Sep 25 17:40:18 2012
@@ -17,6 +17,8 @@
*/
package org.apache.giraph.graph;
+import org.apache.giraph.GiraphConfiguration;
+import org.apache.giraph.ImmutableClassesGiraphConfiguration;
import org.apache.giraph.utils.WritableUtils;
import org.apache.giraph.graph.partition.PartitionOwner;
import org.apache.hadoop.conf.Configuration;
@@ -53,11 +55,14 @@ public class TestEdgeListVertex {
private IFDLEdgeListVertex vertex;
/** Job filled in by setup() */
private GiraphJob job;
+ /** Immutable classes giraph configuration */
+ private ImmutableClassesGiraphConfiguration<IntWritable, FloatWritable,
+ DoubleWritable, LongWritable> configuration;
/**
* Simple instantiable class that extends {@link EdgeListVertex}.
*/
- private static class IFDLEdgeListVertex extends
+ public static class IFDLEdgeListVertex extends
EdgeListVertex<IntWritable, FloatWritable, DoubleWritable,
LongWritable> {
@Override
@@ -72,19 +77,21 @@ public class TestEdgeListVertex {
} catch (IOException e) {
throw new RuntimeException("setUp: Failed", e);
}
- job.setVertexClass(IFDLEdgeListVertex.class);
+ job.getConfiguration().setVertexClass(IFDLEdgeListVertex.class);
Configuration conf = job.getConfiguration();
- conf.setClass(GiraphJob.VERTEX_ID_CLASS, IntWritable.class,
+ conf.setClass(GiraphConfiguration.VERTEX_ID_CLASS, IntWritable.class,
WritableComparable.class);
- conf.setClass(GiraphJob.VERTEX_VALUE_CLASS, FloatWritable.class,
+ conf.setClass(GiraphConfiguration.VERTEX_VALUE_CLASS, FloatWritable.class,
Writable.class);
- conf.setClass(GiraphJob.EDGE_VALUE_CLASS, DoubleWritable.class,
+ conf.setClass(GiraphConfiguration.EDGE_VALUE_CLASS, DoubleWritable.class,
Writable.class);
- conf.setClass(GiraphJob.MESSAGE_VALUE_CLASS, LongWritable.class,
+ conf.setClass(GiraphConfiguration.MESSAGE_VALUE_CLASS, LongWritable.class,
Writable.class);
- vertex = (IFDLEdgeListVertex)
- BspUtils.<IntWritable, FloatWritable, DoubleWritable, LongWritable>
- createVertex(conf);
+ configuration =
+ new ImmutableClassesGiraphConfiguration<IntWritable,
+ FloatWritable, DoubleWritable, LongWritable>(
+ job.getConfiguration());
+ vertex = (IFDLEdgeListVertex) configuration.createVertex();
}
@Test
@@ -182,8 +189,9 @@ public class TestEdgeListVertex {
@Test
public void testSerialize() {
+ final int edgesCount = 1000;
Map<IntWritable, DoubleWritable> edgeMap = Maps.newHashMap();
- for (int i = 1000; i > 0; --i) {
+ for (int i = edgesCount; i > 0; --i) {
edgeMap.put(new IntWritable(i), new DoubleWritable(i * 2.0));
}
List<LongWritable> messageList = Lists.newArrayList();
@@ -193,9 +201,9 @@ public class TestEdgeListVertex {
new IntWritable(2), new FloatWritable(3.0f), edgeMap, messageList);
byte[] byteArray = WritableUtils.writeToByteArray(vertex);
IFDLEdgeListVertex readVertex = (IFDLEdgeListVertex)
- BspUtils.<IntWritable, FloatWritable, DoubleWritable, LongWritable>
- createVertex(job.getConfiguration());
+ configuration.createVertex();
WritableUtils.readFieldsFromByteArray(byteArray, readVertex);
+
assertEquals(vertex.getId(), readVertex.getId());
assertEquals(vertex.getValue(), readVertex.getValue());
assertEquals(Lists.newArrayList(vertex.getEdges()),
Modified: giraph/trunk/src/test/java/org/apache/giraph/graph/partition/TestPartitionStores.java
URL: http://svn.apache.org/viewvc/giraph/trunk/src/test/java/org/apache/giraph/graph/partition/TestPartitionStores.java?rev=1390014&r1=1390013&r2=1390014&view=diff
==============================================================================
--- giraph/trunk/src/test/java/org/apache/giraph/graph/partition/TestPartitionStores.java (original)
+++ giraph/trunk/src/test/java/org/apache/giraph/graph/partition/TestPartitionStores.java Tue Sep 25 17:40:18 2012
@@ -18,6 +18,8 @@
package org.apache.giraph.graph.partition;
+import org.apache.giraph.GiraphConfiguration;
+import org.apache.giraph.ImmutableClassesGiraphConfiguration;
import org.apache.giraph.graph.GiraphJob;
import org.apache.giraph.graph.IntIntNullIntVertex;
import org.apache.giraph.graph.Vertex;
@@ -26,6 +28,7 @@ import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
+import org.junit.Before;
import org.junit.Test;
import static org.junit.Assert.assertEquals;
@@ -41,13 +44,16 @@ import java.io.IOException;
* Test case for partition stores.
*/
public class TestPartitionStores {
- private static class MyVertex extends IntIntNullIntVertex {
+ private ImmutableClassesGiraphConfiguration conf;
+
+ public static class MyVertex extends IntIntNullIntVertex {
@Override
public void compute(Iterable<IntWritable> messages) throws IOException {}
}
private Partition<IntWritable, IntWritable, NullWritable,
- IntWritable> createPartition(Configuration conf, Integer id,
+ IntWritable> createPartition(ImmutableClassesGiraphConfiguration conf,
+ Integer id,
Vertex<IntWritable, IntWritable,
NullWritable, IntWritable>... vertices) {
Partition<IntWritable, IntWritable, NullWritable, IntWritable> partition =
@@ -60,20 +66,15 @@ public class TestPartitionStores {
return partition;
}
+ @Before
+ public void setUp() {
+ GiraphConfiguration configuration = new GiraphConfiguration();
+ configuration.setVertexClass(MyVertex.class);
+ conf = new ImmutableClassesGiraphConfiguration(configuration);
+ }
+
@Test
public void testSimplePartitionStore() {
- Configuration conf = new Configuration();
- conf.setClass(GiraphJob.VERTEX_CLASS, MyVertex.class,
- Vertex.class);
- conf.setClass(GiraphJob.VERTEX_ID_CLASS, IntWritable.class,
- WritableComparable.class);
- conf.setClass(GiraphJob.VERTEX_VALUE_CLASS, IntWritable.class,
- Writable.class);
- conf.setClass(GiraphJob.EDGE_VALUE_CLASS, NullWritable.class,
- Writable.class);
- conf.setClass(GiraphJob.MESSAGE_VALUE_CLASS, IntWritable.class,
- Writable.class);
-
PartitionStore<IntWritable, IntWritable, NullWritable, IntWritable>
partitionStore = new SimplePartitionStore<IntWritable, IntWritable,
NullWritable, IntWritable>(conf);
@@ -82,34 +83,24 @@ public class TestPartitionStores {
@Test
public void testDiskBackedPartitionStore() {
- Configuration conf = new Configuration();
- conf.setClass(GiraphJob.VERTEX_CLASS, MyVertex.class,
- Vertex.class);
- conf.setClass(GiraphJob.VERTEX_ID_CLASS, IntWritable.class,
- WritableComparable.class);
- conf.setClass(GiraphJob.VERTEX_VALUE_CLASS, IntWritable.class,
- Writable.class);
- conf.setClass(GiraphJob.EDGE_VALUE_CLASS, NullWritable.class,
- Writable.class);
- conf.setClass(GiraphJob.MESSAGE_VALUE_CLASS, IntWritable.class,
- Writable.class);
-
- conf.setBoolean(GiraphJob.USE_OUT_OF_CORE_GRAPH, true);
- conf.setInt(GiraphJob.MAX_PARTITIONS_IN_MEMORY, 1);
+ conf.setBoolean(GiraphConfiguration.USE_OUT_OF_CORE_GRAPH, true);
+ conf.setInt(GiraphConfiguration.MAX_PARTITIONS_IN_MEMORY, 1);
PartitionStore<IntWritable, IntWritable, NullWritable, IntWritable>
partitionStore = new DiskBackedPartitionStore<IntWritable,
IntWritable, NullWritable, IntWritable>(conf);
testReadWrite(partitionStore, conf);
- conf.setInt(GiraphJob.MAX_PARTITIONS_IN_MEMORY, 2);
+ conf.setInt(GiraphConfiguration.MAX_PARTITIONS_IN_MEMORY, 2);
partitionStore = new DiskBackedPartitionStore<IntWritable,
IntWritable, NullWritable, IntWritable>(conf);
testReadWrite(partitionStore, conf);
}
- public void testReadWrite(PartitionStore<IntWritable, IntWritable,
- NullWritable, IntWritable> partitionStore, Configuration conf) {
+ public void testReadWrite(
+ PartitionStore<IntWritable, IntWritable,
+ NullWritable, IntWritable> partitionStore,
+ ImmutableClassesGiraphConfiguration conf) {
Vertex<IntWritable, IntWritable, NullWritable, IntWritable> v1 =
new MyVertex();
v1.initialize(new IntWritable(1), new IntWritable(1), null, null);
Modified: giraph/trunk/src/test/java/org/apache/giraph/io/TestLongDoubleDoubleAdjacencyListVertexInputFormat.java
URL: http://svn.apache.org/viewvc/giraph/trunk/src/test/java/org/apache/giraph/io/TestLongDoubleDoubleAdjacencyListVertexInputFormat.java?rev=1390014&r1=1390013&r2=1390014&view=diff
==============================================================================
--- giraph/trunk/src/test/java/org/apache/giraph/io/TestLongDoubleDoubleAdjacencyListVertexInputFormat.java (original)
+++ giraph/trunk/src/test/java/org/apache/giraph/io/TestLongDoubleDoubleAdjacencyListVertexInputFormat.java Tue Sep 25 17:40:18 2012
@@ -18,6 +18,7 @@
package org.apache.giraph.io;
+import org.apache.giraph.GiraphConfiguration;
import org.apache.giraph.graph.Edge;
import org.apache.giraph.graph.EdgeListVertex;
import org.apache.giraph.graph.GiraphJob;
@@ -58,9 +59,9 @@ public class TestLongDoubleDoubleAdjacen
rr = mock(RecordReader.class);
when(rr.nextKeyValue()).thenReturn(true);
conf = new Configuration();
- conf.setClass(GiraphJob.VERTEX_CLASS, DummyVertex.class, Vertex.class);
- conf.setClass(GiraphJob.VERTEX_ID_CLASS, LongWritable.class, Writable.class);
- conf.setClass(GiraphJob.VERTEX_VALUE_CLASS, DoubleWritable.class, Writable.class);
+ conf.setClass(GiraphConfiguration.VERTEX_CLASS, DummyVertex.class, Vertex.class);
+ conf.setClass(GiraphConfiguration.VERTEX_ID_CLASS, LongWritable.class, Writable.class);
+ conf.setClass(GiraphConfiguration.VERTEX_VALUE_CLASS, DoubleWritable.class, Writable.class);
graphState = mock(GraphState.class);
tac = mock(TaskAttemptContext.class);
when(tac.getConfiguration()).thenReturn(conf);
Modified: giraph/trunk/src/test/java/org/apache/giraph/io/TestTextDoubleDoubleAdjacencyListVertexInputFormat.java
URL: http://svn.apache.org/viewvc/giraph/trunk/src/test/java/org/apache/giraph/io/TestTextDoubleDoubleAdjacencyListVertexInputFormat.java?rev=1390014&r1=1390013&r2=1390014&view=diff
==============================================================================
--- giraph/trunk/src/test/java/org/apache/giraph/io/TestTextDoubleDoubleAdjacencyListVertexInputFormat.java (original)
+++ giraph/trunk/src/test/java/org/apache/giraph/io/TestTextDoubleDoubleAdjacencyListVertexInputFormat.java Tue Sep 25 17:40:18 2012
@@ -17,6 +17,7 @@
*/
package org.apache.giraph.io;
+import org.apache.giraph.GiraphConfiguration;
import org.apache.giraph.graph.BspUtils;
import org.apache.giraph.graph.Edge;
import org.apache.giraph.graph.EdgeListVertex;
@@ -65,9 +66,9 @@ public class TestTextDoubleDoubleAdjacen
rr = mock(RecordReader.class);
when(rr.nextKeyValue()).thenReturn(true).thenReturn(false);
conf = new Configuration();
- conf.setClass(GiraphJob.VERTEX_CLASS, DummyVertex.class, Vertex.class);
- conf.setClass(GiraphJob.VERTEX_ID_CLASS, Text.class, Writable.class);
- conf.setClass(GiraphJob.VERTEX_VALUE_CLASS, DoubleWritable.class, Writable.class);
+ conf.setClass(GiraphConfiguration.VERTEX_CLASS, DummyVertex.class, Vertex.class);
+ conf.setClass(GiraphConfiguration.VERTEX_ID_CLASS, Text.class, Writable.class);
+ conf.setClass(GiraphConfiguration.VERTEX_VALUE_CLASS, DoubleWritable.class, Writable.class);
graphState = mock(GraphState.class);
tac = mock(TaskAttemptContext.class);
when(tac.getConfiguration()).thenReturn(conf);
Modified: giraph/trunk/src/test/java/zk/TestZooKeeperManager.java
URL: http://svn.apache.org/viewvc/giraph/trunk/src/test/java/zk/TestZooKeeperManager.java?rev=1390014&r1=1390013&r2=1390014&view=diff
==============================================================================
--- giraph/trunk/src/test/java/zk/TestZooKeeperManager.java (original)
+++ giraph/trunk/src/test/java/zk/TestZooKeeperManager.java Tue Sep 25 17:40:18 2012
@@ -21,6 +21,7 @@ import static org.junit.Assert.assertEqu
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import org.apache.giraph.GiraphConfiguration;
import org.apache.giraph.graph.GiraphJob;
import org.apache.giraph.zk.ZooKeeperManager;
import org.apache.hadoop.conf.Configuration;
@@ -35,17 +36,17 @@ public class TestZooKeeperManager {
assertEquals("Default value for base path should be empty",
"", ZooKeeperManager.getBasePath(conf));
- conf.set(GiraphJob.BASE_ZNODE_KEY, "/howdy");
+ conf.set(GiraphConfiguration.BASE_ZNODE_KEY, "/howdy");
assertEquals("Base path should reflect value of " +
- GiraphJob.BASE_ZNODE_KEY,
+ GiraphConfiguration.BASE_ZNODE_KEY,
"/howdy", ZooKeeperManager.getBasePath(conf));
- conf.set(GiraphJob.BASE_ZNODE_KEY, "no_slash");
+ conf.set(GiraphConfiguration.BASE_ZNODE_KEY, "no_slash");
try {
ZooKeeperManager.getBasePath(conf);
fail("Should not have allowed path without starting slash");
} catch (IllegalArgumentException iae) {
- assertTrue(iae.getMessage().contains(GiraphJob.BASE_ZNODE_KEY));
+ assertTrue(iae.getMessage().contains(GiraphConfiguration.BASE_ZNODE_KEY));
}
}
}