You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2014/08/19 07:40:49 UTC
[1/4] TEZ-1246. Replace constructors with create() methods for DAG,
Vertex, Edge etc in the API. (sseth)
Repository: tez
Updated Branches:
refs/heads/master 5f94db786 -> b30e2bcf8
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
index af82e6c..f856956 100644
--- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/BroadcastAndOneToOneExample.java
@@ -142,21 +142,21 @@ public class BroadcastAndOneToOneExample extends Configured implements Tool {
}
}
byte[] procByte = {(byte) (doLocalityCheck ? 1 : 0), 1};
- UserPayload procPayload = new UserPayload(ByteBuffer.wrap(procByte));
+ UserPayload procPayload = UserPayload.create(ByteBuffer.wrap(procByte));
System.out.println("Using " + numOneToOneTasks + " 1-1 tasks");
- Vertex broadcastVertex = new Vertex("Broadcast", new ProcessorDescriptor(
+ Vertex broadcastVertex = Vertex.create("Broadcast", ProcessorDescriptor.create(
InputProcessor.class.getName()), numBroadcastTasks);
- Vertex inputVertex = new Vertex("Input", new ProcessorDescriptor(
+ Vertex inputVertex = Vertex.create("Input", ProcessorDescriptor.create(
InputProcessor.class.getName()).setUserPayload(procPayload), numOneToOneTasks);
- Vertex oneToOneVertex = new Vertex("OneToOne",
- new ProcessorDescriptor(
+ Vertex oneToOneVertex = Vertex.create("OneToOne",
+ ProcessorDescriptor.create(
OneToOneProcessor.class.getName()).setUserPayload(procPayload));
oneToOneVertex.setVertexManagerPlugin(
- new VertexManagerPluginDescriptor(InputReadyVertexManager.class.getName()));
+ VertexManagerPluginDescriptor.create(InputReadyVertexManager.class.getName()));
UnorderedKVEdgeConfig edgeConf = UnorderedKVEdgeConfig
.newBuilder(Text.class.getName(), IntWritable.class.getName()).build();
@@ -166,9 +166,9 @@ public class BroadcastAndOneToOneExample extends Configured implements Tool {
.addVertex(broadcastVertex)
.addVertex(oneToOneVertex)
.addEdge(
- new Edge(inputVertex, oneToOneVertex, edgeConf.createDefaultOneToOneEdgeProperty()))
+ Edge.create(inputVertex, oneToOneVertex, edgeConf.createDefaultOneToOneEdgeProperty()))
.addEdge(
- new Edge(broadcastVertex, oneToOneVertex,
+ Edge.create(broadcastVertex, oneToOneVertex,
edgeConf.createDefaultBroadcastEdgeProperty()));
return dag;
}
@@ -202,7 +202,7 @@ public class BroadcastAndOneToOneExample extends Configured implements Tool {
// is the same filesystem as the one used for Input/Output.
TezClient tezSession = null;
// needs session or else TaskScheduler does not hold onto containers
- tezSession = new TezClient("broadcastAndOneToOneExample", tezConf);
+ tezSession = TezClient.create("broadcastAndOneToOneExample", tezConf);
tezSession.start();
DAGClient dagClient = null;
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
index ac4cfb1..875bfcb 100644
--- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWord.java
@@ -156,7 +156,7 @@ public class FilterLinesByWord extends Configured implements Tool {
- TezClient tezSession = new TezClient("FilterLinesByWordSession", tezConf,
+ TezClient tezSession = TezClient.create("FilterLinesByWordSession", tezConf,
commonLocalResources, credentials);
tezSession.start(); // Why do I need to start the TezSession.
@@ -169,7 +169,7 @@ public class FilterLinesByWord extends Configured implements Tool {
UserPayload stage1Payload = TezUtils.createUserPayloadFromConf(stage1Conf);
// Setup stage1 Vertex
- Vertex stage1Vertex = new Vertex("stage1", new ProcessorDescriptor(
+ Vertex stage1Vertex = Vertex.create("stage1", ProcessorDescriptor.create(
FilterByWordInputProcessor.class.getName()).setUserPayload(stage1Payload))
.setTaskLocalFiles(commonLocalResources);
@@ -186,22 +186,24 @@ public class FilterLinesByWord extends Configured implements Tool {
stage1Vertex.addDataSource("MRInput", dsd);
// Setup stage2 Vertex
- Vertex stage2Vertex = new Vertex("stage2", new ProcessorDescriptor(
+ Vertex stage2Vertex = Vertex.create("stage2", ProcessorDescriptor.create(
FilterByWordOutputProcessor.class.getName()).setUserPayload(
TezUtils.createUserPayloadFromConf(stage2Conf)), 1);
stage2Vertex.setTaskLocalFiles(commonLocalResources);
// Configure the Output for stage2
- OutputDescriptor od = new OutputDescriptor(MROutput.class.getName())
+ OutputDescriptor od = OutputDescriptor.create(MROutput.class.getName())
.setUserPayload(TezUtils.createUserPayloadFromConf(stage2Conf));
- OutputCommitterDescriptor ocd = new OutputCommitterDescriptor(MROutputCommitter.class.getName());
+ OutputCommitterDescriptor ocd =
+ OutputCommitterDescriptor.create(MROutputCommitter.class.getName());
stage2Vertex.addDataSink("MROutput", new DataSinkDescriptor(od, ocd, null));
UnorderedKVEdgeConfig edgeConf = UnorderedKVEdgeConfig
.newBuilder(Text.class.getName(), TextLongPair.class.getName()).build();
DAG dag = new DAG("FilterLinesByWord");
- Edge edge = new Edge(stage1Vertex, stage2Vertex, edgeConf.createDefaultBroadcastEdgeProperty());
+ Edge edge =
+ Edge.create(stage1Vertex, stage2Vertex, edgeConf.createDefaultBroadcastEdgeProperty());
dag.addVertex(stage1Vertex).addVertex(stage2Vertex).addEdge(edge);
LOG.info("Submitting DAG to Tez Session");
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
index a52cb5a..61d827e 100644
--- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/FilterLinesByWordOneToOne.java
@@ -146,7 +146,7 @@ public class FilterLinesByWordOneToOne extends Configured implements Tool {
- TezClient tezSession = new TezClient("FilterLinesByWordSession", tezConf,
+ TezClient tezSession = TezClient.create("FilterLinesByWordSession", tezConf,
commonLocalResources, null);
tezSession.start(); // Why do I need to start the TezSession.
@@ -160,7 +160,7 @@ public class FilterLinesByWordOneToOne extends Configured implements Tool {
UserPayload stage1Payload = TezUtils.createUserPayloadFromConf(stage1Conf);
// Setup stage1 Vertex
- Vertex stage1Vertex = new Vertex("stage1", new ProcessorDescriptor(
+ Vertex stage1Vertex = Vertex.create("stage1", ProcessorDescriptor.create(
FilterByWordInputProcessor.class.getName()).setUserPayload(stage1Payload))
.setTaskLocalFiles(commonLocalResources);
@@ -177,7 +177,7 @@ public class FilterLinesByWordOneToOne extends Configured implements Tool {
stage1Vertex.addDataSource("MRInput", dsd);
// Setup stage2 Vertex
- Vertex stage2Vertex = new Vertex("stage2", new ProcessorDescriptor(
+ Vertex stage2Vertex = Vertex.create("stage2", ProcessorDescriptor.create(
FilterByWordOutputProcessor.class.getName()).setUserPayload(TezUtils
.createUserPayloadFromConf(stage2Conf)), dsd.getNumberOfShards());
stage2Vertex.setTaskLocalFiles(commonLocalResources);
@@ -185,15 +185,16 @@ public class FilterLinesByWordOneToOne extends Configured implements Tool {
// Configure the Output for stage2
stage2Vertex.addDataSink(
"MROutput",
- new DataSinkDescriptor(new OutputDescriptor(MROutput.class.getName())
+ new DataSinkDescriptor(OutputDescriptor.create(MROutput.class.getName())
.setUserPayload(TezUtils.createUserPayloadFromConf(stage2Conf)),
- new OutputCommitterDescriptor(MROutputCommitter.class.getName()), null));
+ OutputCommitterDescriptor.create(MROutputCommitter.class.getName()), null));
UnorderedKVEdgeConfig edgeConf = UnorderedKVEdgeConfig
.newBuilder(Text.class.getName(), TextLongPair.class.getName()).build();
DAG dag = new DAG("FilterLinesByWord");
- Edge edge = new Edge(stage1Vertex, stage2Vertex, edgeConf.createDefaultOneToOneEdgeProperty());
+ Edge edge =
+ Edge.create(stage1Vertex, stage2Vertex, edgeConf.createDefaultOneToOneEdgeProperty());
dag.addVertex(stage1Vertex).addVertex(stage2Vertex).addEdge(edge);
LOG.info("Submitting DAG to Tez Session");
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
index daad5ba..e4c515b 100644
--- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/MRRSleepJob.java
@@ -524,7 +524,7 @@ public class MRRSleepJob extends Configured implements Tool {
UserPayload mapUserPayload = TezUtils.createUserPayloadFromConf(mapStageConf);
int numTasks = generateSplitsInAM ? -1 : numMapper;
- Vertex mapVertex = new Vertex("map", new ProcessorDescriptor(
+ Vertex mapVertex = Vertex.create("map", ProcessorDescriptor.create(
MapProcessor.class.getName()).setUserPayload(mapUserPayload), numTasks)
.setTaskLocalFiles(commonLocalResources);
mapVertex.addDataSource("MRInput", dataSource);
@@ -536,8 +536,8 @@ public class MRRSleepJob extends Configured implements Tool {
Configuration iconf =
intermediateReduceStageConfs[i];
UserPayload iReduceUserPayload = TezUtils.createUserPayloadFromConf(iconf);
- Vertex ivertex = new Vertex("ireduce" + (i+1),
- new ProcessorDescriptor(ReduceProcessor.class.getName()).
+ Vertex ivertex = Vertex.create("ireduce" + (i + 1),
+ ProcessorDescriptor.create(ReduceProcessor.class.getName()).
setUserPayload(iReduceUserPayload), numIReducer);
ivertex.setTaskLocalFiles(commonLocalResources);
vertices.add(ivertex);
@@ -547,7 +547,7 @@ public class MRRSleepJob extends Configured implements Tool {
Vertex finalReduceVertex = null;
if (numReducer > 0) {
UserPayload reducePayload = TezUtils.createUserPayloadFromConf(finalReduceConf);
- finalReduceVertex = new Vertex("reduce", new ProcessorDescriptor(
+ finalReduceVertex = Vertex.create("reduce", ProcessorDescriptor.create(
ReduceProcessor.class.getName()).setUserPayload(reducePayload), numReducer);
finalReduceVertex.setTaskLocalFiles(commonLocalResources);
finalReduceVertex.addDataSink("MROutput", MROutputLegacy.createConfigBuilder(finalReduceConf,
@@ -571,7 +571,7 @@ public class MRRSleepJob extends Configured implements Tool {
dag.addVertex(vertices.get(i));
if (i != 0) {
dag.addEdge(
- new Edge(vertices.get(i - 1), vertices.get(i), edgeConf.createDefaultEdgeProperty()));
+ Edge.create(vertices.get(i - 1), vertices.get(i), edgeConf.createDefaultEdgeProperty()));
}
}
@@ -733,7 +733,7 @@ public class MRRSleepJob extends Configured implements Tool {
mapSleepTime, mapSleepCount, reduceSleepTime, reduceSleepCount,
iReduceSleepTime, iReduceSleepCount, writeSplitsToDfs, generateSplitsInAM);
- TezClient tezSession = new TezClient("MRRSleep", conf, false, null, credentials);
+ TezClient tezSession = TezClient.create("MRRSleep", conf, false, null, credentials);
tezSession.start();
DAGClient dagClient = tezSession.submitDAG(dag);
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java
index edfd04a..3c4e66f 100644
--- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/TestOrderedWordCount.java
@@ -196,7 +196,7 @@ public class TestOrderedWordCount extends Configured implements Tool {
dsd = MRInputLegacy.createConfigBuilder(mapStageConf, TextInputFormat.class, inputPath).build();
}
- Vertex mapVertex = new Vertex("initialmap", new ProcessorDescriptor(
+ Vertex mapVertex = Vertex.create("initialmap", ProcessorDescriptor.create(
MapProcessor.class.getName()).setUserPayload(
TezUtils.createUserPayloadFromConf(mapStageConf))
.setHistoryText(mapStageHistoryText)).setTaskLocalFiles(commonLocalResources);
@@ -206,10 +206,10 @@ public class TestOrderedWordCount extends Configured implements Tool {
ByteArrayOutputStream iROutputStream = new ByteArrayOutputStream(4096);
iReduceStageConf.writeXml(iROutputStream);
String iReduceStageHistoryText = new String(iROutputStream.toByteArray(), "UTF-8");
- Vertex ivertex = new Vertex("intermediate_reducer", new ProcessorDescriptor(
+ Vertex ivertex = Vertex.create("intermediate_reducer", ProcessorDescriptor.create(
ReduceProcessor.class.getName())
- .setUserPayload(TezUtils.createUserPayloadFromConf(iReduceStageConf))
- .setHistoryText(iReduceStageHistoryText), 2);
+ .setUserPayload(TezUtils.createUserPayloadFromConf(iReduceStageConf))
+ .setHistoryText(iReduceStageHistoryText), 2);
ivertex.setTaskLocalFiles(commonLocalResources);
vertices.add(ivertex);
@@ -217,11 +217,11 @@ public class TestOrderedWordCount extends Configured implements Tool {
finalReduceConf.writeXml(finalReduceOutputStream);
String finalReduceStageHistoryText = new String(finalReduceOutputStream.toByteArray(), "UTF-8");
UserPayload finalReducePayload = TezUtils.createUserPayloadFromConf(finalReduceConf);
- Vertex finalReduceVertex = new Vertex("finalreduce",
- new ProcessorDescriptor(
+ Vertex finalReduceVertex = Vertex.create("finalreduce",
+ ProcessorDescriptor.create(
ReduceProcessor.class.getName())
- .setUserPayload(finalReducePayload)
- .setHistoryText(finalReduceStageHistoryText), 1);
+ .setUserPayload(finalReducePayload)
+ .setHistoryText(finalReduceStageHistoryText), 1);
finalReduceVertex.setTaskLocalFiles(commonLocalResources);
finalReduceVertex.addDataSink("MROutput",
MROutputLegacy.createConfigBuilder(finalReduceConf, TextOutputFormat.class, outputPath)
@@ -238,7 +238,7 @@ public class TestOrderedWordCount extends Configured implements Tool {
HashPartitioner.class.getName()).setFromConfiguration(conf)
.configureInput().useLegacyInput().done().build();
dag.addEdge(
- new Edge(dag.getVertex("initialmap"), dag.getVertex("intermediate_reducer"),
+ Edge.create(dag.getVertex("initialmap"), dag.getVertex("intermediate_reducer"),
edgeConf1.createDefaultEdgeProperty()));
OrderedPartitionedKVEdgeConfig edgeConf2 = OrderedPartitionedKVEdgeConfig
@@ -246,7 +246,7 @@ public class TestOrderedWordCount extends Configured implements Tool {
HashPartitioner.class.getName()).setFromConfiguration(conf)
.configureInput().useLegacyInput().done().build();
dag.addEdge(
- new Edge(dag.getVertex("intermediate_reducer"), dag.getVertex("finalreduce"),
+ Edge.create(dag.getVertex("intermediate_reducer"), dag.getVertex("finalreduce"),
edgeConf2.createDefaultEdgeProperty()));
return dag;
@@ -330,7 +330,7 @@ public class TestOrderedWordCount extends Configured implements Tool {
} else {
tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false);
}
- TezClient tezSession = new TezClient("OrderedWordCountSession", tezConf,
+ TezClient tezSession = TezClient.create("OrderedWordCountSession", tezConf,
null, instance.credentials);
tezSession.start();
@@ -384,7 +384,7 @@ public class TestOrderedWordCount extends Configured implements Tool {
}
if (doPreWarm) {
LOG.info("Pre-warming Session");
- PreWarmVertex preWarmVertex = new PreWarmVertex("PreWarm", preWarmNumContainers, dag
+ PreWarmVertex preWarmVertex = PreWarmVertex.create("PreWarm", preWarmNumContainers, dag
.getVertex("initialmap").getTaskResource());
preWarmVertex.setTaskLocalFiles(dag.getVertex("initialmap").getTaskLocalFiles());
preWarmVertex.setTaskEnvironment(dag.getVertex("initialmap").getTaskEnvironment());
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/UnionExample.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/UnionExample.java b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/UnionExample.java
index e136558..78a5566 100644
--- a/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/UnionExample.java
+++ b/tez-tests/src/main/java/org/apache/tez/mapreduce/examples/UnionExample.java
@@ -173,16 +173,16 @@ public class UnionExample {
inputPath);
DataSourceDescriptor dataSource = configurer.generateSplitsInAM(false).build();
- Vertex mapVertex1 = new Vertex("map1", new ProcessorDescriptor(
+ Vertex mapVertex1 = Vertex.create("map1", ProcessorDescriptor.create(
TokenProcessor.class.getName()), numMaps).addDataSource("MRInput", dataSource);
- Vertex mapVertex2 = new Vertex("map2", new ProcessorDescriptor(
+ Vertex mapVertex2 = Vertex.create("map2", ProcessorDescriptor.create(
TokenProcessor.class.getName()), numMaps).addDataSource("MRInput", dataSource);
- Vertex mapVertex3 = new Vertex("map3", new ProcessorDescriptor(
+ Vertex mapVertex3 = Vertex.create("map3", ProcessorDescriptor.create(
TokenProcessor.class.getName()), numMaps).addDataSource("MRInput", dataSource);
- Vertex checkerVertex = new Vertex("checker", new ProcessorDescriptor(
+ Vertex checkerVertex = Vertex.create("checker", ProcessorDescriptor.create(
UnionProcessor.class.getName()), 1);
Configuration outputConf = new Configuration(tezConf);
@@ -211,10 +211,10 @@ public class UnionExample {
.addVertex(mapVertex3)
.addVertex(checkerVertex)
.addEdge(
- new Edge(mapVertex3, checkerVertex, edgeConf.createDefaultEdgeProperty()))
+ Edge.create(mapVertex3, checkerVertex, edgeConf.createDefaultEdgeProperty()))
.addEdge(
- new GroupInputEdge(unionVertex, checkerVertex, edgeConf.createDefaultEdgeProperty(),
- new InputDescriptor(
+ GroupInputEdge.create(unionVertex, checkerVertex, edgeConf.createDefaultEdgeProperty(),
+ InputDescriptor.create(
ConcatenatedMergedKeyValuesInput.class.getName())));
return dag;
}
@@ -251,7 +251,7 @@ public class UnionExample {
// TEZ-674 Obtain tokens based on the Input / Output paths. For now assuming staging dir
// is the same filesystem as the one used for Input/Output.
- TezClient tezSession = new TezClient("UnionExampleSession", tezConf);
+ TezClient tezSession = TezClient.create("UnionExampleSession", tezConf);
tezSession.start();
DAGClient dagClient = null;
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java b/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
index 4f3eb26..49de6c1 100644
--- a/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
+++ b/tez-tests/src/test/java/org/apache/tez/mapreduce/TestMRRJobsDAGApi.java
@@ -186,8 +186,8 @@ public class TestMRRJobsDAGApi {
SleepProcessorConfig spConf = new SleepProcessorConfig(1);
DAG dag = new DAG("TezSleepProcessor");
- Vertex vertex = new Vertex("SleepVertex", new ProcessorDescriptor(
- SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1,
+ Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(
+ SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1,
Resource.newInstance(1024, 1));
dag.addVertex(vertex);
@@ -197,7 +197,7 @@ public class TestMRRJobsDAGApi {
remoteFs.mkdirs(remoteStagingDir);
tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());
- TezClient tezSession = new TezClient("TezSleepProcessor", tezConf, false);
+ TezClient tezSession = TezClient.create("TezSleepProcessor", tezConf, false);
tezSession.start();
DAGClient dagClient = tezSession.submitDAG(dag);
@@ -229,14 +229,14 @@ public class TestMRRJobsDAGApi {
.nextInt(100000))));
remoteFs.mkdirs(remoteStagingDir);
tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());
- tezSession = new TezClient("OrderedWordCountSession", tezConf, true);
+ tezSession = TezClient.create("OrderedWordCountSession", tezConf, true);
tezSession.start();
SleepProcessorConfig spConf = new SleepProcessorConfig(1);
for (int dagIndex = 1; dagIndex <= 2; dagIndex++) {
DAG dag = new DAG("TezSleepProcessor");
- Vertex vertex = new Vertex("SleepVertex", new ProcessorDescriptor(
- SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1,
+ Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(
+ SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1,
Resource.newInstance(1024, 1));
dag.addVertex(vertex);
@@ -274,8 +274,8 @@ public class TestMRRJobsDAGApi {
SleepProcessorConfig spConf = new SleepProcessorConfig(1);
DAG dag = new DAG("TezSleepProcessor");
- Vertex vertex = new Vertex("SleepVertex", new ProcessorDescriptor(
- SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1,
+ Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(
+ SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 1,
Resource.newInstance(1024, 1));
dag.addVertex(vertex);
@@ -287,7 +287,7 @@ public class TestMRRJobsDAGApi {
localFs.mkdirs(stagingDir);
tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDir.toString());
- TezClient tezSession = new TezClient("TezSleepProcessor", tezConf, false);
+ TezClient tezSession = TezClient.create("TezSleepProcessor", tezConf, false);
tezSession.start();
DAGClient dagClient = tezSession.submitDAG(dag);
@@ -316,8 +316,8 @@ public class TestMRRJobsDAGApi {
SleepProcessorConfig spConf = new SleepProcessorConfig(1);
DAG dag = new DAG("TezSleepProcessorHistoryLogging");
- Vertex vertex = new Vertex("SleepVertex", new ProcessorDescriptor(
- SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 2,
+ Vertex vertex = Vertex.create("SleepVertex", ProcessorDescriptor.create(
+ SleepProcessor.class.getName()).setUserPayload(spConf.toUserPayload()), 2,
Resource.newInstance(1024, 1));
dag.addVertex(vertex);
@@ -335,7 +335,7 @@ public class TestMRRJobsDAGApi {
localFs.makeQualified(historyLogDir).toString());
tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, false);
- TezClient tezSession = new TezClient("TezSleepProcessorHistoryLogging", tezConf);
+ TezClient tezSession = TezClient.create("TezSleepProcessorHistoryLogging", tezConf);
tezSession.start();
DAGClient dagClient = tezSession.submitDAG(dag);
@@ -520,7 +520,7 @@ public class TestMRRJobsDAGApi {
TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, remoteStagingDir.toString());
- TezClient tezSession = new TezClient("testrelocalizationsession", tezConf, true);
+ TezClient tezSession = TezClient.create("testrelocalizationsession", tezConf, true);
tezSession.start();
Assert.assertEquals(TezAppMasterStatus.INITIALIZING, tezSession.getAppMasterStatus());
return tezSession;
@@ -538,7 +538,7 @@ public class TestMRRJobsDAGApi {
tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
remoteStagingDir.toString());
- TezClient tezSession = new TezClient("testsession", tezConf, true);
+ TezClient tezSession = TezClient.create("testsession", tezConf, true);
tezSession.start();
Assert.assertEquals(TezAppMasterStatus.INITIALIZING,
tezSession.getAppMasterStatus());
@@ -674,21 +674,22 @@ public class TestMRRJobsDAGApi {
if (initializerClass == null) {
dsd = MRInputLegacy.createConfigBuilder(stage1Conf, SleepInputFormat.class).build();
} else {
- InputInitializerDescriptor iid = new InputInitializerDescriptor(inputInitializerClazz.getName());
+ InputInitializerDescriptor iid =
+ InputInitializerDescriptor.create(inputInitializerClazz.getName());
dsd = MRInputLegacy.createConfigBuilder(stage1Conf, SleepInputFormat.class)
.setCustomInitializerDescriptor(iid).build();
}
}
- Vertex stage1Vertex = new Vertex("map", new ProcessorDescriptor(
- MapProcessor.class.getName()).setUserPayload(stage1Payload),
+ Vertex stage1Vertex = Vertex.create("map", ProcessorDescriptor.create(
+ MapProcessor.class.getName()).setUserPayload(stage1Payload),
dsd.getNumberOfShards(), Resource.newInstance(256, 1));
stage1Vertex.addDataSource("MRInput", dsd);
- Vertex stage2Vertex = new Vertex("ireduce", new ProcessorDescriptor(
- ReduceProcessor.class.getName()).setUserPayload(stage2Payload),
+ Vertex stage2Vertex = Vertex.create("ireduce", ProcessorDescriptor.create(
+ ReduceProcessor.class.getName()).setUserPayload(stage2Payload),
1, Resource.newInstance(256, 1));
- Vertex stage3Vertex = new Vertex("reduce", new ProcessorDescriptor(
- ReduceProcessor.class.getName()).setUserPayload(stage3Payload),
+ Vertex stage3Vertex = Vertex.create("reduce", ProcessorDescriptor.create(
+ ReduceProcessor.class.getName()).setUserPayload(stage3Payload),
1, Resource.newInstance(256, 1));
stage3Vertex.addDataSink("MROutput",
MROutputLegacy.createConfigBuilder(stage3Conf, NullOutputFormat.class).build());
@@ -699,16 +700,18 @@ public class TestMRRJobsDAGApi {
dag.addVertex(stage2Vertex);
dag.addVertex(stage3Vertex);
- Edge edge1 = new Edge(stage1Vertex, stage2Vertex, new EdgeProperty(
+ Edge edge1 = Edge.create(stage1Vertex, stage2Vertex, EdgeProperty.create(
DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL, new OutputDescriptor(
- OrderedPartitionedKVOutput.class.getName()).setUserPayload(stage2Payload), new InputDescriptor(
- OrderedGroupedInputLegacy.class.getName()).setUserPayload(stage2Payload)));
- Edge edge2 = new Edge(stage2Vertex, stage3Vertex, new EdgeProperty(
+ SchedulingType.SEQUENTIAL, OutputDescriptor.create(
+ OrderedPartitionedKVOutput.class.getName()).setUserPayload(stage2Payload),
+ InputDescriptor.create(
+ OrderedGroupedInputLegacy.class.getName()).setUserPayload(stage2Payload)));
+ Edge edge2 = Edge.create(stage2Vertex, stage3Vertex, EdgeProperty.create(
DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL, new OutputDescriptor(
- OrderedPartitionedKVOutput.class.getName()).setUserPayload(stage3Payload), new InputDescriptor(
- OrderedGroupedInputLegacy.class.getName()).setUserPayload(stage3Payload)));
+ SchedulingType.SEQUENTIAL, OutputDescriptor.create(
+ OrderedPartitionedKVOutput.class.getName()).setUserPayload(stage3Payload),
+ InputDescriptor.create(
+ OrderedGroupedInputLegacy.class.getName()).setUserPayload(stage3Payload)));
dag.addEdge(edge1);
dag.addEdge(edge2);
@@ -731,7 +734,7 @@ public class TestMRRJobsDAGApi {
} else {
tempTezconf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
}
- tezSession = new TezClient("testsession", tempTezconf);
+ tezSession = TezClient.create("testsession", tempTezconf);
tezSession.start();
} else {
tezSession = reUseTezSession;
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-tests/src/test/java/org/apache/tez/test/FaultToleranceTestRunner.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/FaultToleranceTestRunner.java b/tez-tests/src/test/java/org/apache/tez/test/FaultToleranceTestRunner.java
index c43b856..84e5b97 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/FaultToleranceTestRunner.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/FaultToleranceTestRunner.java
@@ -65,7 +65,7 @@ public class FaultToleranceTestRunner {
tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR,
remoteStagingDir.toString());
- tezSession = new TezClient("FaultToleranceTestRunner", tezConf);
+ tezSession = TezClient.create("FaultToleranceTestRunner", tezConf);
tezSession.start();
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-tests/src/test/java/org/apache/tez/test/SimpleTestDAG.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/SimpleTestDAG.java b/tez-tests/src/test/java/org/apache/tez/test/SimpleTestDAG.java
index 3a68fd7..f3a8b9c 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/SimpleTestDAG.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/SimpleTestDAG.java
@@ -46,20 +46,20 @@ public class SimpleTestDAG {
public static DAG createDAG(String name,
Configuration conf) throws Exception {
- UserPayload payload = new UserPayload(null);
+ UserPayload payload = UserPayload.create(null);
int taskCount = TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT;
if (conf != null) {
taskCount = conf.getInt(TEZ_SIMPLE_DAG_NUM_TASKS, TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT);
payload = TezUtils.createUserPayloadFromConf(conf);
}
DAG dag = new DAG(name);
- Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
- Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
- dag.addVertex(v1).addVertex(v2).addEdge(new Edge(v1, v2,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
- DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL,
- TestOutput.getOutputDesc(payload),
+ Vertex v1 = Vertex.create("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
+ Vertex v2 = Vertex.create("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
+ dag.addVertex(v1).addVertex(v2).addEdge(Edge.create(v1, v2,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
+ DataSourceType.PERSISTED,
+ SchedulingType.SEQUENTIAL,
+ TestOutput.getOutputDesc(payload),
TestInput.getInputDesc(payload))));
return dag;
}
@@ -82,7 +82,7 @@ public class SimpleTestDAG {
* @throws Exception
*/
public static DAG createDAGForVertexOrder(String name, Configuration conf) throws Exception{
- UserPayload payload = new UserPayload(null);
+ UserPayload payload = UserPayload.create(null);
int taskCount = TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT;
if (conf != null) {
taskCount = conf.getInt(TEZ_SIMPLE_DAG_NUM_TASKS, TEZ_SIMPLE_DAG_NUM_TASKS_DEFAULT);
@@ -90,12 +90,12 @@ public class SimpleTestDAG {
}
DAG dag = new DAG(name);
- Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
- Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
- Vertex v3 = new Vertex("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
- Vertex v4 = new Vertex("v4", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
- Vertex v5 = new Vertex("v5", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
- Vertex v6 = new Vertex("v6", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
+ Vertex v1 = Vertex.create("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
+ Vertex v2 = Vertex.create("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
+ Vertex v3 = Vertex.create("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
+ Vertex v4 = Vertex.create("v4", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
+ Vertex v5 = Vertex.create("v5", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
+ Vertex v6 = Vertex.create("v6", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
// add vertex not in the topological order, since we are using this dag for testing vertex topological order
dag.addVertex(v4)
@@ -104,42 +104,42 @@ public class SimpleTestDAG {
.addVertex(v1)
.addVertex(v2)
.addVertex(v3)
- .addEdge(new Edge(v1, v3,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
- DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL,
- TestOutput.getOutputDesc(payload),
- TestInput.getInputDesc(payload))))
- .addEdge(new Edge(v2, v3,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
- DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL,
- TestOutput.getOutputDesc(payload),
- TestInput.getInputDesc(payload))))
- .addEdge(new Edge(v3, v4,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
- DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL,
- TestOutput.getOutputDesc(payload),
- TestInput.getInputDesc(payload))))
- .addEdge(new Edge(v3, v5,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
- DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL,
- TestOutput.getOutputDesc(payload),
- TestInput.getInputDesc(payload))))
- .addEdge(new Edge(v4, v6,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
- DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL,
- TestOutput.getOutputDesc(payload),
- TestInput.getInputDesc(payload))))
- .addEdge(new Edge(v5, v6,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
- DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL,
- TestOutput.getOutputDesc(payload),
- TestInput.getInputDesc(payload))));
+ .addEdge(Edge.create(v1, v3,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
+ DataSourceType.PERSISTED,
+ SchedulingType.SEQUENTIAL,
+ TestOutput.getOutputDesc(payload),
+ TestInput.getInputDesc(payload))))
+ .addEdge(Edge.create(v2, v3,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
+ DataSourceType.PERSISTED,
+ SchedulingType.SEQUENTIAL,
+ TestOutput.getOutputDesc(payload),
+ TestInput.getInputDesc(payload))))
+ .addEdge(Edge.create(v3, v4,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
+ DataSourceType.PERSISTED,
+ SchedulingType.SEQUENTIAL,
+ TestOutput.getOutputDesc(payload),
+ TestInput.getInputDesc(payload))))
+ .addEdge(Edge.create(v3, v5,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
+ DataSourceType.PERSISTED,
+ SchedulingType.SEQUENTIAL,
+ TestOutput.getOutputDesc(payload),
+ TestInput.getInputDesc(payload))))
+ .addEdge(Edge.create(v4, v6,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
+ DataSourceType.PERSISTED,
+ SchedulingType.SEQUENTIAL,
+ TestOutput.getOutputDesc(payload),
+ TestInput.getInputDesc(payload))))
+ .addEdge(Edge.create(v5, v6,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
+ DataSourceType.PERSISTED,
+ SchedulingType.SEQUENTIAL,
+ TestOutput.getOutputDesc(payload),
+ TestInput.getInputDesc(payload))));
return dag;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-tests/src/test/java/org/apache/tez/test/SimpleTestDAG3Vertices.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/SimpleTestDAG3Vertices.java b/tez-tests/src/test/java/org/apache/tez/test/SimpleTestDAG3Vertices.java
index 967f0ed..c509fe0 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/SimpleTestDAG3Vertices.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/SimpleTestDAG3Vertices.java
@@ -55,21 +55,21 @@ public class SimpleTestDAG3Vertices {
payload = TezUtils.createUserPayloadFromConf(conf);
}
DAG dag = new DAG(name);
- Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
- Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
- Vertex v3 = new Vertex("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
- dag.addVertex(v1).addVertex(v2).addEdge(new Edge(v1, v2,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
- DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL,
- TestOutput.getOutputDesc(payload),
+ Vertex v1 = Vertex.create("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
+ Vertex v2 = Vertex.create("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
+ Vertex v3 = Vertex.create("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
+ dag.addVertex(v1).addVertex(v2).addEdge(Edge.create(v1, v2,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
+ DataSourceType.PERSISTED,
+ SchedulingType.SEQUENTIAL,
+ TestOutput.getOutputDesc(payload),
+ TestInput.getInputDesc(payload))));
+ dag.addVertex(v3).addEdge(Edge.create(v2, v3,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
+ DataSourceType.PERSISTED,
+ SchedulingType.SEQUENTIAL,
+ TestOutput.getOutputDesc(payload),
TestInput.getInputDesc(payload))));
- dag.addVertex(v3).addEdge(new Edge(v2, v3,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
- DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL,
- TestOutput.getOutputDesc(payload),
- TestInput.getInputDesc(payload))));
return dag;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery.java b/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery.java
index 6972714..c9acdc2 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery.java
@@ -131,7 +131,7 @@ public class TestDAGRecovery {
tezConf.set(TezConfiguration.TEZ_AM_LAUNCH_CMD_OPTS, " -Xmx256m");
tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
- tezSession = new TezClient("TestDAGRecovery", tezConf);
+ tezSession = TezClient.create("TestDAGRecovery", tezConf);
tezSession.start();
}
@@ -184,9 +184,9 @@ public class TestDAGRecovery {
DAG dag = SimpleVTestDAG.createDAG("DelayedInitDAG", null);
dag.getVertex("v1").addDataSource(
"i1",
- new DataSourceDescriptor(
- new InputDescriptor(NoOpInput.class.getName()),
- new InputInitializerDescriptor(FailingInputInitializer.class
+ DataSourceDescriptor.create(
+ InputDescriptor.create(NoOpInput.class.getName()),
+ InputInitializerDescriptor.create(FailingInputInitializer.class
.getName()), null));
runDAGAndVerify(dag, State.SUCCEEDED);
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery2.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery2.java b/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery2.java
index ff22cee..37bea80 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery2.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/TestDAGRecovery2.java
@@ -134,7 +134,7 @@ public class TestDAGRecovery2 {
TezConfiguration tezConf = createSessionConfig(remoteStagingDir);
- tezSession = new TezClient("TestDAGRecovery2", tezConf);
+ tezSession = TezClient.create("TestDAGRecovery2", tezConf);
tezSession.start();
}
@@ -177,10 +177,11 @@ public class TestDAGRecovery2 {
public void testFailingCommitter() throws Exception {
DAG dag = SimpleVTestDAG.createDAG("FailingCommitterDAG", null);
OutputDescriptor od =
- new OutputDescriptor(MultiAttemptDAG.NoOpOutput.class.getName());
- od.setUserPayload(new UserPayload(ByteBuffer.wrap(
- new MultiAttemptDAG.FailingOutputCommitter.FailingOutputCommitterConfig(true).toUserPayload())));
- OutputCommitterDescriptor ocd = new OutputCommitterDescriptor(
+ OutputDescriptor.create(MultiAttemptDAG.NoOpOutput.class.getName());
+ od.setUserPayload(UserPayload.create(ByteBuffer.wrap(
+ new MultiAttemptDAG.FailingOutputCommitter.FailingOutputCommitterConfig(true)
+ .toUserPayload())));
+ OutputCommitterDescriptor ocd = OutputCommitterDescriptor.create(
MultiAttemptDAG.FailingOutputCommitter.class.getName());
dag.getVertex("v3").addDataSink("FailingOutput", new DataSinkDescriptor(od, ocd, null));
runDAGAndVerify(dag, State.FAILED);
@@ -195,7 +196,7 @@ public class TestDAGRecovery2 {
TezConfiguration tezConf = createSessionConfig(remoteStagingDir);
tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true);
tezConf.setBoolean(TezConfiguration.DAG_RECOVERY_ENABLED, false);
- TezClient session = new TezClient("TestDAGRecovery2SingleAttemptOnly", tezConf);
+ TezClient session = TezClient.create("TestDAGRecovery2SingleAttemptOnly", tezConf);
session.start();
// DAG should fail as it never completes on the first attempt
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-tests/src/test/java/org/apache/tez/test/TestFaultTolerance.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestFaultTolerance.java b/tez-tests/src/test/java/org/apache/tez/test/TestFaultTolerance.java
index f3de15d..d894403 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/TestFaultTolerance.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/TestFaultTolerance.java
@@ -89,7 +89,7 @@ public class TestFaultTolerance {
remoteStagingDir.toString());
tezConf.setBoolean(TezConfiguration.TEZ_AM_NODE_BLACKLISTING_ENABLED, false);
- tezSession = new TezClient("TestFaultTolerance", tezConf, true);
+ tezSession = TezClient.create("TestFaultTolerance", tezConf, true);
tezSession.start();
}
}
@@ -135,13 +135,15 @@ public class TestFaultTolerance {
@Test (timeout=60000)
public void testBasicSuccessBroadcast() throws Exception {
DAG dag = new DAG("testBasicSuccessBroadcast");
- Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(null), 2, SimpleTestDAG.defaultResource);
- Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(null), 2, SimpleTestDAG.defaultResource);
- dag.addVertex(v1).addVertex(v2).addEdge(new Edge(v1, v2,
- new EdgeProperty(DataMovementType.BROADCAST,
- DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL,
- TestOutput.getOutputDesc(null),
+ Vertex v1 =
+ Vertex.create("v1", TestProcessor.getProcDesc(null), 2, SimpleTestDAG.defaultResource);
+ Vertex v2 =
+ Vertex.create("v2", TestProcessor.getProcDesc(null), 2, SimpleTestDAG.defaultResource);
+ dag.addVertex(v1).addVertex(v2).addEdge(Edge.create(v1, v2,
+ EdgeProperty.create(DataMovementType.BROADCAST,
+ DataSourceType.PERSISTED,
+ SchedulingType.SEQUENTIAL,
+ TestOutput.getOutputDesc(null),
TestInput.getInputDesc(null))));
runDAGAndVerify(dag, DAGStatus.State.SUCCEEDED);
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-tests/src/test/java/org/apache/tez/test/TestInput.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestInput.java b/tez-tests/src/test/java/org/apache/tez/test/TestInput.java
index 17718bb..29ea7c0 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/TestInput.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/TestInput.java
@@ -122,7 +122,7 @@ public class TestInput extends AbstractLogicalInput {
}
public static InputDescriptor getInputDesc(UserPayload payload) {
- InputDescriptor desc = new InputDescriptor(TestInput.class.getName());
+ InputDescriptor desc = InputDescriptor.create(TestInput.class.getName());
if (payload != null) {
desc.setUserPayload(payload);
}
@@ -158,7 +158,7 @@ public class TestInput extends AbstractLogicalInput {
for (int i=0; i<getNumPhysicalInputs(); ++i) {
String msg = ("FailingInput: " + getContext().getUniqueIdentifier() +
" index: " + i + " version: " + lastInputReadyValue);
- events.add(new InputReadErrorEvent(msg, i, lastInputReadyValue));
+ events.add(InputReadErrorEvent.create(msg, i, lastInputReadyValue));
LOG.info("Failing input: " + msg);
}
} else {
@@ -172,7 +172,7 @@ public class TestInput extends AbstractLogicalInput {
}
String msg = ("FailingInput: " + getContext().getUniqueIdentifier() +
" index: " + index.intValue() + " version: " + lastInputReadyValue);
- events.add(new InputReadErrorEvent(msg, index.intValue(), lastInputReadyValue));
+ events.add(InputReadErrorEvent.create(msg, index.intValue(), lastInputReadyValue));
LOG.info("Failing input: " + msg);
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-tests/src/test/java/org/apache/tez/test/TestOutput.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestOutput.java b/tez-tests/src/test/java/org/apache/tez/test/TestOutput.java
index 5f73933..9d9767b 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/TestOutput.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/TestOutput.java
@@ -42,7 +42,7 @@ public class TestOutput extends AbstractLogicalOutput {
}
public static OutputDescriptor getOutputDesc(UserPayload payload) {
- OutputDescriptor desc = new OutputDescriptor(TestOutput.class.getName());
+ OutputDescriptor desc = OutputDescriptor.create(TestOutput.class.getName());
if (payload != null) {
desc.setUserPayload(payload);
}
@@ -80,7 +80,7 @@ public class TestOutput extends AbstractLogicalOutput {
byte[] result = ByteBuffer.allocate(4).putInt(output).array();
List<Event> events = Lists.newArrayListWithCapacity(getNumPhysicalOutputs());
for (int i = 0; i < getNumPhysicalOutputs(); i++) {
- DataMovementEvent event = new DataMovementEvent(i, result);
+ DataMovementEvent event = DataMovementEvent.create(i, result);
events.add(event);
}
return events;
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-tests/src/test/java/org/apache/tez/test/TestProcessor.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestProcessor.java b/tez-tests/src/test/java/org/apache/tez/test/TestProcessor.java
index fa556e6..ed37ea9 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/TestProcessor.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/TestProcessor.java
@@ -101,8 +101,8 @@ public class TestProcessor extends AbstractLogicalIOProcessor {
}
public static ProcessorDescriptor getProcDesc(UserPayload payload) {
- return new ProcessorDescriptor(TestProcessor.class.getName()).setUserPayload(
- payload == null ? new UserPayload (null) : payload);
+ return ProcessorDescriptor.create(TestProcessor.class.getName()).setUserPayload(
+ payload == null ? UserPayload.create(null) : payload);
}
void throwException(String msg) {
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java b/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java
index 132626a..736c54b 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/TestTezJobs.java
@@ -189,7 +189,7 @@ public class TestTezJobs {
tezConf.set(TezConfiguration.TEZ_AM_STAGING_DIR, stagingDirPath.toString());
TezClient tezSession = null;
try {
- tezSession = new TezClient("IntersectExampleSession", tezConf);
+ tezSession = TezClient.create("IntersectExampleSession", tezConf);
tezSession.start();
IntersectDataGen dataGen = new IntersectDataGen();
@@ -391,7 +391,7 @@ public class TestTezJobs {
@Test (timeout=60000)
public void testVertexOrder() throws Exception {
TezConfiguration tezConf = new TezConfiguration(mrrTezCluster.getConfig());
- TezClient tezClient = new TezClient("TestVertexOrder", tezConf);
+ TezClient tezClient = TezClient.create("TestVertexOrder", tezConf);
tezClient.start();
try {
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-tests/src/test/java/org/apache/tez/test/dag/MultiAttemptDAG.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/dag/MultiAttemptDAG.java b/tez-tests/src/test/java/org/apache/tez/test/dag/MultiAttemptDAG.java
index 34c7a7b..9157beb 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/dag/MultiAttemptDAG.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/dag/MultiAttemptDAG.java
@@ -320,36 +320,36 @@ public class MultiAttemptDAG {
public static DAG createDAG(String name,
Configuration conf) throws Exception {
- UserPayload payload = new UserPayload(null);
+ UserPayload payload = UserPayload.create(null);
int taskCount = MULTI_ATTEMPT_DAG_VERTEX_NUM_TASKS_DEFAULT;
if (conf != null) {
taskCount = conf.getInt(MULTI_ATTEMPT_DAG_VERTEX_NUM_TASKS, MULTI_ATTEMPT_DAG_VERTEX_NUM_TASKS_DEFAULT);
payload = TezUtils.createUserPayloadFromConf(conf);
}
DAG dag = new DAG(name);
- Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
- Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
- Vertex v3 = new Vertex("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
+ Vertex v1 = Vertex.create("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
+ Vertex v2 = Vertex.create("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
+ Vertex v3 = Vertex.create("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
// Make each vertex manager fail on appropriate attempt
- v1.setVertexManagerPlugin(new VertexManagerPluginDescriptor(
+ v1.setVertexManagerPlugin(VertexManagerPluginDescriptor.create(
FailOnAttemptVertexManagerPlugin.class.getName())
- .setUserPayload(new UserPayload(ByteBuffer.wrap(new String("1").getBytes()))));
- v2.setVertexManagerPlugin(new VertexManagerPluginDescriptor(
+ .setUserPayload(UserPayload.create(ByteBuffer.wrap(new String("1").getBytes()))));
+ v2.setVertexManagerPlugin(VertexManagerPluginDescriptor.create(
FailOnAttemptVertexManagerPlugin.class.getName())
- .setUserPayload(new UserPayload(ByteBuffer.wrap(new String("2").getBytes()))));
- v3.setVertexManagerPlugin(new VertexManagerPluginDescriptor(
+ .setUserPayload(UserPayload.create(ByteBuffer.wrap(new String("2").getBytes()))));
+ v3.setVertexManagerPlugin(VertexManagerPluginDescriptor.create(
FailOnAttemptVertexManagerPlugin.class.getName())
- .setUserPayload(new UserPayload(ByteBuffer.wrap(new String("3").getBytes()))));
+ .setUserPayload(UserPayload.create(ByteBuffer.wrap(new String("3").getBytes()))));
dag.addVertex(v1).addVertex(v2).addVertex(v3);
- dag.addEdge(new Edge(v1, v2,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
+ dag.addEdge(Edge.create(v1, v2,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
DataSourceType.PERSISTED,
SchedulingType.SEQUENTIAL,
TestOutput.getOutputDesc(payload),
TestInput.getInputDesc(payload))));
- dag.addEdge(new Edge(v2, v3,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
+ dag.addEdge(Edge.create(v2, v3,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
DataSourceType.PERSISTED,
SchedulingType.SEQUENTIAL,
TestOutput.getOutputDesc(payload),
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-tests/src/test/java/org/apache/tez/test/dag/SimpleReverseVTestDAG.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/dag/SimpleReverseVTestDAG.java b/tez-tests/src/test/java/org/apache/tez/test/dag/SimpleReverseVTestDAG.java
index 526e63b..e5c25ac 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/dag/SimpleReverseVTestDAG.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/dag/SimpleReverseVTestDAG.java
@@ -48,29 +48,29 @@ public class SimpleReverseVTestDAG {
public static DAG createDAG(String name,
Configuration conf) throws Exception {
- UserPayload payload = new UserPayload(null);
+ UserPayload payload = UserPayload.create(null);
int taskCount = TEZ_SIMPLE_REVERSE_V_DAG_NUM_TASKS_DEFAULT;
if (conf != null) {
taskCount = conf.getInt(TEZ_SIMPLE_REVERSE_V_DAG_NUM_TASKS, TEZ_SIMPLE_REVERSE_V_DAG_NUM_TASKS_DEFAULT);
payload = TezUtils.createUserPayloadFromConf(conf);
}
DAG dag = new DAG(name);
- Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
- Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
- Vertex v3 = new Vertex("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
+ Vertex v1 = Vertex.create("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
+ Vertex v2 = Vertex.create("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
+ Vertex v3 = Vertex.create("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
dag.addVertex(v1).addVertex(v2).addVertex(v3);
- dag.addEdge(new Edge(v1, v2,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
- DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL,
- TestOutput.getOutputDesc(payload),
+ dag.addEdge(Edge.create(v1, v2,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
+ DataSourceType.PERSISTED,
+ SchedulingType.SEQUENTIAL,
+ TestOutput.getOutputDesc(payload),
+ TestInput.getInputDesc(payload))));
+ dag.addEdge(Edge.create(v1, v3,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
+ DataSourceType.PERSISTED,
+ SchedulingType.SEQUENTIAL,
+ TestOutput.getOutputDesc(payload),
TestInput.getInputDesc(payload))));
- dag.addEdge(new Edge(v1, v3,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
- DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL,
- TestOutput.getOutputDesc(payload),
- TestInput.getInputDesc(payload))));
return dag;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-tests/src/test/java/org/apache/tez/test/dag/SimpleVTestDAG.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/dag/SimpleVTestDAG.java b/tez-tests/src/test/java/org/apache/tez/test/dag/SimpleVTestDAG.java
index 2bb8971..fb85588 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/dag/SimpleVTestDAG.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/dag/SimpleVTestDAG.java
@@ -48,29 +48,29 @@ public class SimpleVTestDAG {
public static DAG createDAG(String name,
Configuration conf) throws Exception {
- UserPayload payload = new UserPayload(null);
+ UserPayload payload = UserPayload.create(null);
int taskCount = TEZ_SIMPLE_V_DAG_NUM_TASKS_DEFAULT;
if (conf != null) {
taskCount = conf.getInt(TEZ_SIMPLE_V_DAG_NUM_TASKS, TEZ_SIMPLE_V_DAG_NUM_TASKS_DEFAULT);
payload = TezUtils.createUserPayloadFromConf(conf);
}
DAG dag = new DAG(name);
- Vertex v1 = new Vertex("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
- Vertex v2 = new Vertex("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
- Vertex v3 = new Vertex("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
+ Vertex v1 = Vertex.create("v1", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
+ Vertex v2 = Vertex.create("v2", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
+ Vertex v3 = Vertex.create("v3", TestProcessor.getProcDesc(payload), taskCount, defaultResource);
dag.addVertex(v1).addVertex(v2).addVertex(v3);
- dag.addEdge(new Edge(v1, v3,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
- DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL,
- TestOutput.getOutputDesc(payload),
+ dag.addEdge(Edge.create(v1, v3,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
+ DataSourceType.PERSISTED,
+ SchedulingType.SEQUENTIAL,
+ TestOutput.getOutputDesc(payload),
+ TestInput.getInputDesc(payload))));
+ dag.addEdge(Edge.create(v2, v3,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
+ DataSourceType.PERSISTED,
+ SchedulingType.SEQUENTIAL,
+ TestOutput.getOutputDesc(payload),
TestInput.getInputDesc(payload))));
- dag.addEdge(new Edge(v2, v3,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
- DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL,
- TestOutput.getOutputDesc(payload),
- TestInput.getInputDesc(payload))));
return dag;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-tests/src/test/java/org/apache/tez/test/dag/SixLevelsFailingDAG.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/dag/SixLevelsFailingDAG.java b/tez-tests/src/test/java/org/apache/tez/test/dag/SixLevelsFailingDAG.java
index f6a9d88..e2f7919 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/dag/SixLevelsFailingDAG.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/dag/SixLevelsFailingDAG.java
@@ -59,20 +59,20 @@ public class SixLevelsFailingDAG extends ThreeLevelsFailingDAG {
protected static void addDAGVerticesAndEdges() {
ThreeLevelsFailingDAG.addDAGVerticesAndEdges();
- l4v1 = new Vertex("l4v1", TestProcessor.getProcDesc(payload), 10, defaultResource);
+ l4v1 = Vertex.create("l4v1", TestProcessor.getProcDesc(payload), 10, defaultResource);
dag.addVertex(l4v1);
addEdge(l3v1, l4v1, DataMovementType.SCATTER_GATHER);
addEdge(l3v2, l4v1, DataMovementType.SCATTER_GATHER);
- l5v1 = new Vertex("l5v1", TestProcessor.getProcDesc(payload), 2, defaultResource);
+ l5v1 = Vertex.create("l5v1", TestProcessor.getProcDesc(payload), 2, defaultResource);
dag.addVertex(l5v1);
addEdge(l4v1, l5v1, DataMovementType.SCATTER_GATHER);
- l5v2 = new Vertex("l5v2", TestProcessor.getProcDesc(payload), 4, defaultResource);
+ l5v2 = Vertex.create("l5v2", TestProcessor.getProcDesc(payload), 4, defaultResource);
dag.addVertex(l5v2);
addEdge(l4v1, l5v2, DataMovementType.SCATTER_GATHER);
- l5v3 = new Vertex("l5v3", TestProcessor.getProcDesc(payload), 1, defaultResource);
+ l5v3 = Vertex.create("l5v3", TestProcessor.getProcDesc(payload), 1, defaultResource);
dag.addVertex(l5v3);
addEdge(l4v1, l5v3, DataMovementType.SCATTER_GATHER);
- l6v1 = new Vertex("l6v1", TestProcessor.getProcDesc(payload), 4, defaultResource);
+ l6v1 = Vertex.create("l6v1", TestProcessor.getProcDesc(payload), 4, defaultResource);
dag.addVertex(l6v1);
addEdge(l5v1, l6v1, DataMovementType.SCATTER_GATHER);
addEdge(l5v2, l6v1, DataMovementType.SCATTER_GATHER);
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-tests/src/test/java/org/apache/tez/test/dag/ThreeLevelsFailingDAG.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/dag/ThreeLevelsFailingDAG.java b/tez-tests/src/test/java/org/apache/tez/test/dag/ThreeLevelsFailingDAG.java
index 97d2d3f..633c8e7 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/dag/ThreeLevelsFailingDAG.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/dag/ThreeLevelsFailingDAG.java
@@ -48,11 +48,11 @@ public class ThreeLevelsFailingDAG extends TwoLevelsFailingDAG {
protected static void addDAGVerticesAndEdges() {
TwoLevelsFailingDAG.addDAGVerticesAndEdges();
- l3v1 = new Vertex("l3v1", TestProcessor.getProcDesc(payload), 4, defaultResource);
+ l3v1 = Vertex.create("l3v1", TestProcessor.getProcDesc(payload), 4, defaultResource);
dag.addVertex(l3v1);
addEdge(l2v1, l3v1, DataMovementType.SCATTER_GATHER);
addEdge(l2v2, l3v1, DataMovementType.SCATTER_GATHER);
- l3v2 = new Vertex("l3v2", TestProcessor.getProcDesc(payload), 4, defaultResource);
+ l3v2 = Vertex.create("l3v2", TestProcessor.getProcDesc(payload), 4, defaultResource);
dag.addVertex(l3v2);
addEdge(l2v2, l3v2, DataMovementType.BROADCAST);
addEdge(l2v3, l3v2, DataMovementType.SCATTER_GATHER);
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-tests/src/test/java/org/apache/tez/test/dag/TwoLevelsFailingDAG.java
----------------------------------------------------------------------
diff --git a/tez-tests/src/test/java/org/apache/tez/test/dag/TwoLevelsFailingDAG.java b/tez-tests/src/test/java/org/apache/tez/test/dag/TwoLevelsFailingDAG.java
index 63850ec..df3bf97 100644
--- a/tez-tests/src/test/java/org/apache/tez/test/dag/TwoLevelsFailingDAG.java
+++ b/tez-tests/src/test/java/org/apache/tez/test/dag/TwoLevelsFailingDAG.java
@@ -50,7 +50,7 @@ import org.apache.tez.test.TestProcessor;
public class TwoLevelsFailingDAG {
static Resource defaultResource = Resource.newInstance(100, 0);
protected static DAG dag;
- protected static UserPayload payload = new UserPayload(null);
+ protected static UserPayload payload = UserPayload.create(null);
protected static Vertex l1v1, l1v2, l1v3, l1v4;
protected static Vertex l2v1, l2v2, l2v3, l2v4;
@@ -65,17 +65,17 @@ public class TwoLevelsFailingDAG {
}
protected static void addDAGVerticesAndEdges() {
- l1v1 = new Vertex("l1v1", TestProcessor.getProcDesc(payload), 1, defaultResource);
- l2v1 = new Vertex("l2v1", TestProcessor.getProcDesc(payload), 1, defaultResource);
+ l1v1 = Vertex.create("l1v1", TestProcessor.getProcDesc(payload), 1, defaultResource);
+ l2v1 = Vertex.create("l2v1", TestProcessor.getProcDesc(payload), 1, defaultResource);
addVerticesAndEdgeInternal(l1v1, l2v1, DataMovementType.SCATTER_GATHER);
- l1v2 = new Vertex("l1v2", TestProcessor.getProcDesc(payload), 2, defaultResource);
- l2v2 = new Vertex("l2v2", TestProcessor.getProcDesc(payload), 3, defaultResource);
+ l1v2 = Vertex.create("l1v2", TestProcessor.getProcDesc(payload), 2, defaultResource);
+ l2v2 = Vertex.create("l2v2", TestProcessor.getProcDesc(payload), 3, defaultResource);
addVerticesAndEdgeInternal(l1v2, l2v2, DataMovementType.SCATTER_GATHER);
- l1v3 = new Vertex("l1v3", TestProcessor.getProcDesc(payload), 3, defaultResource);
- l2v3 = new Vertex("l2v3", TestProcessor.getProcDesc(payload), 2, defaultResource);
+ l1v3 = Vertex.create("l1v3", TestProcessor.getProcDesc(payload), 3, defaultResource);
+ l2v3 = Vertex.create("l2v3", TestProcessor.getProcDesc(payload), 2, defaultResource);
addVerticesAndEdgeInternal(l1v3, l2v3, DataMovementType.SCATTER_GATHER);
- l1v4 = new Vertex("l1v4", TestProcessor.getProcDesc(payload), 2, defaultResource);
- l2v4 = new Vertex("l2v4", TestProcessor.getProcDesc(payload), 3, defaultResource);
+ l1v4 = Vertex.create("l1v4", TestProcessor.getProcDesc(payload), 2, defaultResource);
+ l2v4 = Vertex.create("l2v4", TestProcessor.getProcDesc(payload), 3, defaultResource);
addVerticesAndEdgeInternal(l1v4, l2v4, DataMovementType.BROADCAST);
}
@@ -99,11 +99,11 @@ public class TwoLevelsFailingDAG {
* @param dataMovementType Data movement type
*/
protected static void addEdge(Vertex v1, Vertex v2, DataMovementType dataMovementType) {
- dag.addEdge(new Edge(v1, v2,
- new EdgeProperty(dataMovementType,
- DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL,
- TestOutput.getOutputDesc(payload),
+ dag.addEdge(Edge.create(v1, v2,
+ EdgeProperty.create(dataMovementType,
+ DataSourceType.PERSISTED,
+ SchedulingType.SEQUENTIAL,
+ TestOutput.getOutputDesc(payload),
TestInput.getInputDesc(payload))));
}
[4/4] git commit: TEZ-1246. Replace constructors with create()
methods for DAG, Vertex, Edge etc in the API. (sseth)
Posted by ss...@apache.org.
TEZ-1246. Replace constructors with create() methods for DAG, Vertex,
Edge etc in the API. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/b30e2bcf
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/b30e2bcf
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/b30e2bcf
Branch: refs/heads/master
Commit: b30e2bcf828dfde341c5edb1a9c8945548e620a2
Parents: 5f94db7
Author: Siddharth Seth <ss...@apache.org>
Authored: Mon Aug 18 22:40:29 2014 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Mon Aug 18 22:40:29 2014 -0700
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../java/org/apache/tez/client/TezClient.java | 73 +-
.../java/org/apache/tez/common/TezUtils.java | 3 +-
.../main/java/org/apache/tez/dag/api/DAG.java | 2 +-
.../apache/tez/dag/api/DagTypeConverters.java | 32 +-
.../apache/tez/dag/api/DataSinkDescriptor.java | 4 +-
.../tez/dag/api/DataSourceDescriptor.java | 48 +-
.../main/java/org/apache/tez/dag/api/Edge.java | 25 +-
.../dag/api/EdgeManagerPluginDescriptor.java | 5 +-
.../org/apache/tez/dag/api/EdgeProperty.java | 64 +-
.../org/apache/tez/dag/api/GroupInputEdge.java | 33 +-
.../org/apache/tez/dag/api/InputDescriptor.java | 6 +-
.../tez/dag/api/InputInitializerDescriptor.java | 6 +-
.../tez/dag/api/OutputCommitterDescriptor.java | 6 +-
.../apache/tez/dag/api/OutputDescriptor.java | 6 +-
.../org/apache/tez/dag/api/PreWarmVertex.java | 57 +-
.../apache/tez/dag/api/ProcessorDescriptor.java | 6 +-
.../org/apache/tez/dag/api/UserPayload.java | 22 +-
.../java/org/apache/tez/dag/api/Vertex.java | 120 ++--
.../apache/tez/dag/api/VertexLocationHint.java | 7 +-
.../dag/api/VertexManagerPluginDescriptor.java | 6 +-
.../api/events/CompositeDataMovementEvent.java | 15 +-
.../runtime/api/events/DataMovementEvent.java | 46 +-
.../events/InputConfigureVertexTasksEvent.java | 9 +-
.../api/events/InputDataInformationEvent.java | 23 +-
.../runtime/api/events/InputFailedEvent.java | 7 +-
.../api/events/InputInitializerEvent.java | 27 +-
.../runtime/api/events/InputReadErrorEvent.java | 9 +-
.../api/events/InputUpdatePayloadEvent.java | 6 +-
.../runtime/api/events/VertexManagerEvent.java | 18 +-
tez-api/src/main/proto/Events.proto | 3 +-
.../org/apache/tez/client/TestTezClient.java | 6 +-
.../org/apache/tez/dag/api/TestDAGPlan.java | 98 +--
.../org/apache/tez/dag/api/TestDAGVerify.java | 687 ++++++++++---------
.../tez/dag/api/TestDagTypeConverters.java | 4 +-
.../event/TestCompositeDataMovementEvent.java | 4 +-
.../org/apache/tez/dag/app/dag/impl/Edge.java | 14 +-
.../app/dag/impl/RootInputVertexManager.java | 2 +-
.../apache/tez/dag/app/dag/impl/VertexImpl.java | 12 +-
.../org/apache/tez/client/TestLocalMode.java | 14 +-
.../tez/dag/app/dag/impl/TestDAGImpl.java | 26 +-
.../apache/tez/dag/app/dag/impl/TestEdge.java | 6 +-
.../dag/impl/TestRootInputVertexManager.java | 8 +-
.../tez/dag/app/dag/impl/TestTaskAttempt.java | 2 +-
.../tez/dag/app/dag/impl/TestTaskImpl.java | 2 +-
.../tez/dag/app/dag/impl/TestVertexImpl.java | 110 +--
.../tez/dag/app/rm/TestContainerReuse.java | 6 +-
.../TestHistoryEventsProtoConversion.java | 13 +-
.../tez/dag/history/utils/TestDAGUtils.java | 30 +-
.../tez/runtime/task/TestTaskExecution.java | 4 +-
.../apache/tez/examples/IntersectDataGen.java | 6 +-
.../apache/tez/examples/IntersectExample.java | 12 +-
.../apache/tez/examples/IntersectValidate.java | 12 +-
.../apache/tez/examples/OrderedWordCount.java | 13 +-
.../tez/examples/SimpleSessionExample.java | 2 +-
.../java/org/apache/tez/examples/WordCount.java | 10 +-
.../apache/tez/mapreduce/client/YARNRunner.java | 15 +-
.../common/MRInputAMSplitGenerator.java | 12 +-
.../common/MRInputSplitDistributor.java | 8 +-
.../tez/mapreduce/hadoop/MRInputHelpers.java | 8 +-
.../org/apache/tez/mapreduce/input/MRInput.java | 21 +-
.../apache/tez/mapreduce/output/MROutput.java | 4 +-
.../common/TestMRInputSplitDistributor.java | 8 +-
.../tez/mapreduce/input/TestMultiMRInput.java | 16 +-
.../tez/mapreduce/processor/MapUtils.java | 2 +-
.../processor/map/TestMapProcessor.java | 9 +-
.../processor/reduce/TestReduceProcessor.java | 12 +-
.../org/apache/tez/common/ProtoConverters.java | 17 +-
.../apache/tez/runtime/api/impl/TezEvent.java | 4 +-
.../runtime/api/impl/TezInputContextImpl.java | 2 +-
.../api/impl/TezMergedInputContextImpl.java | 2 +-
.../runtime/api/impl/TezOutputContextImpl.java | 2 +-
.../api/impl/TezProcessorContextImpl.java | 2 +-
.../TestLogicalIOProcessorRuntimeTask.java | 6 +-
.../vertexmanager/ShuffleVertexManager.java | 6 +-
.../common/shuffle/impl/ShuffleScheduler.java | 6 +-
.../writers/UnorderedPartitionedKVWriter.java | 2 +-
.../conf/OrderedPartitionedKVEdgeConfig.java | 12 +-
.../library/conf/UnorderedKVEdgeConfig.java | 18 +-
.../conf/UnorderedPartitionedKVEdgeConfig.java | 12 +-
.../output/OrderedPartitionedKVOutput.java | 5 +-
.../library/output/UnorderedKVOutput.java | 2 +-
.../library/processor/SleepProcessor.java | 4 +-
.../shuffle/common/impl/ShuffleManager.java | 8 +-
.../TestInputReadyVertexManager.java | 42 +-
.../vertexmanager/TestShuffleVertexManager.java | 60 +-
.../library/common/TestValuesIterator.java | 2 +-
.../impl/TestShuffleInputEventHandler.java | 4 +-
.../impl/TestShuffleInputEventHandlerImpl.java | 2 +-
.../examples/BroadcastAndOneToOneExample.java | 18 +-
.../mapreduce/examples/FilterLinesByWord.java | 14 +-
.../examples/FilterLinesByWordOneToOne.java | 13 +-
.../tez/mapreduce/examples/MRRSleepJob.java | 12 +-
.../examples/TestOrderedWordCount.java | 24 +-
.../tez/mapreduce/examples/UnionExample.java | 16 +-
.../apache/tez/mapreduce/TestMRRJobsDAGApi.java | 63 +-
.../tez/test/FaultToleranceTestRunner.java | 2 +-
.../java/org/apache/tez/test/SimpleTestDAG.java | 102 +--
.../apache/tez/test/SimpleTestDAG3Vertices.java | 28 +-
.../org/apache/tez/test/TestDAGRecovery.java | 8 +-
.../org/apache/tez/test/TestDAGRecovery2.java | 13 +-
.../org/apache/tez/test/TestFaultTolerance.java | 18 +-
.../java/org/apache/tez/test/TestInput.java | 6 +-
.../java/org/apache/tez/test/TestOutput.java | 4 +-
.../java/org/apache/tez/test/TestProcessor.java | 4 +-
.../java/org/apache/tez/test/TestTezJobs.java | 4 +-
.../apache/tez/test/dag/MultiAttemptDAG.java | 28 +-
.../tez/test/dag/SimpleReverseVTestDAG.java | 30 +-
.../org/apache/tez/test/dag/SimpleVTestDAG.java | 30 +-
.../tez/test/dag/SixLevelsFailingDAG.java | 10 +-
.../tez/test/dag/ThreeLevelsFailingDAG.java | 4 +-
.../tez/test/dag/TwoLevelsFailingDAG.java | 28 +-
112 files changed, 1429 insertions(+), 1183 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/CHANGES.txt
----------------------------------------------------------------------
diff --git a/CHANGES.txt b/CHANGES.txt
index f1b03a1..a19682a 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -75,6 +75,7 @@ INCOMPATIBLE CHANGES
TEZ-1417. Rename Configurer* to Config/ConfigBuilder
TEZ-1450. Documentation of TezConfiguration
TEZ-1231. Clean up TezRuntimeConfiguration
+ TEZ-1246. Replace constructors with create() methods for DAG, Vertex, Edge etc in the API
Release 0.4.0-incubating: 2014-04-05
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-api/src/main/java/org/apache/tez/client/TezClient.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClient.java b/tez-api/src/main/java/org/apache/tez/client/TezClient.java
index 0891694..7250379 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClient.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClient.java
@@ -105,26 +105,55 @@ public class TezClient {
private int preWarmDAGCounter = 0;
+
+ private TezClient(String name, TezConfiguration tezConf) {
+ this(name, tezConf, tezConf.getBoolean(
+ TezConfiguration.TEZ_AM_SESSION_MODE, TezConfiguration.TEZ_AM_SESSION_MODE_DEFAULT));
+ }
+
+ @Private
+ TezClient(String name, TezConfiguration tezConf,
+ @Nullable Map<String, LocalResource> localResources,
+ @Nullable Credentials credentials) {
+ this(name, tezConf, tezConf.getBoolean(
+ TezConfiguration.TEZ_AM_SESSION_MODE, TezConfiguration.TEZ_AM_SESSION_MODE_DEFAULT),
+ localResources, credentials);
+ }
+
+ private TezClient(String name, TezConfiguration tezConf, boolean isSession) {
+ this(name, tezConf, isSession, null, null);
+ }
+
+ @Private
+ TezClient(String name, TezConfiguration tezConf, boolean isSession,
+ @Nullable Map<String, LocalResource> localResources,
+ @Nullable Credentials credentials) {
+ this.clientName = name;
+ this.isSession = isSession;
+ // Set in conf for local mode AM to figure out whether in session mode or not
+ tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, isSession);
+ this.amConfig = new AMConfiguration(tezConf, localResources, credentials);
+ }
+
/**
* Create a new TezClient. Session or non-session execution mode will be
- * inferred from configuration.
+ * inferred from configuration.
* @param name
* Name of the client. Used for logging etc. This will also be used
* as app master name is session mode
* @param tezConf
* Configuration for the framework
*/
- public TezClient(String name, TezConfiguration tezConf) {
- this(name, tezConf, tezConf.getBoolean(
- TezConfiguration.TEZ_AM_SESSION_MODE, TezConfiguration.TEZ_AM_SESSION_MODE_DEFAULT));
+ public static TezClient create(String name, TezConfiguration tezConf) {
+ return new TezClient(name, tezConf);
}
-
+
/**
* Create a new TezClient. Session or non-session execution mode will be
* inferred from configuration. Set the initial resources and security
* credentials for the App Master. If app master resources/credentials are
- * needed then this is the recommended constructor for session mode execution.
- *
+ * needed then this is the recommended method for session mode execution.
+ *
* @param name
* Name of the client. Used for logging etc. This will also be used
* as app master name is session mode
@@ -144,14 +173,12 @@ public class TezClient {
* In session mode, credentials, if needed, must be set before
* calling start()
*/
- public TezClient(String name, TezConfiguration tezConf,
- @Nullable Map<String, LocalResource> localResources,
- @Nullable Credentials credentials) {
- this(name, tezConf, tezConf.getBoolean(
- TezConfiguration.TEZ_AM_SESSION_MODE, TezConfiguration.TEZ_AM_SESSION_MODE_DEFAULT),
- localResources, credentials);
+ public static TezClient create(String name, TezConfiguration tezConf,
+ @Nullable Map<String, LocalResource> localResources,
+ @Nullable Credentials credentials) {
+ return new TezClient(name, tezConf, localResources, credentials);
}
-
+
/**
* Create a new TezClient with AM session mode set explicitly. This overrides
* the setting from configuration.
@@ -161,8 +188,8 @@ public class TezClient {
* @param tezConf Configuration for the framework
* @param isSession The AM will run in session mode or not
*/
- public TezClient(String name, TezConfiguration tezConf, boolean isSession) {
- this(name, tezConf, isSession, null, null);
+ public static TezClient create(String name, TezConfiguration tezConf, boolean isSession) {
+ return new TezClient(name, tezConf, isSession);
}
/**
@@ -177,16 +204,12 @@ public class TezClient {
* @param localResources resources for the App Master
* @param credentials credentials for the App Master
*/
- public TezClient(String name, TezConfiguration tezConf, boolean isSession,
- @Nullable Map<String, LocalResource> localResources,
- @Nullable Credentials credentials) {
- this.clientName = name;
- this.isSession = isSession;
- // Set in conf for local mode AM to figure out whether in session mode or not
- tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, isSession);
- this.amConfig = new AMConfiguration(tezConf, localResources, credentials);
+ public static TezClient create(String name, TezConfiguration tezConf, boolean isSession,
+ @Nullable Map<String, LocalResource> localResources,
+ @Nullable Credentials credentials) {
+ return new TezClient(name, tezConf, isSession, localResources, credentials);
}
-
+
/**
* Add local resources for the DAG App Master. <br>
* <p>
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezUtils.java b/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
index 7a85317..c008f35 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezUtils.java
@@ -20,7 +20,6 @@ package org.apache.tez.common;
import java.io.IOException;
import java.io.OutputStream;
-import java.nio.ByteBuffer;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -92,7 +91,7 @@ public class TezUtils {
* @throws java.io.IOException
*/
public static UserPayload createUserPayloadFromConf(Configuration conf) throws IOException {
- return new UserPayload(createByteStringFromConf(conf).asReadOnlyByteBuffer());
+ return UserPayload.create(createByteStringFromConf(conf).asReadOnlyByteBuffer());
}
/**
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
index 2d2ba9f..7b9d76b 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/DAG.java
@@ -253,7 +253,7 @@ public class DAG {
Vertex dstVertex = edge.getOutputVertex();
VertexGroup uv = edge.getInputVertexGroup();
for (Vertex member : uv.getMembers()) {
- newEdges.add(new Edge(member, dstVertex, edge.getEdgeProperty()));
+ newEdges.add(Edge.create(member, dstVertex, edge.getEdgeProperty()));
}
dstVertex.addGroupInput(uv.getGroupName(), uv.getGroupInfo());
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java b/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
index 8b71558..8794b7e 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/DagTypeConverters.java
@@ -179,7 +179,7 @@ public class DagTypeConverters {
new HashSet<String>(inputHint.getRackList()));
outputList.add(outputHint);
}
- return new VertexLocationHint(outputList);
+ return VertexLocationHint.create(outputList);
}
// notes re HDFS URL handling:
@@ -237,15 +237,16 @@ public class DagTypeConverters {
public static EdgeProperty createEdgePropertyMapFromDAGPlan(EdgePlan edge) {
if (edge.getDataMovementType() == PlanEdgeDataMovementType.CUSTOM) {
- return new EdgeProperty(
- (edge.hasEdgeManager() ? convertEdgeManagerPluginDescriptorFromDAGPlan(edge.getEdgeManager()) : null),
+ return EdgeProperty.create(
+ (edge.hasEdgeManager() ?
+ convertEdgeManagerPluginDescriptorFromDAGPlan(edge.getEdgeManager()) : null),
convertFromDAGPlan(edge.getDataSourceType()),
convertFromDAGPlan(edge.getSchedulingType()),
convertOutputDescriptorFromDAGPlan(edge.getEdgeSource()),
convertInputDescriptorFromDAGPlan(edge.getEdgeDestination())
);
} else {
- return new EdgeProperty(
+ return EdgeProperty.create(
convertFromDAGPlan(edge.getDataMovementType()),
convertFromDAGPlan(edge.getDataSourceType()),
convertFromDAGPlan(edge.getSchedulingType()),
@@ -320,9 +321,10 @@ public class DagTypeConverters {
TezEntityDescriptorProto proto) {
UserPayload userPayload = null;
if (proto.hasUserPayload()) {
- userPayload = new UserPayload(proto.getUserPayload().asReadOnlyByteBuffer(), proto.getVersion());
+ userPayload =
+ UserPayload.create(proto.getUserPayload().asReadOnlyByteBuffer(), proto.getVersion());
} else {
- userPayload = new UserPayload(null, -1);
+ userPayload = UserPayload.create(null, -1);
}
return userPayload;
}
@@ -331,49 +333,49 @@ public class DagTypeConverters {
TezEntityDescriptorProto proto) {
String className = proto.getClassName();
UserPayload payload = convertTezUserPayloadFromDAGPlan(proto);
- return new InputDescriptor(className).setUserPayload(payload);
+ return InputDescriptor.create(className).setUserPayload(payload);
}
public static OutputDescriptor convertOutputDescriptorFromDAGPlan(
TezEntityDescriptorProto proto) {
String className = proto.getClassName();
UserPayload payload = convertTezUserPayloadFromDAGPlan(proto);
- return new OutputDescriptor(className).setUserPayload(payload);
+ return OutputDescriptor.create(className).setUserPayload(payload);
}
public static InputInitializerDescriptor convertInputInitializerDescriptorFromDAGPlan(
TezEntityDescriptorProto proto) {
String className = proto.getClassName();
UserPayload payload = convertTezUserPayloadFromDAGPlan(proto);
- return new InputInitializerDescriptor(className).setUserPayload(payload);
+ return InputInitializerDescriptor.create(className).setUserPayload(payload);
}
public static OutputCommitterDescriptor convertOutputCommitterDescriptorFromDAGPlan(
TezEntityDescriptorProto proto) {
String className = proto.getClassName();
UserPayload payload = convertTezUserPayloadFromDAGPlan(proto);
- return new OutputCommitterDescriptor(className).setUserPayload(payload);
+ return OutputCommitterDescriptor.create(className).setUserPayload(payload);
}
public static VertexManagerPluginDescriptor convertVertexManagerPluginDescriptorFromDAGPlan(
TezEntityDescriptorProto proto) {
String className = proto.getClassName();
UserPayload payload = convertTezUserPayloadFromDAGPlan(proto);
- return new VertexManagerPluginDescriptor(className).setUserPayload(payload);
+ return VertexManagerPluginDescriptor.create(className).setUserPayload(payload);
}
public static EdgeManagerPluginDescriptor convertEdgeManagerPluginDescriptorFromDAGPlan(
TezEntityDescriptorProto proto) {
String className = proto.getClassName();
UserPayload payload = convertTezUserPayloadFromDAGPlan(proto);
- return new EdgeManagerPluginDescriptor(className).setUserPayload(payload);
+ return EdgeManagerPluginDescriptor.create(className).setUserPayload(payload);
}
public static ProcessorDescriptor convertProcessorDescriptorFromDAGPlan(
TezEntityDescriptorProto proto) {
String className = proto.getClassName();
UserPayload payload = convertTezUserPayloadFromDAGPlan(proto);
- return new ProcessorDescriptor(className).setUserPayload(payload);
+ return ProcessorDescriptor.create(className).setUserPayload(payload);
}
public static TezAppMasterStatus convertTezSessionStatusFromProto(
@@ -589,7 +591,7 @@ public class DagTypeConverters {
outputList.add(outputHint);
}
- return new VertexLocationHint(outputList);
+ return VertexLocationHint.create(outputList);
}
public static VertexLocationHintProto convertVertexLocationHintToProto(
@@ -614,7 +616,7 @@ public class DagTypeConverters {
}
public static UserPayload convertToTezUserPayload(@Nullable ByteBuffer payload, int version) {
- return new UserPayload(payload, version);
+ return UserPayload.create(payload, version);
}
@Nullable
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-api/src/main/java/org/apache/tez/dag/api/DataSinkDescriptor.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DataSinkDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/DataSinkDescriptor.java
index 849e701..2240384 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/DataSinkDescriptor.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/DataSinkDescriptor.java
@@ -38,7 +38,7 @@ public class DataSinkDescriptor {
* Create a {@link DataSinkDescriptor}
* @param outputDescriptor
* An {@link OutputDescriptor} for the output
- * @param outputCommitterDescriptor
+ * @param committerDescriptor
* Specify a committer to be used for the output. Can be null. After all
* tasks in the vertex (or in the DAG) have completed, the committer
* (if specified) is invoked to commit the outputs. Commit is a data
@@ -52,7 +52,7 @@ public class DataSinkDescriptor {
* @param credentials Credentials needs to access the data sink
*/
public DataSinkDescriptor(OutputDescriptor outputDescriptor,
- @Nullable OutputCommitterDescriptor committerDescriptor,
+ @Nullable OutputCommitterDescriptor committerDescriptor,
@Nullable Credentials credentials) {
this.outputDescriptor = outputDescriptor;
this.committerDescriptor = committerDescriptor;
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-api/src/main/java/org/apache/tez/dag/api/DataSourceDescriptor.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/DataSourceDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/DataSourceDescriptor.java
index ecd36b9..fc9cf23 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/DataSourceDescriptor.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/DataSourceDescriptor.java
@@ -44,8 +44,28 @@ public class DataSourceDescriptor {
private final VertexLocationHint locationHint;
private final Map<String, LocalResource> additionalLocalResources;
+ private DataSourceDescriptor(InputDescriptor inputDescriptor,
+ @Nullable InputInitializerDescriptor initializerDescriptor,
+ @Nullable Credentials credentials) {
+ this(inputDescriptor, initializerDescriptor, -1, credentials, null, null);
+ }
+
+ private DataSourceDescriptor(InputDescriptor inputDescriptor,
+ @Nullable InputInitializerDescriptor initializerDescriptor,
+ int numShards,
+ @Nullable Credentials credentials,
+ @Nullable VertexLocationHint locationHint,
+ @Nullable Map<String, LocalResource> additionalLocalResources) {
+ this.inputDescriptor = inputDescriptor;
+ this.initializerDescriptor = initializerDescriptor;
+ this.numShards = numShards;
+ this.credentials = credentials;
+ this.locationHint = locationHint;
+ this.additionalLocalResources = additionalLocalResources;
+ }
+
/**
- * Create a {@link DataSourceDescriptor} when the data shard calculation
+ * Create a {@link DataSourceDescriptor} when the data shard calculation
* happens in the App Master at runtime
* @param inputDescriptor
* An {@link InputDescriptor} for the Input
@@ -60,10 +80,10 @@ public class DataSourceDescriptor {
* meant to determine the parallelism of the vertex, the initial
* vertex parallelism should be set to -1. Can be null.
*/
- public DataSourceDescriptor(InputDescriptor inputDescriptor,
- @Nullable InputInitializerDescriptor initializerDescriptor,
- @Nullable Credentials credentials) {
- this(inputDescriptor, initializerDescriptor, -1, credentials, null, null);
+ public static DataSourceDescriptor create(InputDescriptor inputDescriptor,
+ @Nullable InputInitializerDescriptor initializerDescriptor,
+ @Nullable Credentials credentials) {
+ return new DataSourceDescriptor(inputDescriptor, initializerDescriptor, credentials);
}
/**
@@ -89,16 +109,14 @@ public class DataSourceDescriptor {
* resources. If a name conflict occurs, a {@link
* org.apache.tez.dag.api.TezException} will be thrown
*/
- public DataSourceDescriptor(InputDescriptor inputDescriptor,
- @Nullable InputInitializerDescriptor initializerDescriptor, int numShards,
- @Nullable Credentials credentials, @Nullable VertexLocationHint locationHint,
- @Nullable Map<String, LocalResource> additionalLocalResources) {
- this.inputDescriptor = inputDescriptor;
- this.initializerDescriptor = initializerDescriptor;
- this.numShards = numShards;
- this.credentials = credentials;
- this.locationHint = locationHint;
- this.additionalLocalResources = additionalLocalResources;
+ public static DataSourceDescriptor create(InputDescriptor inputDescriptor,
+ @Nullable InputInitializerDescriptor initializerDescriptor,
+ int numShards,
+ @Nullable Credentials credentials,
+ @Nullable VertexLocationHint locationHint,
+ @Nullable Map<String, LocalResource> additionalLocalResources) {
+ return new DataSourceDescriptor(inputDescriptor, initializerDescriptor, numShards, credentials,
+ locationHint, additionalLocalResources);
}
public InputDescriptor getInputDescriptor() {
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-api/src/main/java/org/apache/tez/dag/api/Edge.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/Edge.java b/tez-api/src/main/java/org/apache/tez/dag/api/Edge.java
index a7f531f..151d0db 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/Edge.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/Edge.java
@@ -34,15 +34,32 @@ public class Edge {
private final Vertex outputVertex;
private final EdgeProperty edgeProperty;
- // InputVertex(EdgeInput) ----- Edge ----- OutputVertex(EdgeOutput)]
- public Edge(Vertex inputVertex,
- Vertex outputVertex,
+ private Edge(Vertex inputVertex,
+ Vertex outputVertex,
EdgeProperty edgeProperty) {
this.inputVertex = inputVertex;
this.outputVertex = outputVertex;
this.edgeProperty = edgeProperty;
}
-
+
+
+
+ /**
+ * Creates an edge between the specified vertices.
+ *
+ * InputVertex(EdgeInput) ----- Edge ----- OutputVertex(EdgeOutput)]
+ *
+ * @param inputVertex the vertex which generates data to the edge.
+ * @param outputVertex the vertex which consumes data from the edge
+ * @param edgeProperty {@link org.apache.tez.dag.api.EdgeProperty} associated with this edge
+ * @return the {@link org.apache.tez.dag.api.Edge}
+ */
+ public static Edge create(Vertex inputVertex,
+ Vertex outputVertex,
+ EdgeProperty edgeProperty) {
+ return new Edge(inputVertex, outputVertex, edgeProperty);
+ }
+
/**
* The @link {@link Vertex} that provides input to the edge
* @return {@link Vertex}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPluginDescriptor.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPluginDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPluginDescriptor.java
index ff3a5ea..b2d2de6 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPluginDescriptor.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/EdgeManagerPluginDescriptor.java
@@ -26,8 +26,11 @@ import org.apache.hadoop.classification.InterfaceAudience.Public;
@Public
public class EdgeManagerPluginDescriptor extends EntityDescriptor<EdgeManagerPluginDescriptor> {
- public EdgeManagerPluginDescriptor(String edgeManagerPluginClassName) {
+ private EdgeManagerPluginDescriptor(String edgeManagerPluginClassName) {
super(edgeManagerPluginClassName);
}
+ public static EdgeManagerPluginDescriptor create(String edgeManagerPluginClassName) {
+ return new EdgeManagerPluginDescriptor(edgeManagerPluginClassName);
+ }
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-api/src/main/java/org/apache/tez/dag/api/EdgeProperty.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/EdgeProperty.java b/tez-api/src/main/java/org/apache/tez/dag/api/EdgeProperty.java
index c481708..c6ac25f 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/EdgeProperty.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/EdgeProperty.java
@@ -33,7 +33,8 @@ import com.google.common.base.Preconditions;
*/
@Public
public class EdgeProperty {
-
+
+
/**
* Defines the manner of data movement between source and destination tasks.
* Determines which destination tasks have access to data produced on this
@@ -112,17 +113,50 @@ public class EdgeProperty {
final InputDescriptor inputDescriptor;
final OutputDescriptor outputDescriptor;
final EdgeManagerPluginDescriptor edgeManagerDescriptor;
-
+
/**
+ * Setup an EdgeProperty which makes use of one of the provided {@link
+ * org.apache.tez.dag.api.EdgeProperty.DataMovementType}s
+ *
* @param dataMovementType
* @param dataSourceType
* @param schedulingType
+ * @param edgeSource The {@link OutputDescriptor} that generates data on the edge.
+ * @param edgeDestination The {@link InputDescriptor} which will consume data from the edge.
+ */
+ public static EdgeProperty create(DataMovementType dataMovementType,
+ DataSourceType dataSourceType,
+ SchedulingType schedulingType,
+ OutputDescriptor edgeSource,
+ InputDescriptor edgeDestination) {
+ return new EdgeProperty(dataMovementType, dataSourceType, schedulingType, edgeSource,
+ edgeDestination);
+ }
+
+ /**
+ * Setup an Edge which uses a custom EdgeManager
+ *
+ * @param edgeManagerDescriptor
+ * the EdgeManager specifications. This can be null if the edge
+ * manager will be setup at runtime
+ * @param dataSourceType
+ * @param schedulingType
* @param edgeSource
* The {@link OutputDescriptor} that generates data on the edge.
* @param edgeDestination
* The {@link InputDescriptor} which will consume data from the edge.
*/
- public EdgeProperty(DataMovementType dataMovementType,
+ public static EdgeProperty create(EdgeManagerPluginDescriptor edgeManagerDescriptor,
+ DataSourceType dataSourceType,
+ SchedulingType schedulingType,
+ OutputDescriptor edgeSource,
+ InputDescriptor edgeDestination) {
+ return new EdgeProperty(edgeManagerDescriptor, dataSourceType, schedulingType, edgeSource,
+ edgeDestination);
+ }
+
+
+ private EdgeProperty(DataMovementType dataMovementType,
DataSourceType dataSourceType,
SchedulingType schedulingType,
OutputDescriptor edgeSource,
@@ -137,24 +171,12 @@ public class EdgeProperty {
this.edgeManagerDescriptor = null;
}
- /**
- * Setup an Edge which uses a custom EdgeManager
- *
- * @param edgeManagerDescriptor
- * the EdgeManager specifications. This can be null if the edge
- * manager will be setup at runtime
- * @param dataSourceType
- * @param schedulingType
- * @param edgeSource
- * The {@link OutputDescriptor} that generates data on the edge.
- * @param edgeDestination
- * The {@link InputDescriptor} which will consume data from the edge.
- */
- public EdgeProperty(EdgeManagerPluginDescriptor edgeManagerDescriptor,
- DataSourceType dataSourceType,
- SchedulingType schedulingType,
- OutputDescriptor edgeSource,
- InputDescriptor edgeDestination) {
+
+ private EdgeProperty(EdgeManagerPluginDescriptor edgeManagerDescriptor,
+ DataSourceType dataSourceType,
+ SchedulingType schedulingType,
+ OutputDescriptor edgeSource,
+ InputDescriptor edgeDestination) {
this.dataMovementType = DataMovementType.CUSTOM;
this.edgeManagerDescriptor = edgeManagerDescriptor;
this.dataSourceType = dataSourceType;
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-api/src/main/java/org/apache/tez/dag/api/GroupInputEdge.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/GroupInputEdge.java b/tez-api/src/main/java/org/apache/tez/dag/api/GroupInputEdge.java
index eb3aa17..cc062b7 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/GroupInputEdge.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/GroupInputEdge.java
@@ -37,6 +37,20 @@ public class GroupInputEdge {
private final InputDescriptor mergedInput;
// InputVertex(EdgeInput) ----- Edge ----- OutputVertex(EdgeOutput)]
+ private GroupInputEdge(VertexGroup inputVertexGroup,
+ Vertex outputVertex,
+ EdgeProperty edgeProperty,
+ InputDescriptor mergedInput) {
+ this.inputVertexGroup = inputVertexGroup;
+ this.outputVertex = outputVertex;
+ this.edgeProperty = edgeProperty;
+ if (mergedInput == null) {
+ throw new TezUncheckedException(
+ "Merged input must be specified when using GroupInputEdge");
+ }
+ this.mergedInput = mergedInput;
+ }
+
/**
* An Edge that connects a {@link VertexGroup} to a destination Vertex. The
* framework takes care of connecting the {@link VertexGroup} members with the
@@ -44,7 +58,7 @@ public class GroupInputEdge {
* named after the VertexGroup instead of individual inputs from group
* members. These individual inputs are merged using the mergedInput before
* presenting them to the destination task.
- *
+ *
* @param inputVertexGroup
* source {@link VertexGroup}
* @param outputVertex
@@ -56,18 +70,11 @@ public class GroupInputEdge {
* the data from the input vertex tasks to create a single input for
* the output vertex tasks
*/
- public GroupInputEdge(VertexGroup inputVertexGroup,
- Vertex outputVertex,
- EdgeProperty edgeProperty,
- InputDescriptor mergedInput) {
- this.inputVertexGroup = inputVertexGroup;
- this.outputVertex = outputVertex;
- this.edgeProperty = edgeProperty;
- if (mergedInput == null) {
- throw new TezUncheckedException(
- "Merged input must be specified when using GroupInputEdge");
- }
- this.mergedInput = mergedInput;
+ public static GroupInputEdge create(VertexGroup inputVertexGroup,
+ Vertex outputVertex,
+ EdgeProperty edgeProperty,
+ InputDescriptor mergedInput) {
+ return new GroupInputEdge(inputVertexGroup, outputVertex, edgeProperty, mergedInput);
}
public VertexGroup getInputVertexGroup() {
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-api/src/main/java/org/apache/tez/dag/api/InputDescriptor.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/InputDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/InputDescriptor.java
index 2a23c08..012c83e 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/InputDescriptor.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/InputDescriptor.java
@@ -33,7 +33,11 @@ public class InputDescriptor extends EntityDescriptor<InputDescriptor> {
super();
}
- public InputDescriptor(String inputClassName) {
+ private InputDescriptor(String inputClassName) {
super(inputClassName);
}
+
+ public static InputDescriptor create(String inputClassName) {
+ return new InputDescriptor(inputClassName);
+ }
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-api/src/main/java/org/apache/tez/dag/api/InputInitializerDescriptor.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/InputInitializerDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/InputInitializerDescriptor.java
index 6db478d..bb2be5d 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/InputInitializerDescriptor.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/InputInitializerDescriptor.java
@@ -33,7 +33,11 @@ public class InputInitializerDescriptor extends EntityDescriptor<InputInitialize
super();
}
- public InputInitializerDescriptor(String initializerClassName) {
+ private InputInitializerDescriptor(String initializerClassName) {
super(initializerClassName);
}
+
+ public static InputInitializerDescriptor create(String initializerClassName) {
+ return new InputInitializerDescriptor(initializerClassName);
+ }
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-api/src/main/java/org/apache/tez/dag/api/OutputCommitterDescriptor.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/OutputCommitterDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/OutputCommitterDescriptor.java
index b2fa2c8..fd8c86d 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/OutputCommitterDescriptor.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/OutputCommitterDescriptor.java
@@ -33,7 +33,11 @@ public class OutputCommitterDescriptor extends EntityDescriptor<OutputCommitterD
super();
}
- public OutputCommitterDescriptor(String committerClassName) {
+ private OutputCommitterDescriptor(String committerClassName) {
super(committerClassName);
}
+
+ public static OutputCommitterDescriptor create(String committerClassName) {
+ return new OutputCommitterDescriptor(committerClassName);
+ }
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-api/src/main/java/org/apache/tez/dag/api/OutputDescriptor.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/OutputDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/OutputDescriptor.java
index 95c9326..58b6ce4 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/OutputDescriptor.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/OutputDescriptor.java
@@ -33,7 +33,11 @@ public class OutputDescriptor extends EntityDescriptor<OutputDescriptor> {
super();
}
- public OutputDescriptor(String outputClassName) {
+ private OutputDescriptor(String outputClassName) {
super(outputClassName);
}
+
+ public static OutputDescriptor create(String outputClassName) {
+ return new OutputDescriptor(outputClassName);
+ }
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-api/src/main/java/org/apache/tez/dag/api/PreWarmVertex.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/PreWarmVertex.java b/tez-api/src/main/java/org/apache/tez/dag/api/PreWarmVertex.java
index 8d2f0d5..4ec553a 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/PreWarmVertex.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/PreWarmVertex.java
@@ -53,12 +53,34 @@ import org.apache.tez.runtime.api.Processor;
@Public
public class PreWarmVertex extends Vertex {
+
+ private PreWarmVertex(String vertexName, ProcessorDescriptor processorDescriptor, int parallelism,
+ Resource taskResource) {
+ super(vertexName, processorDescriptor, parallelism, taskResource);
+ }
+
+
+ private PreWarmVertex(String vertexName, int parallelism, Resource taskResource) {
+ this(vertexName, ProcessorDescriptor.create(
+ "org.apache.tez.runtime.library.processor.PreWarmProcessor"), parallelism, taskResource);
+ }
+
+ /**
+ * Create a config builder for the @link {@link PreWarmVertex}. This may be used to construct the
+ * pre-warm vertex more flexibly.
+ * @param conf
+ * @return
+ */
+ public static PreWarmVertexConfigBuilder createConfigBuilder(Configuration conf) {
+ return new PreWarmVertexConfigBuilder(conf);
+ }
+
/**
* Create a {@link PreWarmVertex} to be used in
* {@link TezClient#preWarm(PreWarmVertex)} It may be necessary to call more
* methods to add local files etc on the pre-warm vertex post creation so that
* it matches the real DAG vertices.
- *
+ *
* @param vertexName
* Name of the vertex
* @param processorDescriptor
@@ -68,11 +90,12 @@ public class PreWarmVertex extends Vertex {
* @param taskResource
* Execution cpu/memory resources etc needed
*/
- public PreWarmVertex(String vertexName, ProcessorDescriptor processorDescriptor, int parallelism,
- Resource taskResource) {
- super(vertexName, processorDescriptor, parallelism, taskResource);
+ public static PreWarmVertex create(String vertexName, ProcessorDescriptor processorDescriptor,
+ int parallelism,
+ Resource taskResource) {
+ return new PreWarmVertex(vertexName, processorDescriptor, parallelism, taskResource);
}
-
+
/**
* Create a {@link PreWarmVertex} to be used in @link
* {@link TezClient#preWarm(PreWarmVertex)} This uses a built in pre-warm
@@ -82,7 +105,7 @@ public class PreWarmVertex extends Vertex {
* they want the containers to be reused. It may be necessary to call more
* methods to add local files etc on the pre-warm vertex post creation so that
* it matches the real DAG vertices.
- *
+ *
* @param vertexName
* Name of the vertex
* @param parallelism
@@ -90,21 +113,10 @@ public class PreWarmVertex extends Vertex {
* @param taskResource
* Execution cpu/memory resources etc needed
*/
- public PreWarmVertex(String vertexName, int parallelism, Resource taskResource) {
- this(vertexName, new ProcessorDescriptor(
- "org.apache.tez.runtime.library.processor.PreWarmProcessor"), parallelism, taskResource);
+ public static PreWarmVertex create(String vertexName, int parallelism, Resource taskResource) {
+ return new PreWarmVertex(vertexName, parallelism, taskResource);
}
-
- /**
- * Create a config builder for the @link {@link PreWarmVertex}. This may be used to construct the
- * pre-warm vertex more flexibly.
- * @param conf
- * @return
- */
- public static PreWarmVertexConfigBuilder createConfigBuilder(Configuration conf) {
- return new PreWarmVertexConfigBuilder(conf);
- }
-
+
/**
* Setup the prewarm vertex constructor. By default is uses the built-in
* PreWarmProcessor and sets up the prewarm container number equal to
@@ -153,10 +165,11 @@ public class PreWarmVertex extends Vertex {
}
}
if (proc == null) {
- proc = new ProcessorDescriptor("org.apache.tez.runtime.library.processor.PreWarmProcessor");
+ proc =
+ ProcessorDescriptor.create("org.apache.tez.runtime.library.processor.PreWarmProcessor");
}
- return new PreWarmVertex(name, proc, parallelism, resource);
+ return create(name, proc, parallelism, resource);
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-api/src/main/java/org/apache/tez/dag/api/ProcessorDescriptor.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/ProcessorDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/ProcessorDescriptor.java
index a2ec7f3..7f088d8 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/ProcessorDescriptor.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/ProcessorDescriptor.java
@@ -33,7 +33,11 @@ public class ProcessorDescriptor extends EntityDescriptor<ProcessorDescriptor> {
super();
}
- public ProcessorDescriptor(String processorClassName) {
+ private ProcessorDescriptor(String processorClassName) {
super(processorClassName);
}
+
+ public static ProcessorDescriptor create(String processorClassName) {
+ return new ProcessorDescriptor(processorClassName);
+ }
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-api/src/main/java/org/apache/tez/dag/api/UserPayload.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/UserPayload.java b/tez-api/src/main/java/org/apache/tez/dag/api/UserPayload.java
index 3530ea9..fa617b5 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/UserPayload.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/UserPayload.java
@@ -35,16 +35,34 @@ public final class UserPayload {
private final int version;
private static final ByteBuffer EMPTY_BYTE = ByteBuffer.wrap(new byte[0]);
- public UserPayload(@Nullable ByteBuffer payload) {
+ private UserPayload(@Nullable ByteBuffer payload) {
this(payload, 0);
}
- public UserPayload(@Nullable ByteBuffer payload, int version) {
+ private UserPayload(@Nullable ByteBuffer payload, int version) {
this.payload = payload == null ? EMPTY_BYTE : payload;
this.version = version;
}
/**
+ * Create a UserPayload instance which encapsulates a {@link java.nio.ByteBuffer}. The version number used will be 0
+ * @param payload the {@link java.nio.ByteBuffer} payload
+ * @return an instance of {@link org.apache.tez.dag.api.UserPayload}
+ */
+ public static UserPayload create(@Nullable ByteBuffer payload) {
+ return new UserPayload(payload);
+ }
+
+ /**
+ * Create an explicitly versioned UserPayload instance which encapsulates a {@link java.nio.ByteBuffer}.
+ * @param payload the {@link java.nio.ByteBuffer} payload
+ * @return an instance of {@link org.apache.tez.dag.api.UserPayload}
+ */
+ public static UserPayload create(@Nullable ByteBuffer payload, int version) {
+ return new UserPayload(payload, version);
+ }
+
+ /**
* Return the payload as a read-only ByteBuffer.
* @return read-only ByteBuffer.
*/
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java b/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java
index 3a21c96..5e15e16 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/Vertex.java
@@ -24,6 +24,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -68,9 +69,45 @@ public class Vertex {
private String taskLaunchCmdOpts = "";
+ @InterfaceAudience.Private
+ Vertex(String vertexName,
+ ProcessorDescriptor processorDescriptor,
+ int parallelism,
+ Resource taskResource) {
+ this(vertexName, processorDescriptor, parallelism, taskResource, false);
+ }
+
+ private Vertex(String vertexName, ProcessorDescriptor processorDescriptor, int parallelism) {
+ this(vertexName, processorDescriptor, parallelism, null, true);
+ }
+
+
+ private Vertex(String vertexName, ProcessorDescriptor processorDescriptor) {
+ this(vertexName, processorDescriptor, -1);
+ }
+
+ private Vertex(String vertexName,
+ ProcessorDescriptor processorDescriptor,
+ int parallelism,
+ Resource taskResource,
+ boolean allowIncomplete) {
+ this.vertexName = vertexName;
+ this.processorDescriptor = processorDescriptor;
+ this.parallelism = parallelism;
+ this.taskResource = taskResource;
+ if (parallelism < -1) {
+ throw new IllegalArgumentException(
+ "Parallelism should be -1 if determined by the AM"
+ + ", otherwise should be >= 0");
+ }
+ if (!allowIncomplete && taskResource == null) {
+ throw new IllegalArgumentException("Resource cannot be null");
+ }
+ }
+
/**
* Create a new vertex with the given name.
- *
+ *
* @param vertexName
* Name of the vertex
* @param processorDescriptor
@@ -84,13 +121,34 @@ public class Vertex {
* Physical resources like memory/cpu thats used by each task of this
* vertex.
*/
- public Vertex(String vertexName,
- ProcessorDescriptor processorDescriptor,
- int parallelism,
- Resource taskResource) {
- this(vertexName, processorDescriptor, parallelism, taskResource, false);
+ public static Vertex create(String vertexName,
+ ProcessorDescriptor processorDescriptor,
+ int parallelism,
+ Resource taskResource) {
+ return new Vertex(vertexName, processorDescriptor, parallelism, taskResource);
}
-
+
+ /**
+ * Create a new vertex with the given name. <br>
+ * The vertex task resource will be picked from configuration <br>
+ * The vertex parallelism will be inferred. If it cannot be inferred then an
+ * error will be reported. This constructor may be used for vertices that have
+ * data sources, or connected via 1-1 edges or have runtime parallelism
+ * estimation via data source initializers or vertex managers. Calling this
+ * constructor is equivalent to calling
+ * {@link Vertex#Vertex(String, ProcessorDescriptor, int)} with the
+ * parallelism set to -1.
+ *
+ * @param vertexName
+ * Name of the vertex
+ * @param processorDescriptor
+ * Description of the processor that is executed in every task of
+ * this vertex
+ */
+ public static Vertex create(String vertexName, ProcessorDescriptor processorDescriptor) {
+ return new Vertex(vertexName, processorDescriptor);
+ }
+
/**
* Create a new vertex with the given name and parallelism. <br>
* The vertex task resource will be picked from configuration
@@ -100,7 +158,7 @@ public class Vertex {
* own logic to determine task resources and use
* {@link Vertex#Vertex(String, ProcessorDescriptor, int, Resource)} to create
* the Vertex.
- *
+ *
* @param vertexName
* Name of the vertex
* @param processorDescriptor
@@ -111,49 +169,11 @@ public class Vertex {
* decided at runtime. Parallelism may change at runtime due to graph
* reconfigurations.
*/
- public Vertex(String vertexName, ProcessorDescriptor processorDescriptor, int parallelism) {
- this(vertexName, processorDescriptor, parallelism, null, true);
+ public static Vertex create(String vertexName, ProcessorDescriptor processorDescriptor,
+ int parallelism) {
+ return new Vertex(vertexName, processorDescriptor, parallelism);
}
-
- /**
- * Create a new vertex with the given name. <br>
- * The vertex task resource will be picked from configuration <br>
- * The vertex parallelism will be inferred. If it cannot be inferred then an
- * error will be reported. This constructor may be used for vertices that have
- * data sources, or connected via 1-1 edges or have runtime parallelism
- * estimation via data source initializers or vertex managers. Calling this
- * constructor is equivalent to calling
- * {@link Vertex#Vertex(String, ProcessorDescriptor, int)} with the
- * parallelism set to -1.
- *
- * @param vertexName
- * Name of the vertex
- * @param processorDescriptor
- * Description of the processor that is executed in every task of
- * this vertex
- */
- public Vertex(String vertexName, ProcessorDescriptor processorDescriptor) {
- this(vertexName, processorDescriptor, -1);
- }
-
- private Vertex(String vertexName,
- ProcessorDescriptor processorDescriptor,
- int parallelism,
- Resource taskResource,
- boolean allowIncomplete) {
- this.vertexName = vertexName;
- this.processorDescriptor = processorDescriptor;
- this.parallelism = parallelism;
- this.taskResource = taskResource;
- if (parallelism < -1) {
- throw new IllegalArgumentException(
- "Parallelism should be -1 if determined by the AM"
- + ", otherwise should be >= 0");
- }
- if (!allowIncomplete && taskResource == null) {
- throw new IllegalArgumentException("Resource cannot be null");
- }
- }
+
/**
* Get the vertex name
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-api/src/main/java/org/apache/tez/dag/api/VertexLocationHint.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/VertexLocationHint.java b/tez-api/src/main/java/org/apache/tez/dag/api/VertexLocationHint.java
index 898b6be..2af46ea 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/VertexLocationHint.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/VertexLocationHint.java
@@ -38,7 +38,7 @@ public class VertexLocationHint {
private final List<TaskLocationHint> taskLocationHints;
- public VertexLocationHint(List<TaskLocationHint> taskLocationHints) {
+ private VertexLocationHint(List<TaskLocationHint> taskLocationHints) {
if (taskLocationHints != null) {
this.taskLocationHints = Collections.unmodifiableList(taskLocationHints);
} else {
@@ -46,6 +46,11 @@ public class VertexLocationHint {
}
}
+ public static VertexLocationHint create(
+ List<TaskLocationHint> taskLocationHints) {
+ return new VertexLocationHint(taskLocationHints);
+ }
+
public List<TaskLocationHint> getTaskLocationHints() {
return taskLocationHints;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginDescriptor.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginDescriptor.java b/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginDescriptor.java
index c2bce76..687f517 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginDescriptor.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/VertexManagerPluginDescriptor.java
@@ -32,7 +32,11 @@ public class VertexManagerPluginDescriptor extends EntityDescriptor<VertexManage
super();
}
- public VertexManagerPluginDescriptor(String className) {
+ private VertexManagerPluginDescriptor(String className) {
super(className);
}
+
+ public static VertexManagerPluginDescriptor create(String className) {
+ return new VertexManagerPluginDescriptor(className);
+ }
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-api/src/main/java/org/apache/tez/runtime/api/events/CompositeDataMovementEvent.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/CompositeDataMovementEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/CompositeDataMovementEvent.java
index e7322c6..87403de 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/CompositeDataMovementEvent.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/CompositeDataMovementEvent.java
@@ -44,6 +44,12 @@ public class CompositeDataMovementEvent extends Event {
protected final byte[] userPayload;
+ private CompositeDataMovementEvent(int srcIndexStart, int count, byte[] userPayload) {
+ this.sourceIndexStart = srcIndexStart;
+ this.count = count;
+ this.userPayload = userPayload;
+ }
+
/**
* @param srcIndexStart
* the startIndex of the physical source which generated the event
@@ -54,10 +60,9 @@ public class CompositeDataMovementEvent extends Event {
* @param userPayload
* the common payload between all the events.
*/
- public CompositeDataMovementEvent(int srcIndexStart, int count, byte[] userPayload) {
- this.sourceIndexStart = srcIndexStart;
- this.count = count;
- this.userPayload = userPayload;
+ public static CompositeDataMovementEvent create(int srcIndexStart, int count,
+ byte[] userPayload) {
+ return new CompositeDataMovementEvent(srcIndexStart, count, userPayload);
}
public int getSourceIndexStart() {
@@ -97,7 +102,7 @@ public class CompositeDataMovementEvent extends Event {
@Override
public DataMovementEvent next() {
- DataMovementEvent dmEvent = new DataMovementEvent(currentPos, userPayload);
+ DataMovementEvent dmEvent = DataMovementEvent.create(currentPos, userPayload);
currentPos++;
dmEvent.setVersion(version);
return dmEvent;
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-api/src/main/java/org/apache/tez/runtime/api/events/DataMovementEvent.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/DataMovementEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/DataMovementEvent.java
index b737430..88b36cb 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/DataMovementEvent.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/DataMovementEvent.java
@@ -53,35 +53,53 @@ public final class DataMovementEvent extends Event {
*/
private int version;
- /**
- * User Event constructor
- * @param sourceIndex Index to identify the physical edge of the input/output
- * that generated the event
- * @param userPayload User Payload of the User Event
- */
- public DataMovementEvent(int sourceIndex,
- byte[] userPayload) {
+
+ private DataMovementEvent(int sourceIndex,
+ byte[] userPayload) {
this.userPayload = userPayload;
this.sourceIndex = sourceIndex;
}
@Private
- public DataMovementEvent(int sourceIndex,
- int targetIndex,
- int version,
- byte[] userPayload) {
+ private DataMovementEvent(int sourceIndex,
+ int targetIndex,
+ int version,
+ byte[] userPayload) {
this.userPayload = userPayload;
this.sourceIndex = sourceIndex;
this.version = version;
this.targetIndex = targetIndex;
}
+ private DataMovementEvent(byte[] userPayload) {
+ this(-1, userPayload);
+ }
+
+ /**
+ * User Event constructor
+ * @param sourceIndex Index to identify the physical edge of the input/output
+ * that generated the event
+ * @param userPayload User Payload of the User Event
+ */
+ public static DataMovementEvent create(int sourceIndex,
+ byte[] userPayload) {
+ return new DataMovementEvent(sourceIndex, userPayload);
+ }
+
/**
* Constructor for Processor-generated User Events
* @param userPayload
*/
- public DataMovementEvent(byte[] userPayload) {
- this(-1, userPayload);
+ public static DataMovementEvent create(byte[] userPayload) {
+ return new DataMovementEvent(userPayload);
+ }
+
+ @Private
+ public static DataMovementEvent create(int sourceIndex,
+ int targetIndex,
+ int version,
+ byte[] userPayload) {
+ return new DataMovementEvent(sourceIndex, targetIndex, version, userPayload);
}
public byte[] getUserPayload() {
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputConfigureVertexTasksEvent.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputConfigureVertexTasksEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputConfigureVertexTasksEvent.java
index b7a0e01..05857d2 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputConfigureVertexTasksEvent.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputConfigureVertexTasksEvent.java
@@ -38,13 +38,18 @@ public class InputConfigureVertexTasksEvent extends Event {
private final VertexLocationHint locationHint;
private final InputSpecUpdate inputSpecUpdate;
- public InputConfigureVertexTasksEvent(int numTasks, VertexLocationHint locationHint,
- InputSpecUpdate inputSpecUpdate) {
+ private InputConfigureVertexTasksEvent(int numTasks, VertexLocationHint locationHint,
+ InputSpecUpdate inputSpecUpdate) {
this.numTasks = numTasks;
this.locationHint = locationHint;
this.inputSpecUpdate = inputSpecUpdate;
}
+ public static InputConfigureVertexTasksEvent create(int numTasks, VertexLocationHint locationHint,
+ InputSpecUpdate inputSpecUpdate) {
+ return new InputConfigureVertexTasksEvent(numTasks, locationHint, inputSpecUpdate);
+ }
+
public int getNumTasks() {
return numTasks;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputDataInformationEvent.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputDataInformationEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputDataInformationEvent.java
index 8e080be..ec1d85e 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputDataInformationEvent.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputDataInformationEvent.java
@@ -48,23 +48,32 @@ public final class InputDataInformationEvent extends Event {
private final byte[] userPayload;
private final Object userPayloadObject;
- /**
- * Provide a serialized form of the payload
- * @param srcIndex the src index
- * @param userPayload the serialized payload
- */
- public InputDataInformationEvent(int srcIndex, byte[] userPayload) {
+
+ private InputDataInformationEvent(int srcIndex, byte[] userPayload) {
this.sourceIndex = srcIndex;
this.userPayload = userPayload;
this.userPayloadObject = null;
}
- public InputDataInformationEvent(int srcIndex, Object userPayloadDeserialized) {
+ private InputDataInformationEvent(int srcIndex, Object userPayloadDeserialized) {
this.sourceIndex = srcIndex;
this.userPayloadObject = userPayloadDeserialized;
this.userPayload = null;
}
+ /**
+ * Provide a serialized form of the payload
+ * @param srcIndex the src index
+ * @param userPayload the serialized payload
+ */
+ public static InputDataInformationEvent create(int srcIndex, byte[] userPayload) {
+ return new InputDataInformationEvent(srcIndex, userPayload);
+ }
+
+ public static InputDataInformationEvent create(int srcIndex, Object userPayloadDeserialized) {
+ return new InputDataInformationEvent(srcIndex, userPayloadDeserialized);
+ }
+
public int getSourceIndex() {
return this.sourceIndex;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputFailedEvent.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputFailedEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputFailedEvent.java
index 9607eda..639d0b9 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputFailedEvent.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputFailedEvent.java
@@ -46,11 +46,16 @@ public class InputFailedEvent extends Event{
}
@Private
- public InputFailedEvent(int targetIndex, int version) {
+ private InputFailedEvent(int targetIndex, int version) {
this.targetIndex = targetIndex;
this.version = version;
}
+ @Private
+ public static InputFailedEvent create(int targetIndex, int version) {
+ return new InputFailedEvent(targetIndex, version);
+ }
+
public int getTargetIndex() {
return targetIndex;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputInitializerEvent.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputInitializerEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputInitializerEvent.java
index fbe79d8..e4e9ef9 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputInitializerEvent.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputInitializerEvent.java
@@ -38,25 +38,26 @@ public class InputInitializerEvent extends Event {
private String targetVertexName;
private String targetInputName;
- private int version;
private byte[] eventPayload;
+ private InputInitializerEvent(String targetVertexName, String targetInputName,
+ byte[] eventPayload) {
+ Preconditions.checkNotNull(targetVertexName, "TargetVertexName cannot be null");
+ Preconditions.checkNotNull(targetInputName, "TargetInputName cannot be null");
+ this.targetVertexName = targetVertexName;
+ this.targetInputName = targetInputName;
+ this.eventPayload = eventPayload;
+ }
+
/**
* @param targetVertexName the vertex on which the targeted Input exists
* @param targetInputName the name of the input
* @param eventPayload the payload for the event. It is advisable to limit the size of the
* payload to a few KB at max
- * @param version version of the event. Multiple versions may be generated in case of
- * retries
*/
- public InputInitializerEvent(String targetVertexName, String targetInputName,
- byte[] eventPayload, int version) {
- Preconditions.checkNotNull(targetVertexName, "TargetVertexName cannot be null");
- Preconditions.checkNotNull(targetInputName, "TargetInputName cannot be null");
- this.targetVertexName = targetVertexName;
- this.targetInputName = targetInputName;
- this.version = version;
- this.eventPayload = eventPayload;
+ public static InputInitializerEvent create(String targetVertexName, String targetInputName,
+ byte[] eventPayload) {
+ return new InputInitializerEvent(targetVertexName, targetInputName, eventPayload);
}
/**
@@ -77,10 +78,6 @@ public class InputInitializerEvent extends Event {
return this.targetInputName;
}
- public int getVersion() {
- return this.version;
- }
-
/**
* Get the actual user payload
*
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputReadErrorEvent.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputReadErrorEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputReadErrorEvent.java
index e838173..7d2e0d2 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputReadErrorEvent.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputReadErrorEvent.java
@@ -44,14 +44,19 @@ public final class InputReadErrorEvent extends Event {
*/
private final int version;
- public InputReadErrorEvent(String diagnostics, int index,
- int version) {
+ private InputReadErrorEvent(String diagnostics, int index,
+ int version) {
super();
this.diagnostics = diagnostics;
this.index = index;
this.version = version;
}
+ public static InputReadErrorEvent create(String diagnostics, int index,
+ int version) {
+ return new InputReadErrorEvent(diagnostics, index, version);
+ }
+
public String getDiagnostics() {
return diagnostics;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputUpdatePayloadEvent.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputUpdatePayloadEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputUpdatePayloadEvent.java
index 0232607..2d1a8c9 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputUpdatePayloadEvent.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/InputUpdatePayloadEvent.java
@@ -38,11 +38,15 @@ public class InputUpdatePayloadEvent extends Event {
private final byte[] userPayload;
- public InputUpdatePayloadEvent(byte[] userPayload) {
+ private InputUpdatePayloadEvent(byte[] userPayload) {
Preconditions.checkNotNull(userPayload);
this.userPayload = userPayload;
}
+ public static InputUpdatePayloadEvent create(byte[] userPayload) {
+ return new InputUpdatePayloadEvent(userPayload);
+ }
+
public byte[] getUserPayload() {
return userPayload;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-api/src/main/java/org/apache/tez/runtime/api/events/VertexManagerEvent.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/runtime/api/events/VertexManagerEvent.java b/tez-api/src/main/java/org/apache/tez/runtime/api/events/VertexManagerEvent.java
index c2c6129..d93294c 100644
--- a/tez-api/src/main/java/org/apache/tez/runtime/api/events/VertexManagerEvent.java
+++ b/tez-api/src/main/java/org/apache/tez/runtime/api/events/VertexManagerEvent.java
@@ -42,19 +42,23 @@ public class VertexManagerEvent extends Event {
* User payload to be sent
*/
private final byte[] userPayload;
-
+
+ private VertexManagerEvent(String vertexName, byte[] userPayload) {
+ Preconditions.checkArgument(vertexName != null);
+ Preconditions.checkArgument(userPayload != null);
+ this.targetVertexName = vertexName;
+ this.userPayload = userPayload;
+ }
+
/**
* Create a new VertexManagerEvent
* @param vertexName
* @param userPayload This should not be modified since a reference is kept
*/
- public VertexManagerEvent(String vertexName, byte[] userPayload) {
- Preconditions.checkArgument(vertexName != null);
- Preconditions.checkArgument(userPayload != null);
- this.targetVertexName = vertexName;
- this.userPayload = userPayload;
+ public static VertexManagerEvent create(String vertexName, byte[] userPayload) {
+ return new VertexManagerEvent(vertexName, userPayload);
}
-
+
public String getTargetVertexName() {
return targetVertexName;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-api/src/main/proto/Events.proto
----------------------------------------------------------------------
diff --git a/tez-api/src/main/proto/Events.proto b/tez-api/src/main/proto/Events.proto
index cda21d9..490fa3c 100644
--- a/tez-api/src/main/proto/Events.proto
+++ b/tez-api/src/main/proto/Events.proto
@@ -59,6 +59,5 @@ message CompositeEventProto {
message RootInputInitializerEventProto {
optional string target_vertex_name = 1;
optional string target_input_name = 2;
- optional int32 version = 3;
- optional bytes user_payload = 4;
+ optional bytes user_payload = 3;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
index 8e3c397..ddddb16 100644
--- a/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
+++ b/tez-api/src/test/java/org/apache/tez/client/TestTezClient.java
@@ -144,7 +144,7 @@ public class TestTezClient {
}
DAG dag = new DAG("DAG").addVertex(
- new Vertex("Vertex", new ProcessorDescriptor("P"), 1, Resource.newInstance(1, 1)));
+ Vertex.create("Vertex", ProcessorDescriptor.create("P"), 1, Resource.newInstance(1, 1)));
DAGClient dagClient = client.submitDAG(dag);
Assert.assertTrue(dagClient.getExecutionContext().contains(client.mockAppId.toString()));
@@ -178,7 +178,7 @@ public class TestTezClient {
.thenReturn(appId2);
dag = new DAG("DAG").addVertex(
- new Vertex("Vertex", new ProcessorDescriptor("P"), 1, Resource.newInstance(1, 1)));
+ Vertex.create("Vertex", ProcessorDescriptor.create("P"), 1, Resource.newInstance(1, 1)));
dagClient = client.submitDAG(dag);
if (isSession) {
@@ -227,7 +227,7 @@ public class TestTezClient {
configure(client);
client.start();
- PreWarmVertex vertex = new PreWarmVertex("PreWarm", 1, Resource.newInstance(1, 1));
+ PreWarmVertex vertex = PreWarmVertex.create("PreWarm", 1, Resource.newInstance(1, 1));
client.preWarm(vertex);
ArgumentCaptor<SubmitDAGRequestProto> captor1 = ArgumentCaptor.forClass(SubmitDAGRequestProto.class);
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java b/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
index 818581c..6a74553 100644
--- a/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
+++ b/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGPlan.java
@@ -101,24 +101,24 @@ public class TestDAGPlan {
@Test(timeout = 5000)
public void testEdgeManagerSerde() {
DAG dag = new DAG("testDag");
- ProcessorDescriptor pd1 = new ProcessorDescriptor("processor1")
- .setUserPayload(new UserPayload(ByteBuffer.wrap("processor1Bytes".getBytes())));
- ProcessorDescriptor pd2 = new ProcessorDescriptor("processor2")
- .setUserPayload(new UserPayload(ByteBuffer.wrap("processor2Bytes".getBytes())));
- Vertex v1 = new Vertex("v1", pd1, 10, Resource.newInstance(1024, 1));
- Vertex v2 = new Vertex("v2", pd2, 1, Resource.newInstance(1024, 1));
+ ProcessorDescriptor pd1 = ProcessorDescriptor.create("processor1")
+ .setUserPayload(UserPayload.create(ByteBuffer.wrap("processor1Bytes".getBytes())));
+ ProcessorDescriptor pd2 = ProcessorDescriptor.create("processor2")
+ .setUserPayload(UserPayload.create(ByteBuffer.wrap("processor2Bytes".getBytes())));
+ Vertex v1 = Vertex.create("v1", pd1, 10, Resource.newInstance(1024, 1));
+ Vertex v2 = Vertex.create("v2", pd2, 1, Resource.newInstance(1024, 1));
v1.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>())
.setTaskLocalFiles(new HashMap<String, LocalResource>());
v2.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>())
.setTaskLocalFiles(new HashMap<String, LocalResource>());
- InputDescriptor inputDescriptor = new InputDescriptor("input")
- .setUserPayload(new UserPayload(ByteBuffer.wrap("inputBytes".getBytes())));
- OutputDescriptor outputDescriptor = new OutputDescriptor("output")
- .setUserPayload(new UserPayload(ByteBuffer.wrap("outputBytes".getBytes())));
- Edge edge = new Edge(v1, v2, new EdgeProperty(
- new EdgeManagerPluginDescriptor("emClass").setUserPayload(
- new UserPayload(ByteBuffer.wrap("emPayload".getBytes()))),
+ InputDescriptor inputDescriptor = InputDescriptor.create("input")
+ .setUserPayload(UserPayload.create(ByteBuffer.wrap("inputBytes".getBytes())));
+ OutputDescriptor outputDescriptor = OutputDescriptor.create("output")
+ .setUserPayload(UserPayload.create(ByteBuffer.wrap("outputBytes".getBytes())));
+ Edge edge = Edge.create(v1, v2, EdgeProperty.create(
+ EdgeManagerPluginDescriptor.create("emClass").setUserPayload(
+ UserPayload.create(ByteBuffer.wrap("emPayload".getBytes()))),
DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, outputDescriptor, inputDescriptor));
dag.addVertex(v1).addVertex(v2).addEdge(edge);
@@ -137,22 +137,22 @@ public class TestDAGPlan {
@Test(timeout = 5000)
public void testUserPayloadSerde() {
DAG dag = new DAG("testDag");
- ProcessorDescriptor pd1 = new ProcessorDescriptor("processor1").
- setUserPayload(new UserPayload(ByteBuffer.wrap("processor1Bytes".getBytes())));
- ProcessorDescriptor pd2 = new ProcessorDescriptor("processor2").
- setUserPayload(new UserPayload(ByteBuffer.wrap("processor2Bytes".getBytes())));
- Vertex v1 = new Vertex("v1", pd1, 10, Resource.newInstance(1024, 1));
- Vertex v2 = new Vertex("v2", pd2, 1, Resource.newInstance(1024, 1));
+ ProcessorDescriptor pd1 = ProcessorDescriptor.create("processor1").
+ setUserPayload(UserPayload.create(ByteBuffer.wrap("processor1Bytes".getBytes())));
+ ProcessorDescriptor pd2 = ProcessorDescriptor.create("processor2").
+ setUserPayload(UserPayload.create(ByteBuffer.wrap("processor2Bytes".getBytes())));
+ Vertex v1 = Vertex.create("v1", pd1, 10, Resource.newInstance(1024, 1));
+ Vertex v2 = Vertex.create("v2", pd2, 1, Resource.newInstance(1024, 1));
v1.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>())
.setTaskLocalFiles(new HashMap<String, LocalResource>());
v2.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>())
.setTaskLocalFiles(new HashMap<String, LocalResource>());
- InputDescriptor inputDescriptor = new InputDescriptor("input").
- setUserPayload(new UserPayload(ByteBuffer.wrap("inputBytes".getBytes())));
- OutputDescriptor outputDescriptor = new OutputDescriptor("output").
- setUserPayload(new UserPayload(ByteBuffer.wrap("outputBytes".getBytes())));
- Edge edge = new Edge(v1, v2, new EdgeProperty(
+ InputDescriptor inputDescriptor = InputDescriptor.create("input").
+ setUserPayload(UserPayload.create(ByteBuffer.wrap("inputBytes".getBytes())));
+ OutputDescriptor outputDescriptor = OutputDescriptor.create("output").
+ setUserPayload(UserPayload.create(ByteBuffer.wrap("outputBytes".getBytes())));
+ Edge edge = Edge.create(v1, v2, EdgeProperty.create(
DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
SchedulingType.SEQUENTIAL, outputDescriptor, inputDescriptor));
@@ -198,15 +198,15 @@ public class TestDAGPlan {
@Test(timeout = 5000)
public void userVertexOrderingIsMaintained() {
DAG dag = new DAG("testDag");
- ProcessorDescriptor pd1 = new ProcessorDescriptor("processor1").
- setUserPayload(new UserPayload(ByteBuffer.wrap("processor1Bytes".getBytes())));
- ProcessorDescriptor pd2 = new ProcessorDescriptor("processor2").
- setUserPayload(new UserPayload(ByteBuffer.wrap("processor2Bytes".getBytes())));
- ProcessorDescriptor pd3 = new ProcessorDescriptor("processor3").
- setUserPayload(new UserPayload(ByteBuffer.wrap("processor3Bytes".getBytes())));
- Vertex v1 = new Vertex("v1", pd1, 10, Resource.newInstance(1024, 1));
- Vertex v2 = new Vertex("v2", pd2, 1, Resource.newInstance(1024, 1));
- Vertex v3 = new Vertex("v3", pd3, 1, Resource.newInstance(1024, 1));
+ ProcessorDescriptor pd1 = ProcessorDescriptor.create("processor1").
+ setUserPayload(UserPayload.create(ByteBuffer.wrap("processor1Bytes".getBytes())));
+ ProcessorDescriptor pd2 = ProcessorDescriptor.create("processor2").
+ setUserPayload(UserPayload.create(ByteBuffer.wrap("processor2Bytes".getBytes())));
+ ProcessorDescriptor pd3 = ProcessorDescriptor.create("processor3").
+ setUserPayload(UserPayload.create(ByteBuffer.wrap("processor3Bytes".getBytes())));
+ Vertex v1 = Vertex.create("v1", pd1, 10, Resource.newInstance(1024, 1));
+ Vertex v2 = Vertex.create("v2", pd2, 1, Resource.newInstance(1024, 1));
+ Vertex v3 = Vertex.create("v3", pd3, 1, Resource.newInstance(1024, 1));
v1.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>())
.setTaskLocalFiles(new HashMap<String, LocalResource>());
v2.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>())
@@ -214,11 +214,11 @@ public class TestDAGPlan {
v3.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>())
.setTaskLocalFiles(new HashMap<String, LocalResource>());
- InputDescriptor inputDescriptor = new InputDescriptor("input").
- setUserPayload(new UserPayload(ByteBuffer.wrap("inputBytes".getBytes())));
- OutputDescriptor outputDescriptor = new OutputDescriptor("output").
- setUserPayload(new UserPayload(ByteBuffer.wrap("outputBytes".getBytes())));
- Edge edge = new Edge(v1, v2, new EdgeProperty(
+ InputDescriptor inputDescriptor = InputDescriptor.create("input").
+ setUserPayload(UserPayload.create(ByteBuffer.wrap("inputBytes".getBytes())));
+ OutputDescriptor outputDescriptor = OutputDescriptor.create("output").
+ setUserPayload(UserPayload.create(ByteBuffer.wrap("outputBytes".getBytes())));
+ Edge edge = Edge.create(v1, v2, EdgeProperty.create(
DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
SchedulingType.SEQUENTIAL, outputDescriptor, inputDescriptor));
@@ -271,22 +271,22 @@ public class TestDAGPlan {
@Test (timeout=5000)
public void testCredentialsSerde() {
DAG dag = new DAG("testDag");
- ProcessorDescriptor pd1 = new ProcessorDescriptor("processor1").
- setUserPayload(new UserPayload(ByteBuffer.wrap("processor1Bytes".getBytes())));
- ProcessorDescriptor pd2 = new ProcessorDescriptor("processor2").
- setUserPayload(new UserPayload(ByteBuffer.wrap("processor2Bytes".getBytes())));
- Vertex v1 = new Vertex("v1", pd1, 10, Resource.newInstance(1024, 1));
- Vertex v2 = new Vertex("v2", pd2, 1, Resource.newInstance(1024, 1));
+ ProcessorDescriptor pd1 = ProcessorDescriptor.create("processor1").
+ setUserPayload(UserPayload.create(ByteBuffer.wrap("processor1Bytes".getBytes())));
+ ProcessorDescriptor pd2 = ProcessorDescriptor.create("processor2").
+ setUserPayload(UserPayload.create(ByteBuffer.wrap("processor2Bytes".getBytes())));
+ Vertex v1 = Vertex.create("v1", pd1, 10, Resource.newInstance(1024, 1));
+ Vertex v2 = Vertex.create("v2", pd2, 1, Resource.newInstance(1024, 1));
v1.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>())
.setTaskLocalFiles(new HashMap<String, LocalResource>());
v2.setTaskLaunchCmdOpts("").setTaskEnvironment(new HashMap<String, String>())
.setTaskLocalFiles(new HashMap<String, LocalResource>());
- InputDescriptor inputDescriptor = new InputDescriptor("input").
- setUserPayload(new UserPayload(ByteBuffer.wrap("inputBytes".getBytes())));
- OutputDescriptor outputDescriptor = new OutputDescriptor("output").
- setUserPayload(new UserPayload(ByteBuffer.wrap("outputBytes".getBytes())));
- Edge edge = new Edge(v1, v2, new EdgeProperty(
+ InputDescriptor inputDescriptor = InputDescriptor.create("input").
+ setUserPayload(UserPayload.create(ByteBuffer.wrap("inputBytes".getBytes())));
+ OutputDescriptor outputDescriptor = OutputDescriptor.create("output").
+ setUserPayload(UserPayload.create(ByteBuffer.wrap("outputBytes".getBytes())));
+ Edge edge = Edge.create(v1, v2, EdgeProperty.create(
DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
SchedulingType.SEQUENTIAL, outputDescriptor, inputDescriptor));
[3/4] TEZ-1246. Replace constructors with create() methods for DAG,
Vertex, Edge etc in the API. (sseth)
Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGVerify.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGVerify.java b/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGVerify.java
index 711df30..9885272 100644
--- a/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGVerify.java
+++ b/tez-api/src/test/java/org/apache/tez/dag/api/TestDAGVerify.java
@@ -46,17 +46,17 @@ public class TestDAGVerify {
// v2
@Test(timeout = 5000)
public void testVerifyScatterGather() {
- Vertex v1 = new Vertex("v1",
- new ProcessorDescriptor(dummyProcessorClassName),
+ Vertex v1 = Vertex.create("v1",
+ ProcessorDescriptor.create(dummyProcessorClassName),
dummyTaskCount, dummyTaskResource);
- Vertex v2 = new Vertex("v2",
- new ProcessorDescriptor("MapProcessor"),
+ Vertex v2 = Vertex.create("v2",
+ ProcessorDescriptor.create("MapProcessor"),
dummyTaskCount, dummyTaskResource);
- Edge e1 = new Edge(v1, v2,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
- DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- new OutputDescriptor(dummyOutputClassName),
- new InputDescriptor(dummyInputClassName)));
+ Edge e1 = Edge.create(v1, v2,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
+ DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create(dummyOutputClassName),
+ InputDescriptor.create(dummyInputClassName)));
DAG dag = new DAG("testDag");
dag.addVertex(v1);
dag.addVertex(v2);
@@ -66,18 +66,18 @@ public class TestDAGVerify {
@Test(timeout = 5000)
public void testVerifyCustomEdge() {
- Vertex v1 = new Vertex("v1",
- new ProcessorDescriptor(dummyProcessorClassName),
+ Vertex v1 = Vertex.create("v1",
+ ProcessorDescriptor.create(dummyProcessorClassName),
dummyTaskCount, dummyTaskResource);
- Vertex v2 = new Vertex("v2",
- new ProcessorDescriptor("MapProcessor"),
+ Vertex v2 = Vertex.create("v2",
+ ProcessorDescriptor.create("MapProcessor"),
dummyTaskCount, dummyTaskResource);
- Edge e1 = new Edge(v1, v2,
- new EdgeProperty(new EdgeManagerPluginDescriptor("emClass"),
+ Edge e1 = Edge.create(v1, v2,
+ EdgeProperty.create(EdgeManagerPluginDescriptor.create("emClass"),
DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL,
- new OutputDescriptor(dummyOutputClassName),
- new InputDescriptor(dummyInputClassName)));
+ SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create(dummyOutputClassName),
+ InputDescriptor.create(dummyInputClassName)));
DAG dag = new DAG("testDag");
dag.addVertex(v1);
dag.addVertex(v2);
@@ -87,17 +87,17 @@ public class TestDAGVerify {
@Test(timeout = 5000)
public void testVerifyOneToOne() {
- Vertex v1 = new Vertex("v1",
- new ProcessorDescriptor(dummyProcessorClassName),
+ Vertex v1 = Vertex.create("v1",
+ ProcessorDescriptor.create(dummyProcessorClassName),
dummyTaskCount, dummyTaskResource);
- Vertex v2 = new Vertex("v2",
- new ProcessorDescriptor("MapProcessor"),
+ Vertex v2 = Vertex.create("v2",
+ ProcessorDescriptor.create("MapProcessor"),
dummyTaskCount, dummyTaskResource);
- Edge e1 = new Edge(v1, v2,
- new EdgeProperty(DataMovementType.ONE_TO_ONE,
- DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- new OutputDescriptor(dummyOutputClassName),
- new InputDescriptor(dummyInputClassName)));
+ Edge e1 = Edge.create(v1, v2,
+ EdgeProperty.create(DataMovementType.ONE_TO_ONE,
+ DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create(dummyOutputClassName),
+ InputDescriptor.create(dummyInputClassName)));
DAG dag = new DAG("testDag");
dag.addVertex(v1);
dag.addVertex(v2);
@@ -108,25 +108,25 @@ public class TestDAGVerify {
@Test(timeout = 5000)
// v1 (known) -> v2 (-1) -> v3 (-1)
public void testVerifyOneToOneInferParallelism() {
- Vertex v1 = new Vertex("v1",
- new ProcessorDescriptor(dummyProcessorClassName),
+ Vertex v1 = Vertex.create("v1",
+ ProcessorDescriptor.create(dummyProcessorClassName),
dummyTaskCount, dummyTaskResource);
- Vertex v2 = new Vertex("v2",
- new ProcessorDescriptor("MapProcessor"),
+ Vertex v2 = Vertex.create("v2",
+ ProcessorDescriptor.create("MapProcessor"),
-1, dummyTaskResource);
- Vertex v3 = new Vertex("v3",
- new ProcessorDescriptor("MapProcessor"),
+ Vertex v3 = Vertex.create("v3",
+ ProcessorDescriptor.create("MapProcessor"),
-1, dummyTaskResource);
- Edge e1 = new Edge(v1, v2,
- new EdgeProperty(DataMovementType.ONE_TO_ONE,
- DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- new OutputDescriptor(dummyOutputClassName),
- new InputDescriptor(dummyInputClassName)));
- Edge e2 = new Edge(v2, v3,
- new EdgeProperty(DataMovementType.ONE_TO_ONE,
- DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- new OutputDescriptor(dummyOutputClassName),
- new InputDescriptor(dummyInputClassName)));
+ Edge e1 = Edge.create(v1, v2,
+ EdgeProperty.create(DataMovementType.ONE_TO_ONE,
+ DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create(dummyOutputClassName),
+ InputDescriptor.create(dummyInputClassName)));
+ Edge e2 = Edge.create(v2, v3,
+ EdgeProperty.create(DataMovementType.ONE_TO_ONE,
+ DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create(dummyOutputClassName),
+ InputDescriptor.create(dummyInputClassName)));
DAG dag = new DAG("testDag");
dag.addVertex(v1);
dag.addVertex(v2);
@@ -142,25 +142,25 @@ public class TestDAGVerify {
// v1 (known) -> v2 (-1) -> v3 (-1)
// The test checks resiliency to ordering of the vertices/edges
public void testVerifyOneToOneInferParallelismReverseOrder() {
- Vertex v1 = new Vertex("v1",
- new ProcessorDescriptor(dummyProcessorClassName),
+ Vertex v1 = Vertex.create("v1",
+ ProcessorDescriptor.create(dummyProcessorClassName),
dummyTaskCount, dummyTaskResource);
- Vertex v2 = new Vertex("v2",
- new ProcessorDescriptor("MapProcessor"),
+ Vertex v2 = Vertex.create("v2",
+ ProcessorDescriptor.create("MapProcessor"),
-1, dummyTaskResource);
- Vertex v3 = new Vertex("v3",
- new ProcessorDescriptor("MapProcessor"),
+ Vertex v3 = Vertex.create("v3",
+ ProcessorDescriptor.create("MapProcessor"),
-1, dummyTaskResource);
- Edge e1 = new Edge(v1, v2,
- new EdgeProperty(DataMovementType.ONE_TO_ONE,
- DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- new OutputDescriptor(dummyOutputClassName),
- new InputDescriptor(dummyInputClassName)));
- Edge e2 = new Edge(v2, v3,
- new EdgeProperty(DataMovementType.ONE_TO_ONE,
- DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- new OutputDescriptor(dummyOutputClassName),
- new InputDescriptor(dummyInputClassName)));
+ Edge e1 = Edge.create(v1, v2,
+ EdgeProperty.create(DataMovementType.ONE_TO_ONE,
+ DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create(dummyOutputClassName),
+ InputDescriptor.create(dummyInputClassName)));
+ Edge e2 = Edge.create(v2, v3,
+ EdgeProperty.create(DataMovementType.ONE_TO_ONE,
+ DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create(dummyOutputClassName),
+ InputDescriptor.create(dummyInputClassName)));
DAG dag = new DAG("testDag");
dag.addVertex(v3);
dag.addVertex(v1);
@@ -174,17 +174,17 @@ public class TestDAGVerify {
@Test(timeout = 5000)
public void testVerifyOneToOneNoInferParallelism() {
- Vertex v1 = new Vertex("v1",
- new ProcessorDescriptor(dummyProcessorClassName),
+ Vertex v1 = Vertex.create("v1",
+ ProcessorDescriptor.create(dummyProcessorClassName),
-1, dummyTaskResource);
- Vertex v2 = new Vertex("v2",
- new ProcessorDescriptor("MapProcessor"),
+ Vertex v2 = Vertex.create("v2",
+ ProcessorDescriptor.create("MapProcessor"),
-1, dummyTaskResource);
- Edge e1 = new Edge(v1, v2,
- new EdgeProperty(DataMovementType.ONE_TO_ONE,
- DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- new OutputDescriptor(dummyOutputClassName),
- new InputDescriptor(dummyInputClassName)));
+ Edge e1 = Edge.create(v1, v2,
+ EdgeProperty.create(DataMovementType.ONE_TO_ONE,
+ DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create(dummyOutputClassName),
+ InputDescriptor.create(dummyInputClassName)));
DAG dag = new DAG("testDag");
dag.addVertex(v1);
dag.addVertex(v2);
@@ -196,25 +196,25 @@ public class TestDAGVerify {
@Test(timeout = 5000)
// v1 (-1) -> v2 (known) -> v3 (-1)
public void testVerifyOneToOneIncorrectParallelism1() {
- Vertex v1 = new Vertex("v1",
- new ProcessorDescriptor(dummyProcessorClassName),
+ Vertex v1 = Vertex.create("v1",
+ ProcessorDescriptor.create(dummyProcessorClassName),
-1, dummyTaskResource);
- Vertex v2 = new Vertex("v2",
- new ProcessorDescriptor(dummyProcessorClassName),
+ Vertex v2 = Vertex.create("v2",
+ ProcessorDescriptor.create(dummyProcessorClassName),
dummyTaskCount, dummyTaskResource);
- Vertex v3 = new Vertex("v3",
- new ProcessorDescriptor("MapProcessor"),
+ Vertex v3 = Vertex.create("v3",
+ ProcessorDescriptor.create("MapProcessor"),
-1, dummyTaskResource);
- Edge e1 = new Edge(v1, v3,
- new EdgeProperty(DataMovementType.ONE_TO_ONE,
- DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- new OutputDescriptor(dummyOutputClassName),
- new InputDescriptor(dummyInputClassName)));
- Edge e2 = new Edge(v2, v3,
- new EdgeProperty(DataMovementType.ONE_TO_ONE,
- DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- new OutputDescriptor(dummyOutputClassName),
- new InputDescriptor(dummyInputClassName)));
+ Edge e1 = Edge.create(v1, v3,
+ EdgeProperty.create(DataMovementType.ONE_TO_ONE,
+ DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create(dummyOutputClassName),
+ InputDescriptor.create(dummyInputClassName)));
+ Edge e2 = Edge.create(v2, v3,
+ EdgeProperty.create(DataMovementType.ONE_TO_ONE,
+ DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create(dummyOutputClassName),
+ InputDescriptor.create(dummyInputClassName)));
DAG dag = new DAG("testDag");
dag.addVertex(v1);
dag.addVertex(v2);
@@ -234,33 +234,33 @@ public class TestDAGVerify {
// v1 (-1) -> v3 (-1), v2 (known) -> v3 (-1)
// order of edges should not matter
public void testVerifyOneToOneIncorrectParallelism2() {
- Vertex v1 = new Vertex("v1",
- new ProcessorDescriptor(dummyProcessorClassName),
+ Vertex v1 = Vertex.create("v1",
+ ProcessorDescriptor.create(dummyProcessorClassName),
-1, dummyTaskResource);
- Vertex v2 = new Vertex("v2",
- new ProcessorDescriptor(dummyProcessorClassName),
+ Vertex v2 = Vertex.create("v2",
+ ProcessorDescriptor.create(dummyProcessorClassName),
dummyTaskCount, dummyTaskResource);
- Vertex v3 = new Vertex("v3",
- new ProcessorDescriptor(dummyProcessorClassName),
+ Vertex v3 = Vertex.create("v3",
+ ProcessorDescriptor.create(dummyProcessorClassName),
-1, dummyTaskResource);
- Vertex v4 = new Vertex("v4",
- new ProcessorDescriptor(dummyProcessorClassName),
+ Vertex v4 = Vertex.create("v4",
+ ProcessorDescriptor.create(dummyProcessorClassName),
-1, dummyTaskResource);
- Edge e1 = new Edge(v1, v4,
- new EdgeProperty(DataMovementType.ONE_TO_ONE,
- DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- new OutputDescriptor(dummyOutputClassName),
- new InputDescriptor(dummyInputClassName)));
- Edge e2 = new Edge(v2, v4,
- new EdgeProperty(DataMovementType.ONE_TO_ONE,
- DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- new OutputDescriptor(dummyOutputClassName),
- new InputDescriptor(dummyInputClassName)));
- Edge e3 = new Edge(v3, v4,
- new EdgeProperty(DataMovementType.ONE_TO_ONE,
- DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- new OutputDescriptor(dummyOutputClassName),
- new InputDescriptor(dummyInputClassName)));
+ Edge e1 = Edge.create(v1, v4,
+ EdgeProperty.create(DataMovementType.ONE_TO_ONE,
+ DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create(dummyOutputClassName),
+ InputDescriptor.create(dummyInputClassName)));
+ Edge e2 = Edge.create(v2, v4,
+ EdgeProperty.create(DataMovementType.ONE_TO_ONE,
+ DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create(dummyOutputClassName),
+ InputDescriptor.create(dummyInputClassName)));
+ Edge e3 = Edge.create(v3, v4,
+ EdgeProperty.create(DataMovementType.ONE_TO_ONE,
+ DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create(dummyOutputClassName),
+ InputDescriptor.create(dummyInputClassName)));
DAG dag = new DAG("testDag");
dag.addVertex(v1);
dag.addVertex(v2);
@@ -280,17 +280,17 @@ public class TestDAGVerify {
@Test(timeout = 5000)
public void testVerifyBroadcast() {
- Vertex v1 = new Vertex("v1",
- new ProcessorDescriptor(dummyProcessorClassName),
+ Vertex v1 = Vertex.create("v1",
+ ProcessorDescriptor.create(dummyProcessorClassName),
dummyTaskCount, dummyTaskResource);
- Vertex v2 = new Vertex("v2",
- new ProcessorDescriptor("MapProcessor"),
+ Vertex v2 = Vertex.create("v2",
+ ProcessorDescriptor.create("MapProcessor"),
dummyTaskCount, dummyTaskResource);
- Edge e1 = new Edge(v1, v2,
- new EdgeProperty(DataMovementType.BROADCAST,
- DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- new OutputDescriptor(dummyOutputClassName),
- new InputDescriptor(dummyInputClassName)));
+ Edge e1 = Edge.create(v1, v2,
+ EdgeProperty.create(DataMovementType.BROADCAST,
+ DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create(dummyOutputClassName),
+ InputDescriptor.create(dummyInputClassName)));
DAG dag = new DAG("testDag");
dag.addVertex(v1);
dag.addVertex(v2);
@@ -300,17 +300,17 @@ public class TestDAGVerify {
@Test(expected = IllegalStateException.class, timeout = 5000)
public void testVerify3() {
- Vertex v1 = new Vertex("v1",
- new ProcessorDescriptor(dummyProcessorClassName),
+ Vertex v1 = Vertex.create("v1",
+ ProcessorDescriptor.create(dummyProcessorClassName),
dummyTaskCount, dummyTaskResource);
- Vertex v2 = new Vertex("v2",
- new ProcessorDescriptor("MapProcessor"),
+ Vertex v2 = Vertex.create("v2",
+ ProcessorDescriptor.create("MapProcessor"),
dummyTaskCount, dummyTaskResource);
- Edge e1 = new Edge(v1, v2,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
- DataSourceType.EPHEMERAL, SchedulingType.SEQUENTIAL,
- new OutputDescriptor(dummyOutputClassName),
- new InputDescriptor(dummyInputClassName)));
+ Edge e1 = Edge.create(v1, v2,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
+ DataSourceType.EPHEMERAL, SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create(dummyOutputClassName),
+ InputDescriptor.create(dummyInputClassName)));
DAG dag = new DAG("testDag");
dag.addVertex(v1);
dag.addVertex(v2);
@@ -320,17 +320,17 @@ public class TestDAGVerify {
@Test(expected = IllegalStateException.class, timeout = 5000)
public void testVerify4() {
- Vertex v1 = new Vertex("v1",
- new ProcessorDescriptor(dummyProcessorClassName),
+ Vertex v1 = Vertex.create("v1",
+ ProcessorDescriptor.create(dummyProcessorClassName),
dummyTaskCount, dummyTaskResource);
- Vertex v2 = new Vertex("v2",
- new ProcessorDescriptor("MapProcessor"),
+ Vertex v2 = Vertex.create("v2",
+ ProcessorDescriptor.create("MapProcessor"),
dummyTaskCount, dummyTaskResource);
- Edge e1 = new Edge(v1, v2,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
- DataSourceType.EPHEMERAL, SchedulingType.CONCURRENT,
- new OutputDescriptor(dummyOutputClassName),
- new InputDescriptor(dummyInputClassName)));
+ Edge e1 = Edge.create(v1, v2,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
+ DataSourceType.EPHEMERAL, SchedulingType.CONCURRENT,
+ OutputDescriptor.create(dummyOutputClassName),
+ InputDescriptor.create(dummyInputClassName)));
DAG dag = new DAG("testDag");
dag.addVertex(v1);
dag.addVertex(v2);
@@ -346,38 +346,38 @@ public class TestDAGVerify {
@Test(timeout = 5000)
public void testCycle1() {
IllegalStateException ex=null;
- Vertex v1 = new Vertex("v1",
- new ProcessorDescriptor("MapProcessor"),
- dummyTaskCount, dummyTaskResource);
- Vertex v2 = new Vertex("v2",
- new ProcessorDescriptor("MapProcessor"),
- dummyTaskCount, dummyTaskResource);
- Vertex v3 = new Vertex("v3",
- new ProcessorDescriptor("MapProcessor"),
- dummyTaskCount, dummyTaskResource);
- Vertex v4 = new Vertex("v4",
- new ProcessorDescriptor("MapProcessor"),
- dummyTaskCount, dummyTaskResource);
- Edge e1 = new Edge(v1, v2,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
- DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- new OutputDescriptor("dummy output class"),
- new InputDescriptor("dummy input class")));
- Edge e2 = new Edge(v2, v3,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
- DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- new OutputDescriptor("dummy output class"),
- new InputDescriptor("dummy input class")));
- Edge e3 = new Edge(v2, v4,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
- DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- new OutputDescriptor("dummy output class"),
- new InputDescriptor("dummy input class")));
- Edge e4 = new Edge(v4, v1,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
- DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- new OutputDescriptor("dummy output class"),
- new InputDescriptor("dummy input class")));
+ Vertex v1 = Vertex.create("v1",
+ ProcessorDescriptor.create("MapProcessor"),
+ dummyTaskCount, dummyTaskResource);
+ Vertex v2 = Vertex.create("v2",
+ ProcessorDescriptor.create("MapProcessor"),
+ dummyTaskCount, dummyTaskResource);
+ Vertex v3 = Vertex.create("v3",
+ ProcessorDescriptor.create("MapProcessor"),
+ dummyTaskCount, dummyTaskResource);
+ Vertex v4 = Vertex.create("v4",
+ ProcessorDescriptor.create("MapProcessor"),
+ dummyTaskCount, dummyTaskResource);
+ Edge e1 = Edge.create(v1, v2,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
+ DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create("dummy output class"),
+ InputDescriptor.create("dummy input class")));
+ Edge e2 = Edge.create(v2, v3,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
+ DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create("dummy output class"),
+ InputDescriptor.create("dummy input class")));
+ Edge e3 = Edge.create(v2, v4,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
+ DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create("dummy output class"),
+ InputDescriptor.create("dummy input class")));
+ Edge e4 = Edge.create(v4, v1,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
+ DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create("dummy output class"),
+ InputDescriptor.create("dummy input class")));
DAG dag = new DAG("testDag");
dag.addVertex(v1);
dag.addVertex(v2);
@@ -406,38 +406,38 @@ public class TestDAGVerify {
@Test(timeout = 5000)
public void testCycle2() {
IllegalStateException ex=null;
- Vertex v1 = new Vertex("v1",
- new ProcessorDescriptor("MapProcessor"),
- dummyTaskCount, dummyTaskResource);
- Vertex v2 = new Vertex("v2",
- new ProcessorDescriptor("MapProcessor"),
- dummyTaskCount, dummyTaskResource);
- Vertex v3 = new Vertex("v3",
- new ProcessorDescriptor("MapProcessor"),
- dummyTaskCount, dummyTaskResource);
- Vertex v4 = new Vertex("v4",
- new ProcessorDescriptor("MapProcessor"),
- dummyTaskCount, dummyTaskResource);
- Edge e1 = new Edge(v1, v2,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
- DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- new OutputDescriptor("dummy output class"),
- new InputDescriptor("dummy input class")));
- Edge e2 = new Edge(v2, v3,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
- DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- new OutputDescriptor("dummy output class"),
- new InputDescriptor("dummy input class")));
- Edge e3 = new Edge(v2, v4,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
- DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- new OutputDescriptor("dummy output class"),
- new InputDescriptor("dummy input class")));
- Edge e4 = new Edge(v3, v2,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
- DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- new OutputDescriptor("dummy output class"),
- new InputDescriptor("dummy input class")));
+ Vertex v1 = Vertex.create("v1",
+ ProcessorDescriptor.create("MapProcessor"),
+ dummyTaskCount, dummyTaskResource);
+ Vertex v2 = Vertex.create("v2",
+ ProcessorDescriptor.create("MapProcessor"),
+ dummyTaskCount, dummyTaskResource);
+ Vertex v3 = Vertex.create("v3",
+ ProcessorDescriptor.create("MapProcessor"),
+ dummyTaskCount, dummyTaskResource);
+ Vertex v4 = Vertex.create("v4",
+ ProcessorDescriptor.create("MapProcessor"),
+ dummyTaskCount, dummyTaskResource);
+ Edge e1 = Edge.create(v1, v2,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
+ DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create("dummy output class"),
+ InputDescriptor.create("dummy input class")));
+ Edge e2 = Edge.create(v2, v3,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
+ DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create("dummy output class"),
+ InputDescriptor.create("dummy input class")));
+ Edge e3 = Edge.create(v2, v4,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
+ DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create("dummy output class"),
+ InputDescriptor.create("dummy input class")));
+ Edge e4 = Edge.create(v3, v2,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
+ DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create("dummy output class"),
+ InputDescriptor.create("dummy input class")));
DAG dag = new DAG("testDag");
dag.addVertex(v1);
dag.addVertex(v2);
@@ -461,11 +461,11 @@ public class TestDAGVerify {
@Test(timeout = 5000)
public void repeatedVertexName() {
IllegalStateException ex=null;
- Vertex v1 = new Vertex("v1",
- new ProcessorDescriptor("MapProcessor"),
+ Vertex v1 = Vertex.create("v1",
+ ProcessorDescriptor.create("MapProcessor"),
dummyTaskCount, dummyTaskResource);
- Vertex v1repeat = new Vertex("v1",
- new ProcessorDescriptor("MapProcessor"),
+ Vertex v1repeat = Vertex.create("v1",
+ ProcessorDescriptor.create("MapProcessor"),
dummyTaskCount, dummyTaskResource);
try {
DAG dag = new DAG("testDag");
@@ -483,20 +483,20 @@ public class TestDAGVerify {
@Test(expected = IllegalStateException.class, timeout = 5000)
public void testInputAndInputVertexNameCollision() {
- Vertex v1 = new Vertex("v1",
- new ProcessorDescriptor("MapProcessor"),
+ Vertex v1 = Vertex.create("v1",
+ ProcessorDescriptor.create("MapProcessor"),
dummyTaskCount, dummyTaskResource);
- Vertex v2 = new Vertex("v2",
- new ProcessorDescriptor("MapProcessor"),
+ Vertex v2 = Vertex.create("v2",
+ ProcessorDescriptor.create("MapProcessor"),
dummyTaskCount, dummyTaskResource);
- v2.addDataSource("v1", new DataSourceDescriptor(null, null, null));
+ v2.addDataSource("v1", DataSourceDescriptor.create(null, null, null));
- Edge e1 = new Edge(v1, v2,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
- DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- new OutputDescriptor("dummy output class"),
- new InputDescriptor("dummy input class")));
+ Edge e1 = Edge.create(v1, v2,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
+ DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create("dummy output class"),
+ InputDescriptor.create("dummy input class")));
DAG dag = new DAG("testDag");
dag.addVertex(v1);
@@ -507,20 +507,20 @@ public class TestDAGVerify {
@Test(expected = IllegalStateException.class, timeout = 5000)
public void testOutputAndOutputVertexNameCollision() {
- Vertex v1 = new Vertex("v1",
- new ProcessorDescriptor("MapProcessor"),
+ Vertex v1 = Vertex.create("v1",
+ ProcessorDescriptor.create("MapProcessor"),
dummyTaskCount, dummyTaskResource);
- Vertex v2 = new Vertex("v2",
- new ProcessorDescriptor("MapProcessor"),
+ Vertex v2 = Vertex.create("v2",
+ ProcessorDescriptor.create("MapProcessor"),
dummyTaskCount, dummyTaskResource);
v1.addDataSink("v2", new DataSinkDescriptor(null, null, null));
- Edge e1 = new Edge(v1, v2,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
- DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- new OutputDescriptor("dummy output class"),
- new InputDescriptor("dummy input class")));
+ Edge e1 = Edge.create(v1, v2,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
+ DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create("dummy output class"),
+ InputDescriptor.create("dummy input class")));
DAG dag = new DAG("testDag");
dag.addVertex(v1);
@@ -531,11 +531,11 @@ public class TestDAGVerify {
@Test(expected = IllegalStateException.class, timeout = 5000)
public void testOutputAndVertexNameCollision() {
- Vertex v1 = new Vertex("v1",
- new ProcessorDescriptor("MapProcessor"),
+ Vertex v1 = Vertex.create("v1",
+ ProcessorDescriptor.create("MapProcessor"),
dummyTaskCount, dummyTaskResource);
- Vertex v2 = new Vertex("v2",
- new ProcessorDescriptor("MapProcessor"),
+ Vertex v2 = Vertex.create("v2",
+ ProcessorDescriptor.create("MapProcessor"),
dummyTaskCount, dummyTaskResource);
v1.addDataSink("v2", new DataSinkDescriptor(null, null, null));
@@ -548,14 +548,14 @@ public class TestDAGVerify {
@Test(expected = IllegalStateException.class, timeout = 5000)
public void testInputAndVertexNameCollision() {
- Vertex v1 = new Vertex("v1",
- new ProcessorDescriptor("MapProcessor"),
+ Vertex v1 = Vertex.create("v1",
+ ProcessorDescriptor.create("MapProcessor"),
dummyTaskCount, dummyTaskResource);
- Vertex v2 = new Vertex("v2",
- new ProcessorDescriptor("MapProcessor"),
+ Vertex v2 = Vertex.create("v2",
+ ProcessorDescriptor.create("MapProcessor"),
dummyTaskCount, dummyTaskResource);
- v1.addDataSource("v2", new DataSourceDescriptor(null, null, null));
+ v1.addDataSource("v2", DataSourceDescriptor.create(null, null, null));
DAG dag = new DAG("testDag");
dag.addVertex(v1);
@@ -568,25 +568,25 @@ public class TestDAGVerify {
// v3
@Test(timeout = 5000)
public void BinaryInputAllowed() {
- Vertex v1 = new Vertex("v1",
- new ProcessorDescriptor("MapProcessor"),
+ Vertex v1 = Vertex.create("v1",
+ ProcessorDescriptor.create("MapProcessor"),
dummyTaskCount, dummyTaskResource);
- Vertex v2 = new Vertex("v2",
- new ProcessorDescriptor("MapProcessor"),
+ Vertex v2 = Vertex.create("v2",
+ ProcessorDescriptor.create("MapProcessor"),
dummyTaskCount, dummyTaskResource);
- Vertex v3 = new Vertex("v3",
- new ProcessorDescriptor("ReduceProcessor"),
+ Vertex v3 = Vertex.create("v3",
+ ProcessorDescriptor.create("ReduceProcessor"),
dummyTaskCount, dummyTaskResource);
- Edge e1 = new Edge(v1, v3,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
+ Edge e1 = Edge.create(v1, v3,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- new OutputDescriptor("dummy output class"),
- new InputDescriptor("dummy input class")));
- Edge e2 = new Edge(v2, v3,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
- DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- new OutputDescriptor("dummy output class"),
- new InputDescriptor("dummy input class")));
+ OutputDescriptor.create("dummy output class"),
+ InputDescriptor.create("dummy input class")));
+ Edge e2 = Edge.create(v2, v3,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
+ DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create("dummy output class"),
+ InputDescriptor.create("dummy input class")));
DAG dag = new DAG("testDag");
dag.addVertex(v1);
dag.addVertex(v2);
@@ -598,17 +598,17 @@ public class TestDAGVerify {
@Test(timeout = 5000)
public void testVertexGroupWithMultipleOutputEdges() {
- Vertex v1 = new Vertex("v1",
- new ProcessorDescriptor("Processor"),
+ Vertex v1 = Vertex.create("v1",
+ ProcessorDescriptor.create("Processor"),
dummyTaskCount, dummyTaskResource);
- Vertex v2 = new Vertex("v2",
- new ProcessorDescriptor("Processor"),
+ Vertex v2 = Vertex.create("v2",
+ ProcessorDescriptor.create("Processor"),
dummyTaskCount, dummyTaskResource);
- Vertex v3 = new Vertex("v3",
- new ProcessorDescriptor("Processor"),
+ Vertex v3 = Vertex.create("v3",
+ ProcessorDescriptor.create("Processor"),
dummyTaskCount, dummyTaskResource);
- Vertex v4 = new Vertex("v4",
- new ProcessorDescriptor("Processor"),
+ Vertex v4 = Vertex.create("v4",
+ ProcessorDescriptor.create("Processor"),
dummyTaskCount, dummyTaskResource);
DAG dag = new DAG("testDag");
@@ -616,19 +616,19 @@ public class TestDAGVerify {
OutputDescriptor outDesc = new OutputDescriptor();
uv12.addDataSink("uvOut", new DataSinkDescriptor(outDesc, null, null));
- GroupInputEdge e1 = new GroupInputEdge(uv12, v3,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
+ GroupInputEdge e1 = GroupInputEdge.create(uv12, v3,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- new OutputDescriptor("dummy output class"),
- new InputDescriptor("dummy input class")),
- new InputDescriptor("dummy input class"));
+ OutputDescriptor.create("dummy output class"),
+ InputDescriptor.create("dummy input class")),
+ InputDescriptor.create("dummy input class"));
- GroupInputEdge e2 = new GroupInputEdge(uv12, v4,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
+ GroupInputEdge e2 = GroupInputEdge.create(uv12, v4,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- new OutputDescriptor("dummy output class"),
- new InputDescriptor("dummy input class")),
- new InputDescriptor("dummy input class"));
+ OutputDescriptor.create("dummy output class"),
+ InputDescriptor.create("dummy input class")),
+ InputDescriptor.create("dummy input class"));
dag.addVertex(v1);
dag.addVertex(v2);
@@ -651,20 +651,20 @@ public class TestDAGVerify {
@Test(timeout = 5000)
public void testVertexGroup() {
- Vertex v1 = new Vertex("v1",
- new ProcessorDescriptor("Processor"),
+ Vertex v1 = Vertex.create("v1",
+ ProcessorDescriptor.create("Processor"),
dummyTaskCount, dummyTaskResource);
- Vertex v2 = new Vertex("v2",
- new ProcessorDescriptor("Processor"),
+ Vertex v2 = Vertex.create("v2",
+ ProcessorDescriptor.create("Processor"),
dummyTaskCount, dummyTaskResource);
- Vertex v3 = new Vertex("v3",
- new ProcessorDescriptor("Processor"),
+ Vertex v3 = Vertex.create("v3",
+ ProcessorDescriptor.create("Processor"),
dummyTaskCount, dummyTaskResource);
- Vertex v4 = new Vertex("v4",
- new ProcessorDescriptor("Processor"),
+ Vertex v4 = Vertex.create("v4",
+ ProcessorDescriptor.create("Processor"),
dummyTaskCount, dummyTaskResource);
- Vertex v5 = new Vertex("v5",
- new ProcessorDescriptor("Processor"),
+ Vertex v5 = Vertex.create("v5",
+ ProcessorDescriptor.create("Processor"),
dummyTaskCount, dummyTaskResource);
DAG dag = new DAG("testDag");
@@ -676,18 +676,18 @@ public class TestDAGVerify {
String groupName2 = "uv23";
VertexGroup uv23 = dag.createVertexGroup(groupName2, v2, v3);
- GroupInputEdge e1 = new GroupInputEdge(uv12, v4,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
+ GroupInputEdge e1 = GroupInputEdge.create(uv12, v4,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- new OutputDescriptor("dummy output class"),
- new InputDescriptor("dummy input class")),
- new InputDescriptor("dummy input class"));
- GroupInputEdge e2 = new GroupInputEdge(uv23, v5,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
+ OutputDescriptor.create("dummy output class"),
+ InputDescriptor.create("dummy input class")),
+ InputDescriptor.create("dummy input class"));
+ GroupInputEdge e2 = GroupInputEdge.create(uv23, v5,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- new OutputDescriptor("dummy output class"),
- new InputDescriptor("dummy input class")),
- new InputDescriptor("dummy input class"));
+ OutputDescriptor.create("dummy output class"),
+ InputDescriptor.create("dummy input class")),
+ InputDescriptor.create("dummy input class"));
dag.addVertex(v1);
dag.addVertex(v2);
@@ -733,20 +733,20 @@ public class TestDAGVerify {
@Test(timeout = 5000)
public void testVertexGroupOneToOne() {
- Vertex v1 = new Vertex("v1",
- new ProcessorDescriptor("Processor"),
+ Vertex v1 = Vertex.create("v1",
+ ProcessorDescriptor.create("Processor"),
dummyTaskCount, dummyTaskResource);
- Vertex v2 = new Vertex("v2",
- new ProcessorDescriptor("Processor"),
+ Vertex v2 = Vertex.create("v2",
+ ProcessorDescriptor.create("Processor"),
dummyTaskCount, dummyTaskResource);
- Vertex v3 = new Vertex("v3",
- new ProcessorDescriptor("Processor"),
+ Vertex v3 = Vertex.create("v3",
+ ProcessorDescriptor.create("Processor"),
dummyTaskCount, dummyTaskResource);
- Vertex v4 = new Vertex("v4",
- new ProcessorDescriptor("Processor"),
+ Vertex v4 = Vertex.create("v4",
+ ProcessorDescriptor.create("Processor"),
dummyTaskCount, dummyTaskResource);
- Vertex v5 = new Vertex("v5",
- new ProcessorDescriptor("Processor"),
+ Vertex v5 = Vertex.create("v5",
+ ProcessorDescriptor.create("Processor"),
-1, dummyTaskResource);
DAG dag = new DAG("testDag");
@@ -758,18 +758,18 @@ public class TestDAGVerify {
String groupName2 = "uv23";
VertexGroup uv23 = dag.createVertexGroup(groupName2, v2, v3);
- GroupInputEdge e1 = new GroupInputEdge(uv12, v4,
- new EdgeProperty(DataMovementType.ONE_TO_ONE,
+ GroupInputEdge e1 = GroupInputEdge.create(uv12, v4,
+ EdgeProperty.create(DataMovementType.ONE_TO_ONE,
DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- new OutputDescriptor("dummy output class"),
- new InputDescriptor("dummy input class")),
- new InputDescriptor("dummy input class"));
- GroupInputEdge e2 = new GroupInputEdge(uv23, v5,
- new EdgeProperty(DataMovementType.ONE_TO_ONE,
+ OutputDescriptor.create("dummy output class"),
+ InputDescriptor.create("dummy input class")),
+ InputDescriptor.create("dummy input class"));
+ GroupInputEdge e2 = GroupInputEdge.create(uv23, v5,
+ EdgeProperty.create(DataMovementType.ONE_TO_ONE,
DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- new OutputDescriptor("dummy output class"),
- new InputDescriptor("dummy input class")),
- new InputDescriptor("dummy input class"));
+ OutputDescriptor.create("dummy output class"),
+ InputDescriptor.create("dummy input class")),
+ InputDescriptor.create("dummy input class"));
dag.addVertex(v1);
dag.addVertex(v2);
@@ -792,25 +792,25 @@ public class TestDAGVerify {
public void BinaryOutput() {
IllegalStateException ex = null;
try {
- Vertex v1 = new Vertex("v1",
- new ProcessorDescriptor("MapProcessor"),
+ Vertex v1 = Vertex.create("v1",
+ ProcessorDescriptor.create("MapProcessor"),
dummyTaskCount, dummyTaskResource);
- Vertex v2 = new Vertex("v2",
- new ProcessorDescriptor("MapProcessor"),
+ Vertex v2 = Vertex.create("v2",
+ ProcessorDescriptor.create("MapProcessor"),
dummyTaskCount, dummyTaskResource);
- Vertex v3 = new Vertex("v3",
- new ProcessorDescriptor("MapProcessor"),
+ Vertex v3 = Vertex.create("v3",
+ ProcessorDescriptor.create("MapProcessor"),
dummyTaskCount, dummyTaskResource);
- Edge e1 = new Edge(v1, v2,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
- DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- new OutputDescriptor("dummy output class"),
- new InputDescriptor("dummy input class")));
- Edge e2 = new Edge(v1, v2,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
- DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- new OutputDescriptor("dummy output class"),
- new InputDescriptor("dummy input class")));
+ Edge e1 = Edge.create(v1, v2,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
+ DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create("dummy output class"),
+ InputDescriptor.create("dummy input class")));
+ Edge e2 = Edge.create(v1, v2,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
+ DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create("dummy output class"),
+ InputDescriptor.create("dummy input class")));
DAG dag = new DAG("testDag");
dag.addVertex(v1);
dag.addVertex(v2);
@@ -845,16 +845,16 @@ public class TestDAGVerify {
@Test(timeout = 5000)
public void testInvalidVertexConstruction() {
{
- Vertex v1 = new Vertex("v1",
- new ProcessorDescriptor("MapProcessor"),
- 0, dummyTaskResource);
- Vertex v2 = new Vertex("v1",
- new ProcessorDescriptor("MapProcessor"),
- -1, dummyTaskResource);
+ Vertex v1 = Vertex.create("v1",
+ ProcessorDescriptor.create("MapProcessor"),
+ 0, dummyTaskResource);
+ Vertex v2 = Vertex.create("v1",
+ ProcessorDescriptor.create("MapProcessor"),
+ -1, dummyTaskResource);
}
try {
- Vertex v1 = new Vertex("v1",
- new ProcessorDescriptor("MapProcessor"),
+ Vertex v1 = Vertex.create("v1",
+ ProcessorDescriptor.create("MapProcessor"),
-2, dummyTaskResource);
Assert.fail("Expected exception for 0 parallelism");
} catch (IllegalArgumentException e) {
@@ -865,8 +865,8 @@ public class TestDAGVerify {
"Parallelism should be -1 if determined by the AM, otherwise should be >= 0"));
}
try {
- Vertex v1 = new Vertex("v1",
- new ProcessorDescriptor("MapProcessor"),
+ Vertex v1 = Vertex.create("v1",
+ ProcessorDescriptor.create("MapProcessor"),
1, null);
Assert.fail("Expected exception for 0 parallelism");
} catch (IllegalArgumentException e) {
@@ -877,19 +877,20 @@ public class TestDAGVerify {
@Test(timeout = 5000)
public void testMultipleRootInputsAllowed() {
DAG dag = new DAG("testDag");
- ProcessorDescriptor pd1 = new ProcessorDescriptor("processor1")
- .setUserPayload(new UserPayload(ByteBuffer.wrap("processor1Bytes".getBytes())));
- Vertex v1 = new Vertex("v1", pd1, 10, Resource.newInstance(1024, 1));
- VertexManagerPluginDescriptor vertexManagerPluginDescriptor = new VertexManagerPluginDescriptor(
- "TestVertexManager");
+ ProcessorDescriptor pd1 = ProcessorDescriptor.create("processor1")
+ .setUserPayload(UserPayload.create(ByteBuffer.wrap("processor1Bytes".getBytes())));
+ Vertex v1 = Vertex.create("v1", pd1, 10, Resource.newInstance(1024, 1));
+ VertexManagerPluginDescriptor vertexManagerPluginDescriptor =
+ VertexManagerPluginDescriptor.create(
+ "TestVertexManager");
v1.setVertexManagerPlugin(vertexManagerPluginDescriptor);
- InputDescriptor inputDescriptor1 = new InputDescriptor("input1")
- .setUserPayload(new UserPayload(ByteBuffer.wrap("inputBytes".getBytes())));
- InputDescriptor inputDescriptor2 = new InputDescriptor("input2")
- .setUserPayload(new UserPayload(ByteBuffer.wrap("inputBytes".getBytes())));
- v1.addDataSource("input1", new DataSourceDescriptor(inputDescriptor1, null, null));
- v1.addDataSource("input2", new DataSourceDescriptor(inputDescriptor2, null, null));
+ InputDescriptor inputDescriptor1 = InputDescriptor.create("input1")
+ .setUserPayload(UserPayload.create(ByteBuffer.wrap("inputBytes".getBytes())));
+ InputDescriptor inputDescriptor2 = InputDescriptor.create("input2")
+ .setUserPayload(UserPayload.create(ByteBuffer.wrap("inputBytes".getBytes())));
+ v1.addDataSource("input1", DataSourceDescriptor.create(inputDescriptor1, null, null));
+ v1.addDataSource("input2", DataSourceDescriptor.create(inputDescriptor2, null, null));
dag.addVertex(v1);
@@ -900,9 +901,9 @@ public class TestDAGVerify {
@Test(timeout = 5000)
public void testDAGAccessControls() {
DAG dag = new DAG("testDag");
- ProcessorDescriptor pd1 = new ProcessorDescriptor("processor1")
- .setUserPayload(new UserPayload(ByteBuffer.wrap("processor1Bytes".getBytes())));
- Vertex v1 = new Vertex("v1", pd1, 10, Resource.newInstance(1024, 1));
+ ProcessorDescriptor pd1 = ProcessorDescriptor.create("processor1")
+ .setUserPayload(UserPayload.create(ByteBuffer.wrap("processor1Bytes".getBytes())));
+ Vertex v1 = Vertex.create("v1", pd1, 10, Resource.newInstance(1024, 1));
dag.addVertex(v1);
DAGAccessControls dagAccessControls = new DAGAccessControls();
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-api/src/test/java/org/apache/tez/dag/api/TestDagTypeConverters.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/dag/api/TestDagTypeConverters.java b/tez-api/src/test/java/org/apache/tez/dag/api/TestDagTypeConverters.java
index 8049147..9a1cb07 100644
--- a/tez-api/src/test/java/org/apache/tez/dag/api/TestDagTypeConverters.java
+++ b/tez-api/src/test/java/org/apache/tez/dag/api/TestDagTypeConverters.java
@@ -30,10 +30,10 @@ public class TestDagTypeConverters {
@Test
public void testTezEntityDescriptorSerialization() throws IOException {
- UserPayload payload = new UserPayload(ByteBuffer.wrap(new String("Foobar").getBytes()), 100);
+ UserPayload payload = UserPayload.create(ByteBuffer.wrap(new String("Foobar").getBytes()), 100);
String historytext = "Bar123";
EntityDescriptor entityDescriptor =
- new InputDescriptor("inputClazz").setUserPayload(payload)
+ InputDescriptor.create("inputClazz").setUserPayload(payload)
.setHistoryText(historytext);
TezEntityDescriptorProto proto =
DagTypeConverters.convertToDAGPlan(entityDescriptor);
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-api/src/test/java/org/apache/tez/runtime/api/event/TestCompositeDataMovementEvent.java
----------------------------------------------------------------------
diff --git a/tez-api/src/test/java/org/apache/tez/runtime/api/event/TestCompositeDataMovementEvent.java b/tez-api/src/test/java/org/apache/tez/runtime/api/event/TestCompositeDataMovementEvent.java
index 595fb63..20e85ea 100644
--- a/tez-api/src/test/java/org/apache/tez/runtime/api/event/TestCompositeDataMovementEvent.java
+++ b/tez-api/src/test/java/org/apache/tez/runtime/api/event/TestCompositeDataMovementEvent.java
@@ -30,7 +30,7 @@ public class TestCompositeDataMovementEvent {
int numPartitions = 2;
int startIndex = 2;
CompositeDataMovementEvent cdme1 =
- new CompositeDataMovementEvent(startIndex, numPartitions, userPayload);
+ CompositeDataMovementEvent.create(startIndex, numPartitions, userPayload);
Assert.assertEquals(numPartitions, cdme1.getCount());
Assert.assertEquals(startIndex, cdme1.getSourceIndexStart());
}
@@ -40,7 +40,7 @@ public class TestCompositeDataMovementEvent {
int numOutputs = 0;
int startIndex = 1;
CompositeDataMovementEvent cdme2 =
- new CompositeDataMovementEvent(startIndex, numOutputs, userPayload);
+ CompositeDataMovementEvent.create(startIndex, numOutputs, userPayload);
for(DataMovementEvent dme: cdme2.getEvents()){
numOutputs++;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/Edge.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/Edge.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/Edge.java
index 0d26958..b6a3bc1 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/Edge.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/Edge.java
@@ -111,15 +111,15 @@ public class Edge {
private void createEdgeManager() {
switch (edgeProperty.getDataMovementType()) {
case ONE_TO_ONE:
- edgeManagerContext = new EdgeManagerPluginContextImpl(new UserPayload(null));
+ edgeManagerContext = new EdgeManagerPluginContextImpl(UserPayload.create(null));
edgeManager = new OneToOneEdgeManager(edgeManagerContext);
break;
case BROADCAST:
- edgeManagerContext = new EdgeManagerPluginContextImpl(new UserPayload(null));
+ edgeManagerContext = new EdgeManagerPluginContextImpl(UserPayload.create(null));
edgeManager = new BroadcastEdgeManager(edgeManagerContext);
break;
case SCATTER_GATHER:
- edgeManagerContext = new EdgeManagerPluginContextImpl(new UserPayload(null));
+ edgeManagerContext = new EdgeManagerPluginContextImpl(UserPayload.create(null));
edgeManager = new ScatterGatherEdgeManager(edgeManagerContext);
break;
case CUSTOM:
@@ -129,7 +129,7 @@ public class Edge {
edgeProperty.getEdgeManagerDescriptor().getUserPayload().hasPayload()) {
payload = edgeProperty.getEdgeManagerDescriptor().getUserPayload();
} else {
- payload = new UserPayload(null);
+ payload = UserPayload.create(null);
}
edgeManagerContext = new EdgeManagerPluginContextImpl(payload);
String edgeManagerClassName = edgeProperty.getEdgeManagerDescriptor().getClassName();
@@ -157,7 +157,7 @@ public class Edge {
public synchronized void setCustomEdgeManager(EdgeManagerPluginDescriptor descriptor) {
EdgeProperty modifiedEdgeProperty =
- new EdgeProperty(descriptor,
+ EdgeProperty.create(descriptor,
edgeProperty.getDataSourceType(),
edgeProperty.getSchedulingType(),
edgeProperty.getEdgeSource(),
@@ -310,11 +310,11 @@ public class Edge {
Event e;
if (isDataMovementEvent) {
DataMovementEvent dmEvent = (DataMovementEvent) event;
- e = new DataMovementEvent(dmEvent.getSourceIndex(),
+ e = DataMovementEvent.create(dmEvent.getSourceIndex(),
inputIndex, dmEvent.getVersion(), dmEvent.getUserPayload());
} else {
InputFailedEvent ifEvent = ((InputFailedEvent) event);
- e = new InputFailedEvent(inputIndex, ifEvent.getVersion());
+ e = InputFailedEvent.create(inputIndex, ifEvent.getVersion());
}
tezEventToSend = new TezEvent(e, tezEvent.getSourceInfo());
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/RootInputVertexManager.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/RootInputVertexManager.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/RootInputVertexManager.java
index 81df597..34e3af3 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/RootInputVertexManager.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/RootInputVertexManager.java
@@ -97,7 +97,7 @@ public class RootInputVertexManager extends VertexManagerPlugin {
if (event instanceof InputUpdatePayloadEvent) {
// No tasks should have been started yet. Checked by initial state check.
Preconditions.checkState(dataInformationEventSeen == false);
- inputDescriptor.setUserPayload(new UserPayload(
+ inputDescriptor.setUserPayload(UserPayload.create(
ByteBuffer.wrap(((InputUpdatePayloadEvent) event).getUserPayload())));
} else if (event instanceof InputDataInformationEvent) {
dataInformationEventSeen = true;
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
index 6f77d98..6387b8f 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/VertexImpl.java
@@ -1900,15 +1900,15 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
LOG.info("Setting vertexManager to RootInputVertexManager for "
+ logIdentifier);
vertexManager = new VertexManager(
- new VertexManagerPluginDescriptor(RootInputVertexManager.class.getName())
- .setUserPayload(new UserPayload(null)),
+ VertexManagerPluginDescriptor.create(RootInputVertexManager.class.getName())
+ .setUserPayload(UserPayload.create(null)),
this, appContext);
} else if (hasOneToOne && !hasCustom) {
LOG.info("Setting vertexManager to InputReadyVertexManager for "
+ logIdentifier);
vertexManager = new VertexManager(
- new VertexManagerPluginDescriptor(InputReadyVertexManager.class.getName())
- .setUserPayload(new UserPayload(null)),
+ VertexManagerPluginDescriptor.create(InputReadyVertexManager.class.getName())
+ .setUserPayload(UserPayload.create(null)),
this, appContext);
} else if (hasBipartite && !hasCustom) {
LOG.info("Setting vertexManager to ShuffleVertexManager for "
@@ -1921,8 +1921,8 @@ public class VertexImpl implements org.apache.tez.dag.app.dag.Vertex,
LOG.info("Setting vertexManager to ImmediateStartVertexManager for "
+ logIdentifier);
vertexManager = new VertexManager(
- new VertexManagerPluginDescriptor(ImmediateStartVertexManager.class.getName())
- .setUserPayload(new UserPayload(null)),
+ VertexManagerPluginDescriptor.create(ImmediateStartVertexManager.class.getName())
+ .setUserPayload(UserPayload.create(null)),
this, appContext);
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-dag/src/test/java/org/apache/tez/client/TestLocalMode.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/client/TestLocalMode.java b/tez-dag/src/test/java/org/apache/tez/client/TestLocalMode.java
index 5e1b5ba..d25cf09 100644
--- a/tez-dag/src/test/java/org/apache/tez/client/TestLocalMode.java
+++ b/tez-dag/src/test/java/org/apache/tez/client/TestLocalMode.java
@@ -51,7 +51,7 @@ public class TestLocalMode {
tezConf1.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
tezConf1.set("fs.defaultFS", "file:///");
tezConf1.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true);
- TezClient tezClient1 = new TezClient("commonName", tezConf1, true);
+ TezClient tezClient1 = TezClient.create("commonName", tezConf1, true);
tezClient1.start();
DAG dag1 = createSimpleDAG("dag1", SleepProcessor.class.getName());
@@ -69,7 +69,7 @@ public class TestLocalMode {
tezConf2.set("fs.defaultFS", "file:///");
tezConf2.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true);
DAG dag2 = createSimpleDAG("dag2", SleepProcessor.class.getName());
- TezClient tezClient2 = new TezClient("commonName", tezConf2, true);
+ TezClient tezClient2 = TezClient.create("commonName", tezConf2, true);
tezClient2.start();
DAGClient dagClient2 = tezClient2.submitDAG(dag2);
dagClient2.waitForCompletion();
@@ -86,7 +86,7 @@ public class TestLocalMode {
tezConf1.setBoolean(TezConfiguration.TEZ_LOCAL_MODE, true);
tezConf1.set("fs.defaultFS", "file:///");
tezConf1.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true);
- TezClient tezClient1 = new TezClient("commonName", tezConf1, false);
+ TezClient tezClient1 = TezClient.create("commonName", tezConf1, false);
tezClient1.start();
DAG dag1 = createSimpleDAG("dag1", SleepProcessor.class.getName());
@@ -104,7 +104,7 @@ public class TestLocalMode {
tezConf2.set("fs.defaultFS", "file:///");
tezConf2.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true);
DAG dag2 = createSimpleDAG("dag2", SleepProcessor.class.getName());
- TezClient tezClient2 = new TezClient("commonName", tezConf2, false);
+ TezClient tezClient2 = TezClient.create("commonName", tezConf2, false);
tezClient2.start();
DAGClient dagClient2 = tezClient2.submitDAG(dag2);
dagClient2.waitForCompletion();
@@ -122,7 +122,7 @@ public class TestLocalMode {
tezConf1.set("fs.defaultFS", "file:///");
tezConf1.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true);
// Run in non-session mode so that the AM terminates
- TezClient tezClient1 = new TezClient("commonName", tezConf1, false);
+ TezClient tezClient1 = TezClient.create("commonName", tezConf1, false);
tezClient1.start();
DAG dag1 = createSimpleDAG("dag1", SleepProcessor.class.getName());
@@ -146,7 +146,7 @@ public class TestLocalMode {
tezConf1.set("fs.defaultFS", "file:///");
tezConf1.setBoolean(TezRuntimeConfiguration.TEZ_RUNTIME_OPTIMIZE_LOCAL_FETCH, true);
// Run in non-session mode so that the AM terminates
- TezClient tezClient1 = new TezClient("commonName", tezConf1, false);
+ TezClient tezClient1 = TezClient.create("commonName", tezConf1, false);
tezClient1.start();
DAG dag1 = createSimpleDAG("dag1", FailingProcessor.class.getName());
@@ -189,7 +189,7 @@ public class TestLocalMode {
}
private DAG createSimpleDAG(String dagName, String processorName) {
- DAG dag = new DAG(dagName).addVertex(new Vertex("Sleep", new ProcessorDescriptor(
+ DAG dag = new DAG(dagName).addVertex(Vertex.create("Sleep", ProcessorDescriptor.create(
processorName).setUserPayload(
new SleepProcessor.SleepProcessorConfig(1).toUserPayload()), 1));
return dag;
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
index 574b939..b56fbc3 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestDAGImpl.java
@@ -349,31 +349,31 @@ public class TestDAGImpl {
LOG.info("Setting up group dag plan");
int dummyTaskCount = 1;
Resource dummyTaskResource = Resource.newInstance(1, 1);
- org.apache.tez.dag.api.Vertex v1 = new org.apache.tez.dag.api.Vertex("vertex1",
- new ProcessorDescriptor("Processor"),
+ org.apache.tez.dag.api.Vertex v1 = org.apache.tez.dag.api.Vertex.create("vertex1",
+ ProcessorDescriptor.create("Processor"),
dummyTaskCount, dummyTaskResource);
- org.apache.tez.dag.api.Vertex v2 = new org.apache.tez.dag.api.Vertex("vertex2",
- new ProcessorDescriptor("Processor"),
+ org.apache.tez.dag.api.Vertex v2 = org.apache.tez.dag.api.Vertex.create("vertex2",
+ ProcessorDescriptor.create("Processor"),
dummyTaskCount, dummyTaskResource);
- org.apache.tez.dag.api.Vertex v3 = new org.apache.tez.dag.api.Vertex("vertex3",
- new ProcessorDescriptor("Processor"),
+ org.apache.tez.dag.api.Vertex v3 = org.apache.tez.dag.api.Vertex.create("vertex3",
+ ProcessorDescriptor.create("Processor"),
dummyTaskCount, dummyTaskResource);
DAG dag = new DAG("testDag");
String groupName1 = "uv12";
- OutputCommitterDescriptor ocd = new OutputCommitterDescriptor(
+ OutputCommitterDescriptor ocd = OutputCommitterDescriptor.create(
TotalCountingOutputCommitter.class.getName());
org.apache.tez.dag.api.VertexGroup uv12 = dag.createVertexGroup(groupName1, v1, v2);
- OutputDescriptor outDesc = new OutputDescriptor("output.class");
+ OutputDescriptor outDesc = OutputDescriptor.create("output.class");
uv12.addDataSink("uvOut", new DataSinkDescriptor(outDesc, ocd, null));
v3.addDataSink("uvOut", new DataSinkDescriptor(outDesc, ocd, null));
- GroupInputEdge e1 = new GroupInputEdge(uv12, v3,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
+ GroupInputEdge e1 = GroupInputEdge.create(uv12, v3,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- new OutputDescriptor("dummy output class"),
- new InputDescriptor("dummy input class")),
- new InputDescriptor("merge.class"));
+ OutputDescriptor.create("dummy output class"),
+ InputDescriptor.create("dummy input class")),
+ InputDescriptor.create("merge.class"));
dag.addVertex(v1);
dag.addVertex(v2);
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestEdge.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestEdge.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestEdge.java
index dcc135a..19e7067 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestEdge.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestEdge.java
@@ -63,7 +63,7 @@ public class TestEdge {
@Test (timeout = 5000)
public void testCompositeEventHandling() {
EventHandler eventHandler = mock(EventHandler.class);
- EdgeProperty edgeProp = new EdgeProperty(DataMovementType.SCATTER_GATHER,
+ EdgeProperty edgeProp = EdgeProperty.create(DataMovementType.SCATTER_GATHER,
DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL, mock(OutputDescriptor.class),
mock(InputDescriptor.class));
Edge edge = new Edge(edgeProp, eventHandler);
@@ -87,7 +87,7 @@ public class TestEdge {
EventMetaData srcMeta = new EventMetaData(EventProducerConsumerType.OUTPUT, "consumerVertex", "producerVertex", srcTAID);
// Verification via a CompositeEvent
- CompositeDataMovementEvent cdmEvent = new CompositeDataMovementEvent(0, destTasks.size(),
+ CompositeDataMovementEvent cdmEvent = CompositeDataMovementEvent.create(0, destTasks.size(),
"bytes".getBytes());
cdmEvent.setVersion(2); // AttemptNum
TezEvent tezEvent = new TezEvent(cdmEvent, srcMeta);
@@ -104,7 +104,7 @@ public class TestEdge {
// Same Verification via regular DataMovementEvents
reset(eventHandler);
for (int i = 0 ; i < destTasks.size() ; i++) {
- DataMovementEvent dmEvent = new DataMovementEvent(i, "bytes".getBytes());
+ DataMovementEvent dmEvent = DataMovementEvent.create(i, "bytes".getBytes());
dmEvent.setVersion(2);
tezEvent = new TezEvent(dmEvent, srcMeta);
edge.sendTezEventToDestinationTasks(tezEvent);
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestRootInputVertexManager.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestRootInputVertexManager.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestRootInputVertexManager.java
index b7907d6..b9aafa2 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestRootInputVertexManager.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestRootInputVertexManager.java
@@ -48,14 +48,14 @@ public class TestRootInputVertexManager {
InputDescriptor id1 = mock(InputDescriptor.class);
List<Event> events1 = new LinkedList<Event>();
- InputDataInformationEvent diEvent11 = new InputDataInformationEvent(0, null);
+ InputDataInformationEvent diEvent11 = InputDataInformationEvent.create(0, null);
events1.add(diEvent11);
rootInputVertexManager.onRootVertexInitialized("input1", id1, events1);
// All good so far, single input only.
InputDescriptor id2 = mock(InputDescriptor.class);
List<Event> events2 = new LinkedList<Event>();
- InputDataInformationEvent diEvent21 = new InputDataInformationEvent(0, null);
+ InputDataInformationEvent diEvent21 = InputDataInformationEvent.create(0, null);
events2.add(diEvent21);
try {
// Should fail due to second input
@@ -79,7 +79,7 @@ public class TestRootInputVertexManager {
InputDescriptor id1 = mock(InputDescriptor.class);
List<Event> events1 = new LinkedList<Event>();
- InputConfigureVertexTasksEvent diEvent11 = new InputConfigureVertexTasksEvent(1, null,
+ InputConfigureVertexTasksEvent diEvent11 = InputConfigureVertexTasksEvent.create(1, null,
null);
events1.add(diEvent11);
rootInputVertexManager.onRootVertexInitialized("input1", id1, events1);
@@ -87,7 +87,7 @@ public class TestRootInputVertexManager {
InputDescriptor id2 = mock(InputDescriptor.class);
List<Event> events2 = new LinkedList<Event>();
- InputConfigureVertexTasksEvent diEvent21 = new InputConfigureVertexTasksEvent(1, null,
+ InputConfigureVertexTasksEvent diEvent21 = InputConfigureVertexTasksEvent.create(1, null,
null);
events2.add(diEvent21);
try {
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
index 4f5af9a..d0c8f88 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskAttempt.java
@@ -913,7 +913,7 @@ public class TestTaskAttempt {
verify(eventHandler, times(expectedEventsTillSucceeded)).handle(arg.capture());
verifyEventType(arg.getAllValues(), TaskEventTAUpdate.class, 2);
- InputReadErrorEvent mockReEvent = new InputReadErrorEvent("", 0, 1);
+ InputReadErrorEvent mockReEvent = InputReadErrorEvent.create("", 0, 1);
EventMetaData mockMeta = mock(EventMetaData.class);
TezTaskAttemptID mockDestId1 = mock(TezTaskAttemptID.class);
when(mockMeta.getTaskAttemptID()).thenReturn(mockDestId1);
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
index 5bb68c2..0fb593f 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestTaskImpl.java
@@ -167,7 +167,7 @@ public class TestTaskImpl {
private void sendTezEventsToTask(TezTaskID taskId, int numTezEvents) {
TaskEventAddTezEvent event = null;
EventMetaData eventMetaData = new EventMetaData();
- DataMovementEvent dmEvent = new DataMovementEvent(null);
+ DataMovementEvent dmEvent = DataMovementEvent.create(null);
TezEvent tezEvent = new TezEvent(dmEvent, eventMetaData);
for (int i = 0; i < numTezEvents; i++) {
event = new TaskEventAddTezEvent(taskId, tezEvent);
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
index 9cd2cbc..c84c1a6 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/dag/impl/TestVertexImpl.java
@@ -384,43 +384,43 @@ public class TestVertexImpl {
private DAGPlan createDAGPlanWithMixedEdges() {
LOG.info("Setting up mixed edge dag plan");
org.apache.tez.dag.api.DAG dag = new org.apache.tez.dag.api.DAG("MixedEdges");
- org.apache.tez.dag.api.Vertex v1 = new org.apache.tez.dag.api.Vertex("vertex1",
- new ProcessorDescriptor("v1.class"), 1, Resource.newInstance(0, 0));
- org.apache.tez.dag.api.Vertex v2 = new org.apache.tez.dag.api.Vertex("vertex2",
- new ProcessorDescriptor("v2.class"), 1, Resource.newInstance(0, 0));
- org.apache.tez.dag.api.Vertex v3 = new org.apache.tez.dag.api.Vertex("vertex3",
- new ProcessorDescriptor("v3.class"), 1, Resource.newInstance(0, 0));
- org.apache.tez.dag.api.Vertex v4 = new org.apache.tez.dag.api.Vertex("vertex4",
- new ProcessorDescriptor("v4.class"), 1, Resource.newInstance(0, 0));
- org.apache.tez.dag.api.Vertex v5 = new org.apache.tez.dag.api.Vertex("vertex5",
- new ProcessorDescriptor("v5.class"), 1, Resource.newInstance(0, 0));
- org.apache.tez.dag.api.Vertex v6 = new org.apache.tez.dag.api.Vertex("vertex6",
- new ProcessorDescriptor("v6.class"), 1, Resource.newInstance(0, 0));
+ org.apache.tez.dag.api.Vertex v1 = org.apache.tez.dag.api.Vertex.create("vertex1",
+ ProcessorDescriptor.create("v1.class"), 1, Resource.newInstance(0, 0));
+ org.apache.tez.dag.api.Vertex v2 = org.apache.tez.dag.api.Vertex.create("vertex2",
+ ProcessorDescriptor.create("v2.class"), 1, Resource.newInstance(0, 0));
+ org.apache.tez.dag.api.Vertex v3 = org.apache.tez.dag.api.Vertex.create("vertex3",
+ ProcessorDescriptor.create("v3.class"), 1, Resource.newInstance(0, 0));
+ org.apache.tez.dag.api.Vertex v4 = org.apache.tez.dag.api.Vertex.create("vertex4",
+ ProcessorDescriptor.create("v4.class"), 1, Resource.newInstance(0, 0));
+ org.apache.tez.dag.api.Vertex v5 = org.apache.tez.dag.api.Vertex.create("vertex5",
+ ProcessorDescriptor.create("v5.class"), 1, Resource.newInstance(0, 0));
+ org.apache.tez.dag.api.Vertex v6 = org.apache.tez.dag.api.Vertex.create("vertex6",
+ ProcessorDescriptor.create("v6.class"), 1, Resource.newInstance(0, 0));
dag.addVertex(v1).addVertex(v2).addVertex(v3).addVertex(v4).addVertex(v5).addVertex(v6);
- dag.addEdge(new org.apache.tez.dag.api.Edge(v1, v2,
- new EdgeProperty(DataMovementType.BROADCAST, DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL, new OutputDescriptor("out.class"),
- new InputDescriptor("out.class"))));
- dag.addEdge(new org.apache.tez.dag.api.Edge(v1, v3,
- new EdgeProperty(DataMovementType.BROADCAST, DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL, new OutputDescriptor("out.class"),
- new InputDescriptor("out.class"))));
- dag.addEdge(new org.apache.tez.dag.api.Edge(v4, v2,
- new EdgeProperty(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL, new OutputDescriptor("out.class"),
- new InputDescriptor("out.class"))));
- dag.addEdge(new org.apache.tez.dag.api.Edge(v5, v3,
- new EdgeProperty(DataMovementType.ONE_TO_ONE, DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL, new OutputDescriptor("out.class"),
- new InputDescriptor("out.class"))));
- dag.addEdge(new org.apache.tez.dag.api.Edge(v4, v6,
- new EdgeProperty(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL, new OutputDescriptor("out.class"),
- new InputDescriptor("out.class"))));
- dag.addEdge(new org.apache.tez.dag.api.Edge(v5, v6,
- new EdgeProperty(DataMovementType.ONE_TO_ONE, DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL, new OutputDescriptor("out.class"),
- new InputDescriptor("out.class"))));
+ dag.addEdge(org.apache.tez.dag.api.Edge.create(v1, v2,
+ EdgeProperty.create(DataMovementType.BROADCAST, DataSourceType.PERSISTED,
+ SchedulingType.SEQUENTIAL, OutputDescriptor.create("out.class"),
+ InputDescriptor.create("out.class"))));
+ dag.addEdge(org.apache.tez.dag.api.Edge.create(v1, v3,
+ EdgeProperty.create(DataMovementType.BROADCAST, DataSourceType.PERSISTED,
+ SchedulingType.SEQUENTIAL, OutputDescriptor.create("out.class"),
+ InputDescriptor.create("out.class"))));
+ dag.addEdge(org.apache.tez.dag.api.Edge.create(v4, v2,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
+ SchedulingType.SEQUENTIAL, OutputDescriptor.create("out.class"),
+ InputDescriptor.create("out.class"))));
+ dag.addEdge(org.apache.tez.dag.api.Edge.create(v5, v3,
+ EdgeProperty.create(DataMovementType.ONE_TO_ONE, DataSourceType.PERSISTED,
+ SchedulingType.SEQUENTIAL, OutputDescriptor.create("out.class"),
+ InputDescriptor.create("out.class"))));
+ dag.addEdge(org.apache.tez.dag.api.Edge.create(v4, v6,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER, DataSourceType.PERSISTED,
+ SchedulingType.SEQUENTIAL, OutputDescriptor.create("out.class"),
+ InputDescriptor.create("out.class"))));
+ dag.addEdge(org.apache.tez.dag.api.Edge.create(v5, v6,
+ EdgeProperty.create(DataMovementType.ONE_TO_ONE, DataSourceType.PERSISTED,
+ SchedulingType.SEQUENTIAL, OutputDescriptor.create("out.class"),
+ InputDescriptor.create("out.class"))));
return dag.createDag(conf);
}
@@ -1864,7 +1864,7 @@ public class TestVertexImpl {
startVertex(vertices.get("vertex2"));
startVertex(v1);
EdgeManagerPluginDescriptor mockEdgeManagerDescriptor =
- new EdgeManagerPluginDescriptor(EdgeManagerForTest.class.getName());
+ EdgeManagerPluginDescriptor.create(EdgeManagerForTest.class.getName());
Map<String, EdgeManagerPluginDescriptor> edgeManagerDescriptors =
Collections.singletonMap(
@@ -1894,10 +1894,10 @@ public class TestVertexImpl {
List<TezEvent> taskEvents = Lists.newLinkedList();
TezEvent tezEvent1 = new TezEvent(
- new CompositeDataMovementEvent(0, 1, new byte[0]),
+ CompositeDataMovementEvent.create(0, 1, new byte[0]),
new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex2", "vertex3", ta0_t0_v2));
TezEvent tezEvent2 = new TezEvent(
- new DataMovementEvent(0, new byte[0]),
+ DataMovementEvent.create(0, new byte[0]),
new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex2", "vertex3", ta0_t0_v2));
taskEvents.add(tezEvent1);
taskEvents.add(tezEvent2);
@@ -1925,9 +1925,9 @@ public class TestVertexImpl {
Assert.assertTrue(Arrays.equals(edgePayload, originalEm.getEdgeManagerContext()
.getUserPayload().deepCopyAsArray()));
- UserPayload userPayload = new UserPayload(ByteBuffer.wrap(new String("foo").getBytes()));
+ UserPayload userPayload = UserPayload.create(ByteBuffer.wrap(new String("foo").getBytes()));
EdgeManagerPluginDescriptor edgeManagerDescriptor =
- new EdgeManagerPluginDescriptor(EdgeManagerForTest.class.getName());
+ EdgeManagerPluginDescriptor.create(EdgeManagerForTest.class.getName());
edgeManagerDescriptor.setUserPayload(userPayload);
Vertex v3 = vertices.get("vertex3");
@@ -2640,7 +2640,7 @@ public class TestVertexImpl {
// setting the edge manager should vertex1 should not INIT/START it since
// input initialization is not complete. v5 should be inited
EdgeManagerPluginDescriptor mockEdgeManagerDescriptor =
- new EdgeManagerPluginDescriptor(EdgeManagerForTest.class.getName());
+ EdgeManagerPluginDescriptor.create(EdgeManagerForTest.class.getName());
Edge e = v5.sourceVertices.get(v1);
Assert.assertNull(e.getEdgeManager());
e.setCustomEdgeManager(mockEdgeManagerDescriptor);
@@ -2681,7 +2681,7 @@ public class TestVertexImpl {
Assert.assertEquals(VertexState.INITIALIZING, vertices.get("vertex4").getState());
mockEdgeManagerDescriptor =
- new EdgeManagerPluginDescriptor(EdgeManagerForTest.class.getName());
+ EdgeManagerPluginDescriptor.create(EdgeManagerForTest.class.getName());
e = vertices.get("vertex6").sourceVertices.get(vertices.get("vertex4"));
Assert.assertNull(e.getEdgeManager());
e.setCustomEdgeManager(mockEdgeManagerDescriptor);
@@ -2702,7 +2702,7 @@ public class TestVertexImpl {
// fudge vertex manager so that tasks dont start running
v1.vertexManager = new VertexManager(
- new VertexManagerPluginDescriptor(VertexManagerPluginForTest.class.getName()),
+ VertexManagerPluginDescriptor.create(VertexManagerPluginForTest.class.getName()),
v1, appContext);
v1.vertexManager.initialize();
startVertex(v1);
@@ -2739,7 +2739,7 @@ public class TestVertexImpl {
// fudge vertex manager so that tasks dont start running
v1.vertexManager = new VertexManager(
- new VertexManagerPluginDescriptor(VertexManagerPluginForTest.class.getName()),
+ VertexManagerPluginDescriptor.create(VertexManagerPluginForTest.class.getName()),
v1, appContext);
v1.vertexManager.initialize();
@@ -2847,7 +2847,7 @@ public class TestVertexImpl {
Assert.assertFalse(initializer.initComplete.get());
// Signal the initializer by sending an event - via vertex1
- InputInitializerEvent event = new InputInitializerEvent("vertex2", "input1", null, 0);
+ InputInitializerEvent event = InputInitializerEvent.create("vertex2", "input1", null);
TezEvent tezEvent = new TezEvent(event,
new EventMetaData(EventProducerConsumerType.OUTPUT, "vertex1", null, null));
@@ -2911,10 +2911,10 @@ public class TestVertexImpl {
TezTaskID t0_v1 = TezTaskID.getInstance(v1.getVertexId(), 0);
TezTaskAttemptID ta0_t0_v1 = TezTaskAttemptID.getInstance(t0_v1, 0);
events.add(new TezEvent(
- new VertexManagerEvent("vertex2", new byte[0]), new EventMetaData(
+ VertexManagerEvent.create("vertex2", new byte[0]), new EventMetaData(
EventProducerConsumerType.PROCESSOR, "vertex1", "vertex2",
ta0_t0_v1)));
- events.add(new TezEvent(new InputDataInformationEvent(0, new byte[0]),
+ events.add(new TezEvent(InputDataInformationEvent.create(0, new byte[0]),
new EventMetaData(EventProducerConsumerType.INPUT, "vertex2",
"NULL_VERTEX", null)));
dispatcher.getEventHandler().handle(
@@ -2969,7 +2969,7 @@ public class TestVertexImpl {
Assert.assertArrayEquals(payload,
v1.getInputSpecList(0).get(0).getInputDescriptor().getUserPayload().deepCopyAsArray());
EdgeManagerPluginDescriptor mockEdgeManagerDescriptor =
- new EdgeManagerPluginDescriptor(EdgeManagerForTest.class.getName());
+ EdgeManagerPluginDescriptor.create(EdgeManagerForTest.class.getName());
Edge e = v2.sourceVertices.get(v1);
Assert.assertNull(e.getEdgeManager());
e.setCustomEdgeManager(mockEdgeManagerDescriptor);
@@ -3244,7 +3244,7 @@ public class TestVertexImpl {
public void completeInputDistribution(byte[] payload) {
List<Event> events = Lists.newArrayListWithCapacity(1);
- InputUpdatePayloadEvent event = new InputUpdatePayloadEvent(payload);
+ InputUpdatePayloadEvent event = InputUpdatePayloadEvent.create(payload);
events.add(event);
eventHandler.handle(new VertexEventRootInputInitialized(vertexID, inputs
.get(0).getName(), events));
@@ -3255,11 +3255,11 @@ public class TestVertexImpl {
List<TaskLocationHint> locationHints) {
List<Event> events = Lists.newArrayListWithCapacity(targetTasks + 1);
- InputConfigureVertexTasksEvent configEvent = new InputConfigureVertexTasksEvent(
- targetTasks, new VertexLocationHint(locationHints), null);
+ InputConfigureVertexTasksEvent configEvent = InputConfigureVertexTasksEvent.create(
+ targetTasks, VertexLocationHint.create(locationHints), null);
events.add(configEvent);
for (int i = 0; i < targetTasks; i++) {
- InputDataInformationEvent diEvent = new InputDataInformationEvent(i, null);
+ InputDataInformationEvent diEvent = InputDataInformationEvent.create(i, null);
events.add(diEvent);
}
eventHandler.handle(new VertexEventRootInputInitialized(vertexID, inputs
@@ -3321,7 +3321,7 @@ public class TestVertexImpl {
// setting the edge manager should vA to start
EdgeManagerPluginDescriptor mockEdgeManagerDescriptor =
- new EdgeManagerPluginDescriptor(EdgeManagerForTest.class.getName());
+ EdgeManagerPluginDescriptor.create(EdgeManagerForTest.class.getName());
Edge e = vC.sourceVertices.get(vA);
Assert.assertNull(e.getEdgeManager());
e.setCustomEdgeManager(mockEdgeManagerDescriptor);
@@ -3462,7 +3462,7 @@ public class TestVertexImpl {
lock.unlock();
}
initComplete.set(true);
- InputDataInformationEvent diEvent = new InputDataInformationEvent(0, new byte[]{0});
+ InputDataInformationEvent diEvent = InputDataInformationEvent.create(0, new byte[]{0});
List<Event> eventList = new LinkedList<Event>();
eventList.add(diEvent);
return eventList;
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
index 8fdad8e..6eda14f 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/app/rm/TestContainerReuse.java
@@ -1109,11 +1109,11 @@ public class TestContainerReuse {
}
AMSchedulerEventTALaunchRequest lr = new AMSchedulerEventTALaunchRequest(
taID, capability, new TaskSpec(taID, "dagName", "vertexName",
- new ProcessorDescriptor("processorClassName"),
+ ProcessorDescriptor.create("processorClassName"),
Collections.singletonList(new InputSpec("vertexName",
- new InputDescriptor("inputClassName"), 1)),
+ InputDescriptor.create("inputClassName"), 1)),
Collections.singletonList(new OutputSpec("vertexName",
- new OutputDescriptor("outputClassName"), 1)), null), ta, locationHint,
+ OutputDescriptor.create("outputClassName"), 1)), null), ta, locationHint,
priority, containerContext);
return lr;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java b/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java
index edf1c45..8948bc3 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/history/events/TestHistoryEventsProtoConversion.java
@@ -304,15 +304,16 @@ public class TestHistoryEventsProtoConversion {
{
Map<String,EdgeManagerPluginDescriptor> sourceEdgeManagers
= new LinkedHashMap<String, EdgeManagerPluginDescriptor>();
- sourceEdgeManagers.put("foo", new EdgeManagerPluginDescriptor("bar"));
- sourceEdgeManagers.put("foo1", new EdgeManagerPluginDescriptor("bar1")
- .setUserPayload(new UserPayload(ByteBuffer.wrap(new String("payload").getBytes()), 100)));
+ sourceEdgeManagers.put("foo", EdgeManagerPluginDescriptor.create("bar"));
+ sourceEdgeManagers.put("foo1", EdgeManagerPluginDescriptor.create("bar1")
+ .setUserPayload(
+ UserPayload.create(ByteBuffer.wrap(new String("payload").getBytes()), 100)));
VertexParallelismUpdatedEvent event =
new VertexParallelismUpdatedEvent(
TezVertexID.getInstance(
TezDAGID.getInstance(ApplicationId.newInstance(0, 1), 1), 111),
- 100, new VertexLocationHint(Arrays.asList(new TaskLocationHint(
- new HashSet<String>(Arrays.asList("h1")),
+ 100, VertexLocationHint.create(Arrays.asList(new TaskLocationHint(
+ new HashSet<String>(Arrays.asList("h1")),
new HashSet<String>(Arrays.asList("r1"))))),
sourceEdgeManagers, null);
@@ -542,7 +543,7 @@ public class TestHistoryEventsProtoConversion {
// Expected
}
List<TezEvent> events =
- Arrays.asList(new TezEvent(new DataMovementEvent(1, null),
+ Arrays.asList(new TezEvent(DataMovementEvent.create(1, null),
new EventMetaData(EventProducerConsumerType.SYSTEM, "foo", "bar", null)));
event = new VertexDataMovementEventsGeneratedEvent(
TezVertexID.getInstance(
[2/4] TEZ-1246. Replace constructors with create() methods for DAG,
Vertex, Edge etc in the API. (sseth)
Posted by ss...@apache.org.
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java b/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java
index 7ce0de3..9dd0a9a 100644
--- a/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java
+++ b/tez-dag/src/test/java/org/apache/tez/dag/history/utils/TestDAGUtils.java
@@ -38,6 +38,7 @@ import org.apache.tez.dag.api.InputDescriptor;
import org.apache.tez.dag.api.OutputCommitterDescriptor;
import org.apache.tez.dag.api.OutputDescriptor;
import org.apache.tez.dag.api.ProcessorDescriptor;
+import org.apache.tez.dag.api.Vertex;
import org.apache.tez.dag.api.records.DAGProtos.DAGPlan;
import org.apache.tez.runtime.api.OutputCommitter;
import org.codehaus.jettison.json.JSONException;
@@ -53,33 +54,34 @@ public class TestDAGUtils {
Configuration conf = new Configuration(false);
int dummyTaskCount = 1;
Resource dummyTaskResource = Resource.newInstance(1, 1);
- org.apache.tez.dag.api.Vertex v1 = new org.apache.tez.dag.api.Vertex("vertex1",
- new ProcessorDescriptor("Processor").setHistoryText("vertex1 Processor HistoryText"),
+ org.apache.tez.dag.api.Vertex v1 = Vertex.create("vertex1",
+ ProcessorDescriptor.create("Processor").setHistoryText("vertex1 Processor HistoryText"),
dummyTaskCount, dummyTaskResource);
- v1.addDataSource("input1", new DataSourceDescriptor(new InputDescriptor(
+ v1.addDataSource("input1", DataSourceDescriptor.create(InputDescriptor.create(
"input.class").setHistoryText("input HistoryText"), null, null));
- org.apache.tez.dag.api.Vertex v2 = new org.apache.tez.dag.api.Vertex("vertex2",
- new ProcessorDescriptor("Processor").setHistoryText("vertex2 Processor HistoryText"),
+ org.apache.tez.dag.api.Vertex v2 = Vertex.create("vertex2",
+ ProcessorDescriptor.create("Processor").setHistoryText("vertex2 Processor HistoryText"),
dummyTaskCount, dummyTaskResource);
- org.apache.tez.dag.api.Vertex v3 = new org.apache.tez.dag.api.Vertex("vertex3",
- new ProcessorDescriptor("Processor").setHistoryText("vertex3 Processor HistoryText"),
+ org.apache.tez.dag.api.Vertex v3 = Vertex.create("vertex3",
+ ProcessorDescriptor.create("Processor").setHistoryText("vertex3 Processor HistoryText"),
dummyTaskCount, dummyTaskResource);
DAG dag = new DAG("testDag");
String groupName1 = "uv12";
org.apache.tez.dag.api.VertexGroup uv12 = dag.createVertexGroup(groupName1, v1, v2);
- OutputDescriptor outDesc = new OutputDescriptor("output.class")
+ OutputDescriptor outDesc = OutputDescriptor.create("output.class")
.setHistoryText("uvOut HistoryText");
- OutputCommitterDescriptor ocd = new OutputCommitterDescriptor(OutputCommitter.class.getName());
+ OutputCommitterDescriptor ocd =
+ OutputCommitterDescriptor.create(OutputCommitter.class.getName());
uv12.addDataSink("uvOut", new DataSinkDescriptor(outDesc, ocd, null));
v3.addDataSink("uvOut", new DataSinkDescriptor(outDesc, ocd, null));
- GroupInputEdge e1 = new GroupInputEdge(uv12, v3,
- new EdgeProperty(DataMovementType.SCATTER_GATHER,
+ GroupInputEdge e1 = GroupInputEdge.create(uv12, v3,
+ EdgeProperty.create(DataMovementType.SCATTER_GATHER,
DataSourceType.PERSISTED, SchedulingType.SEQUENTIAL,
- new OutputDescriptor("dummy output class").setHistoryText("Dummy History Text"),
- new InputDescriptor("dummy input class").setHistoryText("Dummy History Text")),
- new InputDescriptor("merge.class").setHistoryText("Merge HistoryText"));
+ OutputDescriptor.create("dummy output class").setHistoryText("Dummy History Text"),
+ InputDescriptor.create("dummy input class").setHistoryText("Dummy History Text")),
+ InputDescriptor.create("merge.class").setHistoryText("Merge HistoryText"));
dag.addVertex(v1);
dag.addVertex(v2);
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-dag/src/test/java/org/apache/tez/runtime/task/TestTaskExecution.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/test/java/org/apache/tez/runtime/task/TestTaskExecution.java b/tez-dag/src/test/java/org/apache/tez/runtime/task/TestTaskExecution.java
index 4e012c5..d53ae9f 100644
--- a/tez-dag/src/test/java/org/apache/tez/runtime/task/TestTaskExecution.java
+++ b/tez-dag/src/test/java/org/apache/tez/runtime/task/TestTaskExecution.java
@@ -691,8 +691,8 @@ public class TestTaskExecution {
TezVertexID vertexId = TezVertexID.getInstance(dagId, 1);
TezTaskID taskId = TezTaskID.getInstance(vertexId, 1);
TezTaskAttemptID taskAttemptId = TezTaskAttemptID.getInstance(taskId, 1);
- ProcessorDescriptor processorDescriptor = new ProcessorDescriptor(processorClass)
- .setUserPayload(new UserPayload(ByteBuffer.wrap(processorConf)));
+ ProcessorDescriptor processorDescriptor = ProcessorDescriptor.create(processorClass)
+ .setUserPayload(UserPayload.create(ByteBuffer.wrap(processorConf)));
TaskSpec taskSpec = new TaskSpec(taskAttemptId, "dagName", "vertexName", processorDescriptor,
new ArrayList<InputSpec>(), new ArrayList<OutputSpec>(), null);
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-examples/src/main/java/org/apache/tez/examples/IntersectDataGen.java
----------------------------------------------------------------------
diff --git a/tez-examples/src/main/java/org/apache/tez/examples/IntersectDataGen.java b/tez-examples/src/main/java/org/apache/tez/examples/IntersectDataGen.java
index ec51cf1..435a914 100644
--- a/tez-examples/src/main/java/org/apache/tez/examples/IntersectDataGen.java
+++ b/tez-examples/src/main/java/org/apache/tez/examples/IntersectDataGen.java
@@ -126,7 +126,7 @@ public class IntersectDataGen extends Configured implements Tool {
}
private TezClient createTezSession(TezConfiguration tezConf) throws TezException, IOException {
- TezClient tezSession = new TezClient("IntersectDataGenSession", tezConf);
+ TezClient tezSession = TezClient.create("IntersectDataGenSession", tezConf);
tezSession.start();
return tezSession;
}
@@ -200,9 +200,9 @@ public class IntersectDataGen extends Configured implements Tool {
DAG dag = new DAG("IntersectDataGen");
- Vertex genDataVertex = new Vertex("datagen", new ProcessorDescriptor(
+ Vertex genDataVertex = Vertex.create("datagen", ProcessorDescriptor.create(
GenDataProcessor.class.getName()).setUserPayload(
- new UserPayload(ByteBuffer.wrap(GenDataProcessor.createConfiguration(largeOutSizePerTask,
+ UserPayload.create(ByteBuffer.wrap(GenDataProcessor.createConfiguration(largeOutSizePerTask,
smallOutSizePerTask)))), numTasks);
genDataVertex.addDataSink(STREAM_OUTPUT_NAME,
MROutput.createConfigBuilder(new Configuration(tezConf),
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-examples/src/main/java/org/apache/tez/examples/IntersectExample.java
----------------------------------------------------------------------
diff --git a/tez-examples/src/main/java/org/apache/tez/examples/IntersectExample.java b/tez-examples/src/main/java/org/apache/tez/examples/IntersectExample.java
index 90a232a..bf5897d 100644
--- a/tez-examples/src/main/java/org/apache/tez/examples/IntersectExample.java
+++ b/tez-examples/src/main/java/org/apache/tez/examples/IntersectExample.java
@@ -125,7 +125,7 @@ public class IntersectExample extends Configured implements Tool {
}
private TezClient createTezSession(TezConfiguration tezConf) throws TezException, IOException {
- TezClient tezSession = new TezClient("IntersectExampleSession", tezConf);
+ TezClient tezSession = TezClient.create("IntersectExampleSession", tezConf);
tezSession.start();
return tezSession;
}
@@ -183,28 +183,28 @@ public class IntersectExample extends Configured implements Tool {
HashPartitioner.class.getName()).build();
// Change the way resources are setup - no MRHelpers
- Vertex streamFileVertex = new Vertex("partitioner1", new ProcessorDescriptor(
+ Vertex streamFileVertex = Vertex.create("partitioner1", ProcessorDescriptor.create(
ForwardingProcessor.class.getName())).addDataSource(
"streamfile",
MRInput
.createConfigBuilder(new Configuration(tezConf), TextInputFormat.class,
streamPath.toUri().toString()).groupSplits(false).build());
- Vertex hashFileVertex = new Vertex("partitioner2", new ProcessorDescriptor(
+ Vertex hashFileVertex = Vertex.create("partitioner2", ProcessorDescriptor.create(
ForwardingProcessor.class.getName())).addDataSource(
"hashfile",
MRInput
.createConfigBuilder(new Configuration(tezConf), TextInputFormat.class,
hashPath.toUri().toString()).groupSplits(false).build());
- Vertex intersectVertex = new Vertex("intersect", new ProcessorDescriptor(
+ Vertex intersectVertex = Vertex.create("intersect", ProcessorDescriptor.create(
IntersectProcessor.class.getName()), numPartitions).addDataSink("finalOutput",
MROutput.createConfigBuilder(new Configuration(tezConf),
TextOutputFormat.class, outPath.toUri().toString()).build());
- Edge e1 = new Edge(streamFileVertex, intersectVertex, edgeConf.createDefaultEdgeProperty());
+ Edge e1 = Edge.create(streamFileVertex, intersectVertex, edgeConf.createDefaultEdgeProperty());
- Edge e2 = new Edge(hashFileVertex, intersectVertex, edgeConf.createDefaultEdgeProperty());
+ Edge e2 = Edge.create(hashFileVertex, intersectVertex, edgeConf.createDefaultEdgeProperty());
dag.addVertex(streamFileVertex).addVertex(hashFileVertex).addVertex(intersectVertex)
.addEdge(e1).addEdge(e2);
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-examples/src/main/java/org/apache/tez/examples/IntersectValidate.java
----------------------------------------------------------------------
diff --git a/tez-examples/src/main/java/org/apache/tez/examples/IntersectValidate.java b/tez-examples/src/main/java/org/apache/tez/examples/IntersectValidate.java
index d86fd77..24ba2df 100644
--- a/tez-examples/src/main/java/org/apache/tez/examples/IntersectValidate.java
+++ b/tez-examples/src/main/java/org/apache/tez/examples/IntersectValidate.java
@@ -125,7 +125,7 @@ public class IntersectValidate extends Configured implements Tool {
}
private TezClient createTezSession(TezConfiguration tezConf) throws TezException, IOException {
- TezClient tezSession = new TezClient("IntersectValidateSession", tezConf);
+ TezClient tezSession = TezClient.create("IntersectValidateSession", tezConf);
tezSession.start();
return tezSession;
}
@@ -188,23 +188,23 @@ public class IntersectValidate extends Configured implements Tool {
.newBuilder(Text.class.getName(), NullWritable.class.getName(),
HashPartitioner.class.getName()).build();
- Vertex lhsVertex = new Vertex(LHS_INPUT_NAME, new ProcessorDescriptor(
+ Vertex lhsVertex = Vertex.create(LHS_INPUT_NAME, ProcessorDescriptor.create(
ForwardingProcessor.class.getName())).addDataSource("lhs",
MRInput
.createConfigBuilder(new Configuration(tezConf), TextInputFormat.class,
lhs.toUri().toString()).groupSplits(false).build());
- Vertex rhsVertex = new Vertex(RHS_INPUT_NAME, new ProcessorDescriptor(
+ Vertex rhsVertex = Vertex.create(RHS_INPUT_NAME, ProcessorDescriptor.create(
ForwardingProcessor.class.getName())).addDataSource("rhs",
MRInput
.createConfigBuilder(new Configuration(tezConf), TextInputFormat.class,
rhs.toUri().toString()).groupSplits(false).build());
- Vertex intersectValidateVertex = new Vertex("intersectvalidate", new ProcessorDescriptor(
+ Vertex intersectValidateVertex = Vertex.create("intersectvalidate", ProcessorDescriptor.create(
IntersectValidateProcessor.class.getName()), numPartitions);
- Edge e1 = new Edge(lhsVertex, intersectValidateVertex, edgeConf.createDefaultEdgeProperty());
- Edge e2 = new Edge(rhsVertex, intersectValidateVertex, edgeConf.createDefaultEdgeProperty());
+ Edge e1 = Edge.create(lhsVertex, intersectValidateVertex, edgeConf.createDefaultEdgeProperty());
+ Edge e2 = Edge.create(rhsVertex, intersectValidateVertex, edgeConf.createDefaultEdgeProperty());
dag.addVertex(lhsVertex).addVertex(rhsVertex).addVertex(intersectValidateVertex).addEdge(e1)
.addEdge(e2);
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java
----------------------------------------------------------------------
diff --git a/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java b/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java
index 0bba651..9b3d864 100644
--- a/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java
+++ b/tez-examples/src/main/java/org/apache/tez/examples/OrderedWordCount.java
@@ -121,7 +121,7 @@ public class OrderedWordCount extends Configured implements Tool {
DataSinkDescriptor dataSink = MROutput.createConfigBuilder(new Configuration(tezConf),
TextOutputFormat.class, outputPath).build();
- Vertex tokenizerVertex = new Vertex("Tokenizer", new ProcessorDescriptor(
+ Vertex tokenizerVertex = Vertex.create("Tokenizer", ProcessorDescriptor.create(
TokenProcessor.class.getName()));
tokenizerVertex.addDataSource("MRInput", dataSource);
@@ -132,7 +132,7 @@ public class OrderedWordCount extends Configured implements Tool {
// This vertex will be reading intermediate data via an input edge and writing intermediate data
// via an output edge.
- Vertex summationVertex = new Vertex("Summation", new ProcessorDescriptor(
+ Vertex summationVertex = Vertex.create("Summation", ProcessorDescriptor.create(
SumProcessor.class.getName()), numPartitions);
// Use IntWritable key and Text value to bring all words with the same count in the same
@@ -143,7 +143,7 @@ public class OrderedWordCount extends Configured implements Tool {
// Use 1 task to bring all the data in one place for global sorted order. Essentially the number
// of partitions is 1. So the NoOpSorter can be used to produce the globally ordered output
- Vertex sorterVertex = new Vertex("Sorter", new ProcessorDescriptor(
+ Vertex sorterVertex = Vertex.create("Sorter", ProcessorDescriptor.create(
NoOpSorter.class.getName()), 1);
sorterVertex.addDataSink("MROutput", dataSink);
@@ -154,9 +154,10 @@ public class OrderedWordCount extends Configured implements Tool {
.addVertex(summationVertex)
.addVertex(sorterVertex)
.addEdge(
- new Edge(tokenizerVertex, summationVertex, summationEdgeConf.createDefaultEdgeProperty()))
+ Edge.create(tokenizerVertex, summationVertex,
+ summationEdgeConf.createDefaultEdgeProperty()))
.addEdge(
- new Edge(summationVertex, sorterVertex, sorterEdgeConf.createDefaultEdgeProperty()));
+ Edge.create(summationVertex, sorterVertex, sorterEdgeConf.createDefaultEdgeProperty()));
return dag;
}
@@ -175,7 +176,7 @@ public class OrderedWordCount extends Configured implements Tool {
tezConf = new TezConfiguration();
}
- TezClient tezClient = new TezClient("OrderedWordCount", tezConf);
+ TezClient tezClient = TezClient.create("OrderedWordCount", tezConf);
tezClient.start();
try {
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-examples/src/main/java/org/apache/tez/examples/SimpleSessionExample.java
----------------------------------------------------------------------
diff --git a/tez-examples/src/main/java/org/apache/tez/examples/SimpleSessionExample.java b/tez-examples/src/main/java/org/apache/tez/examples/SimpleSessionExample.java
index f4ed5b3..fd70dad 100644
--- a/tez-examples/src/main/java/org/apache/tez/examples/SimpleSessionExample.java
+++ b/tez-examples/src/main/java/org/apache/tez/examples/SimpleSessionExample.java
@@ -57,7 +57,7 @@ public class SimpleSessionExample extends Configured implements Tool {
// session mode then it can do so in code directly using the appropriate constructor
// tezConf.setBoolean(TezConfiguration.TEZ_AM_SESSION_MODE, true); // via config OR via code
- TezClient tezClient = new TezClient("SimpleSessionExample", tezConf, true);
+ TezClient tezClient = TezClient.create("SimpleSessionExample", tezConf, true);
tezClient.start();
// Session pre-warming allows the user to hide initial startup, resource acquisition latency etc.
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java
----------------------------------------------------------------------
diff --git a/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java b/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java
index ddf0953..52cff91 100644
--- a/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java
+++ b/tez-examples/src/main/java/org/apache/tez/examples/WordCount.java
@@ -150,7 +150,7 @@ public class WordCount extends Configured implements Tool {
// Create a vertex that reads the data from the data source and tokenizes it using the
// TokenProcessor. The number of tasks that will do the work for this vertex will be decided
// using the information provided by the data source descriptor.
- Vertex tokenizerVertex = new Vertex("Tokenizer", new ProcessorDescriptor(
+ Vertex tokenizerVertex = Vertex.create("Tokenizer", ProcessorDescriptor.create(
TokenProcessor.class.getName())).addDataSource("Input", dataSource);
// Create the edge that represents the movement and semantics of data between the producer
@@ -172,8 +172,8 @@ public class WordCount extends Configured implements Tool {
// The number of tasks that do the work of this vertex depends on the number of partitions used
// to distribute the sum processing. In this case, its been made configurable via the
// numPartitions parameter.
- Vertex summationVertex = new Vertex("Summation",
- new ProcessorDescriptor(SumProcessor.class.getName()), numPartitions)
+ Vertex summationVertex = Vertex.create("Summation",
+ ProcessorDescriptor.create(SumProcessor.class.getName()), numPartitions)
.addDataSink("Output", dataSink);
// No need to add jar containing this class as assumed to be part of the Tez jars. Otherwise
@@ -184,7 +184,7 @@ public class WordCount extends Configured implements Tool {
dag.addVertex(tokenizerVertex)
.addVertex(summationVertex)
.addEdge(
- new Edge(tokenizerVertex, summationVertex, edgeConf.createDefaultEdgeProperty()));
+ Edge.create(tokenizerVertex, summationVertex, edgeConf.createDefaultEdgeProperty()));
return dag;
}
@@ -205,7 +205,7 @@ public class WordCount extends Configured implements Tool {
// Create the TezClient to submit the DAG. Pass the tezConf that has all necessary global and
// dag specific configurations
- TezClient tezClient = new TezClient("WordCount", tezConf);
+ TezClient tezClient = TezClient.create("WordCount", tezConf);
// TezClient must be started before it can be used
tezClient.start();
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
index a83b500..94d5818 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/client/YARNRunner.java
@@ -416,7 +416,8 @@ public class YARNRunner implements ClientProtocol {
stageConf.set(MRJobConfig.MROUTPUT_FILE_NAME_PREFIX, "part");
UserPayload vertexUserPayload = TezUtils.createUserPayloadFromConf(stageConf);
- Vertex vertex = new Vertex(vertexName, new ProcessorDescriptor(processorName).setUserPayload(vertexUserPayload),
+ Vertex vertex = Vertex.create(vertexName,
+ ProcessorDescriptor.create(processorName).setUserPayload(vertexUserPayload),
numTasks, taskResource);
if (isMap) {
vertex.addDataSource("MRInput",
@@ -424,10 +425,10 @@ public class YARNRunner implements ClientProtocol {
}
// Map only jobs.
if (stageNum == totalStages -1) {
- OutputDescriptor od = new OutputDescriptor(MROutputLegacy.class.getName())
+ OutputDescriptor od = OutputDescriptor.create(MROutputLegacy.class.getName())
.setUserPayload(vertexUserPayload);
vertex.addDataSink("MROutput", new DataSinkDescriptor(od,
- new OutputCommitterDescriptor(MROutputCommitter.class.getName()), null));
+ OutputCommitterDescriptor.create(MROutputCommitter.class.getName()), null));
}
Map<String, String> taskEnv = new HashMap<String, String>();
@@ -444,7 +445,7 @@ public class YARNRunner implements ClientProtocol {
vertex.setTaskEnvironment(taskEnv)
.setTaskLocalFiles(taskLocalResources)
- .setLocationHint(new VertexLocationHint(locations))
+ .setLocationHint(VertexLocationHint.create(locations))
.setTaskLaunchCmdOpts(taskJavaOpts);
if (!isMap) {
@@ -505,7 +506,7 @@ public class YARNRunner implements ClientProtocol {
MRPartitioner.class.getName(), partitionerConf)
.configureInput().useLegacyInput().done()
.setFromConfiguration(stageConfs[i - 1]).build();
- Edge edge = new Edge(vertices[i-1], vertices[i], edgeConf.createDefaultEdgeProperty());
+ Edge edge = Edge.create(vertices[i - 1], vertices[i], edgeConf.createDefaultEdgeProperty());
dag.addEdge(edge);
}
@@ -796,14 +797,14 @@ public class YARNRunner implements ClientProtocol {
InputDescriptor inputDescriptor;
try {
- inputDescriptor = new InputDescriptor(useLegacyInput ? MRInputLegacy.class
+ inputDescriptor = InputDescriptor.create(useLegacyInput ? MRInputLegacy.class
.getName() : MRInput.class.getName())
.setUserPayload(MRInputHelpersInternal.createMRInputPayload(conf, null));
} catch (IOException e) {
throw new TezUncheckedException(e);
}
- DataSourceDescriptor dsd = new DataSourceDescriptor(inputDescriptor, null, null);
+ DataSourceDescriptor dsd = DataSourceDescriptor.create(inputDescriptor, null, null);
return dsd;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputAMSplitGenerator.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputAMSplitGenerator.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputAMSplitGenerator.java
index 88aed3c..a2777a8 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputAMSplitGenerator.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputAMSplitGenerator.java
@@ -32,7 +32,6 @@ import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapreduce.split.TezMapReduceSplitsGrouper;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.tez.common.TezUtils;
-import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.VertexLocationHint;
import org.apache.tez.mapreduce.hadoop.InputSplitInfoMem;
import org.apache.tez.mapreduce.hadoop.MRInputHelpers;
@@ -132,8 +131,9 @@ public class MRInputAMSplitGenerator extends InputInitializer {
List<Event> events = Lists.newArrayListWithCapacity(inputSplitInfo
.getNumTasks() + 1);
- InputConfigureVertexTasksEvent configureVertexEvent = new InputConfigureVertexTasksEvent(
- inputSplitInfo.getNumTasks(), new VertexLocationHint(inputSplitInfo.getTaskLocationHints()),
+ InputConfigureVertexTasksEvent configureVertexEvent = InputConfigureVertexTasksEvent.create(
+ inputSplitInfo.getNumTasks(),
+ VertexLocationHint.create(inputSplitInfo.getTaskLocationHints()),
InputSpecUpdate.getDefaultSinglePhysicalInputSpecUpdate());
events.add(configureVertexEvent);
@@ -142,7 +142,7 @@ public class MRInputAMSplitGenerator extends InputInitializer {
int count = 0;
for (MRSplitProto mrSplit : splitsProto.getSplitsList()) {
// Unnecessary array copy, can be avoided by using ByteBuffer instead of a raw array.
- InputDataInformationEvent diEvent = new InputDataInformationEvent(count++,
+ InputDataInformationEvent diEvent = InputDataInformationEvent.create(count++,
mrSplit.toByteArray());
events.add(diEvent);
}
@@ -150,12 +150,12 @@ public class MRInputAMSplitGenerator extends InputInitializer {
int count = 0;
if (inputSplitInfo.holdsNewFormatSplits()) {
for (org.apache.hadoop.mapreduce.InputSplit split : inputSplitInfo.getNewFormatSplits()) {
- InputDataInformationEvent diEvent = new InputDataInformationEvent(count++, split);
+ InputDataInformationEvent diEvent = InputDataInformationEvent.create(count++, split);
events.add(diEvent);
}
} else {
for (org.apache.hadoop.mapred.InputSplit split : inputSplitInfo.getOldFormatSplits()) {
- InputDataInformationEvent diEvent = new InputDataInformationEvent(count++, split);
+ InputDataInformationEvent diEvent = InputDataInformationEvent.create(count++, split);
events.add(diEvent);
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputSplitDistributor.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputSplitDistributor.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputSplitDistributor.java
index f23042f..1c28c2b 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputSplitDistributor.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/common/MRInputSplitDistributor.java
@@ -95,7 +95,7 @@ public class MRInputSplitDistributor extends InputInitializer {
updatedPayloadBuilder.clearSplits();
List<Event> events = Lists.newArrayListWithCapacity(this.splitsProto.getSplitsCount() + 1);
- InputUpdatePayloadEvent updatePayloadEvent = new InputUpdatePayloadEvent(
+ InputUpdatePayloadEvent updatePayloadEvent = InputUpdatePayloadEvent.create(
updatedPayloadBuilder.build().toByteArray());
events.add(updatePayloadEvent);
@@ -108,16 +108,16 @@ public class MRInputSplitDistributor extends InputInitializer {
if (sendSerializedEvents) {
// Unnecessary array copy, can be avoided by using ByteBuffer instead of
// a raw array.
- diEvent = new InputDataInformationEvent(count++, mrSplit.toByteArray());
+ diEvent = InputDataInformationEvent.create(count++, mrSplit.toByteArray());
} else {
if (useNewApi) {
org.apache.hadoop.mapreduce.InputSplit newInputSplit = MRInputUtils
.getNewSplitDetailsFromEvent(mrSplit, conf);
- diEvent = new InputDataInformationEvent(count++, newInputSplit);
+ diEvent = InputDataInformationEvent.create(count++, newInputSplit);
} else {
org.apache.hadoop.mapred.InputSplit oldInputSplit = MRInputUtils
.getOldSplitDetailsFromEvent(mrSplit, conf);
- diEvent = new InputDataInformationEvent(count++, oldInputSplit);
+ diEvent = InputDataInformationEvent.create(count++, oldInputSplit);
}
}
events.add(diEvent);
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRInputHelpers.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRInputHelpers.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRInputHelpers.java
index bff94e5..71d294f 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRInputHelpers.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/hadoop/MRInputHelpers.java
@@ -107,16 +107,16 @@ public class MRInputHelpers {
try {
inputSplitInfo = generateInputSplits(conf, splitsDir);
- InputDescriptor inputDescriptor = new InputDescriptor(useLegacyInput ? MRInputLegacy.class
+ InputDescriptor inputDescriptor = InputDescriptor.create(useLegacyInput ? MRInputLegacy.class
.getName() : MRInput.class.getName())
.setUserPayload(createMRInputPayload(conf, null));
Map<String, LocalResource> additionalLocalResources = new HashMap<String, LocalResource>();
updateLocalResourcesForInputSplits(FileSystem.get(conf), inputSplitInfo,
additionalLocalResources);
DataSourceDescriptor dsd =
- new DataSourceDescriptor(inputDescriptor, null, inputSplitInfo.getNumTasks(),
+ DataSourceDescriptor.create(inputDescriptor, null, inputSplitInfo.getNumTasks(),
inputSplitInfo.getCredentials(),
- new VertexLocationHint(inputSplitInfo.getTaskLocationHints()),
+ VertexLocationHint.create(inputSplitInfo.getTaskLocationHints()),
additionalLocalResources);
return dsd;
} catch (IOException e) {
@@ -693,7 +693,7 @@ public class MRInputHelpers {
userPayloadBuilder.setSplits(mrSplitsProto);
}
userPayloadBuilder.setGroupingEnabled(isGrouped);
- return new UserPayload(userPayloadBuilder.build().toByteString().asReadOnlyByteBuffer());
+ return UserPayload.create(userPayloadBuilder.build().toByteString().asReadOnlyByteBuffer());
}
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java
index cdcaba0..ebc2d45 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/input/MRInput.java
@@ -246,11 +246,11 @@ public class MRInput extends MRInputBase {
if (getCredentialsForSourceFilesystem && inputSplitInfo.getCredentials() != null) {
credentials = inputSplitInfo.getCredentials();
}
- return new DataSourceDescriptor(
- new InputDescriptor(inputClassName).setUserPayload(payload),
- new InputInitializerDescriptor(MRInputSplitDistributor.class.getName()),
- inputSplitInfo.getNumTasks(), credentials,
- new VertexLocationHint(inputSplitInfo.getTaskLocationHints()), null);
+ return DataSourceDescriptor.create(
+ InputDescriptor.create(inputClassName).setUserPayload(payload),
+ InputInitializerDescriptor.create(MRInputSplitDistributor.class.getName()),
+ inputSplitInfo.getNumTasks(), credentials,
+ VertexLocationHint.create(inputSplitInfo.getTaskLocationHints()), null);
}
private DataSourceDescriptor createCustomDataSource() throws IOException {
@@ -267,8 +267,9 @@ public class MRInput extends MRInputBase {
payload = MRInputHelpersInternal.createMRInputPayload(conf, null);
}
- return new DataSourceDescriptor(new InputDescriptor(inputClassName).setUserPayload(payload),
- customInitializerDescriptor, credentials);
+ return DataSourceDescriptor
+ .create(InputDescriptor.create(inputClassName).setUserPayload(payload),
+ customInitializerDescriptor, credentials);
}
private DataSourceDescriptor createGeneratorDataSource() throws IOException {
@@ -283,9 +284,9 @@ public class MRInput extends MRInputBase {
} else {
payload = MRInputHelpersInternal.createMRInputPayload(conf, null);
}
- return new DataSourceDescriptor(
- new InputDescriptor(inputClassName).setUserPayload(payload),
- new InputInitializerDescriptor(MRInputAMSplitGenerator.class.getName()), credentials);
+ return DataSourceDescriptor.create(
+ InputDescriptor.create(inputClassName).setUserPayload(payload),
+ InputInitializerDescriptor.create(MRInputAMSplitGenerator.class.getName()), credentials);
}
private void setupBasicConf(Configuration inputConf) {
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java
index a745a7f..fd0ea46 100644
--- a/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java
+++ b/tez-mapreduce/src/main/java/org/apache/tez/mapreduce/output/MROutput.java
@@ -173,8 +173,8 @@ public class MROutput extends AbstractLogicalOutput {
}
return new DataSinkDescriptor(
- new OutputDescriptor(outputClassName).setUserPayload(createUserPayload()),
- (doCommit ? new OutputCommitterDescriptor(
+ OutputDescriptor.create(outputClassName).setUserPayload(createUserPayload()),
+ (doCommit ? OutputCommitterDescriptor.create(
MROutputCommitter.class.getName()) : null), credentials);
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/common/TestMRInputSplitDistributor.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/common/TestMRInputSplitDistributor.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/common/TestMRInputSplitDistributor.java
index c91537c..ebc12d4 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/common/TestMRInputSplitDistributor.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/common/TestMRInputSplitDistributor.java
@@ -68,7 +68,8 @@ public class TestMRInputSplitDistributor {
MRInputUserPayloadProto.Builder payloadProto = MRInputUserPayloadProto.newBuilder();
payloadProto.setSplits(splitsProtoBuilder.build());
payloadProto.setConfigurationBytes(confByteString);
- UserPayload userPayload = new UserPayload(payloadProto.build().toByteString().asReadOnlyByteBuffer());
+ UserPayload userPayload =
+ UserPayload.create(payloadProto.build().toByteString().asReadOnlyByteBuffer());
InputInitializerContext context = new TezRootInputInitializerContextForTest(userPayload);
MRInputSplitDistributor splitDist = new MRInputSplitDistributor(context);
@@ -116,7 +117,8 @@ public class TestMRInputSplitDistributor {
MRInputUserPayloadProto.Builder payloadProto = MRInputUserPayloadProto.newBuilder();
payloadProto.setSplits(splitsProtoBuilder.build());
payloadProto.setConfigurationBytes(confByteString);
- UserPayload userPayload = new UserPayload(payloadProto.build().toByteString().asReadOnlyByteBuffer());
+ UserPayload userPayload =
+ UserPayload.create(payloadProto.build().toByteString().asReadOnlyByteBuffer());
InputInitializerContext context = new TezRootInputInitializerContextForTest(userPayload);
MRInputSplitDistributor splitDist = new MRInputSplitDistributor(context);
@@ -152,7 +154,7 @@ public class TestMRInputSplitDistributor {
TezRootInputInitializerContextForTest(UserPayload payload) throws IOException {
appId = ApplicationId.newInstance(1000, 200);
- this.payload = payload == null ? new UserPayload(null) : payload;
+ this.payload = payload == null ? UserPayload.create(null) : payload;
}
@Override
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/input/TestMultiMRInput.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/input/TestMultiMRInput.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/input/TestMultiMRInput.java
index 6e87011..f418bbb 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/input/TestMultiMRInput.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/input/TestMultiMRInput.java
@@ -111,7 +111,7 @@ public class TestMultiMRInput {
assertEquals(1, splits.length);
MRSplitProto splitProto = MRInputHelpers.createSplitProto(splits[0]);
- InputDataInformationEvent event = new InputDataInformationEvent(0, splitProto.toByteArray());
+ InputDataInformationEvent event = InputDataInformationEvent.create(0, splitProto.toByteArray());
eventList.clear();
eventList.add(event);
@@ -170,10 +170,12 @@ public class TestMultiMRInput {
assertEquals(2, splits.length);
MRSplitProto splitProto1 = MRInputHelpers.createSplitProto(splits[0]);
- InputDataInformationEvent event1 = new InputDataInformationEvent(0, splitProto1.toByteArray());
+ InputDataInformationEvent event1 =
+ InputDataInformationEvent.create(0, splitProto1.toByteArray());
MRSplitProto splitProto2 = MRInputHelpers.createSplitProto(splits[1]);
- InputDataInformationEvent event2 = new InputDataInformationEvent(0, splitProto2.toByteArray());
+ InputDataInformationEvent event2 =
+ InputDataInformationEvent.create(0, splitProto2.toByteArray());
eventList.clear();
eventList.add(event1);
@@ -221,8 +223,10 @@ public class TestMultiMRInput {
assertEquals(1, splits.length);
MRSplitProto splitProto = MRInputHelpers.createSplitProto(splits[0]);
- InputDataInformationEvent event1 = new InputDataInformationEvent(0, splitProto.toByteArray());
- InputDataInformationEvent event2 = new InputDataInformationEvent(1, splitProto.toByteArray());
+ InputDataInformationEvent event1 =
+ InputDataInformationEvent.create(0, splitProto.toByteArray());
+ InputDataInformationEvent event2 =
+ InputDataInformationEvent.create(1, splitProto.toByteArray());
eventList.clear();
eventList.add(event1);
@@ -251,7 +255,7 @@ public class TestMultiMRInput {
doReturn(1).when(inputContext).getTaskIndex();
doReturn(1).when(inputContext).getTaskVertexIndex();
doReturn("taskVertexName").when(inputContext).getTaskVertexName();
- doReturn(new UserPayload(ByteBuffer.wrap(payload))).when(inputContext).getUserPayload();
+ doReturn(UserPayload.create(ByteBuffer.wrap(payload))).when(inputContext).getUserPayload();
return inputContext;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java
index 028820d..eccc343 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/MapUtils.java
@@ -199,7 +199,7 @@ public class MapUtils {
List<OutputSpec> outputSpecs) throws Exception {
jobConf.setInputFormat(SequenceFileInputFormat.class);
- ProcessorDescriptor mapProcessorDesc = new ProcessorDescriptor(
+ ProcessorDescriptor mapProcessorDesc = ProcessorDescriptor.create(
MapProcessor.class.getName()).setUserPayload(
TezUtils.createUserPayloadFromConf(jobConf));
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java
index 66d81c7..caa2663 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/map/TestMapProcessor.java
@@ -124,13 +124,14 @@ public class TestMapProcessor {
MapUtils.generateInputSplit(localFs, workDir, jobConf, mapInput);
InputSpec mapInputSpec = new InputSpec("NullSrcVertex",
- new InputDescriptor(MRInputLegacy.class.getName())
- .setUserPayload(new UserPayload(ByteBuffer.wrap(
+ InputDescriptor.create(MRInputLegacy.class.getName())
+ .setUserPayload(UserPayload.create(ByteBuffer.wrap(
MRRuntimeProtos.MRInputUserPayloadProto.newBuilder()
- .setConfigurationBytes(TezUtils.createByteStringFromConf(jobConf)).build().toByteArray()))),
+ .setConfigurationBytes(TezUtils.createByteStringFromConf(jobConf)).build()
+ .toByteArray()))),
1);
OutputSpec mapOutputSpec = new OutputSpec("NullDestVertex",
- new OutputDescriptor(LocalOnFileSorterOutput.class.getName())
+ OutputDescriptor.create(LocalOnFileSorterOutput.class.getName())
.setUserPayload(TezUtils.createUserPayloadFromConf(jobConf)), 1);
LogicalIOProcessorRuntimeTask task = MapUtils.createLogicalTask(localFs, workDir, jobConf, 0,
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java
----------------------------------------------------------------------
diff --git a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java
index 7cbea59..3021149 100644
--- a/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java
+++ b/tez-mapreduce/src/test/java/org/apache/tez/mapreduce/processor/reduce/TestReduceProcessor.java
@@ -132,14 +132,14 @@ public class TestReduceProcessor {
MapUtils.generateInputSplit(localFs, workDir, jobConf, mapInput);
InputSpec mapInputSpec = new InputSpec("NullSrcVertex",
- new InputDescriptor(MRInputLegacy.class.getName())
- .setUserPayload(new UserPayload(ByteBuffer.wrap(
+ InputDescriptor.create(MRInputLegacy.class.getName())
+ .setUserPayload(UserPayload.create(ByteBuffer.wrap(
MRRuntimeProtos.MRInputUserPayloadProto.newBuilder()
.setConfigurationBytes(TezUtils.createByteStringFromConf(jobConf)).build()
.toByteArray()))),
1);
OutputSpec mapOutputSpec = new OutputSpec("NullDestVertex",
- new OutputDescriptor(LocalOnFileSorterOutput.class.getName()).
+ OutputDescriptor.create(LocalOnFileSorterOutput.class.getName()).
setUserPayload(TezUtils.createUserPayloadFromConf(jobConf)), 1);
// Run a map
LogicalIOProcessorRuntimeTask mapTask = MapUtils.createLogicalTask(localFs, workDir, jobConf, 0,
@@ -159,15 +159,15 @@ public class TestReduceProcessor {
jobConf.set(MRFrameworkConfigs.TASK_LOCAL_RESOURCE_DIR, new Path(workDir,
"localized-resources").toUri().toString());
FileOutputFormat.setOutputPath(jobConf, new Path(workDir, "output"));
- ProcessorDescriptor reduceProcessorDesc = new ProcessorDescriptor(
+ ProcessorDescriptor reduceProcessorDesc = ProcessorDescriptor.create(
ReduceProcessor.class.getName()).setUserPayload(
TezUtils.createUserPayloadFromConf(jobConf));
InputSpec reduceInputSpec = new InputSpec(mapVertexName,
- new InputDescriptor(LocalMergedInput.class.getName())
+ InputDescriptor.create(LocalMergedInput.class.getName())
.setUserPayload(TezUtils.createUserPayloadFromConf(jobConf)), 1);
OutputSpec reduceOutputSpec = new OutputSpec("NullDestinationVertex",
- new OutputDescriptor(MROutputLegacy.class.getName())
+ OutputDescriptor.create(MROutputLegacy.class.getName())
.setUserPayload(TezUtils.createUserPayloadFromConf(jobConf)), 1);
// Now run a reduce
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-runtime-internals/src/main/java/org/apache/tez/common/ProtoConverters.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/common/ProtoConverters.java b/tez-runtime-internals/src/main/java/org/apache/tez/common/ProtoConverters.java
index b682941..4419b53 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/common/ProtoConverters.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/common/ProtoConverters.java
@@ -44,7 +44,7 @@ public class ProtoConverters {
public static DataMovementEvent convertDataMovementEventFromProto(
EventProtos.DataMovementEventProto proto) {
- return new DataMovementEvent(proto.getSourceIndex(),
+ return DataMovementEvent.create(proto.getSourceIndex(),
proto.getTargetIndex(),
proto.getVersion(),
proto.getUserPayload() != null ?
@@ -65,7 +65,7 @@ public class ProtoConverters {
public static CompositeDataMovementEvent convertCompositeDataMovementEventFromProto(
EventProtos.CompositeEventProto proto) {
- return new CompositeDataMovementEvent(proto.getStartIndex(),
+ return CompositeDataMovementEvent.create(proto.getStartIndex(),
proto.getCount(),
proto.hasUserPayload() ? proto.getUserPayload().toByteArray() : null);
}
@@ -82,7 +82,7 @@ public class ProtoConverters {
public static VertexManagerEvent convertVertexManagerEventFromProto(
EventProtos.VertexManagerEventProto vmProto) {
- return new VertexManagerEvent(vmProto.getTargetVertexName(),
+ return VertexManagerEvent.create(vmProto.getTargetVertexName(),
vmProto.hasUserPayload() ? vmProto.getUserPayload().toByteArray() : null);
}
@@ -101,8 +101,9 @@ public class ProtoConverters {
public static InputDataInformationEvent
convertRootInputDataInformationEventFromProto(
EventProtos.RootInputDataInformationEventProto proto) {
- InputDataInformationEvent diEvent = new InputDataInformationEvent(
- proto.getSourceIndex(), proto.hasUserPayload() ? proto.getUserPayload().toByteArray() : null);
+ InputDataInformationEvent diEvent = InputDataInformationEvent.create(
+ proto.getSourceIndex(),
+ proto.hasUserPayload() ? proto.getUserPayload().toByteArray() : null);
diEvent.setTargetIndex(proto.getTargetIndex());
return diEvent;
}
@@ -113,7 +114,6 @@ public class ProtoConverters {
EventProtos.RootInputInitializerEventProto.newBuilder();
builder.setTargetVertexName(event.getTargetVertexName());
builder.setTargetInputName(event.getTargetInputName());
- builder.setVersion(event.getVersion());
if (event.getUserPayload() != null) {
builder.setUserPayload(ByteString.copyFrom(event.getUserPayload()));
}
@@ -123,9 +123,8 @@ public class ProtoConverters {
public static InputInitializerEvent convertRootInputInitializerEventFromProto(
EventProtos.RootInputInitializerEventProto proto) {
InputInitializerEvent event =
- new InputInitializerEvent(proto.getTargetVertexName(), proto.getTargetInputName(),
- (proto.hasUserPayload() ? proto.getUserPayload().toByteArray() : null),
- proto.getVersion());
+ InputInitializerEvent.create(proto.getTargetVertexName(), proto.getTargetInputName(),
+ (proto.hasUserPayload() ? proto.getUserPayload().toByteArray() : null));
return event;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezEvent.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezEvent.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezEvent.java
index 70bfad7..974e190 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezEvent.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezEvent.java
@@ -213,7 +213,7 @@ public class TezEvent implements Writable {
case INPUT_READ_ERROR_EVENT:
InputReadErrorEventProto ideProto =
InputReadErrorEventProto.parseFrom(eventBytes);
- event = new InputReadErrorEvent(ideProto.getDiagnostics(),
+ event = InputReadErrorEvent.create(ideProto.getDiagnostics(),
ideProto.getIndex(), ideProto.getVersion());
break;
case TASK_ATTEMPT_FAILED_EVENT:
@@ -227,7 +227,7 @@ public class TezEvent implements Writable {
case INPUT_FAILED_EVENT:
InputFailedEventProto ifProto =
InputFailedEventProto.parseFrom(eventBytes);
- event = new InputFailedEvent(ifProto.getTargetIndex(), ifProto.getVersion());
+ event = InputFailedEvent.create(ifProto.getTargetIndex(), ifProto.getVersion());
break;
case ROOT_INPUT_DATA_INFORMATION_EVENT:
RootInputDataInformationEventProto difProto = RootInputDataInformationEventProto
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezInputContextImpl.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezInputContextImpl.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezInputContextImpl.java
index 3945d4b..36d2cad 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezInputContextImpl.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezInputContextImpl.java
@@ -74,7 +74,7 @@ public class TezInputContextImpl extends TezTaskContextImpl
checkNotNull(sourceVertexName, "sourceVertexName is null");
checkNotNull(inputs, "input map is null");
checkNotNull(inputReadyTracker, "inputReadyTracker is null");
- this.userPayload = userPayload == null ? new UserPayload(null) : userPayload;
+ this.userPayload = userPayload == null ? UserPayload.create(null) : userPayload;
this.inputIndex = inputIndex;
this.sourceVertexName = sourceVertexName;
this.sourceInfo = new EventMetaData(
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezMergedInputContextImpl.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezMergedInputContextImpl.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezMergedInputContextImpl.java
index 374fbe3..693bde8 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezMergedInputContextImpl.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezMergedInputContextImpl.java
@@ -46,7 +46,7 @@ public class TezMergedInputContextImpl implements MergedInputContext {
checkNotNull(inputReadyTracker, "inputReadyTracker is null");
this.groupInputName = groupInputName;
this.groupInputsMap = groupInputsMap;
- this.userPayload = userPayload == null ? new UserPayload(null) : userPayload;
+ this.userPayload = userPayload == null ? UserPayload.create(null) : userPayload;
this.inputReadyTracker = inputReadyTracker;
this.workDirs = workDirs;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezOutputContextImpl.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezOutputContextImpl.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezOutputContextImpl.java
index 3e88080..e02f41b 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezOutputContextImpl.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezOutputContextImpl.java
@@ -67,7 +67,7 @@ public class TezOutputContextImpl extends TezTaskContextImpl
auxServiceEnv, memDist, outputDescriptor, objectRegistry);
checkNotNull(outputIndex, "outputIndex is null");
checkNotNull(destinationVertexName, "destinationVertexName is null");
- this.userPayload = userPayload == null ? new UserPayload(null) : userPayload;
+ this.userPayload = userPayload == null ? UserPayload.create(null) : userPayload;
this.outputIndex = outputIndex;
this.destinationVertexName = destinationVertexName;
this.sourceInfo = new EventMetaData(EventProducerConsumerType.OUTPUT,
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezProcessorContextImpl.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezProcessorContextImpl.java b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezProcessorContextImpl.java
index 2e8e622..22b9336 100644
--- a/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezProcessorContextImpl.java
+++ b/tez-runtime-internals/src/main/java/org/apache/tez/runtime/api/impl/TezProcessorContextImpl.java
@@ -62,7 +62,7 @@ public class TezProcessorContextImpl extends TezTaskContextImpl implements Proce
counters, runtimeTask, tezUmbilical, serviceConsumerMetadata,
auxServiceEnv, memDist, processorDescriptor, objectRegistry);
checkNotNull(inputReadyTracker, "inputReadyTracker is null");
- this.userPayload = userPayload == null ? new UserPayload(null) : userPayload;
+ this.userPayload = userPayload == null ? UserPayload.create(null) : userPayload;
this.sourceInfo = new EventMetaData(EventProducerConsumerType.PROCESSOR,
taskVertexName, "", taskAttemptID);
this.inputReadyTracker = inputReadyTracker;
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-runtime-internals/src/test/java/org/apache/tez/runtime/TestLogicalIOProcessorRuntimeTask.java
----------------------------------------------------------------------
diff --git a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/TestLogicalIOProcessorRuntimeTask.java b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/TestLogicalIOProcessorRuntimeTask.java
index 0429218..d4374f9 100644
--- a/tez-runtime-internals/src/test/java/org/apache/tez/runtime/TestLogicalIOProcessorRuntimeTask.java
+++ b/tez-runtime-internals/src/test/java/org/apache/tez/runtime/TestLogicalIOProcessorRuntimeTask.java
@@ -108,19 +108,19 @@ public class TestLogicalIOProcessorRuntimeTask {
}
private List<InputSpec> createInputSpecList() {
- InputDescriptor inputDesc = new InputDescriptor(TestInput.class.getName());
+ InputDescriptor inputDesc = InputDescriptor.create(TestInput.class.getName());
InputSpec inputSpec = new InputSpec("inedge", inputDesc, 1);
return Lists.newArrayList(inputSpec);
}
private List<OutputSpec> createOutputSpecList() {
- OutputDescriptor outputtDesc = new OutputDescriptor(TestOutput.class.getName());
+ OutputDescriptor outputtDesc = OutputDescriptor.create(TestOutput.class.getName());
OutputSpec outputSpec = new OutputSpec("outedge", outputtDesc, 1);
return Lists.newArrayList(outputSpec);
}
private ProcessorDescriptor createProcessorDescriptor() {
- ProcessorDescriptor desc = new ProcessorDescriptor(TestProcessor.class.getName());
+ ProcessorDescriptor desc = ProcessorDescriptor.create(TestProcessor.class.getName());
return desc;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java
index d98362c..01de0bd 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/dag/library/vertexmanager/ShuffleVertexManager.java
@@ -280,7 +280,7 @@ public class ShuffleVertexManager extends VertexManagerPlugin {
}
public UserPayload toUserPayload() {
- return new UserPayload(
+ return UserPayload.create(
ByteBuffer.wrap(ShuffleEdgeManagerConfigPayloadProto.newBuilder()
.setNumSourceTaskOutputs(numSourceTaskOutputs)
.setNumDestinationTasks(numDestinationTasks)
@@ -441,7 +441,7 @@ public class ShuffleVertexManager extends VertexManagerPlugin {
((remainderRangeForLastShuffler > 0) ?
remainderRangeForLastShuffler : basePartitionRange));
EdgeManagerPluginDescriptor edgeManagerDescriptor =
- new EdgeManagerPluginDescriptor(CustomShuffleEdgeManager.class.getName());
+ EdgeManagerPluginDescriptor.create(CustomShuffleEdgeManager.class.getName());
edgeManagerDescriptor.setUserPayload(edgeManagerConfig.toUserPayload());
edgeManagers.put(vertex, edgeManagerDescriptor);
}
@@ -660,7 +660,7 @@ public class ShuffleVertexManager extends VertexManagerPlugin {
public VertexManagerPluginDescriptor build() {
VertexManagerPluginDescriptor desc =
- new VertexManagerPluginDescriptor(ShuffleVertexManager.class.getName());
+ VertexManagerPluginDescriptor.create(ShuffleVertexManager.class.getName());
try {
return desc.setUserPayload(TezUtils.createUserPayloadFromConf(this.conf));
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleScheduler.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleScheduler.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleScheduler.java
index b530767..86883dd 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleScheduler.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/impl/ShuffleScheduler.java
@@ -291,10 +291,10 @@ class ShuffleScheduler {
inputContext.getSourceVertexName(), srcAttempt.getInputIdentifier().getInputIndex(),
srcAttempt.getAttemptNumber()) + " to AM.");
List<Event> failedEvents = Lists.newArrayListWithCapacity(1);
- failedEvents.add(new InputReadErrorEvent("Fetch failure for "
+ failedEvents.add(InputReadErrorEvent.create("Fetch failure for "
+ TezRuntimeUtils.getTaskAttemptIdentifier(
- inputContext.getSourceVertexName(), srcAttempt.getInputIdentifier().getInputIndex(),
- srcAttempt.getAttemptNumber()) + " to jobtracker.", srcAttempt.getInputIdentifier()
+ inputContext.getSourceVertexName(), srcAttempt.getInputIdentifier().getInputIndex(),
+ srcAttempt.getAttemptNumber()) + " to jobtracker.", srcAttempt.getInputIdentifier()
.getInputIndex(), srcAttempt.getAttemptNumber()));
inputContext.sendEvents(failedEvents);
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java
index a7a0451..1eb09bc 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/writers/UnorderedPartitionedKVWriter.java
@@ -490,7 +490,7 @@ public class UnorderedPartitionedKVWriter extends BaseUnorderedPartitionedKVWrit
payloadBuidler.setPathComponent(outputContext.getUniqueIdentifier());
}
- CompositeDataMovementEvent cDme = new CompositeDataMovementEvent(0, numPartitions,
+ CompositeDataMovementEvent cDme = CompositeDataMovementEvent.create(0, numPartitions,
payloadBuidler.build().toByteArray());
return cDme;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java
index 090b12f..c495c12 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/OrderedPartitionedKVEdgeConfig.java
@@ -117,11 +117,11 @@ public class OrderedPartitionedKVEdgeConfig
* @return an {@link org.apache.tez.dag.api.EdgeProperty} instance
*/
public EdgeProperty createDefaultEdgeProperty() {
- EdgeProperty edgeProperty = new EdgeProperty(EdgeProperty.DataMovementType.SCATTER_GATHER,
+ EdgeProperty edgeProperty = EdgeProperty.create(EdgeProperty.DataMovementType.SCATTER_GATHER,
EdgeProperty.DataSourceType.PERSISTED, EdgeProperty.SchedulingType.SEQUENTIAL,
- new OutputDescriptor(
+ OutputDescriptor.create(
getOutputClassName()).setUserPayload(getOutputPayload()),
- new InputDescriptor(
+ InputDescriptor.create(
getInputClassName()).setUserPayload(getInputPayload()));
return edgeProperty;
}
@@ -136,10 +136,10 @@ public class OrderedPartitionedKVEdgeConfig
public EdgeProperty createDefaultCustomEdgeProperty(EdgeManagerPluginDescriptor edgeManagerDescriptor) {
Preconditions.checkNotNull(edgeManagerDescriptor, "EdgeManagerDescriptor cannot be null");
EdgeProperty edgeProperty =
- new EdgeProperty(edgeManagerDescriptor, EdgeProperty.DataSourceType.PERSISTED,
+ EdgeProperty.create(edgeManagerDescriptor, EdgeProperty.DataSourceType.PERSISTED,
EdgeProperty.SchedulingType.SEQUENTIAL,
- new OutputDescriptor(getOutputClassName()).setUserPayload(getOutputPayload()),
- new InputDescriptor(getInputClassName()).setUserPayload(getInputPayload()));
+ OutputDescriptor.create(getOutputClassName()).setUserPayload(getOutputPayload()),
+ InputDescriptor.create(getInputClassName()).setUserPayload(getInputPayload()));
return edgeProperty;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java
index 47c0c76..66b2ded 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedKVEdgeConfig.java
@@ -98,11 +98,11 @@ public class UnorderedKVEdgeConfig extends HadoopKeyValuesBasedBaseEdgeConfig {
* @return an {@link org.apache.tez.dag.api.EdgeProperty} instance
*/
public EdgeProperty createDefaultBroadcastEdgeProperty() {
- EdgeProperty edgeProperty = new EdgeProperty(EdgeProperty.DataMovementType.BROADCAST,
+ EdgeProperty edgeProperty = EdgeProperty.create(EdgeProperty.DataMovementType.BROADCAST,
EdgeProperty.DataSourceType.PERSISTED, EdgeProperty.SchedulingType.SEQUENTIAL,
- new OutputDescriptor(
+ OutputDescriptor.create(
getOutputClassName()).setUserPayload(getOutputPayload()),
- new InputDescriptor(
+ InputDescriptor.create(
getInputClassName()).setUserPayload(getInputPayload()));
return edgeProperty;
}
@@ -118,11 +118,11 @@ public class UnorderedKVEdgeConfig extends HadoopKeyValuesBasedBaseEdgeConfig {
* @return an {@link org.apache.tez.dag.api.EdgeProperty} instance
*/
public EdgeProperty createDefaultOneToOneEdgeProperty() {
- EdgeProperty edgeProperty = new EdgeProperty(EdgeProperty.DataMovementType.ONE_TO_ONE,
+ EdgeProperty edgeProperty = EdgeProperty.create(EdgeProperty.DataMovementType.ONE_TO_ONE,
EdgeProperty.DataSourceType.PERSISTED, EdgeProperty.SchedulingType.SEQUENTIAL,
- new OutputDescriptor(
+ OutputDescriptor.create(
getOutputClassName()).setUserPayload(getOutputPayload()),
- new InputDescriptor(
+ InputDescriptor.create(
getInputClassName()).setUserPayload(getInputPayload()));
return edgeProperty;
}
@@ -137,10 +137,10 @@ public class UnorderedKVEdgeConfig extends HadoopKeyValuesBasedBaseEdgeConfig {
public EdgeProperty createDefaultCustomEdgeProperty(EdgeManagerPluginDescriptor edgeManagerDescriptor) {
Preconditions.checkNotNull(edgeManagerDescriptor, "EdgeManagerDescriptor cannot be null");
EdgeProperty edgeProperty =
- new EdgeProperty(edgeManagerDescriptor, EdgeProperty.DataSourceType.PERSISTED,
+ EdgeProperty.create(edgeManagerDescriptor, EdgeProperty.DataSourceType.PERSISTED,
EdgeProperty.SchedulingType.SEQUENTIAL,
- new OutputDescriptor(getOutputClassName()).setUserPayload(getOutputPayload()),
- new InputDescriptor(getInputClassName()).setUserPayload(getInputPayload()));
+ OutputDescriptor.create(getOutputClassName()).setUserPayload(getOutputPayload()),
+ InputDescriptor.create(getInputClassName()).setUserPayload(getInputPayload()));
return edgeProperty;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java
index 3216786..54ea5ce 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/conf/UnorderedPartitionedKVEdgeConfig.java
@@ -123,11 +123,11 @@ public class UnorderedPartitionedKVEdgeConfig
* @return an {@link org.apache.tez.dag.api.EdgeProperty} instance
*/
public EdgeProperty createDefaultEdgeProperty() {
- EdgeProperty edgeProperty = new EdgeProperty(EdgeProperty.DataMovementType.SCATTER_GATHER,
+ EdgeProperty edgeProperty = EdgeProperty.create(EdgeProperty.DataMovementType.SCATTER_GATHER,
EdgeProperty.DataSourceType.PERSISTED, EdgeProperty.SchedulingType.SEQUENTIAL,
- new OutputDescriptor(
+ OutputDescriptor.create(
getOutputClassName()).setUserPayload(getOutputPayload()),
- new InputDescriptor(
+ InputDescriptor.create(
getInputClassName()).setUserPayload(getInputPayload()));
return edgeProperty;
}
@@ -142,10 +142,10 @@ public class UnorderedPartitionedKVEdgeConfig
public EdgeProperty createDefaultCustomEdgeProperty(EdgeManagerPluginDescriptor edgeManagerDescriptor) {
Preconditions.checkNotNull(edgeManagerDescriptor, "EdgeManagerDescriptor cannot be null");
EdgeProperty edgeProperty =
- new EdgeProperty(edgeManagerDescriptor, EdgeProperty.DataSourceType.PERSISTED,
+ EdgeProperty.create(edgeManagerDescriptor, EdgeProperty.DataSourceType.PERSISTED,
EdgeProperty.SchedulingType.SEQUENTIAL,
- new OutputDescriptor(getOutputClassName()).setUserPayload(getOutputPayload()),
- new InputDescriptor(getInputClassName()).setUserPayload(getInputPayload()));
+ OutputDescriptor.create(getOutputClassName()).setUserPayload(getOutputPayload()),
+ InputDescriptor.create(getInputClassName()).setUserPayload(getInputPayload()));
return edgeProperty;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OrderedPartitionedKVOutput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OrderedPartitionedKVOutput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OrderedPartitionedKVOutput.java
index 3c32ec8..8708c6f 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OrderedPartitionedKVOutput.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/OrderedPartitionedKVOutput.java
@@ -199,13 +199,14 @@ public class OrderedPartitionedKVOutput extends AbstractLogicalOutput {
VertexManagerEventPayloadProto.Builder vmBuilder = VertexManagerEventPayloadProto
.newBuilder();
vmBuilder.setOutputSize(outputSize);
- VertexManagerEvent vmEvent = new VertexManagerEvent(
+ VertexManagerEvent vmEvent = VertexManagerEvent.create(
getContext().getDestinationVertexName(), vmBuilder.build().toByteArray());
List<Event> events = Lists.newArrayListWithCapacity(getNumPhysicalOutputs() + 1);
events.add(vmEvent);
- CompositeDataMovementEvent csdme = new CompositeDataMovementEvent(0, getNumPhysicalOutputs(), payload);
+ CompositeDataMovementEvent csdme =
+ CompositeDataMovementEvent.create(0, getNumPhysicalOutputs(), payload);
events.add(csdme);
return events;
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedKVOutput.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedKVOutput.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedKVOutput.java
index 8840ecf..9ad7cd3 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedKVOutput.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/output/UnorderedKVOutput.java
@@ -154,7 +154,7 @@ public class UnorderedKVOutput extends AbstractLogicalOutput {
}
DataMovementEventPayloadProto payloadProto = payloadBuilder.build();
- DataMovementEvent dmEvent = new DataMovementEvent(0, payloadProto.toByteArray());
+ DataMovementEvent dmEvent = DataMovementEvent.create(0, payloadProto.toByteArray());
List<Event> events = Lists.newArrayListWithCapacity(1);
events.add(dmEvent);
return events;
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/processor/SleepProcessor.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/processor/SleepProcessor.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/processor/SleepProcessor.java
index acd32f8..e893631 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/processor/SleepProcessor.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/processor/SleepProcessor.java
@@ -21,7 +21,7 @@ package org.apache.tez.runtime.library.processor;
import java.nio.ByteBuffer;
import java.nio.charset.CharacterCodingException;
import java.nio.charset.Charset;
-import java.nio.charset.CharsetDecoder;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -115,7 +115,7 @@ public class SleepProcessor extends AbstractLogicalIOProcessor {
}
public UserPayload toUserPayload() {
- return new UserPayload(ByteBuffer.wrap(Integer.toString(timeToSleepMS).getBytes()));
+ return UserPayload.create(ByteBuffer.wrap(Integer.toString(timeToSleepMS).getBytes()));
}
public void fromUserPayload(UserPayload userPayload) throws CharacterCodingException {
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/impl/ShuffleManager.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/impl/ShuffleManager.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/impl/ShuffleManager.java
index 3ae3615..be43dc0 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/impl/ShuffleManager.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/shuffle/common/impl/ShuffleManager.java
@@ -487,12 +487,12 @@ public class ShuffleManager implements FetcherCallback {
LOG.fatal(message);
inputContext.fatalError(null, message);
} else {
- InputReadErrorEvent readError = new InputReadErrorEvent(
+ InputReadErrorEvent readError = InputReadErrorEvent.create(
"Fetch failure while fetching from "
+ TezRuntimeUtils.getTaskAttemptIdentifier(
- inputContext.getSourceVertexName(),
- srcAttemptIdentifier.getInputIdentifier().getInputIndex(),
- srcAttemptIdentifier.getAttemptNumber()),
+ inputContext.getSourceVertexName(),
+ srcAttemptIdentifier.getInputIdentifier().getInputIndex(),
+ srcAttemptIdentifier.getAttemptNumber()),
srcAttemptIdentifier.getInputIdentifier().getInputIndex(),
srcAttemptIdentifier.getAttemptNumber());
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestInputReadyVertexManager.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestInputReadyVertexManager.java b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestInputReadyVertexManager.java
index 09b7d4b..f57bdb0 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestInputReadyVertexManager.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestInputReadyVertexManager.java
@@ -59,12 +59,12 @@ public class TestInputReadyVertexManager {
HashMap<String, EdgeProperty> mockInputVertices =
new HashMap<String, EdgeProperty>();
String mockSrcVertexId1 = "Vertex1";
- EdgeProperty eProp1 = new EdgeProperty(
+ EdgeProperty eProp1 = EdgeProperty.create(
EdgeProperty.DataMovementType.SCATTER_GATHER,
EdgeProperty.DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL,
- new OutputDescriptor("out"),
- new InputDescriptor("in"));
+ SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create("out"),
+ InputDescriptor.create("in"));
String mockManagedVertexId = "Vertex";
@@ -93,12 +93,12 @@ public class TestInputReadyVertexManager {
HashMap<String, EdgeProperty> mockInputVertices =
new HashMap<String, EdgeProperty>();
String mockSrcVertexId1 = "Vertex1";
- EdgeProperty eProp1 = new EdgeProperty(
+ EdgeProperty eProp1 = EdgeProperty.create(
EdgeProperty.DataMovementType.ONE_TO_ONE,
EdgeProperty.DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL,
- new OutputDescriptor("out"),
- new InputDescriptor("in"));
+ SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create("out"),
+ InputDescriptor.create("in"));
String mockManagedVertexId = "Vertex";
Container mockContainer1 = mock(Container.class);
@@ -148,26 +148,26 @@ public class TestInputReadyVertexManager {
HashMap<String, EdgeProperty> mockInputVertices =
new HashMap<String, EdgeProperty>();
String mockSrcVertexId1 = "Vertex1";
- EdgeProperty eProp1 = new EdgeProperty(
+ EdgeProperty eProp1 = EdgeProperty.create(
EdgeProperty.DataMovementType.SCATTER_GATHER,
EdgeProperty.DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL,
- new OutputDescriptor("out"),
- new InputDescriptor("in"));
+ SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create("out"),
+ InputDescriptor.create("in"));
String mockSrcVertexId2 = "Vertex2";
- EdgeProperty eProp2 = new EdgeProperty(
+ EdgeProperty eProp2 = EdgeProperty.create(
EdgeProperty.DataMovementType.ONE_TO_ONE,
EdgeProperty.DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL,
- new OutputDescriptor("out"),
- new InputDescriptor("in"));
+ SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create("out"),
+ InputDescriptor.create("in"));
String mockSrcVertexId3 = "Vertex3";
- EdgeProperty eProp3 = new EdgeProperty(
+ EdgeProperty eProp3 = EdgeProperty.create(
EdgeProperty.DataMovementType.ONE_TO_ONE,
- EdgeProperty.DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL,
- new OutputDescriptor("out"),
- new InputDescriptor("in"));
+ EdgeProperty.DataSourceType.PERSISTED,
+ SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create("out"),
+ InputDescriptor.create("in"));
String mockManagedVertexId = "Vertex";
Container mockContainer2 = mock(Container.class);
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java
index 4290cd7..e4b4656 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/dag/library/vertexmanager/TestShuffleVertexManager.java
@@ -68,26 +68,26 @@ public class TestShuffleVertexManager {
HashMap<String, EdgeProperty> mockInputVertices =
new HashMap<String, EdgeProperty>();
String mockSrcVertexId1 = "Vertex1";
- EdgeProperty eProp1 = new EdgeProperty(
+ EdgeProperty eProp1 = EdgeProperty.create(
EdgeProperty.DataMovementType.SCATTER_GATHER,
EdgeProperty.DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL,
- new OutputDescriptor("out"),
- new InputDescriptor("in"));
+ SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create("out"),
+ InputDescriptor.create("in"));
String mockSrcVertexId2 = "Vertex2";
- EdgeProperty eProp2 = new EdgeProperty(
+ EdgeProperty eProp2 = EdgeProperty.create(
EdgeProperty.DataMovementType.SCATTER_GATHER,
EdgeProperty.DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL,
- new OutputDescriptor("out"),
- new InputDescriptor("in"));
+ SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create("out"),
+ InputDescriptor.create("in"));
String mockSrcVertexId3 = "Vertex3";
- EdgeProperty eProp3 = new EdgeProperty(
+ EdgeProperty eProp3 = EdgeProperty.create(
EdgeProperty.DataMovementType.BROADCAST,
- EdgeProperty.DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL,
- new OutputDescriptor("out"),
- new InputDescriptor("in"));
+ EdgeProperty.DataSourceType.PERSISTED,
+ SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create("out"),
+ InputDescriptor.create("in"));
final String mockManagedVertexId = "Vertex4";
@@ -217,7 +217,7 @@ public class TestShuffleVertexManager {
byte[] payload =
VertexManagerEventPayloadProto.newBuilder().setOutputSize(5000L).build().toByteArray();
- VertexManagerEvent vmEvent = new VertexManagerEvent("Vertex", payload);
+ VertexManagerEvent vmEvent = VertexManagerEvent.create("Vertex", payload);
// parallelism not change due to large data size
manager = createManager(conf, mockContext, 0.1f, 0.1f);
manager.onVertexStarted(null);
@@ -237,7 +237,7 @@ public class TestShuffleVertexManager {
scheduledTasks.clear();
payload =
VertexManagerEventPayloadProto.newBuilder().setOutputSize(500L).build().toByteArray();
- vmEvent = new VertexManagerEvent("Vertex", payload);
+ vmEvent = VertexManagerEvent.create("Vertex", payload);
manager = createManager(conf, mockContext, 0.5f, 0.5f);
manager.onVertexStarted(null);
@@ -283,7 +283,7 @@ public class TestShuffleVertexManager {
EdgeManagerPlugin edgeManager = newEdgeManagers.values().iterator().next();
Map<Integer, List<Integer>> targets = Maps.newHashMap();
- DataMovementEvent dmEvent = new DataMovementEvent(1, new byte[0]);
+ DataMovementEvent dmEvent = DataMovementEvent.create(1, new byte[0]);
// 4 source task outputs - same as original number of partitions
Assert.assertEquals(4, edgeManager.getNumSourceTaskPhysicalOutputs(0));
// 4 destination task inputs - 2 source tasks + 2 merged partitions
@@ -295,7 +295,7 @@ public class TestShuffleVertexManager {
Assert.assertEquals(1, e.getValue().size());
Assert.assertEquals(3, e.getValue().get(0).intValue());
targets.clear();
- dmEvent = new DataMovementEvent(2, new byte[0]);
+ dmEvent = DataMovementEvent.create(2, new byte[0]);
edgeManager.routeDataMovementEventToDestination(dmEvent, 0, dmEvent.getSourceIndex(), targets);
Assert.assertEquals(1, targets.size());
e = targets.entrySet().iterator().next();
@@ -321,26 +321,26 @@ public class TestShuffleVertexManager {
HashMap<String, EdgeProperty> mockInputVertices =
new HashMap<String, EdgeProperty>();
String mockSrcVertexId1 = "Vertex1";
- EdgeProperty eProp1 = new EdgeProperty(
+ EdgeProperty eProp1 = EdgeProperty.create(
EdgeProperty.DataMovementType.SCATTER_GATHER,
EdgeProperty.DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL,
- new OutputDescriptor("out"),
- new InputDescriptor("in"));
+ SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create("out"),
+ InputDescriptor.create("in"));
String mockSrcVertexId2 = "Vertex2";
- EdgeProperty eProp2 = new EdgeProperty(
+ EdgeProperty eProp2 = EdgeProperty.create(
EdgeProperty.DataMovementType.SCATTER_GATHER,
EdgeProperty.DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL,
- new OutputDescriptor("out"),
- new InputDescriptor("in"));
+ SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create("out"),
+ InputDescriptor.create("in"));
String mockSrcVertexId3 = "Vertex3";
- EdgeProperty eProp3 = new EdgeProperty(
+ EdgeProperty eProp3 = EdgeProperty.create(
EdgeProperty.DataMovementType.BROADCAST,
- EdgeProperty.DataSourceType.PERSISTED,
- SchedulingType.SEQUENTIAL,
- new OutputDescriptor("out"),
- new InputDescriptor("in"));
+ EdgeProperty.DataSourceType.PERSISTED,
+ SchedulingType.SEQUENTIAL,
+ OutputDescriptor.create("out"),
+ InputDescriptor.create("in"));
String mockManagedVertexId = "Vertex4";
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/TestValuesIterator.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/TestValuesIterator.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/TestValuesIterator.java
index b409e11..faa728f 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/TestValuesIterator.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/TestValuesIterator.java
@@ -382,7 +382,7 @@ public class TestValuesIterator {
doReturn(1).when(inputContext).getInputIndex();
doReturn("srcVertex").when(inputContext).getSourceVertexName();
doReturn(1).when(inputContext).getTaskVertexIndex();
- doReturn(new UserPayload(ByteBuffer.wrap(new byte[1024]))).when(inputContext).getUserPayload();
+ doReturn(UserPayload.create(ByteBuffer.wrap(new byte[1024]))).when(inputContext).getUserPayload();
return inputContext;
}
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestShuffleInputEventHandler.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestShuffleInputEventHandler.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestShuffleInputEventHandler.java
index f31d20e..56e1337 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestShuffleInputEventHandler.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/common/shuffle/impl/TestShuffleInputEventHandler.java
@@ -73,7 +73,7 @@ public class TestShuffleInputEventHandler {
if (emptyPartitionByteString != null) {
builder.setEmptyPartitions(emptyPartitionByteString);
}
- return new DataMovementEvent(srcIndex, targetIndex, 0, builder.build().toByteArray());
+ return DataMovementEvent.create(srcIndex, targetIndex, 0, builder.build().toByteArray());
}
@Before
@@ -106,7 +106,7 @@ public class TestShuffleInputEventHandler {
public void testFailedEvent() throws IOException {
List<Event> events = new LinkedList<Event>();
int targetIdx = 1;
- InputFailedEvent failedEvent = new InputFailedEvent(targetIdx, 0);
+ InputFailedEvent failedEvent = InputFailedEvent.create(targetIdx, 0);
events.add(failedEvent);
handler.handleEvents(events);
InputAttemptIdentifier expectedIdentifier = new InputAttemptIdentifier(targetIdx, 0);
http://git-wip-us.apache.org/repos/asf/tez/blob/b30e2bcf/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/shuffle/common/impl/TestShuffleInputEventHandlerImpl.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/shuffle/common/impl/TestShuffleInputEventHandlerImpl.java b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/shuffle/common/impl/TestShuffleInputEventHandlerImpl.java
index e5a2f99..646dca8 100644
--- a/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/shuffle/common/impl/TestShuffleInputEventHandlerImpl.java
+++ b/tez-runtime-library/src/test/java/org/apache/tez/runtime/library/shuffle/common/impl/TestShuffleInputEventHandlerImpl.java
@@ -148,7 +148,7 @@ public class TestShuffleInputEventHandlerImpl {
if (emptyPartitionByteString != null) {
builder.setEmptyPartitions(emptyPartitionByteString);
}
- Event dme = new DataMovementEvent(srcIndex, targetIndex, 0, builder.build().toByteArray());
+ Event dme = DataMovementEvent.create(srcIndex, targetIndex, 0, builder.build().toByteArray());
return dme;
}