You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by hi...@apache.org on 2013/04/29 20:27:35 UTC
svn commit: r1477252 - in /incubator/tez/branches/TEZ-1:
tez-dag-api/src/main/java/org/apache/tez/dag/api/
tez-dag/src/main/java/org/apache/tez/dag/app/
tez-yarn-client/src/main/java/org/apache/tez/mapreduce/
Author: hitesh
Date: Mon Apr 29 18:27:34 2013
New Revision: 1477252
URL: http://svn.apache.org/r1477252
Log:
TEZ-28. Remove default constructor for EdgeProperty. (hitesh)
Modified:
incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGConfiguration.java
incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/EdgeProperty.java
incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
incubator/tez/branches/TEZ-1/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
Modified: incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGConfiguration.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGConfiguration.java?rev=1477252&r1=1477251&r2=1477252&view=diff
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGConfiguration.java (original)
+++ incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/DAGConfiguration.java Mon Apr 29 18:27:34 2013
@@ -235,7 +235,8 @@ public class DAGConfiguration extends Co
for(Map.Entry<String,LocalResource> entry : lrs.entrySet()) {
LocalResource lr = entry.getValue();
try {
- String lrStr = StringUtils.escapeString(entry.getKey(), StringUtils.ESCAPE_CHAR,
+ String lrStr = StringUtils.escapeString(entry.getKey(),
+ StringUtils.ESCAPE_CHAR,
SEPARATOR.charAt(0))
+ SEPARATOR
+ StringUtils.escapeString(
@@ -243,23 +244,28 @@ public class DAGConfiguration extends Co
StringUtils.ESCAPE_CHAR,
SEPARATOR.charAt(0))
+ SEPARATOR
- + StringUtils.escapeString(String.valueOf(lr.getSize()), StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0))
+ + StringUtils.escapeString(String.valueOf(lr.getSize()),
+ StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0))
+ SEPARATOR
- + StringUtils.escapeString(String.valueOf(lr.getTimestamp()), StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0))
+ + StringUtils.escapeString(String.valueOf(lr.getTimestamp()),
+ StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0))
+ SEPARATOR
- + StringUtils.escapeString(lr.getType().name(), StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0))
+ + StringUtils.escapeString(lr.getType().name(),
+ StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0))
+ SEPARATOR
- + StringUtils.escapeString(lr.getVisibility().name(), StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0))
+ + StringUtils.escapeString(lr.getVisibility().name(),
+ StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0))
+ SEPARATOR
- + StringUtils.escapeString((lr.getPattern() == null ? "" : lr.getPattern()), StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0));
- LOG.info("XXXX serializing local resource:"
- + lrStr);
+ + StringUtils.escapeString(
+ (lr.getPattern() == null ? "" : lr.getPattern()),
+ StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0));
lrStrs[i++] = StringUtils.escapeString(lrStr);
} catch (URISyntaxException e) {
throw new RuntimeException(e);
}
}
- setStrings(TEZ_DAG_VERTEX_TASK_LOCAL_RESOURCE + vertex.getVertexName(), lrStrs);
+ setStrings(TEZ_DAG_VERTEX_TASK_LOCAL_RESOURCE + vertex.getVertexName(),
+ lrStrs);
}
public Map<String, LocalResource> getVertexLocalResources(String vertexName) {
@@ -272,7 +278,6 @@ public class DAGConfiguration extends Co
}
LOG.info("XXXX Found " + lrStrs.length + " local resources");
for (String lrStr : lrStrs) {
- LOG.info("XXXX Parsing local resource from " + lrStr);
String[] tokens =
StringUtils.split(
lrStr, StringUtils.ESCAPE_CHAR, SEPARATOR.charAt(0));
@@ -299,7 +304,8 @@ public class DAGConfiguration extends Co
+ ", vertexName=" + vertexName
+ ", resourceName=" + resourceName
+ ", resourceUrl"
- + ConverterUtils.getPathFromYarnURL(lRsrc.getResource()).toString());
+ + ConverterUtils.getPathFromYarnURL(
+ lRsrc.getResource()).toString());
} catch (URISyntaxException e) {
// Ignore
// FIXME
Modified: incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/EdgeProperty.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/EdgeProperty.java?rev=1477252&r1=1477251&r2=1477252&view=diff
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/EdgeProperty.java (original)
+++ incubator/tez/branches/TEZ-1/tez-dag-api/src/main/java/org/apache/tez/dag/api/EdgeProperty.java Mon Apr 29 18:27:34 2013
@@ -46,14 +46,6 @@ public class EdgeProperty { // FIXME ren
this.outputClass = outputClass;
}
- // default constructor for mapreduce case
- public EdgeProperty() {
- this.connectionPattern = ConnectionPattern.BIPARTITE;
- this.sourceType = SourceType.STABLE;
- this.inputClass = "org.apache.tez.engine.lib.input.ShuffledMergedInput";
- this.outputClass = "org.apache.tez.engine.lib.output.OnFileSortedOutput";
- }
-
public ConnectionPattern getConnectionPattern() {
return connectionPattern;
}
Modified: incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java?rev=1477252&r1=1477251&r2=1477252&view=diff
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java (original)
+++ incubator/tez/branches/TEZ-1/tez-dag/src/main/java/org/apache/tez/dag/app/MRRExampleHelper.java Mon Apr 29 18:27:34 2013
@@ -20,8 +20,12 @@ import org.apache.hadoop.yarn.util.Build
import org.apache.tez.dag.api.DAGConfiguration;
import org.apache.tez.dag.api.Edge;
import org.apache.tez.dag.api.EdgeProperty;
+import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern;
+import org.apache.tez.dag.api.EdgeProperty.SourceType;
import org.apache.tez.dag.api.Vertex;
import org.apache.tez.dag.app.rm.container.AMContainerHelpers;
+import org.apache.tez.engine.lib.input.ShuffledMergedInput;
+import org.apache.tez.engine.lib.output.OnFileSortedOutput;
import org.apache.tez.mapreduce.hadoop.MRJobConfig;
public class MRRExampleHelper {
@@ -88,8 +92,16 @@ public class MRRExampleHelper {
"org.apache.tez.mapreduce.task.IntermediateTask", 3);
Vertex reduce2Vertex = new Vertex("reduce2",
"org.apache.tez.mapreduce.task.FinalTask", 3);
- Edge edge1 = new Edge(mapVertex, reduce1Vertex, new EdgeProperty());
- Edge edge2 = new Edge(reduce1Vertex, reduce2Vertex, new EdgeProperty());
+ Edge edge1 = new Edge(mapVertex, reduce1Vertex,
+ new EdgeProperty(ConnectionPattern.BIPARTITE,
+ SourceType.STABLE,
+ ShuffledMergedInput.class.getName(),
+ OnFileSortedOutput.class.getName()));
+ Edge edge2 = new Edge(reduce1Vertex, reduce2Vertex,
+ new EdgeProperty(ConnectionPattern.BIPARTITE,
+ SourceType.STABLE,
+ ShuffledMergedInput.class.getName(),
+ OnFileSortedOutput.class.getName()));
Map<String, LocalResource> jobRsrcs = createLocalResources(getMRRBaseDir(),
getMRRLocalRsrcList());
@@ -146,7 +158,11 @@ public class MRRExampleHelper {
"org.apache.tez.mapreduce.task.InitialTask", 6);
Vertex reduceVertex = new Vertex("reduce",
"org.apache.tez.mapreduce.task.FinalTask", 1);
- Edge edge = new Edge(mapVertex, reduceVertex, new EdgeProperty());
+ Edge edge = new Edge(mapVertex, reduceVertex,
+ new EdgeProperty(ConnectionPattern.BIPARTITE,
+ SourceType.STABLE,
+ ShuffledMergedInput.class.getName(),
+ OnFileSortedOutput.class.getName()));
Map<String, LocalResource> jobRsrcs = createLocalResources(getMRBaseDir(),
getMRLocalRsrcList());
Modified: incubator/tez/branches/TEZ-1/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
URL: http://svn.apache.org/viewvc/incubator/tez/branches/TEZ-1/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java?rev=1477252&r1=1477251&r2=1477252&view=diff
==============================================================================
--- incubator/tez/branches/TEZ-1/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java (original)
+++ incubator/tez/branches/TEZ-1/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java Mon Apr 29 18:27:34 2013
@@ -59,7 +59,11 @@ import org.apache.tez.dag.api.Edge;
import org.apache.tez.dag.api.EdgeProperty;
import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.Vertex;
+import org.apache.tez.dag.api.EdgeProperty.ConnectionPattern;
+import org.apache.tez.dag.api.EdgeProperty.SourceType;
import org.apache.tez.dag.api.VertexLocationHint.TaskLocationHint;
+import org.apache.tez.engine.lib.input.ShuffledMergedInput;
+import org.apache.tez.engine.lib.output.OnFileSortedOutput;
import org.apache.tez.mapreduce.hadoop.DeprecatedKeys;
import org.apache.tez.mapreduce.hadoop.MRJobConfig;
import org.apache.tez.mapreduce.hadoop.MultiStageMRConfToTezTranslator;
@@ -645,7 +649,11 @@ public class YARNRunner implements Clien
+ ", parrellism=" + reduceVertex.getParallelism());
dag.addVertex(reduceVertex);
- EdgeProperty edgeProperty = new EdgeProperty();
+ EdgeProperty edgeProperty =
+ new EdgeProperty(ConnectionPattern.BIPARTITE,
+ SourceType.STABLE,
+ ShuffledMergedInput.class.getName(),
+ OnFileSortedOutput.class.getName());
Edge edge = null;
if (!isMRR) {
edge = new Edge(mapVertex, reduceVertex, edgeProperty);