You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by se...@apache.org on 2016/01/15 11:53:37 UTC

[07/12] flink git commit: [FLINK-3235] Remove Flink on Tez code

http://git-wip-us.apache.org/repos/asf/flink/blob/8b8dfc16/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/dag/TezDAGGenerator.java
----------------------------------------------------------------------
diff --git a/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/dag/TezDAGGenerator.java b/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/dag/TezDAGGenerator.java
deleted file mode 100644
index 52f39be..0000000
--- a/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/dag/TezDAGGenerator.java
+++ /dev/null
@@ -1,460 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.tez.dag;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.flink.api.common.distributions.DataDistribution;
-import org.apache.flink.api.common.io.InputFormat;
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
-import org.apache.flink.configuration.ConfigConstants;
-import org.apache.flink.configuration.Configuration;
-import org.apache.flink.core.io.InputSplit;
-import org.apache.flink.optimizer.CompilerException;
-import org.apache.flink.optimizer.dag.TempMode;
-import org.apache.flink.optimizer.plan.BulkIterationPlanNode;
-import org.apache.flink.optimizer.plan.Channel;
-import org.apache.flink.optimizer.plan.DualInputPlanNode;
-import org.apache.flink.optimizer.plan.NAryUnionPlanNode;
-import org.apache.flink.optimizer.plan.OptimizedPlan;
-import org.apache.flink.optimizer.plan.PlanNode;
-import org.apache.flink.optimizer.plan.SingleInputPlanNode;
-import org.apache.flink.optimizer.plan.SinkPlanNode;
-import org.apache.flink.optimizer.plan.SourcePlanNode;
-import org.apache.flink.optimizer.plan.WorksetIterationPlanNode;
-import org.apache.flink.runtime.operators.DriverStrategy;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.apache.flink.runtime.operators.util.LocalStrategy;
-import org.apache.flink.runtime.operators.util.TaskConfig;
-import org.apache.flink.tez.runtime.TezTaskConfig;
-import org.apache.flink.util.Visitor;
-import org.apache.tez.dag.api.DAG;
-import org.apache.tez.dag.api.TezConfiguration;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-
-public class TezDAGGenerator implements Visitor<PlanNode> {
-
-	private static final Log LOG = LogFactory.getLog(TezDAGGenerator.class);
-	
-	private Map<PlanNode, FlinkVertex> vertices; // a map from optimizer nodes to Tez vertices
-	private List<FlinkEdge> edges;
-	private final int defaultMaxFan;
-	private final TezConfiguration tezConf;
-
-	private final float defaultSortSpillingThreshold;
-
-	public TezDAGGenerator (TezConfiguration tezConf, Configuration config) {
-		this.defaultMaxFan = config.getInteger(ConfigConstants.DEFAULT_SPILLING_MAX_FAN_KEY,
-				ConfigConstants.DEFAULT_SPILLING_MAX_FAN);
-		this.defaultSortSpillingThreshold = config.getFloat(ConfigConstants.DEFAULT_SORT_SPILLING_THRESHOLD_KEY,
-				ConfigConstants.DEFAULT_SORT_SPILLING_THRESHOLD);
-		this.tezConf = tezConf;
-	}
-
-	public DAG createDAG (OptimizedPlan program) throws Exception {
-		LOG.info ("Creating Tez DAG");
-		this.vertices = new HashMap<PlanNode, FlinkVertex>();
-		this.edges = new ArrayList<FlinkEdge>();
-		program.accept(this);
-
-		DAG dag = DAG.create(program.getJobName());
-		for (FlinkVertex v : vertices.values()) {
-			dag.addVertex(v.createVertex(new TezConfiguration(tezConf)));
-		}
-		for (FlinkEdge e: edges) {
-			dag.addEdge(e.createEdge(new TezConfiguration(tezConf)));
-		}
-
-		/*
-		 * Temporarily throw an error until TEZ-1190 has been fixed or a workaround has been created
-		 */
-		if (containsSelfJoins()) {
-			throw new CompilerException("Dual-input operators with the same input (self-joins) are not yet supported");
-		}
-
-		this.vertices = null;
-		this.edges = null;
-
-		LOG.info ("Tez DAG created");
-		return dag;
-	}
-
-
-	@Override
-	public boolean preVisit(PlanNode node) {
-		if (this.vertices.containsKey(node)) {
-			// return false to prevent further descend
-			return false;
-		}
-
-		if ((node instanceof BulkIterationPlanNode) || (node instanceof WorksetIterationPlanNode)) {
-			throw new CompilerException("Iterations are not yet supported by the Tez execution environment");
-		}
-
-		if ( (node.getBroadcastInputs() != null) && (!node.getBroadcastInputs().isEmpty())) {
-			throw new CompilerException("Broadcast inputs are not yet supported by the Tez execution environment");
-		}
-
-		FlinkVertex vertex = null;
-
-		try {
-			if (node instanceof SourcePlanNode) {
-				vertex = createDataSourceVertex ((SourcePlanNode) node);
-			}
-			else if (node instanceof SinkPlanNode) {
-				vertex = createDataSinkVertex ((SinkPlanNode) node);
-			}
-			else if ((node instanceof SingleInputPlanNode)) {
-				vertex = createSingleInputVertex((SingleInputPlanNode) node);
-			}
-			else if (node instanceof DualInputPlanNode) {
-				vertex = createDualInputVertex((DualInputPlanNode) node);
-			}
-			else if (node instanceof NAryUnionPlanNode) {
-				vertex = createUnionVertex ((NAryUnionPlanNode) node);
-			}
-			else {
-				throw new CompilerException("Unrecognized node type: " + node.getClass().getName());
-			}
-
-		}
-		catch (Exception e) {
-			throw new CompilerException("Error translating node '" + node + "': " + e.getMessage(), e);
-		}
-
-		if (vertex != null) {
-			this.vertices.put(node, vertex);
-		}
-		return true;
-	}
-
-	@Override
-	public void postVisit (PlanNode node) {
-		try {
-			if (node instanceof SourcePlanNode) {
-				return;
-			}
-			final Iterator<Channel> inConns = node.getInputs().iterator();
-			if (!inConns.hasNext()) {
-				throw new CompilerException("Bug: Found a non-source task with no input.");
-			}
-			int inputIndex = 0;
-
-			FlinkVertex targetVertex = this.vertices.get(node);
-			TezTaskConfig targetVertexConfig = targetVertex.getConfig();
-
-
-			while (inConns.hasNext()) {
-				Channel input = inConns.next();
-				inputIndex += translateChannel(input, inputIndex, targetVertex, targetVertexConfig, false);
-			}
-		}
-		catch (Exception e) {
-			e.printStackTrace();
-			throw new CompilerException(
-					"An error occurred while translating the optimized plan to a Tez DAG: " + e.getMessage(), e);
-		}
-	}
-
-	private FlinkVertex createSingleInputVertex(SingleInputPlanNode node) throws CompilerException, IOException {
-
-		final String taskName = node.getNodeName();
-		final DriverStrategy ds = node.getDriverStrategy();
-		final int dop = node.getParallelism();
-
-		final TezTaskConfig config= new TezTaskConfig(new Configuration());
-
-		config.setDriver(ds.getDriverClass());
-		config.setDriverStrategy(ds);
-		config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper());
-		config.setStubParameters(node.getProgramOperator().getParameters());
-
-		for(int i=0;i<ds.getNumRequiredComparators();i++) {
-			config.setDriverComparator(node.getComparator(i), i);
-		}
-		assignDriverResources(node, config);
-
-		return new FlinkProcessorVertex(taskName, dop, config);
-	}
-
-	private FlinkVertex createDualInputVertex(DualInputPlanNode node) throws CompilerException, IOException {
-		final String taskName = node.getNodeName();
-		final DriverStrategy ds = node.getDriverStrategy();
-		final int dop = node.getParallelism();
-
-		final TezTaskConfig config= new TezTaskConfig(new Configuration());
-
-		config.setDriver(ds.getDriverClass());
-		config.setDriverStrategy(ds);
-		config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper());
-		config.setStubParameters(node.getProgramOperator().getParameters());
-
-		if (node.getComparator1() != null) {
-			config.setDriverComparator(node.getComparator1(), 0);
-		}
-		if (node.getComparator2() != null) {
-			config.setDriverComparator(node.getComparator2(), 1);
-		}
-		if (node.getPairComparator() != null) {
-			config.setDriverPairComparator(node.getPairComparator());
-		}
-
-		assignDriverResources(node, config);
-
-		LOG.info("Creating processor vertex " + taskName + " with parallelism " + dop);
-
-		return new FlinkProcessorVertex(taskName, dop, config);
-	}
-
-	private FlinkVertex createDataSinkVertex(SinkPlanNode node) throws CompilerException, IOException {
-		final String taskName = node.getNodeName();
-		final int dop = node.getParallelism();
-
-		final TezTaskConfig config = new TezTaskConfig(new Configuration());
-
-		// set user code
-		config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper());
-		config.setStubParameters(node.getProgramOperator().getParameters());
-
-		LOG.info("Creating data sink vertex " + taskName + " with parallelism " + dop);
-		
-		return new FlinkDataSinkVertex(taskName, dop, config);
-	}
-
-	private FlinkVertex createDataSourceVertex(SourcePlanNode node) throws CompilerException, IOException {
-		final String taskName = node.getNodeName();
-		int dop = node.getParallelism();
-
-		final TezTaskConfig config= new TezTaskConfig(new Configuration());
-
-		config.setStubWrapper(node.getProgramOperator().getUserCodeWrapper());
-		config.setStubParameters(node.getProgramOperator().getParameters());
-
-		InputFormat format = node.getDataSourceNode().getOperator().getFormatWrapper().getUserCodeObject();
-
-		config.setInputFormat(format);
-
-		// Create as many data sources as input splits
-		InputSplit[] splits = format.createInputSplits((dop > 0) ? dop : 1);
-		dop = splits.length;
-
-		LOG.info("Creating data source vertex " + taskName + " with parallelism " + dop);
-		
-		return new FlinkDataSourceVertex(taskName, dop, config);
-	}
-
-	private FlinkVertex createUnionVertex(NAryUnionPlanNode node) throws CompilerException, IOException {
-		final String taskName = node.getNodeName();
-		final int dop = node.getParallelism();
-		final TezTaskConfig config= new TezTaskConfig(new Configuration());
-
-		LOG.info("Creating union vertex " + taskName + " with parallelism " + dop);
-		
-		return new FlinkUnionVertex (taskName, dop, config);
-	}
-
-
-	private void assignDriverResources(PlanNode node, TaskConfig config) {
-		final double relativeMem = node.getRelativeMemoryPerSubTask();
-		if (relativeMem > 0) {
-			config.setRelativeMemoryDriver(relativeMem);
-			config.setFilehandlesDriver(this.defaultMaxFan);
-			config.setSpillingThresholdDriver(this.defaultSortSpillingThreshold);
-		}
-	}
-
-	private void assignLocalStrategyResources(Channel c, TaskConfig config, int inputNum) {
-		if (c.getRelativeMemoryLocalStrategy() > 0) {
-			config.setRelativeMemoryInput(inputNum, c.getRelativeMemoryLocalStrategy());
-			config.setFilehandlesInput(inputNum, this.defaultMaxFan);
-			config.setSpillingThresholdInput(inputNum, this.defaultSortSpillingThreshold);
-		}
-	}
-
-	private int translateChannel(Channel input, int inputIndex, FlinkVertex targetVertex,
-								TezTaskConfig targetVertexConfig, boolean isBroadcast) throws Exception
-	{
-		final PlanNode inputPlanNode = input.getSource();
-		final Iterator<Channel> allInChannels;
-
-
-		allInChannels = Collections.singletonList(input).iterator();
-
-
-		// check that the type serializer is consistent
-		TypeSerializerFactory<?> typeSerFact = null;
-
-		while (allInChannels.hasNext()) {
-			final Channel inConn = allInChannels.next();
-
-			if (typeSerFact == null) {
-				typeSerFact = inConn.getSerializer();
-			} else if (!typeSerFact.equals(inConn.getSerializer())) {
-				throw new CompilerException("Conflicting types in union operator.");
-			}
-
-			final PlanNode sourceNode = inConn.getSource();
-			FlinkVertex sourceVertex = this.vertices.get(sourceNode);
-			TezTaskConfig sourceVertexConfig = sourceVertex.getConfig(); //TODO ??? need to create a new TezConfig ???
-
-			connectJobVertices(
-					inConn, inputIndex, sourceVertex, sourceVertexConfig, targetVertex, targetVertexConfig, isBroadcast);
-		}
-
-		// the local strategy is added only once. in non-union case that is the actual edge,
-		// in the union case, it is the edge between union and the target node
-		addLocalInfoFromChannelToConfig(input, targetVertexConfig, inputIndex, isBroadcast);
-		return 1;
-	}
-
-	private void connectJobVertices(Channel channel, int inputNumber,
-							final FlinkVertex sourceVertex, final TezTaskConfig sourceConfig,
-							final FlinkVertex targetVertex, final TezTaskConfig targetConfig, boolean isBroadcast)
-			throws CompilerException {
-
-		// -------------- configure the source task's ship strategy strategies in task config --------------
-		final int outputIndex = sourceConfig.getNumOutputs();
-		sourceConfig.addOutputShipStrategy(channel.getShipStrategy());
-		if (outputIndex == 0) {
-			sourceConfig.setOutputSerializer(channel.getSerializer());
-		}
-		if (channel.getShipStrategyComparator() != null) {
-			sourceConfig.setOutputComparator(channel.getShipStrategyComparator(), outputIndex);
-		}
-
-		if (channel.getShipStrategy() == ShipStrategyType.PARTITION_RANGE) {
-
-			final DataDistribution dataDistribution = channel.getDataDistribution();
-			if(dataDistribution != null) {
-				sourceConfig.setOutputDataDistribution(dataDistribution, outputIndex);
-			} else {
-				throw new RuntimeException("Range partitioning requires data distribution");
-				// TODO: inject code and configuration for automatic histogram generation
-			}
-		}
-
-		// ---------------- configure the receiver -------------------
-		if (isBroadcast) {
-			targetConfig.addBroadcastInputToGroup(inputNumber);
-		} else {
-			targetConfig.addInputToGroup(inputNumber);
-		}
-
-		//----------------- connect source and target with edge ------------------------------
-
-		FlinkEdge edge;
-		ShipStrategyType shipStrategy = channel.getShipStrategy();
-		TypeSerializer<?> serializer = channel.getSerializer().getSerializer();
-		if ((shipStrategy == ShipStrategyType.FORWARD) || (shipStrategy == ShipStrategyType.NONE)) {
-			edge = new FlinkForwardEdge(sourceVertex, targetVertex, serializer);
-			// For forward edges, create as many tasks in upstream operator as in source operator
-			targetVertex.setParallelism(sourceVertex.getParallelism());
-		}
-		else if (shipStrategy == ShipStrategyType.BROADCAST) {
-			edge = new FlinkBroadcastEdge(sourceVertex, targetVertex, serializer);
-		}
-		else if (shipStrategy == ShipStrategyType.PARTITION_HASH) {
-			edge = new FlinkPartitionEdge(sourceVertex, targetVertex, serializer);
-		}
-		else {
-			throw new CompilerException("Ship strategy between nodes " + sourceVertex.getVertex().getName() + " and " + targetVertex.getVertex().getName() + " currently not supported");
-		}
-
-		// Tez-specific bookkeeping
-		// TODO: This probably will not work for vertices with multiple outputs
-		sourceVertex.addNumberOfSubTasksInOutput(targetVertex.getParallelism(), outputIndex);
-		targetVertex.addInput(sourceVertex, inputNumber);
-
-
-		edges.add(edge);
-	}
-
-	private void addLocalInfoFromChannelToConfig(Channel channel, TaskConfig config, int inputNum, boolean isBroadcastChannel) {
-		// serializer
-		if (isBroadcastChannel) {
-			config.setBroadcastInputSerializer(channel.getSerializer(), inputNum);
-
-			if (channel.getLocalStrategy() != LocalStrategy.NONE || (channel.getTempMode() != null && channel.getTempMode() != TempMode.NONE)) {
-				throw new CompilerException("Found local strategy or temp mode on a broadcast variable channel.");
-			} else {
-				return;
-			}
-		} else {
-			config.setInputSerializer(channel.getSerializer(), inputNum);
-		}
-
-		// local strategy
-		if (channel.getLocalStrategy() != LocalStrategy.NONE) {
-			config.setInputLocalStrategy(inputNum, channel.getLocalStrategy());
-			if (channel.getLocalStrategyComparator() != null) {
-				config.setInputComparator(channel.getLocalStrategyComparator(), inputNum);
-			}
-		}
-
-		assignLocalStrategyResources(channel, config, inputNum);
-
-		// materialization / caching
-		if (channel.getTempMode() != null) {
-			final TempMode tm = channel.getTempMode();
-
-			boolean needsMemory = false;
-			if (tm.breaksPipeline()) {
-				config.setInputAsynchronouslyMaterialized(inputNum, true);
-				needsMemory = true;
-			}
-			if (tm.isCached()) {
-				config.setInputCached(inputNum, true);
-				needsMemory = true;
-			}
-
-			if (needsMemory) {
-				// sanity check
-				if (tm == null || tm == TempMode.NONE || channel.getRelativeTempMemory() <= 0) {
-					throw new CompilerException("Bug in compiler: Inconsistent description of input materialization.");
-				}
-				config.setRelativeInputMaterializationMemory(inputNum, channel.getRelativeTempMemory());
-			}
-		}
-	}
-
-	private boolean containsSelfJoins () {
-		for (FlinkVertex v : vertices.values()) {
-			ArrayList<FlinkVertex> predecessors = new ArrayList<FlinkVertex>();
-			for (FlinkEdge e : edges) {
-				if (e.target == v) {
-					if (predecessors.contains(e.source)) {
-						return true;
-					}
-					predecessors.add(e.source);
-				}
-			}
-		}
-		return false;
-	}
-
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/8b8dfc16/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/examples/ConnectedComponentsStep.java
----------------------------------------------------------------------
diff --git a/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/examples/ConnectedComponentsStep.java b/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/examples/ConnectedComponentsStep.java
deleted file mode 100644
index 707fd47..0000000
--- a/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/examples/ConnectedComponentsStep.java
+++ /dev/null
@@ -1,203 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.tez.examples;
-
-import org.apache.flink.api.common.ProgramDescription;
-import org.apache.flink.api.common.functions.FlatJoinFunction;
-import org.apache.flink.api.common.functions.FlatMapFunction;
-import org.apache.flink.api.common.functions.JoinFunction;
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.aggregation.Aggregations;
-import org.apache.flink.api.java.functions.FunctionAnnotation.ForwardedFields;
-import org.apache.flink.api.java.functions.FunctionAnnotation.ForwardedFieldsFirst;
-import org.apache.flink.api.java.functions.FunctionAnnotation.ForwardedFieldsSecond;
-import org.apache.flink.api.java.tuple.Tuple1;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.examples.java.graph.util.ConnectedComponentsData;
-import org.apache.flink.util.Collector;
-
-
-public class ConnectedComponentsStep implements ProgramDescription {
-
-	// *************************************************************************
-	//     PROGRAM
-	// *************************************************************************
-
-	public static void main(String... args) throws Exception {
-
-		if(!parseParameters(args)) {
-			return;
-		}
-
-		// set up execution environment
-		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
-		// read vertex and edge data
-		DataSet<Long> vertices = getVertexDataSet(env);
-		DataSet<Tuple2<Long, Long>> edges = getEdgeDataSet(env).flatMap(new UndirectEdge());
-
-		// assign the initial components (equal to the vertex id)
-		DataSet<Tuple2<Long, Long>> verticesWithInitialId = vertices.map(new DuplicateValue<Long>());
-
-		DataSet<Tuple2<Long,Long>> nextComponenets = verticesWithInitialId
-				.join(edges)
-				.where(0).equalTo(0)
-				.with(new NeighborWithComponentIDJoin())
-				.groupBy(0).aggregate(Aggregations.MIN, 1)
-				.join(verticesWithInitialId)
-				.where(0).equalTo(0)
-				.with(new ComponentIdFilter());
-
-
-		// emit result
-		if(fileOutput) {
-			nextComponenets.writeAsCsv(outputPath, "\n", " ");
-		} else {
-			nextComponenets.print();
-		}
-
-		// execute program
-		env.execute("Connected Components Example");
-	}
-
-	// *************************************************************************
-	//     USER FUNCTIONS
-	// *************************************************************************
-
-	/**
-	 * Function that turns a value into a 2-tuple where both fields are that value.
-	 */
-	@ForwardedFields("*->f0")
-	public static final class DuplicateValue<T> implements MapFunction<T, Tuple2<T, T>> {
-
-		@Override
-		public Tuple2<T, T> map(T vertex) {
-			return new Tuple2<T, T>(vertex, vertex);
-		}
-	}
-
-	/**
-	 * Undirected edges by emitting for each input edge the input edges itself and an inverted version.
-	 */
-	public static final class UndirectEdge implements FlatMapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> {
-		Tuple2<Long, Long> invertedEdge = new Tuple2<Long, Long>();
-
-		@Override
-		public void flatMap(Tuple2<Long, Long> edge, Collector<Tuple2<Long, Long>> out) {
-			invertedEdge.f0 = edge.f1;
-			invertedEdge.f1 = edge.f0;
-			out.collect(edge);
-			out.collect(invertedEdge);
-		}
-	}
-
-	/**
-	 * UDF that joins a (Vertex-ID, Component-ID) pair that represents the current component that
-	 * a vertex is associated with, with a (Source-Vertex-ID, Target-VertexID) edge. The function
-	 * produces a (Target-vertex-ID, Component-ID) pair.
-	 */
-	@ForwardedFieldsFirst("f1->f1")
-	@ForwardedFieldsSecond("f1->f0")
-	public static final class NeighborWithComponentIDJoin implements JoinFunction<Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>> {
-
-		@Override
-		public Tuple2<Long, Long> join(Tuple2<Long, Long> vertexWithComponent, Tuple2<Long, Long> edge) {
-			return new Tuple2<Long, Long>(edge.f1, vertexWithComponent.f1);
-		}
-	}
-
-
-
-	@ForwardedFieldsFirst("*")
-	public static final class ComponentIdFilter implements FlatJoinFunction<Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>> {
-
-		@Override
-		public void join(Tuple2<Long, Long> candidate, Tuple2<Long, Long> old, Collector<Tuple2<Long, Long>> out) {
-			if (candidate.f1 < old.f1) {
-				out.collect(candidate);
-			}
-		}
-	}
-
-
-
-	@Override
-	public String getDescription() {
-		return "Parameters: <vertices-path> <edges-path> <result-path> <max-number-of-iterations>";
-	}
-
-	// *************************************************************************
-	//     UTIL METHODS
-	// *************************************************************************
-
-	private static boolean fileOutput = false;
-	private static String verticesPath = null;
-	private static String edgesPath = null;
-	private static String outputPath = null;
-	private static int maxIterations = 10;
-
-	private static boolean parseParameters(String[] programArguments) {
-
-		if(programArguments.length > 0) {
-			// parse input arguments
-			fileOutput = true;
-			if(programArguments.length == 4) {
-				verticesPath = programArguments[0];
-				edgesPath = programArguments[1];
-				outputPath = programArguments[2];
-				maxIterations = Integer.parseInt(programArguments[3]);
-			} else {
-				System.err.println("Usage: ConnectedComponents <vertices path> <edges path> <result path> <max number of iterations>");
-				return false;
-			}
-		} else {
-			System.out.println("Executing Connected Components example with default parameters and built-in default data.");
-			System.out.println("  Provide parameters to read input data from files.");
-			System.out.println("  See the documentation for the correct format of input files.");
-			System.out.println("  Usage: ConnectedComponents <vertices path> <edges path> <result path> <max number of iterations>");
-		}
-		return true;
-	}
-
-	private static DataSet<Long> getVertexDataSet(ExecutionEnvironment env) {
-
-		if(fileOutput) {
-			return env.readCsvFile(verticesPath).types(Long.class)
-					.map(
-							new MapFunction<Tuple1<Long>, Long>() {
-								public Long map(Tuple1<Long> value) { return value.f0; }
-							});
-		} else {
-			return ConnectedComponentsData.getDefaultVertexDataSet(env);
-		}
-	}
-
-	private static DataSet<Tuple2<Long, Long>> getEdgeDataSet(ExecutionEnvironment env) {
-
-		if(fileOutput) {
-			return env.readCsvFile(edgesPath).fieldDelimiter(' ').types(Long.class, Long.class);
-		} else {
-			return ConnectedComponentsData.getDefaultEdgeDataSet(env);
-		}
-	}
-
-
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/8b8dfc16/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/examples/ExampleDriver.java
----------------------------------------------------------------------
diff --git a/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/examples/ExampleDriver.java b/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/examples/ExampleDriver.java
deleted file mode 100644
index c65fb69..0000000
--- a/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/examples/ExampleDriver.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.tez.examples;
-
-import org.apache.hadoop.util.ProgramDriver;
-import org.apache.tez.common.counters.TezCounters;
-import org.apache.tez.dag.api.TezException;
-import org.apache.tez.dag.api.client.DAGClient;
-import org.apache.tez.dag.api.client.DAGStatus;
-import org.apache.tez.dag.api.client.Progress;
-import org.apache.tez.dag.api.client.StatusGetOpts;
-import org.apache.tez.dag.api.client.VertexStatus;
-
-import java.io.IOException;
-import java.text.DecimalFormat;
-import java.util.EnumSet;
-import java.util.Set;
-
-public class ExampleDriver {
-
-	private static final DecimalFormat formatter = new DecimalFormat("###.##%");
-
-	public static void main(String [] args){
-		int exitCode = -1;
-		ProgramDriver pgd = new ProgramDriver();
-		try {
-			pgd.addClass("wc", WordCount.class,
-					"Wordcount");
-			pgd.addClass("tpch3", TPCHQuery3.class,
-					"Modified TPC-H 3 query");
-			pgd.addClass("tc", TransitiveClosureNaiveStep.class,
-					"One step of transitive closure");
-			pgd.addClass("pr", PageRankBasicStep.class,
-					"One step of PageRank");
-			pgd.addClass("cc", ConnectedComponentsStep.class,
-					"One step of connected components");
-			exitCode = pgd.run(args);
-		} catch(Throwable e){
-			e.printStackTrace();
-		}
-		System.exit(exitCode);
-	}
-
-	public static void printDAGStatus(DAGClient dagClient, String[] vertexNames)
-			throws IOException, TezException {
-		printDAGStatus(dagClient, vertexNames, false, false);
-	}
-
-	public static void printDAGStatus(DAGClient dagClient, String[] vertexNames, boolean displayDAGCounters, boolean displayVertexCounters)
-			throws IOException, TezException {
-		Set<StatusGetOpts> opts = EnumSet.of(StatusGetOpts.GET_COUNTERS);
-		DAGStatus dagStatus = dagClient.getDAGStatus(
-				(displayDAGCounters ? opts : null));
-		Progress progress = dagStatus.getDAGProgress();
-		double vProgressFloat = 0.0f;
-		if (progress != null) {
-			System.out.println("");
-			System.out.println("DAG: State: "
-					+ dagStatus.getState()
-					+ " Progress: "
-					+ (progress.getTotalTaskCount() < 0 ? formatter.format(0.0f) :
-					formatter.format((double)(progress.getSucceededTaskCount())
-							/progress.getTotalTaskCount())));
-			for (String vertexName : vertexNames) {
-				VertexStatus vStatus = dagClient.getVertexStatus(vertexName,
-						(displayVertexCounters ? opts : null));
-				if (vStatus == null) {
-					System.out.println("Could not retrieve status for vertex: "
-							+ vertexName);
-					continue;
-				}
-				Progress vProgress = vStatus.getProgress();
-				if (vProgress != null) {
-					vProgressFloat = 0.0f;
-					if (vProgress.getTotalTaskCount() == 0) {
-						vProgressFloat = 1.0f;
-					} else if (vProgress.getTotalTaskCount() > 0) {
-						vProgressFloat = (double)vProgress.getSucceededTaskCount()
-								/vProgress.getTotalTaskCount();
-					}
-					System.out.println("VertexStatus:"
-							+ " VertexName: "
-							+ (vertexName.equals("ivertex1") ? "intermediate-reducer"
-							: vertexName)
-							+ " Progress: " + formatter.format(vProgressFloat));
-				}
-				if (displayVertexCounters) {
-					TezCounters counters = vStatus.getVertexCounters();
-					if (counters != null) {
-						System.out.println("Vertex Counters for " + vertexName + ": "
-								+ counters);
-					}
-				}
-			}
-		}
-		if (displayDAGCounters) {
-			TezCounters counters = dagStatus.getDAGCounters();
-			if (counters != null) {
-				System.out.println("DAG Counters: " + counters);
-			}
-		}
-	}
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/8b8dfc16/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/examples/PageRankBasicStep.java
----------------------------------------------------------------------
diff --git a/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/examples/PageRankBasicStep.java b/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/examples/PageRankBasicStep.java
deleted file mode 100644
index 031893d..0000000
--- a/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/examples/PageRankBasicStep.java
+++ /dev/null
@@ -1,241 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.tez.examples;
-
-
-import org.apache.flink.api.common.functions.FilterFunction;
-import org.apache.flink.api.common.functions.FlatMapFunction;
-import org.apache.flink.api.common.functions.GroupReduceFunction;
-import org.apache.flink.api.common.functions.MapFunction;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.functions.FunctionAnnotation.ForwardedFields;
-import org.apache.flink.api.java.tuple.Tuple1;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.examples.java.graph.util.PageRankData;
-import org.apache.flink.util.Collector;
-
-import java.util.ArrayList;
-
-import static org.apache.flink.api.java.aggregation.Aggregations.SUM;
-
-public class PageRankBasicStep {
-
-	private static final double DAMPENING_FACTOR = 0.85;
-	private static final double EPSILON = 0.0001;
-
-	// *************************************************************************
-	//     PROGRAM
-	// *************************************************************************
-
-	public static void main(String[] args) throws Exception {
-
-		if(!parseParameters(args)) {
-			return;
-		}
-
-		// set up execution environment
-		final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
-		// get input data
-		DataSet<Long> pagesInput = getPagesDataSet(env);
-		DataSet<Tuple2<Long, Long>> linksInput = getLinksDataSet(env);
-
-		// assign initial rank to pages
-		DataSet<Tuple2<Long, Double>> pagesWithRanks = pagesInput.
-				map(new RankAssigner((1.0d / numPages)));
-
-		// build adjacency list from link input
-		DataSet<Tuple2<Long, Long[]>> adjacencyListInput =
-				linksInput.groupBy(0).reduceGroup(new BuildOutgoingEdgeList());
-
-		DataSet<Tuple2<Long, Double>> newRanks = pagesWithRanks
-				.join(adjacencyListInput).where(0).equalTo(0)
-				.flatMap(new JoinVertexWithEdgesMatch())
-				.groupBy(0).aggregate(SUM, 1)
-				.map(new Dampener(DAMPENING_FACTOR, numPages));
-
-
-		// emit result
-		if(fileOutput) {
-			newRanks.writeAsCsv(outputPath, "\n", " ");
-		} else {
-			newRanks.print();
-		}
-
-		// execute program
-		env.execute("Basic Page Rank Example");
-
-	}
-
-	// *************************************************************************
-	//     USER FUNCTIONS
-	// *************************************************************************
-
-	/**
-	 * A map function that assigns an initial rank to all pages.
-	 */
-	public static final class RankAssigner implements MapFunction<Long, Tuple2<Long, Double>> {
-		Tuple2<Long, Double> outPageWithRank;
-
-		public RankAssigner(double rank) {
-			this.outPageWithRank = new Tuple2<Long, Double>(-1l, rank);
-		}
-
-		@Override
-		public Tuple2<Long, Double> map(Long page) {
-			outPageWithRank.f0 = page;
-			return outPageWithRank;
-		}
-	}
-
-	/**
-	 * A reduce function that takes a sequence of edges and builds the adjacency list for the vertex where the edges
-	 * originate. Run as a pre-processing step.
-	 */
-	@ForwardedFields("0")
-	public static final class BuildOutgoingEdgeList implements GroupReduceFunction<Tuple2<Long, Long>, Tuple2<Long, Long[]>> {
-
-		private final ArrayList<Long> neighbors = new ArrayList<Long>();
-
-		@Override
-		public void reduce(Iterable<Tuple2<Long, Long>> values, Collector<Tuple2<Long, Long[]>> out) {
-			neighbors.clear();
-			Long id = 0L;
-
-			for (Tuple2<Long, Long> n : values) {
-				id = n.f0;
-				neighbors.add(n.f1);
-			}
-			out.collect(new Tuple2<Long, Long[]>(id, neighbors.toArray(new Long[neighbors.size()])));
-		}
-	}
-
-	/**
-	 * Join function that distributes a fraction of a vertex's rank to all neighbors.
-	 */
-	public static final class JoinVertexWithEdgesMatch implements FlatMapFunction<Tuple2<Tuple2<Long, Double>, Tuple2<Long, Long[]>>, Tuple2<Long, Double>> {
-
-		@Override
-		public void flatMap(Tuple2<Tuple2<Long, Double>, Tuple2<Long, Long[]>> value, Collector<Tuple2<Long, Double>> out){
-			Long[] neigbors = value.f1.f1;
-			double rank = value.f0.f1;
-			double rankToDistribute = rank / ((double) neigbors.length);
-
-			for (int i = 0; i < neigbors.length; i++) {
-				out.collect(new Tuple2<Long, Double>(neigbors[i], rankToDistribute));
-			}
-		}
-	}
-
-	/**
-	 * The function that applies the page rank dampening formula
-	 */
-	@ForwardedFields("0")
-	public static final class Dampener implements MapFunction<Tuple2<Long,Double>, Tuple2<Long,Double>> {
-
-		private final double dampening;
-		private final double randomJump;
-
-		public Dampener(double dampening, double numVertices) {
-			this.dampening = dampening;
-			this.randomJump = (1 - dampening) / numVertices;
-		}
-
-		@Override
-		public Tuple2<Long, Double> map(Tuple2<Long, Double> value) {
-			value.f1 = (value.f1 * dampening) + randomJump;
-			return value;
-		}
-	}
-
-	/**
-	 * Filter that filters vertices where the rank difference is below a threshold.
-	 */
-	public static final class EpsilonFilter implements FilterFunction<Tuple2<Tuple2<Long, Double>, Tuple2<Long, Double>>> {
-
-		@Override
-		public boolean filter(Tuple2<Tuple2<Long, Double>, Tuple2<Long, Double>> value) {
-			return Math.abs(value.f0.f1 - value.f1.f1) > EPSILON;
-		}
-	}
-
-	// *************************************************************************
-	//     UTIL METHODS
-	// *************************************************************************
-
-	private static boolean fileOutput = false;
-	private static String pagesInputPath = null;
-	private static String linksInputPath = null;
-	private static String outputPath = null;
-	private static long numPages = 0;
-	private static int maxIterations = 10;
-
-	private static boolean parseParameters(String[] args) {
-
-		if(args.length > 0) {
-			if(args.length == 5) {
-				fileOutput = true;
-				pagesInputPath = args[0];
-				linksInputPath = args[1];
-				outputPath = args[2];
-				numPages = Integer.parseInt(args[3]);
-				maxIterations = Integer.parseInt(args[4]);
-			} else {
-				System.err.println("Usage: PageRankBasic <pages path> <links path> <output path> <num pages> <num iterations>");
-				return false;
-			}
-		} else {
-			System.out.println("Executing PageRank Basic example with default parameters and built-in default data.");
-			System.out.println("  Provide parameters to read input data from files.");
-			System.out.println("  See the documentation for the correct format of input files.");
-			System.out.println("  Usage: PageRankBasic <pages path> <links path> <output path> <num pages> <num iterations>");
-
-			numPages = PageRankData.getNumberOfPages();
-		}
-		return true;
-	}
-
-	private static DataSet<Long> getPagesDataSet(ExecutionEnvironment env) {
-		if(fileOutput) {
-			return env
-					.readCsvFile(pagesInputPath)
-					.fieldDelimiter(' ')
-					.lineDelimiter("\n")
-					.types(Long.class)
-					.map(new MapFunction<Tuple1<Long>, Long>() {
-						@Override
-						public Long map(Tuple1<Long> v) { return v.f0; }
-					});
-		} else {
-			return PageRankData.getDefaultPagesDataSet(env);
-		}
-	}
-
-	private static DataSet<Tuple2<Long, Long>> getLinksDataSet(ExecutionEnvironment env) {
-		if(fileOutput) {
-			return env.readCsvFile(linksInputPath)
-					.fieldDelimiter(' ')
-					.lineDelimiter("\n")
-					.types(Long.class, Long.class);
-		} else {
-			return PageRankData.getDefaultEdgeDataSet(env);
-		}
-	}
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/8b8dfc16/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/examples/TPCHQuery3.java
----------------------------------------------------------------------
diff --git a/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/examples/TPCHQuery3.java b/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/examples/TPCHQuery3.java
deleted file mode 100644
index d61f80e..0000000
--- a/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/examples/TPCHQuery3.java
+++ /dev/null
@@ -1,224 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.tez.examples;
-
-import org.apache.flink.api.common.functions.FilterFunction;
-import org.apache.flink.api.common.functions.JoinFunction;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.aggregation.Aggregations;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.api.java.tuple.Tuple4;
-import org.apache.flink.tez.client.RemoteTezEnvironment;
-
-import java.text.DateFormat;
-import java.text.ParseException;
-import java.text.SimpleDateFormat;
-import java.util.Date;
-
-public class TPCHQuery3 {
-
-	// *************************************************************************
-	//     PROGRAM
-	// *************************************************************************
-
-	public static void main(String[] args) throws Exception {
-
-		if(!parseParameters(args)) {
-			return;
-		}
-
-		final RemoteTezEnvironment env = RemoteTezEnvironment.create();
-		env.setParallelism(400);
-
-
-		// get input data
-		DataSet<Lineitem> lineitems = getLineitemDataSet(env);
-		DataSet<Order> orders = getOrdersDataSet(env);
-		DataSet<Customer> customers = getCustomerDataSet(env);
-
-		// Filter market segment "AUTOMOBILE"
-		customers = customers.filter(
-				new FilterFunction<Customer>() {
-					@Override
-					public boolean filter(Customer c) {
-						return c.getMktsegment().equals("AUTOMOBILE");
-					}
-				});
-
-		// Filter all Orders with o_orderdate < 12.03.1995
-		orders = orders.filter(
-				new FilterFunction<Order>() {
-					private final DateFormat format = new SimpleDateFormat("yyyy-MM-dd");
-					private final Date date = format.parse("1995-03-12");
-
-					@Override
-					public boolean filter(Order o) throws ParseException {
-						return format.parse(o.getOrderdate()).before(date);
-					}
-				});
-
-		// Filter all Lineitems with l_shipdate > 12.03.1995
-		lineitems = lineitems.filter(
-				new FilterFunction<Lineitem>() {
-					private final DateFormat format = new SimpleDateFormat("yyyy-MM-dd");
-					private final Date date = format.parse("1995-03-12");
-
-					@Override
-					public boolean filter(Lineitem l) throws ParseException {
-						return format.parse(l.getShipdate()).after(date);
-					}
-				});
-
-		// Join customers with orders and package them into a ShippingPriorityItem
-		DataSet<ShippingPriorityItem> customerWithOrders =
-				customers.join(orders).where(0).equalTo(1)
-						.with(
-								new JoinFunction<Customer, Order, ShippingPriorityItem>() {
-									@Override
-									public ShippingPriorityItem join(Customer c, Order o) {
-										return new ShippingPriorityItem(o.getOrderKey(), 0.0, o.getOrderdate(),
-												o.getShippriority());
-									}
-								});
-
-		// Join the last join result with Lineitems
-		DataSet<ShippingPriorityItem> result =
-				customerWithOrders.join(lineitems).where(0).equalTo(0)
-						.with(
-								new JoinFunction<ShippingPriorityItem, Lineitem, ShippingPriorityItem>() {
-									@Override
-									public ShippingPriorityItem join(ShippingPriorityItem i, Lineitem l) {
-										i.setRevenue(l.getExtendedprice() * (1 - l.getDiscount()));
-										return i;
-									}
-								})
-								// Group by l_orderkey, o_orderdate and o_shippriority and compute revenue sum
-						.groupBy(0, 2, 3)
-						.aggregate(Aggregations.SUM, 1);
-
-		// emit result
-		result.writeAsCsv(outputPath, "\n", "|");
-
-		// execute program
-		env.registerMainClass(TPCHQuery3.class);
-		env.execute("TPCH Query 3 Example");
-
-	}
-
-	// *************************************************************************
-	//     DATA TYPES
-	// *************************************************************************
-
-	public static class Lineitem extends Tuple4<Integer, Double, Double, String> {
-
-		public Integer getOrderkey() { return this.f0; }
-		public Double getDiscount() { return this.f2; }
-		public Double getExtendedprice() { return this.f1; }
-		public String getShipdate() { return this.f3; }
-	}
-
-	public static class Customer extends Tuple2<Integer, String> {
-
-		public Integer getCustKey() { return this.f0; }
-		public String getMktsegment() { return this.f1; }
-	}
-
-	public static class Order extends Tuple4<Integer, Integer, String, Integer> {
-
-		public Integer getOrderKey() { return this.f0; }
-		public Integer getCustKey() { return this.f1; }
-		public String getOrderdate() { return this.f2; }
-		public Integer getShippriority() { return this.f3; }
-	}
-
-	public static class ShippingPriorityItem extends Tuple4<Integer, Double, String, Integer> {
-
-		public ShippingPriorityItem() { }
-
-		public ShippingPriorityItem(Integer o_orderkey, Double revenue,
-									String o_orderdate, Integer o_shippriority) {
-			this.f0 = o_orderkey;
-			this.f1 = revenue;
-			this.f2 = o_orderdate;
-			this.f3 = o_shippriority;
-		}
-
-		public Integer getOrderkey() { return this.f0; }
-		public void setOrderkey(Integer orderkey) { this.f0 = orderkey; }
-		public Double getRevenue() { return this.f1; }
-		public void setRevenue(Double revenue) { this.f1 = revenue; }
-
-		public String getOrderdate() { return this.f2; }
-		public Integer getShippriority() { return this.f3; }
-	}
-
-	// *************************************************************************
-	//     UTIL METHODS
-	// *************************************************************************
-
-	private static String lineitemPath;
-	private static String customerPath;
-	private static String ordersPath;
-	private static String outputPath;
-
-	private static boolean parseParameters(String[] programArguments) {
-
-		if(programArguments.length > 0) {
-			if(programArguments.length == 4) {
-				lineitemPath = programArguments[0];
-				customerPath = programArguments[1];
-				ordersPath = programArguments[2];
-				outputPath = programArguments[3];
-			} else {
-				System.err.println("Usage: TPCHQuery3 <lineitem-csv path> <customer-csv path> <orders-csv path> <result path>");
-				return false;
-			}
-		} else {
-			System.err.println("This program expects data from the TPC-H benchmark as input data.\n" +
-					"  Due to legal restrictions, we can not ship generated data.\n" +
-					"  You can find the TPC-H data generator at http://www.tpc.org/tpch/.\n" +
-					"  Usage: TPCHQuery3 <lineitem-csv path> <customer-csv path> <orders-csv path> <result path>");
-			return false;
-		}
-		return true;
-	}
-
-	private static DataSet<Lineitem> getLineitemDataSet(ExecutionEnvironment env) {
-		return env.readCsvFile(lineitemPath)
-				.fieldDelimiter('|')
-				.includeFields("1000011000100000")
-				.tupleType(Lineitem.class);
-	}
-
-	private static DataSet<Customer> getCustomerDataSet(ExecutionEnvironment env) {
-		return env.readCsvFile(customerPath)
-				.fieldDelimiter('|')
-				.includeFields("10000010")
-				.tupleType(Customer.class);
-	}
-
-	private static DataSet<Order> getOrdersDataSet(ExecutionEnvironment env) {
-		return env.readCsvFile(ordersPath)
-				.fieldDelimiter('|')
-				.includeFields("110010010")
-				.tupleType(Order.class);
-	}
-
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/8b8dfc16/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/examples/TransitiveClosureNaiveStep.java
----------------------------------------------------------------------
diff --git a/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/examples/TransitiveClosureNaiveStep.java b/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/examples/TransitiveClosureNaiveStep.java
deleted file mode 100644
index b014c3e..0000000
--- a/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/examples/TransitiveClosureNaiveStep.java
+++ /dev/null
@@ -1,135 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.tez.examples;
-
-import org.apache.flink.api.common.ProgramDescription;
-import org.apache.flink.api.common.functions.GroupReduceFunction;
-import org.apache.flink.api.common.functions.JoinFunction;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.examples.java.graph.util.ConnectedComponentsData;
-import org.apache.flink.util.Collector;
-
-
-/*
- * NOTE:
- * This program is currently supposed to throw a Compiler Exception due to TEZ-1190
- */
-
-public class TransitiveClosureNaiveStep implements ProgramDescription {
-
-
-	public static void main (String... args) throws Exception{
-
-		if (!parseParameters(args)) {
-			return;
-		}
-
-		// set up execution environment
-		ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment();
-
-		DataSet<Tuple2<Long, Long>> edges = getEdgeDataSet(env);
-
-		DataSet<Tuple2<Long,Long>> nextPaths = edges
-				.join(edges)
-				.where(1)
-				.equalTo(0)
-				.with(new JoinFunction<Tuple2<Long, Long>, Tuple2<Long, Long>, Tuple2<Long, Long>>() {
-					@Override
-					/**
-					 left: Path (z,x) - x is reachable by z
-					 right: Edge (x,y) - edge x-->y exists
-					 out: Path (z,y) - y is reachable by z
-					 */
-					public Tuple2<Long, Long> join(Tuple2<Long, Long> left, Tuple2<Long, Long> right) throws Exception {
-						return new Tuple2<Long, Long>(
-								new Long(left.f0),
-								new Long(right.f1));
-					}
-				})
-				.union(edges)
-				.groupBy(0, 1)
-				.reduceGroup(new GroupReduceFunction<Tuple2<Long, Long>, Tuple2<Long, Long>>() {
-					@Override
-					public void reduce(Iterable<Tuple2<Long, Long>> values, Collector<Tuple2<Long, Long>> out) throws Exception {
-						out.collect(values.iterator().next());
-					}
-				});
-
-		// emit result
-		if (fileOutput) {
-			nextPaths.writeAsCsv(outputPath, "\n", " ");
-		} else {
-			nextPaths.print();
-		}
-
-		// execute program
-		env.execute("Transitive Closure Example");
-
-	}
-
-	@Override
-	public String getDescription() {
-		return "Parameters: <edges-path> <result-path> <max-number-of-iterations>";
-	}
-
-	// *************************************************************************
-	//     UTIL METHODS
-	// *************************************************************************
-
-	private static boolean fileOutput = false;
-	private static String edgesPath = null;
-	private static String outputPath = null;
-	private static int maxIterations = 10;
-
-	private static boolean parseParameters(String[] programArguments) {
-
-		if (programArguments.length > 0) {
-			// parse input arguments
-			fileOutput = true;
-			if (programArguments.length == 3) {
-				edgesPath = programArguments[0];
-				outputPath = programArguments[1];
-				maxIterations = Integer.parseInt(programArguments[2]);
-			} else {
-				System.err.println("Usage: TransitiveClosure <edges path> <result path> <max number of iterations>");
-				return false;
-			}
-		} else {
-			System.out.println("Executing TransitiveClosure example with default parameters and built-in default data.");
-			System.out.println("  Provide parameters to read input data from files.");
-			System.out.println("  See the documentation for the correct format of input files.");
-			System.out.println("  Usage: TransitiveClosure <edges path> <result path> <max number of iterations>");
-		}
-		return true;
-	}
-
-
-	private static DataSet<Tuple2<Long, Long>> getEdgeDataSet(ExecutionEnvironment env) {
-
-		if(fileOutput) {
-			return env.readCsvFile(edgesPath).fieldDelimiter(' ').types(Long.class, Long.class);
-		} else {
-			return ConnectedComponentsData.getDefaultEdgeDataSet(env);
-		}
-	}
-
-}
-

http://git-wip-us.apache.org/repos/asf/flink/blob/8b8dfc16/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/examples/WordCount.java
----------------------------------------------------------------------
diff --git a/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/examples/WordCount.java b/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/examples/WordCount.java
deleted file mode 100644
index e758156..0000000
--- a/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/examples/WordCount.java
+++ /dev/null
@@ -1,129 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.tez.examples;
-
-import org.apache.flink.api.common.functions.FlatMapFunction;
-import org.apache.flink.api.java.DataSet;
-import org.apache.flink.api.java.ExecutionEnvironment;
-import org.apache.flink.api.java.tuple.Tuple2;
-import org.apache.flink.examples.java.wordcount.util.WordCountData;
-import org.apache.flink.tez.client.RemoteTezEnvironment;
-import org.apache.flink.util.Collector;
-
-public class WordCount {
-
-	// *************************************************************************
-	//     PROGRAM
-	// *************************************************************************
-
-	public static void main(String[] args) throws Exception {
-
-		if(!parseParameters(args)) {
-			return;
-		}
-
-		// set up the execution environment
-		final RemoteTezEnvironment env = RemoteTezEnvironment.create();
-		env.setParallelism(8);
-
-		// get input data
-		DataSet<String> text = getTextDataSet(env);
-
-		DataSet<Tuple2<String, Integer>> counts =
-				// split up the lines in pairs (2-tuples) containing: (word,1)
-				text.flatMap(new Tokenizer())
-						// group by the tuple field "0" and sum up tuple field "1"
-						.groupBy(0)
-						.sum(1);
-
-		// emit result
-		if(fileOutput) {
-			counts.writeAsCsv(outputPath, "\n", " ");
-		} else {
-			counts.print();
-		}
-
-		// execute program
-		env.registerMainClass(WordCount.class);
-		env.execute("WordCount Example");
-	}
-
-	// *************************************************************************
-	//     USER FUNCTIONS
-	// *************************************************************************
-
-	/**
-	 * Implements the string tokenizer that splits sentences into words as a user-defined
-	 * FlatMapFunction. The function takes a line (String) and splits it into
-	 * multiple pairs in the form of "(word,1)" (Tuple2<String, Integer>).
-	 */
-	public static final class Tokenizer implements FlatMapFunction<String, Tuple2<String, Integer>> {
-
-		@Override
-		public void flatMap(String value, Collector<Tuple2<String, Integer>> out) {
-			// normalize and split the line
-			String[] tokens = value.toLowerCase().split("\\W+");
-
-			// emit the pairs
-			for (String token : tokens) {
-				if (token.length() > 0) {
-					out.collect(new Tuple2<String, Integer>(token, 1));
-				}
-			}
-		}
-	}
-
-	// *************************************************************************
-	//     UTIL METHODS
-	// *************************************************************************
-
-	private static boolean fileOutput = false;
-	private static String textPath;
-	private static String outputPath;
-
-	private static boolean parseParameters(String[] args) {
-
-		if(args.length > 0) {
-			// parse input arguments
-			fileOutput = true;
-			if(args.length == 2) {
-				textPath = args[0];
-				outputPath = args[1];
-			} else {
-				System.err.println("Usage: WordCount <text path> <result path>");
-				return false;
-			}
-		} else {
-			System.out.println("Executing WordCount example with built-in default data.");
-			System.out.println("  Provide parameters to read input data from a file.");
-			System.out.println("  Usage: WordCount <text path> <result path>");
-		}
-		return true;
-	}
-
-	private static DataSet<String> getTextDataSet(ExecutionEnvironment env) {
-		if(fileOutput) {
-			// read the text file from given input path
-			return env.readTextFile(textPath);
-		} else {
-			// get default test text data
-			return WordCountData.getDefaultTextLineDataSet(env);
-		}
-	}
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/8b8dfc16/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/runtime/DataSinkProcessor.java
----------------------------------------------------------------------
diff --git a/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/runtime/DataSinkProcessor.java b/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/runtime/DataSinkProcessor.java
deleted file mode 100644
index 8011d21..0000000
--- a/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/runtime/DataSinkProcessor.java
+++ /dev/null
@@ -1,228 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.tez.runtime;
-
-
-import com.google.common.base.Preconditions;
-import org.apache.flink.api.common.io.OutputFormat;
-import org.apache.flink.api.common.typeutils.TypeComparatorFactory;
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
-import org.apache.flink.runtime.jobgraph.tasks.AbstractInvokable;
-import org.apache.flink.runtime.operators.sort.UnilateralSortMerger;
-import org.apache.flink.runtime.operators.util.CloseableInputProvider;
-import org.apache.flink.tez.runtime.input.TezReaderIterator;
-import org.apache.flink.tez.util.DummyInvokable;
-import org.apache.flink.tez.util.EncodingUtils;
-import org.apache.flink.util.MutableObjectIterator;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.tez.common.TezUtils;
-import org.apache.tez.dag.api.UserPayload;
-import org.apache.tez.runtime.api.AbstractLogicalIOProcessor;
-import org.apache.tez.runtime.api.Event;
-import org.apache.tez.runtime.api.LogicalInput;
-import org.apache.tez.runtime.api.LogicalOutput;
-import org.apache.tez.runtime.api.ProcessorContext;
-import org.apache.tez.runtime.library.api.KeyValueReader;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-public class DataSinkProcessor<IT> extends AbstractLogicalIOProcessor {
-
-	// Tez stuff
-	private TezTaskConfig config;
-	protected Map<String, LogicalInput> inputs;
-	private List<KeyValueReader> readers;
-	private int numInputs;
-	private TezRuntimeEnvironment runtimeEnvironment;
-	AbstractInvokable invokable = new DummyInvokable();
-
-	// Flink stuff
-	private OutputFormat<IT> format;
-	private ClassLoader userCodeClassLoader = this.getClass().getClassLoader();
-	private CloseableInputProvider<IT> localStrategy;
-	// input reader
-	private MutableObjectIterator<IT> reader;
-	// input iterator
-	private MutableObjectIterator<IT> input;
-	private TypeSerializerFactory<IT> inputTypeSerializerFactory;
-
-
-
-
-	public DataSinkProcessor(ProcessorContext context) {
-		super(context);
-	}
-
-	@Override
-	public void initialize() throws Exception {
-		UserPayload payload = getContext().getUserPayload();
-		Configuration conf = TezUtils.createConfFromUserPayload(payload);
-
-		this.config = (TezTaskConfig) EncodingUtils.decodeObjectFromString(conf.get(TezTaskConfig.TEZ_TASK_CONFIG), getClass().getClassLoader());
-		config.setTaskName(getContext().getTaskVertexName());
-
-		this.runtimeEnvironment = new TezRuntimeEnvironment((long) (0.7 * this.getContext().getTotalMemoryAvailableToTask()));
-
-		this.inputTypeSerializerFactory = this.config.getInputSerializer(0, this.userCodeClassLoader);
-
-		initOutputFormat();
-	}
-
-	@Override
-	public void run(Map<String, LogicalInput> inputs, Map<String, LogicalOutput> outputs) throws Exception {
-
-		Preconditions.checkArgument((outputs == null) || (outputs.size() == 0));
-		Preconditions.checkArgument(inputs.size() == 1);
-
-		this.inputs = inputs;
-		this.numInputs = inputs.size();
-		this.readers = new ArrayList<KeyValueReader>(numInputs);
-		if (this.inputs != null) {
-			for (LogicalInput input: this.inputs.values()) {
-				//if (input instanceof AbstractLogicalInput) {
-				//	((AbstractLogicalInput) input).initialize();
-				//}
-				input.start();
-				readers.add((KeyValueReader) input.getReader());
-			}
-		}
-
-		this.reader = new TezReaderIterator<IT>(readers.get(0));
-
-		this.invoke();
-	}
-
-	@Override
-	public void handleEvents(List<Event> processorEvents) {
-
-	}
-
-	@Override
-	public void close() throws Exception {
-		this.runtimeEnvironment.getIOManager().shutdown();
-	}
-
-	private void invoke () {
-		try {
-			// initialize local strategies
-			switch (this.config.getInputLocalStrategy(0)) {
-				case NONE:
-					// nothing to do
-					localStrategy = null;
-					input = reader;
-					break;
-				case SORT:
-					// initialize sort local strategy
-					try {
-						// get type comparator
-						TypeComparatorFactory<IT> compFact = this.config.getInputComparator(0, this.userCodeClassLoader);
-						if (compFact == null) {
-							throw new Exception("Missing comparator factory for local strategy on input " + 0);
-						}
-
-						// initialize sorter
-						UnilateralSortMerger<IT> sorter = new UnilateralSortMerger<IT>(
-								this.runtimeEnvironment.getMemoryManager(),
-								this.runtimeEnvironment.getIOManager(),
-								this.reader, this.invokable, this.inputTypeSerializerFactory, compFact.createComparator(),
-								this.config.getRelativeMemoryInput(0), this.config.getFilehandlesInput(0),
-								this.config.getSpillingThresholdInput(0), false);
-
-						this.localStrategy = sorter;
-						this.input = sorter.getIterator();
-					} catch (Exception e) {
-						throw new RuntimeException("Initializing the input processing failed" +
-								e.getMessage() == null ? "." : ": " + e.getMessage(), e);
-					}
-					break;
-				default:
-					throw new RuntimeException("Invalid local strategy for DataSinkTask");
-			}
-
-			final TypeSerializer<IT> serializer = this.inputTypeSerializerFactory.getSerializer();
-			final MutableObjectIterator<IT> input = this.input;
-			final OutputFormat<IT> format = this.format;
-
-
-			IT record = serializer.createInstance();
-			format.open (this.getContext().getTaskIndex(), this.getContext().getVertexParallelism());
-
-			// work!
-			while (((record = input.next(record)) != null)) {
-				format.writeRecord(record);
-			}
-
-			this.format.close();
-			this.format = null;
-		}
-		catch (IOException e) {
-			e.printStackTrace();
-			throw new RuntimeException();
-		}
-		finally {
-			if (this.format != null) {
-				// close format, if it has not been closed, yet.
-				// This should only be the case if we had a previous error, or were canceled.
-				try {
-					this.format.close();
-				}
-				catch (Throwable t) {
-					//TODO log warning message
-				}
-			}
-			// close local strategy if necessary
-			if (localStrategy != null) {
-				try {
-					this.localStrategy.close();
-				} catch (Throwable t) {
-					//TODO log warning message
-				}
-			}
-		}
-	}
-
-	private void initOutputFormat () {
-		try {
-			this.format = this.config.<OutputFormat<IT>>getStubWrapper(this.userCodeClassLoader).getUserCodeObject(OutputFormat.class, this.userCodeClassLoader);
-
-			// check if the class is a subclass, if the check is required
-			if (!OutputFormat.class.isAssignableFrom(this.format.getClass())) {
-				throw new RuntimeException("The class '" + this.format.getClass().getName() + "' is not a subclass of '" +
-						OutputFormat.class.getName() + "' as is required.");
-			}
-		}
-		catch (ClassCastException ccex) {
-			throw new RuntimeException("The stub class is not a proper subclass of " + OutputFormat.class.getName(), ccex);
-		}
-
-		// configure the stub. catch exceptions here extra, to report them as originating from the user code
-		try {
-			this.format.configure(this.config.getStubParameters());
-		}
-		catch (Throwable t) {
-			throw new RuntimeException("The user defined 'configure()' method in the Output Format caused an error: "
-					+ t.getMessage(), t);
-		}
-	}
-
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/8b8dfc16/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/runtime/DataSourceProcessor.java
----------------------------------------------------------------------
diff --git a/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/runtime/DataSourceProcessor.java b/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/runtime/DataSourceProcessor.java
deleted file mode 100644
index dd3f843..0000000
--- a/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/runtime/DataSourceProcessor.java
+++ /dev/null
@@ -1,190 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.tez.runtime;
-
-import com.google.common.base.Preconditions;
-import org.apache.flink.api.common.distributions.DataDistribution;
-import org.apache.flink.api.common.io.InputFormat;
-import org.apache.flink.api.common.typeutils.TypeComparator;
-import org.apache.flink.api.common.typeutils.TypeComparatorFactory;
-import org.apache.flink.api.common.typeutils.TypeSerializer;
-import org.apache.flink.api.common.typeutils.TypeSerializerFactory;
-import org.apache.flink.core.io.InputSplit;
-import org.apache.flink.runtime.operators.shipping.ShipStrategyType;
-import org.apache.flink.tez.runtime.input.FlinkInput;
-import org.apache.flink.tez.runtime.output.TezChannelSelector;
-import org.apache.flink.tez.runtime.output.TezOutputCollector;
-import org.apache.flink.tez.runtime.output.TezOutputEmitter;
-import org.apache.flink.tez.util.EncodingUtils;
-import org.apache.flink.util.Collector;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.tez.common.TezUtils;
-import org.apache.tez.dag.api.UserPayload;
-import org.apache.tez.runtime.api.AbstractLogicalIOProcessor;
-import org.apache.tez.runtime.api.Event;
-import org.apache.tez.runtime.api.LogicalInput;
-import org.apache.tez.runtime.api.LogicalOutput;
-import org.apache.tez.runtime.api.ProcessorContext;
-import org.apache.tez.runtime.library.api.KeyValueWriter;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Map;
-
-
-public class DataSourceProcessor<OT> extends AbstractLogicalIOProcessor {
-
-	private TezTaskConfig config;
-	protected Map<String, LogicalOutput> outputs;
-	private List<KeyValueWriter> writers;
-	private int numOutputs;
-	private Collector<OT> collector;
-
-	private InputFormat<OT, InputSplit> format;
-	private TypeSerializerFactory<OT> serializerFactory;
-	private FlinkInput input;
-	private ClassLoader userCodeClassLoader = getClass().getClassLoader();
-
-
-	public DataSourceProcessor(ProcessorContext context) {
-		super(context);
-	}
-
-	@Override
-	public void initialize() throws Exception {
-		UserPayload payload = getContext().getUserPayload();
-		Configuration conf = TezUtils.createConfFromUserPayload(payload);
-
-		this.config = (TezTaskConfig) EncodingUtils.decodeObjectFromString(conf.get(TezTaskConfig.TEZ_TASK_CONFIG), getClass().getClassLoader());
-		config.setTaskName(getContext().getTaskVertexName());
-
-		this.serializerFactory = config.getOutputSerializer(this.userCodeClassLoader);
-
-		initInputFormat();
-	}
-
-	@Override
-	public void handleEvents(List<Event> processorEvents) {
-		int i = 0;
-	}
-
-	@Override
-	public void close() throws Exception {
-
-	}
-
-	@Override
-	public void run(Map<String, LogicalInput> inputs, Map<String, LogicalOutput> outputs) throws Exception {
-
-		Preconditions.checkArgument(inputs.size() == 1);
-		LogicalInput logicalInput = inputs.values().iterator().next();
-		if (!(logicalInput instanceof FlinkInput)) {
-			throw new RuntimeException("Input to Flink Data Source Processor should be of type FlinkInput");
-		}
-		this.input = (FlinkInput) logicalInput;
-		//this.reader = (KeyValueReader) input.getReader();
-
-		// Initialize inputs, get readers and writers
-		this.outputs = outputs;
-		this.numOutputs = outputs.size();
-		this.writers = new ArrayList<KeyValueWriter>(numOutputs);
-		if (this.outputs != null) {
-			for (LogicalOutput output : this.outputs.values()) {
-				output.start();
-				writers.add((KeyValueWriter) output.getWriter());
-			}
-		}
-		this.invoke();
-	}
-
-
-	private void invoke () {
-		final TypeSerializer<OT> serializer = this.serializerFactory.getSerializer();
-		try {
-			InputSplit split = input.getSplit();
-
-			OT record = serializer.createInstance();
-			final InputFormat<OT, InputSplit> format = this.format;
-			format.open(split);
-
-			int numOutputs = outputs.size();
-			ArrayList<TezChannelSelector<OT>> channelSelectors = new ArrayList<TezChannelSelector<OT>>(numOutputs);
-			ArrayList<Integer> numStreamsInOutputs = this.config.getNumberSubtasksInOutput();
-			for (int i = 0; i < numOutputs; i++) {
-				final ShipStrategyType strategy = config.getOutputShipStrategy(i);
-				final TypeComparatorFactory<OT> compFactory = config.getOutputComparator(i, this.userCodeClassLoader);
-				final DataDistribution dataDist = config.getOutputDataDistribution(i, this.userCodeClassLoader);
-				if (compFactory == null) {
-					channelSelectors.add(i, new TezOutputEmitter<OT>(strategy));
-				} else if (dataDist == null){
-					final TypeComparator<OT> comparator = compFactory.createComparator();
-					channelSelectors.add(i, new TezOutputEmitter<OT>(strategy, comparator));
-				} else {
-					final TypeComparator<OT> comparator = compFactory.createComparator();
-					channelSelectors.add(i,new TezOutputEmitter<OT>(strategy, comparator, dataDist));
-				}
-			}
-			collector = new TezOutputCollector<OT>(writers, channelSelectors, serializerFactory.getSerializer(), numStreamsInOutputs);
-
-			while (!format.reachedEnd()) {
-				// build next pair and ship pair if it is valid
-				if ((record = format.nextRecord(record)) != null) {
-					collector.collect(record);
-				}
-			}
-			format.close();
-
-			collector.close();
-
-		}
-		catch (Exception ex) {
-			// close the input, but do not report any exceptions, since we already have another root cause
-			try {
-				this.format.close();
-			} catch (Throwable t) {}
-		}
-	}
-
-
-	private void initInputFormat() {
-		try {
-			this.format = config.<InputFormat<OT, InputSplit>>getStubWrapper(this.userCodeClassLoader)
-					.getUserCodeObject(InputFormat.class, this.userCodeClassLoader);
-
-			// check if the class is a subclass, if the check is required
-			if (!InputFormat.class.isAssignableFrom(this.format.getClass())) {
-				throw new RuntimeException("The class '" + this.format.getClass().getName() + "' is not a subclass of '" +
-						InputFormat.class.getName() + "' as is required.");
-			}
-		}
-		catch (ClassCastException ccex) {
-			throw new RuntimeException("The stub class is not a proper subclass of " + InputFormat.class.getName(),
-					ccex);
-		}
-		// configure the stub. catch exceptions here extra, to report them as originating from the user code
-		try {
-			this.format.configure(this.config.getStubParameters());
-		}
-		catch (Throwable t) {
-			throw new RuntimeException("The user defined 'configure()' method caused an error: " + t.getMessage(), t);
-		}
-	}
-
-
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/8b8dfc16/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/runtime/RegularProcessor.java
----------------------------------------------------------------------
diff --git a/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/runtime/RegularProcessor.java b/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/runtime/RegularProcessor.java
deleted file mode 100644
index 14d9cde..0000000
--- a/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/runtime/RegularProcessor.java
+++ /dev/null
@@ -1,138 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.tez.runtime;
-
-import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.api.common.TaskInfo;
-import org.apache.flink.api.common.accumulators.Accumulator;
-import org.apache.flink.api.common.functions.Function;
-import org.apache.flink.core.fs.Path;
-import org.apache.flink.runtime.operators.Driver;
-import org.apache.flink.api.common.functions.util.RuntimeUDFContext;
-import org.apache.flink.tez.util.EncodingUtils;
-import org.apache.flink.util.InstantiationUtil;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.tez.common.TezUtils;
-import org.apache.tez.dag.api.UserPayload;
-import org.apache.tez.runtime.api.AbstractLogicalIOProcessor;
-import org.apache.tez.runtime.api.Event;
-import org.apache.tez.runtime.api.LogicalInput;
-import org.apache.tez.runtime.api.LogicalOutput;
-import org.apache.tez.runtime.api.ProcessorContext;
-import org.apache.tez.runtime.library.api.KeyValueReader;
-import org.apache.tez.runtime.library.api.KeyValueWriter;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.Future;
-
-
-public class RegularProcessor<S extends Function, OT> extends AbstractLogicalIOProcessor {
-
-	private TezTask<S,OT> task;
-	protected Map<String, LogicalInput> inputs;
-	protected Map<String, LogicalOutput> outputs;
-	private List<KeyValueReader> readers;
-	private List<KeyValueWriter> writers;
-	private int numInputs;
-	private int numOutputs;
-
-
-	public RegularProcessor(ProcessorContext context) {
-		super(context);
-	}
-
-	@Override
-	public void initialize() throws Exception {
-		UserPayload payload = getContext().getUserPayload();
-		Configuration conf = TezUtils.createConfFromUserPayload(payload);
-
-		TezTaskConfig taskConfig = (TezTaskConfig) EncodingUtils.decodeObjectFromString(conf.get(TezTaskConfig.TEZ_TASK_CONFIG), getClass().getClassLoader());
-		taskConfig.setTaskName(getContext().getTaskVertexName());
-
-		RuntimeUDFContext runtimeUdfContext = new RuntimeUDFContext(
-				new TaskInfo(
-						getContext().getTaskVertexName(),
-						getContext().getTaskIndex(),
-						getContext().getVertexParallelism(),
-						getContext().getTaskAttemptNumber()
-				),
-				getClass().getClassLoader(),
-				new ExecutionConfig(),
-				new HashMap<String, Future<Path>>(),
-				new HashMap<String, Accumulator<?, ?>>());
-
-		this.task = new TezTask<S, OT>(taskConfig, runtimeUdfContext, this.getContext().getTotalMemoryAvailableToTask());
-	}
-
-	@Override
-	public void handleEvents(List<Event> processorEvents) {
-
-	}
-
-	@Override
-	public void close() throws Exception {
-		task.getIOManager().shutdown();
-	}
-
-	@Override
-	public void run(Map<String, LogicalInput> inputs, Map<String, LogicalOutput> outputs) throws Exception {
-
-		this.inputs = inputs;
-		this.outputs = outputs;
-		final Class<? extends Driver<S, OT>> driverClass = this.task.getTaskConfig().getDriver();
-		Driver<S,OT> driver = InstantiationUtil.instantiate(driverClass, Driver.class);
-		this.numInputs = driver.getNumberOfInputs();
-		this.numOutputs = outputs.size();
-
-
-		this.readers = new ArrayList<KeyValueReader>(numInputs);
-		//Ensure size of list is = numInputs
-		for (int i = 0; i < numInputs; i++)
-			this.readers.add(null);
-		HashMap<String, ArrayList<Integer>> inputPositions = ((TezTaskConfig) this.task.getTaskConfig()).getInputPositions();
-		if (this.inputs != null) {
-			for (String name : this.inputs.keySet()) {
-				LogicalInput input = this.inputs.get(name);
-				//if (input instanceof AbstractLogicalInput) {
-				//	((AbstractLogicalInput) input).initialize();
-				//}
-				input.start();
-				ArrayList<Integer> positions = inputPositions.get(name);
-				for (Integer pos : positions) {
-					//int pos = inputPositions.get(name);
-					readers.set(pos, (KeyValueReader) input.getReader());
-				}
-			}
-		}
-
-		this.writers = new ArrayList<KeyValueWriter>(numOutputs);
-		if (this.outputs != null) {
-			for (LogicalOutput output : this.outputs.values()) {
-				output.start();
-				writers.add((KeyValueWriter) output.getWriter());
-			}
-		}
-
-		// Do the work
-		task.invoke (readers, writers);
-	}
-}

http://git-wip-us.apache.org/repos/asf/flink/blob/8b8dfc16/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/runtime/TezRuntimeEnvironment.java
----------------------------------------------------------------------
diff --git a/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/runtime/TezRuntimeEnvironment.java b/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/runtime/TezRuntimeEnvironment.java
deleted file mode 100644
index b61a9b6..0000000
--- a/flink-contrib/flink-tez/src/main/java/org/apache/flink/tez/runtime/TezRuntimeEnvironment.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.flink.tez.runtime;
-
-import org.apache.flink.core.memory.MemoryType;
-import org.apache.flink.runtime.io.disk.iomanager.IOManager;
-import org.apache.flink.runtime.io.disk.iomanager.IOManagerAsync;
-import org.apache.flink.runtime.memory.MemoryManager;
-
-public class TezRuntimeEnvironment {
-	
-	private final IOManager ioManager;
-
-	private final MemoryManager memoryManager;
-
-	public TezRuntimeEnvironment(long totalMemory) {
-		this.memoryManager = new MemoryManager(totalMemory, 1, MemoryManager.DEFAULT_PAGE_SIZE, MemoryType.HEAP, true);
-		this.ioManager = new IOManagerAsync();
-	}
-
-	public IOManager getIOManager() {
-		return ioManager;
-	}
-
-	public MemoryManager getMemoryManager() {
-		return memoryManager;
-	}
-}