You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ac...@apache.org on 2013/03/15 22:26:48 UTC
svn commit: r1457129 [4/38] - in /incubator/tez: ./ tez-ampool/
tez-ampool/src/ tez-ampool/src/main/ tez-ampool/src/main/bin/
tez-ampool/src/main/conf/ tez-ampool/src/main/java/
tez-ampool/src/main/java/org/ tez-ampool/src/main/java/org/apache/ tez-amp...
Added: incubator/tez/tez-ampool/src/main/java/org/apache/tez/ampool/rest/ApplicationPollService.java
URL: http://svn.apache.org/viewvc/incubator/tez/tez-ampool/src/main/java/org/apache/tez/ampool/rest/ApplicationPollService.java?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-ampool/src/main/java/org/apache/tez/ampool/rest/ApplicationPollService.java (added)
+++ incubator/tez/tez-ampool/src/main/java/org/apache/tez/ampool/rest/ApplicationPollService.java Fri Mar 15 21:26:36 2013
@@ -0,0 +1,116 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.ampool.rest;
+
+import java.util.Map;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.GET;
+import javax.ws.rs.Path;
+import javax.ws.rs.PathParam;
+import javax.ws.rs.Produces;
+import javax.ws.rs.WebApplicationException;
+import javax.ws.rs.core.Context;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response.Status;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.tez.ampool.AMContext;
+import org.apache.tez.ampool.manager.AMPoolManager;
+import org.apache.tez.mapreduce.hadoop.MRJobConfig;
+
+@Path("/applications")
+public class ApplicationPollService {
+
+ private static final Log LOG =
+ LogFactory.getLog(ApplicationPollService.class);
+
+ private static AMPoolManager manager;
+
+ public static void init(AMPoolManager manager) {
+ ApplicationPollService.manager = manager;
+ }
+
+ @Path("poll/{applicationAttemptId}")
+ @GET
+ @Consumes(MediaType.APPLICATION_JSON)
+ @Produces({MediaType.APPLICATION_JSON})
+ public ApplicationPollResponse pollForApplication(
+ @PathParam("applicationAttemptId") String applicationAttemptIdStr,
+ @Context HttpServletRequest req) {
+ ApplicationAttemptId applicationAttemptId;
+ LOG.info("Received a poll from launchedAM"
+ + ", appAttemptId=" + applicationAttemptIdStr);
+ try {
+ applicationAttemptId =
+ ConverterUtils.toApplicationAttemptId(applicationAttemptIdStr);
+ } catch (IllegalArgumentException e) {
+ LOG.warn("Received an invalid attempt id in the poll request"
+ + ", appAttemptId=" + applicationAttemptIdStr);
+ throw new WebApplicationException(Status.BAD_REQUEST);
+ }
+
+ AMContext amContext = null;
+ try {
+ amContext =
+ manager.getAMContext(applicationAttemptId);
+ } catch (Exception e) {
+ throw new WebApplicationException(e, Status.INTERNAL_SERVER_ERROR);
+ }
+
+ if (amContext == null
+ || amContext.getSubmissionContext() == null) {
+ throw new WebApplicationException(Status.NO_CONTENT);
+ }
+ try {
+ return convertAMContext(amContext);
+ } catch (Exception e) {
+ throw new WebApplicationException(e, Status.INTERNAL_SERVER_ERROR);
+ }
+ }
+
+ private ApplicationPollResponse convertAMContext(
+ AMContext amContext) throws Exception {
+ ApplicationSubmissionContext context =
+ amContext.getSubmissionContext();
+ Map<String, LocalResource> resources =
+ context.getAMContainerSpec().getLocalResources();
+ ApplicationPollResponse pollResponse =
+ new ApplicationPollResponse(context.getApplicationId(),
+ amContext.getApplicationSubmissionTime());
+ if (resources.containsKey(MRJobConfig.JOB_CONF_FILE)) {
+ LocalResource rsrc = resources.get(MRJobConfig.JOB_CONF_FILE);
+ pollResponse.setConfigurationFileLocation(
+ ConverterUtils.getPathFromYarnURL(rsrc.getResource()).toString());
+ }
+ if (resources.containsKey(MRJobConfig.JOB_JAR)) {
+ LocalResource rsrc = resources.get(MRJobConfig.JOB_JAR);
+ pollResponse.setApplicationJarLocation(
+ ConverterUtils.getPathFromYarnURL(rsrc.getResource()).toString());
+ }
+ return pollResponse;
+ }
+
+}
Added: incubator/tez/tez-ampool/src/main/resources/lazy-mram-default.xml
URL: http://svn.apache.org/viewvc/incubator/tez/tez-ampool/src/main/resources/lazy-mram-default.xml?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-ampool/src/main/resources/lazy-mram-default.xml (added)
+++ incubator/tez/tez-ampool/src/main/resources/lazy-mram-default.xml Fri Mar 15 21:26:36 2013
@@ -0,0 +1,35 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+
+ <property>
+ <description>How often to poll AMPoolService for job assignment</description>
+ <name>yarn.app.mapreduce.am.lazy.polling-interval.secs</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <description>How many containers to pre-allocate after starting up</description>
+ <name>yarn.app.mapreduce.am.lazy.prealloc-container-count</name>
+ <value>0</value>
+ </property>
+
+</configuration>
Added: incubator/tez/tez-ampool/src/main/resources/tez-ampool-default.xml
URL: http://svn.apache.org/viewvc/incubator/tez/tez-ampool/src/main/resources/tez-ampool-default.xml?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-ampool/src/main/resources/tez-ampool-default.xml (added)
+++ incubator/tez/tez-ampool/src/main/resources/tez-ampool-default.xml Fri Mar 15 21:26:36 2013
@@ -0,0 +1,114 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+
+ <property>
+ <description>Memory to use for AMPoolService AM</description>
+ <name>tez.ampool.am.master_memory</name>
+ <value>1024</value>
+ </property>
+
+ <property>
+ <description>Queue to launch AMPoolService AM</description>
+ <name>tez.ampool.am.master_queue</name>
+ <value>default</value>
+ </property>
+
+ <property>
+ <description>Port to use for AMPoolService status</description>
+ <name>tez.ampool.ws.port</name>
+ <value>12999</value>
+ </property>
+
+ <property>
+ <description>Minimum size of AM Pool</description>
+ <name>tez.ampool.am-pool-size</name>
+ <value>3</value>
+ </property>
+
+ <property>
+ <description>Maximum size of AM Pool</description>
+ <name>tez.ampool.max-am-pool-size</name>
+ <value>5</value>
+ </property>
+
+ <property>
+ <description>When to launch new AM. If true, launched after an existing AM in pool completes. Else, launched as soon as a job is assigned to an AM from the pool</description>
+ <name>tez.ampool.launch-new-am-after-app-completion</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <description>How many launch failures to account for unassigned AMs before shutting down AMPoolService</description>
+ <name>tez.ampool.max-am-launch-failures</name>
+ <value>10</value>
+ </property>
+
+ <property>
+ <description>No. of threads to use to serve the ClientRMProtocol proxy fronted by the AMPoolService</description>
+ <name>tez.ampool.rm-proxy-client.thread-count</name>
+ <value>10</value>
+ </property>
+
+ <property>
+ <description>Address on which to run the ClientRMProtocol proxy</description>
+ <name>tez.ampool.address</name>
+ <value>0.0.0.0:10030</value>
+ </property>
+
+ <property>
+ <description>Memory to use when launching the lazy MR AM</description>
+ <name>tez.ampool.mr-am.memory-allocation-mb</name>
+ <value>1536</value>
+ </property>
+
+ <property>
+ <description>Queue to which the Lazy MRAM is to be submitted to</description>
+ <name>tez.ampool.mr-am.queue-name</name>
+ <value>default</value>
+ </property>
+
+ <property>
+ <description>Comma-separated paths to job jars on DFS (optional)</description>
+ <name>tez.ampool.mr-am.job-jar-path</name>
+ <value></value>
+ </property>
+
+ <property>
+ <description>Main class for Lazy MRAM</description>
+ <name>tez.ampool.mr-am.application-master-class</name>
+ <value>org.apache.hadoop.mapreduce.v2.app2.lazy.LazyMRAppMaster</value>
+ </property>
+
+ <property>
+ <description>Local filesystem path for staging local data used by AMPoolClient/AMPoolService</description>
+ <name>tez.ampool.tmp-dir-path</name>
+ <value>/tmp/ampoolservice/</value>
+ </property>
+
+ <property>
+ <description>Path on FS used by AMPoolService to upload lazy-mr-am config</description>
+ <name>tez.ampool.am.staging-dir</name>
+ <value>/tmp/tez/ampool/staging/</value>
+ </property>
+
+</configuration>
+
Added: incubator/tez/tez-ampool/src/test/java/org/apache/tez/ampool/TestAMPool.java
URL: http://svn.apache.org/viewvc/incubator/tez/tez-ampool/src/test/java/org/apache/tez/ampool/TestAMPool.java?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-ampool/src/test/java/org/apache/tez/ampool/TestAMPool.java (added)
+++ incubator/tez/tez-ampool/src/test/java/org/apache/tez/ampool/TestAMPool.java Fri Mar 15 21:26:36 2013
@@ -0,0 +1,151 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.tez.ampool;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.net.URL;
+import java.util.List;
+
+import junit.framework.Assert;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.JarFinder;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.client.YarnClient;
+import org.apache.hadoop.yarn.client.YarnClientImpl;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.MiniYARNCluster;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler;
+import org.apache.tez.ampool.AMPoolService;
+import org.apache.tez.ampool.client.AMPoolClient;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class TestAMPool {
+
+ private static final Log LOG =
+ LogFactory.getLog(TestAMPool.class);
+
+ protected static MiniYARNCluster yarnCluster = null;
+ protected static Configuration conf = new YarnConfiguration();
+ protected static YarnClient yarnClient = null;
+
+ protected static String APPMASTER_JAR =
+ JarFinder.getJar(AMPoolService.class);
+
+ @BeforeClass
+ public static void setup() throws InterruptedException, IOException {
+ LOG.info("Starting up YARN cluster");
+ conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
+ conf.setClass(YarnConfiguration.RM_SCHEDULER,
+ FifoScheduler.class, ResourceScheduler.class);
+ if (yarnCluster == null) {
+ yarnCluster = new MiniYARNCluster(TestAMPool.class.getName(),
+ 1, 1, 1);
+ yarnCluster.init(conf);
+ yarnCluster.start();
+ URL url = Thread.currentThread().getContextClassLoader().getResource("yarn-site.xml");
+ if (url == null) {
+ throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath");
+ }
+ yarnCluster.getConfig().set("yarn.application.classpath", new File(url.getPath()).getParent());
+ OutputStream os = new FileOutputStream(new File(url.getPath()));
+ yarnCluster.getConfig().writeXml(os);
+ yarnClient = new YarnClientImpl();
+ yarnClient.init(yarnCluster.getConfig());
+ yarnClient.start();
+ os.close();
+ }
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ LOG.info("setup thread sleep interrupted. message=" + e.getMessage());
+ }
+ }
+
+ @AfterClass
+ public static void tearDown() throws IOException {
+ if (yarnCluster != null) {
+ yarnCluster.stop();
+ yarnCluster = null;
+ }
+ if (yarnClient != null) {
+ yarnClient.stop();
+ yarnClient = null;
+ }
+ }
+
+ @Test
+ @Ignore
+ public void testSimpleRun() throws Exception {
+
+ String[] args = {
+ "--jar",
+ APPMASTER_JAR,
+ "--master_memory",
+ "512",
+ };
+
+ LOG.info("Initializing AMPool Client");
+ AMPoolClient client = new AMPoolClient();
+ boolean initSuccess = client.init(args);
+ Assert.assertTrue(initSuccess);
+ LOG.info("Running AMPool Client");
+ boolean result = client.run();
+
+ LOG.info("Client run completed. Result=" + result);
+ Assert.assertTrue(result);
+
+ while(true) {
+ try {
+ Thread.sleep(2000);
+ } catch (InterruptedException e) {
+ LOG.info("thread sleep interrupted. message=" + e.getMessage());
+ }
+
+ List<ApplicationReport> reports = yarnClient.getApplicationList();
+ if (reports.isEmpty()) {
+ LOG.error("Did not find any applications");
+ break;
+ }
+ FinalApplicationStatus status =
+ reports.get(0).getFinalApplicationStatus();
+ if (status == FinalApplicationStatus.UNDEFINED) {
+ LOG.info("Application is still running");
+ continue;
+ } else {
+ LOG.info("Application completed with status"
+ + status);
+ break;
+ }
+
+ }
+
+ }
+
+}
Added: incubator/tez/tez-ampool/src/test/resources/log4j.properties
URL: http://svn.apache.org/viewvc/incubator/tez/tez-ampool/src/test/resources/log4j.properties?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-ampool/src/test/resources/log4j.properties (added)
+++ incubator/tez/tez-ampool/src/test/resources/log4j.properties Fri Mar 15 21:26:36 2013
@@ -0,0 +1,19 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# log4j configuration used during build and unit tests
+
+log4j.rootLogger=info,stdout
+log4j.threshhold=ALL
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %c{2} (%F:%M(%L)) - %m%n
Added: incubator/tez/tez-ampool/src/test/resources/yarn-ampool-site.xml
URL: http://svn.apache.org/viewvc/incubator/tez/tez-ampool/src/test/resources/yarn-ampool-site.xml?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-ampool/src/test/resources/yarn-ampool-site.xml (added)
+++ incubator/tez/tez-ampool/src/test/resources/yarn-ampool-site.xml Fri Mar 15 21:26:36 2013
@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<configuration>
+</configuration>
+
Added: incubator/tez/tez-ampool/src/test/resources/yarn-site.xml
URL: http://svn.apache.org/viewvc/incubator/tez/tez-ampool/src/test/resources/yarn-site.xml?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-ampool/src/test/resources/yarn-site.xml (added)
+++ incubator/tez/tez-ampool/src/test/resources/yarn-site.xml Fri Mar 15 21:26:36 2013
@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<configuration>
+</configuration>
+
Added: incubator/tez/tez-api/.classpath
URL: http://svn.apache.org/viewvc/incubator/tez/tez-api/.classpath?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-api/.classpath (added)
+++ incubator/tez/tez-api/.classpath Fri Mar 15 21:26:36 2013
@@ -0,0 +1,72 @@
+<classpath>
+ <classpathentry kind="src" path="src/main/java" including="**/*.java"/>
+ <classpathentry kind="output" path="target/classes"/>
+ <classpathentry kind="var" path="M2_REPO/javax/activation/activation/1.1/activation-1.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2.jar"/>
+ <classpathentry kind="var" path="M2_REPO/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/javax/servlet/servlet-api/2.5/servlet-api-2.5.jar"/>
+ <classpathentry kind="var" path="M2_REPO/asm/asm/3.1/asm-3.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/apache/avro/avro/1.5.3/avro-1.5.3.jar"/>
+ <classpathentry kind="var" path="M2_REPO/commons-beanutils/commons-beanutils/1.7.0/commons-beanutils-1.7.0.jar"/>
+ <classpathentry kind="var" path="M2_REPO/commons-beanutils/commons-beanutils-core/1.8.0/commons-beanutils-core-1.8.0.jar"/>
+ <classpathentry kind="var" path="M2_REPO/commons-cli/commons-cli/1.2/commons-cli-1.2.jar"/>
+ <classpathentry kind="var" path="M2_REPO/commons-codec/commons-codec/1.4/commons-codec-1.4.jar"/>
+ <classpathentry kind="var" path="M2_REPO/commons-collections/commons-collections/3.2.1/commons-collections-3.2.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/apache/commons/commons-compress/1.4/commons-compress-1.4.jar"/>
+ <classpathentry kind="var" path="M2_REPO/commons-configuration/commons-configuration/1.6/commons-configuration-1.6.jar"/>
+ <classpathentry kind="var" path="M2_REPO/commons-digester/commons-digester/1.8/commons-digester-1.8.jar"/>
+ <classpathentry kind="var" path="M2_REPO/commons-el/commons-el/1.0/commons-el-1.0.jar"/>
+ <classpathentry kind="var" path="M2_REPO/commons-httpclient/commons-httpclient/3.1/commons-httpclient-3.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/commons-io/commons-io/2.1/commons-io-2.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/commons-lang/commons-lang/2.5/commons-lang-2.5.jar"/>
+ <classpathentry kind="var" path="M2_REPO/commons-logging/commons-logging/1.1.1/commons-logging-1.1.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/apache/commons/commons-math/2.1/commons-math-2.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/commons-net/commons-net/3.1/commons-net-3.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/com/google/guava/guava/11.0.2/guava-11.0.2.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/apache/hadoop/hadoop-annotations/3.0.0-SNAPSHOT/hadoop-annotations-3.0.0-SNAPSHOT.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/apache/hadoop/hadoop-auth/3.0.0-SNAPSHOT/hadoop-auth-3.0.0-SNAPSHOT.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/apache/hadoop/hadoop-common/3.0.0-SNAPSHOT/hadoop-common-3.0.0-SNAPSHOT.jar" sourcepath="M2_REPO/org/apache/hadoop/hadoop-common/3.0.0-SNAPSHOT/hadoop-common-3.0.0-SNAPSHOT-sources.jar">
+ <attributes>
+ <attribute value="jar:file:/Users/acmurthy/.m2/repository/org/apache/hadoop/hadoop-common/3.0.0-SNAPSHOT/hadoop-common-3.0.0-SNAPSHOT-javadoc.jar!/" name="javadoc_location"/>
+ </attributes>
+ </classpathentry>
+ <classpathentry kind="var" path="M2_REPO/org/apache/hadoop/hadoop-yarn-api/3.0.0-SNAPSHOT/hadoop-yarn-api-3.0.0-SNAPSHOT.jar" sourcepath="M2_REPO/org/apache/hadoop/hadoop-yarn-api/3.0.0-SNAPSHOT/hadoop-yarn-api-3.0.0-SNAPSHOT-sources.jar">
+ <attributes>
+ <attribute value="jar:file:/Users/acmurthy/.m2/repository/org/apache/hadoop/hadoop-yarn-api/3.0.0-SNAPSHOT/hadoop-yarn-api-3.0.0-SNAPSHOT-javadoc.jar!/" name="javadoc_location"/>
+ </attributes>
+ </classpathentry>
+ <classpathentry kind="var" path="M2_REPO/org/apache/hadoop/hadoop-yarn-common/3.0.0-SNAPSHOT/hadoop-yarn-common-3.0.0-SNAPSHOT.jar" sourcepath="M2_REPO/org/apache/hadoop/hadoop-yarn-common/3.0.0-SNAPSHOT/hadoop-yarn-common-3.0.0-SNAPSHOT-sources.jar">
+ <attributes>
+ <attribute value="jar:file:/Users/acmurthy/.m2/repository/org/apache/hadoop/hadoop-yarn-common/3.0.0-SNAPSHOT/hadoop-yarn-common-3.0.0-SNAPSHOT-javadoc.jar!/" name="javadoc_location"/>
+ </attributes>
+ </classpathentry>
+ <classpathentry kind="var" path="M2_REPO/org/codehaus/jackson/jackson-core-asl/1.8.8/jackson-core-asl-1.8.8.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/codehaus/jackson/jackson-jaxrs/1.7.1/jackson-jaxrs-1.7.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/codehaus/jackson/jackson-mapper-asl/1.8.8/jackson-mapper-asl-1.8.8.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/codehaus/jackson/jackson-xc/1.7.1/jackson-xc-1.7.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/tomcat/jasper-compiler/5.5.23/jasper-compiler-5.5.23.jar"/>
+ <classpathentry kind="var" path="M2_REPO/tomcat/jasper-runtime/5.5.23/jasper-runtime-5.5.23.jar"/>
+ <classpathentry kind="var" path="M2_REPO/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/com/sun/jersey/jersey-core/1.8/jersey-core-1.8.jar"/>
+ <classpathentry kind="var" path="M2_REPO/com/sun/jersey/jersey-json/1.8/jersey-json-1.8.jar"/>
+ <classpathentry kind="var" path="M2_REPO/com/sun/jersey/jersey-server/1.8/jersey-server-1.8.jar"/>
+ <classpathentry kind="var" path="M2_REPO/net/java/dev/jets3t/jets3t/0.6.1/jets3t-0.6.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/mortbay/jetty/jetty/6.1.26/jetty-6.1.26.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/mortbay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar"/>
+ <classpathentry kind="var" path="M2_REPO/jline/jline/0.9.94/jline-0.9.94.jar"/>
+ <classpathentry kind="var" path="M2_REPO/com/jcraft/jsch/0.1.42/jsch-0.1.42.jar"/>
+ <classpathentry kind="var" path="M2_REPO/com/google/code/findbugs/jsr305/1.3.9/jsr305-1.3.9.jar"/>
+ <classpathentry kind="var" path="M2_REPO/log4j/log4j/1.2.17/log4j-1.2.17.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/jboss/netty/netty/3.2.2.Final/netty-3.2.2.Final.jar"/>
+ <classpathentry kind="var" path="M2_REPO/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar"/>
+ <classpathentry kind="var" path="M2_REPO/com/google/protobuf/protobuf-java/2.4.0a/protobuf-java-2.4.0a.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/slf4j/slf4j-api/1.6.1/slf4j-api-1.6.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/slf4j/slf4j-log4j12/1.6.1/slf4j-log4j12-1.6.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/xerial/snappy/snappy-java/1.0.3.2/snappy-java-1.0.3.2.jar"/>
+ <classpathentry kind="var" path="M2_REPO/stax/stax-api/1.0.1/stax-api-1.0.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/xmlenc/xmlenc/0.52/xmlenc-0.52.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/tukaani/xz/1.0/xz-1.0.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/apache/zookeeper/zookeeper/3.4.2/zookeeper-3.4.2.jar"/>
+ <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
+</classpath>
\ No newline at end of file
Added: incubator/tez/tez-api/.project
URL: http://svn.apache.org/viewvc/incubator/tez/tez-api/.project?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-api/.project (added)
+++ incubator/tez/tez-api/.project Fri Mar 15 21:26:36 2013
@@ -0,0 +1,13 @@
+<projectDescription>
+ <name>tez-api</name>
+ <comment>NO_M2ECLIPSE_SUPPORT: Project files created with the maven-eclipse-plugin are not supported in M2Eclipse.</comment>
+ <projects/>
+ <buildSpec>
+ <buildCommand>
+ <name>org.eclipse.jdt.core.javabuilder</name>
+ </buildCommand>
+ </buildSpec>
+ <natures>
+ <nature>org.eclipse.jdt.core.javanature</nature>
+ </natures>
+</projectDescription>
\ No newline at end of file
Added: incubator/tez/tez-api/pom.xml
URL: http://svn.apache.org/viewvc/incubator/tez/tez-api/pom.xml?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-api/pom.xml (added)
+++ incubator/tez/tez-api/pom.xml Fri Mar 15 21:26:36 2013
@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez</artifactId>
+ <version>0.1.0</version>
+ </parent>
+ <artifactId>tez-api</artifactId>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+ </dependencies>
+</project>
Added: incubator/tez/tez-api/src/main/java/org/apache/tez/api/Input.java
URL: http://svn.apache.org/viewvc/incubator/tez/tez-api/src/main/java/org/apache/tez/api/Input.java?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-api/src/main/java/org/apache/tez/api/Input.java (added)
+++ incubator/tez/tez-api/src/main/java/org/apache/tez/api/Input.java Fri Mar 15 21:26:36 2013
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.api;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * {@link Input} represents a pipe through which an <em>tez</em> task
+ * can get input key/value pairs.
+ */
+public interface Input {
+
+ /**
+ * Initialize <code>Input</code>.
+ *
+ * @param conf job configuration
+ * @param master master process controlling the task
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ public void initialize(Configuration conf, Master master)
+ throws IOException, InterruptedException;
+
+ /**
+ * Check if there is another key/value pair.
+ *
+ * @return true if a key/value pair was read
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ public boolean hasNext() throws IOException, InterruptedException;
+
+ /**
+ * Get the next key.
+ *
+ * @return the current key or null if there is no current key
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ public Object getNextKey() throws IOException, InterruptedException;
+
+ /**
+ * Get the next values.
+ *
+ * @return the object that was read
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ public Iterable<Object> getNextValues()
+ throws IOException, InterruptedException;
+
+ /**
+ * The current progress of the {@link Input} through its data.
+ *
+ * @return a number between 0.0 and 1.0 that is the fraction of the data read
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ public float getProgress() throws IOException, InterruptedException;
+
+ /**
+ * Close this <code>Input</code> for future operations.
+ */
+ public void close() throws IOException;
+
+}
Added: incubator/tez/tez-api/src/main/java/org/apache/tez/api/Master.java
URL: http://svn.apache.org/viewvc/incubator/tez/tez-api/src/main/java/org/apache/tez/api/Master.java?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-api/src/main/java/org/apache/tez/api/Master.java (added)
+++ incubator/tez/tez-api/src/main/java/org/apache/tez/api/Master.java Fri Mar 15 21:26:36 2013
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.api;
+
+import org.apache.hadoop.ipc.ProtocolInfo;
+import org.apache.hadoop.ipc.VersionedProtocol;
+import org.apache.tez.records.TezJobID;
+import org.apache.tez.records.TezTaskAttemptID;
+import org.apache.tez.records.TezTaskDependencyCompletionEventsUpdate;
+
+/**
+ * {@link Master} represents the master controlling the {@link Task}.
+ */
+@ProtocolInfo(protocolName = "Master", protocolVersion = 1)
+public interface Master extends VersionedProtocol {
+
+ // TODO TEZAM3 This likely needs to change to be a little more generic.
+ // Many output / input relationships cannot be captured via this. The current
+ // form works primarily works for the existing MR
+
+ TezTaskDependencyCompletionEventsUpdate getDependentTasksCompletionEvents(
+ TezJobID jobID, int fromEventIdx, int maxEventsToFetch,
+ TezTaskAttemptID taskAttemptId);
+
+}
Added: incubator/tez/tez-api/src/main/java/org/apache/tez/api/Output.java
URL: http://svn.apache.org/viewvc/incubator/tez/tez-api/src/main/java/org/apache/tez/api/Output.java?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-api/src/main/java/org/apache/tez/api/Output.java (added)
+++ incubator/tez/tez-api/src/main/java/org/apache/tez/api/Output.java Fri Mar 15 21:26:36 2013
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.api;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.records.OutputContext;
+
+/**
+ * {@link Output} represents a pipe through which an <em>tez</em> task
+ * can send out outputs.
+ */
+public interface Output {
+
+ /**
+ * Initialize <code>Output</code>.
+ *
+ * @param conf job configuration
+ * @param master master process controlling the task
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ public void initialize(Configuration conf, Master master)
+ throws IOException, InterruptedException;
+
+ /**
+ * Writes a key/value pair.
+ *
+ * @param key the key to write.
+ * @param value the value to write.
+ * @throws IOException
+ */
+ public void write(Object key, Object value
+ ) throws IOException, InterruptedException;
+
+ /**
+ * Returns the OutputContext for the particular <code>Output</code>.
+ *
+ * @return the OutputContext for this Output if it exists, otherwise null.
+ */
+ public OutputContext getOutputContext();
+
+ /**
+ * Close this <code>Output</code> for future operations.
+ *
+ * @throws IOException
+ */
+ public void close() throws IOException, InterruptedException;
+}
Added: incubator/tez/tez-api/src/main/java/org/apache/tez/api/Partitioner.java
URL: http://svn.apache.org/viewvc/incubator/tez/tez-api/src/main/java/org/apache/tez/api/Partitioner.java?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-api/src/main/java/org/apache/tez/api/Partitioner.java (added)
+++ incubator/tez/tez-api/src/main/java/org/apache/tez/api/Partitioner.java Fri Mar 15 21:26:36 2013
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.api;
+
+/**
+ * {@link Partitioner} is used by the TEZ framework to partition
+ * output key/value pairs.
+ */
+public interface Partitioner {
+
+ /**
+ * Get partition for given key/value.
+ * @param key key
+ * @param value value
+ * @param numPartitions number of partitions
+ * @return
+ */
+ int getPartition(Object key, Object value, int numPartitions);
+
+}
Added: incubator/tez/tez-api/src/main/java/org/apache/tez/api/Processor.java
URL: http://svn.apache.org/viewvc/incubator/tez/tez-api/src/main/java/org/apache/tez/api/Processor.java?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-api/src/main/java/org/apache/tez/api/Processor.java (added)
+++ incubator/tez/tez-api/src/main/java/org/apache/tez/api/Processor.java Fri Mar 15 21:26:36 2013
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.api;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * {@link Processor} represents the <em>tez</em> entity responsible for
+ * consuming {@link Input} and producing {@link Output}.
+ */
+public interface Processor {
+
+ /**
+ * Initialize the <code>Processor</code>.
+ *
+ * @param conf job-configuration
+ * @param master master process controlling the task
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ public void initialize(Configuration conf, Master master)
+ throws IOException, InterruptedException;
+
+ /**
+ * Process input data from <code>input</code> and
+ * send it to <code>output</code>.
+ *
+ * @param in input
+ * @param out output
+ * @param master master process controlling the task
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ public void process(Input in, Output out)
+ throws IOException, InterruptedException;
+
+ /**
+ * Close the {@link Processor}.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ public void close() throws IOException, InterruptedException;
+
+}
Added: incubator/tez/tez-api/src/main/java/org/apache/tez/api/Task.java
URL: http://svn.apache.org/viewvc/incubator/tez/tez-api/src/main/java/org/apache/tez/api/Task.java?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-api/src/main/java/org/apache/tez/api/Task.java (added)
+++ incubator/tez/tez-api/src/main/java/org/apache/tez/api/Task.java Fri Mar 15 21:26:36 2013
@@ -0,0 +1,77 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.api;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ * {@link Task} is the base <em>tez</em> entity which consumes
+ * input key/value pairs through an {@link Input} pipe,
+ * processes them via a {@link Processor} and
+ * produces output key/value pairs for an {@link Output} pipe.
+ */
+public interface Task {
+
+ /**
+ * Initialize the {@link Task}.
+ *
+ * @param conf task configuration
+ * @param master master controlling the task
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ public void initialize(Configuration conf, Master master)
+ throws IOException, InterruptedException;
+
+ /**
+ * Get {@link Input} of the task.
+ * @return <code>Input</code> of the task
+ */
+ public Input getInput();
+
+ /**
+ * Get {@link Processor} of the task.
+ * @return <code>Processor</code> of the task
+ */
+ public Processor getProcessor();
+
+ /**
+ * Get {@link Output} of the task.
+ * @return <code>Output</code> of the task
+ */
+ public Output getOutput();
+
+ /**
+ * Run the {@link Task}.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ public void run() throws IOException, InterruptedException;
+
+ /**
+ * Stop the {@link Task}.
+ *
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ public void close() throws IOException, InterruptedException;
+
+}
Added: incubator/tez/tez-api/src/main/java/org/apache/tez/records/OutputContext.java
URL: http://svn.apache.org/viewvc/incubator/tez/tez-api/src/main/java/org/apache/tez/records/OutputContext.java?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-api/src/main/java/org/apache/tez/records/OutputContext.java (added)
+++ incubator/tez/tez-api/src/main/java/org/apache/tez/records/OutputContext.java Fri Mar 15 21:26:36 2013
@@ -0,0 +1,61 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.tez.records;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.io.Writable;
+
+//TODO EVENTUALLY Add more interfaces. Maybe make this an abstract class.
+/**
+ * Contains context information for Output. For example, shuffle headers, size
+ * of output, etc. TODO Ideally should be Output specific.
+ */
+
+public class OutputContext implements Writable {
+
+ public OutputContext(int shufflePort) {
+ this.shufflePort = shufflePort;
+ }
+
+ public OutputContext() {
+ }
+
+ public int shufflePort;
+
+ public int getShufflePort() {
+ return this.shufflePort;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeInt(shufflePort);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ shufflePort = in.readInt();
+ }
+
+ @Override
+ public String toString() {
+ return "shufflePort: " + shufflePort;
+ }
+}
Added: incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezDependentTaskCompletionEvent.java
URL: http://svn.apache.org/viewvc/incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezDependentTaskCompletionEvent.java?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezDependentTaskCompletionEvent.java (added)
+++ incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezDependentTaskCompletionEvent.java Fri Mar 15 21:26:36 2013
@@ -0,0 +1,217 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.records;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.WritableUtils;
+
+/**
+ * This is used to track task completion events on
+ * job tracker.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+// TODO TEZAM3 This needs to be more generic. Maybe some kind of a serialized
+// blob - which can be interpretted by the Input plugin.
+public class TezDependentTaskCompletionEvent implements Writable {
+ @InterfaceAudience.Public
+ @InterfaceStability.Evolving
+ // TODO EVENTUALLY - Remove TIPFAILED state ?
+ static public enum Status {FAILED, KILLED, SUCCEEDED, OBSOLETE, TIPFAILED};
+
+ private int eventId;
+ // TODO EVENTUALLY - rename.
+ private String taskTrackerHttp;
+ private int taskRunTime; // using int since runtime is the time difference
+ private TezTaskAttemptID taskAttemptId;
+ Status status;
+ // TODO TEZAM2 Get rid of the isMap field. Job specific type information can be determined from TaskAttemptId.getTaskType
+ boolean isMap = false;
+ public static final TezDependentTaskCompletionEvent[] EMPTY_ARRAY =
+ new TezDependentTaskCompletionEvent[0];
+ /**
+ * Default constructor for Writable.
+ *
+ */
+ public TezDependentTaskCompletionEvent(){
+ taskAttemptId = new TezTaskAttemptID();
+ }
+
+ /**
+ * Constructor. eventId should be created externally and incremented
+ * per event for each job.
+ * @param eventId event id, event id should be unique and assigned in
+ * incrementally, starting from 0.
+ * @param taskId task id
+ * @param status task's status
+ * @param taskTrackerHttp task tracker's host:port for http.
+ */
+ public TezDependentTaskCompletionEvent(int eventId,
+ TezTaskAttemptID taskId,
+ boolean isMap,
+ Status status,
+ String taskTrackerHttp){
+
+ this.taskAttemptId = taskId;
+ this.isMap = isMap;
+ this.eventId = eventId;
+ this.status =status;
+ this.taskTrackerHttp = taskTrackerHttp;
+ }
+ /**
+ * Returns event Id.
+ * @return event id
+ */
+ public int getEventId() {
+ return eventId;
+ }
+
+ /**
+ * Returns task id.
+ * @return task id
+ */
+ public TezTaskAttemptID getTaskAttemptID() {
+ return taskAttemptId;
+ }
+
+ /**
+ * Returns enum Status.SUCESS or Status.FAILURE.
+ * @return task tracker status
+ */
+ public Status getStatus() {
+ return status;
+ }
+ /**
+ * http location of the tasktracker where this task ran.
+ * @return http location of tasktracker user logs
+ */
+ public String getTaskTrackerHttp() {
+ return taskTrackerHttp;
+ }
+
+ /**
+ * Returns time (in millisec) the task took to complete.
+ */
+ public int getTaskRunTime() {
+ return taskRunTime;
+ }
+
+ /**
+ * Set the task completion time
+ * @param taskCompletionTime time (in millisec) the task took to complete
+ */
+ protected void setTaskRunTime(int taskCompletionTime) {
+ this.taskRunTime = taskCompletionTime;
+ }
+
+ /**
+ * set event Id. should be assigned incrementally starting from 0.
+ * @param eventId
+ */
+ protected void setEventId(int eventId) {
+ this.eventId = eventId;
+ }
+
+ /**
+ * Sets task id.
+ * @param taskId
+ */
+ protected void setTaskAttemptID(TezTaskAttemptID taskId) {
+ this.taskAttemptId = taskId;
+ }
+
+ /**
+ * Set task status.
+ * @param status
+ */
+ protected void setTaskStatus(Status status) {
+ this.status = status;
+ }
+
+ /**
+ * Set task tracker http location.
+ * @param taskTrackerHttp
+ */
+ protected void setTaskTrackerHttp(String taskTrackerHttp) {
+ this.taskTrackerHttp = taskTrackerHttp;
+ }
+
+ @Override
+ public String toString(){
+ StringBuffer buf = new StringBuffer();
+ buf.append("Task Id : ");
+ buf.append(taskAttemptId);
+ buf.append(", Status : ");
+ buf.append(status.name());
+ return buf.toString();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if(o == null)
+ return false;
+ if(o.getClass().equals(this.getClass())) {
+ TezDependentTaskCompletionEvent event = (TezDependentTaskCompletionEvent) o;
+ return this.isMap == event.isMapTask()
+ && this.eventId == event.getEventId()
+ && this.status.equals(event.getStatus())
+ && this.taskAttemptId.equals(event.getTaskAttemptID())
+ && this.taskRunTime == event.getTaskRunTime()
+ && this.taskTrackerHttp.equals(event.getTaskTrackerHttp());
+ }
+ return false;
+ }
+
+ @Override
+ public int hashCode() {
+ return toString().hashCode();
+ }
+
+ public boolean isMapTask() {
+ return isMap;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ taskAttemptId.write(out);
+ out.writeBoolean(isMap);
+ WritableUtils.writeEnum(out, status);
+ WritableUtils.writeString(out, taskTrackerHttp);
+ WritableUtils.writeVInt(out, taskRunTime);
+ WritableUtils.writeVInt(out, eventId);
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ taskAttemptId.readFields(in);
+ isMap = in.readBoolean();
+ status = WritableUtils.readEnum(in, Status.class);
+ taskTrackerHttp = WritableUtils.readString(in);
+ taskRunTime = WritableUtils.readVInt(in);
+ eventId = WritableUtils.readVInt(in);
+
+ }
+
+}
Added: incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezID.java
URL: http://svn.apache.org/viewvc/incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezID.java?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezID.java (added)
+++ incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezID.java Fri Mar 15 21:26:36 2013
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.records;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.io.WritableComparable;
+
+/**
+ * A general identifier, which internally stores the id
+ * as an integer. This is the super class of {@link TezJobID},
+ * {@link TezTaskID} and {@link TezTaskAttemptID}.
+ *
+ * @see TezJobID
+ * @see TezTaskID
+ * @see TezTaskAttemptID
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public abstract class TezID implements WritableComparable<TezID> {
+ public static final char SEPARATOR = '_';
+ protected int id;
+
+ /** constructs an ID object from the given int */
+ public TezID(int id) {
+ this.id = id;
+ }
+
+ protected TezID() {
+ }
+
+ /** returns the int which represents the identifier */
+ public int getId() {
+ return id;
+ }
+
+ @Override
+ public String toString() {
+ return String.valueOf(id);
+ }
+
+ @Override
+ public int hashCode() {
+ return id;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if(o == null)
+ return false;
+ if (o.getClass() == this.getClass()) {
+ TezID that = (TezID) o;
+ return this.id == that.id;
+ }
+ else
+ return false;
+ }
+
+ /** Compare IDs by associated numbers */
+ public int compareTo(TezID that) {
+ return this.id - that.id;
+ }
+
+ public void readFields(DataInput in) throws IOException {
+ this.id = in.readInt();
+ }
+
+ public void write(DataOutput out) throws IOException {
+ out.writeInt(id);
+ }
+
+}
Added: incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezJobID.java
URL: http://svn.apache.org/viewvc/incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezJobID.java?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezJobID.java (added)
+++ incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezJobID.java Fri Mar 15 21:26:36 2013
@@ -0,0 +1,159 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.records;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.text.NumberFormat;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.io.Text;
+
+/**
+ * JobID represents the immutable and unique identifier for
+ * the job. JobID consists of two parts. First part
+ * represents the jobtracker identifier, so that jobID to jobtracker map
+ * is defined. For cluster setup this string is the jobtracker
+ * start time, for local setting, it is "local".
+ * Second part of the JobID is the job number. <br>
+ * An example JobID is :
+ * <code>job_200707121733_0003</code> , which represents the third job
+ * running at the jobtracker started at <code>200707121733</code>.
+ * <p>
+ * Applications should never construct or parse JobID strings, but rather
+ * use appropriate constructors or {@link #forName(String)} method.
+ *
+ * @see TezTaskID
+ * @see TezTaskAttemptID
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class TezJobID extends TezID
+ implements Comparable<TezID> {
+ protected static final String JOB = "job";
+
+ // Jobid regex for various tools and framework components
+ public static final String JOBID_REGEX =
+ JOB + SEPARATOR + "[0-9]+" + SEPARATOR + "[0-9]+";
+
+ private final Text jtIdentifier;
+
+ protected static final NumberFormat idFormat = NumberFormat.getInstance();
+ static {
+ idFormat.setGroupingUsed(false);
+ idFormat.setMinimumIntegerDigits(4);
+ }
+
+ /**
+ * Constructs a JobID object
+ * @param jtIdentifier jobTracker identifier
+ * @param id job number
+ */
+ public TezJobID(String jtIdentifier, int id) {
+ super(id);
+ this.jtIdentifier = new Text(jtIdentifier);
+ }
+
+ public TezJobID() {
+ jtIdentifier = new Text();
+ }
+
+ public String getJtIdentifier() {
+ return jtIdentifier.toString();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!super.equals(o))
+ return false;
+
+ TezJobID that = (TezJobID)o;
+ return this.jtIdentifier.equals(that.jtIdentifier);
+ }
+
+ /**Compare JobIds by first jtIdentifiers, then by job numbers*/
+ @Override
+ public int compareTo(TezID o) {
+ TezJobID that = (TezJobID)o;
+ int jtComp = this.jtIdentifier.compareTo(that.jtIdentifier);
+ if(jtComp == 0) {
+ return this.id - that.id;
+ }
+ else return jtComp;
+ }
+
+ /**
+ * Add the stuff after the "job" prefix to the given builder. This is useful,
+ * because the sub-ids use this substring at the start of their string.
+ * @param builder the builder to append to
+ * @return the builder that was passed in
+ */
+ public StringBuilder appendTo(StringBuilder builder) {
+ builder.append(SEPARATOR);
+ builder.append(jtIdentifier);
+ builder.append(SEPARATOR);
+ builder.append(idFormat.format(id));
+ return builder;
+ }
+
+ @Override
+ public int hashCode() {
+ return jtIdentifier.hashCode() + id;
+ }
+
+ @Override
+ public String toString() {
+ return appendTo(new StringBuilder(JOB)).toString();
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ super.readFields(in);
+ this.jtIdentifier.readFields(in);
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ super.write(out);
+ jtIdentifier.write(out);
+ }
+
+ /** Construct a JobId object from given string
+ * @return constructed JobId object or null if the given String is null
+ * @throws IllegalArgumentException if the given string is malformed
+ */
+ public static TezJobID forName(String str) throws IllegalArgumentException {
+ if(str == null)
+ return null;
+ try {
+ String[] parts = str.split("_");
+ if(parts.length == 3) {
+ if(parts[0].equals(JOB)) {
+ return new TezJobID(parts[1], Integer.parseInt(parts[2]));
+ }
+ }
+ }catch (Exception ex) {//fall below
+ }
+ throw new IllegalArgumentException("JobId string : " + str
+ + " is not properly formed");
+ }
+
+}
Added: incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezTaskAttemptID.java
URL: http://svn.apache.org/viewvc/incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezTaskAttemptID.java?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezTaskAttemptID.java (added)
+++ incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezTaskAttemptID.java Fri Mar 15 21:26:36 2013
@@ -0,0 +1,154 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.records;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * TaskAttemptID represents the immutable and unique identifier for
+ * a task attempt. Each task attempt is one particular instance of a Map or
+ * Reduce Task identified by its TaskID.
+ *
+ * TaskAttemptID consists of 2 parts. First part is the
+ * {@link TezTaskID}, that this TaskAttemptID belongs to.
+ * Second part is the task attempt number. <br>
+ * An example TaskAttemptID is :
+ * <code>attempt_200707121733_0003_m_000005_0</code> , which represents the
+ * zeroth task attempt for the fifth map task in the third job
+ * running at the jobtracker started at <code>200707121733</code>.
+ * <p>
+ * Applications should never construct or parse TaskAttemptID strings
+ * , but rather use appropriate constructors or {@link #forName(String)}
+ * method.
+ *
+ * @see TezJobID
+ * @see TezTaskID
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class TezTaskAttemptID extends TezID {
+ public static final String ATTEMPT = "attempt";
+ private TezTaskID taskId;
+
+ /**
+ * Constructs a TaskAttemptID object from given {@link TezTaskID}.
+ * @param taskId TaskID that this task belongs to
+ * @param id the task attempt number
+ */
+ public TezTaskAttemptID(TezTaskID taskId, int id) {
+ super(id);
+ if(taskId == null) {
+ throw new IllegalArgumentException("taskId cannot be null");
+ }
+ this.taskId = taskId;
+ }
+
+ /**
+ * Constructs a TaskId object from given parts.
+ * @param jtIdentifier jobTracker identifier
+ * @param jobId job number
+ * @param type the TaskType
+ * @param taskId taskId number
+ * @param id the task attempt number
+ */
+ public TezTaskAttemptID(String jtIdentifier, int jobId, String type,
+ int taskId, int id) {
+ this(new TezTaskID(jtIdentifier, jobId, type, taskId), id);
+ }
+
+ public TezTaskAttemptID() {
+ taskId = new TezTaskID();
+ }
+
+ /** Returns the {@link TezJobID} object that this task attempt belongs to */
+ public TezJobID getJobID() {
+ return taskId.getJobID();
+ }
+
+ /** Returns the {@link TezTaskID} object that this task attempt belongs to */
+ public TezTaskID getTaskID() {
+ return taskId;
+ }
+
+ /**Returns the TaskType of the TaskAttemptID */
+ public String getTaskType() {
+ return taskId.getTaskType();
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!super.equals(o))
+ return false;
+
+ TezTaskAttemptID that = (TezTaskAttemptID)o;
+ return this.taskId.equals(that.taskId);
+ }
+
+ /**
+ * Add the unique string to the StringBuilder
+ * @param builder the builder to append ot
+ * @return the builder that was passed in.
+ */
+ protected StringBuilder appendTo(StringBuilder builder) {
+ return taskId.appendTo(builder).append(SEPARATOR).append(id);
+ }
+
+ public static TezTaskAttemptID read(DataInput in) throws IOException {
+ TezTaskAttemptID taId = new TezTaskAttemptID();
+ taId.readFields(in);
+ return taId;
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ super.readFields(in);
+ taskId.readFields(in);
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ super.write(out);
+ taskId.write(out);
+ }
+
+ @Override
+ public int hashCode() {
+ return taskId.hashCode() * 5 + id;
+ }
+
+ /**Compare TaskIds by first tipIds, then by task numbers. */
+ @Override
+ public int compareTo(TezID o) {
+ TezTaskAttemptID that = (TezTaskAttemptID)o;
+ int tipComp = this.taskId.compareTo(that.taskId);
+ if(tipComp == 0) {
+ return this.id - that.id;
+ }
+ else return tipComp;
+ }
+ @Override
+ public String toString() {
+ return appendTo(new StringBuilder(ATTEMPT)).toString();
+ }
+}
Added: incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezTaskDependencyCompletionEventsUpdate.java
URL: http://svn.apache.org/viewvc/incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezTaskDependencyCompletionEventsUpdate.java?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezTaskDependencyCompletionEventsUpdate.java (added)
+++ incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezTaskDependencyCompletionEventsUpdate.java Fri Mar 15 21:26:36 2013
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.records;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.io.Writable;
+
+public class TezTaskDependencyCompletionEventsUpdate implements Writable {
+ TezDependentTaskCompletionEvent[] events;
+ boolean reset;
+
+ public TezTaskDependencyCompletionEventsUpdate() { }
+
+ public TezTaskDependencyCompletionEventsUpdate(
+ TezDependentTaskCompletionEvent[] events, boolean reset) {
+ this.events = events;
+ this.reset = reset;
+ }
+
+ public boolean shouldReset() {
+ return reset;
+ }
+
+ public TezDependentTaskCompletionEvent[] getDependentTaskCompletionEvents() {
+ return events;
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeBoolean(reset);
+ out.writeInt(events.length);
+ for (TezDependentTaskCompletionEvent event : events) {
+ event.write(out);
+ }
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ reset = in.readBoolean();
+ events = new TezDependentTaskCompletionEvent[in.readInt()];
+ for (int i = 0; i < events.length; ++i) {
+ events[i] = new TezDependentTaskCompletionEvent();
+ events[i].readFields(in);
+ }
+ }
+
+}
Added: incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezTaskID.java
URL: http://svn.apache.org/viewvc/incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezTaskID.java?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezTaskID.java (added)
+++ incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezTaskID.java Fri Mar 15 21:26:36 2013
@@ -0,0 +1,192 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.records;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.text.NumberFormat;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.io.WritableUtils;
+
+
+/**
+ * TaskID represents the immutable and unique identifier for
+ * a Map or Reduce Task. Each TaskID encompasses multiple attempts made to
+ * execute the Map or Reduce Task, each of which are uniquely indentified by
+ * their TaskAttemptID.
+ *
+ * TaskID consists of 3 parts. First part is the {@link TezJobID}, that this
+ * TaskInProgress belongs to. Second part of the TaskID is either 'm' or 'r'
+ * representing whether the task is a map task or a reduce task.
+ * And the third part is the task number. <br>
+ * An example TaskID is :
+ * <code>task_200707121733_0003_m_000005</code> , which represents the
+ * fifth map task in the third job running at the jobtracker
+ * started at <code>200707121733</code>.
+ * <p>
+ * Applications should never construct or parse TaskID strings
+ * , but rather use appropriate constructors or {@link #forName(String)}
+ * method.
+ *
+ * @see TezJobID
+ * @see TezTaskAttemptID
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Stable
+public class TezTaskID extends TezID {
+ public static final String TASK = "task";
+ protected static final NumberFormat idFormat = NumberFormat.getInstance();
+ static {
+ idFormat.setGroupingUsed(false);
+ idFormat.setMinimumIntegerDigits(6);
+ }
+
+ private TezJobID jobId;
+ private String type;
+
+ /**
+ * Constructs a TaskID object from given {@link TezJobID}.
+ * @param jobId JobID that this tip belongs to
+ * @param type the {@link TezTaskType} of the task
+ * @param id the tip number
+ */
+ public TezTaskID(TezJobID jobId, String type, int id) {
+ super(id);
+ if(jobId == null) {
+ throw new IllegalArgumentException("jobId cannot be null");
+ }
+ this.jobId = jobId;
+ this.type = type;
+ }
+
+ /**
+ * Constructs a TaskInProgressId object from given parts.
+ * @param jtIdentifier jobTracker identifier
+ * @param jobId job number
+ * @param type the TaskType
+ * @param id the tip number
+ */
+ public TezTaskID(String jtIdentifier, int jobId, String type, int id) {
+ this(new TezJobID(jtIdentifier, jobId), type, id);
+ }
+
+ public TezTaskID() {
+ jobId = new TezJobID();
+ }
+
+ /** Returns the {@link TezJobID} object that this tip belongs to */
+ public TezJobID getJobID() {
+ return jobId;
+ }
+
+ /**
+ * Get the type of the task
+ */
+ public String getTaskType() {
+ return type;
+ }
+
+ public void readFields(DataInput in) throws IOException {
+ super.readFields(in);
+ jobId.readFields(in);
+ type = WritableUtils.readString(in);
+// String typeClassName = WritableUtils.readString(in);
+// String enumVal = WritableUtils.readString(in);
+//
+// // TODO EVENTUALLY Move this into RelcetionUtils
+// try {
+// Class<?> cls = Class.forName(typeClassName);
+// if (TezTaskType.class.isAssignableFrom(cls)) {
+// Method m = cls.getMethod("valueOf", String.class);
+// m.setAccessible(true);
+// type = (TezTaskType) m.invoke(null, enumVal);
+//
+// } else {
+// throw new RuntimeException("Type: + " + typeClassName
+// + " should be a subclass of " + TezTaskType.class.getName());
+// }
+// } catch (ClassNotFoundException e) {
+// throw new RuntimeException("Unable to load typeClass: " + typeClassName,
+// e);
+// } catch (Exception e) {
+// throw new RuntimeException("Failed to create instance of: "
+// + typeClassName, e);
+// }
+ }
+
+ public void write(DataOutput out) throws IOException {
+ super.write(out);
+ jobId.write(out);
+ WritableUtils.writeString(out, type);
+// WritableUtils.writeString(out, type.getClass().getName());
+// // TODO Maybe define a method for default instantiation.
+// WritableUtils.writeString(out, type.toSerializedString());
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!super.equals(o))
+ return false;
+
+ TezTaskID that = (TezTaskID)o;
+ return this.type.equals(that.type) && this.jobId.equals(that.jobId);
+ }
+
+ /**Compare TaskInProgressIds by first jobIds, then by tip numbers and type.*/
+ @Override
+ public int compareTo(TezID o) {
+ TezTaskID that = (TezTaskID)o;
+ int jobComp = this.jobId.compareTo(that.jobId);
+ if(jobComp == 0) {
+ if(this.type == that.type) {
+ return this.id - that.id;
+ }
+ else {
+ return this.type.toString().compareTo(that.type.toString());
+ }
+ }
+ else return jobComp;
+ }
+ @Override
+ public String toString() {
+ return appendTo(new StringBuilder(TASK)).toString();
+ }
+
+ /**
+ * Add the unique string to the given builder.
+ * @param builder the builder to append to
+ * @return the builder that was passed in
+ */
+ protected StringBuilder appendTo(StringBuilder builder) {
+ return jobId.appendTo(builder).
+ append(SEPARATOR).
+ append(type.toString()).
+ append(SEPARATOR).
+ append(idFormat.format(id));
+ }
+
+ @Override
+ public int hashCode() {
+ return jobId.hashCode() * 524287 + id;
+ }
+
+}
Added: incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezTaskType.java
URL: http://svn.apache.org/viewvc/incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezTaskType.java?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezTaskType.java (added)
+++ incubator/tez/tez-api/src/main/java/org/apache/tez/records/TezTaskType.java Fri Mar 15 21:26:36 2013
@@ -0,0 +1,26 @@
+package org.apache.tez.records;
+///**
+// * Licensed to the Apache Software Foundation (ASF) under one or more
+// * contributor license agreements. See the NOTICE file distributed with this
+// * work for additional information regarding copyright ownership. The ASF
+// * licenses this file to you under the Apache License, Version 2.0 (the
+// * "License"); you may not use this file except in compliance with the License.
+// * You may obtain a copy of the License at
+// *
+// * http://www.apache.org/licenses/LICENSE-2.0
+// *
+// * Unless required by applicable law or agreed to in writing, software
+// * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// * License for the specific language governing permissions and limitations under
+// * the License.
+// */
+//
+//package org.apache.tez.records;
+//
+//
+//public interface TezTaskType {
+//
+// public String toSerializedString();
+//
+//}
Added: incubator/tez/tez-common/.classpath
URL: http://svn.apache.org/viewvc/incubator/tez/tez-common/.classpath?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-common/.classpath (added)
+++ incubator/tez/tez-common/.classpath Fri Mar 15 21:26:36 2013
@@ -0,0 +1,73 @@
+<classpath>
+ <classpathentry kind="src" path="src/main/java" including="**/*.java"/>
+ <classpathentry kind="output" path="target/classes"/>
+ <classpathentry kind="var" path="M2_REPO/javax/activation/activation/1.1/activation-1.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/javax/xml/bind/jaxb-api/2.2.2/jaxb-api-2.2.2.jar"/>
+ <classpathentry kind="var" path="M2_REPO/javax/servlet/jsp/jsp-api/2.1/jsp-api-2.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/javax/servlet/servlet-api/2.5/servlet-api-2.5.jar"/>
+ <classpathentry kind="var" path="M2_REPO/asm/asm/3.1/asm-3.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/apache/avro/avro/1.5.3/avro-1.5.3.jar"/>
+ <classpathentry kind="var" path="M2_REPO/commons-beanutils/commons-beanutils/1.7.0/commons-beanutils-1.7.0.jar"/>
+ <classpathentry kind="var" path="M2_REPO/commons-beanutils/commons-beanutils-core/1.8.0/commons-beanutils-core-1.8.0.jar"/>
+ <classpathentry kind="var" path="M2_REPO/commons-cli/commons-cli/1.2/commons-cli-1.2.jar"/>
+ <classpathentry kind="var" path="M2_REPO/commons-codec/commons-codec/1.4/commons-codec-1.4.jar"/>
+ <classpathentry kind="var" path="M2_REPO/commons-collections/commons-collections/3.2.1/commons-collections-3.2.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/apache/commons/commons-compress/1.4/commons-compress-1.4.jar"/>
+ <classpathentry kind="var" path="M2_REPO/commons-configuration/commons-configuration/1.6/commons-configuration-1.6.jar"/>
+ <classpathentry kind="var" path="M2_REPO/commons-digester/commons-digester/1.8/commons-digester-1.8.jar"/>
+ <classpathentry kind="var" path="M2_REPO/commons-el/commons-el/1.0/commons-el-1.0.jar"/>
+ <classpathentry kind="var" path="M2_REPO/commons-httpclient/commons-httpclient/3.1/commons-httpclient-3.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/commons-io/commons-io/2.1/commons-io-2.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/commons-lang/commons-lang/2.5/commons-lang-2.5.jar"/>
+ <classpathentry kind="var" path="M2_REPO/commons-logging/commons-logging/1.1.1/commons-logging-1.1.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/apache/commons/commons-math/2.1/commons-math-2.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/commons-net/commons-net/3.1/commons-net-3.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/com/google/guava/guava/11.0.2/guava-11.0.2.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/apache/hadoop/hadoop-annotations/3.0.0-SNAPSHOT/hadoop-annotations-3.0.0-SNAPSHOT.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/apache/hadoop/hadoop-auth/3.0.0-SNAPSHOT/hadoop-auth-3.0.0-SNAPSHOT.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/apache/hadoop/hadoop-common/3.0.0-SNAPSHOT/hadoop-common-3.0.0-SNAPSHOT.jar" sourcepath="M2_REPO/org/apache/hadoop/hadoop-common/3.0.0-SNAPSHOT/hadoop-common-3.0.0-SNAPSHOT-sources.jar">
+ <attributes>
+ <attribute value="jar:file:/Users/acmurthy/.m2/repository/org/apache/hadoop/hadoop-common/3.0.0-SNAPSHOT/hadoop-common-3.0.0-SNAPSHOT-javadoc.jar!/" name="javadoc_location"/>
+ </attributes>
+ </classpathentry>
+ <classpathentry kind="var" path="M2_REPO/org/apache/hadoop/hadoop-yarn-api/3.0.0-SNAPSHOT/hadoop-yarn-api-3.0.0-SNAPSHOT.jar" sourcepath="M2_REPO/org/apache/hadoop/hadoop-yarn-api/3.0.0-SNAPSHOT/hadoop-yarn-api-3.0.0-SNAPSHOT-sources.jar">
+ <attributes>
+ <attribute value="jar:file:/Users/acmurthy/.m2/repository/org/apache/hadoop/hadoop-yarn-api/3.0.0-SNAPSHOT/hadoop-yarn-api-3.0.0-SNAPSHOT-javadoc.jar!/" name="javadoc_location"/>
+ </attributes>
+ </classpathentry>
+ <classpathentry kind="var" path="M2_REPO/org/apache/hadoop/hadoop-yarn-common/3.0.0-SNAPSHOT/hadoop-yarn-common-3.0.0-SNAPSHOT.jar" sourcepath="M2_REPO/org/apache/hadoop/hadoop-yarn-common/3.0.0-SNAPSHOT/hadoop-yarn-common-3.0.0-SNAPSHOT-sources.jar">
+ <attributes>
+ <attribute value="jar:file:/Users/acmurthy/.m2/repository/org/apache/hadoop/hadoop-yarn-common/3.0.0-SNAPSHOT/hadoop-yarn-common-3.0.0-SNAPSHOT-javadoc.jar!/" name="javadoc_location"/>
+ </attributes>
+ </classpathentry>
+ <classpathentry kind="var" path="M2_REPO/org/codehaus/jackson/jackson-core-asl/1.8.8/jackson-core-asl-1.8.8.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/codehaus/jackson/jackson-jaxrs/1.7.1/jackson-jaxrs-1.7.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/codehaus/jackson/jackson-mapper-asl/1.8.8/jackson-mapper-asl-1.8.8.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/codehaus/jackson/jackson-xc/1.7.1/jackson-xc-1.7.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/tomcat/jasper-compiler/5.5.23/jasper-compiler-5.5.23.jar"/>
+ <classpathentry kind="var" path="M2_REPO/tomcat/jasper-runtime/5.5.23/jasper-runtime-5.5.23.jar"/>
+ <classpathentry kind="var" path="M2_REPO/com/sun/xml/bind/jaxb-impl/2.2.3-1/jaxb-impl-2.2.3-1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/com/sun/jersey/jersey-core/1.8/jersey-core-1.8.jar"/>
+ <classpathentry kind="var" path="M2_REPO/com/sun/jersey/jersey-json/1.8/jersey-json-1.8.jar"/>
+ <classpathentry kind="var" path="M2_REPO/com/sun/jersey/jersey-server/1.8/jersey-server-1.8.jar"/>
+ <classpathentry kind="var" path="M2_REPO/net/java/dev/jets3t/jets3t/0.6.1/jets3t-0.6.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/codehaus/jettison/jettison/1.1/jettison-1.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/mortbay/jetty/jetty/6.1.26/jetty-6.1.26.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/mortbay/jetty/jetty-util/6.1.26/jetty-util-6.1.26.jar"/>
+ <classpathentry kind="var" path="M2_REPO/jline/jline/0.9.94/jline-0.9.94.jar"/>
+ <classpathentry kind="var" path="M2_REPO/com/jcraft/jsch/0.1.42/jsch-0.1.42.jar"/>
+ <classpathentry kind="var" path="M2_REPO/com/google/code/findbugs/jsr305/1.3.9/jsr305-1.3.9.jar"/>
+ <classpathentry kind="var" path="M2_REPO/log4j/log4j/1.2.17/log4j-1.2.17.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/jboss/netty/netty/3.2.2.Final/netty-3.2.2.Final.jar"/>
+ <classpathentry kind="var" path="M2_REPO/com/thoughtworks/paranamer/paranamer/2.3/paranamer-2.3.jar"/>
+ <classpathentry kind="var" path="M2_REPO/com/google/protobuf/protobuf-java/2.4.0a/protobuf-java-2.4.0a.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/slf4j/slf4j-api/1.6.1/slf4j-api-1.6.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/slf4j/slf4j-log4j12/1.6.1/slf4j-log4j12-1.6.1.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/xerial/snappy/snappy-java/1.0.3.2/snappy-java-1.0.3.2.jar"/>
+ <classpathentry kind="var" path="M2_REPO/stax/stax-api/1.0.1/stax-api-1.0.1.jar"/>
+ <classpathentry kind="src" path="/tez-api"/>
+ <classpathentry kind="var" path="M2_REPO/xmlenc/xmlenc/0.52/xmlenc-0.52.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/tukaani/xz/1.0/xz-1.0.jar"/>
+ <classpathentry kind="var" path="M2_REPO/org/apache/zookeeper/zookeeper/3.4.2/zookeeper-3.4.2.jar"/>
+ <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
+</classpath>
\ No newline at end of file
Added: incubator/tez/tez-common/.project
URL: http://svn.apache.org/viewvc/incubator/tez/tez-common/.project?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-common/.project (added)
+++ incubator/tez/tez-common/.project Fri Mar 15 21:26:36 2013
@@ -0,0 +1,15 @@
+<projectDescription>
+ <name>tez-common</name>
+ <comment>NO_M2ECLIPSE_SUPPORT: Project files created with the maven-eclipse-plugin are not supported in M2Eclipse.</comment>
+ <projects>
+ <project>tez-api</project>
+ </projects>
+ <buildSpec>
+ <buildCommand>
+ <name>org.eclipse.jdt.core.javabuilder</name>
+ </buildCommand>
+ </buildSpec>
+ <natures>
+ <nature>org.eclipse.jdt.core.javanature</nature>
+ </natures>
+</projectDescription>
\ No newline at end of file
Added: incubator/tez/tez-common/pom.xml
URL: http://svn.apache.org/viewvc/incubator/tez/tez-common/pom.xml?rev=1457129&view=auto
==============================================================================
--- incubator/tez/tez-common/pom.xml (added)
+++ incubator/tez/tez-common/pom.xml Fri Mar 15 21:26:36 2013
@@ -0,0 +1,45 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez</artifactId>
+ <version>0.1.0</version>
+ </parent>
+ <artifactId>tez-common</artifactId>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.tez</groupId>
+ <artifactId>tez-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ </dependency>
+ </dependencies>
+</project>