You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@twill.apache.org by ch...@apache.org on 2013/11/21 22:54:24 UTC
[01/15] Initial import commit.
Updated Branches:
refs/heads/master [created] 1925ffafc
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/zookeeper/src/test/resources/logback-test.xml
----------------------------------------------------------------------
diff --git a/zookeeper/src/test/resources/logback-test.xml b/zookeeper/src/test/resources/logback-test.xml
new file mode 100644
index 0000000..157df6e
--- /dev/null
+++ b/zookeeper/src/test/resources/logback-test.xml
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!-- Default logback configuration for twill library -->
+<configuration>
+ <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n</pattern>
+ </encoder>
+ </appender>
+
+ <logger name="org.apache.hadoop" level="WARN" />
+ <logger name="org.apache.zookeeper" level="WARN" />
+
+ <root level="INFO">
+ <appender-ref ref="STDOUT"/>
+ </root>
+</configuration>
[08/15] Initial import commit.
Posted by ch...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
new file mode 100644
index 0000000..e6f0522
--- /dev/null
+++ b/pom.xml
@@ -0,0 +1,556 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>org.apache.twill</groupId>
+ <artifactId>twill-parent</artifactId>
+ <version>1.3.0-SNAPSHOT</version>
+ <packaging>pom</packaging>
+ <name>Twill library parent project</name>
+
+ <modules>
+ <module>common</module>
+ <module>discovery-api</module>
+ <module>api</module>
+ <module>zookeeper</module>
+ <module>discovery-core</module>
+ <module>core</module>
+ <module>yarn</module>
+ </modules>
+
+ <repositories>
+ <repository>
+ <id>cloudera-releases</id>
+ <url>https://repository.cloudera.com/artifactory/cloudera-repos</url>
+ <releases>
+ <enabled>true</enabled>
+ </releases>
+ <snapshots>
+ <enabled>false</enabled>
+ </snapshots>
+ </repository>
+ </repositories>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <surefire.redirectTestOutputToFile>true</surefire.redirectTestOutputToFile>
+ <hadoop.version>[2.0.2-alpha,2.2.0]</hadoop.version>
+ <hadoop20.output.dir>target/hadoop20-classes</hadoop20.output.dir>
+ </properties>
+
+ <scm>
+ <connection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-twill.git</connection>
+ <url>https://git-wip-us.apache.org/repos/asf?p=incubator-twill.git;a=summary</url>
+ <developerConnection>scm:git:https://git-wip-us.apache.org/repos/asf/incubator-twill.git</developerConnection>
+ </scm>
+
+ <build>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-source-plugin</artifactId>
+ <version>2.2.1</version>
+ <configuration>
+ <excludeResources>true</excludeResources>
+ </configuration>
+ <executions>
+ <execution>
+ <id>attach-sources</id>
+ <phase>package</phase>
+ <goals>
+ <goal>jar-no-fork</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <version>2.9</version>
+ <configuration>
+ <excludePackageNames>*.internal.*</excludePackageNames>
+ <links>
+ <link>http://download.oracle.com/javase/6/docs/api/</link>
+ </links>
+ <bottom>
+ <![CDATA[Copyright © 2013 <a href="http://www.apache.org">The Apache Software Foundation</a>. All rights reserved.]]>
+ </bottom>
+ </configuration>
+ <executions>
+ <execution>
+ <id>attach-javadoc</id>
+ <phase>package</phase>
+ <goals>
+ <goal>jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.1</version>
+ <configuration>
+ <source>1.6</source>
+ <target>1.6</target>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-source-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-release-plugin</artifactId>
+ <version>2.4.1</version>
+ <configuration>
+ <autoVersionSubmodules>true</autoVersionSubmodules>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-site-plugin</artifactId>
+ <version>3.2</version>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.14.1</version>
+ <configuration>
+ <argLine>-Xmx512m</argLine>
+ <redirectTestOutputToFile>${surefire.redirectTestOutputToFile}</redirectTestOutputToFile>
+ <systemPropertyVariables>
+ <java.io.tmpdir>${project.build.directory}</java.io.tmpdir>
+ </systemPropertyVariables>
+ <includes>
+ <include>**/*TestSuite.java</include>
+ <include>**/Test*.java</include>
+ <include>**/*Test.java</include>
+ <include>**/*TestCase.java</include>
+ </includes>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-deploy-plugin</artifactId>
+ <version>2.8</version>
+ <configuration>
+ <deployAtEnd>true</deployAtEnd>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
+
+ <profiles>
+ <profile>
+ <id>hadoop-2.0</id>
+ <properties>
+ <hadoop.version>2.0.2-alpha</hadoop.version>
+ </properties>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.8</version>
+ <executions>
+ <execution>
+ <id>add-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>src/main/hadoop20</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <profile>
+ <id>cdh-4.4.0</id>
+ <properties>
+ <hadoop.version>2.0.0-cdh4.4.0</hadoop.version>
+ </properties>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.8</version>
+ <executions>
+ <execution>
+ <id>add-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>src/main/hadoop20</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <profile>
+ <id>hadoop-2.1</id>
+ <properties>
+ <hadoop.version>2.1.0-beta</hadoop.version>
+ </properties>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.8</version>
+ <executions>
+ <execution>
+ <id>add-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>src/main/hadoop21</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <profile>
+ <id>hadoop-2.2</id>
+ <properties>
+ <hadoop.version>2.2.0</hadoop.version>
+ </properties>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ </activation>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <version>1.8</version>
+ <executions>
+ <execution>
+ <id>add-source</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>add-source</goal>
+ </goals>
+ <configuration>
+ <sources>
+ <source>src/main/hadoop21</source>
+ </sources>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+
+ </profiles>
+
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>13.0.1</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ <version>2.0.1</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>2.2.4</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ <version>3.4.5</version>
+ <exclusions>
+ <exclusion>
+ <artifactId>slf4j-api</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>slf4j-log4j12</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>log4j</artifactId>
+ <groupId>log4j</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>junit</artifactId>
+ <groupId>junit</groupId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.jboss.netty</groupId>
+ <artifactId>netty</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ <version>3.6.6.Final</version>
+ </dependency>
+ <dependency>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ <version>1.0.4.1</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>1.7.5</version>
+ </dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-core</artifactId>
+ <version>1.0.9</version>
+ </dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ <version>1.0.9</version>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jcl-over-slf4j</artifactId>
+ <version>1.7.2</version>
+ </dependency>
+ <dependency>
+ <groupId>org.ow2.asm</groupId>
+ <artifactId>asm-all</artifactId>
+ <version>4.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.jboss.netty</groupId>
+ <artifactId>netty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-client</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ </exclusion>
+ <exclusion>
+ <artifactId>guava</artifactId>
+ <groupId>com.google.guava</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jersey-core</artifactId>
+ <groupId>com.sun.jersey</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jersey-json</artifactId>
+ <groupId>com.sun.jersey</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jersey-server</artifactId>
+ <groupId>com.sun.jersey</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jasper-compiler</artifactId>
+ <groupId>tomcat</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jasper-runtime</artifactId>
+ <groupId>tomcat</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>jsp-api</artifactId>
+ <groupId>javax.servlet.jsp</groupId>
+ </exclusion>
+ <exclusion>
+ <artifactId>slf4j-api</artifactId>
+ <groupId>org.slf4j</groupId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ <version>${hadoop.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>log4j</groupId>
+ <artifactId>log4j</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>tomcat</groupId>
+ <artifactId>jasper-runtime</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.servlet.jsp</groupId>
+ <artifactId>jsp-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-daemon</groupId>
+ <artifactId>commons-daemon</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ <version>${hadoop.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-log4j12</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ <version>4.11</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ <version>1.5</version>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+
+ <reporting>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-report-plugin</artifactId>
+ <version>2.14.1</version>
+ </plugin>
+ </plugins>
+ </reporting>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/pom.xml
----------------------------------------------------------------------
diff --git a/yarn/pom.xml b/yarn/pom.xml
new file mode 100644
index 0000000..382d104
--- /dev/null
+++ b/yarn/pom.xml
@@ -0,0 +1,127 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>twill-parent</artifactId>
+ <groupId>org.apache.twill</groupId>
+ <version>1.3.0-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>twill-yarn</artifactId>
+ <name>Twill Apache Hadoop YARN library</name>
+
+ <properties>
+ <output.dir>target/classes</output.dir>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>twill-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>twill-discovery-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jcl-over-slf4j</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-yarn-client</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-common</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-hdfs</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-minicluster</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <outputDirectory>${output.dir}</outputDirectory>
+ </build>
+
+ <profiles>
+ <profile>
+ <id>hadoop-2.0</id>
+ <properties>
+ <output.dir>${hadoop20.output.dir}</output.dir>
+ </properties>
+ </profile>
+ <profile>
+ <id>hadoop-2.1</id>
+ <build>
+ <resources>
+ <resource>
+ <directory>${hadoop20.output.dir}</directory>
+ </resource>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+ </build>
+ </profile>
+ <profile>
+ <id>hadoop-2.2</id>
+ <build>
+ <resources>
+ <resource>
+ <directory>${hadoop20.output.dir}</directory>
+ </resource>
+ <resource>
+ <directory>src/main/resources</directory>
+ </resource>
+ </resources>
+ </build>
+ </profile>
+ </profiles>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAMClient.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAMClient.java b/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAMClient.java
new file mode 100644
index 0000000..d98dee1
--- /dev/null
+++ b/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAMClient.java
@@ -0,0 +1,213 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn;
+
+import org.apache.twill.internal.ProcessLauncher;
+import org.apache.twill.internal.appmaster.RunnableProcessLauncher;
+import org.apache.twill.internal.yarn.ports.AMRMClient;
+import org.apache.twill.internal.yarn.ports.AMRMClientImpl;
+import org.apache.twill.internal.yarn.ports.AllocationResponse;
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Multimap;
+import com.google.common.util.concurrent.AbstractIdleService;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.InetSocketAddress;
+import java.net.URL;
+import java.util.List;
+import java.util.UUID;
+
+/**
+ *
+ */
+public final class Hadoop20YarnAMClient extends AbstractIdleService implements YarnAMClient {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Hadoop20YarnAMClient.class);
+ private static final Function<ContainerStatus, YarnContainerStatus> STATUS_TRANSFORM;
+
+ static {
+ STATUS_TRANSFORM = new Function<ContainerStatus, YarnContainerStatus>() {
+ @Override
+ public YarnContainerStatus apply(ContainerStatus status) {
+ return new Hadoop20YarnContainerStatus(status);
+ }
+ };
+ }
+
+ private final ContainerId containerId;
+ private final Multimap<String, AMRMClient.ContainerRequest> containerRequests;
+ private final AMRMClient amrmClient;
+ private final YarnNMClient nmClient;
+ private InetSocketAddress trackerAddr;
+ private URL trackerUrl;
+ private Resource maxCapability;
+ private Resource minCapability;
+
+ public Hadoop20YarnAMClient(Configuration conf) {
+ String masterContainerId = System.getenv().get(ApplicationConstants.AM_CONTAINER_ID_ENV);
+ Preconditions.checkArgument(masterContainerId != null,
+ "Missing %s from environment", ApplicationConstants.AM_CONTAINER_ID_ENV);
+ this.containerId = ConverterUtils.toContainerId(masterContainerId);
+ this.containerRequests = ArrayListMultimap.create();
+
+ this.amrmClient = new AMRMClientImpl(containerId.getApplicationAttemptId());
+ this.amrmClient.init(conf);
+ this.nmClient = new Hadoop20YarnNMClient(YarnRPC.create(conf), conf);
+ }
+
+ @Override
+ protected void startUp() throws Exception {
+ Preconditions.checkNotNull(trackerAddr, "Tracker address not set.");
+ Preconditions.checkNotNull(trackerUrl, "Tracker URL not set.");
+
+ amrmClient.start();
+
+ RegisterApplicationMasterResponse response = amrmClient.registerApplicationMaster(trackerAddr.getHostName(),
+ trackerAddr.getPort(),
+ trackerUrl.toString());
+ maxCapability = response.getMaximumResourceCapability();
+ minCapability = response.getMinimumResourceCapability();
+ }
+
+ @Override
+ protected void shutDown() throws Exception {
+ amrmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null, trackerUrl.toString());
+ amrmClient.stop();
+ }
+
+ @Override
+ public ContainerId getContainerId() {
+ return containerId;
+ }
+
+ @Override
+ public String getHost() {
+ return System.getenv().get(ApplicationConstants.NM_HOST_ENV);
+ }
+
+ @Override
+ public void setTracker(InetSocketAddress trackerAddr, URL trackerUrl) {
+ this.trackerAddr = trackerAddr;
+ this.trackerUrl = trackerUrl;
+ }
+
+ @Override
+ public synchronized void allocate(float progress, AllocateHandler handler) throws Exception {
+ AllocationResponse response = amrmClient.allocate(progress);
+ List<ProcessLauncher<YarnContainerInfo>> launchers
+ = Lists.newArrayListWithCapacity(response.getAllocatedContainers().size());
+
+ for (Container container : response.getAllocatedContainers()) {
+ launchers.add(new RunnableProcessLauncher(new Hadoop20YarnContainerInfo(container), nmClient));
+ }
+
+ if (!launchers.isEmpty()) {
+ handler.acquired(launchers);
+
+ // If no process has been launched through the given launcher, return the container.
+ for (ProcessLauncher<YarnContainerInfo> l : launchers) {
+ // This cast always works.
+ RunnableProcessLauncher launcher = (RunnableProcessLauncher) l;
+ if (!launcher.isLaunched()) {
+ Container container = launcher.getContainerInfo().getContainer();
+ LOG.info("Nothing to run in container, releasing it: {}", container);
+ amrmClient.releaseAssignedContainer(container.getId());
+ }
+ }
+ }
+
+ List<YarnContainerStatus> completed = ImmutableList.copyOf(
+ Iterables.transform(response.getCompletedContainersStatuses(), STATUS_TRANSFORM));
+ if (!completed.isEmpty()) {
+ handler.completed(completed);
+ }
+ }
+
+ @Override
+ public ContainerRequestBuilder addContainerRequest(Resource capability) {
+ return addContainerRequest(capability, 1);
+ }
+
+ @Override
+ public ContainerRequestBuilder addContainerRequest(Resource capability, int count) {
+ return new ContainerRequestBuilder(adjustCapability(capability), count) {
+ @Override
+ public String apply() {
+ synchronized (Hadoop20YarnAMClient.this) {
+ String id = UUID.randomUUID().toString();
+
+ String[] hosts = this.hosts.isEmpty() ? null : this.hosts.toArray(new String[this.hosts.size()]);
+ String[] racks = this.racks.isEmpty() ? null : this.racks.toArray(new String[this.racks.size()]);
+
+ for (int i = 0; i < count; i++) {
+ AMRMClient.ContainerRequest request = new AMRMClient.ContainerRequest(capability, hosts, racks,
+ priority, 1);
+ containerRequests.put(id, request);
+ amrmClient.addContainerRequest(request);
+ }
+
+ return id;
+ }
+ }
+ };
+ }
+
+ @Override
+ public synchronized void completeContainerRequest(String id) {
+ for (AMRMClient.ContainerRequest request : containerRequests.removeAll(id)) {
+ amrmClient.removeContainerRequest(request);
+ }
+ }
+
+ private Resource adjustCapability(Resource resource) {
+ int cores = YarnUtils.getVirtualCores(resource);
+ int updatedCores = Math.max(Math.min(cores, YarnUtils.getVirtualCores(maxCapability)),
+ YarnUtils.getVirtualCores(minCapability));
+ // Try and set the virtual cores, which older versions of YARN don't support this.
+ if (cores != updatedCores && YarnUtils.setVirtualCores(resource, updatedCores)) {
+ LOG.info("Adjust virtual cores requirement from {} to {}.", cores, updatedCores);
+ }
+
+ int updatedMemory = Math.min(resource.getMemory(), maxCapability.getMemory());
+ int minMemory = minCapability.getMemory();
+ updatedMemory = (int) Math.ceil(((double) updatedMemory / minMemory)) * minMemory;
+
+ if (resource.getMemory() != updatedMemory) {
+ resource.setMemory(updatedMemory);
+ LOG.info("Adjust memory requirement from {} to {} MB.", resource.getMemory(), updatedMemory);
+ }
+
+ return resource;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAppClient.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAppClient.java b/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAppClient.java
new file mode 100644
index 0000000..bfec34e
--- /dev/null
+++ b/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnAppClient.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn;
+
+import org.apache.twill.api.TwillSpecification;
+import org.apache.twill.internal.ProcessController;
+import org.apache.twill.internal.ProcessLauncher;
+import org.apache.twill.internal.appmaster.ApplicationMasterProcessLauncher;
+import org.apache.twill.internal.appmaster.ApplicationSubmitter;
+import com.google.common.base.Throwables;
+import com.google.common.util.concurrent.AbstractIdleService;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.DelegationToken;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.client.YarnClient;
+import org.apache.hadoop.yarn.client.YarnClientImpl;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.util.Records;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.InetSocketAddress;
+
+/**
+ *
+ */
+public final class Hadoop20YarnAppClient extends AbstractIdleService implements YarnAppClient {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Hadoop20YarnAppClient.class);
+ private final YarnClient yarnClient;
+ private String user;
+
+ public Hadoop20YarnAppClient(Configuration configuration) {
+ this.yarnClient = new YarnClientImpl();
+ yarnClient.init(configuration);
+ this.user = System.getProperty("user.name");
+ }
+
+ @Override
+ public ProcessLauncher<ApplicationId> createLauncher(TwillSpecification twillSpec) throws Exception {
+ // Request for new application
+ final GetNewApplicationResponse response = yarnClient.getNewApplication();
+ final ApplicationId appId = response.getApplicationId();
+
+ // Setup the context for application submission
+ final ApplicationSubmissionContext appSubmissionContext = Records.newRecord(ApplicationSubmissionContext.class);
+ appSubmissionContext.setApplicationId(appId);
+ appSubmissionContext.setApplicationName(twillSpec.getName());
+ appSubmissionContext.setUser(user);
+
+ ApplicationSubmitter submitter = new ApplicationSubmitter() {
+
+ @Override
+ public ProcessController<YarnApplicationReport> submit(YarnLaunchContext launchContext, Resource capability) {
+ ContainerLaunchContext context = launchContext.getLaunchContext();
+ addRMToken(context);
+ context.setUser(appSubmissionContext.getUser());
+ context.setResource(adjustMemory(response, capability));
+ appSubmissionContext.setAMContainerSpec(context);
+
+ try {
+ yarnClient.submitApplication(appSubmissionContext);
+ return new ProcessControllerImpl(yarnClient, appId);
+ } catch (YarnRemoteException e) {
+ LOG.error("Failed to submit application {}", appId, e);
+ throw Throwables.propagate(e);
+ }
+ }
+ };
+
+ return new ApplicationMasterProcessLauncher(appId, submitter);
+ }
+
+ private Resource adjustMemory(GetNewApplicationResponse response, Resource capability) {
+ int minMemory = response.getMinimumResourceCapability().getMemory();
+
+ int updatedMemory = Math.min(capability.getMemory(), response.getMaximumResourceCapability().getMemory());
+ updatedMemory = (int) Math.ceil(((double) updatedMemory / minMemory)) * minMemory;
+
+ if (updatedMemory != capability.getMemory()) {
+ capability.setMemory(updatedMemory);
+ }
+
+ return capability;
+ }
+
+ private void addRMToken(ContainerLaunchContext context) {
+ if (!UserGroupInformation.isSecurityEnabled()) {
+ return;
+ }
+
+ try {
+ Credentials credentials = YarnUtils.decodeCredentials(context.getContainerTokens());
+
+ Configuration config = yarnClient.getConfig();
+ Token<TokenIdentifier> token = convertToken(
+ yarnClient.getRMDelegationToken(new Text(YarnUtils.getYarnTokenRenewer(config))),
+ YarnUtils.getRMAddress(config));
+
+ LOG.info("Added RM delegation token {}", token);
+ credentials.addToken(token.getService(), token);
+
+ context.setContainerTokens(YarnUtils.encodeCredentials(credentials));
+
+ } catch (Exception e) {
+ LOG.error("Fails to create credentials.", e);
+ throw Throwables.propagate(e);
+ }
+ }
+
+ private <T extends TokenIdentifier> Token<T> convertToken(DelegationToken protoToken, InetSocketAddress serviceAddr) {
+ Token<T> token = new Token<T>(protoToken.getIdentifier().array(),
+ protoToken.getPassword().array(),
+ new Text(protoToken.getKind()),
+ new Text(protoToken.getService()));
+ if (serviceAddr != null) {
+ SecurityUtil.setTokenService(token, serviceAddr);
+ }
+ return token;
+ }
+
+ @Override
+ public ProcessLauncher<ApplicationId> createLauncher(String user, TwillSpecification twillSpec) throws Exception {
+ this.user = user;
+ return createLauncher(twillSpec);
+ }
+
+ @Override
+ public ProcessController<YarnApplicationReport> createProcessController(ApplicationId appId) {
+ return new ProcessControllerImpl(yarnClient, appId);
+ }
+
+ @Override
+ protected void startUp() throws Exception {
+ yarnClient.start();
+ }
+
+ @Override
+ protected void shutDown() throws Exception {
+ yarnClient.stop();
+ }
+
+ private static final class ProcessControllerImpl implements ProcessController<YarnApplicationReport> {
+ private final YarnClient yarnClient;
+ private final ApplicationId appId;
+
+ public ProcessControllerImpl(YarnClient yarnClient, ApplicationId appId) {
+ this.yarnClient = yarnClient;
+ this.appId = appId;
+ }
+
+ @Override
+ public YarnApplicationReport getReport() {
+ try {
+ return new Hadoop20YarnApplicationReport(yarnClient.getApplicationReport(appId));
+ } catch (YarnRemoteException e) {
+ LOG.error("Failed to get application report {}", appId, e);
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ public void cancel() {
+ try {
+ yarnClient.killApplication(appId);
+ } catch (YarnRemoteException e) {
+ LOG.error("Failed to kill application {}", appId, e);
+ throw Throwables.propagate(e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnApplicationReport.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnApplicationReport.java b/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnApplicationReport.java
new file mode 100644
index 0000000..6c1b764
--- /dev/null
+++ b/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnApplicationReport.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+
+/**
+ *
+ */
+public final class Hadoop20YarnApplicationReport implements YarnApplicationReport {
+
+ private final ApplicationReport report;
+
+ public Hadoop20YarnApplicationReport(ApplicationReport report) {
+ this.report = report;
+ }
+
+ @Override
+ public ApplicationId getApplicationId() {
+ return report.getApplicationId();
+ }
+
+ @Override
+ public ApplicationAttemptId getCurrentApplicationAttemptId() {
+ return report.getCurrentApplicationAttemptId();
+ }
+
+ @Override
+ public String getQueue() {
+ return report.getQueue();
+ }
+
+ @Override
+ public String getName() {
+ return report.getName();
+ }
+
+ @Override
+ public String getHost() {
+ return report.getHost();
+ }
+
+ @Override
+ public int getRpcPort() {
+ return report.getRpcPort();
+ }
+
+ @Override
+ public YarnApplicationState getYarnApplicationState() {
+ return report.getYarnApplicationState();
+ }
+
+ @Override
+ public String getDiagnostics() {
+ return report.getDiagnostics();
+ }
+
+ @Override
+ public String getTrackingUrl() {
+ return report.getTrackingUrl();
+ }
+
+ @Override
+ public String getOriginalTrackingUrl() {
+ return report.getOriginalTrackingUrl();
+ }
+
+ @Override
+ public long getStartTime() {
+ return report.getStartTime();
+ }
+
+ @Override
+ public long getFinishTime() {
+ return report.getFinishTime();
+ }
+
+ @Override
+ public FinalApplicationStatus getFinalApplicationStatus() {
+ return report.getFinalApplicationStatus();
+ }
+
+ @Override
+ public ApplicationResourceUsageReport getApplicationResourceUsageReport() {
+ return report.getApplicationResourceUsageReport();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnContainerInfo.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnContainerInfo.java b/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnContainerInfo.java
new file mode 100644
index 0000000..79b2cb5
--- /dev/null
+++ b/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnContainerInfo.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn;
+
+import com.google.common.base.Throwables;
+import org.apache.hadoop.yarn.api.records.Container;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
+/**
+ *
+ */
+public final class Hadoop20YarnContainerInfo implements YarnContainerInfo {
+
+ private final Container container;
+
+ public Hadoop20YarnContainerInfo(Container container) {
+ this.container = container;
+ }
+
+ @Override
+ public <T> T getContainer() {
+ return (T) container;
+ }
+
+ @Override
+ public String getId() {
+ return container.getId().toString();
+ }
+
+ @Override
+ public InetAddress getHost() {
+ try {
+ return InetAddress.getByName(container.getNodeId().getHost());
+ } catch (UnknownHostException e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ public int getPort() {
+ return container.getNodeId().getPort();
+ }
+
+ @Override
+ public int getMemoryMB() {
+ return container.getResource().getMemory();
+ }
+
+ @Override
+ public int getVirtualCores() {
+ return YarnUtils.getVirtualCores(container.getResource());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnContainerStatus.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnContainerStatus.java b/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnContainerStatus.java
new file mode 100644
index 0000000..cc61856
--- /dev/null
+++ b/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnContainerStatus.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn;
+
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+
+/**
+ *
+ */
+public final class Hadoop20YarnContainerStatus implements YarnContainerStatus {
+
+ private final ContainerStatus containerStatus;
+
+ public Hadoop20YarnContainerStatus(ContainerStatus containerStatus) {
+ this.containerStatus = containerStatus;
+ }
+
+ @Override
+ public String getContainerId() {
+ return containerStatus.getContainerId().toString();
+ }
+
+ @Override
+ public ContainerState getState() {
+ return containerStatus.getState();
+ }
+
+ @Override
+ public int getExitStatus() {
+ return containerStatus.getExitStatus();
+ }
+
+ @Override
+ public String getDiagnostics() {
+ return containerStatus.getDiagnostics();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnLaunchContext.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnLaunchContext.java b/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnLaunchContext.java
new file mode 100644
index 0000000..b1f6d66
--- /dev/null
+++ b/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnLaunchContext.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Maps;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.util.Records;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Map;
+
+/**
+ *
+ */
+public final class Hadoop20YarnLaunchContext implements YarnLaunchContext {
+
+ private static final Function<YarnLocalResource, LocalResource> RESOURCE_TRANSFORM;
+
+ static {
+ // Creates transform function from YarnLocalResource -> LocalResource
+ RESOURCE_TRANSFORM = new Function<YarnLocalResource, LocalResource>() {
+ @Override
+ public LocalResource apply(YarnLocalResource input) {
+ return input.getLocalResource();
+ }
+ };
+ }
+
+ private final ContainerLaunchContext launchContext;
+
+ public Hadoop20YarnLaunchContext() {
+ launchContext = Records.newRecord(ContainerLaunchContext.class);
+ }
+
+ @Override
+ public <T> T getLaunchContext() {
+ return (T) launchContext;
+ }
+
+ @Override
+ public void setCredentials(Credentials credentials) {
+ launchContext.setContainerTokens(YarnUtils.encodeCredentials(credentials));
+ }
+
+ @Override
+ public void setLocalResources(Map<String, YarnLocalResource> localResources) {
+ launchContext.setLocalResources(Maps.transformValues(localResources, RESOURCE_TRANSFORM));
+ }
+
+ @Override
+ public void setServiceData(Map<String, ByteBuffer> serviceData) {
+ launchContext.setServiceData(serviceData);
+ }
+
+ @Override
+ public Map<String, String> getEnvironment() {
+ return launchContext.getEnvironment();
+ }
+
+ @Override
+ public void setEnvironment(Map<String, String> environment) {
+ launchContext.setEnvironment(environment);
+ }
+
+ @Override
+ public List<String> getCommands() {
+ return launchContext.getCommands();
+ }
+
+ @Override
+ public void setCommands(List<String> commands) {
+ launchContext.setCommands(commands);
+ }
+
+ @Override
+ public void setApplicationACLs(Map<ApplicationAccessType, String> acls) {
+ launchContext.setApplicationACLs(acls);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnLocalResource.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnLocalResource.java b/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnLocalResource.java
new file mode 100644
index 0000000..b327b94
--- /dev/null
+++ b/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnLocalResource.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn;
+
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.URL;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ *
+ */
+public final class Hadoop20YarnLocalResource implements YarnLocalResource {
+
+ private final LocalResource localResource;
+
+ public Hadoop20YarnLocalResource() {
+ this.localResource = Records.newRecord(LocalResource.class);
+ }
+
+ @Override
+ public <T> T getLocalResource() {
+ return (T) localResource;
+ }
+
+ @Override
+ public URL getResource() {
+ return localResource.getResource();
+ }
+
+ @Override
+ public void setResource(URL resource) {
+ localResource.setResource(resource);
+ }
+
+ @Override
+ public long getSize() {
+ return localResource.getSize();
+ }
+
+ @Override
+ public void setSize(long size) {
+ localResource.setSize(size);
+ }
+
+ @Override
+ public long getTimestamp() {
+ return localResource.getTimestamp();
+ }
+
+ @Override
+ public void setTimestamp(long timestamp) {
+ localResource.setTimestamp(timestamp);
+ }
+
+ @Override
+ public LocalResourceType getType() {
+ return localResource.getType();
+ }
+
+ @Override
+ public void setType(LocalResourceType type) {
+ localResource.setType(type);
+ }
+
+ @Override
+ public LocalResourceVisibility getVisibility() {
+ return localResource.getVisibility();
+ }
+
+ @Override
+ public void setVisibility(LocalResourceVisibility visibility) {
+ localResource.setVisibility(visibility);
+ }
+
+ @Override
+ public String getPattern() {
+ return localResource.getPattern();
+ }
+
+ @Override
+ public void setPattern(String pattern) {
+ localResource.setPattern(pattern);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnNMClient.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnNMClient.java b/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnNMClient.java
new file mode 100644
index 0000000..98ecc67
--- /dev/null
+++ b/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/Hadoop20YarnNMClient.java
@@ -0,0 +1,121 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn;
+
+import org.apache.twill.common.Cancellable;
+import com.google.common.base.Throwables;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.yarn.api.ContainerManager;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.util.Records;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.InetSocketAddress;
+
+/**
+ *
+ */
+public final class Hadoop20YarnNMClient implements YarnNMClient {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Hadoop20YarnNMClient.class);
+
+ private final YarnRPC yarnRPC;
+ private final Configuration yarnConf;
+
+ public Hadoop20YarnNMClient(YarnRPC yarnRPC, Configuration yarnConf) {
+ this.yarnRPC = yarnRPC;
+ this.yarnConf = yarnConf;
+ }
+
+ @Override
+ public Cancellable start(YarnContainerInfo containerInfo, YarnLaunchContext launchContext) {
+ ContainerLaunchContext context = launchContext.getLaunchContext();
+ context.setUser(System.getProperty("user.name"));
+
+ Container container = containerInfo.getContainer();
+
+ context.setContainerId(container.getId());
+ context.setResource(container.getResource());
+
+ StartContainerRequest startRequest = Records.newRecord(StartContainerRequest.class);
+ startRequest.setContainerLaunchContext(context);
+
+ ContainerManager manager = connectContainerManager(container);
+ try {
+ manager.startContainer(startRequest);
+ return new ContainerTerminator(container, manager);
+ } catch (YarnRemoteException e) {
+ LOG.error("Error in launching process", e);
+ throw Throwables.propagate(e);
+ }
+
+ }
+
+ /**
+ * Helper to connect to container manager (node manager).
+ */
+ private ContainerManager connectContainerManager(Container container) {
+ String cmIpPortStr = String.format("%s:%d", container.getNodeId().getHost(), container.getNodeId().getPort());
+ InetSocketAddress cmAddress = NetUtils.createSocketAddr(cmIpPortStr);
+ return ((ContainerManager) yarnRPC.getProxy(ContainerManager.class, cmAddress, yarnConf));
+ }
+
+ private static final class ContainerTerminator implements Cancellable {
+
+ private final Container container;
+ private final ContainerManager manager;
+
+ private ContainerTerminator(Container container, ContainerManager manager) {
+ this.container = container;
+ this.manager = manager;
+ }
+
+ @Override
+ public void cancel() {
+ LOG.info("Request to stop container {}.", container.getId());
+ StopContainerRequest stopRequest = Records.newRecord(StopContainerRequest.class);
+ stopRequest.setContainerId(container.getId());
+ try {
+ manager.stopContainer(stopRequest);
+ boolean completed = false;
+ while (!completed) {
+ GetContainerStatusRequest statusRequest = Records.newRecord(GetContainerStatusRequest.class);
+ statusRequest.setContainerId(container.getId());
+ GetContainerStatusResponse statusResponse = manager.getContainerStatus(statusRequest);
+ LOG.info("Container status: {} {}", statusResponse.getStatus(), statusResponse.getStatus().getDiagnostics());
+
+ completed = (statusResponse.getStatus().getState() == ContainerState.COMPLETE);
+ }
+ LOG.info("Container {} stopped.", container.getId());
+ } catch (YarnRemoteException e) {
+ LOG.error("Fail to stop container {}", container.getId(), e);
+ throw Throwables.propagate(e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/ports/AMRMClient.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/ports/AMRMClient.java b/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/ports/AMRMClient.java
new file mode 100644
index 0000000..26b6fa2
--- /dev/null
+++ b/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/ports/AMRMClient.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn.ports;
+
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.service.Service;
+
+/**
+ * Ported from Apache Hadoop YARN.
+ */
+public interface AMRMClient extends Service {
+
+ /**
+ * Value used to define no locality.
+ */
+ static final String ANY = "*";
+
+ /**
+ * Object to represent container request for resources.
+ * Resources may be localized to nodes and racks.
+ * Resources may be assigned priorities.
+ * Can ask for multiple containers of a given type.
+ */
+ public static class ContainerRequest {
+ Resource capability;
+ String[] hosts;
+ String[] racks;
+ Priority priority;
+ int containerCount;
+
+ public ContainerRequest(Resource capability, String[] hosts,
+ String[] racks, Priority priority, int containerCount) {
+ this.capability = capability;
+ this.hosts = (hosts != null ? hosts.clone() : null);
+ this.racks = (racks != null ? racks.clone() : null);
+ this.priority = priority;
+ this.containerCount = containerCount;
+ }
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("Capability[").append(capability).append("]");
+ sb.append("Priority[").append(priority).append("]");
+ sb.append("ContainerCount[").append(containerCount).append("]");
+ return sb.toString();
+ }
+ }
+
+ /**
+ * Register the application master. This must be called before any
+ * other interaction
+ * @param appHostName Name of the host on which master is running
+ * @param appHostPort Port master is listening on
+ * @param appTrackingUrl URL at which the master info can be seen
+ * @return <code>RegisterApplicationMasterResponse</code>
+ * @throws org.apache.hadoop.yarn.exceptions.YarnRemoteException
+ */
+ public RegisterApplicationMasterResponse
+ registerApplicationMaster(String appHostName,
+ int appHostPort,
+ String appTrackingUrl)
+ throws YarnRemoteException;
+
+ /**
+ * Request additional containers and receive new container allocations.
+ * Requests made via <code>addContainerRequest</code> are sent to the
+ * <code>ResourceManager</code>. New containers assigned to the master are
+ * retrieved. Status of completed containers and node health updates are
+ * also retrieved.
+ * This also doubles as a heartbeat to the ResourceManager and must be
+ * made periodically.
+ * The call may not always return any new allocations of containers.
+ * App should not make concurrent allocate requests. May cause request loss.
+ * @param progressIndicator Indicates progress made by the master
+ * @return the response of the allocate request
+ * @throws YarnRemoteException
+ */
+ public AllocationResponse allocate(float progressIndicator)
+ throws YarnRemoteException;
+
+ /**
+ * Unregister the Application Master. This must be called in the end.
+ * @param appStatus Success/Failure status of the master
+ * @param appMessage Diagnostics message on failure
+ * @param appTrackingUrl New URL to get master info
+ * @throws YarnRemoteException
+ */
+ public void unregisterApplicationMaster(FinalApplicationStatus appStatus,
+ String appMessage,
+ String appTrackingUrl)
+ throws YarnRemoteException;
+
+ /**
+ * Request containers for resources before calling <code>allocate</code>.
+ * @param req Resource request
+ */
+ public void addContainerRequest(ContainerRequest req);
+
+ /**
+ * Remove previous container request. The previous container request may have
+ * already been sent to the ResourceManager. So even after the remove request
+ * the app must be prepared to receive an allocation for the previous request
+ * even after the remove request
+ * @param req Resource request
+ */
+ public void removeContainerRequest(ContainerRequest req);
+
+ /**
+ * Release containers assigned by the Resource Manager. If the app cannot use
+ * the container or wants to give up the container then it can release it.
+ * The app needs to make new requests for the released resource capability if
+ * it still needs it. For example, if it released non-local resources
+ * @param containerId
+ */
+ public void releaseAssignedContainer(ContainerId containerId);
+
+ /**
+ * Get the currently available resources in the cluster.
+ * A valid value is available after a call to allocate has been made
+ * @return Currently available resources
+ */
+ public Resource getClusterAvailableResources();
+
+ /**
+ * Get the current number of nodes in the cluster.
+ * A valid values is available after a call to allocate has been made
+ * @return Current number of nodes in the cluster
+ */
+ public int getClusterNodeCount();
+}
[11/15] Initial import commit.
Posted by ch...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/kafka/client/AbstractMessageSetEncoder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/kafka/client/AbstractMessageSetEncoder.java b/core/src/main/java/org/apache/twill/internal/kafka/client/AbstractMessageSetEncoder.java
new file mode 100644
index 0000000..9955d6a
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/kafka/client/AbstractMessageSetEncoder.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.kafka.client;
+
+import org.jboss.netty.buffer.ChannelBuffer;
+import org.jboss.netty.buffer.ChannelBuffers;
+
+import java.util.zip.CRC32;
+
+/**
+ * A base implementation of {@link MessageSetEncoder}.
+ */
+abstract class AbstractMessageSetEncoder implements MessageSetEncoder {
+
+ private static final ThreadLocal<CRC32> CRC32_LOCAL = new ThreadLocal<CRC32>() {
+ @Override
+ protected CRC32 initialValue() {
+ return new CRC32();
+ }
+ };
+
+ protected final int computeCRC32(ChannelBuffer buffer) {
+ CRC32 crc32 = CRC32_LOCAL.get();
+ crc32.reset();
+
+ if (buffer.hasArray()) {
+ crc32.update(buffer.array(), buffer.arrayOffset() + buffer.readerIndex(), buffer.readableBytes());
+ } else {
+ byte[] bytes = new byte[buffer.readableBytes()];
+ buffer.getBytes(buffer.readerIndex(), bytes);
+ crc32.update(bytes);
+ }
+ return (int) crc32.getValue();
+ }
+
+ protected final ChannelBuffer encodePayload(ChannelBuffer payload) {
+ return encodePayload(payload, Compression.NONE);
+ }
+
+ protected final ChannelBuffer encodePayload(ChannelBuffer payload, Compression compression) {
+ ChannelBuffer header = ChannelBuffers.buffer(10);
+
+ int crc = computeCRC32(payload);
+
+ int magic = ((compression == Compression.NONE) ? 0 : 1);
+
+ // Message length = 1 byte magic + (optional 1 compression byte) + 4 bytes crc + payload length
+ header.writeInt(5 + magic + payload.readableBytes());
+ // Magic number = 0 for non-compressed data
+ header.writeByte(magic);
+ if (magic > 0) {
+ header.writeByte(compression.getCode());
+ }
+ header.writeInt(crc);
+
+ return ChannelBuffers.wrappedBuffer(header, payload);
+ }
+
+ protected final ChannelBuffer prefixLength(ChannelBuffer buffer) {
+ ChannelBuffer sizeBuf = ChannelBuffers.buffer(4);
+ sizeBuf.writeInt(buffer.readableBytes());
+ return ChannelBuffers.wrappedBuffer(sizeBuf, buffer);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/kafka/client/BasicFetchedMessage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/kafka/client/BasicFetchedMessage.java b/core/src/main/java/org/apache/twill/internal/kafka/client/BasicFetchedMessage.java
new file mode 100644
index 0000000..286bf82
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/kafka/client/BasicFetchedMessage.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.kafka.client;
+
+import org.apache.twill.kafka.client.FetchedMessage;
+
+import java.nio.ByteBuffer;
+
+/**
+ *
+ */
+final class BasicFetchedMessage implements FetchedMessage {
+
+ private final long offset;
+ private final ByteBuffer buffer;
+
+ BasicFetchedMessage(long offset, ByteBuffer buffer) {
+ this.offset = offset;
+ this.buffer = buffer;
+ }
+
+ @Override
+ public long getOffset() {
+ return offset;
+ }
+
+ @Override
+ public ByteBuffer getBuffer() {
+ return buffer;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/kafka/client/Bufferer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/kafka/client/Bufferer.java b/core/src/main/java/org/apache/twill/internal/kafka/client/Bufferer.java
new file mode 100644
index 0000000..c1fb4f2
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/kafka/client/Bufferer.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.kafka.client;
+
+import org.jboss.netty.buffer.ChannelBuffer;
+import org.jboss.netty.buffer.ChannelBuffers;
+
+/**
+ * A class to help buffering data of format [len][payload-of-len].
+ */
+final class Bufferer {
+
+ private ChannelBuffer currentBuffer = null;
+ private int currentSize = -1;
+
+ void apply(ChannelBuffer buffer) {
+ currentBuffer = concatBuffer(currentBuffer, buffer);
+ }
+
+ /**
+ * Returns the buffer if the buffer data is ready to be consumed,
+ * otherwise return {@link ChannelBuffers#EMPTY_BUFFER}.
+ */
+ ChannelBuffer getNext() {
+ if (currentSize < 0) {
+ if (currentBuffer.readableBytes() < 4) {
+ return ChannelBuffers.EMPTY_BUFFER;
+ }
+ currentSize = currentBuffer.readInt();
+ }
+
+ // Keep buffering if less then required number of bytes
+ if (currentBuffer.readableBytes() < currentSize) {
+ return ChannelBuffers.EMPTY_BUFFER;
+ }
+
+ ChannelBuffer result = currentBuffer.readSlice(currentSize);
+ currentSize = -1;
+
+ return result;
+ }
+
+ private ChannelBuffer concatBuffer(ChannelBuffer current, ChannelBuffer buffer) {
+ return current == null ? buffer : ChannelBuffers.wrappedBuffer(current, buffer);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/kafka/client/Compression.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/kafka/client/Compression.java b/core/src/main/java/org/apache/twill/internal/kafka/client/Compression.java
new file mode 100644
index 0000000..3355b9f
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/kafka/client/Compression.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.kafka.client;
+
+/**
+ * Enum for indicating compression method.
+ */
+public enum Compression {
+ NONE(0),
+ GZIP(1),
+ SNAPPY(2);
+
+ private final int code;
+
+ Compression(int code) {
+ this.code = code;
+ }
+
+ public int getCode() {
+ return code;
+ }
+
+ public static Compression fromCode(int code) {
+ switch (code) {
+ case 0:
+ return NONE;
+ case 1:
+ return GZIP;
+ case 2:
+ return SNAPPY;
+ }
+ throw new IllegalArgumentException("Unknown compression code.");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/kafka/client/ConnectionPool.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/kafka/client/ConnectionPool.java b/core/src/main/java/org/apache/twill/internal/kafka/client/ConnectionPool.java
new file mode 100644
index 0000000..c2865ba
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/kafka/client/ConnectionPool.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.kafka.client;
+
+import com.google.common.collect.Maps;
+import org.jboss.netty.bootstrap.ClientBootstrap;
+import org.jboss.netty.channel.ChannelFuture;
+import org.jboss.netty.channel.ChannelFutureListener;
+import org.jboss.netty.channel.group.ChannelGroup;
+import org.jboss.netty.channel.group.ChannelGroupFuture;
+import org.jboss.netty.channel.group.ChannelGroupFutureListener;
+import org.jboss.netty.channel.group.DefaultChannelGroup;
+
+import java.net.InetSocketAddress;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.ConcurrentMap;
+
+/**
+ * Provides netty socket connection reuse.
+ */
+final class ConnectionPool {
+
+ private final ClientBootstrap bootstrap;
+ private final ChannelGroup channelGroup;
+ private final ConcurrentMap<InetSocketAddress, Queue<ChannelFuture>> connections;
+
+ /**
+ * For releasing a connection back to the pool.
+ */
+ interface ConnectionReleaser {
+ void release();
+ }
+
+ /**
+ * Result of a connect request.
+ */
+ interface ConnectResult extends ConnectionReleaser {
+ ChannelFuture getChannelFuture();
+ }
+
+ ConnectionPool(ClientBootstrap bootstrap) {
+ this.bootstrap = bootstrap;
+ this.channelGroup = new DefaultChannelGroup();
+ this.connections = Maps.newConcurrentMap();
+ }
+
+ ConnectResult connect(InetSocketAddress address) {
+ Queue<ChannelFuture> channelFutures = connections.get(address);
+ if (channelFutures == null) {
+ channelFutures = new ConcurrentLinkedQueue<ChannelFuture>();
+ Queue<ChannelFuture> result = connections.putIfAbsent(address, channelFutures);
+ channelFutures = result == null ? channelFutures : result;
+ }
+
+ ChannelFuture channelFuture = channelFutures.poll();
+ while (channelFuture != null) {
+ if (channelFuture.isSuccess() && channelFuture.getChannel().isConnected()) {
+ return new SimpleConnectResult(address, channelFuture);
+ }
+ channelFuture = channelFutures.poll();
+ }
+
+ channelFuture = bootstrap.connect(address);
+ channelFuture.addListener(new ChannelFutureListener() {
+ @Override
+ public void operationComplete(ChannelFuture future) throws Exception {
+ if (future.isSuccess()) {
+ channelGroup.add(future.getChannel());
+ }
+ }
+ });
+ return new SimpleConnectResult(address, channelFuture);
+ }
+
+ ChannelGroupFuture close() {
+ ChannelGroupFuture result = channelGroup.close();
+ result.addListener(new ChannelGroupFutureListener() {
+ @Override
+ public void operationComplete(ChannelGroupFuture future) throws Exception {
+ bootstrap.releaseExternalResources();
+ }
+ });
+ return result;
+ }
+
+ private final class SimpleConnectResult implements ConnectResult {
+
+ private final InetSocketAddress address;
+ private final ChannelFuture future;
+
+
+ private SimpleConnectResult(InetSocketAddress address, ChannelFuture future) {
+ this.address = address;
+ this.future = future;
+ }
+
+ @Override
+ public ChannelFuture getChannelFuture() {
+ return future;
+ }
+
+ @Override
+ public void release() {
+ if (future.isSuccess()) {
+ connections.get(address).offer(future);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/kafka/client/GZipMessageSetEncoder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/kafka/client/GZipMessageSetEncoder.java b/core/src/main/java/org/apache/twill/internal/kafka/client/GZipMessageSetEncoder.java
new file mode 100644
index 0000000..daa0c2c
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/kafka/client/GZipMessageSetEncoder.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.kafka.client;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.zip.GZIPOutputStream;
+
+/**
+ * A {@link MessageSetEncoder} that compress message set using GZIP.
+ */
+final class GZipMessageSetEncoder extends AbstractCompressedMessageSetEncoder {
+
+ GZipMessageSetEncoder() {
+ super(Compression.GZIP);
+ }
+
+ @Override
+ protected OutputStream createCompressedStream(OutputStream os) throws IOException {
+ return new GZIPOutputStream(os);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/kafka/client/IdentityMessageSetEncoder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/kafka/client/IdentityMessageSetEncoder.java b/core/src/main/java/org/apache/twill/internal/kafka/client/IdentityMessageSetEncoder.java
new file mode 100644
index 0000000..51dc746
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/kafka/client/IdentityMessageSetEncoder.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.kafka.client;
+
+import org.jboss.netty.buffer.ChannelBuffer;
+import org.jboss.netty.buffer.ChannelBuffers;
+
+/**
+ * A pass-through {@link MessageSetEncoder}.
+ */
+final class IdentityMessageSetEncoder extends AbstractMessageSetEncoder {
+
+ private ChannelBuffer messageSets = ChannelBuffers.EMPTY_BUFFER;
+
+ @Override
+ public MessageSetEncoder add(ChannelBuffer payload) {
+ messageSets = ChannelBuffers.wrappedBuffer(messageSets, encodePayload(payload));
+ return this;
+ }
+
+ @Override
+ public ChannelBuffer finish() {
+ ChannelBuffer buf = prefixLength(messageSets);
+ messageSets = ChannelBuffers.EMPTY_BUFFER;
+ return buf;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/kafka/client/KafkaBrokerCache.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/kafka/client/KafkaBrokerCache.java b/core/src/main/java/org/apache/twill/internal/kafka/client/KafkaBrokerCache.java
new file mode 100644
index 0000000..f2bb815
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/kafka/client/KafkaBrokerCache.java
@@ -0,0 +1,326 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.kafka.client;
+
+import org.apache.twill.common.Threads;
+import org.apache.twill.zookeeper.NodeChildren;
+import org.apache.twill.zookeeper.NodeData;
+import org.apache.twill.zookeeper.ZKClient;
+import com.google.common.base.Charsets;
+import com.google.common.base.Function;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSortedMap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.AbstractIdleService;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.InetSocketAddress;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.SortedMap;
+
+/**
+ * A Service to cache kafka broker information by subscribing to ZooKeeper.
+ */
+final class KafkaBrokerCache extends AbstractIdleService {
+
+ private static final Logger LOG = LoggerFactory.getLogger(KafkaBrokerCache.class);
+
+ private static final String BROKERS_PATH = "/brokers";
+
+ private final ZKClient zkClient;
+ private final Map<String, InetSocketAddress> brokers;
+ // topicBrokers is from topic->partition size->brokerId
+ private final Map<String, SortedMap<Integer, Set<String>>> topicBrokers;
+ private final Runnable invokeGetBrokers = new Runnable() {
+ @Override
+ public void run() {
+ getBrokers();
+ }
+ };
+ private final Runnable invokeGetTopics = new Runnable() {
+ @Override
+ public void run() {
+ getTopics();
+ }
+ };
+
+ KafkaBrokerCache(ZKClient zkClient) {
+ this.zkClient = zkClient;
+ this.brokers = Maps.newConcurrentMap();
+ this.topicBrokers = Maps.newConcurrentMap();
+ }
+
+ @Override
+ protected void startUp() throws Exception {
+ getBrokers();
+ getTopics();
+ }
+
+ @Override
+ protected void shutDown() throws Exception {
+ // No-op
+ }
+
+ public int getPartitionSize(String topic) {
+ SortedMap<Integer, Set<String>> partitionBrokers = topicBrokers.get(topic);
+ if (partitionBrokers == null || partitionBrokers.isEmpty()) {
+ return 1;
+ }
+ return partitionBrokers.lastKey();
+ }
+
+ public TopicBroker getBrokerAddress(String topic, int partition) {
+ SortedMap<Integer, Set<String>> partitionBrokers = topicBrokers.get(topic);
+ if (partitionBrokers == null || partitionBrokers.isEmpty()) {
+ return pickRandomBroker(topic);
+ }
+
+ // If the requested partition is greater than supported partition size, randomly pick one
+ if (partition >= partitionBrokers.lastKey()) {
+ return pickRandomBroker(topic);
+ }
+
+ // Randomly pick a partition size and randomly pick a broker from it
+ Random random = new Random();
+ partitionBrokers = partitionBrokers.tailMap(partition + 1);
+ List<Integer> sizes = Lists.newArrayList(partitionBrokers.keySet());
+ Integer partitionSize = pickRandomItem(sizes, random);
+ List<String> ids = Lists.newArrayList(partitionBrokers.get(partitionSize));
+ InetSocketAddress address = brokers.get(ids.get(new Random().nextInt(ids.size())));
+ return address == null ? pickRandomBroker(topic) : new TopicBroker(topic, address, partitionSize);
+ }
+
+ private TopicBroker pickRandomBroker(String topic) {
+ Map.Entry<String, InetSocketAddress> entry = Iterables.getFirst(brokers.entrySet(), null);
+ if (entry == null) {
+ return null;
+ }
+ InetSocketAddress address = entry.getValue();
+ return new TopicBroker(topic, address, 0);
+ }
+
+ private <T> T pickRandomItem(List<T> list, Random random) {
+ return list.get(random.nextInt(list.size()));
+ }
+
+ private void getBrokers() {
+ final String idsPath = BROKERS_PATH + "/ids";
+
+ Futures.addCallback(zkClient.getChildren(idsPath, new Watcher() {
+ @Override
+ public void process(WatchedEvent event) {
+ getBrokers();
+ }
+ }), new ExistsOnFailureFutureCallback<NodeChildren>(idsPath, invokeGetBrokers) {
+ @Override
+ public void onSuccess(NodeChildren result) {
+ Set<String> children = ImmutableSet.copyOf(result.getChildren());
+ for (String child : children) {
+ getBrokenData(idsPath + "/" + child, child);
+ }
+ // Remove all removed brokers
+ removeDiff(children, brokers);
+ }
+ });
+ }
+
+ private void getTopics() {
+ final String topicsPath = BROKERS_PATH + "/topics";
+ Futures.addCallback(zkClient.getChildren(topicsPath, new Watcher() {
+ @Override
+ public void process(WatchedEvent event) {
+ getTopics();
+ }
+ }), new ExistsOnFailureFutureCallback<NodeChildren>(topicsPath, invokeGetTopics) {
+ @Override
+ public void onSuccess(NodeChildren result) {
+ Set<String> children = ImmutableSet.copyOf(result.getChildren());
+
+ // Process new children
+ for (String topic : ImmutableSet.copyOf(Sets.difference(children, topicBrokers.keySet()))) {
+ getTopic(topicsPath + "/" + topic, topic);
+ }
+
+ // Remove old children
+ removeDiff(children, topicBrokers);
+ }
+ });
+ }
+
+ private void getBrokenData(String path, final String brokerId) {
+ Futures.addCallback(zkClient.getData(path), new FutureCallback<NodeData>() {
+ @Override
+ public void onSuccess(NodeData result) {
+ String data = new String(result.getData(), Charsets.UTF_8);
+ String hostPort = data.substring(data.indexOf(':') + 1);
+ int idx = hostPort.indexOf(':');
+ brokers.put(brokerId, new InetSocketAddress(hostPort.substring(0, idx),
+ Integer.parseInt(hostPort.substring(idx + 1))));
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ // No-op, the watch on the parent node will handle it.
+ }
+ });
+ }
+
+ private void getTopic(final String path, final String topic) {
+ Futures.addCallback(zkClient.getChildren(path, new Watcher() {
+ @Override
+ public void process(WatchedEvent event) {
+ // Other event type changes are either could be ignored or handled by parent watcher
+ if (event.getType() == Event.EventType.NodeChildrenChanged) {
+ getTopic(path, topic);
+ }
+ }
+ }), new FutureCallback<NodeChildren>() {
+ @Override
+ public void onSuccess(NodeChildren result) {
+ List<String> children = result.getChildren();
+ final List<ListenableFuture<BrokerPartition>> futures = Lists.newArrayListWithCapacity(children.size());
+
+ // Fetch data from each broken node
+ for (final String brokerId : children) {
+ Futures.transform(zkClient.getData(path + "/" + brokerId), new Function<NodeData, BrokerPartition>() {
+ @Override
+ public BrokerPartition apply(NodeData input) {
+ return new BrokerPartition(brokerId, Integer.parseInt(new String(input.getData(), Charsets.UTF_8)));
+ }
+ });
+ }
+
+ // When all fetching is done, build the partition size->broker map for this topic
+ Futures.successfulAsList(futures).addListener(new Runnable() {
+ @Override
+ public void run() {
+ Map<Integer, Set<String>> partitionBrokers = Maps.newHashMap();
+ for (ListenableFuture<BrokerPartition> future : futures) {
+ try {
+ BrokerPartition info = future.get();
+ Set<String> brokerSet = partitionBrokers.get(info.getPartitionSize());
+ if (brokerSet == null) {
+ brokerSet = Sets.newHashSet();
+ partitionBrokers.put(info.getPartitionSize(), brokerSet);
+ }
+ brokerSet.add(info.getBrokerId());
+ } catch (Exception e) {
+ // Exception is ignored, as it will be handled by parent watcher
+ }
+ }
+ topicBrokers.put(topic, ImmutableSortedMap.copyOf(partitionBrokers));
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ // No-op. Failure would be handled by parent watcher already (e.g. node not exists -> children change in parent)
+ }
+ });
+ }
+
+ private <K, V> void removeDiff(Set<K> keys, Map<K, V> map) {
+ for (K key : ImmutableSet.copyOf(Sets.difference(map.keySet(), keys))) {
+ map.remove(key);
+ }
+ }
+
+ private abstract class ExistsOnFailureFutureCallback<V> implements FutureCallback<V> {
+
+ private final String path;
+ private final Runnable action;
+
+ protected ExistsOnFailureFutureCallback(String path, Runnable action) {
+ this.path = path;
+ this.action = action;
+ }
+
+ @Override
+ public final void onFailure(Throwable t) {
+ if (!isNotExists(t)) {
+ LOG.error("Fail to watch for kafka brokers: " + path, t);
+ return;
+ }
+
+ waitExists(path);
+ }
+
+ private boolean isNotExists(Throwable t) {
+ return ((t instanceof KeeperException) && ((KeeperException) t).code() == KeeperException.Code.NONODE);
+ }
+
+ private void waitExists(String path) {
+ LOG.info("Path " + path + " not exists. Watch for creation.");
+
+ // If the node doesn't exists, use the "exists" call to watch for node creation.
+ Futures.addCallback(zkClient.exists(path, new Watcher() {
+ @Override
+ public void process(WatchedEvent event) {
+ if (event.getType() == Event.EventType.NodeCreated || event.getType() == Event.EventType.NodeDeleted) {
+ action.run();
+ }
+ }
+ }), new FutureCallback<Stat>() {
+ @Override
+ public void onSuccess(Stat result) {
+ // If path exists, get children again, otherwise wait for watch to get triggered
+ if (result != null) {
+ action.run();
+ }
+ }
+ @Override
+ public void onFailure(Throwable t) {
+ action.run();
+ }
+ });
+ }
+ }
+
+ private static final class BrokerPartition {
+ private final String brokerId;
+ private final int partitionSize;
+
+ private BrokerPartition(String brokerId, int partitionSize) {
+ this.brokerId = brokerId;
+ this.partitionSize = partitionSize;
+ }
+
+ public String getBrokerId() {
+ return brokerId;
+ }
+
+ public int getPartitionSize() {
+ return partitionSize;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/kafka/client/KafkaRequest.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/kafka/client/KafkaRequest.java b/core/src/main/java/org/apache/twill/internal/kafka/client/KafkaRequest.java
new file mode 100644
index 0000000..7b43f8a
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/kafka/client/KafkaRequest.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.kafka.client;
+
+import org.jboss.netty.buffer.ChannelBuffer;
+
+/**
+ *
+ */
+final class KafkaRequest {
+
+ public enum Type {
+ PRODUCE(0),
+ FETCH(1),
+ MULTI_FETCH(2),
+ MULTI_PRODUCE(3),
+ OFFSETS(4);
+
+ private final short id;
+
+ private Type(int id) {
+ this.id = (short) id;
+ }
+
+ public short getId() {
+ return id;
+ }
+ }
+
+ private final Type type;
+ private final String topic;
+ private final int partition;
+ private final ChannelBuffer body;
+ private final ResponseHandler responseHandler;
+
+
+ public static KafkaRequest createProduce(String topic, int partition, ChannelBuffer body) {
+ return new KafkaRequest(Type.PRODUCE, topic, partition, body, ResponseHandler.NO_OP);
+ }
+
+ public static KafkaRequest createFetch(String topic, int partition, ChannelBuffer body, ResponseHandler handler) {
+ return new KafkaRequest(Type.FETCH, topic, partition, body, handler);
+ }
+
+ public static KafkaRequest createOffsets(String topic, int partition, ChannelBuffer body, ResponseHandler handler) {
+ return new KafkaRequest(Type.OFFSETS, topic, partition, body, handler);
+ }
+
+ private KafkaRequest(Type type, String topic, int partition, ChannelBuffer body, ResponseHandler responseHandler) {
+ this.type = type;
+ this.topic = topic;
+ this.partition = partition;
+ this.body = body;
+ this.responseHandler = responseHandler;
+ }
+
+ Type getType() {
+ return type;
+ }
+
+ String getTopic() {
+ return topic;
+ }
+
+ int getPartition() {
+ return partition;
+ }
+
+ ChannelBuffer getBody() {
+ return body;
+ }
+
+ ResponseHandler getResponseHandler() {
+ return responseHandler;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/kafka/client/KafkaRequestEncoder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/kafka/client/KafkaRequestEncoder.java b/core/src/main/java/org/apache/twill/internal/kafka/client/KafkaRequestEncoder.java
new file mode 100644
index 0000000..ef78c76
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/kafka/client/KafkaRequestEncoder.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.kafka.client;
+
+import com.google.common.base.Charsets;
+import org.jboss.netty.buffer.ChannelBuffer;
+import org.jboss.netty.buffer.ChannelBuffers;
+import org.jboss.netty.channel.Channel;
+import org.jboss.netty.channel.ChannelHandlerContext;
+import org.jboss.netty.handler.codec.oneone.OneToOneEncoder;
+
+import java.nio.ByteBuffer;
+
+/**
+ *
+ */
+final class KafkaRequestEncoder extends OneToOneEncoder {
+
+ @Override
+ protected Object encode(ChannelHandlerContext ctx, Channel channel, Object msg) throws Exception {
+ if (!(msg instanceof KafkaRequest)) {
+ return msg;
+ }
+ KafkaRequest req = (KafkaRequest) msg;
+ ByteBuffer topic = Charsets.UTF_8.encode(req.getTopic());
+
+ ChannelBuffer buffer = ChannelBuffers.dynamicBuffer(16 + topic.remaining() + req.getBody().readableBytes());
+ int writerIdx = buffer.writerIndex();
+ buffer.writerIndex(writerIdx + 4); // Reserves 4 bytes for message length
+
+ // Write out <REQUEST_TYPE>, <TOPIC_LENGTH>, <TOPIC>, <PARTITION>
+ buffer.writeShort(req.getType().getId());
+ buffer.writeShort(topic.remaining());
+ buffer.writeBytes(topic);
+ buffer.writeInt(req.getPartition());
+
+ // Write out the size of the whole buffer (excluding the size field) at the beginning
+ buffer.setInt(writerIdx, buffer.readableBytes() - 4 + req.getBody().readableBytes());
+
+ ChannelBuffer buf = ChannelBuffers.wrappedBuffer(buffer, req.getBody());
+ buf = buf.readBytes(buf.readableBytes());
+
+ return buf;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/kafka/client/KafkaRequestSender.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/kafka/client/KafkaRequestSender.java b/core/src/main/java/org/apache/twill/internal/kafka/client/KafkaRequestSender.java
new file mode 100644
index 0000000..fbc552c
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/kafka/client/KafkaRequestSender.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.kafka.client;
+
+/**
+ *
+ */
+interface KafkaRequestSender {
+
+ void send(KafkaRequest request);
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/kafka/client/KafkaResponse.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/kafka/client/KafkaResponse.java b/core/src/main/java/org/apache/twill/internal/kafka/client/KafkaResponse.java
new file mode 100644
index 0000000..68c1bd8
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/kafka/client/KafkaResponse.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.kafka.client;
+
+import org.apache.twill.kafka.client.FetchException;
+import org.jboss.netty.buffer.ChannelBuffer;
+
+/**
+ *
+ */
+final class KafkaResponse {
+
+ private final FetchException.ErrorCode errorCode;
+ private final ChannelBuffer body;
+ private final int size;
+
+ KafkaResponse(FetchException.ErrorCode errorCode, ChannelBuffer body, int size) {
+ this.errorCode = errorCode;
+ this.body = body;
+ this.size = size;
+ }
+
+ public int getSize() {
+ return size;
+ }
+
+ public FetchException.ErrorCode getErrorCode() {
+ return errorCode;
+ }
+
+ public ChannelBuffer getBody() {
+ return body;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/kafka/client/KafkaResponseDispatcher.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/kafka/client/KafkaResponseDispatcher.java b/core/src/main/java/org/apache/twill/internal/kafka/client/KafkaResponseDispatcher.java
new file mode 100644
index 0000000..47f70ce
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/kafka/client/KafkaResponseDispatcher.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.kafka.client;
+
+import org.jboss.netty.channel.ChannelHandlerContext;
+import org.jboss.netty.channel.ExceptionEvent;
+import org.jboss.netty.channel.MessageEvent;
+import org.jboss.netty.channel.SimpleChannelHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.SocketException;
+import java.nio.channels.ClosedChannelException;
+
+/**
+ *
+ */
+final class KafkaResponseDispatcher extends SimpleChannelHandler {
+
+ private static final Logger LOG = LoggerFactory.getLogger(KafkaResponseDispatcher.class);
+
+ @Override
+ public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception {
+ Object attachment = ctx.getAttachment();
+ if (e.getMessage() instanceof KafkaResponse && attachment instanceof ResponseHandler) {
+ ((ResponseHandler) attachment).received((KafkaResponse) e.getMessage());
+ } else {
+ super.messageReceived(ctx, e);
+ }
+ }
+
+ @Override
+ public void writeRequested(ChannelHandlerContext ctx, MessageEvent e) throws Exception {
+ if (e.getMessage() instanceof KafkaRequest) {
+ ctx.setAttachment(((KafkaRequest) e.getMessage()).getResponseHandler());
+ }
+ super.writeRequested(ctx, e);
+ }
+
+ @Override
+ public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) throws Exception {
+ if (e.getCause() instanceof ClosedChannelException || e.getCause() instanceof SocketException) {
+ // No need to log for socket exception as the client has logic to retry.
+ return;
+ }
+ LOG.warn("Exception caught in kafka client connection.", e.getCause());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/kafka/client/KafkaResponseHandler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/kafka/client/KafkaResponseHandler.java b/core/src/main/java/org/apache/twill/internal/kafka/client/KafkaResponseHandler.java
new file mode 100644
index 0000000..5251e65
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/kafka/client/KafkaResponseHandler.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.kafka.client;
+
+import org.apache.twill.kafka.client.FetchException;
+import org.jboss.netty.buffer.ChannelBuffer;
+import org.jboss.netty.channel.ChannelHandlerContext;
+import org.jboss.netty.channel.Channels;
+import org.jboss.netty.channel.MessageEvent;
+import org.jboss.netty.channel.SimpleChannelHandler;
+
+/**
+ *
+ */
+final class KafkaResponseHandler extends SimpleChannelHandler {
+
+ private final Bufferer bufferer = new Bufferer();
+
+ @Override
+ public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception {
+ Object msg = e.getMessage();
+ if (!(msg instanceof ChannelBuffer)) {
+ super.messageReceived(ctx, e);
+ return;
+ }
+
+ bufferer.apply((ChannelBuffer) msg);
+ ChannelBuffer buffer = bufferer.getNext();
+ while (buffer.readable()) {
+ // Send the response object upstream
+ Channels.fireMessageReceived(ctx, new KafkaResponse(FetchException.ErrorCode.fromCode(buffer.readShort()),
+ buffer, buffer.readableBytes() + 6));
+ buffer = bufferer.getNext();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/kafka/client/MessageFetcher.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/kafka/client/MessageFetcher.java b/core/src/main/java/org/apache/twill/internal/kafka/client/MessageFetcher.java
new file mode 100644
index 0000000..0814917
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/kafka/client/MessageFetcher.java
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.kafka.client;
+
+import org.apache.twill.common.Threads;
+import org.apache.twill.kafka.client.FetchException;
+import org.apache.twill.kafka.client.FetchedMessage;
+import com.google.common.base.Throwables;
+import com.google.common.collect.AbstractIterator;
+import com.google.common.io.ByteStreams;
+import org.jboss.netty.buffer.ChannelBuffer;
+import org.jboss.netty.buffer.ChannelBufferInputStream;
+import org.jboss.netty.buffer.ChannelBufferOutputStream;
+import org.jboss.netty.buffer.ChannelBuffers;
+import org.xerial.snappy.SnappyInputStream;
+
+import java.io.IOException;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Executors;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.zip.GZIPInputStream;
+
+/**
+ * This class is for consuming messages from a kafka topic.
+ */
+final class MessageFetcher extends AbstractIterator<FetchedMessage> implements ResponseHandler {
+
+ private static final long BACKOFF_INTERVAL_MS = 100;
+
+ private final KafkaRequestSender sender;
+ private final String topic;
+ private final int partition;
+ private final int maxSize;
+ private final AtomicLong offset;
+ private final BlockingQueue<FetchResult> messages;
+ private final ScheduledExecutorService scheduler;
+ private volatile long backoffMillis;
+ private final Runnable sendFetchRequest = new Runnable() {
+ @Override
+ public void run() {
+ sendFetchRequest();
+ }
+ };
+
+ MessageFetcher(String topic, int partition, long offset, int maxSize, KafkaRequestSender sender) {
+ this.topic = topic;
+ this.partition = partition;
+ this.sender = sender;
+ this.offset = new AtomicLong(offset);
+ this.maxSize = maxSize;
+ this.messages = new LinkedBlockingQueue<FetchResult>();
+ this.scheduler = Executors.newSingleThreadScheduledExecutor(
+ Threads.createDaemonThreadFactory("kafka-" + topic + "-consumer"));
+ }
+
+ @Override
+ public void received(KafkaResponse response) {
+ if (response.getErrorCode() != FetchException.ErrorCode.OK) {
+ messages.add(FetchResult.failure(new FetchException("Error in fetching: " + response.getErrorCode(),
+ response.getErrorCode())));
+ return;
+ }
+
+ try {
+ if (decodeResponse(response.getBody(), -1)) {
+ backoffMillis = 0;
+ } else {
+ backoffMillis = Math.max(backoffMillis + BACKOFF_INTERVAL_MS, 1000);
+ scheduler.schedule(sendFetchRequest, backoffMillis, TimeUnit.MILLISECONDS);
+ }
+ } catch (Throwable t) {
+ messages.add(FetchResult.failure(t));
+ }
+ }
+
+ private boolean decodeResponse(ChannelBuffer buffer, long nextOffset) {
+ boolean hasMessage = false;
+ boolean computeOffset = nextOffset < 0;
+ while (buffer.readableBytes() >= 4) {
+ int size = buffer.readInt();
+ if (buffer.readableBytes() < size) {
+ if (!hasMessage) {
+ throw new IllegalStateException("Size too small");
+ }
+ break;
+ }
+ nextOffset = computeOffset ? offset.addAndGet(size + 4) : nextOffset;
+ decodeMessage(size, buffer, nextOffset);
+ hasMessage = true;
+ }
+ return hasMessage;
+
+ }
+
+ private void decodeMessage(int size, ChannelBuffer buffer, long nextOffset) {
+ int readerIdx = buffer.readerIndex();
+ int magic = buffer.readByte();
+ Compression compression = magic == 0 ? Compression.NONE : Compression.fromCode(buffer.readByte());
+ int crc = buffer.readInt();
+
+ ChannelBuffer payload = buffer.readSlice(size - (buffer.readerIndex() - readerIdx));
+
+ // Verify CRC?
+ enqueueMessage(compression, payload, nextOffset);
+ }
+
+ private void enqueueMessage(Compression compression, ChannelBuffer payload, long nextOffset) {
+ switch (compression) {
+ case NONE:
+ messages.add(FetchResult.success(new BasicFetchedMessage(nextOffset, payload.toByteBuffer())));
+ break;
+ case GZIP:
+ decodeResponse(gunzip(payload), nextOffset);
+ break;
+ case SNAPPY:
+ decodeResponse(unsnappy(payload), nextOffset);
+ break;
+ }
+ }
+
+ private ChannelBuffer gunzip(ChannelBuffer source) {
+ ChannelBufferOutputStream output = new ChannelBufferOutputStream(
+ ChannelBuffers.dynamicBuffer(source.readableBytes() * 2));
+ try {
+ try {
+ GZIPInputStream gzipInput = new GZIPInputStream(new ChannelBufferInputStream(source));
+ try {
+ ByteStreams.copy(gzipInput, output);
+ return output.buffer();
+ } finally {
+ gzipInput.close();
+ }
+ } finally {
+ output.close();
+ }
+ } catch (IOException e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ private ChannelBuffer unsnappy(ChannelBuffer source) {
+ ChannelBufferOutputStream output = new ChannelBufferOutputStream(
+ ChannelBuffers.dynamicBuffer(source.readableBytes() * 2));
+ try {
+ try {
+ SnappyInputStream snappyInput = new SnappyInputStream(new ChannelBufferInputStream(source));
+ try {
+ ByteStreams.copy(snappyInput, output);
+ return output.buffer();
+ } finally {
+ snappyInput.close();
+ }
+ } finally {
+ output.close();
+ }
+ } catch (IOException e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ private void sendFetchRequest() {
+ ChannelBuffer fetchBody = ChannelBuffers.buffer(12);
+ fetchBody.writeLong(offset.get());
+ fetchBody.writeInt(maxSize);
+ sender.send(KafkaRequest.createFetch(topic, partition, fetchBody, MessageFetcher.this));
+ }
+
+ @Override
+ protected FetchedMessage computeNext() {
+ FetchResult result = messages.poll();
+ if (result != null) {
+ return getMessage(result);
+ }
+
+ try {
+ sendFetchRequest();
+ return getMessage(messages.take());
+ } catch (InterruptedException e) {
+ scheduler.shutdownNow();
+ return endOfData();
+ }
+ }
+
+ private FetchedMessage getMessage(FetchResult result) {
+ try {
+ if (result.isSuccess()) {
+ return result.getMessage();
+ } else {
+ throw result.getErrorCause();
+ }
+ } catch (Throwable t) {
+ throw Throwables.propagate(t);
+ }
+ }
+
+ private static final class FetchResult {
+ private final FetchedMessage message;
+ private final Throwable errorCause;
+
+ static FetchResult success(FetchedMessage message) {
+ return new FetchResult(message, null);
+ }
+
+ static FetchResult failure(Throwable cause) {
+ return new FetchResult(null, cause);
+ }
+
+ private FetchResult(FetchedMessage message, Throwable errorCause) {
+ this.message = message;
+ this.errorCause = errorCause;
+ }
+
+ public FetchedMessage getMessage() {
+ return message;
+ }
+
+ public Throwable getErrorCause() {
+ return errorCause;
+ }
+
+ public boolean isSuccess() {
+ return message != null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/kafka/client/MessageSetEncoder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/kafka/client/MessageSetEncoder.java b/core/src/main/java/org/apache/twill/internal/kafka/client/MessageSetEncoder.java
new file mode 100644
index 0000000..49008cc
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/kafka/client/MessageSetEncoder.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.kafka.client;
+
+import org.jboss.netty.buffer.ChannelBuffer;
+
+/**
+ * This represents a set of messages that goes into the same message set and get encoded as
+ * single kafka message set.
+ */
+interface MessageSetEncoder {
+
+ MessageSetEncoder add(ChannelBuffer payload);
+
+ ChannelBuffer finish();
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/kafka/client/ResponseHandler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/kafka/client/ResponseHandler.java b/core/src/main/java/org/apache/twill/internal/kafka/client/ResponseHandler.java
new file mode 100644
index 0000000..f681b85
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/kafka/client/ResponseHandler.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.kafka.client;
+
+/**
+ * Represents handler for kafka response.
+ */
+interface ResponseHandler {
+
+ ResponseHandler NO_OP = new ResponseHandler() {
+ @Override
+ public void received(KafkaResponse response) {
+ // No-op
+ }
+ };
+
+ void received(KafkaResponse response);
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/kafka/client/SimpleKafkaClient.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/kafka/client/SimpleKafkaClient.java b/core/src/main/java/org/apache/twill/internal/kafka/client/SimpleKafkaClient.java
new file mode 100644
index 0000000..8ff4856
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/kafka/client/SimpleKafkaClient.java
@@ -0,0 +1,304 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.kafka.client;
+
+import org.apache.twill.common.Threads;
+import org.apache.twill.kafka.client.FetchException;
+import org.apache.twill.kafka.client.FetchedMessage;
+import org.apache.twill.kafka.client.KafkaClient;
+import org.apache.twill.kafka.client.PreparePublish;
+import org.apache.twill.zookeeper.ZKClient;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.primitives.Ints;
+import com.google.common.primitives.Longs;
+import com.google.common.util.concurrent.AbstractIdleService;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.SettableFuture;
+import org.jboss.netty.bootstrap.ClientBootstrap;
+import org.jboss.netty.buffer.ChannelBuffer;
+import org.jboss.netty.buffer.ChannelBuffers;
+import org.jboss.netty.channel.Channel;
+import org.jboss.netty.channel.ChannelFuture;
+import org.jboss.netty.channel.ChannelFutureListener;
+import org.jboss.netty.channel.ChannelPipeline;
+import org.jboss.netty.channel.ChannelPipelineFactory;
+import org.jboss.netty.channel.Channels;
+import org.jboss.netty.channel.socket.nio.NioClientBossPool;
+import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory;
+import org.jboss.netty.channel.socket.nio.NioWorkerPool;
+import org.jboss.netty.util.HashedWheelTimer;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * Basic implementation of {@link KafkaClient}.
+ */
+public final class SimpleKafkaClient extends AbstractIdleService implements KafkaClient {
+
+ private static final Logger LOG = LoggerFactory.getLogger(SimpleKafkaClient.class);
+ private static final int BROKER_POLL_INTERVAL = 100;
+
+ private final KafkaBrokerCache brokerCache;
+ private ClientBootstrap bootstrap;
+ private ConnectionPool connectionPool;
+
+ public SimpleKafkaClient(ZKClient zkClient) {
+ this.brokerCache = new KafkaBrokerCache(zkClient);
+ }
+
+ @Override
+ protected void startUp() throws Exception {
+ brokerCache.startAndWait();
+ ThreadFactory threadFactory = Threads.createDaemonThreadFactory("kafka-client-netty-%d");
+ NioClientBossPool bossPool = new NioClientBossPool(Executors.newSingleThreadExecutor(threadFactory), 1,
+ new HashedWheelTimer(threadFactory), null);
+ NioWorkerPool workerPool = new NioWorkerPool(Executors.newFixedThreadPool(4, threadFactory), 4);
+
+ bootstrap = new ClientBootstrap(new NioClientSocketChannelFactory(bossPool, workerPool));
+ bootstrap.setPipelineFactory(new KafkaChannelPipelineFactory());
+ connectionPool = new ConnectionPool(bootstrap);
+ }
+
+ @Override
+ protected void shutDown() throws Exception {
+ connectionPool.close();
+ bootstrap.releaseExternalResources();
+ brokerCache.stopAndWait();
+ }
+
+ @Override
+ public PreparePublish preparePublish(final String topic, final Compression compression) {
+ final Map<Integer, MessageSetEncoder> encoders = Maps.newHashMap();
+
+ return new PreparePublish() {
+ @Override
+ public PreparePublish add(byte[] payload, Object partitionKey) {
+ return add(ByteBuffer.wrap(payload), partitionKey);
+ }
+
+ @Override
+ public PreparePublish add(ByteBuffer payload, Object partitionKey) {
+ // TODO: Partition
+ int partition = 0;
+
+ MessageSetEncoder encoder = encoders.get(partition);
+ if (encoder == null) {
+ encoder = getEncoder(compression);
+ encoders.put(partition, encoder);
+ }
+ encoder.add(ChannelBuffers.wrappedBuffer(payload));
+
+ return this;
+ }
+
+ @Override
+ public ListenableFuture<?> publish() {
+ List<ListenableFuture<?>> futures = Lists.newArrayListWithCapacity(encoders.size());
+ for (Map.Entry<Integer, MessageSetEncoder> entry : encoders.entrySet()) {
+ futures.add(doPublish(topic, entry.getKey(), entry.getValue().finish()));
+ }
+ encoders.clear();
+ return Futures.allAsList(futures);
+ }
+
+ private ListenableFuture<?> doPublish(String topic, int partition, ChannelBuffer messageSet) {
+ final KafkaRequest request = KafkaRequest.createProduce(topic, partition, messageSet);
+ final SettableFuture<?> result = SettableFuture.create();
+ final ConnectionPool.ConnectResult connection =
+ connectionPool.connect(getTopicBroker(topic, partition).getAddress());
+
+ connection.getChannelFuture().addListener(new ChannelFutureListener() {
+ @Override
+ public void operationComplete(ChannelFuture future) throws Exception {
+ try {
+ future.getChannel().write(request).addListener(getPublishChannelFutureListener(result, null, connection));
+ } catch (Exception e) {
+ result.setException(e);
+ }
+ }
+ });
+
+ return result;
+ }
+ };
+ }
+
+ @Override
+ public Iterator<FetchedMessage> consume(final String topic, final int partition, long offset, int maxSize) {
+ Preconditions.checkArgument(maxSize >= 10, "Message size cannot be smaller than 10.");
+
+ // Connect to broker. Consumer connection are long connection. No need to worry about reuse.
+ final AtomicReference<ChannelFuture> channelFutureRef = new AtomicReference<ChannelFuture>(
+ connectionPool.connect(getTopicBroker(topic, partition).getAddress()).getChannelFuture());
+
+ return new MessageFetcher(topic, partition, offset, maxSize, new KafkaRequestSender() {
+
+ @Override
+ public void send(final KafkaRequest request) {
+ if (!isRunning()) {
+ return;
+ }
+ try {
+ // Try to send the request
+ Channel channel = channelFutureRef.get().getChannel();
+ if (!channel.write(request).await().isSuccess()) {
+ // If failed, retry
+ channel.close();
+ ChannelFuture channelFuture = connectionPool.connect(
+ getTopicBroker(topic, partition).getAddress()).getChannelFuture();
+ channelFutureRef.set(channelFuture);
+ channelFuture.addListener(new ChannelFutureListener() {
+ @Override
+ public void operationComplete(ChannelFuture channelFuture) throws Exception {
+ send(request);
+ }
+ });
+ }
+ } catch (InterruptedException e) {
+ // Ignore it
+ LOG.info("Interrupted when sending consume request", e);
+ }
+ }
+ });
+ }
+
+ @Override
+ public ListenableFuture<long[]> getOffset(final String topic, final int partition, long time, int maxOffsets) {
+ final SettableFuture<long[]> resultFuture = SettableFuture.create();
+ final ChannelBuffer body = ChannelBuffers.buffer(Longs.BYTES + Ints.BYTES);
+ body.writeLong(time);
+ body.writeInt(maxOffsets);
+
+ connectionPool.connect(getTopicBroker(topic, partition).getAddress())
+ .getChannelFuture().addListener(new ChannelFutureListener() {
+ @Override
+ public void operationComplete(ChannelFuture future) throws Exception {
+ if (checkFailure(future)) {
+ return;
+ }
+
+ future.getChannel().write(KafkaRequest.createOffsets(topic, partition, body, new ResponseHandler() {
+ @Override
+ public void received(KafkaResponse response) {
+ if (response.getErrorCode() != FetchException.ErrorCode.OK) {
+ resultFuture.setException(new FetchException("Failed to fetch offset.", response.getErrorCode()));
+ } else {
+ // Decode the offset response, which contains 4 bytes number of offsets, followed by number of offsets,
+ // each 8 bytes in size.
+ ChannelBuffer resultBuffer = response.getBody();
+ int size = resultBuffer.readInt();
+ long[] result = new long[size];
+ for (int i = 0; i < size; i++) {
+ result[i] = resultBuffer.readLong();
+ }
+ resultFuture.set(result);
+ }
+ }
+ })).addListener(new ChannelFutureListener() {
+ @Override
+ public void operationComplete(ChannelFuture future) throws Exception {
+ checkFailure(future);
+ }
+ });
+ }
+
+ private boolean checkFailure(ChannelFuture future) {
+ if (!future.isSuccess()) {
+ if (future.isCancelled()) {
+ resultFuture.cancel(true);
+ } else {
+ resultFuture.setException(future.getCause());
+ }
+ return true;
+ }
+ return false;
+ }
+ });
+
+ return resultFuture;
+ }
+
+ private TopicBroker getTopicBroker(String topic, int partition) {
+ TopicBroker topicBroker = brokerCache.getBrokerAddress(topic, partition);
+ while (topicBroker == null) {
+ try {
+ TimeUnit.MILLISECONDS.sleep(BROKER_POLL_INTERVAL);
+ } catch (InterruptedException e) {
+ return null;
+ }
+ topicBroker = brokerCache.getBrokerAddress(topic, partition);
+ }
+ return topicBroker;
+ }
+
+ private MessageSetEncoder getEncoder(Compression compression) {
+ switch (compression) {
+ case GZIP:
+ return new GZipMessageSetEncoder();
+ case SNAPPY:
+ return new SnappyMessageSetEncoder();
+ default:
+ return new IdentityMessageSetEncoder();
+ }
+ }
+
+ private <V> ChannelFutureListener getPublishChannelFutureListener(final SettableFuture<V> result, final V resultObj,
+ final ConnectionPool.ConnectionReleaser releaser) {
+ return new ChannelFutureListener() {
+ @Override
+ public void operationComplete(ChannelFuture future) throws Exception {
+ try {
+ if (future.isSuccess()) {
+ result.set(resultObj);
+ } else if (future.isCancelled()) {
+ result.cancel(true);
+ } else {
+ result.setException(future.getCause());
+ }
+ } finally {
+ releaser.release();
+ }
+ }
+ };
+ }
+
+ private static final class KafkaChannelPipelineFactory implements ChannelPipelineFactory {
+
+ @Override
+ public ChannelPipeline getPipeline() throws Exception {
+ ChannelPipeline pipeline = Channels.pipeline();
+
+ pipeline.addLast("encoder", new KafkaRequestEncoder());
+ pipeline.addLast("decoder", new KafkaResponseHandler());
+ pipeline.addLast("dispatcher", new KafkaResponseDispatcher());
+ return pipeline;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/kafka/client/SnappyMessageSetEncoder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/kafka/client/SnappyMessageSetEncoder.java b/core/src/main/java/org/apache/twill/internal/kafka/client/SnappyMessageSetEncoder.java
new file mode 100644
index 0000000..bf18c08
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/kafka/client/SnappyMessageSetEncoder.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.kafka.client;
+
+import org.xerial.snappy.SnappyOutputStream;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * A {@link MessageSetEncoder} that compress messages using snappy.
+ */
+final class SnappyMessageSetEncoder extends AbstractCompressedMessageSetEncoder {
+
+ SnappyMessageSetEncoder() {
+ super(Compression.SNAPPY);
+ }
+
+ @Override
+ protected OutputStream createCompressedStream(OutputStream os) throws IOException {
+ return new SnappyOutputStream(os);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/kafka/client/TopicBroker.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/kafka/client/TopicBroker.java b/core/src/main/java/org/apache/twill/internal/kafka/client/TopicBroker.java
new file mode 100644
index 0000000..fd4bf03
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/kafka/client/TopicBroker.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.kafka.client;
+
+import java.net.InetSocketAddress;
+
+/**
+ * Represents broker information of a given topic.
+ */
+final class TopicBroker {
+
+ private final String topic;
+ private final InetSocketAddress address;
+ private final int partitionSize;
+
+ TopicBroker(String topic, InetSocketAddress address, int partitionSize) {
+ this.topic = topic;
+ this.address = address;
+ this.partitionSize = partitionSize;
+ }
+
+ String getTopic() {
+ return topic;
+ }
+
+ InetSocketAddress getAddress() {
+ return address;
+ }
+
+ int getPartitionSize() {
+ return partitionSize;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/kafka/client/package-info.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/kafka/client/package-info.java b/core/src/main/java/org/apache/twill/internal/kafka/client/package-info.java
new file mode 100644
index 0000000..f3f615c
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/kafka/client/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This package provides pure java kafka client implementation.
+ */
+package org.apache.twill.internal.kafka.client;
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/logging/KafkaAppender.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/logging/KafkaAppender.java b/core/src/main/java/org/apache/twill/internal/logging/KafkaAppender.java
new file mode 100644
index 0000000..12818ef
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/logging/KafkaAppender.java
@@ -0,0 +1,303 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.logging;
+
+import org.apache.twill.common.Services;
+import org.apache.twill.common.Threads;
+import org.apache.twill.internal.kafka.client.Compression;
+import org.apache.twill.internal.kafka.client.SimpleKafkaClient;
+import org.apache.twill.kafka.client.KafkaClient;
+import org.apache.twill.kafka.client.PreparePublish;
+import org.apache.twill.zookeeper.RetryStrategies;
+import org.apache.twill.zookeeper.ZKClientService;
+import org.apache.twill.zookeeper.ZKClientServices;
+import org.apache.twill.zookeeper.ZKClients;
+import ch.qos.logback.classic.pattern.ClassOfCallerConverter;
+import ch.qos.logback.classic.pattern.FileOfCallerConverter;
+import ch.qos.logback.classic.pattern.LineOfCallerConverter;
+import ch.qos.logback.classic.pattern.MethodOfCallerConverter;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.classic.spi.IThrowableProxy;
+import ch.qos.logback.classic.spi.StackTraceElementProxy;
+import ch.qos.logback.core.AppenderBase;
+import com.google.common.base.Charsets;
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Throwables;
+import com.google.common.collect.Iterables;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.gson.stream.JsonWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.StringWriter;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ *
+ */
+public final class KafkaAppender extends AppenderBase<ILoggingEvent> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(KafkaAppender.class);
+
+ private final LogEventConverter eventConverter;
+ private final AtomicReference<PreparePublish> publisher;
+ private final Runnable flushTask;
+ /**
+ * Rough count of how many entries are being buffered. It's just approximate, not exact.
+ */
+ private final AtomicInteger bufferedSize;
+
+ private ZKClientService zkClientService;
+ private KafkaClient kafkaClient;
+ private String zkConnectStr;
+ private String hostname;
+ private String topic;
+ private Queue<String> buffer;
+ private int flushLimit = 20;
+ private int flushPeriod = 100;
+ private ScheduledExecutorService scheduler;
+
+ public KafkaAppender() {
+ eventConverter = new LogEventConverter();
+ publisher = new AtomicReference<PreparePublish>();
+ flushTask = createFlushTask();
+ bufferedSize = new AtomicInteger();
+ buffer = new ConcurrentLinkedQueue<String>();
+ }
+
+ /**
+ * Sets the zookeeper connection string. Called by slf4j.
+ */
+ @SuppressWarnings("unused")
+ public void setZookeeper(String zkConnectStr) {
+ this.zkConnectStr = zkConnectStr;
+ }
+
+ /**
+ * Sets the hostname. Called by slf4j.
+ */
+ @SuppressWarnings("unused")
+ public void setHostname(String hostname) {
+ this.hostname = hostname;
+ }
+
+ /**
+ * Sets the topic name for publishing logs. Called by slf4j.
+ */
+ @SuppressWarnings("unused")
+ public void setTopic(String topic) {
+ this.topic = topic;
+ }
+
+ /**
+ * Sets the maximum number of cached log entries before performing an force flush. Called by slf4j.
+ */
+ @SuppressWarnings("unused")
+ public void setFlushLimit(int flushLimit) {
+ this.flushLimit = flushLimit;
+ }
+
+ /**
+ * Sets the periodic flush time in milliseconds. Called by slf4j.
+ */
+ @SuppressWarnings("unused")
+ public void setFlushPeriod(int flushPeriod) {
+ this.flushPeriod = flushPeriod;
+ }
+
+ @Override
+ public void start() {
+ Preconditions.checkNotNull(zkConnectStr);
+
+ scheduler = Executors.newSingleThreadScheduledExecutor(Threads.createDaemonThreadFactory("kafka-logger"));
+
+ zkClientService = ZKClientServices.delegate(
+ ZKClients.reWatchOnExpire(
+ ZKClients.retryOnFailure(ZKClientService.Builder.of(zkConnectStr).build(),
+ RetryStrategies.fixDelay(1, TimeUnit.SECONDS))));
+
+ kafkaClient = new SimpleKafkaClient(zkClientService);
+ Futures.addCallback(Services.chainStart(zkClientService, kafkaClient), new FutureCallback<Object>() {
+ @Override
+ public void onSuccess(Object result) {
+ LOG.info("Kafka client started: " + zkConnectStr);
+ publisher.set(kafkaClient.preparePublish(topic, Compression.SNAPPY));
+ scheduler.scheduleWithFixedDelay(flushTask, 0, flushPeriod, TimeUnit.MILLISECONDS);
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ // Fail to talk to kafka. Other than logging, what can be done?
+ LOG.error("Failed to start kafka client.", t);
+ }
+ });
+
+ super.start();
+ }
+
+ @Override
+ public void stop() {
+ super.stop();
+ scheduler.shutdownNow();
+ Futures.getUnchecked(Services.chainStop(kafkaClient, zkClientService));
+ }
+
+ public void forceFlush() {
+ try {
+ publishLogs().get(2, TimeUnit.SECONDS);
+ } catch (Exception e) {
+ LOG.error("Failed to publish last batch of log.", e);
+ }
+ }
+
+ @Override
+ protected void append(ILoggingEvent eventObject) {
+ buffer.offer(eventConverter.convert(eventObject));
+ if (bufferedSize.incrementAndGet() >= flushLimit && publisher.get() != null) {
+ // Try to do a extra flush
+ scheduler.submit(flushTask);
+ }
+ }
+
+ private ListenableFuture<Integer> publishLogs() {
+ // If the publisher is not available, simply returns a completed future.
+ PreparePublish publisher = KafkaAppender.this.publisher.get();
+ if (publisher == null) {
+ return Futures.immediateFuture(0);
+ }
+
+ int count = 0;
+ for (String json : Iterables.consumingIterable(buffer)) {
+ publisher.add(Charsets.UTF_8.encode(json), 0);
+ count++;
+ }
+ // Nothing to publish, simply returns a completed future.
+ if (count == 0) {
+ return Futures.immediateFuture(0);
+ }
+
+ bufferedSize.set(0);
+ final int finalCount = count;
+ return Futures.transform(publisher.publish(), new Function<Object, Integer>() {
+ @Override
+ public Integer apply(Object input) {
+ return finalCount;
+ }
+ });
+ }
+
+ /**
+ * Creates a {@link Runnable} that writes all logs in the buffer into kafka.
+ * @return The Runnable task
+ */
+ private Runnable createFlushTask() {
+ return new Runnable() {
+ @Override
+ public void run() {
+ Futures.addCallback(publishLogs(), new FutureCallback<Integer>() {
+ @Override
+ public void onSuccess(Integer result) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Log entries published, size=" + result);
+ }
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ LOG.error("Failed to push logs to kafka. Log entries dropped.", t);
+ }
+ });
+ }
+ };
+ }
+
+ /**
+ * Helper class to convert {@link ILoggingEvent} into json string.
+ */
+ private final class LogEventConverter {
+
+ private final ClassOfCallerConverter classNameConverter = new ClassOfCallerConverter();
+ private final MethodOfCallerConverter methodConverter = new MethodOfCallerConverter();
+ private final FileOfCallerConverter fileConverter = new FileOfCallerConverter();
+ private final LineOfCallerConverter lineConverter = new LineOfCallerConverter();
+
+ private String convert(ILoggingEvent event) {
+ StringWriter result = new StringWriter();
+ JsonWriter writer = new JsonWriter(result);
+
+ try {
+ try {
+ writer.beginObject();
+ writer.name("name").value(event.getLoggerName());
+ writer.name("host").value(hostname);
+ writer.name("timestamp").value(Long.toString(event.getTimeStamp()));
+ writer.name("level").value(event.getLevel().toString());
+ writer.name("className").value(classNameConverter.convert(event));
+ writer.name("method").value(methodConverter.convert(event));
+ writer.name("file").value(fileConverter.convert(event));
+ writer.name("line").value(lineConverter.convert(event));
+ writer.name("thread").value(event.getThreadName());
+ writer.name("message").value(event.getFormattedMessage());
+ writer.name("stackTraces");
+ encodeStackTraces(event.getThrowableProxy(), writer);
+
+ writer.endObject();
+ } finally {
+ writer.close();
+ }
+ } catch (IOException e) {
+ throw Throwables.propagate(e);
+ }
+
+ return result.toString();
+ }
+
+ private void encodeStackTraces(IThrowableProxy throwable, JsonWriter writer) throws IOException {
+ writer.beginArray();
+ try {
+ if (throwable == null) {
+ return;
+ }
+
+ for (StackTraceElementProxy stackTrace : throwable.getStackTraceElementProxyArray()) {
+ writer.beginObject();
+
+ StackTraceElement element = stackTrace.getStackTraceElement();
+ writer.name("className").value(element.getClassName());
+ writer.name("method").value(element.getMethodName());
+ writer.name("file").value(element.getFileName());
+ writer.name("line").value(element.getLineNumber());
+
+ writer.endObject();
+ }
+ } finally {
+ writer.endArray();
+ }
+ }
+ }
+}
[02/15] Initial import commit.
Posted by ch...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/RewatchOnExpireWatcher.java
----------------------------------------------------------------------
diff --git a/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/RewatchOnExpireWatcher.java b/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/RewatchOnExpireWatcher.java
new file mode 100644
index 0000000..181ca2b
--- /dev/null
+++ b/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/RewatchOnExpireWatcher.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.zookeeper;
+
+import org.apache.twill.zookeeper.NodeChildren;
+import org.apache.twill.zookeeper.NodeData;
+import org.apache.twill.zookeeper.ZKClient;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.atomic.AtomicMarkableReference;
+
+/**
+ * A wrapper for {@link Watcher} that will re-set the watch automatically until it is successful.
+ */
+final class RewatchOnExpireWatcher implements Watcher {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RewatchOnExpireWatcher.class);
+
+ enum ActionType {
+ EXISTS,
+ CHILDREN,
+ DATA
+ }
+
+ private final ZKClient client;
+ private final ActionType actionType;
+ private final String path;
+ private final Watcher delegate;
+ private final AtomicMarkableReference<Object> lastResult;
+
+ RewatchOnExpireWatcher(ZKClient client, ActionType actionType, String path, Watcher delegate) {
+ this.client = client;
+ this.actionType = actionType;
+ this.path = path;
+ this.delegate = delegate;
+ this.lastResult = new AtomicMarkableReference<Object>(null, false);
+ }
+
+ /**
+ * Sets the result from the operation that causes this watcher to be set.
+ */
+ void setLastResult(Object result) {
+ lastResult.compareAndSet(null, result, false, true);
+ }
+
+ @Override
+ public void process(WatchedEvent event) {
+ if (delegate != null && event.getType() != Event.EventType.None) {
+ try {
+ delegate.process(event);
+ } catch (Throwable t) {
+ LOG.error("Watcher throws exception.", t);
+ }
+ }
+
+ if (event.getState() != Event.KeeperState.Expired) {
+ return;
+ }
+ switch (actionType) {
+ case EXISTS:
+ exists();
+ break;
+ case CHILDREN:
+ children();
+ break;
+ case DATA:
+ data();
+ break;
+ }
+ }
+
+ private void exists() {
+ Futures.addCallback(client.exists(path, this), new FutureCallback<Stat>() {
+ @Override
+ public void onSuccess(Stat stat) {
+ // Since we know all callbacks and watcher are triggered from single event thread, there is no race condition.
+ Object oldResult = lastResult.getReference();
+ lastResult.compareAndSet(oldResult, null, true, false);
+
+ if (stat != oldResult && (stat == null || !stat.equals(oldResult))) {
+ if (stat == null) {
+ // previous stat is not null, means node deleted
+ process(new WatchedEvent(Event.EventType.NodeDeleted, Event.KeeperState.SyncConnected, path));
+ } else if (oldResult == null) {
+ // previous stat is null, means node created
+ process(new WatchedEvent(Event.EventType.NodeCreated, Event.KeeperState.SyncConnected, path));
+ } else {
+ // Otherwise, something changed on the node
+ process(new WatchedEvent(Event.EventType.NodeDataChanged, Event.KeeperState.SyncConnected, path));
+ }
+ }
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ if (RetryUtils.canRetry(t)) {
+ exists();
+ } else {
+ lastResult.set(null, false);
+ LOG.error("Fail to re-set watch on exists for path " + path, t);
+ }
+ }
+ });
+ }
+
+ private void children() {
+ Futures.addCallback(client.getChildren(path, this), new FutureCallback<NodeChildren>() {
+ @Override
+ public void onSuccess(NodeChildren result) {
+ Object oldResult = lastResult.getReference();
+ lastResult.compareAndSet(oldResult, null, true, false);
+
+ if (result.equals(oldResult)) {
+ return;
+ }
+
+ if (!(oldResult instanceof NodeChildren)) {
+ // Something very wrong
+ LOG.error("The same watcher has been used for different event type.");
+ return;
+ }
+
+ NodeChildren oldNodeChildren = (NodeChildren) oldResult;
+ if (!result.getChildren().equals(oldNodeChildren.getChildren())) {
+ process(new WatchedEvent(Event.EventType.NodeChildrenChanged, Event.KeeperState.SyncConnected, path));
+ } else {
+ process(new WatchedEvent(Event.EventType.NodeDataChanged, Event.KeeperState.SyncConnected, path));
+ }
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ if (RetryUtils.canRetry(t)) {
+ children();
+ return;
+ }
+
+ lastResult.set(null, false);
+ if (t instanceof KeeperException) {
+ KeeperException.Code code = ((KeeperException) t).code();
+ if (code == KeeperException.Code.NONODE) {
+ // Node deleted
+ process(new WatchedEvent(Event.EventType.NodeDeleted, Event.KeeperState.SyncConnected, path));
+ return;
+ }
+ }
+ LOG.error("Fail to re-set watch on getChildren for path " + path, t);
+ }
+ });
+ }
+
+ private void data() {
+ Futures.addCallback(client.getData(path, this), new FutureCallback<NodeData>() {
+ @Override
+ public void onSuccess(NodeData result) {
+ Object oldResult = lastResult.getReference();
+ lastResult.compareAndSet(oldResult, null, true, false);
+
+ if (!result.equals(oldResult)) {
+ // Whenever something changed, treated it as data changed.
+ process(new WatchedEvent(Event.EventType.NodeDataChanged, Event.KeeperState.SyncConnected, path));
+ }
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ if (RetryUtils.canRetry(t)) {
+ data();
+ return;
+ }
+
+ lastResult.set(null, false);
+ if (t instanceof KeeperException) {
+ KeeperException.Code code = ((KeeperException) t).code();
+ if (code == KeeperException.Code.NONODE) {
+ // Node deleted
+ process(new WatchedEvent(Event.EventType.NodeDeleted, Event.KeeperState.SyncConnected, path));
+ return;
+ }
+ }
+ LOG.error("Fail to re-set watch on getData for path " + path, t);
+ }
+ });
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/RewatchOnExpireZKClient.java
----------------------------------------------------------------------
diff --git a/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/RewatchOnExpireZKClient.java b/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/RewatchOnExpireZKClient.java
new file mode 100644
index 0000000..402c153
--- /dev/null
+++ b/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/RewatchOnExpireZKClient.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.zookeeper;
+
+import org.apache.twill.internal.zookeeper.RewatchOnExpireWatcher.ActionType;
+import org.apache.twill.zookeeper.ForwardingZKClient;
+import org.apache.twill.zookeeper.NodeChildren;
+import org.apache.twill.zookeeper.NodeData;
+import org.apache.twill.zookeeper.OperationFuture;
+import org.apache.twill.zookeeper.ZKClient;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.data.Stat;
+
+/**
+ * A {@link ZKClient} that will rewatch automatically when session expired and reconnect.
+ * The rewatch logic is mainly done in {@link RewatchOnExpireWatcher}.
+ */
+public final class RewatchOnExpireZKClient extends ForwardingZKClient {
+
+ public RewatchOnExpireZKClient(ZKClient delegate) {
+ super(delegate);
+ }
+
+ @Override
+ public OperationFuture<Stat> exists(String path, Watcher watcher) {
+ final RewatchOnExpireWatcher wrappedWatcher = new RewatchOnExpireWatcher(this, ActionType.EXISTS, path, watcher);
+ OperationFuture<Stat> result = super.exists(path, wrappedWatcher);
+ Futures.addCallback(result, new FutureCallback<Stat>() {
+ @Override
+ public void onSuccess(Stat result) {
+ wrappedWatcher.setLastResult(result);
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ // No-op
+ }
+ });
+ return result;
+ }
+
+ @Override
+ public OperationFuture<NodeChildren> getChildren(String path, Watcher watcher) {
+ final RewatchOnExpireWatcher wrappedWatcher = new RewatchOnExpireWatcher(this, ActionType.CHILDREN, path, watcher);
+ OperationFuture<NodeChildren> result = super.getChildren(path, wrappedWatcher);
+ Futures.addCallback(result, new FutureCallback<NodeChildren>() {
+ @Override
+ public void onSuccess(NodeChildren result) {
+ wrappedWatcher.setLastResult(result);
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ // No-op
+ }
+ });
+ return result;
+ }
+
+ @Override
+ public OperationFuture<NodeData> getData(String path, Watcher watcher) {
+ final RewatchOnExpireWatcher wrappedWatcher = new RewatchOnExpireWatcher(this, ActionType.DATA, path, watcher);
+ OperationFuture<NodeData> result = super.getData(path, wrappedWatcher);
+ Futures.addCallback(result, new FutureCallback<NodeData>() {
+ @Override
+ public void onSuccess(NodeData result) {
+ wrappedWatcher.setLastResult(result);
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ // No-op
+ }
+ });
+ return result;
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/SettableOperationFuture.java
----------------------------------------------------------------------
diff --git a/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/SettableOperationFuture.java b/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/SettableOperationFuture.java
new file mode 100644
index 0000000..7544e56
--- /dev/null
+++ b/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/SettableOperationFuture.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.zookeeper;
+
+import org.apache.twill.zookeeper.OperationFuture;
+import com.google.common.util.concurrent.AbstractFuture;
+
+import javax.annotation.Nullable;
+import java.util.concurrent.Executor;
+
+/**
+ * An implementation for {@link OperationFuture} that allows setting result directly.
+ * Also, all listener callback will be fired from the given executor.
+ */
+public final class SettableOperationFuture<V> extends AbstractFuture<V> implements OperationFuture<V> {
+
+ private final String requestPath;
+ private final Executor executor;
+
+ public static <V> SettableOperationFuture<V> create(String path, Executor executor) {
+ return new SettableOperationFuture<V>(path, executor);
+ }
+
+ private SettableOperationFuture(String requestPath, Executor executor) {
+ this.requestPath = requestPath;
+ this.executor = executor;
+ }
+
+ @Override
+ public String getRequestPath() {
+ return requestPath;
+ }
+
+ @Override
+ public void addListener(final Runnable listener, final Executor exec) {
+ super.addListener(new Runnable() {
+ @Override
+ public void run() {
+ exec.execute(listener);
+ }
+ }, executor);
+ }
+
+ @Override
+ public boolean setException(Throwable throwable) {
+ return super.setException(throwable);
+ }
+
+ @Override
+ public boolean set(@Nullable V value) {
+ return super.set(value);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/package-info.java
----------------------------------------------------------------------
diff --git a/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/package-info.java b/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/package-info.java
new file mode 100644
index 0000000..d2afa11
--- /dev/null
+++ b/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Internal classes for zookeeper.
+ */
+package org.apache.twill.internal.zookeeper;
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/zookeeper/src/main/java/org/apache/twill/zookeeper/ForwardingZKClient.java
----------------------------------------------------------------------
diff --git a/zookeeper/src/main/java/org/apache/twill/zookeeper/ForwardingZKClient.java b/zookeeper/src/main/java/org/apache/twill/zookeeper/ForwardingZKClient.java
new file mode 100644
index 0000000..3f3003d
--- /dev/null
+++ b/zookeeper/src/main/java/org/apache/twill/zookeeper/ForwardingZKClient.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.zookeeper;
+
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.data.Stat;
+
+import javax.annotation.Nullable;
+
+/**
+ *
+ */
+public abstract class ForwardingZKClient implements ZKClient {
+
+ private final ZKClient delegate;
+
+ protected ForwardingZKClient(ZKClient delegate) {
+ this.delegate = delegate;
+ }
+
+ public final ZKClient getDelegate() {
+ return delegate;
+ }
+
+ @Override
+ public Long getSessionId() {
+ return delegate.getSessionId();
+ }
+
+ @Override
+ public String getConnectString() {
+ return delegate.getConnectString();
+ }
+
+ @Override
+ public void addConnectionWatcher(Watcher watcher) {
+ delegate.addConnectionWatcher(watcher);
+ }
+
+ @Override
+ public OperationFuture<String> create(String path, @Nullable byte[] data, CreateMode createMode) {
+ return create(path, data, createMode, true);
+ }
+
+ @Override
+ public OperationFuture<String> create(String path, @Nullable byte[] data, CreateMode createMode,
+ boolean createParent) {
+ return delegate.create(path, data, createMode, createParent);
+ }
+
+ @Override
+ public OperationFuture<Stat> exists(String path) {
+ return exists(path, null);
+ }
+
+ @Override
+ public OperationFuture<Stat> exists(String path, @Nullable Watcher watcher) {
+ return delegate.exists(path, watcher);
+ }
+
+ @Override
+ public OperationFuture<NodeChildren> getChildren(String path) {
+ return getChildren(path, null);
+ }
+
+ @Override
+ public OperationFuture<NodeChildren> getChildren(String path, @Nullable Watcher watcher) {
+ return delegate.getChildren(path, watcher);
+ }
+
+ @Override
+ public OperationFuture<NodeData> getData(String path) {
+ return getData(path, null);
+ }
+
+ @Override
+ public OperationFuture<NodeData> getData(String path, @Nullable Watcher watcher) {
+ return delegate.getData(path, watcher);
+ }
+
+ @Override
+ public OperationFuture<Stat> setData(String path, byte[] data) {
+ return setData(path, data, -1);
+ }
+
+ @Override
+ public OperationFuture<Stat> setData(String dataPath, byte[] data, int version) {
+ return delegate.setData(dataPath, data, version);
+ }
+
+ @Override
+ public OperationFuture<String> delete(String path) {
+ return delete(path, -1);
+ }
+
+ @Override
+ public OperationFuture<String> delete(String deletePath, int version) {
+ return delegate.delete(deletePath, version);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/zookeeper/src/main/java/org/apache/twill/zookeeper/ForwardingZKClientService.java
----------------------------------------------------------------------
diff --git a/zookeeper/src/main/java/org/apache/twill/zookeeper/ForwardingZKClientService.java b/zookeeper/src/main/java/org/apache/twill/zookeeper/ForwardingZKClientService.java
new file mode 100644
index 0000000..10391b2
--- /dev/null
+++ b/zookeeper/src/main/java/org/apache/twill/zookeeper/ForwardingZKClientService.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.zookeeper;
+
+import com.google.common.base.Supplier;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import org.apache.zookeeper.ZooKeeper;
+
+import java.util.concurrent.Executor;
+
+/**
+ *
+ */
+public abstract class ForwardingZKClientService extends ForwardingZKClient implements ZKClientService {
+
+ private final ZKClientService delegate;
+
+ protected ForwardingZKClientService(ZKClientService delegate) {
+ super(delegate);
+ this.delegate = delegate;
+ }
+
+ @Override
+ public Supplier<ZooKeeper> getZooKeeperSupplier() {
+ return delegate.getZooKeeperSupplier();
+ }
+
+ @Override
+ public ListenableFuture<State> start() {
+ return delegate.start();
+ }
+
+ @Override
+ public State startAndWait() {
+ return Futures.getUnchecked(start());
+ }
+
+ @Override
+ public boolean isRunning() {
+ return delegate.isRunning();
+ }
+
+ @Override
+ public State state() {
+ return delegate.state();
+ }
+
+ @Override
+ public ListenableFuture<State> stop() {
+ return delegate.stop();
+ }
+
+ @Override
+ public State stopAndWait() {
+ return Futures.getUnchecked(stop());
+ }
+
+ @Override
+ public void addListener(Listener listener, Executor executor) {
+ delegate.addListener(listener, executor);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/zookeeper/src/main/java/org/apache/twill/zookeeper/NodeChildren.java
----------------------------------------------------------------------
diff --git a/zookeeper/src/main/java/org/apache/twill/zookeeper/NodeChildren.java b/zookeeper/src/main/java/org/apache/twill/zookeeper/NodeChildren.java
new file mode 100644
index 0000000..b432c01
--- /dev/null
+++ b/zookeeper/src/main/java/org/apache/twill/zookeeper/NodeChildren.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.zookeeper;
+
+import org.apache.zookeeper.data.Stat;
+
+import java.util.List;
+
+/**
+ * Represents result of call to {@link ZKClientService#getChildren(String, org.apache.zookeeper.Watcher)} method.
+ */
+public interface NodeChildren {
+
+ /**
+ * @return The {@link Stat} of the node.
+ */
+ Stat getStat();
+
+ /**
+ * @return List of children node names.
+ */
+ List<String> getChildren();
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/zookeeper/src/main/java/org/apache/twill/zookeeper/NodeData.java
----------------------------------------------------------------------
diff --git a/zookeeper/src/main/java/org/apache/twill/zookeeper/NodeData.java b/zookeeper/src/main/java/org/apache/twill/zookeeper/NodeData.java
new file mode 100644
index 0000000..ac15957
--- /dev/null
+++ b/zookeeper/src/main/java/org/apache/twill/zookeeper/NodeData.java
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.zookeeper;
+
+import org.apache.zookeeper.data.Stat;
+
+import javax.annotation.Nullable;
+
+/**
+ * Represents result of call to {@link ZKClientService#getData(String, org.apache.zookeeper.Watcher)}.
+ */
+public interface NodeData {
+
+ /**
+ * @return The {@link Stat} of the node.
+ */
+ Stat getStat();
+
+ /**
+ * @return Data stored in the node, or {@code null} if there is no data.
+ */
+ @Nullable
+ byte[] getData();
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/zookeeper/src/main/java/org/apache/twill/zookeeper/OperationFuture.java
----------------------------------------------------------------------
diff --git a/zookeeper/src/main/java/org/apache/twill/zookeeper/OperationFuture.java b/zookeeper/src/main/java/org/apache/twill/zookeeper/OperationFuture.java
new file mode 100644
index 0000000..fafaa7a
--- /dev/null
+++ b/zookeeper/src/main/java/org/apache/twill/zookeeper/OperationFuture.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.zookeeper;
+
+import com.google.common.util.concurrent.ListenableFuture;
+
+/**
+ * A {@link ListenableFuture} that also provides the requested path for a operation.
+ *
+ * @param <V> The result type returned by this Future's {@link #get()} method.
+ */
+public interface OperationFuture<V> extends ListenableFuture<V> {
+
+ /**
+ * @return The path being requested for the ZooKeeper operation.
+ */
+ String getRequestPath();
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/zookeeper/src/main/java/org/apache/twill/zookeeper/RetryStrategies.java
----------------------------------------------------------------------
diff --git a/zookeeper/src/main/java/org/apache/twill/zookeeper/RetryStrategies.java b/zookeeper/src/main/java/org/apache/twill/zookeeper/RetryStrategies.java
new file mode 100644
index 0000000..56474b7
--- /dev/null
+++ b/zookeeper/src/main/java/org/apache/twill/zookeeper/RetryStrategies.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.zookeeper;
+
+import com.google.common.base.Preconditions;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Factory for creating common {@link RetryStrategy} implementation.
+ */
+public final class RetryStrategies {
+
+ /**
+ * @return A {@link RetryStrategy} that doesn't do any retry.
+ */
+ public static RetryStrategy noRetry() {
+ return new RetryStrategy() {
+ @Override
+ public long nextRetry(int failureCount, long startTime, OperationType type, String path) {
+ return -1;
+ }
+ };
+ }
+
+ /**
+ * Creates a {@link RetryStrategy} that retries maximum given number of times, with the actual
+ * delay behavior delegated to another {@link RetryStrategy}.
+ * @param limit Maximum number of retries allowed.
+ * @param strategy When failure count is less than or equal to the limit, this strategy will be called.
+ * @return A {@link RetryStrategy}.
+ */
+ public static RetryStrategy limit(final int limit, final RetryStrategy strategy) {
+ Preconditions.checkArgument(limit >= 0, "limit must be >= 0");
+ return new RetryStrategy() {
+ @Override
+ public long nextRetry(int failureCount, long startTime, OperationType type, String path) {
+ return (failureCount <= limit) ? strategy.nextRetry(failureCount, startTime, type, path) : -1L;
+ }
+ };
+ }
+
+ /**
+ * Creates a {@link RetryStrategy} that imposes a fix delay between each retries.
+ * @param delay delay time
+ * @param delayUnit {@link TimeUnit} for the delay.
+ * @return A {@link RetryStrategy}.
+ */
+ public static RetryStrategy fixDelay(final long delay, final TimeUnit delayUnit) {
+ Preconditions.checkArgument(delay >= 0, "delay must be >= 0");
+ return new RetryStrategy() {
+ @Override
+ public long nextRetry(int failureCount, long startTime, OperationType type, String path) {
+ return TimeUnit.MILLISECONDS.convert(delay, delayUnit);
+ }
+ };
+ }
+
+ /**
+ * Creates a {@link RetryStrategy} that will increase delay exponentially between each retries.
+ * @param baseDelay delay to start with.
+ * @param maxDelay cap of the delay.
+ * @param delayUnit {@link TimeUnit} for the delays.
+ * @return A {@link RetryStrategy}.
+ */
+ public static RetryStrategy exponentialDelay(final long baseDelay, final long maxDelay, final TimeUnit delayUnit) {
+ Preconditions.checkArgument(baseDelay >= 0, "base delay must be >= 0");
+ Preconditions.checkArgument(maxDelay >= 0, "max delay must be >= 0");
+ return new RetryStrategy() {
+ @Override
+ public long nextRetry(int failureCount, long startTime, OperationType type, String path) {
+ long power = failureCount > Long.SIZE ? Long.MAX_VALUE : (1L << (failureCount - 1));
+ long delay = Math.min(baseDelay * power, maxDelay);
+ delay = delay < 0 ? maxDelay : delay;
+ return TimeUnit.MILLISECONDS.convert(delay, delayUnit);
+ }
+ };
+ }
+
+ /**
+ * Creates a {@link RetryStrategy} that will retry until maximum amount of time has been passed since the request,
+ * with the actual delay behavior delegated to another {@link RetryStrategy}.
+ * @param maxElapseTime Maximum amount of time until giving up retry.
+ * @param timeUnit {@link TimeUnit} for the max elapse time.
+ * @param strategy When time elapsed is less than or equal to the limit, this strategy will be called.
+ * @return A {@link RetryStrategy}.
+ */
+ public static RetryStrategy timeLimit(long maxElapseTime, TimeUnit timeUnit, final RetryStrategy strategy) {
+ Preconditions.checkArgument(maxElapseTime >= 0, "max elapse time must be >= 0");
+ final long maxElapseMs = TimeUnit.MILLISECONDS.convert(maxElapseTime, timeUnit);
+ return new RetryStrategy() {
+ @Override
+ public long nextRetry(int failureCount, long startTime, OperationType type, String path) {
+ long elapseTime = System.currentTimeMillis() - startTime;
+ return elapseTime <= maxElapseMs ? strategy.nextRetry(failureCount, startTime, type, path) : -1L;
+ }
+ };
+ }
+
+ private RetryStrategies() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/zookeeper/src/main/java/org/apache/twill/zookeeper/RetryStrategy.java
----------------------------------------------------------------------
diff --git a/zookeeper/src/main/java/org/apache/twill/zookeeper/RetryStrategy.java b/zookeeper/src/main/java/org/apache/twill/zookeeper/RetryStrategy.java
new file mode 100644
index 0000000..3301e8a
--- /dev/null
+++ b/zookeeper/src/main/java/org/apache/twill/zookeeper/RetryStrategy.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.zookeeper;
+
+/**
+ * Provides strategy to use for operation retries.
+ */
+public interface RetryStrategy {
+
+ /**
+ * Defines ZooKeeper operation type that triggers retry.
+ */
+ enum OperationType {
+ CREATE,
+ EXISTS,
+ GET_CHILDREN,
+ GET_DATA,
+ SET_DATA,
+ DELETE
+ }
+
+ /**
+ * Returns the number of milliseconds to wait before retrying the operation.
+ *
+ * @param failureCount Number of times that the request has been failed.
+ * @param startTime Timestamp in milliseconds that the request starts.
+ * @param type Type of operation tried to perform.
+ * @param path The path that the operation is acting on.
+ * @return Number of milliseconds to wait before retrying the operation. Returning {@code 0} means
+ * retry it immediately, while negative means abort the operation.
+ */
+ long nextRetry(int failureCount, long startTime, OperationType type, String path);
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/zookeeper/src/main/java/org/apache/twill/zookeeper/ZKClient.java
----------------------------------------------------------------------
diff --git a/zookeeper/src/main/java/org/apache/twill/zookeeper/ZKClient.java b/zookeeper/src/main/java/org/apache/twill/zookeeper/ZKClient.java
new file mode 100644
index 0000000..d60182e
--- /dev/null
+++ b/zookeeper/src/main/java/org/apache/twill/zookeeper/ZKClient.java
@@ -0,0 +1,161 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.zookeeper;
+
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.data.Stat;
+
+import javax.annotation.Nullable;
+
+/**
+ * A ZooKeeper client that provides asynchronous zookeeper operations.
+ */
+public interface ZKClient {
+
+ /**
+ * Returns the current Zookeeper session ID of this client.
+ * If this ZKClient is not connected, {@code null} is returned.
+ */
+ Long getSessionId();
+
+ /**
+ * Returns the connection string used for connecting to Zookeeper.
+ */
+ String getConnectString();
+
+ /**
+ * Adds a {@link Watcher} that will be called whenever connection state change.
+ * @param watcher The watcher to set.
+ */
+ void addConnectionWatcher(Watcher watcher);
+
+ /**
+ * Same as calling
+ * {@link #create(String, byte[], org.apache.zookeeper.CreateMode, boolean) create(path, data, createMode, true)}.
+ *
+ * @see #create(String, byte[], org.apache.zookeeper.CreateMode, boolean)
+ */
+ OperationFuture<String> create(String path, @Nullable byte[] data, CreateMode createMode);
+
+ /**
+ * Creates a path in zookeeper, with given data and create mode.
+ *
+ * @param path Path to be created
+ * @param data Data to be stored in the node, or {@code null} if no data to store.
+ * @param createMode The {@link org.apache.zookeeper.CreateMode} for the node.
+ * @param createParent If {@code true} and parent nodes are missing, it will create all parent nodes as normal
+ * persistent node before creating the request node.
+ * @return A {@link OperationFuture} that will be completed when the
+ * creation is done. If there is error during creation, it will be reflected as error in the future.
+ */
+ OperationFuture<String> create(String path, @Nullable byte[] data, CreateMode createMode, boolean createParent);
+
+ /**
+ * Checks if the path exists. Same as calling
+ * {@link #exists(String, org.apache.zookeeper.Watcher) exists(path, null)}.
+ *
+ * @see #exists(String, org.apache.zookeeper.Watcher)
+ */
+ OperationFuture<Stat> exists(String path);
+
+ /**
+ * Checks if the given path exists and leave a watcher on the node for watching creation/deletion/data changes
+ * on the node.
+ *
+ * @param path The path to check for existence.
+ * @param watcher Watcher for watching changes, or {@code null} if no watcher to set.
+ * @return A {@link OperationFuture} that will be completed when the exists check is done. If the path
+ * does exists, the node {@link Stat} is set into the future. If the path doesn't exists,
+ * a {@code null} value is set into the future.
+ */
+ OperationFuture<Stat> exists(String path, @Nullable Watcher watcher);
+
+ /**
+ * Gets the list of children nodes under the given path. Same as calling
+ * {@link #getChildren(String, org.apache.zookeeper.Watcher) getChildren(path, null)}.
+ *
+ * @see #getChildren(String, org.apache.zookeeper.Watcher)
+ */
+ OperationFuture<NodeChildren> getChildren(String path);
+
+ /**
+ * Gets the list of children nodes under the given path and leave a watcher on the node for watching node
+ * deletion and children nodes creation/deletion.
+ *
+ * @param path The path to fetch for children nodes
+ * @param watcher Watcher for watching changes, or {@code null} if no watcher to set.
+ * @return A {@link OperationFuture} that will be completed when the getChildren call is done, with the result
+ * given as {@link NodeChildren}. If there is error, it will be reflected as error in the future.
+ */
+ OperationFuture<NodeChildren> getChildren(String path, @Nullable Watcher watcher);
+
+ /**
+ * Gets the data stored in the given path. Same as calling
+ * {@link #getData(String, org.apache.zookeeper.Watcher) getData(path, null)}.
+ */
+ OperationFuture<NodeData> getData(String path);
+
+ /**
+ * Gets the data stored in the given path and leave a watcher on the node for watching deletion/data changes on
+ * the node.
+ *
+ * @param path The path to get data from.
+ * @param watcher Watcher for watching changes, or {@code null} if no watcher to set.
+ * @return A {@link OperationFuture} that will be completed when the getData call is done, with the result
+ * given as {@link NodeData}. If there is error, it will be reflected as error in the future.
+ */
+ OperationFuture<NodeData> getData(String path, @Nullable Watcher watcher);
+
+ /**
+ * Sets the data for the given path without matching version. Same as calling
+ * {@link #setData(String, byte[], int) setData(path, data, -1)}.
+ */
+ OperationFuture<Stat> setData(String path, byte[] data);
+
+ /**
+ * Sets the data for the given path that match the given version. If the version given is {@code -1}, it matches
+ * any version.
+ *
+ * @param dataPath The path to set data to.
+ * @param data Data to be set.
+ * @param version Matching version.
+ * @return A {@link OperationFuture} that will be completed when the setData call is done, with node {@link Stat}
+ * given as the future result. If there is error, it will be reflected as error in the future.
+ */
+ OperationFuture<Stat> setData(String dataPath, byte[] data, int version);
+
+ /**
+ * Deletes the node of the given path without matching version. Same as calling
+ * {@link #delete(String, int) delete(path, -1)}.
+ *
+ * @see #delete(String, int)
+ */
+ OperationFuture<String> delete(String path);
+
+ /**
+ * Deletes the node of the given path that match the given version. If the version given is {@code -1}, it matches
+ * any version.
+ *
+ * @param deletePath The path to set data to.
+ * @param version Matching version.
+ * @return A {@link OperationFuture} that will be completed when the setData call is done, with node path
+ * given as the future result. If there is error, it will be reflected as error in the future.
+ */
+ OperationFuture<String> delete(String deletePath, int version);
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/zookeeper/src/main/java/org/apache/twill/zookeeper/ZKClientService.java
----------------------------------------------------------------------
diff --git a/zookeeper/src/main/java/org/apache/twill/zookeeper/ZKClientService.java b/zookeeper/src/main/java/org/apache/twill/zookeeper/ZKClientService.java
new file mode 100644
index 0000000..63f27fb
--- /dev/null
+++ b/zookeeper/src/main/java/org/apache/twill/zookeeper/ZKClientService.java
@@ -0,0 +1,96 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.zookeeper;
+
+import org.apache.twill.internal.zookeeper.DefaultZKClientService;
+import com.google.common.base.Supplier;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimap;
+import com.google.common.util.concurrent.Service;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.data.ACL;
+
+/**
+ * A {@link ZKClient} that extends from {@link Service} to provide lifecycle management functions.
+ * The {@link #start()} method needed to be called before calling any other method on this interface.
+ * When the client is no longer needed, call {@link #stop()} to release any resources that it holds.
+ */
+public interface ZKClientService extends ZKClient, Service {
+
+ /**
+ * Returns a {@link Supplier} of {@link ZooKeeper} that gives the current {@link ZooKeeper} in use at the moment
+ * when {@link com.google.common.base.Supplier#get()} get called.
+ *
+ * @return A {@link Supplier Supplier<ZooKeeper>}
+ */
+ Supplier<ZooKeeper> getZooKeeperSupplier();
+
+ /**
+ * Builder for creating an implementation of {@link ZKClientService}.
+ * The default client timeout is 10000ms.
+ */
+ public static final class Builder {
+
+ private final String connectStr;
+ private int timeout = 10000;
+ private Watcher connectionWatcher;
+ private Multimap<String, ACL> acls = HashMultimap.create();
+
+ /**
+ * Creates a {@link Builder} with the given ZooKeeper connection string.
+ * @param connectStr The connection string.
+ * @return A new instance of Builder.
+ */
+ public static Builder of(String connectStr) {
+ return new Builder(connectStr);
+ }
+
+ /**
+ * Sets the client timeout to the give milliseconds.
+ * @param timeout timeout in milliseconds.
+ * @return This builder
+ */
+ public Builder setSessionTimeout(int timeout) {
+ this.timeout = timeout;
+ return this;
+ }
+
+ /**
+ * Sets a {@link Watcher} that will be called whenever connection state change.
+ * @param watcher The watcher to set.
+ * @return This builder.
+ */
+ public Builder setConnectionWatcher(Watcher watcher) {
+ this.connectionWatcher = watcher;
+ return this;
+ }
+
+ /**
+ * Creates an instance of {@link ZKClientService} with the settings of this builder.
+ * @return A new instance of {@link ZKClientService}.
+ */
+ public ZKClientService build() {
+ return new DefaultZKClientService(connectStr, timeout, connectionWatcher);
+ }
+
+ private Builder(String connectStr) {
+ this.connectStr = connectStr;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/zookeeper/src/main/java/org/apache/twill/zookeeper/ZKClientServices.java
----------------------------------------------------------------------
diff --git a/zookeeper/src/main/java/org/apache/twill/zookeeper/ZKClientServices.java b/zookeeper/src/main/java/org/apache/twill/zookeeper/ZKClientServices.java
new file mode 100644
index 0000000..cc38c76
--- /dev/null
+++ b/zookeeper/src/main/java/org/apache/twill/zookeeper/ZKClientServices.java
@@ -0,0 +1,145 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.zookeeper;
+
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.data.Stat;
+
+import javax.annotation.Nullable;
+
+/**
+ * Provides static factory method to create {@link ZKClientService} with modified behaviors.
+ */
+public final class ZKClientServices {
+
+ /**
+ * Creates a {@link ZKClientService} from the given {@link ZKClient} if the given {@link ZKClient} is an instance of
+ * {@link ZKClientService} or is a {@link ForwardingZKClient} that eventually trace back to a delegate of type
+ * {@link ZKClientService}. If such a {@link ZKClientService} instance is found, this method returns
+ * an instance by invoking {@link #delegate(ZKClient, ZKClientService)} with the given {@link ZKClient} and
+ * the {@link ZKClientService} found respectively.
+ *
+ * @param client The {@link ZKClient}.
+ * @return A {@link ZKClientService}.
+ * @throws IllegalArgumentException If no {@link ZKClientService} is found.
+ */
+ public static ZKClientService delegate(ZKClient client) {
+ ZKClient zkClient = client;
+ while (!(zkClient instanceof ZKClientService) && zkClient instanceof ForwardingZKClient) {
+ zkClient = ((ForwardingZKClient) zkClient).getDelegate();
+ }
+ if (zkClient instanceof ZKClientService) {
+ return delegate(client, (ZKClientService) zkClient);
+ }
+ throw new IllegalArgumentException("No ZKClientService found from the delegation hierarchy");
+ }
+
+ /**
+ * Creates a {@link ZKClientService} that for all {@link ZKClient} methods would be delegated to another
+ * {@link ZKClient}, while methods for {@link ZKClientService} would be delegated to another {@link ZKClientService},
+ * which the given {@link ZKClient} and {@link ZKClientService} could be different instances.
+ *
+ * @param client The {@link ZKClient} for delegation
+ * @param clientService The {@link ZKClientService} for delegation.
+ * @return A {@link ZKClientService}.
+ */
+ public static ZKClientService delegate(final ZKClient client, ZKClientService clientService) {
+ return new ForwardingZKClientService(clientService) {
+
+ @Override
+ public Long getSessionId() {
+ return client.getSessionId();
+ }
+
+ @Override
+ public String getConnectString() {
+ return client.getConnectString();
+ }
+
+ @Override
+ public void addConnectionWatcher(Watcher watcher) {
+ client.addConnectionWatcher(watcher);
+ }
+
+ @Override
+ public OperationFuture<String> create(String path, @Nullable byte[] data, CreateMode createMode) {
+ return client.create(path, data, createMode);
+ }
+
+ @Override
+ public OperationFuture<String> create(String path, @Nullable byte[] data, CreateMode createMode,
+ boolean createParent) {
+ return client.create(path, data, createMode, createParent);
+ }
+
+ @Override
+ public OperationFuture<Stat> exists(String path) {
+ return client.exists(path);
+ }
+
+ @Override
+ public OperationFuture<Stat> exists(String path, @Nullable Watcher watcher) {
+ return client.exists(path, watcher);
+ }
+
+ @Override
+ public OperationFuture<NodeChildren> getChildren(String path) {
+ return client.getChildren(path);
+ }
+
+ @Override
+ public OperationFuture<NodeChildren> getChildren(String path, @Nullable Watcher watcher) {
+ return client.getChildren(path, watcher);
+ }
+
+ @Override
+ public OperationFuture<NodeData> getData(String path) {
+ return client.getData(path);
+ }
+
+ @Override
+ public OperationFuture<NodeData> getData(String path, @Nullable Watcher watcher) {
+ return client.getData(path, watcher);
+ }
+
+ @Override
+ public OperationFuture<Stat> setData(String path, byte[] data) {
+ return client.setData(path, data);
+ }
+
+ @Override
+ public OperationFuture<Stat> setData(String dataPath, byte[] data, int version) {
+ return client.setData(dataPath, data, version);
+ }
+
+ @Override
+ public OperationFuture<String> delete(String path) {
+ return client.delete(path);
+ }
+
+ @Override
+ public OperationFuture<String> delete(String deletePath, int version) {
+ return client.delete(deletePath, version);
+ }
+ };
+ }
+
+ private ZKClientServices() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/zookeeper/src/main/java/org/apache/twill/zookeeper/ZKClients.java
----------------------------------------------------------------------
diff --git a/zookeeper/src/main/java/org/apache/twill/zookeeper/ZKClients.java b/zookeeper/src/main/java/org/apache/twill/zookeeper/ZKClients.java
new file mode 100644
index 0000000..f67c1bd
--- /dev/null
+++ b/zookeeper/src/main/java/org/apache/twill/zookeeper/ZKClients.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.zookeeper;
+
+import org.apache.twill.internal.zookeeper.FailureRetryZKClient;
+import org.apache.twill.internal.zookeeper.NamespaceZKClient;
+import org.apache.twill.internal.zookeeper.RewatchOnExpireZKClient;
+
+/**
+ *
+ */
+public final class ZKClients {
+
+ /**
+ * Creates a {@link ZKClient} that will perform auto re-watch on all existing watches
+ * when reconnection happens after session expiration. All {@link org.apache.zookeeper.Watcher Watchers}
+ * set through the returned {@link ZKClient} would not receive any connection events.
+ *
+ * @param client The {@link ZKClient} for operations delegation.
+ * @return A {@link ZKClient} that will do auto re-watch on all methods that accept a
+ * {@link org.apache.zookeeper.Watcher} upon session expiration.
+ */
+ public static ZKClient reWatchOnExpire(ZKClient client) {
+ return new RewatchOnExpireZKClient(client);
+ }
+
+ /**
+ * Creates a {@link ZKClient} that will retry interim failure (e.g. connection loss, session expiration)
+ * based on the given {@link RetryStrategy}.
+ *
+ * @param client The {@link ZKClient} for operations delegation.
+ * @param retryStrategy The {@link RetryStrategy} to be invoke when there is operation failure.
+ * @return A {@link ZKClient}.
+ */
+ public static ZKClient retryOnFailure(ZKClient client, RetryStrategy retryStrategy) {
+ return new FailureRetryZKClient(client, retryStrategy);
+ }
+
+
+ public static ZKClient namespace(ZKClient zkClient, String namespace) {
+ return new NamespaceZKClient(zkClient, namespace);
+ }
+
+ private ZKClients() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/zookeeper/src/main/java/org/apache/twill/zookeeper/ZKOperations.java
----------------------------------------------------------------------
diff --git a/zookeeper/src/main/java/org/apache/twill/zookeeper/ZKOperations.java b/zookeeper/src/main/java/org/apache/twill/zookeeper/ZKOperations.java
new file mode 100644
index 0000000..6dcd1a7
--- /dev/null
+++ b/zookeeper/src/main/java/org/apache/twill/zookeeper/ZKOperations.java
@@ -0,0 +1,355 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.zookeeper;
+
+import org.apache.twill.common.Cancellable;
+import org.apache.twill.common.Threads;
+import org.apache.twill.internal.zookeeper.SettableOperationFuture;
+import com.google.common.collect.Lists;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.SettableFuture;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.concurrent.CancellationException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Collection of helper methods for common operations that usually needed when interacting with ZooKeeper.
+ */
+public final class ZKOperations {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ZKOperations.class);
+
+ /**
+ * Represents a ZK operation updates callback.
+ * @param <T> Type of updated data.
+ */
+ public interface Callback<T> {
+ void updated(T data);
+ }
+
+ /**
+ * Interface for defining callback method to receive node data updates.
+ */
+ public interface DataCallback extends Callback<NodeData> {
+ /**
+ * Invoked when data of the node changed.
+ * @param nodeData New data of the node, or {@code null} if the node has been deleted.
+ */
+ @Override
+ void updated(NodeData nodeData);
+ }
+
+ /**
+ * Interface for defining callback method to receive children nodes updates.
+ */
+ public interface ChildrenCallback extends Callback<NodeChildren> {
+ @Override
+ void updated(NodeChildren nodeChildren);
+ }
+
+ private interface Operation<T> {
+ ZKClient getZKClient();
+
+ OperationFuture<T> exec(String path, Watcher watcher);
+ }
+
+ /**
+ * Watch for data changes of the given path. The callback will be triggered whenever changes has been
+ * detected. Note that the callback won't see every single changes, as that's not the guarantee of ZooKeeper.
+ * If the node doesn't exists, it will watch for its creation then starts watching for data changes.
+ * When the node is deleted afterwards,
+ *
+ * @param zkClient The {@link ZKClient} for the operation
+ * @param path Path to watch
+ * @param callback Callback to be invoked when data changes is detected.
+ * @return A {@link Cancellable} to cancel the watch.
+ */
+ public static Cancellable watchData(final ZKClient zkClient, final String path, final DataCallback callback) {
+ final AtomicBoolean cancelled = new AtomicBoolean(false);
+ watchChanges(new Operation<NodeData>() {
+
+ @Override
+ public ZKClient getZKClient() {
+ return zkClient;
+ }
+
+ @Override
+ public OperationFuture<NodeData> exec(String path, Watcher watcher) {
+ return zkClient.getData(path, watcher);
+ }
+ }, path, callback, cancelled);
+
+ return new Cancellable() {
+ @Override
+ public void cancel() {
+ cancelled.set(true);
+ }
+ };
+ }
+
+ public static ListenableFuture<String> watchDeleted(final ZKClient zkClient, final String path) {
+ SettableFuture<String> completion = SettableFuture.create();
+ watchDeleted(zkClient, path, completion);
+ return completion;
+ }
+
+ public static void watchDeleted(final ZKClient zkClient, final String path,
+ final SettableFuture<String> completion) {
+
+ Futures.addCallback(zkClient.exists(path, new Watcher() {
+ @Override
+ public void process(WatchedEvent event) {
+ if (!completion.isDone()) {
+ if (event.getType() == Event.EventType.NodeDeleted) {
+ completion.set(path);
+ } else {
+ watchDeleted(zkClient, path, completion);
+ }
+ }
+ }
+ }), new FutureCallback<Stat>() {
+ @Override
+ public void onSuccess(Stat result) {
+ if (result == null) {
+ completion.set(path);
+ }
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ completion.setException(t);
+ }
+ });
+ }
+
+ public static Cancellable watchChildren(final ZKClient zkClient, String path, ChildrenCallback callback) {
+ final AtomicBoolean cancelled = new AtomicBoolean(false);
+ watchChanges(new Operation<NodeChildren>() {
+
+ @Override
+ public ZKClient getZKClient() {
+ return zkClient;
+ }
+
+ @Override
+ public OperationFuture<NodeChildren> exec(String path, Watcher watcher) {
+ return zkClient.getChildren(path, watcher);
+ }
+ }, path, callback, cancelled);
+
+ return new Cancellable() {
+ @Override
+ public void cancel() {
+ cancelled.set(true);
+ }
+ };
+ }
+
+ /**
+ * Returns a new {@link OperationFuture} that the result will be the same as the given future, except that when
+ * the source future is having an exception matching the giving exception type, the errorResult will be set
+ * in to the returned {@link OperationFuture}.
+ * @param future The source future.
+ * @param exceptionType Type of {@link KeeperException} to be ignored.
+ * @param errorResult Object to be set into the resulting future on a matching exception.
+ * @param <V> Type of the result.
+ * @return A new {@link OperationFuture}.
+ */
+ public static <V> OperationFuture<V> ignoreError(OperationFuture<V> future,
+ final Class<? extends KeeperException> exceptionType,
+ final V errorResult) {
+ final SettableOperationFuture<V> resultFuture = SettableOperationFuture.create(future.getRequestPath(),
+ Threads.SAME_THREAD_EXECUTOR);
+
+ Futures.addCallback(future, new FutureCallback<V>() {
+ @Override
+ public void onSuccess(V result) {
+ resultFuture.set(result);
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ if (exceptionType.isAssignableFrom(t.getClass())) {
+ resultFuture.set(errorResult);
+ } else if (t instanceof CancellationException) {
+ resultFuture.cancel(true);
+ } else {
+ resultFuture.setException(t);
+ }
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+
+ return resultFuture;
+ }
+
+ /**
+ * Deletes the given path recursively. The delete method will keep running until the given path is successfully
+ * removed, which means if there are new node created under the given path while deleting, they'll get deleted
+ * again. If there is {@link KeeperException} during the deletion other than
+ * {@link KeeperException.NotEmptyException} or {@link KeeperException.NoNodeException},
+ * the exception would be reflected in the result future and deletion process will stop,
+ * leaving the given path with intermediate state.
+ *
+ * @param path The path to delete.
+ * @return An {@link OperationFuture} that will be completed when the given path is deleted or bailed due to
+ * exception.
+ */
+ public static OperationFuture<String> recursiveDelete(final ZKClient zkClient, final String path) {
+ final SettableOperationFuture<String> resultFuture =
+ SettableOperationFuture.create(path, Threads.SAME_THREAD_EXECUTOR);
+
+ // Try to delete the given path.
+ Futures.addCallback(zkClient.delete(path), new FutureCallback<String>() {
+ private final FutureCallback<String> deleteCallback = this;
+
+ @Override
+ public void onSuccess(String result) {
+ // Path deleted successfully. Operation done.
+ resultFuture.set(result);
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ // Failed to delete the given path
+ if (!(t instanceof KeeperException.NotEmptyException || t instanceof KeeperException.NoNodeException)) {
+ // For errors other than NotEmptyException, treat the operation as failed.
+ resultFuture.setException(t);
+ return;
+ }
+
+ // If failed because of NotEmptyException, get the list of children under the given path
+ Futures.addCallback(zkClient.getChildren(path), new FutureCallback<NodeChildren>() {
+
+ @Override
+ public void onSuccess(NodeChildren result) {
+ // Delete all children nodes recursively.
+ final List<OperationFuture<String>> deleteFutures = Lists.newLinkedList();
+ for (String child :result.getChildren()) {
+ deleteFutures.add(recursiveDelete(zkClient, path + "/" + child));
+ }
+
+ // When deletion of all children succeeded, delete the given path again.
+ Futures.successfulAsList(deleteFutures).addListener(new Runnable() {
+ @Override
+ public void run() {
+ for (OperationFuture<String> deleteFuture : deleteFutures) {
+ try {
+ // If any exception when deleting children, treat the operation as failed.
+ deleteFuture.get();
+ } catch (Exception e) {
+ resultFuture.setException(e.getCause());
+ }
+ }
+ Futures.addCallback(zkClient.delete(path), deleteCallback, Threads.SAME_THREAD_EXECUTOR);
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ // If failed to get list of children, treat the operation as failed.
+ resultFuture.setException(t);
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+
+ return resultFuture;
+ }
+
+ /**
+ * Watch for the given path until it exists.
+ * @param zkClient The {@link ZKClient} to use.
+ * @param path A ZooKeeper path to watch for existent.
+ */
+ private static void watchExists(final ZKClient zkClient, final String path, final SettableFuture<String> completion) {
+ Futures.addCallback(zkClient.exists(path, new Watcher() {
+ @Override
+ public void process(WatchedEvent event) {
+ if (!completion.isDone()) {
+ watchExists(zkClient, path, completion);
+ }
+ }
+ }), new FutureCallback<Stat>() {
+ @Override
+ public void onSuccess(Stat result) {
+ if (result != null) {
+ completion.set(path);
+ }
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ completion.setException(t);
+ }
+ });
+ }
+
+ private static <T> void watchChanges(final Operation<T> operation, final String path,
+ final Callback<T> callback, final AtomicBoolean cancelled) {
+ Futures.addCallback(operation.exec(path, new Watcher() {
+ @Override
+ public void process(WatchedEvent event) {
+ if (!cancelled.get()) {
+ watchChanges(operation, path, callback, cancelled);
+ }
+ }
+ }), new FutureCallback<T>() {
+ @Override
+ public void onSuccess(T result) {
+ if (!cancelled.get()) {
+ callback.updated(result);
+ }
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ if (t instanceof KeeperException && ((KeeperException) t).code() == KeeperException.Code.NONODE) {
+ final SettableFuture<String> existCompletion = SettableFuture.create();
+ existCompletion.addListener(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ if (!cancelled.get()) {
+ watchChanges(operation, existCompletion.get(), callback, cancelled);
+ }
+ } catch (Exception e) {
+ LOG.error("Failed to watch children for path " + path, e);
+ }
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+ watchExists(operation.getZKClient(), path, existCompletion);
+ return;
+ }
+ LOG.error("Failed to watch data for path " + path + " " + t, t);
+ }
+ });
+ }
+
+ private ZKOperations() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/zookeeper/src/main/java/org/apache/twill/zookeeper/package-info.java
----------------------------------------------------------------------
diff --git a/zookeeper/src/main/java/org/apache/twill/zookeeper/package-info.java b/zookeeper/src/main/java/org/apache/twill/zookeeper/package-info.java
new file mode 100644
index 0000000..e5bd237
--- /dev/null
+++ b/zookeeper/src/main/java/org/apache/twill/zookeeper/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package provides functionality for ZooKeeper interactions.
+ */
+package org.apache.twill.zookeeper;
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/zookeeper/src/test/java/org/apache/twill/zookeeper/RetryStrategyTest.java
----------------------------------------------------------------------
diff --git a/zookeeper/src/test/java/org/apache/twill/zookeeper/RetryStrategyTest.java b/zookeeper/src/test/java/org/apache/twill/zookeeper/RetryStrategyTest.java
new file mode 100644
index 0000000..601f0bd
--- /dev/null
+++ b/zookeeper/src/test/java/org/apache/twill/zookeeper/RetryStrategyTest.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.zookeeper;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ *
+ */
+public class RetryStrategyTest {
+
+ @Test
+ public void testNoRetry() {
+ RetryStrategy strategy = RetryStrategies.noRetry();
+ long startTime = System.currentTimeMillis();
+ for (int i = 1; i <= 10; i++) {
+ Assert.assertEquals(-1L, strategy.nextRetry(i, startTime, RetryStrategy.OperationType.CREATE, "/"));
+ }
+ }
+
+ @Test
+ public void testLimit() {
+ RetryStrategy strategy = RetryStrategies.limit(10, RetryStrategies.fixDelay(1, TimeUnit.MILLISECONDS));
+ long startTime = System.currentTimeMillis();
+ for (int i = 1; i <= 10; i++) {
+ Assert.assertEquals(1L, strategy.nextRetry(i, startTime, RetryStrategy.OperationType.CREATE, "/"));
+ }
+ Assert.assertEquals(-1L, strategy.nextRetry(11, startTime, RetryStrategy.OperationType.CREATE, "/"));
+ }
+
+ @Test
+ public void testUnlimited() {
+ RetryStrategy strategy = RetryStrategies.fixDelay(1, TimeUnit.MILLISECONDS);
+ long startTime = System.currentTimeMillis();
+ for (int i = 1; i <= 10; i++) {
+ Assert.assertEquals(1L, strategy.nextRetry(i, startTime, RetryStrategy.OperationType.CREATE, "/"));
+ }
+ Assert.assertEquals(1L, strategy.nextRetry(100000, startTime, RetryStrategy.OperationType.CREATE, "/"));
+ }
+
+ @Test
+ public void testExponential() {
+ RetryStrategy strategy = RetryStrategies.exponentialDelay(1, 60000, TimeUnit.MILLISECONDS);
+ long startTime = System.currentTimeMillis();
+ for (int i = 1; i <= 16; i++) {
+ Assert.assertEquals(1L << (i - 1), strategy.nextRetry(i, startTime, RetryStrategy.OperationType.CREATE, "/"));
+ }
+ for (int i = 60; i <= 80; i++) {
+ Assert.assertEquals(60000, strategy.nextRetry(i, startTime, RetryStrategy.OperationType.CREATE, "/"));
+ }
+ }
+
+ @Test
+ public void testExponentialLimit() {
+ RetryStrategy strategy = RetryStrategies.limit(99,
+ RetryStrategies.exponentialDelay(1, 60000, TimeUnit.MILLISECONDS));
+ long startTime = System.currentTimeMillis();
+ for (int i = 1; i <= 16; i++) {
+ Assert.assertEquals(1L << (i - 1), strategy.nextRetry(i, startTime, RetryStrategy.OperationType.CREATE, "/"));
+ }
+ for (int i = 60; i <= 80; i++) {
+ Assert.assertEquals(60000, strategy.nextRetry(i, startTime, RetryStrategy.OperationType.CREATE, "/"));
+ }
+ Assert.assertEquals(-1L, strategy.nextRetry(100, startTime, RetryStrategy.OperationType.CREATE, "/"));
+ }
+
+ @Test
+ public void testTimeLimit() throws InterruptedException {
+ RetryStrategy strategy = RetryStrategies.timeLimit(1, TimeUnit.SECONDS,
+ RetryStrategies.fixDelay(1, TimeUnit.MILLISECONDS));
+ long startTime = System.currentTimeMillis();
+ Assert.assertEquals(1L, strategy.nextRetry(1, startTime, RetryStrategy.OperationType.CREATE, "/"));
+ TimeUnit.MILLISECONDS.sleep(1100);
+ Assert.assertEquals(-1L, strategy.nextRetry(2, startTime, RetryStrategy.OperationType.CREATE, "/"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/zookeeper/src/test/java/org/apache/twill/zookeeper/ZKClientTest.java
----------------------------------------------------------------------
diff --git a/zookeeper/src/test/java/org/apache/twill/zookeeper/ZKClientTest.java b/zookeeper/src/test/java/org/apache/twill/zookeeper/ZKClientTest.java
new file mode 100644
index 0000000..f1db74a
--- /dev/null
+++ b/zookeeper/src/test/java/org/apache/twill/zookeeper/ZKClientTest.java
@@ -0,0 +1,254 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.zookeeper;
+
+import org.apache.twill.internal.zookeeper.InMemoryZKServer;
+import org.apache.twill.internal.zookeeper.KillZKSession;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
+import com.google.common.io.Files;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ *
+ */
+public class ZKClientTest {
+
+ @Test
+ public void testChroot() throws Exception {
+ InMemoryZKServer zkServer = InMemoryZKServer.builder().setTickTime(1000).build();
+ zkServer.startAndWait();
+
+ try {
+ ZKClientService client = ZKClientService.Builder.of(zkServer.getConnectionStr() + "/chroot").build();
+ client.startAndWait();
+ try {
+ List<OperationFuture<String>> futures = Lists.newArrayList();
+ futures.add(client.create("/test1/test2", null, CreateMode.PERSISTENT));
+ futures.add(client.create("/test1/test3", null, CreateMode.PERSISTENT));
+ Futures.successfulAsList(futures).get();
+
+ Assert.assertNotNull(client.exists("/test1/test2").get());
+ Assert.assertNotNull(client.exists("/test1/test3").get());
+
+ } finally {
+ client.stopAndWait();
+ }
+ } finally {
+ zkServer.stopAndWait();
+ }
+ }
+
+ @Test
+ public void testCreateParent() throws ExecutionException, InterruptedException {
+ InMemoryZKServer zkServer = InMemoryZKServer.builder().setTickTime(1000).build();
+ zkServer.startAndWait();
+
+ try {
+ ZKClientService client = ZKClientService.Builder.of(zkServer.getConnectionStr()).build();
+ client.startAndWait();
+
+ try {
+ String path = client.create("/test1/test2/test3/test4/test5",
+ "testing".getBytes(), CreateMode.PERSISTENT_SEQUENTIAL).get();
+ Assert.assertTrue(path.startsWith("/test1/test2/test3/test4/test5"));
+
+ String dataPath = "";
+ for (int i = 1; i <= 4; i++) {
+ dataPath = dataPath + "/test" + i;
+ Assert.assertNull(client.getData(dataPath).get().getData());
+ }
+ Assert.assertTrue(Arrays.equals("testing".getBytes(), client.getData(path).get().getData()));
+ } finally {
+ client.stopAndWait();
+ }
+ } finally {
+ zkServer.stopAndWait();
+ }
+ }
+
+ @Test
+ public void testGetChildren() throws ExecutionException, InterruptedException {
+ InMemoryZKServer zkServer = InMemoryZKServer.builder().setTickTime(1000).build();
+ zkServer.startAndWait();
+
+ try {
+ ZKClientService client = ZKClientService.Builder.of(zkServer.getConnectionStr()).build();
+ client.startAndWait();
+
+ try {
+ client.create("/test", null, CreateMode.PERSISTENT).get();
+ Assert.assertTrue(client.getChildren("/test").get().getChildren().isEmpty());
+
+ Futures.allAsList(ImmutableList.of(client.create("/test/c1", null, CreateMode.EPHEMERAL),
+ client.create("/test/c2", null, CreateMode.EPHEMERAL))).get();
+
+ NodeChildren nodeChildren = client.getChildren("/test").get();
+ Assert.assertEquals(2, nodeChildren.getChildren().size());
+
+ Assert.assertEquals(ImmutableSet.of("c1", "c2"), ImmutableSet.copyOf(nodeChildren.getChildren()));
+
+ } finally {
+ client.stopAndWait();
+ }
+ } finally {
+ zkServer.stopAndWait();
+ }
+ }
+
+ @Test
+ public void testSetData() throws ExecutionException, InterruptedException {
+ InMemoryZKServer zkServer = InMemoryZKServer.builder().setTickTime(1000).build();
+ zkServer.startAndWait();
+
+ try {
+ ZKClientService client = ZKClientService.Builder.of(zkServer.getConnectionStr()).build();
+ client.startAndWait();
+
+ client.create("/test", null, CreateMode.PERSISTENT).get();
+ Assert.assertNull(client.getData("/test").get().getData());
+
+ client.setData("/test", "testing".getBytes()).get();
+ Assert.assertTrue(Arrays.equals("testing".getBytes(), client.getData("/test").get().getData()));
+
+ } finally {
+ zkServer.stopAndWait();
+ }
+ }
+
+ @Test
+ public void testExpireRewatch() throws InterruptedException, IOException, ExecutionException {
+ InMemoryZKServer zkServer = InMemoryZKServer.builder().setTickTime(1000).build();
+ zkServer.startAndWait();
+
+ try {
+ final CountDownLatch expireReconnectLatch = new CountDownLatch(1);
+ final AtomicBoolean expired = new AtomicBoolean(false);
+ final ZKClientService client = ZKClientServices.delegate(ZKClients.reWatchOnExpire(
+ ZKClientService.Builder.of(zkServer.getConnectionStr())
+ .setSessionTimeout(2000)
+ .setConnectionWatcher(new Watcher() {
+ @Override
+ public void process(WatchedEvent event) {
+ if (event.getState() == Event.KeeperState.Expired) {
+ expired.set(true);
+ } else if (event.getState() == Event.KeeperState.SyncConnected && expired.compareAndSet(true, true)) {
+ expireReconnectLatch.countDown();
+ }
+ }
+ }).build()));
+ client.startAndWait();
+
+ try {
+ final BlockingQueue<Watcher.Event.EventType> events = new LinkedBlockingQueue<Watcher.Event.EventType>();
+ client.exists("/expireRewatch", new Watcher() {
+ @Override
+ public void process(WatchedEvent event) {
+ client.exists("/expireRewatch", this);
+ events.add(event.getType());
+ }
+ });
+
+ client.create("/expireRewatch", null, CreateMode.PERSISTENT);
+ Assert.assertEquals(Watcher.Event.EventType.NodeCreated, events.poll(2, TimeUnit.SECONDS));
+
+ KillZKSession.kill(client.getZooKeeperSupplier().get(), zkServer.getConnectionStr(), 1000);
+
+ Assert.assertTrue(expireReconnectLatch.await(5, TimeUnit.SECONDS));
+
+ client.delete("/expireRewatch");
+ Assert.assertEquals(Watcher.Event.EventType.NodeDeleted, events.poll(4, TimeUnit.SECONDS));
+ } finally {
+ client.stopAndWait();
+ }
+ } finally {
+ zkServer.stopAndWait();
+ }
+ }
+
+ @Test
+ public void testRetry() throws ExecutionException, InterruptedException, TimeoutException {
+ File dataDir = Files.createTempDir();
+ InMemoryZKServer zkServer = InMemoryZKServer.builder().setDataDir(dataDir).setTickTime(1000).build();
+ zkServer.startAndWait();
+ int port = zkServer.getLocalAddress().getPort();
+
+ final CountDownLatch disconnectLatch = new CountDownLatch(1);
+ ZKClientService client = ZKClientServices.delegate(ZKClients.retryOnFailure(
+ ZKClientService.Builder.of(zkServer.getConnectionStr()).setConnectionWatcher(new Watcher() {
+ @Override
+ public void process(WatchedEvent event) {
+ if (event.getState() == Event.KeeperState.Disconnected) {
+ disconnectLatch.countDown();
+ }
+ }
+ }).build(), RetryStrategies.fixDelay(0, TimeUnit.SECONDS)));
+ client.startAndWait();
+
+ zkServer.stopAndWait();
+
+ Assert.assertTrue(disconnectLatch.await(1, TimeUnit.SECONDS));
+
+ final CountDownLatch createLatch = new CountDownLatch(1);
+ Futures.addCallback(client.create("/testretry/test", null, CreateMode.PERSISTENT), new FutureCallback<String>() {
+ @Override
+ public void onSuccess(String result) {
+ createLatch.countDown();
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ t.printStackTrace(System.out);
+ }
+ });
+
+ TimeUnit.SECONDS.sleep(2);
+ zkServer = InMemoryZKServer.builder()
+ .setDataDir(dataDir)
+ .setAutoCleanDataDir(true)
+ .setPort(port)
+ .setTickTime(1000)
+ .build();
+ zkServer.startAndWait();
+
+ try {
+ Assert.assertTrue(createLatch.await(5, TimeUnit.SECONDS));
+ } finally {
+ zkServer.stopAndWait();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/zookeeper/src/test/java/org/apache/twill/zookeeper/ZKOperationsTest.java
----------------------------------------------------------------------
diff --git a/zookeeper/src/test/java/org/apache/twill/zookeeper/ZKOperationsTest.java b/zookeeper/src/test/java/org/apache/twill/zookeeper/ZKOperationsTest.java
new file mode 100644
index 0000000..9518d6e
--- /dev/null
+++ b/zookeeper/src/test/java/org/apache/twill/zookeeper/ZKOperationsTest.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.zookeeper;
+
+import org.apache.twill.internal.zookeeper.InMemoryZKServer;
+import org.apache.zookeeper.CreateMode;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+/**
+ *
+ */
+public class ZKOperationsTest {
+
+ @Test
+ public void recursiveDelete() throws ExecutionException, InterruptedException, TimeoutException {
+ InMemoryZKServer zkServer = InMemoryZKServer.builder().setTickTime(1000).build();
+ zkServer.startAndWait();
+
+ try {
+ ZKClientService client = ZKClientService.Builder.of(zkServer.getConnectionStr()).build();
+ client.startAndWait();
+
+ try {
+ client.create("/test1/test10/test101", null, CreateMode.PERSISTENT).get();
+ client.create("/test1/test10/test102", null, CreateMode.PERSISTENT).get();
+ client.create("/test1/test10/test103", null, CreateMode.PERSISTENT).get();
+
+ client.create("/test1/test11/test111", null, CreateMode.PERSISTENT).get();
+ client.create("/test1/test11/test112", null, CreateMode.PERSISTENT).get();
+ client.create("/test1/test11/test113", null, CreateMode.PERSISTENT).get();
+
+ ZKOperations.recursiveDelete(client, "/test1").get(2, TimeUnit.SECONDS);
+
+ Assert.assertNull(client.exists("/test1").get(2, TimeUnit.SECONDS));
+
+ } finally {
+ client.stopAndWait();
+ }
+ } finally {
+ zkServer.stopAndWait();
+ }
+ }
+}
[06/15] Initial import commit.
Posted by ch...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterProcessLauncher.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterProcessLauncher.java b/yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterProcessLauncher.java
new file mode 100644
index 0000000..b51bb63
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterProcessLauncher.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.appmaster;
+
+import org.apache.twill.internal.Constants;
+import org.apache.twill.internal.EnvKeys;
+import org.apache.twill.internal.ProcessController;
+import org.apache.twill.internal.yarn.AbstractYarnProcessLauncher;
+import org.apache.twill.internal.yarn.YarnLaunchContext;
+import org.apache.twill.internal.yarn.YarnUtils;
+import com.google.common.collect.ImmutableMap;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.Records;
+
+import java.util.Map;
+
+/**
+ * A {@link org.apache.twill.internal.ProcessLauncher} for launching Application Master from the client.
+ */
+public final class ApplicationMasterProcessLauncher extends AbstractYarnProcessLauncher<ApplicationId> {
+
+ private final ApplicationSubmitter submitter;
+
+ public ApplicationMasterProcessLauncher(ApplicationId appId, ApplicationSubmitter submitter) {
+ super(appId);
+ this.submitter = submitter;
+ }
+
+ @Override
+ protected boolean useArchiveSuffix() {
+ return false;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ protected <R> ProcessController<R> doLaunch(YarnLaunchContext launchContext) {
+ final ApplicationId appId = getContainerInfo();
+
+ // Set the resource requirement for AM
+ Resource capability = Records.newRecord(Resource.class);
+ capability.setMemory(Constants.APP_MASTER_MEMORY_MB);
+ YarnUtils.setVirtualCores(capability, 1);
+
+ // Put in extra environments
+ Map<String, String> env = ImmutableMap.<String, String>builder()
+ .putAll(launchContext.getEnvironment())
+ .put(EnvKeys.YARN_APP_ID, Integer.toString(appId.getId()))
+ .put(EnvKeys.YARN_APP_ID_CLUSTER_TIME, Long.toString(appId.getClusterTimestamp()))
+ .put(EnvKeys.YARN_APP_ID_STR, appId.toString())
+ .put(EnvKeys.YARN_CONTAINER_MEMORY_MB, Integer.toString(Constants.APP_MASTER_MEMORY_MB))
+ .put(EnvKeys.YARN_CONTAINER_VIRTUAL_CORES, Integer.toString(YarnUtils.getVirtualCores(capability)))
+ .build();
+
+ launchContext.setEnvironment(env);
+ return (ProcessController<R>) submitter.submit(launchContext, capability);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java b/yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
new file mode 100644
index 0000000..51c8503
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterService.java
@@ -0,0 +1,799 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.appmaster;
+
+import org.apache.twill.api.Command;
+import org.apache.twill.api.EventHandler;
+import org.apache.twill.api.EventHandlerSpecification;
+import org.apache.twill.api.LocalFile;
+import org.apache.twill.api.ResourceSpecification;
+import org.apache.twill.api.RunId;
+import org.apache.twill.api.RuntimeSpecification;
+import org.apache.twill.api.TwillRunResources;
+import org.apache.twill.api.TwillSpecification;
+import org.apache.twill.common.Threads;
+import org.apache.twill.filesystem.Location;
+import org.apache.twill.internal.AbstractTwillService;
+import org.apache.twill.internal.Configs;
+import org.apache.twill.internal.Constants;
+import org.apache.twill.internal.DefaultTwillRunResources;
+import org.apache.twill.internal.EnvKeys;
+import org.apache.twill.internal.ProcessLauncher;
+import org.apache.twill.internal.TwillContainerLauncher;
+import org.apache.twill.internal.ZKServiceDecorator;
+import org.apache.twill.internal.json.LocalFileCodec;
+import org.apache.twill.internal.json.TwillSpecificationAdapter;
+import org.apache.twill.internal.kafka.EmbeddedKafkaServer;
+import org.apache.twill.internal.logging.Loggings;
+import org.apache.twill.internal.state.Message;
+import org.apache.twill.internal.state.MessageCallback;
+import org.apache.twill.internal.utils.Instances;
+import org.apache.twill.internal.utils.Networks;
+import org.apache.twill.internal.yarn.YarnAMClient;
+import org.apache.twill.internal.yarn.YarnAMClientFactory;
+import org.apache.twill.internal.yarn.YarnContainerInfo;
+import org.apache.twill.internal.yarn.YarnContainerStatus;
+import org.apache.twill.internal.yarn.YarnUtils;
+import org.apache.twill.zookeeper.ZKClient;
+import org.apache.twill.zookeeper.ZKClients;
+import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Supplier;
+import com.google.common.base.Throwables;
+import com.google.common.collect.HashMultiset;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableMultimap;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multiset;
+import com.google.common.collect.Sets;
+import com.google.common.io.CharStreams;
+import com.google.common.io.Files;
+import com.google.common.io.InputSupplier;
+import com.google.common.reflect.TypeToken;
+import com.google.common.util.concurrent.AbstractExecutionThreadService;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.Service;
+import com.google.common.util.concurrent.SettableFuture;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonElement;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.zookeeper.CreateMode;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.Reader;
+import java.net.URI;
+import java.net.URL;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+/**
+ *
+ */
+public final class ApplicationMasterService extends AbstractTwillService {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ApplicationMasterService.class);
+
+ // Copied from org.apache.hadoop.yarn.security.AMRMTokenIdentifier.KIND_NAME since it's missing in Hadoop-2.0
+ private static final Text AMRM_TOKEN_KIND_NAME = new Text("YARN_AM_RM_TOKEN");
+
+ private final RunId runId;
+ private final ZKClient zkClient;
+ private final TwillSpecification twillSpec;
+ private final ApplicationMasterLiveNodeData amLiveNode;
+ private final ZKServiceDecorator serviceDelegate;
+ private final RunningContainers runningContainers;
+ private final ExpectedContainers expectedContainers;
+ private final TrackerService trackerService;
+ private final YarnAMClient amClient;
+ private final String jvmOpts;
+ private final int reservedMemory;
+ private final EventHandler eventHandler;
+ private final Location applicationLocation;
+
+ private EmbeddedKafkaServer kafkaServer;
+ private Queue<RunnableContainerRequest> runnableContainerRequests;
+ private ExecutorService instanceChangeExecutor;
+
+ public ApplicationMasterService(RunId runId, ZKClient zkClient, File twillSpecFile,
+ YarnAMClientFactory amClientFactory, Location applicationLocation) throws Exception {
+ super(applicationLocation);
+
+ this.runId = runId;
+ this.twillSpec = TwillSpecificationAdapter.create().fromJson(twillSpecFile);
+ this.zkClient = zkClient;
+ this.applicationLocation = applicationLocation;
+ this.amClient = amClientFactory.create();
+ this.credentials = createCredentials();
+ this.jvmOpts = loadJvmOptions();
+ this.reservedMemory = getReservedMemory();
+
+ amLiveNode = new ApplicationMasterLiveNodeData(Integer.parseInt(System.getenv(EnvKeys.YARN_APP_ID)),
+ Long.parseLong(System.getenv(EnvKeys.YARN_APP_ID_CLUSTER_TIME)),
+ amClient.getContainerId().toString());
+
+ serviceDelegate = new ZKServiceDecorator(zkClient, runId, createLiveNodeDataSupplier(),
+ new ServiceDelegate(), new Runnable() {
+ @Override
+ public void run() {
+ amClient.stopAndWait();
+ }
+ });
+ expectedContainers = initExpectedContainers(twillSpec);
+ runningContainers = initRunningContainers(amClient.getContainerId(), amClient.getHost());
+ trackerService = new TrackerService(runningContainers.getResourceReport(), amClient.getHost());
+ eventHandler = createEventHandler(twillSpec);
+ }
+
+ private String loadJvmOptions() throws IOException {
+ final File jvmOptsFile = new File(Constants.Files.JVM_OPTIONS);
+ if (!jvmOptsFile.exists()) {
+ return "";
+ }
+
+ return CharStreams.toString(new InputSupplier<Reader>() {
+ @Override
+ public Reader getInput() throws IOException {
+ return new FileReader(jvmOptsFile);
+ }
+ });
+ }
+
+ private int getReservedMemory() {
+ String value = System.getenv(EnvKeys.TWILL_RESERVED_MEMORY_MB);
+ if (value == null) {
+ return Configs.Defaults.JAVA_RESERVED_MEMORY_MB;
+ }
+ try {
+ return Integer.parseInt(value);
+ } catch (Exception e) {
+ return Configs.Defaults.JAVA_RESERVED_MEMORY_MB;
+ }
+ }
+
+ private EventHandler createEventHandler(TwillSpecification twillSpec) {
+ try {
+ // Should be able to load by this class ClassLoader, as they packaged in the same jar.
+ EventHandlerSpecification handlerSpec = twillSpec.getEventHandler();
+
+ Class<?> handlerClass = getClass().getClassLoader().loadClass(handlerSpec.getClassName());
+ Preconditions.checkArgument(EventHandler.class.isAssignableFrom(handlerClass),
+ "Class {} does not implements {}",
+ handlerClass, EventHandler.class.getName());
+ return Instances.newInstance((Class<? extends EventHandler>) handlerClass);
+ } catch (Exception e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ private Supplier<? extends JsonElement> createLiveNodeDataSupplier() {
+ return new Supplier<JsonElement>() {
+ @Override
+ public JsonElement get() {
+ return new Gson().toJsonTree(amLiveNode);
+ }
+ };
+ }
+
+ private RunningContainers initRunningContainers(ContainerId appMasterContainerId,
+ String appMasterHost) throws Exception {
+ TwillRunResources appMasterResources = new DefaultTwillRunResources(
+ 0,
+ appMasterContainerId.toString(),
+ Integer.parseInt(System.getenv(EnvKeys.YARN_CONTAINER_VIRTUAL_CORES)),
+ Integer.parseInt(System.getenv(EnvKeys.YARN_CONTAINER_MEMORY_MB)),
+ appMasterHost);
+ String appId = appMasterContainerId.getApplicationAttemptId().getApplicationId().toString();
+ return new RunningContainers(appId, appMasterResources);
+ }
+
+ private ExpectedContainers initExpectedContainers(TwillSpecification twillSpec) {
+ Map<String, Integer> expectedCounts = Maps.newHashMap();
+ for (RuntimeSpecification runtimeSpec : twillSpec.getRunnables().values()) {
+ expectedCounts.put(runtimeSpec.getName(), runtimeSpec.getResourceSpecification().getInstances());
+ }
+ return new ExpectedContainers(expectedCounts);
+ }
+
+ private void doStart() throws Exception {
+ LOG.info("Start application master with spec: " + TwillSpecificationAdapter.create().toJson(twillSpec));
+
+ // initialize the event handler, if it fails, it will fail the application.
+ eventHandler.initialize(new BasicEventHandlerContext(twillSpec.getEventHandler()));
+
+ instanceChangeExecutor = Executors.newSingleThreadExecutor(Threads.createDaemonThreadFactory("instanceChanger"));
+
+ kafkaServer = new EmbeddedKafkaServer(new File(Constants.Files.KAFKA), generateKafkaConfig());
+
+ // Must start tracker before start AMClient
+ LOG.info("Starting application master tracker server");
+ trackerService.startAndWait();
+ URL trackerUrl = trackerService.getUrl();
+ LOG.info("Started application master tracker server on " + trackerUrl);
+
+ amClient.setTracker(trackerService.getBindAddress(), trackerUrl);
+ amClient.startAndWait();
+
+ // Creates ZK path for runnable and kafka logging service
+ Futures.allAsList(ImmutableList.of(
+ zkClient.create("/" + runId.getId() + "/runnables", null, CreateMode.PERSISTENT),
+ zkClient.create("/" + runId.getId() + "/kafka", null, CreateMode.PERSISTENT))
+ ).get();
+
+ // Starts kafka server
+ LOG.info("Starting kafka server");
+
+ kafkaServer.startAndWait();
+ LOG.info("Kafka server started");
+
+ runnableContainerRequests = initContainerRequests();
+ }
+
+ private void doStop() throws Exception {
+ Thread.interrupted(); // This is just to clear the interrupt flag
+
+ LOG.info("Stop application master with spec: {}", TwillSpecificationAdapter.create().toJson(twillSpec));
+
+ try {
+ // call event handler destroy. If there is error, only log and not affected stop sequence.
+ eventHandler.destroy();
+ } catch (Throwable t) {
+ LOG.warn("Exception when calling {}.destroy()", twillSpec.getEventHandler().getClassName(), t);
+ }
+
+ instanceChangeExecutor.shutdownNow();
+
+ // For checking if all containers are stopped.
+ final Set<String> ids = Sets.newHashSet(runningContainers.getContainerIds());
+ YarnAMClient.AllocateHandler handler = new YarnAMClient.AllocateHandler() {
+ @Override
+ public void acquired(List<ProcessLauncher<YarnContainerInfo>> launchers) {
+ // no-op
+ }
+
+ @Override
+ public void completed(List<YarnContainerStatus> completed) {
+ for (YarnContainerStatus status : completed) {
+ ids.remove(status.getContainerId());
+ }
+ }
+ };
+
+ runningContainers.stopAll();
+
+ // Poll for 5 seconds to wait for containers to stop.
+ int count = 0;
+ while (!ids.isEmpty() && count++ < 5) {
+ amClient.allocate(0.0f, handler);
+ TimeUnit.SECONDS.sleep(1);
+ }
+
+ LOG.info("Stopping application master tracker server");
+ try {
+ trackerService.stopAndWait();
+ LOG.info("Stopped application master tracker server");
+ } catch (Exception e) {
+ LOG.error("Failed to stop tracker service.", e);
+ } finally {
+ try {
+ // App location cleanup
+ cleanupDir(URI.create(System.getenv(EnvKeys.TWILL_APP_DIR)));
+ Loggings.forceFlush();
+ // Sleep a short while to let kafka clients to have chance to fetch the log
+ TimeUnit.SECONDS.sleep(1);
+ } finally {
+ kafkaServer.stopAndWait();
+ LOG.info("Kafka server stopped");
+ }
+ }
+ }
+
+ private void cleanupDir(URI appDir) {
+ try {
+ if (applicationLocation.delete(true)) {
+ LOG.info("Application directory deleted: {}", appDir);
+ } else {
+ LOG.warn("Failed to cleanup directory {}.", appDir);
+ }
+ } catch (Exception e) {
+ LOG.warn("Exception while cleanup directory {}.", appDir, e);
+ }
+ }
+
+
+ private void doRun() throws Exception {
+ // The main loop
+ Map.Entry<Resource, ? extends Collection<RuntimeSpecification>> currentRequest = null;
+ final Queue<ProvisionRequest> provisioning = Lists.newLinkedList();
+
+ YarnAMClient.AllocateHandler allocateHandler = new YarnAMClient.AllocateHandler() {
+ @Override
+ public void acquired(List<ProcessLauncher<YarnContainerInfo>> launchers) {
+ launchRunnable(launchers, provisioning);
+ }
+
+ @Override
+ public void completed(List<YarnContainerStatus> completed) {
+ handleCompleted(completed);
+ }
+ };
+
+ long nextTimeoutCheck = System.currentTimeMillis() + Constants.PROVISION_TIMEOUT;
+ while (isRunning()) {
+ // Call allocate. It has to be made at first in order to be able to get cluster resource availability.
+ amClient.allocate(0.0f, allocateHandler);
+
+ // Looks for containers requests.
+ if (provisioning.isEmpty() && runnableContainerRequests.isEmpty() && runningContainers.isEmpty()) {
+ LOG.info("All containers completed. Shutting down application master.");
+ break;
+ }
+
+ // If nothing is in provisioning, and no pending request, move to next one
+ while (provisioning.isEmpty() && currentRequest == null && !runnableContainerRequests.isEmpty()) {
+ currentRequest = runnableContainerRequests.peek().takeRequest();
+ if (currentRequest == null) {
+ // All different types of resource request from current order is done, move to next one
+ // TODO: Need to handle order type as well
+ runnableContainerRequests.poll();
+ }
+ }
+ // Nothing in provision, makes the next batch of provision request
+ if (provisioning.isEmpty() && currentRequest != null) {
+ addContainerRequests(currentRequest.getKey(), currentRequest.getValue(), provisioning);
+ currentRequest = null;
+ }
+
+ nextTimeoutCheck = checkProvisionTimeout(nextTimeoutCheck);
+
+ if (isRunning()) {
+ TimeUnit.SECONDS.sleep(1);
+ }
+ }
+ }
+
+ /**
+ * Handling containers that are completed.
+ */
+ private void handleCompleted(List<YarnContainerStatus> completedContainersStatuses) {
+ Multiset<String> restartRunnables = HashMultiset.create();
+ for (YarnContainerStatus status : completedContainersStatuses) {
+ LOG.info("Container {} completed with {}:{}.",
+ status.getContainerId(), status.getState(), status.getDiagnostics());
+ runningContainers.handleCompleted(status, restartRunnables);
+ }
+
+ for (Multiset.Entry<String> entry : restartRunnables.entrySet()) {
+ LOG.info("Re-request container for {} with {} instances.", entry.getElement(), entry.getCount());
+ for (int i = 0; i < entry.getCount(); i++) {
+ runnableContainerRequests.add(createRunnableContainerRequest(entry.getElement()));
+ }
+ }
+
+ // For all runnables that needs to re-request for containers, update the expected count timestamp
+ // so that the EventHandler would triggered with the right expiration timestamp.
+ expectedContainers.updateRequestTime(restartRunnables.elementSet());
+ }
+
+ /**
+ * Check for containers provision timeout and invoke eventHandler if necessary.
+ *
+ * @return the timestamp for the next time this method needs to be called.
+ */
+ private long checkProvisionTimeout(long nextTimeoutCheck) {
+ if (System.currentTimeMillis() < nextTimeoutCheck) {
+ return nextTimeoutCheck;
+ }
+
+ // Invoke event handler for provision request timeout
+ Map<String, ExpectedContainers.ExpectedCount> expiredRequests = expectedContainers.getAll();
+ Map<String, Integer> runningCounts = runningContainers.countAll();
+
+ List<EventHandler.TimeoutEvent> timeoutEvents = Lists.newArrayList();
+ for (Map.Entry<String, ExpectedContainers.ExpectedCount> entry : expiredRequests.entrySet()) {
+ String runnableName = entry.getKey();
+ ExpectedContainers.ExpectedCount expectedCount = entry.getValue();
+ int runningCount = runningCounts.containsKey(runnableName) ? runningCounts.get(runnableName) : 0;
+ if (expectedCount.getCount() != runningCount) {
+ timeoutEvents.add(new EventHandler.TimeoutEvent(runnableName, expectedCount.getCount(),
+ runningCount, expectedCount.getTimestamp()));
+ }
+ }
+
+ if (!timeoutEvents.isEmpty()) {
+ try {
+ EventHandler.TimeoutAction action = eventHandler.launchTimeout(timeoutEvents);
+ if (action.getTimeout() < 0) {
+ // Abort application
+ stop();
+ } else {
+ return nextTimeoutCheck + action.getTimeout();
+ }
+ } catch (Throwable t) {
+ LOG.warn("Exception when calling EventHandler {}. Ignore the result.", t);
+ }
+ }
+ return nextTimeoutCheck + Constants.PROVISION_TIMEOUT;
+ }
+
+ private Credentials createCredentials() {
+ Credentials credentials = new Credentials();
+ if (!UserGroupInformation.isSecurityEnabled()) {
+ return credentials;
+ }
+
+ try {
+ credentials.addAll(UserGroupInformation.getCurrentUser().getCredentials());
+
+ // Remove the AM->RM tokens
+ Iterator<Token<?>> iter = credentials.getAllTokens().iterator();
+ while (iter.hasNext()) {
+ Token<?> token = iter.next();
+ if (token.getKind().equals(AMRM_TOKEN_KIND_NAME)) {
+ iter.remove();
+ }
+ }
+ } catch (IOException e) {
+ LOG.warn("Failed to get current user. No credentials will be provided to containers.", e);
+ }
+
+ return credentials;
+ }
+
+ private Queue<RunnableContainerRequest> initContainerRequests() {
+ // Orderly stores container requests.
+ Queue<RunnableContainerRequest> requests = Lists.newLinkedList();
+ // For each order in the twillSpec, create container request for each runnable.
+ for (TwillSpecification.Order order : twillSpec.getOrders()) {
+ // Group container requests based on resource requirement.
+ ImmutableMultimap.Builder<Resource, RuntimeSpecification> builder = ImmutableMultimap.builder();
+ for (String runnableName : order.getNames()) {
+ RuntimeSpecification runtimeSpec = twillSpec.getRunnables().get(runnableName);
+ Resource capability = createCapability(runtimeSpec.getResourceSpecification());
+ builder.put(capability, runtimeSpec);
+ }
+ requests.add(new RunnableContainerRequest(order.getType(), builder.build()));
+ }
+ return requests;
+ }
+
+ /**
+ * Adds container requests with the given resource capability for each runtime.
+ */
+ private void addContainerRequests(Resource capability,
+ Collection<RuntimeSpecification> runtimeSpecs,
+ Queue<ProvisionRequest> provisioning) {
+ for (RuntimeSpecification runtimeSpec : runtimeSpecs) {
+ String name = runtimeSpec.getName();
+ int newContainers = expectedContainers.getExpected(name) - runningContainers.count(name);
+ if (newContainers > 0) {
+ // TODO: Allow user to set priority?
+ LOG.info("Request {} container with capability {}", newContainers, capability);
+ String requestId = amClient.addContainerRequest(capability, newContainers).setPriority(0).apply();
+ provisioning.add(new ProvisionRequest(runtimeSpec, requestId, newContainers));
+ }
+ }
+ }
+
+ /**
+ * Launches runnables in the provisioned containers.
+ */
+ private void launchRunnable(List<ProcessLauncher<YarnContainerInfo>> launchers,
+ Queue<ProvisionRequest> provisioning) {
+ for (ProcessLauncher<YarnContainerInfo> processLauncher : launchers) {
+ LOG.info("Got container {}", processLauncher.getContainerInfo().getId());
+ ProvisionRequest provisionRequest = provisioning.peek();
+ if (provisionRequest == null) {
+ continue;
+ }
+
+ String runnableName = provisionRequest.getRuntimeSpec().getName();
+ LOG.info("Starting runnable {} with {}", runnableName, processLauncher);
+
+ int containerCount = expectedContainers.getExpected(runnableName);
+
+ ProcessLauncher.PrepareLaunchContext launchContext = processLauncher.prepareLaunch(
+ ImmutableMap.<String, String>builder()
+ .put(EnvKeys.TWILL_APP_DIR, System.getenv(EnvKeys.TWILL_APP_DIR))
+ .put(EnvKeys.TWILL_FS_USER, System.getenv(EnvKeys.TWILL_FS_USER))
+ .put(EnvKeys.TWILL_APP_RUN_ID, runId.getId())
+ .put(EnvKeys.TWILL_APP_NAME, twillSpec.getName())
+ .put(EnvKeys.TWILL_ZK_CONNECT, zkClient.getConnectString())
+ .put(EnvKeys.TWILL_LOG_KAFKA_ZK, getKafkaZKConnect())
+ .build()
+ , getLocalizeFiles(), credentials
+ );
+
+ TwillContainerLauncher launcher = new TwillContainerLauncher(
+ twillSpec.getRunnables().get(runnableName), launchContext,
+ ZKClients.namespace(zkClient, getZKNamespace(runnableName)),
+ containerCount, jvmOpts, reservedMemory, getSecureStoreLocation());
+
+ runningContainers.start(runnableName, processLauncher.getContainerInfo(), launcher);
+
+ // Need to call complete to workaround bug in YARN AMRMClient
+ if (provisionRequest.containerAcquired()) {
+ amClient.completeContainerRequest(provisionRequest.getRequestId());
+ }
+
+ if (expectedContainers.getExpected(runnableName) == runningContainers.count(runnableName)) {
+ LOG.info("Runnable " + runnableName + " fully provisioned with " + containerCount + " instances.");
+ provisioning.poll();
+ }
+ }
+ }
+
+ private List<LocalFile> getLocalizeFiles() {
+ try {
+ Reader reader = Files.newReader(new File(Constants.Files.LOCALIZE_FILES), Charsets.UTF_8);
+ try {
+ return new GsonBuilder().registerTypeAdapter(LocalFile.class, new LocalFileCodec())
+ .create().fromJson(reader, new TypeToken<List<LocalFile>>() {}.getType());
+ } finally {
+ reader.close();
+ }
+ } catch (IOException e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ private String getZKNamespace(String runnableName) {
+ return String.format("/%s/runnables/%s", runId.getId(), runnableName);
+ }
+
+ private String getKafkaZKConnect() {
+ return String.format("%s/%s/kafka", zkClient.getConnectString(), runId.getId());
+ }
+
+ private Properties generateKafkaConfig() {
+ int port = Networks.getRandomPort();
+ Preconditions.checkState(port > 0, "Failed to get random port.");
+
+ Properties prop = new Properties();
+ prop.setProperty("log.dir", new File("kafka-logs").getAbsolutePath());
+ prop.setProperty("zk.connect", getKafkaZKConnect());
+ prop.setProperty("num.threads", "8");
+ prop.setProperty("port", Integer.toString(port));
+ prop.setProperty("log.flush.interval", "10000");
+ prop.setProperty("max.socket.request.bytes", "104857600");
+ prop.setProperty("log.cleanup.interval.mins", "1");
+ prop.setProperty("log.default.flush.scheduler.interval.ms", "1000");
+ prop.setProperty("zk.connectiontimeout.ms", "1000000");
+ prop.setProperty("socket.receive.buffer", "1048576");
+ prop.setProperty("enable.zookeeper", "true");
+ prop.setProperty("log.retention.hours", "24");
+ prop.setProperty("brokerid", "0");
+ prop.setProperty("socket.send.buffer", "1048576");
+ prop.setProperty("num.partitions", "1");
+ prop.setProperty("log.file.size", "536870912");
+ prop.setProperty("log.default.flush.interval.ms", "1000");
+ return prop;
+ }
+
+ private ListenableFuture<String> processMessage(final String messageId, Message message) {
+ LOG.debug("Message received: {} {}.", messageId, message);
+
+ SettableFuture<String> result = SettableFuture.create();
+ Runnable completion = getMessageCompletion(messageId, result);
+
+ if (handleSecureStoreUpdate(message)) {
+ runningContainers.sendToAll(message, completion);
+ return result;
+ }
+
+ if (handleSetInstances(message, completion)) {
+ return result;
+ }
+
+ // Replicate messages to all runnables
+ if (message.getScope() == Message.Scope.ALL_RUNNABLE) {
+ runningContainers.sendToAll(message, completion);
+ return result;
+ }
+
+ // Replicate message to a particular runnable.
+ if (message.getScope() == Message.Scope.RUNNABLE) {
+ runningContainers.sendToRunnable(message.getRunnableName(), message, completion);
+ return result;
+ }
+
+ LOG.info("Message ignored. {}", message);
+ return Futures.immediateFuture(messageId);
+ }
+
+ /**
+ * Attempts to change the number of running instances.
+ * @return {@code true} if the message does requests for changes in number of running instances of a runnable,
+ * {@code false} otherwise.
+ */
+ private boolean handleSetInstances(final Message message, final Runnable completion) {
+ if (message.getType() != Message.Type.SYSTEM || message.getScope() != Message.Scope.RUNNABLE) {
+ return false;
+ }
+
+ Command command = message.getCommand();
+ Map<String, String> options = command.getOptions();
+ if (!"instances".equals(command.getCommand()) || !options.containsKey("count")) {
+ return false;
+ }
+
+ final String runnableName = message.getRunnableName();
+ if (runnableName == null || runnableName.isEmpty() || !twillSpec.getRunnables().containsKey(runnableName)) {
+ LOG.info("Unknown runnable {}", runnableName);
+ return false;
+ }
+
+ final int newCount = Integer.parseInt(options.get("count"));
+ final int oldCount = expectedContainers.getExpected(runnableName);
+
+ LOG.info("Received change instances request for {}, from {} to {}.", runnableName, oldCount, newCount);
+
+ if (newCount == oldCount) { // Nothing to do, simply complete the request.
+ completion.run();
+ return true;
+ }
+
+ instanceChangeExecutor.execute(createSetInstanceRunnable(message, completion, oldCount, newCount));
+ return true;
+ }
+
+ /**
+ * Creates a Runnable for execution of change instance request.
+ */
+ private Runnable createSetInstanceRunnable(final Message message, final Runnable completion,
+ final int oldCount, final int newCount) {
+ return new Runnable() {
+ @Override
+ public void run() {
+ final String runnableName = message.getRunnableName();
+
+ LOG.info("Processing change instance request for {}, from {} to {}.", runnableName, oldCount, newCount);
+ try {
+ // Wait until running container count is the same as old count
+ runningContainers.waitForCount(runnableName, oldCount);
+ LOG.info("Confirmed {} containers running for {}.", oldCount, runnableName);
+
+ expectedContainers.setExpected(runnableName, newCount);
+
+ try {
+ if (newCount < oldCount) {
+ // Shutdown some running containers
+ for (int i = 0; i < oldCount - newCount; i++) {
+ runningContainers.removeLast(runnableName);
+ }
+ } else {
+ // Increase the number of instances
+ runnableContainerRequests.add(createRunnableContainerRequest(runnableName));
+ }
+ } finally {
+ runningContainers.sendToRunnable(runnableName, message, completion);
+ LOG.info("Change instances request completed. From {} to {}.", oldCount, newCount);
+ }
+ } catch (InterruptedException e) {
+ // If the wait is being interrupted, discard the message.
+ completion.run();
+ }
+ }
+ };
+ }
+
+ private RunnableContainerRequest createRunnableContainerRequest(final String runnableName) {
+ // Find the current order of the given runnable in order to create a RunnableContainerRequest.
+ TwillSpecification.Order order = Iterables.find(twillSpec.getOrders(), new Predicate<TwillSpecification.Order>() {
+ @Override
+ public boolean apply(TwillSpecification.Order input) {
+ return (input.getNames().contains(runnableName));
+ }
+ });
+
+ RuntimeSpecification runtimeSpec = twillSpec.getRunnables().get(runnableName);
+ Resource capability = createCapability(runtimeSpec.getResourceSpecification());
+ return new RunnableContainerRequest(order.getType(), ImmutableMultimap.of(capability, runtimeSpec));
+ }
+
+ private Runnable getMessageCompletion(final String messageId, final SettableFuture<String> future) {
+ return new Runnable() {
+ @Override
+ public void run() {
+ future.set(messageId);
+ }
+ };
+ }
+
+ private Resource createCapability(ResourceSpecification resourceSpec) {
+ Resource capability = Records.newRecord(Resource.class);
+
+ if (!YarnUtils.setVirtualCores(capability, resourceSpec.getVirtualCores())) {
+ LOG.debug("Virtual cores limit not supported.");
+ }
+
+ capability.setMemory(resourceSpec.getMemorySize());
+ return capability;
+ }
+
+ @Override
+ protected Service getServiceDelegate() {
+ return serviceDelegate;
+ }
+
+ /**
+ * A private class for service lifecycle. It's done this way so that we can have {@link ZKServiceDecorator} to
+ * wrap around this to reflect status in ZK.
+ */
+ private final class ServiceDelegate extends AbstractExecutionThreadService implements MessageCallback {
+
+ private volatile Thread runThread;
+
+ @Override
+ protected void run() throws Exception {
+ runThread = Thread.currentThread();
+ try {
+ doRun();
+ } catch (InterruptedException e) {
+ // It's ok to get interrupted exception, as it's a signal to stop
+ Thread.currentThread().interrupt();
+ }
+ }
+
+ @Override
+ protected void startUp() throws Exception {
+ doStart();
+ }
+
+ @Override
+ protected void shutDown() throws Exception {
+ doStop();
+ }
+
+ @Override
+ protected void triggerShutdown() {
+ Thread runThread = this.runThread;
+ if (runThread != null) {
+ runThread.interrupt();
+ }
+ }
+
+ @Override
+ public ListenableFuture<String> onReceived(String messageId, Message message) {
+ return processMessage(messageId, message);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationSubmitter.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationSubmitter.java b/yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationSubmitter.java
new file mode 100644
index 0000000..931c5ef
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationSubmitter.java
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.appmaster;
+
+import org.apache.twill.internal.ProcessController;
+import org.apache.twill.internal.yarn.YarnApplicationReport;
+import org.apache.twill.internal.yarn.YarnLaunchContext;
+import org.apache.hadoop.yarn.api.records.Resource;
+
+/**
+ * Interface for submitting a new application to run.
+ */
+public interface ApplicationSubmitter {
+
+ ProcessController<YarnApplicationReport> submit(YarnLaunchContext launchContext, Resource capability);
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/internal/appmaster/BasicEventHandlerContext.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/internal/appmaster/BasicEventHandlerContext.java b/yarn/src/main/java/org/apache/twill/internal/appmaster/BasicEventHandlerContext.java
new file mode 100644
index 0000000..1769910
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/internal/appmaster/BasicEventHandlerContext.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.appmaster;
+
+import org.apache.twill.api.EventHandlerContext;
+import org.apache.twill.api.EventHandlerSpecification;
+
+/**
+ *
+ */
+final class BasicEventHandlerContext implements EventHandlerContext {
+
+ private final EventHandlerSpecification specification;
+
+ BasicEventHandlerContext(EventHandlerSpecification specification) {
+ this.specification = specification;
+ }
+
+ @Override
+ public EventHandlerSpecification getSpecification() {
+ return specification;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/internal/appmaster/ExpectedContainers.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/internal/appmaster/ExpectedContainers.java b/yarn/src/main/java/org/apache/twill/internal/appmaster/ExpectedContainers.java
new file mode 100644
index 0000000..f4ebbd0
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/internal/appmaster/ExpectedContainers.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.appmaster;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.Maps;
+
+import java.util.Map;
+
+/**
+ * This class hold information about the expected container count for each runnable. It also
+ * keep track of the timestamp where the expected count has been updated.
+ */
+final class ExpectedContainers {
+
+ private final Map<String, ExpectedCount> expectedCounts;
+
+ ExpectedContainers(Map<String, Integer> expected) {
+ expectedCounts = Maps.newHashMap();
+ long now = System.currentTimeMillis();
+
+ for (Map.Entry<String, Integer> entry : expected.entrySet()) {
+ expectedCounts.put(entry.getKey(), new ExpectedCount(entry.getValue(), now));
+ }
+ }
+
+ synchronized void setExpected(String runnable, int expected) {
+ expectedCounts.put(runnable, new ExpectedCount(expected, System.currentTimeMillis()));
+ }
+
+ /**
+ * Updates the ExpectCount timestamp to current time.
+ * @param runnables List of runnable names.
+ */
+ synchronized void updateRequestTime(Iterable<String> runnables) {
+ for (String runnable : runnables) {
+ ExpectedCount oldCount = expectedCounts.get(runnable);
+ expectedCounts.put(runnable, new ExpectedCount(oldCount.getCount(), System.currentTimeMillis()));
+ }
+ }
+
+ synchronized int getExpected(String runnable) {
+ return expectedCounts.get(runnable).getCount();
+ }
+
+ synchronized Map<String, ExpectedCount> getAll() {
+ return ImmutableMap.copyOf(expectedCounts);
+ }
+
+ static final class ExpectedCount {
+ private final int count;
+ private final long timestamp;
+
+ private ExpectedCount(int count, long timestamp) {
+ this.count = count;
+ this.timestamp = timestamp;
+ }
+
+ int getCount() {
+ return count;
+ }
+
+ long getTimestamp() {
+ return timestamp;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/internal/appmaster/LoggerContextListenerAdapter.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/internal/appmaster/LoggerContextListenerAdapter.java b/yarn/src/main/java/org/apache/twill/internal/appmaster/LoggerContextListenerAdapter.java
new file mode 100644
index 0000000..2d41aa6
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/internal/appmaster/LoggerContextListenerAdapter.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.appmaster;
+
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.LoggerContext;
+import ch.qos.logback.classic.spi.LoggerContextListener;
+
+/**
+ *
+ */
+abstract class LoggerContextListenerAdapter implements LoggerContextListener {
+
+ private final boolean resetResistant;
+
+ protected LoggerContextListenerAdapter(boolean resetResistant) {
+ this.resetResistant = resetResistant;
+ }
+
+ @Override
+ public final boolean isResetResistant() {
+ return resetResistant;
+ }
+
+ @Override
+ public void onStart(LoggerContext context) {
+ }
+
+ @Override
+ public void onReset(LoggerContext context) {
+ }
+
+ @Override
+ public void onStop(LoggerContext context) {
+ }
+
+ @Override
+ public void onLevelChange(Logger logger, Level level) {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/internal/appmaster/ProvisionRequest.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/internal/appmaster/ProvisionRequest.java b/yarn/src/main/java/org/apache/twill/internal/appmaster/ProvisionRequest.java
new file mode 100644
index 0000000..002d2a5
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/internal/appmaster/ProvisionRequest.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.appmaster;
+
+import org.apache.twill.api.RuntimeSpecification;
+
+/**
+ * Package private class to help AM to track in progress container request.
+ */
+final class ProvisionRequest {
+ private final RuntimeSpecification runtimeSpec;
+ private final String requestId;
+ private int requestCount;
+
+ ProvisionRequest(RuntimeSpecification runtimeSpec, String requestId, int requestCount) {
+ this.runtimeSpec = runtimeSpec;
+ this.requestId = requestId;
+ this.requestCount = requestCount;
+ }
+
+ RuntimeSpecification getRuntimeSpec() {
+ return runtimeSpec;
+ }
+
+ String getRequestId() {
+ return requestId;
+ }
+
+ /**
+ * Called to notify a container has been provision for this request.
+ * @return {@code true} if the requested container count has been provisioned.
+ */
+ boolean containerAcquired() {
+ requestCount--;
+ return requestCount == 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/internal/appmaster/RunnableContainerRequest.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/internal/appmaster/RunnableContainerRequest.java b/yarn/src/main/java/org/apache/twill/internal/appmaster/RunnableContainerRequest.java
new file mode 100644
index 0000000..7f28443
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/internal/appmaster/RunnableContainerRequest.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.appmaster;
+
+import org.apache.twill.api.RuntimeSpecification;
+import org.apache.twill.api.TwillSpecification;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+import org.apache.hadoop.yarn.api.records.Resource;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Map;
+
+/**
+ * Data structure for holding set of runnable specifications based on resource capability.
+ */
+final class RunnableContainerRequest {
+ private final TwillSpecification.Order.Type orderType;
+ private final Iterator<Map.Entry<Resource, Collection<RuntimeSpecification>>> requests;
+
+ RunnableContainerRequest(TwillSpecification.Order.Type orderType,
+ Multimap<Resource, RuntimeSpecification> requests) {
+ this.orderType = orderType;
+ this.requests = requests.asMap().entrySet().iterator();
+ }
+
+ TwillSpecification.Order.Type getOrderType() {
+ return orderType;
+ }
+
+ /**
+ * Remove a resource request and return it.
+ * @return The {@link Resource} and {@link Collection} of {@link RuntimeSpecification} or
+ * {@code null} if there is no more request.
+ */
+ Map.Entry<Resource, ? extends Collection<RuntimeSpecification>> takeRequest() {
+ Map.Entry<Resource, Collection<RuntimeSpecification>> next = Iterators.getNext(requests, null);
+ return next == null ? null : Maps.immutableEntry(next.getKey(), ImmutableList.copyOf(next.getValue()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/internal/appmaster/RunnableProcessLauncher.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/internal/appmaster/RunnableProcessLauncher.java b/yarn/src/main/java/org/apache/twill/internal/appmaster/RunnableProcessLauncher.java
new file mode 100644
index 0000000..b4b27a9
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/internal/appmaster/RunnableProcessLauncher.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.appmaster;
+
+import org.apache.twill.common.Cancellable;
+import org.apache.twill.internal.EnvKeys;
+import org.apache.twill.internal.ProcessController;
+import org.apache.twill.internal.yarn.AbstractYarnProcessLauncher;
+import org.apache.twill.internal.yarn.YarnContainerInfo;
+import org.apache.twill.internal.yarn.YarnLaunchContext;
+import org.apache.twill.internal.yarn.YarnNMClient;
+import com.google.common.base.Objects;
+import com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+
+/**
+ *
+ */
+public final class RunnableProcessLauncher extends AbstractYarnProcessLauncher<YarnContainerInfo> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(RunnableProcessLauncher.class);
+
+ private final YarnContainerInfo containerInfo;
+ private final YarnNMClient nmClient;
+ private boolean launched;
+
+ public RunnableProcessLauncher(YarnContainerInfo containerInfo, YarnNMClient nmClient) {
+ super(containerInfo);
+ this.containerInfo = containerInfo;
+ this.nmClient = nmClient;
+ }
+
+ @Override
+ public String toString() {
+ return Objects.toStringHelper(this)
+ .add("container", containerInfo)
+ .toString();
+ }
+
+ @Override
+ protected <R> ProcessController<R> doLaunch(YarnLaunchContext launchContext) {
+ Map<String, String> env = Maps.newHashMap(launchContext.getEnvironment());
+
+ // Set extra environments
+ env.put(EnvKeys.YARN_CONTAINER_ID, containerInfo.getId());
+ env.put(EnvKeys.YARN_CONTAINER_HOST, containerInfo.getHost().getHostName());
+ env.put(EnvKeys.YARN_CONTAINER_PORT, Integer.toString(containerInfo.getPort()));
+ env.put(EnvKeys.YARN_CONTAINER_MEMORY_MB, Integer.toString(containerInfo.getMemoryMB()));
+ env.put(EnvKeys.YARN_CONTAINER_VIRTUAL_CORES, Integer.toString(containerInfo.getVirtualCores()));
+
+ launchContext.setEnvironment(env);
+
+ LOG.info("Launching in container {}, {}", containerInfo.getId(), launchContext.getCommands());
+ final Cancellable cancellable = nmClient.start(containerInfo, launchContext);
+ launched = true;
+
+ return new ProcessController<R>() {
+ @Override
+ public R getReport() {
+ // No reporting support for runnable launch yet.
+ return null;
+
+ }
+
+ @Override
+ public void cancel() {
+ cancellable.cancel();
+ }
+ };
+ }
+
+ public boolean isLaunched() {
+ return launched;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/internal/appmaster/RunningContainers.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/internal/appmaster/RunningContainers.java b/yarn/src/main/java/org/apache/twill/internal/appmaster/RunningContainers.java
new file mode 100644
index 0000000..beef0d4
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/internal/appmaster/RunningContainers.java
@@ -0,0 +1,427 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.appmaster;
+
+import org.apache.twill.api.ResourceReport;
+import org.apache.twill.api.RunId;
+import org.apache.twill.api.ServiceController;
+import org.apache.twill.api.TwillRunResources;
+import org.apache.twill.internal.ContainerInfo;
+import org.apache.twill.internal.DefaultResourceReport;
+import org.apache.twill.internal.DefaultTwillRunResources;
+import org.apache.twill.internal.RunIds;
+import org.apache.twill.internal.TwillContainerController;
+import org.apache.twill.internal.TwillContainerLauncher;
+import org.apache.twill.internal.container.TwillContainerMain;
+import org.apache.twill.internal.state.Message;
+import org.apache.twill.internal.yarn.YarnContainerStatus;
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.HashBasedTable;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multiset;
+import com.google.common.collect.Table;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.BitSet;
+import java.util.Collection;
+import java.util.Deque;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.Condition;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * A helper class for ApplicationMasterService to keep track of running containers and to interact
+ * with them.
+ */
+final class RunningContainers {
+ private static final Logger LOG = LoggerFactory.getLogger(RunningContainers.class);
+
+ /**
+ * Function to return cardinality of a given BitSet.
+ */
+ private static final Function<BitSet, Integer> BITSET_CARDINALITY = new Function<BitSet, Integer>() {
+ @Override
+ public Integer apply(BitSet input) {
+ return input.cardinality();
+ }
+ };
+
+ // Table of <runnableName, containerId, controller>
+ private final Table<String, String, TwillContainerController> containers;
+
+ // Map from runnableName to a BitSet, with the <instanceId> bit turned on for having an instance running.
+ private final Map<String, BitSet> runnableInstances;
+ private final DefaultResourceReport resourceReport;
+ private final Deque<String> startSequence;
+ private final Lock containerLock;
+ private final Condition containerChange;
+
+ RunningContainers(String appId, TwillRunResources appMasterResources) {
+ containers = HashBasedTable.create();
+ runnableInstances = Maps.newHashMap();
+ startSequence = Lists.newLinkedList();
+ containerLock = new ReentrantLock();
+ containerChange = containerLock.newCondition();
+ resourceReport = new DefaultResourceReport(appId, appMasterResources);
+ }
+
+ /**
+ * Returns {@code true} if there is no live container.
+ */
+ boolean isEmpty() {
+ containerLock.lock();
+ try {
+ return runnableInstances.isEmpty();
+ } finally {
+ containerLock.unlock();
+ }
+ }
+
+ void start(String runnableName, ContainerInfo containerInfo, TwillContainerLauncher launcher) {
+ containerLock.lock();
+ try {
+ int instanceId = getStartInstanceId(runnableName);
+ RunId runId = getRunId(runnableName, instanceId);
+ TwillContainerController controller = launcher.start(runId, instanceId,
+ TwillContainerMain.class, "$HADOOP_CONF_DIR");
+ containers.put(runnableName, containerInfo.getId(), controller);
+
+ TwillRunResources resources = new DefaultTwillRunResources(instanceId,
+ containerInfo.getId(),
+ containerInfo.getVirtualCores(),
+ containerInfo.getMemoryMB(),
+ containerInfo.getHost().getHostName());
+ resourceReport.addRunResources(runnableName, resources);
+
+ if (startSequence.isEmpty() || !runnableName.equals(startSequence.peekLast())) {
+ startSequence.addLast(runnableName);
+ }
+ containerChange.signalAll();
+
+ } finally {
+ containerLock.unlock();
+ }
+ }
+
+ ResourceReport getResourceReport() {
+ return resourceReport;
+ }
+
+ /**
+ * Stops and removes the last running container of the given runnable.
+ */
+ void removeLast(String runnableName) {
+ containerLock.lock();
+ try {
+ int maxInstanceId = getMaxInstanceId(runnableName);
+ if (maxInstanceId < 0) {
+ LOG.warn("No running container found for {}", runnableName);
+ return;
+ }
+
+ String lastContainerId = null;
+ TwillContainerController lastController = null;
+
+ // Find the controller with the maxInstanceId
+ for (Map.Entry<String, TwillContainerController> entry : containers.row(runnableName).entrySet()) {
+ if (getInstanceId(entry.getValue().getRunId()) == maxInstanceId) {
+ lastContainerId = entry.getKey();
+ lastController = entry.getValue();
+ break;
+ }
+ }
+
+ Preconditions.checkState(lastContainerId != null,
+ "No container found for {} with instanceId = {}", runnableName, maxInstanceId);
+
+ LOG.info("Stopping service: {} {}", runnableName, lastController.getRunId());
+ lastController.stopAndWait();
+ containers.remove(runnableName, lastContainerId);
+ removeInstanceId(runnableName, maxInstanceId);
+ resourceReport.removeRunnableResources(runnableName, lastContainerId);
+ containerChange.signalAll();
+ } finally {
+ containerLock.unlock();
+ }
+ }
+
+ /**
+ * Blocks until there are changes in running containers.
+ */
+ void waitForCount(String runnableName, int count) throws InterruptedException {
+ containerLock.lock();
+ try {
+ while (getRunningInstances(runnableName) != count) {
+ containerChange.await();
+ }
+ } finally {
+ containerLock.unlock();
+ }
+ }
+
+ /**
+ * Returns the number of running instances of the given runnable.
+ */
+ int count(String runnableName) {
+ containerLock.lock();
+ try {
+ return getRunningInstances(runnableName);
+ } finally {
+ containerLock.unlock();
+ }
+ }
+
+ /**
+ * Returns a Map contains running instances of all runnables.
+ */
+ Map<String, Integer> countAll() {
+ containerLock.lock();
+ try {
+ return ImmutableMap.copyOf(Maps.transformValues(runnableInstances, BITSET_CARDINALITY));
+ } finally {
+ containerLock.unlock();
+ }
+ }
+
+ void sendToAll(Message message, Runnable completion) {
+ containerLock.lock();
+ try {
+ if (containers.isEmpty()) {
+ completion.run();
+ }
+
+ // Sends the command to all running containers
+ AtomicInteger count = new AtomicInteger(containers.size());
+ for (Map.Entry<String, Map<String, TwillContainerController>> entry : containers.rowMap().entrySet()) {
+ for (TwillContainerController controller : entry.getValue().values()) {
+ sendMessage(entry.getKey(), message, controller, count, completion);
+ }
+ }
+ } finally {
+ containerLock.unlock();
+ }
+ }
+
+ void sendToRunnable(String runnableName, Message message, Runnable completion) {
+ containerLock.lock();
+ try {
+ Collection<TwillContainerController> controllers = containers.row(runnableName).values();
+ if (controllers.isEmpty()) {
+ completion.run();
+ }
+
+ AtomicInteger count = new AtomicInteger(controllers.size());
+ for (TwillContainerController controller : controllers) {
+ sendMessage(runnableName, message, controller, count, completion);
+ }
+ } finally {
+ containerLock.unlock();
+ }
+ }
+
+ /**
+ * Stops all running services. Only called when the AppMaster stops.
+ */
+ void stopAll() {
+ containerLock.lock();
+ try {
+ // Stop it one by one in reverse order of start sequence
+ Iterator<String> itor = startSequence.descendingIterator();
+ List<ListenableFuture<ServiceController.State>> futures = Lists.newLinkedList();
+ while (itor.hasNext()) {
+ String runnableName = itor.next();
+ LOG.info("Stopping all instances of " + runnableName);
+
+ futures.clear();
+ // Parallel stops all running containers of the current runnable.
+ for (TwillContainerController controller : containers.row(runnableName).values()) {
+ futures.add(controller.stop());
+ }
+ // Wait for containers to stop. Assumes the future returned by Futures.successfulAsList won't throw exception.
+ Futures.getUnchecked(Futures.successfulAsList(futures));
+
+ LOG.info("Terminated all instances of " + runnableName);
+ }
+ containers.clear();
+ runnableInstances.clear();
+ } finally {
+ containerLock.unlock();
+ }
+ }
+
+ Set<String> getContainerIds() {
+ containerLock.lock();
+ try {
+ return ImmutableSet.copyOf(containers.columnKeySet());
+ } finally {
+ containerLock.unlock();
+ }
+ }
+
+ /**
+ * Handle completion of container.
+ * @param status The completion status.
+ * @param restartRunnables Set of runnable names that requires restart.
+ */
+ void handleCompleted(YarnContainerStatus status, Multiset<String> restartRunnables) {
+ containerLock.lock();
+ String containerId = status.getContainerId();
+ int exitStatus = status.getExitStatus();
+ ContainerState state = status.getState();
+
+ try {
+ Map<String, TwillContainerController> lookup = containers.column(containerId);
+ if (lookup.isEmpty()) {
+ // It's OK because if a container is stopped through removeLast, this would be empty.
+ return;
+ }
+
+ if (lookup.size() != 1) {
+ LOG.warn("More than one controller found for container {}", containerId);
+ }
+
+ if (exitStatus != 0) {
+ LOG.warn("Container {} exited abnormally with state {}, exit code {}. Re-request the container.",
+ containerId, state, exitStatus);
+ restartRunnables.add(lookup.keySet().iterator().next());
+ } else {
+ LOG.info("Container {} exited normally with state {}", containerId, state);
+ }
+
+ for (Map.Entry<String, TwillContainerController> completedEntry : lookup.entrySet()) {
+ String runnableName = completedEntry.getKey();
+ TwillContainerController controller = completedEntry.getValue();
+ controller.completed(exitStatus);
+
+ removeInstanceId(runnableName, getInstanceId(controller.getRunId()));
+ resourceReport.removeRunnableResources(runnableName, containerId);
+ }
+
+ lookup.clear();
+ containerChange.signalAll();
+ } finally {
+ containerLock.unlock();
+ }
+ }
+
+ /**
+ * Sends a command through the given {@link org.apache.twill.internal.TwillContainerController} of a runnable. Decrements the count
+ * when the sending of command completed. Triggers completion when count reaches zero.
+ */
+ private void sendMessage(final String runnableName, final Message message,
+ final TwillContainerController controller, final AtomicInteger count,
+ final Runnable completion) {
+ Futures.addCallback(controller.sendMessage(message), new FutureCallback<Message>() {
+ @Override
+ public void onSuccess(Message result) {
+ if (count.decrementAndGet() == 0) {
+ completion.run();
+ }
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ try {
+ LOG.error("Failed to send message. Runnable: {}, RunId: {}, Message: {}.",
+ runnableName, controller.getRunId(), message, t);
+ } finally {
+ if (count.decrementAndGet() == 0) {
+ completion.run();
+ }
+ }
+ }
+ });
+ }
+
+ /**
+ * Returns the instanceId to start the given runnable.
+ */
+ private int getStartInstanceId(String runnableName) {
+ BitSet instances = runnableInstances.get(runnableName);
+ if (instances == null) {
+ instances = new BitSet();
+ runnableInstances.put(runnableName, instances);
+ }
+ int instanceId = instances.nextClearBit(0);
+ instances.set(instanceId);
+ return instanceId;
+ }
+
+ private void removeInstanceId(String runnableName, int instanceId) {
+ BitSet instances = runnableInstances.get(runnableName);
+ if (instances == null) {
+ return;
+ }
+ instances.clear(instanceId);
+ if (instances.isEmpty()) {
+ runnableInstances.remove(runnableName);
+ }
+ }
+
+ /**
+ * Returns the largest instanceId for the given runnable. Returns -1 if no container is running.
+ */
+ private int getMaxInstanceId(String runnableName) {
+ BitSet instances = runnableInstances.get(runnableName);
+ if (instances == null || instances.isEmpty()) {
+ return -1;
+ }
+ return instances.length() - 1;
+ }
+
+ /**
+ * Returns nnumber of running instances for the given runnable.
+ */
+ private int getRunningInstances(String runableName) {
+ BitSet instances = runnableInstances.get(runableName);
+ return instances == null ? 0 : instances.cardinality();
+ }
+
+ private RunId getRunId(String runnableName, int instanceId) {
+ RunId baseId;
+
+ Collection<TwillContainerController> controllers = containers.row(runnableName).values();
+ if (controllers.isEmpty()) {
+ baseId = RunIds.generate();
+ } else {
+ String id = controllers.iterator().next().getRunId().getId();
+ baseId = RunIds.fromString(id.substring(0, id.lastIndexOf('-')));
+ }
+
+ return RunIds.fromString(baseId.getId() + '-' + instanceId);
+ }
+
+ private int getInstanceId(RunId runId) {
+ String id = runId.getId();
+ return Integer.parseInt(id.substring(id.lastIndexOf('-') + 1));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/internal/appmaster/TrackerService.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/internal/appmaster/TrackerService.java b/yarn/src/main/java/org/apache/twill/internal/appmaster/TrackerService.java
new file mode 100644
index 0000000..ca299e0
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/internal/appmaster/TrackerService.java
@@ -0,0 +1,222 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.appmaster;
+
+import org.apache.twill.api.ResourceReport;
+import org.apache.twill.internal.json.ResourceReportAdapter;
+import com.google.common.util.concurrent.AbstractIdleService;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.jboss.netty.bootstrap.ServerBootstrap;
+import org.jboss.netty.buffer.ChannelBuffer;
+import org.jboss.netty.buffer.ChannelBufferOutputStream;
+import org.jboss.netty.buffer.ChannelBuffers;
+import org.jboss.netty.channel.Channel;
+import org.jboss.netty.channel.ChannelFactory;
+import org.jboss.netty.channel.ChannelFuture;
+import org.jboss.netty.channel.ChannelFutureListener;
+import org.jboss.netty.channel.ChannelHandlerContext;
+import org.jboss.netty.channel.ChannelPipeline;
+import org.jboss.netty.channel.ChannelPipelineFactory;
+import org.jboss.netty.channel.Channels;
+import org.jboss.netty.channel.ExceptionEvent;
+import org.jboss.netty.channel.MessageEvent;
+import org.jboss.netty.channel.SimpleChannelUpstreamHandler;
+import org.jboss.netty.channel.group.ChannelGroup;
+import org.jboss.netty.channel.group.DefaultChannelGroup;
+import org.jboss.netty.channel.socket.nio.NioServerSocketChannelFactory;
+import org.jboss.netty.handler.codec.http.DefaultHttpResponse;
+import org.jboss.netty.handler.codec.http.HttpChunkAggregator;
+import org.jboss.netty.handler.codec.http.HttpContentCompressor;
+import org.jboss.netty.handler.codec.http.HttpHeaders;
+import org.jboss.netty.handler.codec.http.HttpMethod;
+import org.jboss.netty.handler.codec.http.HttpRequest;
+import org.jboss.netty.handler.codec.http.HttpRequestDecoder;
+import org.jboss.netty.handler.codec.http.HttpResponse;
+import org.jboss.netty.handler.codec.http.HttpResponseEncoder;
+import org.jboss.netty.handler.codec.http.HttpResponseStatus;
+import org.jboss.netty.handler.codec.http.HttpVersion;
+import org.jboss.netty.util.CharsetUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.net.URL;
+import java.util.concurrent.Executor;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Webservice that the Application Master will register back to the resource manager
+ * for clients to track application progress. Currently used purely for getting a
+ * breakdown of resource usage as a {@link org.apache.twill.api.ResourceReport}.
+ */
+public final class TrackerService extends AbstractIdleService {
+
+ // TODO: This is temporary. When support more REST API, this would get moved.
+ public static final String PATH = "/resources";
+
+ private static final Logger LOG = LoggerFactory.getLogger(TrackerService.class);
+ private static final int NUM_BOSS_THREADS = 1;
+ private static final int CLOSE_CHANNEL_TIMEOUT = 5;
+ private static final int MAX_INPUT_SIZE = 100 * 1024 * 1024;
+
+ private final String host;
+ private ServerBootstrap bootstrap;
+ private InetSocketAddress bindAddress;
+ private URL url;
+ private final ChannelGroup channelGroup;
+ private final ResourceReport resourceReport;
+
+ /**
+ * Initialize the service.
+ *
+ * @param resourceReport live report that the service will return to clients.
+ * @param appMasterHost the application master host.
+ */
+ public TrackerService(ResourceReport resourceReport, String appMasterHost) {
+ this.channelGroup = new DefaultChannelGroup("appMasterTracker");
+ this.resourceReport = resourceReport;
+ this.host = appMasterHost;
+ }
+
+ /**
+ * Returns the address this tracker service is bounded to.
+ */
+ public InetSocketAddress getBindAddress() {
+ return bindAddress;
+ }
+
+ /**
+ * @return tracker url.
+ */
+ public URL getUrl() {
+ return url;
+ }
+
+ @Override
+ protected void startUp() throws Exception {
+ Executor bossThreads = Executors.newFixedThreadPool(NUM_BOSS_THREADS,
+ new ThreadFactoryBuilder()
+ .setDaemon(true)
+ .setNameFormat("boss-thread")
+ .build());
+
+ Executor workerThreads = Executors.newCachedThreadPool(new ThreadFactoryBuilder()
+ .setDaemon(true)
+ .setNameFormat("worker-thread#%d")
+ .build());
+
+ ChannelFactory factory = new NioServerSocketChannelFactory(bossThreads, workerThreads);
+
+ bootstrap = new ServerBootstrap(factory);
+
+ bootstrap.setPipelineFactory(new ChannelPipelineFactory() {
+ public ChannelPipeline getPipeline() {
+ ChannelPipeline pipeline = Channels.pipeline();
+
+ pipeline.addLast("decoder", new HttpRequestDecoder());
+ pipeline.addLast("aggregator", new HttpChunkAggregator(MAX_INPUT_SIZE));
+ pipeline.addLast("encoder", new HttpResponseEncoder());
+ pipeline.addLast("compressor", new HttpContentCompressor());
+ pipeline.addLast("handler", new ReportHandler(resourceReport));
+
+ return pipeline;
+ }
+ });
+
+ Channel channel = bootstrap.bind(new InetSocketAddress(host, 0));
+ bindAddress = (InetSocketAddress) channel.getLocalAddress();
+ url = URI.create(String.format("http://%s:%d", host, bindAddress.getPort()))
+ .resolve(TrackerService.PATH).toURL();
+ channelGroup.add(channel);
+ }
+
+ @Override
+ protected void shutDown() throws Exception {
+ try {
+ if (!channelGroup.close().await(CLOSE_CHANNEL_TIMEOUT, TimeUnit.SECONDS)) {
+ LOG.warn("Timeout when closing all channels.");
+ }
+ } finally {
+ bootstrap.releaseExternalResources();
+ }
+ }
+
+ /**
+ * Handler to return resources used by this application master, which will be available through
+ * the host and port set when this application master registered itself to the resource manager.
+ */
+ public class ReportHandler extends SimpleChannelUpstreamHandler {
+ private final ResourceReport report;
+ private final ResourceReportAdapter reportAdapter;
+
+ public ReportHandler(ResourceReport report) {
+ this.report = report;
+ this.reportAdapter = ResourceReportAdapter.create();
+ }
+
+ @Override
+ public void messageReceived(ChannelHandlerContext ctx, MessageEvent e) throws Exception {
+ HttpRequest request = (HttpRequest) e.getMessage();
+ if (!isValid(request)) {
+ write404(e);
+ return;
+ }
+
+ writeResponse(e);
+ }
+
+ // only accepts GET on /resources for now
+ private boolean isValid(HttpRequest request) {
+ return (request.getMethod() == HttpMethod.GET) && PATH.equals(request.getUri());
+ }
+
+ private void write404(MessageEvent e) {
+ HttpResponse response = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.NOT_FOUND);
+ ChannelFuture future = e.getChannel().write(response);
+ future.addListener(ChannelFutureListener.CLOSE);
+ }
+
+ private void writeResponse(MessageEvent e) {
+ HttpResponse response = new DefaultHttpResponse(HttpVersion.HTTP_1_1, HttpResponseStatus.OK);
+ response.setHeader(HttpHeaders.Names.CONTENT_TYPE, "application/json; charset=UTF-8");
+
+ ChannelBuffer content = ChannelBuffers.dynamicBuffer();
+ Writer writer = new OutputStreamWriter(new ChannelBufferOutputStream(content), CharsetUtil.UTF_8);
+ reportAdapter.toJson(report, writer);
+ try {
+ writer.close();
+ } catch (IOException e1) {
+ LOG.error("error writing resource report", e1);
+ }
+ response.setContent(content);
+ ChannelFuture future = e.getChannel().write(response);
+ future.addListener(ChannelFutureListener.CLOSE);
+ }
+
+ @Override
+ public void exceptionCaught(ChannelHandlerContext ctx, ExceptionEvent e) {
+ e.getChannel().close();
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/internal/appmaster/package-info.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/internal/appmaster/package-info.java b/yarn/src/main/java/org/apache/twill/internal/appmaster/package-info.java
new file mode 100644
index 0000000..bf8e677
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/internal/appmaster/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This package contains implementation of Twill application master.
+ */
+package org.apache.twill.internal.appmaster;
[13/15] Initial import commit.
Posted by ch...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/common/src/main/java/org/apache/twill/filesystem/LocationFactories.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/twill/filesystem/LocationFactories.java b/common/src/main/java/org/apache/twill/filesystem/LocationFactories.java
new file mode 100644
index 0000000..751a632
--- /dev/null
+++ b/common/src/main/java/org/apache/twill/filesystem/LocationFactories.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.filesystem;
+
+import com.google.common.base.Throwables;
+
+import java.io.IOException;
+import java.net.URI;
+
+/**
+ * Providers helper methods for creating different {@link LocationFactory}.
+ */
+public final class LocationFactories {
+
+ /**
+ * Creates a {@link LocationFactory} that always applies the giving namespace prefix.
+ */
+ public static LocationFactory namespace(LocationFactory delegate, final String namespace) {
+ return new ForwardingLocationFactory(delegate) {
+ @Override
+ public Location create(String path) {
+ try {
+ Location base = getDelegate().create(namespace);
+ return base.append(path);
+ } catch (IOException e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ public Location create(URI uri) {
+ if (uri.isAbsolute()) {
+ return getDelegate().create(uri);
+ }
+ try {
+ Location base = getDelegate().create(namespace);
+ return base.append(uri.getPath());
+ } catch (IOException e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ public Location getHomeLocation() {
+ return getDelegate().getHomeLocation();
+ }
+ };
+ }
+
+ private LocationFactories() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/common/src/main/java/org/apache/twill/filesystem/LocationFactory.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/twill/filesystem/LocationFactory.java b/common/src/main/java/org/apache/twill/filesystem/LocationFactory.java
new file mode 100644
index 0000000..f88d94d
--- /dev/null
+++ b/common/src/main/java/org/apache/twill/filesystem/LocationFactory.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.filesystem;
+
+import java.net.URI;
+
+/**
+ * Factory for creating instance of {@link Location}.
+ */
+public interface LocationFactory {
+
+ /**
+ * Creates an instance of {@link Location} of the given path.
+ * @param path The path representing the location.
+ * @return An instance of {@link Location}.
+ */
+ Location create(String path);
+
+ /**
+ * Creates an instance of {@link Location} based on {@link java.net.URI} <code>uri</code>.
+ *
+ * @param uri to the resource on the filesystem.
+ * @return An instance of {@link Location}
+ */
+ Location create(URI uri);
+
+ /**
+ * Returns the home location.
+ */
+ Location getHomeLocation();
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/common/src/test/java/org/apache/twill/common/ServicesTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/twill/common/ServicesTest.java b/common/src/test/java/org/apache/twill/common/ServicesTest.java
new file mode 100644
index 0000000..c0aa7ee
--- /dev/null
+++ b/common/src/test/java/org/apache/twill/common/ServicesTest.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.common;
+
+import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.AbstractIdleService;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.Service;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * Unit test for {@link Services} methods.
+ */
+public class ServicesTest {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ServicesTest.class);
+
+ @Test
+ public void testChain() throws ExecutionException, InterruptedException {
+ AtomicBoolean transiting = new AtomicBoolean(false);
+ Service s1 = new DummyService("s1", transiting);
+ Service s2 = new DummyService("s2", transiting);
+ Service s3 = new DummyService("s3", transiting);
+
+ Futures.allAsList(Services.chainStart(s1, s2, s3).get()).get();
+ Futures.allAsList(Services.chainStop(s3, s2, s1).get()).get();
+ }
+
+ @Test
+ public void testCompletion() throws ExecutionException, InterruptedException {
+ Service service = new DummyService("s1", new AtomicBoolean());
+ ListenableFuture<Service.State> completion = Services.getCompletionFuture(service);
+
+ service.start();
+ service.stop();
+
+ completion.get();
+
+ AtomicBoolean transiting = new AtomicBoolean();
+ service = new DummyService("s2", transiting);
+ completion = Services.getCompletionFuture(service);
+
+ service.startAndWait();
+ transiting.set(true);
+ service.stop();
+
+ try {
+ completion.get();
+ Assert.assertTrue(false);
+ } catch (ExecutionException e) {
+ // Expected
+ }
+ }
+
+ private static final class DummyService extends AbstractIdleService {
+
+ private final String name;
+ private final AtomicBoolean transiting;
+
+ private DummyService(String name, AtomicBoolean transiting) {
+ this.name = name;
+ this.transiting = transiting;
+ }
+
+ @Override
+ protected void startUp() throws Exception {
+ Preconditions.checkState(transiting.compareAndSet(false, true));
+ LOG.info("Starting: " + name);
+ TimeUnit.MILLISECONDS.sleep(500);
+ LOG.info("Started: " + name);
+ Preconditions.checkState(transiting.compareAndSet(true, false));
+ }
+
+ @Override
+ protected void shutDown() throws Exception {
+ Preconditions.checkState(transiting.compareAndSet(false, true));
+ LOG.info("Stopping: " + name);
+ TimeUnit.MILLISECONDS.sleep(500);
+ LOG.info("Stopped: " + name);
+ Preconditions.checkState(transiting.compareAndSet(true, false));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/common/src/test/java/org/apache/twill/filesystem/LocalLocationTest.java
----------------------------------------------------------------------
diff --git a/common/src/test/java/org/apache/twill/filesystem/LocalLocationTest.java b/common/src/test/java/org/apache/twill/filesystem/LocalLocationTest.java
new file mode 100644
index 0000000..198f77f
--- /dev/null
+++ b/common/src/test/java/org/apache/twill/filesystem/LocalLocationTest.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.filesystem;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URI;
+
+/**
+ *
+ */
+public class LocalLocationTest {
+
+ @Test
+ public void testDelete() throws IOException {
+ LocationFactory factory = new LocalLocationFactory(new File(System.getProperty("java.io.tmpdir")));
+
+ Location base = factory.create("test").getTempFile(".tmp");
+ Assert.assertTrue(base.mkdirs());
+
+ Assert.assertTrue(base.append("test1").getTempFile(".tmp").createNew());
+ Assert.assertTrue(base.append("test2").getTempFile(".tmp").createNew());
+
+ Location subDir = base.append("test3");
+ Assert.assertTrue(subDir.mkdirs());
+
+ Assert.assertTrue(subDir.append("test4").getTempFile(".tmp").createNew());
+ Assert.assertTrue(subDir.append("test5").getTempFile(".tmp").createNew());
+
+ Assert.assertTrue(base.delete(true));
+ Assert.assertFalse(base.exists());
+ }
+
+ @Test
+ public void testHelper() {
+ LocationFactory factory = LocationFactories.namespace(
+ new LocalLocationFactory(new File(System.getProperty("java.io.tmpdir"))),
+ "testhelper");
+
+ Location location = factory.create("test");
+ Assert.assertTrue(location.toURI().getPath().endsWith("testhelper/test"));
+
+ location = factory.create(URI.create("test2"));
+ Assert.assertTrue(location.toURI().getPath().endsWith("testhelper/test2"));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/pom.xml
----------------------------------------------------------------------
diff --git a/core/pom.xml b/core/pom.xml
new file mode 100644
index 0000000..051fa38
--- /dev/null
+++ b/core/pom.xml
@@ -0,0 +1,89 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>twill-parent</artifactId>
+ <groupId>org.apache.twill</groupId>
+ <version>1.3.0-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>twill-core</artifactId>
+ <name>Twill core library</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>twill-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>twill-zookeeper</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>twill-discovery-core</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>io.netty</groupId>
+ <artifactId>netty</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.xerial.snappy</groupId>
+ <artifactId>snappy-java</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.ow2.asm</groupId>
+ <artifactId>asm-all</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.commons</groupId>
+ <artifactId>commons-compress</artifactId>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/AbstractExecutionServiceController.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/AbstractExecutionServiceController.java b/core/src/main/java/org/apache/twill/internal/AbstractExecutionServiceController.java
new file mode 100644
index 0000000..974639d
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/AbstractExecutionServiceController.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal;
+
+import org.apache.twill.api.RunId;
+import org.apache.twill.api.ServiceController;
+import org.apache.twill.common.Threads;
+import com.google.common.util.concurrent.AbstractIdleService;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.Service;
+
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.Executor;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ *
+ */
+public abstract class AbstractExecutionServiceController implements ServiceController {
+
+ private final RunId runId;
+ private final ListenerExecutors listenerExecutors;
+ private final Service serviceDelegate;
+
+ protected AbstractExecutionServiceController(RunId runId) {
+ this.runId = runId;
+ this.listenerExecutors = new ListenerExecutors();
+ this.serviceDelegate = new ServiceDelegate();
+ }
+
+ protected abstract void startUp();
+
+ protected abstract void shutDown();
+
+ @Override
+ public final RunId getRunId() {
+ return runId;
+ }
+
+ @Override
+ public final void addListener(Listener listener, Executor executor) {
+ listenerExecutors.addListener(new ListenerExecutor(listener, executor));
+ }
+
+ @Override
+ public final ListenableFuture<State> start() {
+ serviceDelegate.addListener(listenerExecutors, Threads.SAME_THREAD_EXECUTOR);
+ return serviceDelegate.start();
+ }
+
+ @Override
+ public final State startAndWait() {
+ return Futures.getUnchecked(start());
+ }
+
+ @Override
+ public final boolean isRunning() {
+ return serviceDelegate.isRunning();
+ }
+
+ @Override
+ public final State state() {
+ return serviceDelegate.state();
+ }
+
+ @Override
+ public final State stopAndWait() {
+ return Futures.getUnchecked(stop());
+ }
+
+ @Override
+ public final ListenableFuture<State> stop() {
+ return serviceDelegate.stop();
+ }
+
+ protected Executor executor(final State state) {
+ return new Executor() {
+ @Override
+ public void execute(Runnable command) {
+ Thread t = new Thread(command, getClass().getSimpleName() + " " + state);
+ t.setDaemon(true);
+ t.start();
+ }
+ };
+ }
+
+
+ private final class ServiceDelegate extends AbstractIdleService {
+ @Override
+ protected void startUp() throws Exception {
+ AbstractExecutionServiceController.this.startUp();
+ }
+
+ @Override
+ protected void shutDown() throws Exception {
+ AbstractExecutionServiceController.this.shutDown();
+ }
+
+ @Override
+ protected Executor executor(State state) {
+ return AbstractExecutionServiceController.this.executor(state);
+ }
+ }
+
+ /**
+ * Inner class for dispatching listener call back to a list of listeners
+ */
+ private static final class ListenerExecutors implements Listener {
+
+ private interface Callback {
+ void call(Listener listener);
+ }
+
+ private final Queue<ListenerExecutor> listeners = new ConcurrentLinkedQueue<ListenerExecutor>();
+ private final AtomicReference<Callback> lastState = new AtomicReference<Callback>();
+
+ private synchronized void addListener(final ListenerExecutor listener) {
+ listeners.add(listener);
+ Callback callback = lastState.get();
+ if (callback != null) {
+ callback.call(listener);
+ }
+ }
+
+ @Override
+ public synchronized void starting() {
+ lastState.set(new Callback() {
+ @Override
+ public void call(Listener listener) {
+ listener.starting();
+ }
+ });
+ for (ListenerExecutor listener : listeners) {
+ listener.starting();
+ }
+ }
+
+ @Override
+ public synchronized void running() {
+ lastState.set(new Callback() {
+ @Override
+ public void call(Listener listener) {
+ listener.running();
+ }
+ });
+ for (ListenerExecutor listener : listeners) {
+ listener.running();
+ }
+ }
+
+ @Override
+ public synchronized void stopping(final State from) {
+ lastState.set(new Callback() {
+ @Override
+ public void call(Listener listener) {
+ listener.stopping(from);
+ }
+ });
+ for (ListenerExecutor listener : listeners) {
+ listener.stopping(from);
+ }
+ }
+
+ @Override
+ public synchronized void terminated(final State from) {
+ lastState.set(new Callback() {
+ @Override
+ public void call(Listener listener) {
+ listener.terminated(from);
+ }
+ });
+ for (ListenerExecutor listener : listeners) {
+ listener.terminated(from);
+ }
+ }
+
+ @Override
+ public synchronized void failed(final State from, final Throwable failure) {
+ lastState.set(new Callback() {
+ @Override
+ public void call(Listener listener) {
+ listener.failed(from, failure);
+ }
+ });
+ for (ListenerExecutor listener : listeners) {
+ listener.failed(from, failure);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/AbstractTwillController.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/AbstractTwillController.java b/core/src/main/java/org/apache/twill/internal/AbstractTwillController.java
new file mode 100644
index 0000000..5806f9d
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/AbstractTwillController.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal;
+
+import org.apache.twill.api.RunId;
+import org.apache.twill.api.TwillController;
+import org.apache.twill.api.logging.LogEntry;
+import org.apache.twill.api.logging.LogHandler;
+import org.apache.twill.discovery.Discoverable;
+import org.apache.twill.discovery.DiscoveryServiceClient;
+import org.apache.twill.discovery.ZKDiscoveryService;
+import org.apache.twill.internal.json.StackTraceElementCodec;
+import org.apache.twill.internal.kafka.client.SimpleKafkaClient;
+import org.apache.twill.internal.logging.LogEntryDecoder;
+import org.apache.twill.internal.state.SystemMessages;
+import org.apache.twill.kafka.client.FetchedMessage;
+import org.apache.twill.kafka.client.KafkaClient;
+import org.apache.twill.zookeeper.ZKClient;
+import org.apache.twill.zookeeper.ZKClients;
+import com.google.common.base.Charsets;
+import com.google.common.collect.Iterables;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Iterator;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A abstract base class for {@link org.apache.twill.api.TwillController} implementation that uses Zookeeper to controller a
+ * running twill application.
+ */
+public abstract class AbstractTwillController extends AbstractZKServiceController implements TwillController {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AbstractTwillController.class);
+ private static final int MAX_KAFKA_FETCH_SIZE = 1048576;
+ private static final long SHUTDOWN_TIMEOUT_MS = 2000;
+ private static final long LOG_FETCH_TIMEOUT_MS = 5000;
+
+ private final Queue<LogHandler> logHandlers;
+ private final KafkaClient kafkaClient;
+ private final DiscoveryServiceClient discoveryServiceClient;
+ private final LogPollerThread logPoller;
+
+ public AbstractTwillController(RunId runId, ZKClient zkClient, Iterable<LogHandler> logHandlers) {
+ super(runId, zkClient);
+ this.logHandlers = new ConcurrentLinkedQueue<LogHandler>();
+ this.kafkaClient = new SimpleKafkaClient(ZKClients.namespace(zkClient, "/" + runId.getId() + "/kafka"));
+ this.discoveryServiceClient = new ZKDiscoveryService(zkClient);
+ Iterables.addAll(this.logHandlers, logHandlers);
+ this.logPoller = new LogPollerThread(runId, kafkaClient, logHandlers);
+ }
+
+ @Override
+ protected void doStartUp() {
+ if (!logHandlers.isEmpty()) {
+ logPoller.start();
+ }
+ }
+
+ @Override
+ protected void doShutDown() {
+ logPoller.terminate();
+ try {
+ // Wait for the poller thread to stop.
+ logPoller.join(SHUTDOWN_TIMEOUT_MS);
+ } catch (InterruptedException e) {
+ LOG.warn("Joining of log poller thread interrupted.", e);
+ }
+ }
+
+ @Override
+ public final synchronized void addLogHandler(LogHandler handler) {
+ logHandlers.add(handler);
+ if (!logPoller.isAlive()) {
+ logPoller.start();
+ }
+ }
+
+ @Override
+ public final Iterable<Discoverable> discoverService(String serviceName) {
+ return discoveryServiceClient.discover(serviceName);
+ }
+
+ @Override
+ public final ListenableFuture<Integer> changeInstances(String runnable, int newCount) {
+ return sendMessage(SystemMessages.setInstances(runnable, newCount), newCount);
+ }
+
+ private static final class LogPollerThread extends Thread {
+
+ private final KafkaClient kafkaClient;
+ private final Iterable<LogHandler> logHandlers;
+ private volatile boolean running = true;
+
+ LogPollerThread(RunId runId, KafkaClient kafkaClient, Iterable<LogHandler> logHandlers) {
+ super("twill-log-poller-" + runId.getId());
+ setDaemon(true);
+ this.kafkaClient = kafkaClient;
+ this.logHandlers = logHandlers;
+ }
+
+ @Override
+ public void run() {
+ LOG.info("Twill log poller thread '{}' started.", getName());
+ kafkaClient.startAndWait();
+ Gson gson = new GsonBuilder().registerTypeAdapter(LogEntry.class, new LogEntryDecoder())
+ .registerTypeAdapter(StackTraceElement.class, new StackTraceElementCodec())
+ .create();
+
+ while (running && !isInterrupted()) {
+ long offset;
+ try {
+ // Get the earliest offset
+ long[] offsets = kafkaClient.getOffset(Constants.LOG_TOPIC, 0, -2, 1).get(LOG_FETCH_TIMEOUT_MS,
+ TimeUnit.MILLISECONDS);
+ // Should have one entry
+ offset = offsets[0];
+ } catch (Throwable t) {
+ // Keep retrying
+ LOG.warn("Failed to fetch offsets from Kafka. Retrying.", t);
+ continue;
+ }
+
+ // Now fetch log messages from Kafka
+ Iterator<FetchedMessage> messageIterator = kafkaClient.consume(Constants.LOG_TOPIC, 0,
+ offset, MAX_KAFKA_FETCH_SIZE);
+ try {
+ while (messageIterator.hasNext()) {
+ String json = Charsets.UTF_8.decode(messageIterator.next().getBuffer()).toString();
+ try {
+ LogEntry entry = gson.fromJson(json, LogEntry.class);
+ if (entry != null) {
+ invokeHandlers(entry);
+ }
+ } catch (Exception e) {
+ LOG.error("Failed to decode log entry {}", json, e);
+ }
+ }
+ } catch (Throwable t) {
+ LOG.warn("Exception while fetching log message from Kafka. Retrying.", t);
+ continue;
+ }
+ }
+
+ kafkaClient.stopAndWait();
+ LOG.info("Twill log poller thread stopped.");
+ }
+
+ void terminate() {
+ running = false;
+ interrupt();
+ }
+
+ private void invokeHandlers(LogEntry entry) {
+ for (LogHandler handler : logHandlers) {
+ handler.onLog(entry);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/AbstractZKServiceController.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/AbstractZKServiceController.java b/core/src/main/java/org/apache/twill/internal/AbstractZKServiceController.java
new file mode 100644
index 0000000..98cc2b8
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/AbstractZKServiceController.java
@@ -0,0 +1,314 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal;
+
+import org.apache.twill.api.Command;
+import org.apache.twill.api.RunId;
+import org.apache.twill.api.ServiceController;
+import org.apache.twill.common.Threads;
+import org.apache.twill.internal.json.StackTraceElementCodec;
+import org.apache.twill.internal.json.StateNodeCodec;
+import org.apache.twill.internal.state.Message;
+import org.apache.twill.internal.state.Messages;
+import org.apache.twill.internal.state.StateNode;
+import org.apache.twill.internal.state.SystemMessages;
+import org.apache.twill.zookeeper.NodeData;
+import org.apache.twill.zookeeper.ZKClient;
+import com.google.common.base.Charsets;
+import com.google.common.collect.Lists;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.gson.GsonBuilder;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * An abstract base class for implementing a {@link ServiceController} using ZooKeeper as a means for
+ * communicating with the remote service. This is designed to work in pair with the {@link ZKServiceDecorator}.
+ */
+public abstract class AbstractZKServiceController extends AbstractExecutionServiceController {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AbstractZKServiceController.class);
+
+ private final ZKClient zkClient;
+ private final InstanceNodeDataCallback instanceNodeDataCallback;
+ private final StateNodeDataCallback stateNodeDataCallback;
+ private final List<ListenableFuture<?>> messageFutures;
+ private ListenableFuture<State> stopMessageFuture;
+
+ protected AbstractZKServiceController(RunId runId, ZKClient zkClient) {
+ super(runId);
+ this.zkClient = zkClient;
+ this.instanceNodeDataCallback = new InstanceNodeDataCallback();
+ this.stateNodeDataCallback = new StateNodeDataCallback();
+ this.messageFutures = Lists.newLinkedList();
+ }
+
+ @Override
+ public final ListenableFuture<Command> sendCommand(Command command) {
+ return sendMessage(Messages.createForAll(command), command);
+ }
+
+ @Override
+ public final ListenableFuture<Command> sendCommand(String runnableName, Command command) {
+ return sendMessage(Messages.createForRunnable(runnableName, command), command);
+ }
+
+ @Override
+ protected final void startUp() {
+ // Watch for instance node existence.
+ actOnExists(getInstancePath(), new Runnable() {
+ @Override
+ public void run() {
+ watchInstanceNode();
+ }
+ });
+
+ // Watch for state node data
+ actOnExists(getZKPath("state"), new Runnable() {
+ @Override
+ public void run() {
+ watchStateNode();
+ }
+ });
+
+ doStartUp();
+ }
+
+ @Override
+ protected final synchronized void shutDown() {
+ if (stopMessageFuture == null) {
+ stopMessageFuture = ZKMessages.sendMessage(zkClient, getMessagePrefix(),
+ SystemMessages.stopApplication(), State.TERMINATED);
+ }
+
+ // Cancel all pending message futures.
+ for (ListenableFuture<?> future : messageFutures) {
+ future.cancel(true);
+ }
+
+ doShutDown();
+ }
+
+ /**
+ * Sends a {@link Message} to the remote service. Returns a future that will be completed when the message
+ * has been processed.
+ * @param message The message to send.
+ * @param result Object to set into the future when message is being processed.
+ * @param <V> Type of the result.
+ * @return A {@link ListenableFuture} that will be completed when the message has been processed.
+ */
+ protected final synchronized <V> ListenableFuture<V> sendMessage(Message message, V result) {
+ if (!isRunning()) {
+ return Futures.immediateFailedFuture(new IllegalStateException("Cannot send message to non-running application"));
+ }
+ final ListenableFuture<V> messageFuture = ZKMessages.sendMessage(zkClient, getMessagePrefix(), message, result);
+ messageFutures.add(messageFuture);
+ messageFuture.addListener(new Runnable() {
+ @Override
+ public void run() {
+ // If the completion is triggered when stopping, do nothing.
+ if (state() == State.STOPPING) {
+ return;
+ }
+ synchronized (AbstractZKServiceController.this) {
+ messageFutures.remove(messageFuture);
+ }
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+
+ return messageFuture;
+ }
+
+ protected final ListenableFuture<State> getStopMessageFuture() {
+ return stopMessageFuture;
+ }
+
+ /**
+ * Called during startup. Executed in the startup thread.
+ */
+ protected abstract void doStartUp();
+
+ /**
+ * Called during shutdown. Executed in the shutdown thread.
+ */
+ protected abstract void doShutDown();
+
+ /**
+ * Called when an update on the live instance node is detected.
+ * @param nodeData The updated live instance node data or {@code null} if there is an error when fetching
+ * the node data.
+ */
+ protected abstract void instanceNodeUpdated(NodeData nodeData);
+
+ /**
+ * Called when an update on the state node is detected.
+ * @param stateNode The update state node data or {@code null} if there is an error when fetching the node data.
+ */
+ protected abstract void stateNodeUpdated(StateNode stateNode);
+
+ protected synchronized void forceShutDown() {
+ if (stopMessageFuture == null) {
+ // In force shutdown, don't send message.
+ stopMessageFuture = Futures.immediateFuture(State.TERMINATED);
+ }
+ stop();
+ }
+
+
+ private void actOnExists(final String path, final Runnable action) {
+ // Watch for node existence.
+ final AtomicBoolean nodeExists = new AtomicBoolean(false);
+ Futures.addCallback(zkClient.exists(path, new Watcher() {
+ @Override
+ public void process(WatchedEvent event) {
+ // When node is created, call the action.
+ // Other event type would be handled by the action.
+ if (event.getType() == Event.EventType.NodeCreated && nodeExists.compareAndSet(false, true)) {
+ action.run();
+ }
+ }
+ }), new FutureCallback<Stat>() {
+ @Override
+ public void onSuccess(Stat result) {
+ if (result != null && nodeExists.compareAndSet(false, true)) {
+ action.run();
+ }
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ LOG.error("Failed in exists call to {}. Shutting down service.", path, t);
+ forceShutDown();
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+ }
+
+ private void watchInstanceNode() {
+ Futures.addCallback(zkClient.getData(getInstancePath(), new Watcher() {
+ @Override
+ public void process(WatchedEvent event) {
+ State state = state();
+ if (state != State.NEW && state != State.STARTING && state != State.RUNNING) {
+ // Ignore ZK node events when it is in stopping sequence.
+ return;
+ }
+ switch (event.getType()) {
+ case NodeDataChanged:
+ watchInstanceNode();
+ break;
+ case NodeDeleted:
+ // When the ephemeral node goes away, treat the remote service stopped.
+ forceShutDown();
+ break;
+ default:
+ LOG.info("Ignore ZK event for instance node: {}", event);
+ }
+ }
+ }), instanceNodeDataCallback, Threads.SAME_THREAD_EXECUTOR);
+ }
+
+ private void watchStateNode() {
+ Futures.addCallback(zkClient.getData(getZKPath("state"), new Watcher() {
+ @Override
+ public void process(WatchedEvent event) {
+ State state = state();
+ if (state != State.NEW && state != State.STARTING && state != State.RUNNING) {
+ // Ignore ZK node events when it is in stopping sequence.
+ return;
+ }
+ switch (event.getType()) {
+ case NodeDataChanged:
+ watchStateNode();
+ break;
+ default:
+ LOG.info("Ignore ZK event for state node: {}", event);
+ }
+ }
+ }), stateNodeDataCallback, Threads.SAME_THREAD_EXECUTOR);
+ }
+
+ /**
+ * Returns the path prefix for creating sequential message node for the remote service.
+ */
+ private String getMessagePrefix() {
+ return getZKPath("messages/msg");
+ }
+
+ /**
+ * Returns the zookeeper node path for the ephemeral instance node for this runId.
+ */
+ private String getInstancePath() {
+ return String.format("/instances/%s", getRunId().getId());
+ }
+
+ private String getZKPath(String path) {
+ return String.format("/%s/%s", getRunId().getId(), path);
+ }
+
+ private final class InstanceNodeDataCallback implements FutureCallback<NodeData> {
+
+ @Override
+ public void onSuccess(NodeData result) {
+ instanceNodeUpdated(result);
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ LOG.error("Failed in fetching instance node data.", t);
+ if (t instanceof KeeperException && ((KeeperException) t).code() == KeeperException.Code.NONODE) {
+ // If the node is gone, treat the remote service stopped.
+ forceShutDown();
+ } else {
+ instanceNodeUpdated(null);
+ }
+ }
+ }
+
+ private final class StateNodeDataCallback implements FutureCallback<NodeData> {
+
+ @Override
+ public void onSuccess(NodeData result) {
+ byte[] data = result.getData();
+ if (data == null) {
+ stateNodeUpdated(null);
+ return;
+ }
+ StateNode stateNode = new GsonBuilder().registerTypeAdapter(StateNode.class, new StateNodeCodec())
+ .registerTypeAdapter(StackTraceElement.class, new StackTraceElementCodec())
+ .create()
+ .fromJson(new String(data, Charsets.UTF_8), StateNode.class);
+
+ stateNodeUpdated(stateNode);
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ LOG.error("Failed in fetching state node data.", t);
+ stateNodeUpdated(null);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/ApplicationBundler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/ApplicationBundler.java b/core/src/main/java/org/apache/twill/internal/ApplicationBundler.java
new file mode 100644
index 0000000..b1a1e93
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/ApplicationBundler.java
@@ -0,0 +1,339 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal;
+
+import org.apache.twill.filesystem.Location;
+import org.apache.twill.internal.utils.Dependencies;
+import com.google.common.base.Function;
+import com.google.common.base.Splitter;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.common.io.ByteStreams;
+import com.google.common.io.Files;
+
+import java.io.BufferedOutputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.net.URL;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Queue;
+import java.util.Set;
+import java.util.jar.JarEntry;
+import java.util.jar.JarOutputStream;
+import java.util.zip.CRC32;
+import java.util.zip.CheckedOutputStream;
+
+/**
+ * This class builds jar files based on class dependencies.
+ */
+public final class ApplicationBundler {
+
+ private final List<String> excludePackages;
+ private final List<String> includePackages;
+ private final Set<String> bootstrapClassPaths;
+ private final CRC32 crc32;
+
+ /**
+ * Constructs a ApplicationBundler.
+ *
+ * @param excludePackages Class packages to exclude
+ */
+ public ApplicationBundler(Iterable<String> excludePackages) {
+ this(excludePackages, ImmutableList.<String>of());
+ }
+
+ /**
+ * Constructs a ApplicationBundler.
+ *
+ * @param excludePackages Class packages to exclude
+ * @param includePackages Class packages that should be included. Anything in this list will override the
+ * one provided in excludePackages.
+ */
+ public ApplicationBundler(Iterable<String> excludePackages, Iterable<String> includePackages) {
+ this.excludePackages = ImmutableList.copyOf(excludePackages);
+ this.includePackages = ImmutableList.copyOf(includePackages);
+
+ ImmutableSet.Builder<String> builder = ImmutableSet.builder();
+ for (String classpath : Splitter.on(File.pathSeparatorChar).split(System.getProperty("sun.boot.class.path"))) {
+ File file = new File(classpath);
+ builder.add(file.getAbsolutePath());
+ try {
+ builder.add(file.getCanonicalPath());
+ } catch (IOException e) {
+ // Ignore the exception and proceed.
+ }
+ }
+ this.bootstrapClassPaths = builder.build();
+ this.crc32 = new CRC32();
+
+ }
+
+ public void createBundle(Location target, Iterable<Class<?>> classes) throws IOException {
+ createBundle(target, classes, ImmutableList.<URI>of());
+ }
+
+ /**
+ * Same as calling {@link #createBundle(Location, Iterable)}.
+ */
+ public void createBundle(Location target, Class<?> clz, Class<?>...classes) throws IOException {
+ createBundle(target, ImmutableSet.<Class<?>>builder().add(clz).add(classes).build());
+ }
+
+ /**
+ * Creates a jar file which includes all the given classes and all the classes that they depended on.
+ * The jar will also include all classes and resources under the packages as given as include packages
+ * in the constructor.
+ *
+ * @param target Where to save the target jar file.
+ * @param resources Extra resources to put into the jar file. If resource is a jar file, it'll be put under
+ * lib/ entry, otherwise under the resources/ entry.
+ * @param classes Set of classes to start the dependency traversal.
+ * @throws IOException
+ */
+ public void createBundle(Location target, Iterable<Class<?>> classes, Iterable<URI> resources) throws IOException {
+ // Write the jar to local tmp file first
+ File tmpJar = File.createTempFile(target.getName(), ".tmp");
+ try {
+ Set<String> entries = Sets.newHashSet();
+ JarOutputStream jarOut = new JarOutputStream(new FileOutputStream(tmpJar));
+ try {
+ // Find class dependencies
+ findDependencies(classes, entries, jarOut);
+
+ // Add extra resources
+ for (URI resource : resources) {
+ copyResource(resource, entries, jarOut);
+ }
+ } finally {
+ jarOut.close();
+ }
+ // Copy the tmp jar into destination.
+ OutputStream os = new BufferedOutputStream(target.getOutputStream());
+ try {
+ Files.copy(tmpJar, os);
+ } finally {
+ os.close();
+ }
+ } finally {
+ tmpJar.delete();
+ }
+ }
+
+ private void findDependencies(Iterable<Class<?>> classes, final Set<String> entries,
+ final JarOutputStream jarOut) throws IOException {
+
+ Iterable<String> classNames = Iterables.transform(classes, new Function<Class<?>, String>() {
+ @Override
+ public String apply(Class<?> input) {
+ return input.getName();
+ }
+ });
+
+ ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+ if (classLoader == null) {
+ classLoader = getClass().getClassLoader();
+ }
+ Dependencies.findClassDependencies(classLoader, new Dependencies.ClassAcceptor() {
+ @Override
+ public boolean accept(String className, URL classUrl, URL classPathUrl) {
+ if (bootstrapClassPaths.contains(classPathUrl.getFile())) {
+ return false;
+ }
+
+ boolean shouldInclude = false;
+ for (String include : includePackages) {
+ if (className.startsWith(include)) {
+ shouldInclude = true;
+ break;
+ }
+ }
+
+ if (!shouldInclude) {
+ for (String exclude : excludePackages) {
+ if (className.startsWith(exclude)) {
+ return false;
+ }
+ }
+ }
+
+ putEntry(className, classUrl, classPathUrl, entries, jarOut);
+ return true;
+ }
+ }, classNames);
+ }
+
+ private void putEntry(String className, URL classUrl, URL classPathUrl, Set<String> entries, JarOutputStream jarOut) {
+ String classPath = classPathUrl.getFile();
+ if (classPath.endsWith(".jar")) {
+ saveDirEntry("lib/", entries, jarOut);
+ saveEntry("lib/" + classPath.substring(classPath.lastIndexOf('/') + 1), classPathUrl, entries, jarOut, false);
+ } else {
+ // Class file, put it under the classes directory
+ saveDirEntry("classes/", entries, jarOut);
+ if ("file".equals(classPathUrl.getProtocol())) {
+ // Copy every files under the classPath
+ try {
+ copyDir(new File(classPathUrl.toURI()), "classes/", entries, jarOut);
+ } catch (Exception e) {
+ throw Throwables.propagate(e);
+ }
+ } else {
+ String entry = "classes/" + className.replace('.', '/') + ".class";
+ saveDirEntry(entry.substring(0, entry.lastIndexOf('/') + 1), entries, jarOut);
+ saveEntry(entry, classUrl, entries, jarOut, true);
+ }
+ }
+ }
+
+ /**
+ * Saves a directory entry to the jar output.
+ */
+ private void saveDirEntry(String path, Set<String> entries, JarOutputStream jarOut) {
+ if (entries.contains(path)) {
+ return;
+ }
+
+ try {
+ String entry = "";
+ for (String dir : Splitter.on('/').omitEmptyStrings().split(path)) {
+ entry += dir + '/';
+ if (entries.add(entry)) {
+ JarEntry jarEntry = new JarEntry(entry);
+ jarEntry.setMethod(JarOutputStream.STORED);
+ jarEntry.setSize(0L);
+ jarEntry.setCrc(0L);
+ jarOut.putNextEntry(jarEntry);
+ jarOut.closeEntry();
+ }
+ }
+ } catch (IOException e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ /**
+ * Saves a class entry to the jar output.
+ */
+ private void saveEntry(String entry, URL url, Set<String> entries, JarOutputStream jarOut, boolean compress) {
+ if (!entries.add(entry)) {
+ return;
+ }
+ try {
+ JarEntry jarEntry = new JarEntry(entry);
+ InputStream is = url.openStream();
+
+ try {
+ if (compress) {
+ jarOut.putNextEntry(jarEntry);
+ ByteStreams.copy(is, jarOut);
+ } else {
+ crc32.reset();
+ TransferByteOutputStream os = new TransferByteOutputStream();
+ CheckedOutputStream checkedOut = new CheckedOutputStream(os, crc32);
+ ByteStreams.copy(is, checkedOut);
+ checkedOut.close();
+
+ long size = os.size();
+ jarEntry.setMethod(JarEntry.STORED);
+ jarEntry.setSize(size);
+ jarEntry.setCrc(checkedOut.getChecksum().getValue());
+ jarOut.putNextEntry(jarEntry);
+ os.transfer(jarOut);
+ }
+ } finally {
+ is.close();
+ }
+ jarOut.closeEntry();
+ } catch (Exception e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+
+ /**
+ * Copies all entries under the file path.
+ */
+ private void copyDir(File baseDir, String entryPrefix,
+ Set<String> entries, JarOutputStream jarOut) throws IOException {
+ URI baseUri = baseDir.toURI();
+ Queue<File> queue = Lists.newLinkedList();
+ Collections.addAll(queue, baseDir.listFiles());
+ while (!queue.isEmpty()) {
+ File file = queue.remove();
+
+ String entry = entryPrefix + baseUri.relativize(file.toURI()).getPath();
+ if (entries.add(entry)) {
+ jarOut.putNextEntry(new JarEntry(entry));
+ if (file.isFile()) {
+ Files.copy(file, jarOut);
+ }
+ jarOut.closeEntry();
+ }
+
+ if (file.isDirectory()) {
+ File[] files = file.listFiles();
+ if (files != null) {
+ queue.addAll(Arrays.asList(files));
+ }
+ }
+ }
+ }
+
+ private void copyResource(URI resource, Set<String> entries, JarOutputStream jarOut) throws IOException {
+ if ("file".equals(resource.getScheme())) {
+ File file = new File(resource);
+ if (file.isDirectory()) {
+ saveDirEntry("resources/", entries, jarOut);
+ copyDir(file, "resources/", entries, jarOut);
+ return;
+ }
+ }
+
+ URL url = resource.toURL();
+ String path = url.getFile();
+ String prefix = path.endsWith(".jar") ? "lib/" : "resources/";
+ path = prefix + path.substring(path.lastIndexOf('/') + 1);
+
+ saveDirEntry(prefix, entries, jarOut);
+ jarOut.putNextEntry(new JarEntry(path));
+ InputStream is = url.openStream();
+ try {
+ ByteStreams.copy(is, jarOut);
+ } finally {
+ is.close();
+ }
+ }
+
+ private static final class TransferByteOutputStream extends ByteArrayOutputStream {
+
+ public void transfer(OutputStream os) throws IOException {
+ os.write(buf, 0, count);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/Arguments.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/Arguments.java b/core/src/main/java/org/apache/twill/internal/Arguments.java
new file mode 100644
index 0000000..a78547c
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/Arguments.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMultimap;
+import com.google.common.collect.Multimap;
+
+import java.util.List;
+
+/**
+ * Class that encapsulate application arguments and per runnable arguments.
+ */
+public final class Arguments {
+
+ private final List<String> arguments;
+ private final Multimap<String, String> runnableArguments;
+
+ public Arguments(List<String> arguments, Multimap<String, String> runnableArguments) {
+ this.arguments = ImmutableList.copyOf(arguments);
+ this.runnableArguments = ImmutableMultimap.copyOf(runnableArguments);
+ }
+
+ public List<String> getArguments() {
+ return arguments;
+ }
+
+ public Multimap<String, String> getRunnableArguments() {
+ return runnableArguments;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/BasicTwillContext.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/BasicTwillContext.java b/core/src/main/java/org/apache/twill/internal/BasicTwillContext.java
new file mode 100644
index 0000000..61bdaef
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/BasicTwillContext.java
@@ -0,0 +1,131 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal;
+
+import org.apache.twill.api.RunId;
+import org.apache.twill.api.TwillContext;
+import org.apache.twill.api.TwillRunnableSpecification;
+import org.apache.twill.common.Cancellable;
+import org.apache.twill.discovery.Discoverable;
+import org.apache.twill.discovery.DiscoveryService;
+
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+
+/**
+ *
+ */
+public final class BasicTwillContext implements TwillContext {
+
+ private final RunId runId;
+ private final RunId appRunId;
+ private final InetAddress host;
+ private final String[] args;
+ private final String[] appArgs;
+ private final TwillRunnableSpecification spec;
+ private final int instanceId;
+ private final DiscoveryService discoveryService;
+ private final int allowedMemoryMB;
+ private final int virtualCores;
+ private volatile int instanceCount;
+
+ public BasicTwillContext(RunId runId, RunId appRunId, InetAddress host, String[] args, String[] appArgs,
+ TwillRunnableSpecification spec, int instanceId, DiscoveryService discoveryService,
+ int instanceCount, int allowedMemoryMB, int virtualCores) {
+ this.runId = runId;
+ this.appRunId = appRunId;
+ this.host = host;
+ this.args = args;
+ this.appArgs = appArgs;
+ this.spec = spec;
+ this.instanceId = instanceId;
+ this.discoveryService = discoveryService;
+ this.instanceCount = instanceCount;
+ this.allowedMemoryMB = allowedMemoryMB;
+ this.virtualCores = virtualCores;
+ }
+
+ @Override
+ public RunId getRunId() {
+ return runId;
+ }
+
+ @Override
+ public RunId getApplicationRunId() {
+ return appRunId;
+ }
+
+ @Override
+ public int getInstanceCount() {
+ return instanceCount;
+ }
+
+ public void setInstanceCount(int count) {
+ this.instanceCount = count;
+ }
+
+ @Override
+ public InetAddress getHost() {
+ return host;
+ }
+
+ @Override
+ public String[] getArguments() {
+ return args;
+ }
+
+ @Override
+ public String[] getApplicationArguments() {
+ return appArgs;
+ }
+
+ @Override
+ public TwillRunnableSpecification getSpecification() {
+ return spec;
+ }
+
+ @Override
+ public int getInstanceId() {
+ return instanceId;
+ }
+
+ @Override
+ public int getVirtualCores() {
+ return virtualCores;
+ }
+
+ @Override
+ public int getMaxMemoryMB() {
+ return allowedMemoryMB;
+ }
+
+ @Override
+ public Cancellable announce(final String serviceName, final int port) {
+ return discoveryService.register(new Discoverable() {
+ @Override
+ public String getName() {
+ return serviceName;
+ }
+
+ @Override
+ public InetSocketAddress getSocketAddress() {
+ return new InetSocketAddress(getHost(), port);
+ }
+ });
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/Configs.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/Configs.java b/core/src/main/java/org/apache/twill/internal/Configs.java
new file mode 100644
index 0000000..0fa1df8
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/Configs.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal;
+
+/**
+ *
+ */
+public final class Configs {
+
+ public static final class Keys {
+ /**
+ * Size in MB of reserved memory for Java process (non-heap memory).
+ */
+ public static final String JAVA_RESERVED_MEMORY_MB = "twill.java.reserved.memory.mb";
+
+ private Keys() {
+ }
+ }
+
+ public static final class Defaults {
+ // By default have 200MB reserved for Java process.
+ public static final int JAVA_RESERVED_MEMORY_MB = 200;
+
+ private Defaults() {
+ }
+ }
+
+ private Configs() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/Constants.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/Constants.java b/core/src/main/java/org/apache/twill/internal/Constants.java
new file mode 100644
index 0000000..0387d3e
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/Constants.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal;
+
+/**
+ * This class contains collection of common constants used in Twill.
+ */
+public final class Constants {
+
+ public static final String LOG_TOPIC = "log";
+
+ /** Maximum number of seconds for AM to start. */
+ public static final int APPLICATION_MAX_START_SECONDS = 60;
+ /** Maximum number of seconds for AM to stop. */
+ public static final int APPLICATION_MAX_STOP_SECONDS = 60;
+
+ public static final long PROVISION_TIMEOUT = 30000;
+
+ /** Memory size of AM */
+ public static final int APP_MASTER_MEMORY_MB = 512;
+
+ public static final int APP_MASTER_RESERVED_MEMORY_MB = 150;
+
+ public static final String STDOUT = "stdout";
+ public static final String STDERR = "stderr";
+
+ /**
+ * Constants for names of internal files that are shared between client, AM and containers.
+ */
+ public static final class Files {
+
+ public static final String LAUNCHER_JAR = "launcher.jar";
+ public static final String APP_MASTER_JAR = "appMaster.jar";
+ public static final String CONTAINER_JAR = "container.jar";
+ public static final String LOCALIZE_FILES = "localizeFiles.json";
+ public static final String TWILL_SPEC = "twillSpec.json";
+ public static final String ARGUMENTS = "arguments.json";
+ public static final String LOGBACK_TEMPLATE = "logback-template.xml";
+ public static final String KAFKA = "kafka.tgz";
+ public static final String JVM_OPTIONS = "jvm.opts";
+ public static final String CREDENTIALS = "credentials.store";
+
+ private Files() {
+ }
+ }
+
+ private Constants() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/ContainerInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/ContainerInfo.java b/core/src/main/java/org/apache/twill/internal/ContainerInfo.java
new file mode 100644
index 0000000..67c21d3
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/ContainerInfo.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal;
+
+import java.net.InetAddress;
+
+/**
+ * Represents information of the container that the processing is/will be running in.
+ */
+public interface ContainerInfo {
+
+ String getId();
+
+ InetAddress getHost();
+
+ int getPort();
+
+ int getMemoryMB();
+
+ int getVirtualCores();
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/ContainerLiveNodeData.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/ContainerLiveNodeData.java b/core/src/main/java/org/apache/twill/internal/ContainerLiveNodeData.java
new file mode 100644
index 0000000..705943c
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/ContainerLiveNodeData.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal;
+
+/**
+ *
+ */
+public final class ContainerLiveNodeData {
+
+ private final String containerId;
+ private final String host;
+
+ public ContainerLiveNodeData(String containerId, String host) {
+ this.containerId = containerId;
+ this.host = host;
+ }
+
+ public String getContainerId() {
+ return containerId;
+ }
+
+ public String getHost() {
+ return host;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/EnvContainerInfo.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/EnvContainerInfo.java b/core/src/main/java/org/apache/twill/internal/EnvContainerInfo.java
new file mode 100644
index 0000000..fd50028
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/EnvContainerInfo.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
+/**
+ * A {@link ContainerInfo} based on information on the environment.
+ */
+public final class EnvContainerInfo implements ContainerInfo {
+ private final String id;
+ private final InetAddress host;
+ private final int port;
+ private final int virtualCores;
+ private final int memoryMB;
+
+ public EnvContainerInfo() throws UnknownHostException {
+ id = System.getenv(EnvKeys.YARN_CONTAINER_ID);
+ host = InetAddress.getByName(System.getenv(EnvKeys.YARN_CONTAINER_HOST));
+ port = Integer.parseInt(System.getenv(EnvKeys.YARN_CONTAINER_PORT));
+ virtualCores = Integer.parseInt(System.getenv(EnvKeys.YARN_CONTAINER_VIRTUAL_CORES));
+ memoryMB = Integer.parseInt(System.getenv(EnvKeys.YARN_CONTAINER_MEMORY_MB));
+ }
+
+ @Override
+ public String getId() {
+ return id;
+ }
+
+ @Override
+ public InetAddress getHost() {
+ return host;
+ }
+
+ @Override
+ public int getPort() {
+ return port;
+ }
+
+ @Override
+ public int getMemoryMB() {
+ return memoryMB;
+ }
+
+ @Override
+ public int getVirtualCores() {
+ return virtualCores;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/EnvKeys.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/EnvKeys.java b/core/src/main/java/org/apache/twill/internal/EnvKeys.java
new file mode 100644
index 0000000..9bf6523
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/EnvKeys.java
@@ -0,0 +1,59 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal;
+
+/**
+ * Places for define common environment keys.
+ */
+public final class EnvKeys {
+
+ public static final String TWILL_ZK_CONNECT = "TWILL_ZK_CONNECT";
+ public static final String TWILL_APP_RUN_ID = "TWILL_APP_RUN_ID";
+ public static final String TWILL_RUN_ID = "TWILL_RUN_ID";
+ public static final String TWILL_INSTANCE_ID = "TWILL_INSTANCE_ID";
+ public static final String TWILL_INSTANCE_COUNT = "TWILL_INSTANCE_COUNT";
+ public static final String TWILL_RESERVED_MEMORY_MB = "TWILL_RESERVED_MEMORY_MB";
+
+ public static final String TWILL_FS_USER = "TWILL_FS_USER";
+
+ /**
+ * Cluster filesystem directory for storing twill app related files.
+ */
+ public static final String TWILL_APP_DIR = "TWILL_APP_DIR";
+
+ public static final String TWILL_APP_NAME = "TWILL_APP_NAME";
+ public static final String TWILL_RUNNABLE_NAME = "TWILL_RUNNABLE_NAME";
+
+ public static final String TWILL_LOG_KAFKA_ZK = "TWILL_LOG_KAFKA_ZK";
+
+ public static final String YARN_APP_ID = "YARN_APP_ID";
+ public static final String YARN_APP_ID_CLUSTER_TIME = "YARN_APP_ID_CLUSTER_TIME";
+ public static final String YARN_APP_ID_STR = "YARN_APP_ID_STR";
+
+ public static final String YARN_CONTAINER_ID = "YARN_CONTAINER_ID";
+ public static final String YARN_CONTAINER_HOST = "YARN_CONTAINER_HOST";
+ public static final String YARN_CONTAINER_PORT = "YARN_CONTAINER_PORT";
+ /**
+ * Used to inform runnables of their resource usage.
+ */
+ public static final String YARN_CONTAINER_VIRTUAL_CORES = "YARN_CONTAINER_VIRTUAL_CORES";
+ public static final String YARN_CONTAINER_MEMORY_MB = "YARN_CONTAINER_MEMORY_MB";
+
+ private EnvKeys() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/ListenerExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/ListenerExecutor.java b/core/src/main/java/org/apache/twill/internal/ListenerExecutor.java
new file mode 100644
index 0000000..9d3e156
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/ListenerExecutor.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal;
+
+import com.google.common.collect.Maps;
+import com.google.common.util.concurrent.Service;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.ConcurrentMap;
+import java.util.concurrent.Executor;
+
+/**
+ * Wrapper for {@link Service.Listener} to have callback executed on a given {@link Executor}.
+ * Also make sure each method is called at most once.
+ */
+final class ListenerExecutor implements Service.Listener {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ListenerExecutor.class);
+
+ private final Service.Listener delegate;
+ private final Executor executor;
+ private final ConcurrentMap<Service.State, Boolean> callStates = Maps.newConcurrentMap();
+
+ ListenerExecutor(Service.Listener delegate, Executor executor) {
+ this.delegate = delegate;
+ this.executor = executor;
+ }
+
+ @Override
+ public void starting() {
+ if (hasCalled(Service.State.STARTING)) {
+ return;
+ }
+ executor.execute(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ delegate.starting();
+ } catch (Throwable t) {
+ LOG.warn("Exception thrown from listener", t);
+ }
+ }
+ });
+ }
+
+ @Override
+ public void running() {
+ if (hasCalled(Service.State.RUNNING)) {
+ return;
+ }
+ executor.execute(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ delegate.running();
+ } catch (Throwable t) {
+ LOG.warn("Exception thrown from listener", t);
+ }
+ }
+ });
+ }
+
+ @Override
+ public void stopping(final Service.State from) {
+ if (hasCalled(Service.State.STOPPING)) {
+ return;
+ }
+ executor.execute(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ delegate.stopping(from);
+ } catch (Throwable t) {
+ LOG.warn("Exception thrown from listener", t);
+ }
+ }
+ });
+ }
+
+ @Override
+ public void terminated(final Service.State from) {
+ if (hasCalled(Service.State.TERMINATED)) {
+ return;
+ }
+ executor.execute(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ delegate.terminated(from);
+ } catch (Throwable t) {
+ LOG.warn("Exception thrown from listener", t);
+ }
+ }
+ });
+ }
+
+ @Override
+ public void failed(final Service.State from, final Throwable failure) {
+ // Both failed and terminate are using the same state for checking as only either one could be called.
+ if (hasCalled(Service.State.TERMINATED)) {
+ return;
+ }
+ executor.execute(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ delegate.failed(from, failure);
+ } catch (Throwable t) {
+ LOG.warn("Exception thrown from listener", t);
+ }
+ }
+ });
+ }
+
+ private boolean hasCalled(Service.State state) {
+ return callStates.putIfAbsent(state, true) != null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/LogOnlyEventHandler.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/LogOnlyEventHandler.java b/core/src/main/java/org/apache/twill/internal/LogOnlyEventHandler.java
new file mode 100644
index 0000000..4f71a05
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/LogOnlyEventHandler.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal;
+
+import org.apache.twill.api.EventHandler;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ *
+ */
+public final class LogOnlyEventHandler extends EventHandler {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LogOnlyEventHandler.class);
+
+ @Override
+ public TimeoutAction launchTimeout(Iterable<TimeoutEvent> timeoutEvents) {
+ for (TimeoutEvent event : timeoutEvents) {
+ LOG.info("Requested {} containers for runnable {}, only got {} after {} ms.",
+ event.getExpectedInstances(), event.getRunnableName(),
+ event.getActualInstances(), System.currentTimeMillis() - event.getRequestTime());
+ }
+ return TimeoutAction.recheck(Constants.PROVISION_TIMEOUT, TimeUnit.MILLISECONDS);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/ProcessController.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/ProcessController.java b/core/src/main/java/org/apache/twill/internal/ProcessController.java
new file mode 100644
index 0000000..4453838
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/ProcessController.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal;
+
+import org.apache.twill.common.Cancellable;
+
+/**
+ * For controlling a launch yarn process.
+ *
+ * @param <R> Report type.
+ */
+public interface ProcessController<R> extends Cancellable {
+
+ R getReport();
+
+ /**
+ * Request to stop the running process.
+ */
+ void cancel();
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/ProcessLauncher.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/ProcessLauncher.java b/core/src/main/java/org/apache/twill/internal/ProcessLauncher.java
new file mode 100644
index 0000000..e48a226
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/ProcessLauncher.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal;
+
+import org.apache.twill.api.LocalFile;
+
+import java.util.Map;
+
+/**
+ * Class for launching container process.
+ *
+ * @param <T> Type of the object that contains information about the container that the process is going to launch.
+ */
+public interface ProcessLauncher<T> {
+
+ /**
+ * Returns information about the container that this launch would launch process in.
+ */
+ T getContainerInfo();
+
+ /**
+ * Returns a preparer with the given default set of environments, resources and credentials.
+ */
+ <C> PrepareLaunchContext prepareLaunch(Map<String, String> environments,
+ Iterable<LocalFile> resources, C credentials);
+
+ /**
+ * For setting up the launcher.
+ */
+ interface PrepareLaunchContext {
+
+ ResourcesAdder withResources();
+
+ AfterResources noResources();
+
+ interface ResourcesAdder {
+ MoreResources add(LocalFile localFile);
+ }
+
+ interface AfterResources {
+ EnvironmentAdder withEnvironment();
+
+ AfterEnvironment noEnvironment();
+ }
+
+ interface EnvironmentAdder {
+ <V> MoreEnvironment add(String key, V value);
+ }
+
+ interface MoreEnvironment extends EnvironmentAdder, AfterEnvironment {
+ }
+
+ interface AfterEnvironment {
+ CommandAdder withCommands();
+ }
+
+ interface MoreResources extends ResourcesAdder, AfterResources { }
+
+ interface CommandAdder {
+ StdOutSetter add(String cmd, String...args);
+ }
+
+ interface StdOutSetter {
+ StdErrSetter redirectOutput(String stdout);
+
+ StdErrSetter noOutput();
+ }
+
+ interface StdErrSetter {
+ MoreCommand redirectError(String stderr);
+
+ MoreCommand noError();
+ }
+
+ interface MoreCommand extends CommandAdder {
+ <R> ProcessController<R> launch();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/SingleRunnableApplication.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/SingleRunnableApplication.java b/core/src/main/java/org/apache/twill/internal/SingleRunnableApplication.java
new file mode 100644
index 0000000..a52afe1
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/SingleRunnableApplication.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal;
+
+import org.apache.twill.api.ResourceSpecification;
+import org.apache.twill.api.TwillApplication;
+import org.apache.twill.api.TwillRunnable;
+import org.apache.twill.api.TwillRunnableSpecification;
+import org.apache.twill.api.TwillSpecification;
+
+/**
+ * A simple {@link org.apache.twill.api.TwillApplication} that contains only one {@link org.apache.twill.api.TwillRunnable}.
+ */
+public class SingleRunnableApplication implements TwillApplication {
+
+ private final TwillRunnable runnable;
+ private final ResourceSpecification resourceSpec;
+
+ public SingleRunnableApplication(TwillRunnable runnable, ResourceSpecification resourceSpec) {
+ this.runnable = runnable;
+ this.resourceSpec = resourceSpec;
+ }
+
+ @Override
+ public TwillSpecification configure() {
+ TwillRunnableSpecification runnableSpec = runnable.configure();
+ return TwillSpecification.Builder.with()
+ .setName(runnableSpec.getName())
+ .withRunnable().add(runnableSpec.getName(), runnable, resourceSpec)
+ .noLocalFiles()
+ .anyOrder()
+ .build();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/TwillContainerController.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/TwillContainerController.java b/core/src/main/java/org/apache/twill/internal/TwillContainerController.java
new file mode 100644
index 0000000..8b090bd
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/TwillContainerController.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal;
+
+import org.apache.twill.api.ServiceController;
+import org.apache.twill.internal.state.Message;
+import com.google.common.util.concurrent.ListenableFuture;
+
+/**
+ * A {@link ServiceController} that allows sending a message directly. Internal use only.
+ */
+public interface TwillContainerController extends ServiceController {
+
+ ListenableFuture<Message> sendMessage(Message message);
+
+ /**
+ * Calls to indicated that the container that this controller is associated with is completed.
+ * Any resources it hold will be releases and all pending futures will be cancelled.
+ */
+ void completed(int exitStatus);
+}
[09/15] Initial import commit.
Posted by ch...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/test/java/org/apache/twill/internal/utils/ApplicationBundlerTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/twill/internal/utils/ApplicationBundlerTest.java b/core/src/test/java/org/apache/twill/internal/utils/ApplicationBundlerTest.java
new file mode 100644
index 0000000..508cadb
--- /dev/null
+++ b/core/src/test/java/org/apache/twill/internal/utils/ApplicationBundlerTest.java
@@ -0,0 +1,113 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.utils;
+
+import org.apache.twill.filesystem.LocalLocationFactory;
+import org.apache.twill.filesystem.Location;
+import org.apache.twill.internal.ApplicationBundler;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.io.ByteStreams;
+import com.google.common.io.Files;
+import org.junit.Assert;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.List;
+import java.util.jar.JarEntry;
+import java.util.jar.JarInputStream;
+
+/**
+ *
+ */
+public class ApplicationBundlerTest {
+
+ @Rule
+ public TemporaryFolder tmpDir = new TemporaryFolder();
+
+ @Test
+ public void testFindDependencies() throws IOException, ClassNotFoundException {
+ Location location = new LocalLocationFactory(tmpDir.newFolder()).create("test.jar");
+
+ // Create a jar file with by tracing dependency
+ ApplicationBundler bundler = new ApplicationBundler(ImmutableList.<String>of());
+ bundler.createBundle(location, ApplicationBundler.class);
+
+ File targetDir = tmpDir.newFolder();
+ unjar(new File(location.toURI()), targetDir);
+
+ // Load the class back, it should be loaded by the custom classloader
+ ClassLoader classLoader = createClassLoader(targetDir);
+ Class<?> clz = classLoader.loadClass(ApplicationBundler.class.getName());
+ Assert.assertSame(classLoader, clz.getClassLoader());
+
+ // For system classes, they shouldn't be packaged, hence loaded by different classloader.
+ clz = classLoader.loadClass(Object.class.getName());
+ Assert.assertNotSame(classLoader, clz.getClassLoader());
+ }
+
+ private void unjar(File jarFile, File targetDir) throws IOException {
+ JarInputStream jarInput = new JarInputStream(new FileInputStream(jarFile));
+ try {
+ JarEntry jarEntry = jarInput.getNextJarEntry();
+ while (jarEntry != null) {
+ File target = new File(targetDir, jarEntry.getName());
+ if (jarEntry.isDirectory()) {
+ target.mkdirs();
+ } else {
+ target.getParentFile().mkdirs();
+ ByteStreams.copy(jarInput, Files.newOutputStreamSupplier(target));
+ }
+
+ jarEntry = jarInput.getNextJarEntry();
+ }
+ } finally {
+ jarInput.close();
+ }
+ }
+
+ private ClassLoader createClassLoader(File dir) throws MalformedURLException {
+ List<URL> urls = Lists.newArrayList();
+ urls.add(new File(dir, "classes").toURI().toURL());
+ File[] libFiles = new File(dir, "lib").listFiles();
+ if (libFiles != null) {
+ for (File file : libFiles) {
+ urls.add(file.toURI().toURL());
+ }
+ }
+ return new URLClassLoader(urls.toArray(new URL[0])) {
+ @Override
+ protected synchronized Class<?> loadClass(String name, boolean resolve) throws ClassNotFoundException {
+ // Load class from the given URLs first before delegating to parent.
+ try {
+ return super.findClass(name);
+ } catch (ClassNotFoundException e) {
+ ClassLoader parent = getParent();
+ return parent == null ? ClassLoader.getSystemClassLoader().loadClass(name) : parent.loadClass(name);
+ }
+ }
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/test/java/org/apache/twill/kafka/client/KafkaTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/twill/kafka/client/KafkaTest.java b/core/src/test/java/org/apache/twill/kafka/client/KafkaTest.java
new file mode 100644
index 0000000..40fc3ed
--- /dev/null
+++ b/core/src/test/java/org/apache/twill/kafka/client/KafkaTest.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.kafka.client;
+
+import org.apache.twill.common.Services;
+import org.apache.twill.internal.kafka.EmbeddedKafkaServer;
+import org.apache.twill.internal.kafka.client.Compression;
+import org.apache.twill.internal.kafka.client.SimpleKafkaClient;
+import org.apache.twill.internal.utils.Networks;
+import org.apache.twill.internal.zookeeper.InMemoryZKServer;
+import org.apache.twill.zookeeper.ZKClientService;
+import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
+import com.google.common.io.ByteStreams;
+import com.google.common.io.Files;
+import com.google.common.util.concurrent.Futures;
+import org.apache.commons.compress.archivers.ArchiveEntry;
+import org.apache.commons.compress.archivers.ArchiveException;
+import org.apache.commons.compress.archivers.ArchiveInputStream;
+import org.apache.commons.compress.archivers.ArchiveStreamFactory;
+import org.apache.commons.compress.compressors.CompressorException;
+import org.apache.commons.compress.compressors.CompressorStreamFactory;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Iterator;
+import java.util.Properties;
+import java.util.concurrent.TimeUnit;
+
+/**
+ *
+ */
+public class KafkaTest {
+
+ private static final Logger LOG = LoggerFactory.getLogger(KafkaTest.class);
+
+ @ClassRule
+ public static final TemporaryFolder TMP_FOLDER = new TemporaryFolder();
+
+ private static InMemoryZKServer zkServer;
+ private static EmbeddedKafkaServer kafkaServer;
+ private static ZKClientService zkClientService;
+ private static KafkaClient kafkaClient;
+
+ @BeforeClass
+ public static void init() throws Exception {
+ zkServer = InMemoryZKServer.builder().setDataDir(TMP_FOLDER.newFolder()).build();
+ zkServer.startAndWait();
+
+ // Extract the kafka.tgz and start the kafka server
+ kafkaServer = new EmbeddedKafkaServer(extractKafka(), generateKafkaConfig(zkServer.getConnectionStr()));
+ kafkaServer.startAndWait();
+
+ zkClientService = ZKClientService.Builder.of(zkServer.getConnectionStr()).build();
+
+ kafkaClient = new SimpleKafkaClient(zkClientService);
+ Services.chainStart(zkClientService, kafkaClient).get();
+ }
+
+ @AfterClass
+ public static void finish() throws Exception {
+ Services.chainStop(kafkaClient, zkClientService).get();
+ kafkaServer.stopAndWait();
+ zkServer.stopAndWait();
+ }
+
+ @Test
+ public void testKafkaClient() throws Exception {
+ String topic = "testClient";
+
+ Thread t1 = createPublishThread(kafkaClient, topic, Compression.GZIP, "GZIP Testing message", 10);
+ Thread t2 = createPublishThread(kafkaClient, topic, Compression.NONE, "Testing message", 10);
+
+ t1.start();
+ t2.start();
+
+ Thread t3 = createPublishThread(kafkaClient, topic, Compression.SNAPPY, "Snappy Testing message", 10);
+ t2.join();
+ t3.start();
+
+ Iterator<FetchedMessage> consumer = kafkaClient.consume(topic, 0, 0, 1048576);
+ int count = 0;
+ long startTime = System.nanoTime();
+ while (count < 30 && consumer.hasNext() && secondsPassed(startTime, TimeUnit.NANOSECONDS) < 5) {
+ LOG.info(Charsets.UTF_8.decode(consumer.next().getBuffer()).toString());
+ count++;
+ }
+
+ Assert.assertEquals(30, count);
+ }
+
+ @Test (timeout = 10000)
+ public void testOffset() throws Exception {
+ String topic = "testOffset";
+
+ // Initial earliest offset should be 0.
+ long[] offsets = kafkaClient.getOffset(topic, 0, -2, 10).get();
+ Assert.assertArrayEquals(new long[]{0L}, offsets);
+
+ // Publish some messages
+ Thread publishThread = createPublishThread(kafkaClient, topic, Compression.NONE, "Testing", 2000);
+ publishThread.start();
+ publishThread.join();
+
+ // Fetch earliest offset, should still be 0.
+ offsets = kafkaClient.getOffset(topic, 0, -2, 10).get();
+ Assert.assertArrayEquals(new long[]{0L}, offsets);
+
+ // Fetch latest offset
+ offsets = kafkaClient.getOffset(topic, 0, -1, 10).get();
+ Iterator<FetchedMessage> consumer = kafkaClient.consume(topic, 0, offsets[0], 1048576);
+
+ // Publish one more message, the consumer should see the new message being published.
+ publishThread = createPublishThread(kafkaClient, topic, Compression.NONE, "Testing", 1, 3000);
+ publishThread.start();
+ publishThread.join();
+
+ // Should see the last message being published.
+ Assert.assertTrue(consumer.hasNext());
+ Assert.assertEquals("3000 Testing", Charsets.UTF_8.decode(consumer.next().getBuffer()).toString());
+ }
+
+ private Thread createPublishThread(final KafkaClient kafkaClient, final String topic,
+ final Compression compression, final String message, final int count) {
+ return createPublishThread(kafkaClient, topic, compression, message, count, 0);
+ }
+
+ private Thread createPublishThread(final KafkaClient kafkaClient, final String topic, final Compression compression,
+ final String message, final int count, final int base) {
+ return new Thread() {
+ public void run() {
+ PreparePublish preparePublish = kafkaClient.preparePublish(topic, compression);
+ for (int i = 0; i < count; i++) {
+ preparePublish.add(((base + i) + " " + message).getBytes(Charsets.UTF_8), 0);
+ }
+ Futures.getUnchecked(preparePublish.publish());
+ }
+ };
+ }
+
+ private long secondsPassed(long startTime, TimeUnit startUnit) {
+ return TimeUnit.SECONDS.convert(System.nanoTime() - TimeUnit.NANOSECONDS.convert(startTime, startUnit),
+ TimeUnit.NANOSECONDS);
+ }
+
+ private static File extractKafka() throws IOException, ArchiveException, CompressorException {
+ File kafkaExtract = TMP_FOLDER.newFolder();
+ InputStream kakfaResource = KafkaTest.class.getClassLoader().getResourceAsStream("kafka-0.7.2.tgz");
+ ArchiveInputStream archiveInput = new ArchiveStreamFactory()
+ .createArchiveInputStream(ArchiveStreamFactory.TAR,
+ new CompressorStreamFactory()
+ .createCompressorInputStream(CompressorStreamFactory.GZIP, kakfaResource));
+
+ try {
+ ArchiveEntry entry = archiveInput.getNextEntry();
+ while (entry != null) {
+ File file = new File(kafkaExtract, entry.getName());
+ if (entry.isDirectory()) {
+ file.mkdirs();
+ } else {
+ ByteStreams.copy(archiveInput, Files.newOutputStreamSupplier(file));
+ }
+ entry = archiveInput.getNextEntry();
+ }
+ } finally {
+ archiveInput.close();
+ }
+ return kafkaExtract;
+ }
+
+ private static Properties generateKafkaConfig(String zkConnectStr) throws IOException {
+ int port = Networks.getRandomPort();
+ Preconditions.checkState(port > 0, "Failed to get random port.");
+
+ Properties prop = new Properties();
+ prop.setProperty("log.dir", TMP_FOLDER.newFolder().getAbsolutePath());
+ prop.setProperty("zk.connect", zkConnectStr);
+ prop.setProperty("num.threads", "8");
+ prop.setProperty("port", Integer.toString(port));
+ prop.setProperty("log.flush.interval", "1000");
+ prop.setProperty("max.socket.request.bytes", "104857600");
+ prop.setProperty("log.cleanup.interval.mins", "1");
+ prop.setProperty("log.default.flush.scheduler.interval.ms", "1000");
+ prop.setProperty("zk.connectiontimeout.ms", "1000000");
+ prop.setProperty("socket.receive.buffer", "1048576");
+ prop.setProperty("enable.zookeeper", "true");
+ prop.setProperty("log.retention.hours", "24");
+ prop.setProperty("brokerid", "0");
+ prop.setProperty("socket.send.buffer", "1048576");
+ prop.setProperty("num.partitions", "1");
+ // Use a really small file size to force some flush to happen
+ prop.setProperty("log.file.size", "1024");
+ prop.setProperty("log.default.flush.interval.ms", "1000");
+ return prop;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/test/resources/logback-test.xml
----------------------------------------------------------------------
diff --git a/core/src/test/resources/logback-test.xml b/core/src/test/resources/logback-test.xml
new file mode 100644
index 0000000..3c36660
--- /dev/null
+++ b/core/src/test/resources/logback-test.xml
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!-- Default logback configuration for twill library -->
+<configuration>
+ <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n</pattern>
+ </encoder>
+ </appender>
+
+ <logger name="org.apache.hadoop" level="WARN" />
+ <logger name="org.apache.zookeeper" level="WARN" />
+
+ <root level="INFO">
+ <appender-ref ref="STDOUT"/>
+ </root>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/discovery-api/pom.xml
----------------------------------------------------------------------
diff --git a/discovery-api/pom.xml b/discovery-api/pom.xml
new file mode 100644
index 0000000..400be4a
--- /dev/null
+++ b/discovery-api/pom.xml
@@ -0,0 +1,39 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>twill-parent</artifactId>
+ <groupId>org.apache.twill</groupId>
+ <version>1.3.0-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>twill-discovery-api</artifactId>
+ <name>Twill discovery service API</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>twill-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/discovery-api/src/main/java/org/apache/twill/discovery/Discoverable.java
----------------------------------------------------------------------
diff --git a/discovery-api/src/main/java/org/apache/twill/discovery/Discoverable.java b/discovery-api/src/main/java/org/apache/twill/discovery/Discoverable.java
new file mode 100644
index 0000000..a5529fe
--- /dev/null
+++ b/discovery-api/src/main/java/org/apache/twill/discovery/Discoverable.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.twill.discovery;
+
+import java.net.InetSocketAddress;
+
+/**
+ * Discoverable defines the attributes of service to be discovered.
+ */
+public interface Discoverable {
+
+ /**
+ * @return Name of the service
+ */
+ String getName();
+
+ /**
+ * @return An {@link InetSocketAddress} representing the host+port of the service.
+ */
+ InetSocketAddress getSocketAddress();
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/discovery-api/src/main/java/org/apache/twill/discovery/DiscoveryService.java
----------------------------------------------------------------------
diff --git a/discovery-api/src/main/java/org/apache/twill/discovery/DiscoveryService.java b/discovery-api/src/main/java/org/apache/twill/discovery/DiscoveryService.java
new file mode 100644
index 0000000..a26fff8
--- /dev/null
+++ b/discovery-api/src/main/java/org/apache/twill/discovery/DiscoveryService.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.discovery;
+
+
+import org.apache.twill.common.Cancellable;
+
+/**
+ * DiscoveryService defines interface for registering {@link Discoverable}.
+ */
+public interface DiscoveryService {
+
+ /**
+ * Registers a {@link Discoverable} service.
+ * @param discoverable Information of the service provider that could be discovered.
+ * @return A {@link Cancellable} for un-registration.
+ */
+ Cancellable register(Discoverable discoverable);
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/discovery-api/src/main/java/org/apache/twill/discovery/DiscoveryServiceClient.java
----------------------------------------------------------------------
diff --git a/discovery-api/src/main/java/org/apache/twill/discovery/DiscoveryServiceClient.java b/discovery-api/src/main/java/org/apache/twill/discovery/DiscoveryServiceClient.java
new file mode 100644
index 0000000..89cf269
--- /dev/null
+++ b/discovery-api/src/main/java/org/apache/twill/discovery/DiscoveryServiceClient.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.discovery;
+
+/**
+ * Interface for {@link DiscoveryServiceClient} to discover services registered with {@link DiscoveryService}.
+ */
+public interface DiscoveryServiceClient {
+
+ /**
+ * Retrieves a list of {@link Discoverable} for the a service with the given name.
+ *
+ * @param name Name of the service
+ * @return A live {@link Iterable} that on each call to {@link Iterable#iterator()} returns
+ * an {@link java.util.Iterator Iterator} that reflects the latest set of
+ * available {@link Discoverable} services.
+ */
+ Iterable<Discoverable> discover(String name);
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/discovery-core/pom.xml
----------------------------------------------------------------------
diff --git a/discovery-core/pom.xml b/discovery-core/pom.xml
new file mode 100644
index 0000000..cd258dc
--- /dev/null
+++ b/discovery-core/pom.xml
@@ -0,0 +1,52 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>twill-parent</artifactId>
+ <groupId>org.apache.twill</groupId>
+ <version>1.3.0-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>twill-discovery-core</artifactId>
+ <name>Twill discovery service implementations</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>twill-discovery-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>twill-zookeeper</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/discovery-core/src/main/java/org/apache/twill/discovery/DiscoverableWrapper.java
----------------------------------------------------------------------
diff --git a/discovery-core/src/main/java/org/apache/twill/discovery/DiscoverableWrapper.java b/discovery-core/src/main/java/org/apache/twill/discovery/DiscoverableWrapper.java
new file mode 100644
index 0000000..5fa97d1
--- /dev/null
+++ b/discovery-core/src/main/java/org/apache/twill/discovery/DiscoverableWrapper.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.discovery;
+
+import java.net.InetSocketAddress;
+
+/**
+ * Wrapper for a discoverable.
+ */
+final class DiscoverableWrapper implements Discoverable {
+ private final String name;
+ private final InetSocketAddress address;
+
+ DiscoverableWrapper(Discoverable discoverable) {
+ this.name = discoverable.getName();
+ this.address = discoverable.getSocketAddress();
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public InetSocketAddress getSocketAddress() {
+ return address;
+ }
+
+ @Override
+ public String toString() {
+ return "{name=" + name + ", address=" + address;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+
+ Discoverable other = (Discoverable) o;
+
+ return name.equals(other.getName()) && address.equals(other.getSocketAddress());
+ }
+
+ @Override
+ public int hashCode() {
+ int result = name.hashCode();
+ result = 31 * result + address.hashCode();
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/discovery-core/src/main/java/org/apache/twill/discovery/InMemoryDiscoveryService.java
----------------------------------------------------------------------
diff --git a/discovery-core/src/main/java/org/apache/twill/discovery/InMemoryDiscoveryService.java b/discovery-core/src/main/java/org/apache/twill/discovery/InMemoryDiscoveryService.java
new file mode 100644
index 0000000..7a9e984
--- /dev/null
+++ b/discovery-core/src/main/java/org/apache/twill/discovery/InMemoryDiscoveryService.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.discovery;
+
+import org.apache.twill.common.Cancellable;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Multimap;
+
+import java.util.Iterator;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * A simple in memory implementation of {@link DiscoveryService} and {@link DiscoveryServiceClient}.
+ */
+public class InMemoryDiscoveryService implements DiscoveryService, DiscoveryServiceClient {
+
+ private final Multimap<String, Discoverable> services = HashMultimap.create();
+ private final Lock lock = new ReentrantLock();
+
+ @Override
+ public Cancellable register(final Discoverable discoverable) {
+ lock.lock();
+ try {
+ final Discoverable wrapper = new DiscoverableWrapper(discoverable);
+ services.put(wrapper.getName(), wrapper);
+ return new Cancellable() {
+ @Override
+ public void cancel() {
+ lock.lock();
+ try {
+ services.remove(wrapper.getName(), wrapper);
+ } finally {
+ lock.unlock();
+ }
+ }
+ };
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ @Override
+ public Iterable<Discoverable> discover(final String name) {
+ return new Iterable<Discoverable>() {
+ @Override
+ public Iterator<Discoverable> iterator() {
+ lock.lock();
+ try {
+ return ImmutableList.copyOf(services.get(name)).iterator();
+ } finally {
+ lock.unlock();
+ }
+ }
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/discovery-core/src/main/java/org/apache/twill/discovery/ZKDiscoveryService.java
----------------------------------------------------------------------
diff --git a/discovery-core/src/main/java/org/apache/twill/discovery/ZKDiscoveryService.java b/discovery-core/src/main/java/org/apache/twill/discovery/ZKDiscoveryService.java
new file mode 100644
index 0000000..e2f9bc0
--- /dev/null
+++ b/discovery-core/src/main/java/org/apache/twill/discovery/ZKDiscoveryService.java
@@ -0,0 +1,511 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.discovery;
+
+import org.apache.twill.common.Cancellable;
+import org.apache.twill.common.Threads;
+import org.apache.twill.zookeeper.NodeChildren;
+import org.apache.twill.zookeeper.NodeData;
+import org.apache.twill.zookeeper.OperationFuture;
+import org.apache.twill.zookeeper.ZKClient;
+import org.apache.twill.zookeeper.ZKClients;
+import org.apache.twill.zookeeper.ZKOperations;
+import com.google.common.base.Charsets;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.CacheLoader;
+import com.google.common.cache.LoadingCache;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Multimap;
+import com.google.common.hash.Hashing;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.SettableFuture;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonSerializationContext;
+import com.google.gson.JsonSerializer;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.Type;
+import java.net.InetSocketAddress;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+/**
+ * Zookeeper implementation of {@link DiscoveryService} and {@link DiscoveryServiceClient}.
+ * <p>
+ * Discoverable services are registered within Zookeeper under the namespace 'discoverable' by default.
+ * If you would like to change the namespace under which the services are registered then you can pass
+ * in the namespace during construction of {@link ZKDiscoveryService}.
+ * </p>
+ *
+ * <p>
+ * Following is a simple example of how {@link ZKDiscoveryService} can be used for registering services
+ * and also for discovering the registered services.
+ * <blockquote>
+ * <pre>
+ * {@code
+ *
+ * DiscoveryService service = new ZKDiscoveryService(zkClient);
+ * service.register(new Discoverable() {
+ * @Override
+ * public String getName() {
+ * return 'service-name';
+ * }
+ *
+ * @Override
+ * public InetSocketAddress getSocketAddress() {
+ * return new InetSocketAddress(hostname, port);
+ * }
+ * });
+ * ...
+ * ...
+ * Iterable<Discoverable> services = service.discovery("service-name");
+ * ...
+ * }
+ * </pre>
+ * </blockquote>
+ * </p>
+ */
+public class ZKDiscoveryService implements DiscoveryService, DiscoveryServiceClient {
+ private static final Logger LOG = LoggerFactory.getLogger(ZKDiscoveryService.class);
+ private static final String NAMESPACE = "/discoverable";
+
+ private static final long RETRY_MILLIS = 1000;
+
+ // In memory map for recreating ephemeral nodes after session expires.
+ // It map from discoverable to the corresponding Cancellable
+ private final Multimap<Discoverable, DiscoveryCancellable> discoverables;
+ private final Lock lock;
+
+ private final LoadingCache<String, Iterable<Discoverable>> services;
+ private final ZKClient zkClient;
+ private final ScheduledExecutorService retryExecutor;
+
+ /**
+ * Constructs ZKDiscoveryService using the provided zookeeper client for storing service registry.
+ * @param zkClient The {@link ZKClient} for interacting with zookeeper.
+ */
+ public ZKDiscoveryService(ZKClient zkClient) {
+ this(zkClient, NAMESPACE);
+ }
+
+ /**
+ * Constructs ZKDiscoveryService using the provided zookeeper client for storing service registry under namepsace.
+ * @param zkClient of zookeeper quorum
+ * @param namespace under which the service registered would be stored in zookeeper.
+ * If namespace is {@code null}, no namespace will be used.
+ */
+ public ZKDiscoveryService(ZKClient zkClient, String namespace) {
+ this.discoverables = HashMultimap.create();
+ this.lock = new ReentrantLock();
+ this.retryExecutor = Executors.newSingleThreadScheduledExecutor(
+ Threads.createDaemonThreadFactory("zk-discovery-retry"));
+ this.zkClient = namespace == null ? zkClient : ZKClients.namespace(zkClient, namespace);
+ this.services = CacheBuilder.newBuilder().build(createServiceLoader());
+ this.zkClient.addConnectionWatcher(createConnectionWatcher());
+ }
+
+ /**
+ * Registers a {@link Discoverable} in zookeeper.
+ * <p>
+ * Registering a {@link Discoverable} will create a node <base>/<service-name>
+ * in zookeeper as a ephemeral node. If the node already exists (timeout associated with emphemeral, then a runtime
+ * exception is thrown to make sure that a service with an intent to register is not started without registering.
+ * When a runtime is thrown, expectation is that the process being started with fail and would be started again
+ * by the monitoring service.
+ * </p>
+ * @param discoverable Information of the service provider that could be discovered.
+ * @return An instance of {@link Cancellable}
+ */
+ @Override
+ public Cancellable register(final Discoverable discoverable) {
+ final Discoverable wrapper = new DiscoverableWrapper(discoverable);
+ final SettableFuture<String> future = SettableFuture.create();
+ final DiscoveryCancellable cancellable = new DiscoveryCancellable(wrapper);
+
+ // Create the zk ephemeral node.
+ Futures.addCallback(doRegister(wrapper), new FutureCallback<String>() {
+ @Override
+ public void onSuccess(String result) {
+ // Set the sequence node path to cancellable for future cancellation.
+ cancellable.setPath(result);
+ lock.lock();
+ try {
+ discoverables.put(wrapper, cancellable);
+ } finally {
+ lock.unlock();
+ }
+ LOG.debug("Service registered: {} {}", wrapper, result);
+ future.set(result);
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ if (t instanceof KeeperException.NodeExistsException) {
+ handleRegisterFailure(discoverable, future, this, t);
+ } else {
+ LOG.warn("Failed to register: {}", wrapper, t);
+ future.setException(t);
+ }
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+
+ Futures.getUnchecked(future);
+ return cancellable;
+ }
+
+ @Override
+ public Iterable<Discoverable> discover(String service) {
+ return services.getUnchecked(service);
+ }
+
+ /**
+ * Handle registration failure.
+ *
+ * @param discoverable The discoverable to register.
+ * @param completion A settable future to set when registration is completed / failed.
+ * @param creationCallback A future callback for path creation.
+ * @param failureCause The original cause of failure.
+ */
+ private void handleRegisterFailure(final Discoverable discoverable,
+ final SettableFuture<String> completion,
+ final FutureCallback<String> creationCallback,
+ final Throwable failureCause) {
+
+ final String path = getNodePath(discoverable);
+ Futures.addCallback(zkClient.exists(path), new FutureCallback<Stat>() {
+ @Override
+ public void onSuccess(Stat result) {
+ if (result == null) {
+ // If the node is gone, simply retry.
+ LOG.info("Node {} is gone. Retry registration for {}.", path, discoverable);
+ retryRegister(discoverable, creationCallback);
+ return;
+ }
+
+ long ephemeralOwner = result.getEphemeralOwner();
+ if (ephemeralOwner == 0) {
+ // it is not an ephemeral node, something wrong.
+ LOG.error("Node {} already exists and is not an ephemeral node. Discoverable registration failed: {}.",
+ path, discoverable);
+ completion.setException(failureCause);
+ return;
+ }
+ Long sessionId = zkClient.getSessionId();
+ if (sessionId == null || ephemeralOwner != sessionId) {
+ // This zkClient is not valid or doesn't own the ephemeral node, simply keep retrying.
+ LOG.info("Owner of {} is different. Retry registration for {}.", path, discoverable);
+ retryRegister(discoverable, creationCallback);
+ } else {
+ // This client owned the node, treat the registration as completed.
+ // This could happen if same client tries to register twice (due to mistake or failure race condition).
+ completion.set(path);
+ }
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ // If exists call failed, simply retry creation.
+ LOG.warn("Error when getting stats on {}. Retry registration for {}.", path, discoverable);
+ retryRegister(discoverable, creationCallback);
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+ }
+
+ private OperationFuture<String> doRegister(Discoverable discoverable) {
+ byte[] discoverableBytes = encode(discoverable);
+ return zkClient.create(getNodePath(discoverable), discoverableBytes, CreateMode.EPHEMERAL, true);
+ }
+
+ private void retryRegister(final Discoverable discoverable, final FutureCallback<String> creationCallback) {
+ retryExecutor.schedule(new Runnable() {
+
+ @Override
+ public void run() {
+ Futures.addCallback(doRegister(discoverable), creationCallback, Threads.SAME_THREAD_EXECUTOR);
+ }
+ }, RETRY_MILLIS, TimeUnit.MILLISECONDS);
+ }
+
+
+ /**
+ * Generate unique node path for a given {@link Discoverable}.
+ * @param discoverable An instance of {@link Discoverable}.
+ * @return A node name based on the discoverable.
+ */
+ private String getNodePath(Discoverable discoverable) {
+ InetSocketAddress socketAddress = discoverable.getSocketAddress();
+ String node = Hashing.md5()
+ .newHasher()
+ .putBytes(socketAddress.getAddress().getAddress())
+ .putInt(socketAddress.getPort())
+ .hash().toString();
+
+ return String.format("/%s/%s", discoverable.getName(), node);
+ }
+
+ private Watcher createConnectionWatcher() {
+ return new Watcher() {
+ // Watcher is invoked from single event thread, hence safe to use normal mutable variable.
+ private boolean expired;
+
+ @Override
+ public void process(WatchedEvent event) {
+ if (event.getState() == Event.KeeperState.Expired) {
+ LOG.warn("ZK Session expired: {}", zkClient.getConnectString());
+ expired = true;
+ } else if (event.getState() == Event.KeeperState.SyncConnected && expired) {
+ LOG.info("Reconnected after expiration: {}", zkClient.getConnectString());
+ expired = false;
+
+ // Re-register all services
+ lock.lock();
+ try {
+ for (final Map.Entry<Discoverable, DiscoveryCancellable> entry : discoverables.entries()) {
+ LOG.info("Re-registering service: {}", entry.getKey());
+
+ // Must be non-blocking in here.
+ Futures.addCallback(doRegister(entry.getKey()), new FutureCallback<String>() {
+ @Override
+ public void onSuccess(String result) {
+ // Updates the cancellable to the newly created sequential node.
+ entry.getValue().setPath(result);
+ LOG.debug("Service re-registered: {} {}", entry.getKey(), result);
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ // When failed to create the node, there would be no retry and simply make the cancellable do nothing.
+ entry.getValue().setPath(null);
+ LOG.error("Failed to re-register service: {}", entry.getKey(), t);
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+ }
+ } finally {
+ lock.unlock();
+ }
+ }
+ }
+ };
+ }
+
+ /**
+ * Creates a CacheLoader for creating live Iterable for watching instances changes for a given service.
+ */
+ private CacheLoader<String, Iterable<Discoverable>> createServiceLoader() {
+ return new CacheLoader<String, Iterable<Discoverable>>() {
+ @Override
+ public Iterable<Discoverable> load(String service) throws Exception {
+ // The atomic reference is to keep the resulting Iterable live. It always contains a
+ // immutable snapshot of the latest detected set of Discoverable.
+ final AtomicReference<Iterable<Discoverable>> iterable =
+ new AtomicReference<Iterable<Discoverable>>(ImmutableList.<Discoverable>of());
+ final String serviceBase = "/" + service;
+
+ // Watch for children changes in /service
+ ZKOperations.watchChildren(zkClient, serviceBase, new ZKOperations.ChildrenCallback() {
+ @Override
+ public void updated(NodeChildren nodeChildren) {
+ // Fetch data of all children nodes in parallel.
+ List<String> children = nodeChildren.getChildren();
+ List<OperationFuture<NodeData>> dataFutures = Lists.newArrayListWithCapacity(children.size());
+ for (String child : children) {
+ dataFutures.add(zkClient.getData(serviceBase + "/" + child));
+ }
+
+ // Update the service map when all fetching are done.
+ final ListenableFuture<List<NodeData>> fetchFuture = Futures.successfulAsList(dataFutures);
+ fetchFuture.addListener(new Runnable() {
+ @Override
+ public void run() {
+ ImmutableList.Builder<Discoverable> builder = ImmutableList.builder();
+ for (NodeData nodeData : Futures.getUnchecked(fetchFuture)) {
+ // For successful fetch, decode the content.
+ if (nodeData != null) {
+ Discoverable discoverable = decode(nodeData.getData());
+ if (discoverable != null) {
+ builder.add(discoverable);
+ }
+ }
+ }
+ iterable.set(builder.build());
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+ }
+ });
+
+ return new Iterable<Discoverable>() {
+ @Override
+ public Iterator<Discoverable> iterator() {
+ return iterable.get().iterator();
+ }
+ };
+ }
+ };
+ }
+
+ /**
+ * Static helper function for decoding array of bytes into a {@link DiscoverableWrapper} object.
+ * @param bytes representing serialized {@link DiscoverableWrapper}
+ * @return null if bytes are null; else an instance of {@link DiscoverableWrapper}
+ */
+ private static Discoverable decode(byte[] bytes) {
+ if (bytes == null) {
+ return null;
+ }
+ String content = new String(bytes, Charsets.UTF_8);
+ return new GsonBuilder().registerTypeAdapter(Discoverable.class, new DiscoverableCodec())
+ .create()
+ .fromJson(content, Discoverable.class);
+ }
+
+ /**
+ * Static helper function for encoding an instance of {@link DiscoverableWrapper} into array of bytes.
+ * @param discoverable An instance of {@link Discoverable}
+ * @return array of bytes representing an instance of <code>discoverable</code>
+ */
+ private static byte[] encode(Discoverable discoverable) {
+ return new GsonBuilder().registerTypeAdapter(DiscoverableWrapper.class, new DiscoverableCodec())
+ .create()
+ .toJson(discoverable, DiscoverableWrapper.class)
+ .getBytes(Charsets.UTF_8);
+ }
+
+ /**
+ * Inner class for cancelling (un-register) discovery service.
+ */
+ private final class DiscoveryCancellable implements Cancellable {
+
+ private final Discoverable discoverable;
+ private final AtomicBoolean cancelled;
+ private volatile String path;
+
+ DiscoveryCancellable(Discoverable discoverable) {
+ this.discoverable = discoverable;
+ this.cancelled = new AtomicBoolean();
+ }
+
+ /**
+ * Set the zk node path representing the ephemeral sequence node of this registered discoverable.
+ * Called from ZK event thread when creating of the node completed, either from normal registration or
+ * re-registration due to session expiration.
+ *
+ * @param path The path to ephemeral sequence node.
+ */
+ void setPath(String path) {
+ this.path = path;
+ if (cancelled.get() && path != null) {
+ // Simply delete the path if it's already cancelled
+ // It's for the case when session expire happened and re-registration completed after this has been cancelled.
+ // Not bother with the result as if there is error, nothing much we could do.
+ zkClient.delete(path);
+ }
+ }
+
+ @Override
+ public void cancel() {
+ if (!cancelled.compareAndSet(false, true)) {
+ return;
+ }
+
+ // Take a snapshot of the volatile path.
+ String path = this.path;
+
+ // If it is null, meaning cancel() is called before the ephemeral node is created, hence
+ // setPath() will be called in future (through zk callback when creation is completed)
+ // so that deletion will be done in setPath().
+ if (path == null) {
+ return;
+ }
+
+ // Remove this Cancellable from the map so that upon session expiration won't try to register.
+ lock.lock();
+ try {
+ discoverables.remove(discoverable, this);
+ } finally {
+ lock.unlock();
+ }
+
+ // Delete the path. It's ok if the path not exists
+ // (e.g. what session expired and before node has been re-created)
+ Futures.getUnchecked(ZKOperations.ignoreError(zkClient.delete(path),
+ KeeperException.NoNodeException.class, path));
+ LOG.debug("Service unregistered: {} {}", discoverable, path);
+ }
+ }
+
+ /**
+ * SerDe for converting a {@link DiscoverableWrapper} into a JSON object
+ * or from a JSON object into {@link DiscoverableWrapper}.
+ */
+ private static final class DiscoverableCodec implements JsonSerializer<Discoverable>, JsonDeserializer<Discoverable> {
+
+ @Override
+ public Discoverable deserialize(JsonElement json, Type typeOfT,
+ JsonDeserializationContext context) throws JsonParseException {
+ JsonObject jsonObj = json.getAsJsonObject();
+ final String service = jsonObj.get("service").getAsString();
+ String hostname = jsonObj.get("hostname").getAsString();
+ int port = jsonObj.get("port").getAsInt();
+ final InetSocketAddress address = new InetSocketAddress(hostname, port);
+ return new Discoverable() {
+ @Override
+ public String getName() {
+ return service;
+ }
+
+ @Override
+ public InetSocketAddress getSocketAddress() {
+ return address;
+ }
+ };
+ }
+
+ @Override
+ public JsonElement serialize(Discoverable src, Type typeOfSrc, JsonSerializationContext context) {
+ JsonObject jsonObj = new JsonObject();
+ jsonObj.addProperty("service", src.getName());
+ jsonObj.addProperty("hostname", src.getSocketAddress().getHostName());
+ jsonObj.addProperty("port", src.getSocketAddress().getPort());
+ return jsonObj;
+ }
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/discovery-core/src/main/java/org/apache/twill/discovery/package-info.java
----------------------------------------------------------------------
diff --git a/discovery-core/src/main/java/org/apache/twill/discovery/package-info.java b/discovery-core/src/main/java/org/apache/twill/discovery/package-info.java
new file mode 100644
index 0000000..a1d6e0c
--- /dev/null
+++ b/discovery-core/src/main/java/org/apache/twill/discovery/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Classes in this package provides service discovery implementations.
+ */
+package org.apache.twill.discovery;
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/discovery-core/src/test/java/org/apache/twill/discovery/InMemoryDiscoveryServiceTest.java
----------------------------------------------------------------------
diff --git a/discovery-core/src/test/java/org/apache/twill/discovery/InMemoryDiscoveryServiceTest.java b/discovery-core/src/test/java/org/apache/twill/discovery/InMemoryDiscoveryServiceTest.java
new file mode 100644
index 0000000..d8cc375
--- /dev/null
+++ b/discovery-core/src/test/java/org/apache/twill/discovery/InMemoryDiscoveryServiceTest.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.twill.discovery;
+
+import org.apache.twill.common.Cancellable;
+import com.google.common.collect.Iterables;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.net.InetSocketAddress;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Test memory based service discovery service.
+ */
+public class InMemoryDiscoveryServiceTest {
+ private Cancellable register(DiscoveryService service, final String name, final String host, final int port) {
+ return service.register(new Discoverable() {
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public InetSocketAddress getSocketAddress() {
+ return new InetSocketAddress(host, port);
+ }
+ });
+ }
+
+ @Test
+ public void simpleDiscoverable() throws Exception {
+ DiscoveryService discoveryService = new InMemoryDiscoveryService();
+ DiscoveryServiceClient discoveryServiceClient = (DiscoveryServiceClient) discoveryService;
+
+ // Register one service running on one host:port
+ Cancellable cancellable = register(discoveryService, "foo", "localhost", 8090);
+ Iterable<Discoverable> discoverables = discoveryServiceClient.discover("foo");
+
+ // Discover that registered host:port.
+ Assert.assertTrue(Iterables.size(discoverables) == 1);
+
+ // Remove the service
+ cancellable.cancel();
+
+ // There should be no service.
+ discoverables = discoveryServiceClient.discover("foo");
+ TimeUnit.MILLISECONDS.sleep(100);
+ Assert.assertTrue(Iterables.size(discoverables) == 0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/discovery-core/src/test/java/org/apache/twill/discovery/ZKDiscoveryServiceTest.java
----------------------------------------------------------------------
diff --git a/discovery-core/src/test/java/org/apache/twill/discovery/ZKDiscoveryServiceTest.java b/discovery-core/src/test/java/org/apache/twill/discovery/ZKDiscoveryServiceTest.java
new file mode 100644
index 0000000..feee8db
--- /dev/null
+++ b/discovery-core/src/test/java/org/apache/twill/discovery/ZKDiscoveryServiceTest.java
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.discovery;
+
+import org.apache.twill.common.Cancellable;
+import org.apache.twill.common.Services;
+import org.apache.twill.internal.zookeeper.InMemoryZKServer;
+import org.apache.twill.internal.zookeeper.KillZKSession;
+import org.apache.twill.zookeeper.RetryStrategies;
+import org.apache.twill.zookeeper.ZKClientService;
+import org.apache.twill.zookeeper.ZKClientServices;
+import org.apache.twill.zookeeper.ZKClients;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.util.concurrent.Futures;
+import org.junit.AfterClass;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.InetSocketAddress;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Test Zookeeper based discovery service.
+ */
+public class ZKDiscoveryServiceTest {
+ private static final Logger LOG = LoggerFactory.getLogger(ZKDiscoveryServiceTest.class);
+
+ private static InMemoryZKServer zkServer;
+ private static ZKClientService zkClient;
+
+ @BeforeClass
+ public static void beforeClass() {
+ zkServer = InMemoryZKServer.builder().setTickTime(100000).build();
+ zkServer.startAndWait();
+
+ zkClient = ZKClientServices.delegate(
+ ZKClients.retryOnFailure(
+ ZKClients.reWatchOnExpire(
+ ZKClientService.Builder.of(zkServer.getConnectionStr()).build()),
+ RetryStrategies.fixDelay(1, TimeUnit.SECONDS)));
+ zkClient.startAndWait();
+ }
+
+ @AfterClass
+ public static void afterClass() {
+ Futures.getUnchecked(Services.chainStop(zkClient, zkServer));
+ }
+
+ private Cancellable register(DiscoveryService service, final String name, final String host, final int port) {
+ return service.register(new Discoverable() {
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public InetSocketAddress getSocketAddress() {
+ return new InetSocketAddress(host, port);
+ }
+ });
+ }
+
+
+ private boolean waitTillExpected(int expected, Iterable<Discoverable> discoverables) throws Exception {
+ for (int i = 0; i < 10; ++i) {
+ TimeUnit.MILLISECONDS.sleep(10);
+ if (Iterables.size(discoverables) == expected) {
+ return true;
+ }
+ }
+ return (Iterables.size(discoverables) == expected);
+ }
+
+ @Test (timeout = 5000)
+ public void testDoubleRegister() throws Exception {
+ ZKDiscoveryService discoveryService = new ZKDiscoveryService(zkClient);
+ DiscoveryServiceClient discoveryServiceClient = discoveryService;
+
+ // Register on the same host port, it shouldn't fail.
+ Cancellable cancellable = register(discoveryService, "test_double_reg", "localhost", 54321);
+ Cancellable cancellable2 = register(discoveryService, "test_double_reg", "localhost", 54321);
+
+ Iterable<Discoverable> discoverables = discoveryServiceClient.discover("test_double_reg");
+
+ Assert.assertTrue(waitTillExpected(1, discoverables));
+
+ cancellable.cancel();
+ cancellable2.cancel();
+
+ // Register again with two different clients, but killing session of the first one.
+ final ZKClientService zkClient2 = ZKClientServices.delegate(
+ ZKClients.retryOnFailure(
+ ZKClients.reWatchOnExpire(
+ ZKClientService.Builder.of(zkServer.getConnectionStr()).build()),
+ RetryStrategies.fixDelay(1, TimeUnit.SECONDS)));
+ zkClient2.startAndWait();
+
+ try {
+ ZKDiscoveryService discoveryService2 = new ZKDiscoveryService(zkClient2);
+ cancellable2 = register(discoveryService2, "test_multi_client", "localhost", 54321);
+
+ // Schedule a thread to shutdown zkClient2.
+ new Thread() {
+ @Override
+ public void run() {
+ try {
+ TimeUnit.SECONDS.sleep(2);
+ zkClient2.stopAndWait();
+ } catch (InterruptedException e) {
+ LOG.error(e.getMessage(), e);
+ }
+ }
+ }.start();
+
+ // This call would block until zkClient2 is shutdown.
+ cancellable = register(discoveryService, "test_multi_client", "localhost", 54321);
+ cancellable.cancel();
+
+ } finally {
+ zkClient2.stopAndWait();
+ }
+ }
+
+ @Test
+ public void testSessionExpires() throws Exception {
+ ZKDiscoveryService discoveryService = new ZKDiscoveryService(zkClient);
+ DiscoveryServiceClient discoveryServiceClient = discoveryService;
+
+ Cancellable cancellable = register(discoveryService, "test_expires", "localhost", 54321);
+
+ Iterable<Discoverable> discoverables = discoveryServiceClient.discover("test_expires");
+
+ // Discover that registered host:port.
+ Assert.assertTrue(waitTillExpected(1, discoverables));
+
+ KillZKSession.kill(zkClient.getZooKeeperSupplier().get(), zkServer.getConnectionStr(), 5000);
+
+ // Register one more endpoint to make sure state has been reflected after reconnection
+ Cancellable cancellable2 = register(discoveryService, "test_expires", "localhost", 54322);
+
+ // Reconnection would trigger re-registration.
+ Assert.assertTrue(waitTillExpected(2, discoverables));
+
+ cancellable.cancel();
+ cancellable2.cancel();
+
+ // Verify that both are now gone.
+ Assert.assertTrue(waitTillExpected(0, discoverables));
+ }
+
+ @Test
+ public void simpleDiscoverable() throws Exception {
+ DiscoveryService discoveryService = new ZKDiscoveryService(zkClient);
+ DiscoveryServiceClient discoveryServiceClient = new ZKDiscoveryService(zkClient);
+
+ // Register one service running on one host:port
+ Cancellable cancellable = register(discoveryService, "foo", "localhost", 8090);
+ Iterable<Discoverable> discoverables = discoveryServiceClient.discover("foo");
+
+ // Discover that registered host:port.
+ Assert.assertTrue(waitTillExpected(1, discoverables));
+
+ // Remove the service
+ cancellable.cancel();
+
+ // There should be no service.
+
+ discoverables = discoveryServiceClient.discover("foo");
+
+ Assert.assertTrue(waitTillExpected(0, discoverables));
+ }
+
+ @Test
+ public void manySameDiscoverable() throws Exception {
+ List<Cancellable> cancellables = Lists.newArrayList();
+ DiscoveryService discoveryService = new ZKDiscoveryService(zkClient);
+ DiscoveryServiceClient discoveryServiceClient = new ZKDiscoveryService(zkClient);
+
+ cancellables.add(register(discoveryService, "manyDiscoverable", "localhost", 1));
+ cancellables.add(register(discoveryService, "manyDiscoverable", "localhost", 2));
+ cancellables.add(register(discoveryService, "manyDiscoverable", "localhost", 3));
+ cancellables.add(register(discoveryService, "manyDiscoverable", "localhost", 4));
+ cancellables.add(register(discoveryService, "manyDiscoverable", "localhost", 5));
+
+ Iterable<Discoverable> discoverables = discoveryServiceClient.discover("manyDiscoverable");
+ Assert.assertTrue(waitTillExpected(5, discoverables));
+
+ for (int i = 0; i < 5; i++) {
+ cancellables.get(i).cancel();
+ Assert.assertTrue(waitTillExpected(4 - i, discoverables));
+ }
+ }
+
+ @Test
+ public void multiServiceDiscoverable() throws Exception {
+ List<Cancellable> cancellables = Lists.newArrayList();
+ DiscoveryService discoveryService = new ZKDiscoveryService(zkClient);
+ DiscoveryServiceClient discoveryServiceClient = new ZKDiscoveryService(zkClient);
+
+ cancellables.add(register(discoveryService, "service1", "localhost", 1));
+ cancellables.add(register(discoveryService, "service1", "localhost", 2));
+ cancellables.add(register(discoveryService, "service1", "localhost", 3));
+ cancellables.add(register(discoveryService, "service1", "localhost", 4));
+ cancellables.add(register(discoveryService, "service1", "localhost", 5));
+
+ cancellables.add(register(discoveryService, "service2", "localhost", 1));
+ cancellables.add(register(discoveryService, "service2", "localhost", 2));
+ cancellables.add(register(discoveryService, "service2", "localhost", 3));
+
+ cancellables.add(register(discoveryService, "service3", "localhost", 1));
+ cancellables.add(register(discoveryService, "service3", "localhost", 2));
+
+ Iterable<Discoverable> discoverables = discoveryServiceClient.discover("service1");
+ Assert.assertTrue(waitTillExpected(5, discoverables));
+
+ discoverables = discoveryServiceClient.discover("service2");
+ Assert.assertTrue(waitTillExpected(3, discoverables));
+
+ discoverables = discoveryServiceClient.discover("service3");
+ Assert.assertTrue(waitTillExpected(2, discoverables));
+
+ cancellables.add(register(discoveryService, "service3", "localhost", 3));
+ Assert.assertTrue(waitTillExpected(3, discoverables)); // Shows live iterator.
+
+ for (Cancellable cancellable : cancellables) {
+ cancellable.cancel();
+ }
+
+ Assert.assertTrue(waitTillExpected(0, discoveryServiceClient.discover("service1")));
+ Assert.assertTrue(waitTillExpected(0, discoveryServiceClient.discover("service2")));
+ Assert.assertTrue(waitTillExpected(0, discoveryServiceClient.discover("service3")));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/discovery-core/src/test/resources/logback-test.xml
----------------------------------------------------------------------
diff --git a/discovery-core/src/test/resources/logback-test.xml b/discovery-core/src/test/resources/logback-test.xml
new file mode 100644
index 0000000..2615cb4
--- /dev/null
+++ b/discovery-core/src/test/resources/logback-test.xml
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!-- Default logback configuration for twill library -->
+<configuration>
+ <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n</pattern>
+ </encoder>
+ </appender>
+
+ <logger name="org.apache.twill" level="DEBUG" />
+
+ <root level="WARN">
+ <appender-ref ref="STDOUT"/>
+ </root>
+
+</configuration>
[07/15] Initial import commit.
Posted by ch...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/ports/AMRMClientImpl.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/ports/AMRMClientImpl.java b/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/ports/AMRMClientImpl.java
new file mode 100644
index 0000000..c1bd75a
--- /dev/null
+++ b/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/ports/AMRMClientImpl.java
@@ -0,0 +1,412 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn.ports;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.yarn.YarnException;
+import org.apache.hadoop.yarn.api.AMRMProtocol;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.factories.RecordFactory;
+import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.service.AbstractService;
+import org.apache.hadoop.yarn.util.BuilderUtils;
+
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.security.PrivilegedAction;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.TreeSet;
+
+/**
+ * Ported from Apache Hadoop YARN.
+ */
+public final class AMRMClientImpl extends AbstractService implements AMRMClient {
+
+ private static final Log LOG = LogFactory.getLog(AMRMClientImpl.class);
+
+ private final RecordFactory recordFactory =
+ RecordFactoryProvider.getRecordFactory(null);
+
+ private int lastResponseId = 0;
+
+ protected AMRMProtocol rmClient;
+ protected final ApplicationAttemptId appAttemptId;
+ protected Resource clusterAvailableResources;
+ protected int clusterNodeCount;
+
+ //Key -> Priority
+ //Value -> Map
+ //Key->ResourceName (e.g., hostname, rackname, *)
+ //Value->Map
+ //Key->Resource Capability
+ //Value->ResourceRequest
+ protected final
+ Map<Priority, Map<String, Map<Resource, ResourceRequest>>>
+ remoteRequestsTable =
+ new TreeMap<Priority, Map<String, Map<Resource, ResourceRequest>>>();
+
+ protected final Set<ResourceRequest> ask = new TreeSet<ResourceRequest>(
+ new org.apache.hadoop.yarn.util.BuilderUtils.ResourceRequestComparator());
+ protected final Set<ContainerId> release = new TreeSet<ContainerId>();
+
+ public AMRMClientImpl(ApplicationAttemptId appAttemptId) {
+ super(AMRMClientImpl.class.getName());
+ this.appAttemptId = appAttemptId;
+ }
+
+ @Override
+ public synchronized void init(Configuration conf) {
+ super.init(conf);
+ }
+
+ @Override
+ public synchronized void start() {
+ final YarnConfiguration conf = new YarnConfiguration(getConfig());
+ final YarnRPC rpc = YarnRPC.create(conf);
+ final InetSocketAddress rmAddress = conf.getSocketAddr(
+ YarnConfiguration.RM_SCHEDULER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_PORT);
+
+ UserGroupInformation currentUser;
+ try {
+ currentUser = UserGroupInformation.getCurrentUser();
+ } catch (IOException e) {
+ throw new YarnException(e);
+ }
+
+ if (UserGroupInformation.isSecurityEnabled()) {
+ String tokenURLEncodedStr = System.getenv().get(
+ ApplicationConstants.APPLICATION_MASTER_TOKEN_ENV_NAME);
+ Token<? extends TokenIdentifier> token = new Token<TokenIdentifier>();
+
+ try {
+ token.decodeFromUrlString(tokenURLEncodedStr);
+ } catch (IOException e) {
+ throw new YarnException(e);
+ }
+
+ SecurityUtil.setTokenService(token, rmAddress);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("AppMasterToken is " + token);
+ }
+ currentUser.addToken(token);
+ }
+
+ rmClient = currentUser.doAs(new PrivilegedAction<AMRMProtocol>() {
+ @Override
+ public AMRMProtocol run() {
+ return (AMRMProtocol) rpc.getProxy(AMRMProtocol.class, rmAddress,
+ conf);
+ }
+ });
+ LOG.debug("Connecting to ResourceManager at " + rmAddress);
+ super.start();
+ }
+
+ @Override
+ public synchronized void stop() {
+ if (this.rmClient != null) {
+ RPC.stopProxy(this.rmClient);
+ }
+ super.stop();
+ }
+
+ @Override
+ public RegisterApplicationMasterResponse registerApplicationMaster(
+ String appHostName, int appHostPort, String appTrackingUrl)
+ throws YarnRemoteException {
+ // do this only once ???
+ RegisterApplicationMasterRequest request = recordFactory
+ .newRecordInstance(RegisterApplicationMasterRequest.class);
+ synchronized (this) {
+ request.setApplicationAttemptId(appAttemptId);
+ }
+ request.setHost(appHostName);
+ request.setRpcPort(appHostPort);
+ if (appTrackingUrl != null) {
+ request.setTrackingUrl(appTrackingUrl);
+ }
+ RegisterApplicationMasterResponse response = rmClient
+ .registerApplicationMaster(request);
+ return response;
+ }
+
+ @Override
+ public AllocationResponse allocate(float progressIndicator)
+ throws YarnRemoteException {
+ AllocateResponse allocateResponse = null;
+ ArrayList<ResourceRequest> askList = null;
+ ArrayList<ContainerId> releaseList = null;
+ AllocateRequest allocateRequest = null;
+
+ try {
+ synchronized (this) {
+ askList = new ArrayList<ResourceRequest>(ask);
+ releaseList = new ArrayList<ContainerId>(release);
+ // optimistically clear this collection assuming no RPC failure
+ ask.clear();
+ release.clear();
+ allocateRequest = BuilderUtils
+ .newAllocateRequest(appAttemptId, lastResponseId, progressIndicator,
+ askList, releaseList);
+ }
+
+ allocateResponse = rmClient.allocate(allocateRequest);
+ AllocationResponse response = AllocationResponses.create(allocateResponse);
+
+ synchronized (this) {
+ // update these on successful RPC
+ clusterNodeCount = allocateResponse.getNumClusterNodes();
+ lastResponseId = response.getResponseId();
+ clusterAvailableResources = response.getAvailableResources();
+ }
+
+ return response;
+ } finally {
+ // TODO how to differentiate remote YARN exception vs error in RPC
+ if (allocateResponse == null) {
+ // We hit an exception in allocate()
+ // Preserve ask and release for next call to allocate()
+ synchronized (this) {
+ release.addAll(releaseList);
+ // Requests could have been added or deleted during call to allocate.
+ // If requests were added/removed then there is nothing to do since
+ // the ResourceRequest object in ask would have the actual new value.
+ // If ask does not have this ResourceRequest then it was unchanged and
+ // so we can add the value back safely.
+ // This assumes that there will no concurrent calls to allocate() and
+ // so we don't have to worry about ask being changed in the
+ // synchronized block at the beginning of this method.
+ for (ResourceRequest oldAsk : askList) {
+ if (!ask.contains(oldAsk)) {
+ ask.add(oldAsk);
+ }
+ }
+ }
+ }
+ }
+ }
+
+ @Override
+ public void unregisterApplicationMaster(FinalApplicationStatus appStatus,
+ String appMessage, String appTrackingUrl) throws YarnRemoteException {
+ FinishApplicationMasterRequest request = recordFactory
+ .newRecordInstance(FinishApplicationMasterRequest.class);
+ request.setAppAttemptId(appAttemptId);
+ request.setFinishApplicationStatus(appStatus);
+ if (appMessage != null) {
+ request.setDiagnostics(appMessage);
+ }
+ if (appTrackingUrl != null) {
+ request.setTrackingUrl(appTrackingUrl);
+ }
+ rmClient.finishApplicationMaster(request);
+ }
+
+ @Override
+ public synchronized void addContainerRequest(ContainerRequest req) {
+ // Create resource requests
+ if (req.hosts != null) {
+ for (String host : req.hosts) {
+ addResourceRequest(req.priority, host, req.capability, req.containerCount);
+ }
+ }
+
+ if (req.racks != null) {
+ for (String rack : req.racks) {
+ addResourceRequest(req.priority, rack, req.capability, req.containerCount);
+ }
+ }
+
+ // Off switch
+ addResourceRequest(req.priority, ANY, req.capability, req.containerCount);
+ }
+
+ @Override
+ public synchronized void removeContainerRequest(ContainerRequest req) {
+ // Update resource requests
+ if (req.hosts != null) {
+ for (String hostName : req.hosts) {
+ decResourceRequest(req.priority, hostName, req.capability, req.containerCount);
+ }
+ }
+
+ if (req.racks != null) {
+ for (String rack : req.racks) {
+ decResourceRequest(req.priority, rack, req.capability, req.containerCount);
+ }
+ }
+
+ decResourceRequest(req.priority, ANY, req.capability, req.containerCount);
+ }
+
+ @Override
+ public synchronized void releaseAssignedContainer(ContainerId containerId) {
+ release.add(containerId);
+ }
+
+ @Override
+ public synchronized Resource getClusterAvailableResources() {
+ return clusterAvailableResources;
+ }
+
+ @Override
+ public synchronized int getClusterNodeCount() {
+ return clusterNodeCount;
+ }
+
+ private void addResourceRequestToAsk(ResourceRequest remoteRequest) {
+ // This code looks weird but is needed because of the following scenario.
+ // A ResourceRequest is removed from the remoteRequestTable. A 0 container
+ // request is added to 'ask' to notify the RM about not needing it any more.
+ // Before the call to allocate, the user now requests more containers. If
+ // the locations of the 0 size request and the new request are the same
+ // (with the difference being only container count), then the set comparator
+ // will consider both to be the same and not add the new request to ask. So
+ // we need to check for the "same" request being present and remove it and
+ // then add it back. The comparator is container count agnostic.
+ // This should happen only rarely but we do need to guard against it.
+ if (ask.contains(remoteRequest)) {
+ ask.remove(remoteRequest);
+ }
+ ask.add(remoteRequest);
+ }
+
+ private void addResourceRequest(Priority priority, String resourceName,
+ Resource capability, int containerCount) {
+ Map<String, Map<Resource, ResourceRequest>> remoteRequests =
+ this.remoteRequestsTable.get(priority);
+ if (remoteRequests == null) {
+ remoteRequests = new HashMap<String, Map<Resource, ResourceRequest>>();
+ this.remoteRequestsTable.put(priority, remoteRequests);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Added priority=" + priority);
+ }
+ }
+ Map<Resource, ResourceRequest> reqMap = remoteRequests.get(resourceName);
+ if (reqMap == null) {
+ reqMap = new HashMap<Resource, ResourceRequest>();
+ remoteRequests.put(resourceName, reqMap);
+ }
+ ResourceRequest remoteRequest = reqMap.get(capability);
+ if (remoteRequest == null) {
+ remoteRequest = BuilderUtils.
+ newResourceRequest(priority, resourceName, capability, 0);
+ reqMap.put(capability, remoteRequest);
+ }
+
+ remoteRequest.setNumContainers(remoteRequest.getNumContainers() + containerCount);
+
+ // Note this down for next interaction with ResourceManager
+ addResourceRequestToAsk(remoteRequest);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("addResourceRequest:" + " applicationId="
+ + appAttemptId + " priority=" + priority.getPriority()
+ + " resourceName=" + resourceName + " numContainers="
+ + remoteRequest.getNumContainers() + " #asks=" + ask.size());
+ }
+ }
+
+ private void decResourceRequest(Priority priority, String resourceName,
+ Resource capability, int containerCount) {
+ Map<String, Map<Resource, ResourceRequest>> remoteRequests =
+ this.remoteRequestsTable.get(priority);
+
+ if (remoteRequests == null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Not decrementing resource as priority " + priority
+ + " is not present in request table");
+ }
+ return;
+ }
+
+ Map<Resource, ResourceRequest> reqMap = remoteRequests.get(resourceName);
+ if (reqMap == null) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Not decrementing resource as " + resourceName
+ + " is not present in request table");
+ }
+ return;
+ }
+ ResourceRequest remoteRequest = reqMap.get(capability);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("BEFORE decResourceRequest:" + " applicationId="
+ + appAttemptId + " priority=" + priority.getPriority()
+ + " resourceName=" + resourceName + " numContainers="
+ + remoteRequest.getNumContainers() + " #asks=" + ask.size());
+ }
+
+ remoteRequest.
+ setNumContainers(remoteRequest.getNumContainers() - containerCount);
+ if (remoteRequest.getNumContainers() < 0) {
+ // guard against spurious removals
+ remoteRequest.setNumContainers(0);
+ }
+ // Send the ResourceRequest to RM even if is 0 because it needs to override
+ // a previously sent value. If ResourceRequest was not sent previously then
+ // sending 0 ought to be a no-op on RM.
+ addResourceRequestToAsk(remoteRequest);
+
+ // Delete entries from map if no longer needed.
+ if (remoteRequest.getNumContainers() == 0) {
+ reqMap.remove(capability);
+ if (reqMap.size() == 0) {
+ remoteRequests.remove(resourceName);
+ }
+ if (remoteRequests.size() == 0) {
+ remoteRequestsTable.remove(priority);
+ }
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.info("AFTER decResourceRequest:" + " applicationId="
+ + appAttemptId + " priority=" + priority.getPriority()
+ + " resourceName=" + resourceName + " numContainers="
+ + remoteRequest.getNumContainers() + " #asks=" + ask.size());
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/ports/AllocationResponse.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/ports/AllocationResponse.java b/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/ports/AllocationResponse.java
new file mode 100644
index 0000000..89734fc
--- /dev/null
+++ b/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/ports/AllocationResponse.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn.ports;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.Resource;
+
+import java.util.List;
+
+/**
+ * This interface is to abstract the differences in Vanilla Hadoop YARN 2.0 and CDH 4.4
+ */
+public interface AllocationResponse {
+
+ int getResponseId();
+
+ Resource getAvailableResources();
+
+ List<Container> getAllocatedContainers();
+
+ List<ContainerStatus> getCompletedContainersStatuses();
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/ports/AllocationResponses.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/ports/AllocationResponses.java b/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/ports/AllocationResponses.java
new file mode 100644
index 0000000..ea46c3b
--- /dev/null
+++ b/yarn/src/main/hadoop20/org/apache/twill/internal/yarn/ports/AllocationResponses.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn.ports;
+
+import com.google.common.base.Throwables;
+import com.google.common.reflect.TypeToken;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.Resource;
+
+import java.util.List;
+
+/**
+ * Factory for building instance of {@link AllocationResponse} based on the response type.
+ */
+public final class AllocationResponses {
+
+ /**
+ * A hack for CDH 4.4.0, as the AllocateResponse class is being rewritten and diverted from YARN 2.0
+ */
+ private static final boolean IS_CDH_4_4;
+
+ static {
+ boolean result = false;
+ try {
+ try {
+ // See if it is standard YARN 2.0 AllocateResponse object.
+ AllocateResponse.class.getMethod("getAMResponse");
+ } catch (NoSuchMethodException e) {
+ // See if it is CDH 4.4 AllocateResponse object.
+ AllocationResponse.class.getMethod("getAllocatedContainers");
+ result = true;
+ }
+ } catch (Exception e) {
+ // Something very wrong in here, as it shouldn't arrive here.
+ e.printStackTrace();
+ throw Throwables.propagate(e);
+ }
+
+ IS_CDH_4_4 = result;
+ }
+
+ public static AllocationResponse create(Object response) {
+ if (IS_CDH_4_4) {
+ return new ReflectionAllocationResponse(response);
+ }
+
+ try {
+ Object amResponse = response.getClass().getMethod("getAMResponse").invoke(response);
+ return new ReflectionAllocationResponse(amResponse);
+ } catch (Exception e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ private static final class ReflectionAllocationResponse implements AllocationResponse {
+
+ private final Object response;
+
+ private ReflectionAllocationResponse(Object response) {
+ this.response = response;
+ }
+
+ @Override
+ public int getResponseId() {
+ return call("getResponseId", TypeToken.of(Integer.class));
+ }
+
+ @Override
+ public Resource getAvailableResources() {
+ return call("getAvailableResources", TypeToken.of(Resource.class));
+ }
+
+ @Override
+ public List<Container> getAllocatedContainers() {
+ return call("getAllocatedContainers", new TypeToken<List<Container>>() {});
+ }
+
+ @Override
+ public List<ContainerStatus> getCompletedContainersStatuses() {
+ return call("getCompletedContainersStatuses", new TypeToken<List<ContainerStatus>>() {});
+ }
+
+ private <T> T call(String methodName, TypeToken<T> resultType) {
+ try {
+ return (T) resultType.getRawType().cast(response.getClass().getMethod(methodName).invoke(response));
+ } catch (Exception e) {
+ throw Throwables.propagate(e);
+ }
+ }
+ }
+
+ private AllocationResponses() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAMClient.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAMClient.java b/yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAMClient.java
new file mode 100644
index 0000000..ce8f90f
--- /dev/null
+++ b/yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAMClient.java
@@ -0,0 +1,207 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn;
+
+import org.apache.twill.internal.ProcessLauncher;
+import org.apache.twill.internal.appmaster.RunnableProcessLauncher;
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Multimap;
+import com.google.common.util.concurrent.AbstractIdleService;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.client.api.AMRMClient;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.net.InetSocketAddress;
+import java.net.URL;
+import java.util.List;
+import java.util.UUID;
+
+/**
+ *
+ */
+public final class Hadoop21YarnAMClient extends AbstractIdleService implements YarnAMClient {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Hadoop21YarnAMClient.class);
+
+ private static final Function<ContainerStatus, YarnContainerStatus> STATUS_TRANSFORM;
+
+ static {
+ STATUS_TRANSFORM = new Function<ContainerStatus, YarnContainerStatus>() {
+ @Override
+ public YarnContainerStatus apply(ContainerStatus status) {
+ return new Hadoop21YarnContainerStatus(status);
+ }
+ };
+ }
+
+ private final ContainerId containerId;
+ private final Multimap<String, AMRMClient.ContainerRequest> containerRequests;
+ private final AMRMClient<AMRMClient.ContainerRequest> amrmClient;
+ private final Hadoop21YarnNMClient nmClient;
+ private InetSocketAddress trackerAddr;
+ private URL trackerUrl;
+ private Resource maxCapability;
+
+ public Hadoop21YarnAMClient(Configuration conf) {
+ String masterContainerId = System.getenv().get(ApplicationConstants.Environment.CONTAINER_ID.name());
+ Preconditions.checkArgument(masterContainerId != null,
+ "Missing %s from environment", ApplicationConstants.Environment.CONTAINER_ID.name());
+ this.containerId = ConverterUtils.toContainerId(masterContainerId);
+ this.containerRequests = ArrayListMultimap.create();
+
+ this.amrmClient = AMRMClient.createAMRMClient();
+ this.amrmClient.init(conf);
+ this.nmClient = new Hadoop21YarnNMClient(conf);
+ }
+
+ @Override
+ protected void startUp() throws Exception {
+ Preconditions.checkNotNull(trackerAddr, "Tracker address not set.");
+ Preconditions.checkNotNull(trackerUrl, "Tracker URL not set.");
+
+ amrmClient.start();
+ RegisterApplicationMasterResponse response = amrmClient.registerApplicationMaster(trackerAddr.getHostName(),
+ trackerAddr.getPort(),
+ trackerUrl.toString());
+ maxCapability = response.getMaximumResourceCapability();
+ nmClient.startAndWait();
+ }
+
+ @Override
+ protected void shutDown() throws Exception {
+ nmClient.stopAndWait();
+ amrmClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED, null, trackerUrl.toString());
+ amrmClient.stop();
+ }
+
+ @Override
+ public ContainerId getContainerId() {
+ return containerId;
+ }
+
+ @Override
+ public String getHost() {
+ return System.getenv().get(ApplicationConstants.Environment.NM_HOST.name());
+ }
+
+ @Override
+ public void setTracker(InetSocketAddress trackerAddr, URL trackerUrl) {
+ this.trackerAddr = trackerAddr;
+ this.trackerUrl = trackerUrl;
+ }
+
+ @Override
+ public synchronized void allocate(float progress, AllocateHandler handler) throws Exception {
+ AllocateResponse allocateResponse = amrmClient.allocate(progress);
+ List<ProcessLauncher<YarnContainerInfo>> launchers
+ = Lists.newArrayListWithCapacity(allocateResponse.getAllocatedContainers().size());
+
+ for (Container container : allocateResponse.getAllocatedContainers()) {
+ launchers.add(new RunnableProcessLauncher(new Hadoop21YarnContainerInfo(container), nmClient));
+ }
+
+ if (!launchers.isEmpty()) {
+ handler.acquired(launchers);
+
+ // If no process has been launched through the given launcher, return the container.
+ for (ProcessLauncher<YarnContainerInfo> l : launchers) {
+ // This cast always works.
+ RunnableProcessLauncher launcher = (RunnableProcessLauncher) l;
+ if (!launcher.isLaunched()) {
+ Container container = launcher.getContainerInfo().getContainer();
+ LOG.info("Nothing to run in container, releasing it: {}", container);
+ amrmClient.releaseAssignedContainer(container.getId());
+ }
+ }
+ }
+
+ List<YarnContainerStatus> completed = ImmutableList.copyOf(
+ Iterables.transform(allocateResponse.getCompletedContainersStatuses(), STATUS_TRANSFORM));
+ if (!completed.isEmpty()) {
+ handler.completed(completed);
+ }
+ }
+
+ @Override
+ public ContainerRequestBuilder addContainerRequest(Resource capability) {
+ return addContainerRequest(capability, 1);
+ }
+
+ @Override
+ public ContainerRequestBuilder addContainerRequest(Resource capability, int count) {
+ return new ContainerRequestBuilder(adjustCapability(capability), count) {
+ @Override
+ public String apply() {
+ synchronized (Hadoop21YarnAMClient.this) {
+ String id = UUID.randomUUID().toString();
+
+ String[] hosts = this.hosts.isEmpty() ? null : this.hosts.toArray(new String[this.hosts.size()]);
+ String[] racks = this.racks.isEmpty() ? null : this.racks.toArray(new String[this.racks.size()]);
+
+ for (int i = 0; i < count; i++) {
+ AMRMClient.ContainerRequest request = new AMRMClient.ContainerRequest(capability, hosts, racks, priority);
+ containerRequests.put(id, request);
+ amrmClient.addContainerRequest(request);
+ }
+
+ return id;
+ }
+ }
+ };
+ }
+
+ @Override
+ public synchronized void completeContainerRequest(String id) {
+ for (AMRMClient.ContainerRequest request : containerRequests.removeAll(id)) {
+ amrmClient.removeContainerRequest(request);
+ }
+ }
+
+ private Resource adjustCapability(Resource resource) {
+ int cores = resource.getVirtualCores();
+ int updatedCores = Math.min(resource.getVirtualCores(), maxCapability.getVirtualCores());
+
+ if (cores != updatedCores) {
+ resource.setVirtualCores(updatedCores);
+ LOG.info("Adjust virtual cores requirement from {} to {}.", cores, updatedCores);
+ }
+
+ int updatedMemory = Math.min(resource.getMemory(), maxCapability.getMemory());
+ if (resource.getMemory() != updatedMemory) {
+ resource.setMemory(updatedMemory);
+ LOG.info("Adjust memory requirement from {} to {} MB.", resource.getMemory(), updatedMemory);
+ }
+
+ return resource;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAppClient.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAppClient.java b/yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAppClient.java
new file mode 100644
index 0000000..50b212d
--- /dev/null
+++ b/yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnAppClient.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn;
+
+import org.apache.twill.api.TwillSpecification;
+import org.apache.twill.internal.ProcessController;
+import org.apache.twill.internal.ProcessLauncher;
+import org.apache.twill.internal.appmaster.ApplicationMasterProcessLauncher;
+import org.apache.twill.internal.appmaster.ApplicationSubmitter;
+import com.google.common.base.Throwables;
+import com.google.common.util.concurrent.AbstractIdleService;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.client.api.YarnClientApplication;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ */
+public final class Hadoop21YarnAppClient extends AbstractIdleService implements YarnAppClient {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Hadoop21YarnAppClient.class);
+ private final YarnClient yarnClient;
+
+ public Hadoop21YarnAppClient(Configuration configuration) {
+ this.yarnClient = YarnClient.createYarnClient();
+ yarnClient.init(configuration);
+ }
+
+ @Override
+ public ProcessLauncher<ApplicationId> createLauncher(TwillSpecification twillSpec) throws Exception {
+ // Request for new application
+ YarnClientApplication application = yarnClient.createApplication();
+ final GetNewApplicationResponse response = application.getNewApplicationResponse();
+ final ApplicationId appId = response.getApplicationId();
+
+ // Setup the context for application submission
+ final ApplicationSubmissionContext appSubmissionContext = application.getApplicationSubmissionContext();
+ appSubmissionContext.setApplicationId(appId);
+ appSubmissionContext.setApplicationName(twillSpec.getName());
+
+ ApplicationSubmitter submitter = new ApplicationSubmitter() {
+ @Override
+ public ProcessController<YarnApplicationReport> submit(YarnLaunchContext context, Resource capability) {
+ ContainerLaunchContext launchContext = context.getLaunchContext();
+
+ addRMToken(launchContext);
+ appSubmissionContext.setAMContainerSpec(launchContext);
+ appSubmissionContext.setResource(adjustMemory(response, capability));
+ appSubmissionContext.setMaxAppAttempts(2);
+
+ try {
+ yarnClient.submitApplication(appSubmissionContext);
+ return new ProcessControllerImpl(yarnClient, appId);
+ } catch (Exception e) {
+ LOG.error("Failed to submit application {}", appId, e);
+ throw Throwables.propagate(e);
+ }
+ }
+ };
+
+ return new ApplicationMasterProcessLauncher(appId, submitter);
+ }
+
+ private Resource adjustMemory(GetNewApplicationResponse response, Resource capability) {
+ int maxMemory = response.getMaximumResourceCapability().getMemory();
+ int updatedMemory = capability.getMemory();
+
+ if (updatedMemory > maxMemory) {
+ capability.setMemory(maxMemory);
+ }
+
+ return capability;
+ }
+
+ private void addRMToken(ContainerLaunchContext context) {
+ if (!UserGroupInformation.isSecurityEnabled()) {
+ return;
+ }
+
+ try {
+ Credentials credentials = YarnUtils.decodeCredentials(context.getTokens());
+
+ Configuration config = yarnClient.getConfig();
+ Token<TokenIdentifier> token = ConverterUtils.convertFromYarn(
+ yarnClient.getRMDelegationToken(new Text(YarnUtils.getYarnTokenRenewer(config))),
+ YarnUtils.getRMAddress(config));
+
+ LOG.info("Added RM delegation token {}", token);
+ credentials.addToken(token.getService(), token);
+
+ context.setTokens(YarnUtils.encodeCredentials(credentials));
+
+ } catch (Exception e) {
+ LOG.error("Fails to create credentials.", e);
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ public ProcessLauncher<ApplicationId> createLauncher(String user, TwillSpecification twillSpec) throws Exception {
+ // Ignore user
+ return createLauncher(twillSpec);
+ }
+
+ @Override
+ public ProcessController<YarnApplicationReport> createProcessController(ApplicationId appId) {
+ return new ProcessControllerImpl(yarnClient, appId);
+ }
+
+ @Override
+ protected void startUp() throws Exception {
+ yarnClient.start();
+ }
+
+ @Override
+ protected void shutDown() throws Exception {
+ yarnClient.stop();
+ }
+
+ private static final class ProcessControllerImpl implements ProcessController<YarnApplicationReport> {
+ private final YarnClient yarnClient;
+ private final ApplicationId appId;
+
+ public ProcessControllerImpl(YarnClient yarnClient, ApplicationId appId) {
+ this.yarnClient = yarnClient;
+ this.appId = appId;
+ }
+
+ @Override
+ public YarnApplicationReport getReport() {
+ try {
+ return new Hadoop21YarnApplicationReport(yarnClient.getApplicationReport(appId));
+ } catch (Exception e) {
+ LOG.error("Failed to get application report {}", appId, e);
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ public void cancel() {
+ try {
+ yarnClient.killApplication(appId);
+ } catch (Exception e) {
+ LOG.error("Failed to kill application {}", appId, e);
+ throw Throwables.propagate(e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnApplicationReport.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnApplicationReport.java b/yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnApplicationReport.java
new file mode 100644
index 0000000..6e614f5
--- /dev/null
+++ b/yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnApplicationReport.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+
+/**
+ *
+ */
+public final class Hadoop21YarnApplicationReport implements YarnApplicationReport {
+
+ private final ApplicationReport report;
+
+ public Hadoop21YarnApplicationReport(ApplicationReport report) {
+ this.report = report;
+ }
+
+ @Override
+ public ApplicationId getApplicationId() {
+ return report.getApplicationId();
+ }
+
+ @Override
+ public ApplicationAttemptId getCurrentApplicationAttemptId() {
+ return report.getCurrentApplicationAttemptId();
+ }
+
+ @Override
+ public String getQueue() {
+ return report.getQueue();
+ }
+
+ @Override
+ public String getName() {
+ return report.getName();
+ }
+
+ @Override
+ public String getHost() {
+ return report.getHost();
+ }
+
+ @Override
+ public int getRpcPort() {
+ return report.getRpcPort();
+ }
+
+ @Override
+ public YarnApplicationState getYarnApplicationState() {
+ return report.getYarnApplicationState();
+ }
+
+ @Override
+ public String getDiagnostics() {
+ return report.getDiagnostics();
+ }
+
+ @Override
+ public String getTrackingUrl() {
+ return report.getTrackingUrl();
+ }
+
+ @Override
+ public String getOriginalTrackingUrl() {
+ return report.getOriginalTrackingUrl();
+ }
+
+ @Override
+ public long getStartTime() {
+ return report.getStartTime();
+ }
+
+ @Override
+ public long getFinishTime() {
+ return report.getFinishTime();
+ }
+
+ @Override
+ public FinalApplicationStatus getFinalApplicationStatus() {
+ return report.getFinalApplicationStatus();
+ }
+
+ @Override
+ public ApplicationResourceUsageReport getApplicationResourceUsageReport() {
+ return report.getApplicationResourceUsageReport();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnContainerInfo.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnContainerInfo.java b/yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnContainerInfo.java
new file mode 100644
index 0000000..86903c1
--- /dev/null
+++ b/yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnContainerInfo.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn;
+
+import com.google.common.base.Throwables;
+import org.apache.hadoop.yarn.api.records.Container;
+
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+
+/**
+ *
+ */
+public final class Hadoop21YarnContainerInfo implements YarnContainerInfo {
+
+ private final Container container;
+
+ public Hadoop21YarnContainerInfo(Container container) {
+ this.container = container;
+ }
+
+ @Override
+ public <T> T getContainer() {
+ return (T) container;
+ }
+
+ @Override
+ public String getId() {
+ return container.getId().toString();
+ }
+
+ @Override
+ public InetAddress getHost() {
+ try {
+ return InetAddress.getByName(container.getNodeId().getHost());
+ } catch (UnknownHostException e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ public int getPort() {
+ return container.getNodeId().getPort();
+ }
+
+ @Override
+ public int getMemoryMB() {
+ return container.getResource().getMemory();
+ }
+
+ @Override
+ public int getVirtualCores() {
+ return container.getResource().getVirtualCores();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnContainerStatus.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnContainerStatus.java b/yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnContainerStatus.java
new file mode 100644
index 0000000..f5758c7
--- /dev/null
+++ b/yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnContainerStatus.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn;
+
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+
+/**
+ *
+ */
+public final class Hadoop21YarnContainerStatus implements YarnContainerStatus {
+
+ private final ContainerStatus containerStatus;
+
+ public Hadoop21YarnContainerStatus(ContainerStatus containerStatus) {
+ this.containerStatus = containerStatus;
+ }
+
+ @Override
+ public String getContainerId() {
+ return containerStatus.getContainerId().toString();
+ }
+
+ @Override
+ public ContainerState getState() {
+ return containerStatus.getState();
+ }
+
+ @Override
+ public int getExitStatus() {
+ return containerStatus.getExitStatus();
+ }
+
+ @Override
+ public String getDiagnostics() {
+ return containerStatus.getDiagnostics();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnLaunchContext.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnLaunchContext.java b/yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnLaunchContext.java
new file mode 100644
index 0000000..8621f93
--- /dev/null
+++ b/yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnLaunchContext.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn;
+
+import com.google.common.base.Function;
+import com.google.common.collect.Maps;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.util.Records;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Map;
+
+/**
+ *
+ */
+public final class Hadoop21YarnLaunchContext implements YarnLaunchContext {
+
+ private static final Function<YarnLocalResource, LocalResource> RESOURCE_TRANSFORM;
+
+ static {
+ // Creates transform function from YarnLocalResource -> LocalResource
+ RESOURCE_TRANSFORM = new Function<YarnLocalResource, LocalResource>() {
+ @Override
+ public LocalResource apply(YarnLocalResource input) {
+ return input.getLocalResource();
+ }
+ };
+ }
+
+ private final ContainerLaunchContext launchContext;
+
+ public Hadoop21YarnLaunchContext() {
+ launchContext = Records.newRecord(ContainerLaunchContext.class);
+ }
+
+ @Override
+ public <T> T getLaunchContext() {
+ return (T) launchContext;
+ }
+
+ @Override
+ public void setCredentials(Credentials credentials) {
+ launchContext.setTokens(YarnUtils.encodeCredentials(credentials));
+ }
+
+ @Override
+ public void setLocalResources(Map<String, YarnLocalResource> localResources) {
+ launchContext.setLocalResources(Maps.transformValues(localResources, RESOURCE_TRANSFORM));
+ }
+
+ @Override
+ public void setServiceData(Map<String, ByteBuffer> serviceData) {
+ launchContext.setServiceData(serviceData);
+ }
+
+ @Override
+ public Map<String, String> getEnvironment() {
+ return launchContext.getEnvironment();
+ }
+
+ @Override
+ public void setEnvironment(Map<String, String> environment) {
+ launchContext.setEnvironment(environment);
+ }
+
+ @Override
+ public List<String> getCommands() {
+ return launchContext.getCommands();
+ }
+
+ @Override
+ public void setCommands(List<String> commands) {
+ launchContext.setCommands(commands);
+ }
+
+ @Override
+ public void setApplicationACLs(Map<ApplicationAccessType, String> acls) {
+ launchContext.setApplicationACLs(acls);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnLocalResource.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnLocalResource.java b/yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnLocalResource.java
new file mode 100644
index 0000000..3f756bd
--- /dev/null
+++ b/yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnLocalResource.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn;
+
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.URL;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ *
+ */
+public final class Hadoop21YarnLocalResource implements YarnLocalResource {
+
+ private final LocalResource localResource;
+
+ public Hadoop21YarnLocalResource() {
+ this.localResource = Records.newRecord(LocalResource.class);
+ }
+
+ @Override
+ public <T> T getLocalResource() {
+ return (T) localResource;
+ }
+
+ @Override
+ public URL getResource() {
+ return localResource.getResource();
+ }
+
+ @Override
+ public void setResource(URL resource) {
+ localResource.setResource(resource);
+ }
+
+ @Override
+ public long getSize() {
+ return localResource.getSize();
+ }
+
+ @Override
+ public void setSize(long size) {
+ localResource.setSize(size);
+ }
+
+ @Override
+ public long getTimestamp() {
+ return localResource.getTimestamp();
+ }
+
+ @Override
+ public void setTimestamp(long timestamp) {
+ localResource.setTimestamp(timestamp);
+ }
+
+ @Override
+ public LocalResourceType getType() {
+ return localResource.getType();
+ }
+
+ @Override
+ public void setType(LocalResourceType type) {
+ localResource.setType(type);
+ }
+
+ @Override
+ public LocalResourceVisibility getVisibility() {
+ return localResource.getVisibility();
+ }
+
+ @Override
+ public void setVisibility(LocalResourceVisibility visibility) {
+ localResource.setVisibility(visibility);
+ }
+
+ @Override
+ public String getPattern() {
+ return localResource.getPattern();
+ }
+
+ @Override
+ public void setPattern(String pattern) {
+ localResource.setPattern(pattern);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnNMClient.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnNMClient.java b/yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnNMClient.java
new file mode 100644
index 0000000..d3a6a80
--- /dev/null
+++ b/yarn/src/main/hadoop21/org/apache/twill/internal/yarn/Hadoop21YarnNMClient.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn;
+
+import org.apache.twill.common.Cancellable;
+import com.google.common.base.Throwables;
+import com.google.common.util.concurrent.AbstractIdleService;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
+import org.apache.hadoop.yarn.api.records.ContainerState;
+import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.client.api.NMClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ */
+public final class Hadoop21YarnNMClient extends AbstractIdleService implements YarnNMClient {
+
+ private static final Logger LOG = LoggerFactory.getLogger(Hadoop21YarnNMClient.class);
+
+ private final NMClient nmClient;
+
+ public Hadoop21YarnNMClient(Configuration configuration) {
+ this.nmClient = NMClient.createNMClient();
+ nmClient.init(configuration);
+ }
+
+ @Override
+ public Cancellable start(YarnContainerInfo containerInfo, YarnLaunchContext launchContext) {
+ try {
+ Container container = containerInfo.getContainer();
+ nmClient.startContainer(container, launchContext.<ContainerLaunchContext>getLaunchContext());
+ return new ContainerTerminator(container, nmClient);
+ } catch (Exception e) {
+ LOG.error("Error in launching process", e);
+ throw Throwables.propagate(e);
+ }
+
+ }
+
+ @Override
+ protected void startUp() throws Exception {
+ nmClient.start();
+ }
+
+ @Override
+ protected void shutDown() throws Exception {
+ nmClient.stop();
+ }
+
+ private static final class ContainerTerminator implements Cancellable {
+
+ private final Container container;
+ private final NMClient nmClient;
+
+ private ContainerTerminator(Container container, NMClient nmClient) {
+ this.container = container;
+ this.nmClient = nmClient;
+ }
+
+ @Override
+ public void cancel() {
+ LOG.info("Request to stop container {}.", container.getId());
+
+ try {
+ nmClient.stopContainer(container.getId(), container.getNodeId());
+ boolean completed = false;
+ while (!completed) {
+ ContainerStatus status = nmClient.getContainerStatus(container.getId(), container.getNodeId());
+ LOG.info("Container status: {} {}", status, status.getDiagnostics());
+
+ completed = (status.getState() == ContainerState.COMPLETE);
+ }
+ LOG.info("Container {} stopped.", container.getId());
+ } catch (Exception e) {
+ LOG.error("Fail to stop container {}", container.getId(), e);
+ throw Throwables.propagate(e);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/filesystem/HDFSLocation.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/filesystem/HDFSLocation.java b/yarn/src/main/java/org/apache/twill/filesystem/HDFSLocation.java
new file mode 100644
index 0000000..b0eeb43
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/filesystem/HDFSLocation.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.filesystem;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Options;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.util.UUID;
+
+/**
+ * A concrete implementation of {@link Location} for the HDFS filesystem.
+ */
+final class HDFSLocation implements Location {
+ private final FileSystem fs;
+ private final Path path;
+
+ /**
+ * Constructs a HDFSLocation.
+ *
+ * @param fs An instance of {@link FileSystem}
+ * @param path of the file.
+ */
+ HDFSLocation(FileSystem fs, Path path) {
+ this.fs = fs;
+ this.path = path;
+ }
+
+ /**
+ * Checks if this location exists on HDFS.
+ *
+ * @return true if found; false otherwise.
+ * @throws IOException
+ */
+ @Override
+ public boolean exists() throws IOException {
+ return fs.exists(path);
+ }
+
+ /**
+ * @return An {@link InputStream} for this location on HDFS.
+ * @throws IOException
+ */
+ @Override
+ public InputStream getInputStream() throws IOException {
+ return fs.open(path);
+ }
+
+ /**
+ * @return An {@link OutputStream} for this location on HDFS.
+ * @throws IOException
+ */
+ @Override
+ public OutputStream getOutputStream() throws IOException {
+ return fs.create(path);
+ }
+
+ @Override
+ public OutputStream getOutputStream(String permission) throws IOException {
+ Configuration conf = fs.getConf();
+ return fs.create(path,
+ new FsPermission(permission),
+ true,
+ conf.getInt("io.file.buffer.size", 4096),
+ fs.getDefaultReplication(path),
+ fs.getDefaultBlockSize(path),
+ null);
+ }
+
+ /**
+ * Appends the child to the current {@link Location} on HDFS.
+ * <p>
+ * Returns a new instance of Location.
+ * </p>
+ *
+ * @param child to be appended to this location.
+ * @return A new instance of {@link Location}
+ * @throws IOException
+ */
+ @Override
+ public Location append(String child) throws IOException {
+ if (child.startsWith("/")) {
+ child = child.substring(1);
+ }
+ return new HDFSLocation(fs, new Path(URI.create(path.toUri() + "/" + child)));
+ }
+
+ @Override
+ public Location getTempFile(String suffix) throws IOException {
+ Path path = new Path(
+ URI.create(this.path.toUri() + "." + UUID.randomUUID() + (suffix == null ? TEMP_FILE_SUFFIX : suffix)));
+ return new HDFSLocation(fs, path);
+ }
+
+ /**
+ * @return Returns the name of the file or directory denoteed by this abstract pathname.
+ */
+ @Override
+ public String getName() {
+ return path.getName();
+ }
+
+ @Override
+ public boolean createNew() throws IOException {
+ return fs.createNewFile(path);
+ }
+
+ /**
+ * @return A {@link URI} for this location on HDFS.
+ */
+ @Override
+ public URI toURI() {
+ return path.toUri();
+ }
+
+ /**
+ * Deletes the file or directory denoted by this abstract pathname. If this
+ * pathname denotes a directory, then the directory must be empty in order
+ * to be deleted.
+ *
+ * @return true if and only if the file or directory is successfully deleted; false otherwise.
+ */
+ @Override
+ public boolean delete() throws IOException {
+ return fs.delete(path, false);
+ }
+
+ @Override
+ public boolean delete(boolean recursive) throws IOException {
+ return fs.delete(path, true);
+ }
+
+ @Override
+ public Location renameTo(Location destination) throws IOException {
+ // Destination will always be of the same type as this location.
+ if (fs instanceof DistributedFileSystem) {
+ ((DistributedFileSystem) fs).rename(path, ((HDFSLocation) destination).path, Options.Rename.OVERWRITE);
+ return new HDFSLocation(fs, new Path(destination.toURI()));
+ }
+
+ if (fs.rename(path, ((HDFSLocation) destination).path)) {
+ return new HDFSLocation(fs, new Path(destination.toURI()));
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Creates the directory named by this abstract pathname, including any necessary
+ * but nonexistent parent directories.
+ *
+ * @return true if and only if the renaming succeeded; false otherwise
+ */
+ @Override
+ public boolean mkdirs() throws IOException {
+ return fs.mkdirs(path);
+ }
+
+ /**
+ * @return Length of file.
+ */
+ @Override
+ public long length() throws IOException {
+ return fs.getFileStatus(path).getLen();
+ }
+
+ @Override
+ public long lastModified() throws IOException {
+ return fs.getFileStatus(path).getModificationTime();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/filesystem/HDFSLocationFactory.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/filesystem/HDFSLocationFactory.java b/yarn/src/main/java/org/apache/twill/filesystem/HDFSLocationFactory.java
new file mode 100644
index 0000000..fa79391
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/filesystem/HDFSLocationFactory.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.filesystem;
+
+import com.google.common.base.Throwables;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+import java.io.IOException;
+import java.net.URI;
+
+/**
+ * A {@link LocationFactory} that creates HDFS {@link Location}.
+ */
+public final class HDFSLocationFactory implements LocationFactory {
+
+ private final FileSystem fileSystem;
+ private final String pathBase;
+
+ public HDFSLocationFactory(Configuration configuration) {
+ this(getFileSystem(configuration));
+ }
+
+ public HDFSLocationFactory(Configuration configuration, String pathBase) {
+ this(getFileSystem(configuration), pathBase);
+ }
+
+ public HDFSLocationFactory(FileSystem fileSystem) {
+ this(fileSystem, "/");
+ }
+
+ public HDFSLocationFactory(FileSystem fileSystem, String pathBase) {
+ String base = pathBase.equals("/") ? "" : pathBase;
+ base = base.endsWith("/") ? base.substring(0, base.length() - 1) : base;
+
+ this.fileSystem = fileSystem;
+ this.pathBase = base;
+ }
+
+ @Override
+ public Location create(String path) {
+ if (path.startsWith("/")) {
+ path = path.substring(1);
+ }
+ return new HDFSLocation(fileSystem, new Path(fileSystem.getUri() + "/" + pathBase + "/" + path));
+ }
+
+ @Override
+ public Location create(URI uri) {
+ if (!uri.toString().startsWith(fileSystem.getUri().toString())) {
+ // It's a full URI
+ return new HDFSLocation(fileSystem, new Path(uri));
+ }
+ if (uri.isAbsolute()) {
+ return new HDFSLocation(fileSystem, new Path(fileSystem.getUri() + uri.getPath()));
+ }
+ return new HDFSLocation(fileSystem, new Path(fileSystem.getUri() + "/" + pathBase + "/" + uri.getPath()));
+ }
+
+ @Override
+ public Location getHomeLocation() {
+ return new HDFSLocation(fileSystem, fileSystem.getHomeDirectory());
+ }
+
+ /**
+ * Returns the underlying {@link FileSystem} object.
+ */
+ public FileSystem getFileSystem() {
+ return fileSystem;
+ }
+
+ private static FileSystem getFileSystem(Configuration configuration) {
+ try {
+ return FileSystem.get(configuration);
+ } catch (IOException e) {
+ throw Throwables.propagate(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/filesystem/package-info.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/filesystem/package-info.java b/yarn/src/main/java/org/apache/twill/filesystem/package-info.java
new file mode 100644
index 0000000..2ca09fd
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/filesystem/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Contains HDFS location classes.
+ */
+package org.apache.twill.filesystem;
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/internal/AbstractTwillService.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/internal/AbstractTwillService.java b/yarn/src/main/java/org/apache/twill/internal/AbstractTwillService.java
new file mode 100644
index 0000000..47dd07c
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/internal/AbstractTwillService.java
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal;
+
+import org.apache.twill.filesystem.Location;
+import org.apache.twill.internal.state.Message;
+import org.apache.twill.internal.state.SystemMessages;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.Service;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedInputStream;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.util.concurrent.Executor;
+
+/**
+ * A base implementation of {@link Service} handle secure token update.
+ */
+public abstract class AbstractTwillService implements Service {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AbstractTwillService.class);
+
+ protected final Location applicationLocation;
+
+ protected volatile Credentials credentials;
+
+ protected AbstractTwillService(Location applicationLocation) {
+ this.applicationLocation = applicationLocation;
+ }
+
+ protected abstract Service getServiceDelegate();
+
+ /**
+ * Returns the location of the secure store, or {@code null} if either not running in secure mode or an error
+ * occur when trying to acquire the location.
+ */
+ protected final Location getSecureStoreLocation() {
+ if (!UserGroupInformation.isSecurityEnabled()) {
+ return null;
+ }
+ try {
+ return applicationLocation.append(Constants.Files.CREDENTIALS);
+ } catch (IOException e) {
+ LOG.error("Failed to create secure store location.", e);
+ return null;
+ }
+ }
+
+ /**
+ * Attempts to handle secure store update.
+ *
+ * @param message The message received
+ * @return {@code true} if the message requests for secure store update, {@code false} otherwise.
+ */
+ protected final boolean handleSecureStoreUpdate(Message message) {
+ if (!SystemMessages.SECURE_STORE_UPDATED.equals(message)) {
+ return false;
+ }
+
+ // If not in secure mode, simply ignore the message.
+ if (!UserGroupInformation.isSecurityEnabled()) {
+ return true;
+ }
+
+ try {
+ Credentials credentials = new Credentials();
+ Location location = getSecureStoreLocation();
+ DataInputStream input = new DataInputStream(new BufferedInputStream(location.getInputStream()));
+ try {
+ credentials.readTokenStorageStream(input);
+ } finally {
+ input.close();
+ }
+
+ UserGroupInformation.getCurrentUser().addCredentials(credentials);
+ this.credentials = credentials;
+
+ LOG.info("Secure store updated from {}.", location.toURI());
+
+ } catch (Throwable t) {
+ LOG.error("Failed to update secure store.", t);
+ }
+
+ return true;
+ }
+
+ @Override
+ public final ListenableFuture<State> start() {
+ return getServiceDelegate().start();
+ }
+
+ @Override
+ public final State startAndWait() {
+ return Futures.getUnchecked(start());
+ }
+
+ @Override
+ public final boolean isRunning() {
+ return getServiceDelegate().isRunning();
+ }
+
+ @Override
+ public final State state() {
+ return getServiceDelegate().state();
+ }
+
+ @Override
+ public final ListenableFuture<State> stop() {
+ return getServiceDelegate().stop();
+ }
+
+ @Override
+ public final State stopAndWait() {
+ return Futures.getUnchecked(stop());
+ }
+
+ @Override
+ public final void addListener(Listener listener, Executor executor) {
+ getServiceDelegate().addListener(listener, executor);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/internal/ServiceMain.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/internal/ServiceMain.java b/yarn/src/main/java/org/apache/twill/internal/ServiceMain.java
new file mode 100644
index 0000000..4ffb023
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/internal/ServiceMain.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal;
+
+import org.apache.twill.common.Services;
+import org.apache.twill.filesystem.HDFSLocationFactory;
+import org.apache.twill.filesystem.LocalLocationFactory;
+import org.apache.twill.filesystem.Location;
+import org.apache.twill.internal.logging.KafkaAppender;
+import org.apache.twill.zookeeper.ZKClientService;
+import ch.qos.logback.classic.LoggerContext;
+import ch.qos.logback.classic.joran.JoranConfigurator;
+import ch.qos.logback.classic.util.ContextInitializer;
+import ch.qos.logback.core.joran.spi.JoranException;
+import com.google.common.base.Throwables;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.Service;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.slf4j.ILoggerFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.xml.sax.InputSource;
+
+import java.io.File;
+import java.io.StringReader;
+import java.net.URI;
+import java.util.concurrent.ExecutionException;
+
+/**
+ * Class for main method that starts a service.
+ */
+public abstract class ServiceMain {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ServiceMain.class);
+
+ static {
+ // This is to work around detection of HADOOP_HOME (HADOOP-9422)
+ if (!System.getenv().containsKey("HADOOP_HOME") && System.getProperty("hadoop.home.dir") == null) {
+ System.setProperty("hadoop.home.dir", new File("").getAbsolutePath());
+ }
+ }
+
+ protected final void doMain(final ZKClientService zkClientService,
+ final Service service) throws ExecutionException, InterruptedException {
+ configureLogger();
+
+ final String serviceName = service.toString();
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ Services.chainStop(service, zkClientService);
+ }
+ });
+
+ // Listener for state changes of the service
+ ListenableFuture<Service.State> completion = Services.getCompletionFuture(service);
+
+ // Starts the service
+ LOG.info("Starting service {}.", serviceName);
+ Futures.getUnchecked(Services.chainStart(zkClientService, service));
+ LOG.info("Service {} started.", serviceName);
+ try {
+ completion.get();
+ LOG.info("Service {} completed.", serviceName);
+ } catch (Throwable t) {
+ LOG.warn("Exception thrown from service {}.", serviceName, t);
+ throw Throwables.propagate(t);
+ } finally {
+ ILoggerFactory loggerFactory = LoggerFactory.getILoggerFactory();
+ if (loggerFactory instanceof LoggerContext) {
+ ((LoggerContext) loggerFactory).stop();
+ }
+ }
+ }
+
+ protected abstract String getHostname();
+
+ protected abstract String getKafkaZKConnect();
+
+ /**
+ * Returns the {@link Location} for the application based on the env {@link EnvKeys#TWILL_APP_DIR}.
+ */
+ protected static Location createAppLocation(Configuration conf) {
+ // Note: It's a little bit hacky based on the uri schema to create the LocationFactory, refactor it later.
+ URI appDir = URI.create(System.getenv(EnvKeys.TWILL_APP_DIR));
+
+ try {
+ if ("file".equals(appDir.getScheme())) {
+ return new LocalLocationFactory().create(appDir);
+ }
+
+ if ("hdfs".equals(appDir.getScheme())) {
+ if (UserGroupInformation.isSecurityEnabled()) {
+ return new HDFSLocationFactory(FileSystem.get(conf)).create(appDir);
+ }
+
+ String fsUser = System.getenv(EnvKeys.TWILL_FS_USER);
+ if (fsUser == null) {
+ throw new IllegalStateException("Missing environment variable " + EnvKeys.TWILL_FS_USER);
+ }
+ return new HDFSLocationFactory(FileSystem.get(FileSystem.getDefaultUri(conf), conf, fsUser)).create(appDir);
+ }
+
+ LOG.warn("Unsupported location type {}.", appDir);
+ throw new IllegalArgumentException("Unsupported location type " + appDir);
+
+ } catch (Exception e) {
+ LOG.error("Failed to create application location for {}.", appDir);
+ throw Throwables.propagate(e);
+ }
+ }
+
+ private void configureLogger() {
+ // Check if SLF4J is bound to logback in the current environment
+ ILoggerFactory loggerFactory = LoggerFactory.getILoggerFactory();
+ if (!(loggerFactory instanceof LoggerContext)) {
+ return;
+ }
+
+ LoggerContext context = (LoggerContext) loggerFactory;
+ context.reset();
+ JoranConfigurator configurator = new JoranConfigurator();
+ configurator.setContext(context);
+
+ try {
+ File twillLogback = new File(Constants.Files.LOGBACK_TEMPLATE);
+ if (twillLogback.exists()) {
+ configurator.doConfigure(twillLogback);
+ }
+ new ContextInitializer(context).autoConfig();
+ } catch (JoranException e) {
+ throw Throwables.propagate(e);
+ }
+ doConfigure(configurator, getLogConfig(getLoggerLevel(context.getLogger(Logger.ROOT_LOGGER_NAME))));
+ }
+
+ private void doConfigure(JoranConfigurator configurator, String config) {
+ try {
+ configurator.doConfigure(new InputSource(new StringReader(config)));
+ } catch (Exception e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ private String getLogConfig(String rootLevel) {
+ return
+ "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" +
+ "<configuration>\n" +
+ " <appender name=\"KAFKA\" class=\"" + KafkaAppender.class.getName() + "\">\n" +
+ " <topic>" + Constants.LOG_TOPIC + "</topic>\n" +
+ " <hostname>" + getHostname() + "</hostname>\n" +
+ " <zookeeper>" + getKafkaZKConnect() + "</zookeeper>\n" +
+ " </appender>\n" +
+ " <logger name=\"org.apache.twill.internal.logging\" additivity=\"false\" />\n" +
+ " <root level=\"" + rootLevel + "\">\n" +
+ " <appender-ref ref=\"KAFKA\"/>\n" +
+ " </root>\n" +
+ "</configuration>";
+ }
+
+ private String getLoggerLevel(Logger logger) {
+ if (logger instanceof ch.qos.logback.classic.Logger) {
+ return ((ch.qos.logback.classic.Logger) logger).getLevel().toString();
+ }
+
+ if (logger.isTraceEnabled()) {
+ return "TRACE";
+ }
+ if (logger.isDebugEnabled()) {
+ return "DEBUG";
+ }
+ if (logger.isInfoEnabled()) {
+ return "INFO";
+ }
+ if (logger.isWarnEnabled()) {
+ return "WARN";
+ }
+ if (logger.isErrorEnabled()) {
+ return "ERROR";
+ }
+ return "OFF";
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterLiveNodeData.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterLiveNodeData.java b/yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterLiveNodeData.java
new file mode 100644
index 0000000..028df7b
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterLiveNodeData.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.appmaster;
+
+/**
+ * Represents data being stored in the live node of the application master.
+ */
+public final class ApplicationMasterLiveNodeData {
+
+ private final int appId;
+ private final long appIdClusterTime;
+ private final String containerId;
+
+ public ApplicationMasterLiveNodeData(int appId, long appIdClusterTime, String containerId) {
+ this.appId = appId;
+ this.appIdClusterTime = appIdClusterTime;
+ this.containerId = containerId;
+ }
+
+ public int getAppId() {
+ return appId;
+ }
+
+ public long getAppIdClusterTime() {
+ return appIdClusterTime;
+ }
+
+ public String getContainerId() {
+ return containerId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterMain.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterMain.java b/yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterMain.java
new file mode 100644
index 0000000..b34a7a2
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterMain.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.appmaster;
+
+import org.apache.twill.api.RunId;
+import org.apache.twill.internal.Constants;
+import org.apache.twill.internal.EnvKeys;
+import org.apache.twill.internal.RunIds;
+import org.apache.twill.internal.ServiceMain;
+import org.apache.twill.internal.yarn.VersionDetectYarnAMClientFactory;
+import org.apache.twill.zookeeper.RetryStrategies;
+import org.apache.twill.zookeeper.ZKClientService;
+import org.apache.twill.zookeeper.ZKClientServices;
+import org.apache.twill.zookeeper.ZKClients;
+import com.google.common.util.concurrent.Service;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+
+import java.io.File;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Main class for launching {@link ApplicationMasterService}.
+ */
+public final class ApplicationMasterMain extends ServiceMain {
+
+ private final String kafkaZKConnect;
+
+ private ApplicationMasterMain(String kafkaZKConnect) {
+ this.kafkaZKConnect = kafkaZKConnect;
+ }
+
+ /**
+ * Starts the application master.
+ */
+ public static void main(String[] args) throws Exception {
+ String zkConnect = System.getenv(EnvKeys.TWILL_ZK_CONNECT);
+ File twillSpec = new File(Constants.Files.TWILL_SPEC);
+ RunId runId = RunIds.fromString(System.getenv(EnvKeys.TWILL_RUN_ID));
+
+ ZKClientService zkClientService =
+ ZKClientServices.delegate(
+ ZKClients.reWatchOnExpire(
+ ZKClients.retryOnFailure(
+ ZKClientService.Builder.of(zkConnect).build(),
+ RetryStrategies.fixDelay(1, TimeUnit.SECONDS))));
+
+ Configuration conf = new YarnConfiguration(new HdfsConfiguration(new Configuration()));
+ Service service = new ApplicationMasterService(runId, zkClientService, twillSpec,
+ new VersionDetectYarnAMClientFactory(conf), createAppLocation(conf));
+ new ApplicationMasterMain(String.format("%s/%s/kafka", zkConnect, runId.getId())).doMain(zkClientService, service);
+ }
+
+ @Override
+ protected String getHostname() {
+ try {
+ return InetAddress.getLocalHost().getCanonicalHostName();
+ } catch (UnknownHostException e) {
+ return "unknown";
+ }
+ }
+
+ @Override
+ protected String getKafkaZKConnect() {
+ return kafkaZKConnect;
+ }
+}
[03/15] Initial import commit.
Posted by ch...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/test/java/org/apache/twill/yarn/ResourceReportTestRun.java
----------------------------------------------------------------------
diff --git a/yarn/src/test/java/org/apache/twill/yarn/ResourceReportTestRun.java b/yarn/src/test/java/org/apache/twill/yarn/ResourceReportTestRun.java
new file mode 100644
index 0000000..131f90a
--- /dev/null
+++ b/yarn/src/test/java/org/apache/twill/yarn/ResourceReportTestRun.java
@@ -0,0 +1,268 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.yarn;
+
+import org.apache.twill.api.ResourceReport;
+import org.apache.twill.api.ResourceSpecification;
+import org.apache.twill.api.TwillApplication;
+import org.apache.twill.api.TwillController;
+import org.apache.twill.api.TwillRunResources;
+import org.apache.twill.api.TwillRunner;
+import org.apache.twill.api.TwillSpecification;
+import org.apache.twill.api.logging.PrinterLogHandler;
+import org.apache.twill.common.ServiceListenerAdapter;
+import org.apache.twill.common.Threads;
+import org.apache.twill.discovery.Discoverable;
+import org.apache.twill.internal.EnvKeys;
+import com.google.common.base.Charsets;
+import com.google.common.collect.Maps;
+import com.google.common.io.LineReader;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.net.Socket;
+import java.net.URISyntaxException;
+import java.util.Collection;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * Using echo server to test resource reports.
+ * This test is executed by {@link org.apache.twill.yarn.YarnTestSuite}.
+ */
+public class ResourceReportTestRun {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ResourceReportTestRun.class);
+
+ private class ResourceApplication implements TwillApplication {
+ @Override
+ public TwillSpecification configure() {
+ return TwillSpecification.Builder.with()
+ .setName("ResourceApplication")
+ .withRunnable()
+ .add("echo1", new EchoServer(), ResourceSpecification.Builder.with()
+ .setVirtualCores(1)
+ .setMemory(128, ResourceSpecification.SizeUnit.MEGA)
+ .setInstances(2).build()).noLocalFiles()
+ .add("echo2", new EchoServer(), ResourceSpecification.Builder.with()
+ .setVirtualCores(2)
+ .setMemory(256, ResourceSpecification.SizeUnit.MEGA)
+ .setInstances(1).build()).noLocalFiles()
+ .anyOrder()
+ .build();
+ }
+ }
+
+ @Test
+ public void testRunnablesGetAllowedResourcesInEnv() throws InterruptedException, IOException,
+ TimeoutException, ExecutionException {
+ TwillRunner runner = YarnTestSuite.getTwillRunner();
+
+ ResourceSpecification resourceSpec = ResourceSpecification.Builder.with()
+ .setVirtualCores(1)
+ .setMemory(2048, ResourceSpecification.SizeUnit.MEGA)
+ .setInstances(1)
+ .build();
+ TwillController controller = runner.prepare(new EnvironmentEchoServer(), resourceSpec)
+ .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
+ .withApplicationArguments("envecho")
+ .withArguments("EnvironmentEchoServer", "echo2")
+ .start();
+
+ final CountDownLatch running = new CountDownLatch(1);
+ controller.addListener(new ServiceListenerAdapter() {
+ @Override
+ public void running() {
+ running.countDown();
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+
+ Assert.assertTrue(running.await(30, TimeUnit.SECONDS));
+
+ Iterable<Discoverable> envEchoServices = controller.discoverService("envecho");
+ Assert.assertTrue(YarnTestSuite.waitForSize(envEchoServices, 1, 30));
+
+ // TODO: check virtual cores once yarn adds the ability
+ Map<String, String> expectedValues = Maps.newHashMap();
+ expectedValues.put(EnvKeys.YARN_CONTAINER_MEMORY_MB, "2048");
+ expectedValues.put(EnvKeys.TWILL_INSTANCE_COUNT, "1");
+
+ // check environment of the runnable.
+ Discoverable discoverable = envEchoServices.iterator().next();
+ for (Map.Entry<String, String> expected : expectedValues.entrySet()) {
+ Socket socket = new Socket(discoverable.getSocketAddress().getHostName(),
+ discoverable.getSocketAddress().getPort());
+ try {
+ PrintWriter writer = new PrintWriter(new OutputStreamWriter(socket.getOutputStream(), Charsets.UTF_8), true);
+ LineReader reader = new LineReader(new InputStreamReader(socket.getInputStream(), Charsets.UTF_8));
+ writer.println(expected.getKey());
+ Assert.assertEquals(expected.getValue(), reader.readLine());
+ } finally {
+ socket.close();
+ }
+ }
+
+ controller.stop().get(30, TimeUnit.SECONDS);
+ // Sleep a bit before exiting.
+ TimeUnit.SECONDS.sleep(2);
+ }
+
+ @Test
+ public void testResourceReportWithFailingContainers() throws InterruptedException, IOException,
+ TimeoutException, ExecutionException {
+ TwillRunner runner = YarnTestSuite.getTwillRunner();
+
+ ResourceSpecification resourceSpec = ResourceSpecification.Builder.with()
+ .setVirtualCores(1)
+ .setMemory(128, ResourceSpecification.SizeUnit.MEGA)
+ .setInstances(2)
+ .build();
+ TwillController controller = runner.prepare(new BuggyServer(), resourceSpec)
+ .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
+ .withApplicationArguments("echo")
+ .withArguments("BuggyServer", "echo2")
+ .start();
+
+ final CountDownLatch running = new CountDownLatch(1);
+ controller.addListener(new ServiceListenerAdapter() {
+ @Override
+ public void running() {
+ running.countDown();
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+
+ Assert.assertTrue(running.await(30, TimeUnit.SECONDS));
+
+ Iterable<Discoverable> echoServices = controller.discoverService("echo");
+ Assert.assertTrue(YarnTestSuite.waitForSize(echoServices, 2, 60));
+ // check that we have 2 runnables.
+ ResourceReport report = controller.getResourceReport();
+ Assert.assertEquals(2, report.getRunnableResources("BuggyServer").size());
+
+ // cause a divide by 0 in one server
+ Discoverable discoverable = echoServices.iterator().next();
+ Socket socket = new Socket(discoverable.getSocketAddress().getAddress(),
+ discoverable.getSocketAddress().getPort());
+ try {
+ PrintWriter writer = new PrintWriter(new OutputStreamWriter(socket.getOutputStream(), Charsets.UTF_8), true);
+ writer.println("0");
+ } finally {
+ socket.close();
+ }
+
+ // takes some time for app master to find out the container completed...
+ TimeUnit.SECONDS.sleep(5);
+ // check that we have 1 runnable, not 2.
+ report = controller.getResourceReport();
+ Assert.assertEquals(1, report.getRunnableResources("BuggyServer").size());
+
+ controller.stop().get(30, TimeUnit.SECONDS);
+ // Sleep a bit before exiting.
+ TimeUnit.SECONDS.sleep(2);
+ }
+
+ @Test
+ public void testResourceReport() throws InterruptedException, ExecutionException, IOException,
+ URISyntaxException, TimeoutException {
+ TwillRunner runner = YarnTestSuite.getTwillRunner();
+
+ TwillController controller = runner.prepare(new ResourceApplication())
+ .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
+ .withApplicationArguments("echo")
+ .withArguments("echo1", "echo1")
+ .withArguments("echo2", "echo2")
+ .start();
+
+ final CountDownLatch running = new CountDownLatch(1);
+ controller.addListener(new ServiceListenerAdapter() {
+ @Override
+ public void running() {
+ running.countDown();
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+
+ Assert.assertTrue(running.await(30, TimeUnit.SECONDS));
+
+ // wait for 3 echo servers to come up
+ Iterable<Discoverable> echoServices = controller.discoverService("echo");
+ Assert.assertTrue(YarnTestSuite.waitForSize(echoServices, 3, 60));
+ ResourceReport report = controller.getResourceReport();
+ // make sure resources for echo1 and echo2 are there
+ Map<String, Collection<TwillRunResources>> usedResources = report.getResources();
+ Assert.assertEquals(2, usedResources.keySet().size());
+ Assert.assertTrue(usedResources.containsKey("echo1"));
+ Assert.assertTrue(usedResources.containsKey("echo2"));
+
+ Collection<TwillRunResources> echo1Resources = usedResources.get("echo1");
+ // 2 instances of echo1
+ Assert.assertEquals(2, echo1Resources.size());
+ // TODO: check cores after hadoop-2.1.0
+ for (TwillRunResources resources : echo1Resources) {
+ Assert.assertEquals(128, resources.getMemoryMB());
+ }
+
+ Collection<TwillRunResources> echo2Resources = usedResources.get("echo2");
+ // 2 instances of echo1
+ Assert.assertEquals(1, echo2Resources.size());
+ // TODO: check cores after hadoop-2.1.0
+ for (TwillRunResources resources : echo2Resources) {
+ Assert.assertEquals(256, resources.getMemoryMB());
+ }
+
+ // Decrease number of instances of echo1 from 2 to 1
+ controller.changeInstances("echo1", 1);
+ echoServices = controller.discoverService("echo1");
+ Assert.assertTrue(YarnTestSuite.waitForSize(echoServices, 1, 60));
+ report = controller.getResourceReport();
+
+ // make sure resources for echo1 and echo2 are there
+ usedResources = report.getResources();
+ Assert.assertEquals(2, usedResources.keySet().size());
+ Assert.assertTrue(usedResources.containsKey("echo1"));
+ Assert.assertTrue(usedResources.containsKey("echo2"));
+
+ echo1Resources = usedResources.get("echo1");
+ // 1 instance of echo1 now
+ Assert.assertEquals(1, echo1Resources.size());
+ // TODO: check cores after hadoop-2.1.0
+ for (TwillRunResources resources : echo1Resources) {
+ Assert.assertEquals(128, resources.getMemoryMB());
+ }
+
+ echo2Resources = usedResources.get("echo2");
+ // 2 instances of echo1
+ Assert.assertEquals(1, echo2Resources.size());
+ // TODO: check cores after hadoop-2.1.0
+ for (TwillRunResources resources : echo2Resources) {
+ Assert.assertEquals(256, resources.getMemoryMB());
+ }
+
+ controller.stop().get(30, TimeUnit.SECONDS);
+ // Sleep a bit before exiting.
+ TimeUnit.SECONDS.sleep(2);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/test/java/org/apache/twill/yarn/SocketServer.java
----------------------------------------------------------------------
diff --git a/yarn/src/test/java/org/apache/twill/yarn/SocketServer.java b/yarn/src/test/java/org/apache/twill/yarn/SocketServer.java
new file mode 100644
index 0000000..5148ed2
--- /dev/null
+++ b/yarn/src/test/java/org/apache/twill/yarn/SocketServer.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.yarn;
+
+import org.apache.twill.api.AbstractTwillRunnable;
+import org.apache.twill.api.TwillContext;
+import org.apache.twill.api.TwillContext;
+import org.apache.twill.common.Cancellable;
+import com.google.common.base.Charsets;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.net.ServerSocket;
+import java.net.Socket;
+import java.net.SocketException;
+import java.util.List;
+
+/**
+ * Boilerplate for a server that announces itself and talks to clients through a socket.
+ */
+public abstract class SocketServer extends AbstractTwillRunnable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(SocketServer.class);
+
+ protected volatile boolean running;
+ protected volatile Thread runThread;
+ protected ServerSocket serverSocket;
+ protected Cancellable canceller;
+
+ @Override
+ public void initialize(TwillContext context) {
+ super.initialize(context);
+ running = true;
+ try {
+ serverSocket = new ServerSocket(0);
+ LOG.info("Server started: " + serverSocket.getLocalSocketAddress() +
+ ", id: " + context.getInstanceId() +
+ ", count: " + context.getInstanceCount());
+
+ final List<Cancellable> cancellables = ImmutableList.of(
+ context.announce(context.getApplicationArguments()[0], serverSocket.getLocalPort()),
+ context.announce(context.getArguments()[0], serverSocket.getLocalPort())
+ );
+ canceller = new Cancellable() {
+ @Override
+ public void cancel() {
+ for (Cancellable c : cancellables) {
+ c.cancel();
+ }
+ }
+ };
+ } catch (IOException e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ public void run() {
+ try {
+ runThread = Thread.currentThread();
+ while (running) {
+ try {
+ Socket socket = serverSocket.accept();
+ try {
+ BufferedReader reader = new BufferedReader(new InputStreamReader(socket.getInputStream(), Charsets.UTF_8));
+ PrintWriter writer = new PrintWriter(new OutputStreamWriter(socket.getOutputStream()), true);
+ handleRequest(reader, writer);
+ } finally {
+ socket.close();
+ }
+ } catch (SocketException e) {
+ LOG.info("Socket exception: " + e);
+ }
+ }
+ } catch (Exception e) {
+ LOG.error(e.getMessage(), e);
+ }
+ }
+
+ @Override
+ public void stop() {
+ LOG.info("Stopping server");
+ canceller.cancel();
+ running = false;
+ Thread t = runThread;
+ if (t != null) {
+ t.interrupt();
+ }
+ try {
+ serverSocket.close();
+ } catch (IOException e) {
+ LOG.error("Exception while closing socket.", e);
+ throw Throwables.propagate(e);
+ }
+ serverSocket = null;
+ }
+
+ @Override
+ public void destroy() {
+ try {
+ if (serverSocket != null) {
+ serverSocket.close();
+ }
+ } catch (IOException e) {
+ LOG.error("Exception while closing socket.", e);
+ throw Throwables.propagate(e);
+ }
+ }
+
+ abstract public void handleRequest(BufferedReader reader, PrintWriter writer) throws IOException;
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/test/java/org/apache/twill/yarn/TaskCompletedTestRun.java
----------------------------------------------------------------------
diff --git a/yarn/src/test/java/org/apache/twill/yarn/TaskCompletedTestRun.java b/yarn/src/test/java/org/apache/twill/yarn/TaskCompletedTestRun.java
new file mode 100644
index 0000000..5a93271
--- /dev/null
+++ b/yarn/src/test/java/org/apache/twill/yarn/TaskCompletedTestRun.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.yarn;
+
+import org.apache.twill.api.AbstractTwillRunnable;
+import org.apache.twill.api.ResourceSpecification;
+import org.apache.twill.api.TwillController;
+import org.apache.twill.api.TwillRunner;
+import org.apache.twill.api.logging.PrinterLogHandler;
+import org.apache.twill.common.ServiceListenerAdapter;
+import org.apache.twill.common.Threads;
+import com.google.common.base.Throwables;
+import com.google.common.util.concurrent.Service;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.PrintWriter;
+import java.util.Random;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Testing application master will shutdown itself when all tasks are completed.
+ * This test is executed by {@link YarnTestSuite}.
+ */
+public class TaskCompletedTestRun {
+
+ public static final class SleepTask extends AbstractTwillRunnable {
+
+ @Override
+ public void run() {
+ // Randomly sleep for 3-5 seconds.
+ try {
+ TimeUnit.SECONDS.sleep(new Random().nextInt(3) + 3);
+ } catch (InterruptedException e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ public void stop() {
+ // No-op
+ }
+ }
+
+ @Test
+ public void testTaskCompleted() throws InterruptedException {
+ TwillRunner twillRunner = YarnTestSuite.getTwillRunner();
+ TwillController controller = twillRunner.prepare(new SleepTask(),
+ ResourceSpecification.Builder.with()
+ .setVirtualCores(1)
+ .setMemory(512, ResourceSpecification.SizeUnit.MEGA)
+ .setInstances(3).build())
+ .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
+ .start();
+
+ final CountDownLatch runLatch = new CountDownLatch(1);
+ final CountDownLatch stopLatch = new CountDownLatch(1);
+ controller.addListener(new ServiceListenerAdapter() {
+
+ @Override
+ public void running() {
+ runLatch.countDown();
+ }
+
+ @Override
+ public void terminated(Service.State from) {
+ stopLatch.countDown();
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+
+ Assert.assertTrue(runLatch.await(1, TimeUnit.MINUTES));
+
+ Assert.assertTrue(stopLatch.await(1, TimeUnit.MINUTES));
+
+ TimeUnit.SECONDS.sleep(2);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/test/java/org/apache/twill/yarn/TwillSpecificationTest.java
----------------------------------------------------------------------
diff --git a/yarn/src/test/java/org/apache/twill/yarn/TwillSpecificationTest.java b/yarn/src/test/java/org/apache/twill/yarn/TwillSpecificationTest.java
new file mode 100644
index 0000000..8be907b
--- /dev/null
+++ b/yarn/src/test/java/org/apache/twill/yarn/TwillSpecificationTest.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.yarn;
+
+import org.apache.twill.api.AbstractTwillRunnable;
+import org.apache.twill.api.TwillSpecification;
+import com.google.common.collect.ImmutableSet;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.List;
+
+/**
+ *
+ */
+public class TwillSpecificationTest {
+
+ /**
+ * Dummy for test.
+ */
+ public static final class DummyRunnable extends AbstractTwillRunnable {
+
+ @Override
+ public void stop() {
+ // no-op
+ }
+
+ @Override
+ public void run() {
+ // no-op
+ }
+ }
+
+ @Test
+ public void testAnyOrder() {
+ TwillSpecification spec =
+ TwillSpecification.Builder.with()
+ .setName("Testing")
+ .withRunnable()
+ .add("r1", new DummyRunnable()).noLocalFiles()
+ .add("r2", new DummyRunnable()).noLocalFiles()
+ .add("r3", new DummyRunnable()).noLocalFiles()
+ .anyOrder()
+ .build();
+
+ Assert.assertEquals(3, spec.getRunnables().size());
+ List<TwillSpecification.Order> orders = spec.getOrders();
+ Assert.assertEquals(1, orders.size());
+ Assert.assertEquals(ImmutableSet.of("r1", "r2", "r3"), orders.get(0).getNames());
+ }
+
+ @Test
+ public void testOrder() {
+ TwillSpecification spec =
+ TwillSpecification.Builder.with()
+ .setName("Testing")
+ .withRunnable()
+ .add("r1", new DummyRunnable()).noLocalFiles()
+ .add("r2", new DummyRunnable()).noLocalFiles()
+ .add("r3", new DummyRunnable()).noLocalFiles()
+ .add("r4", new DummyRunnable()).noLocalFiles()
+ .withOrder().begin("r1", "r2").nextWhenStarted("r3")
+ .build();
+
+ Assert.assertEquals(4, spec.getRunnables().size());
+ List<TwillSpecification.Order> orders = spec.getOrders();
+ Assert.assertEquals(3, orders.size());
+ Assert.assertEquals(ImmutableSet.of("r1", "r2"), orders.get(0).getNames());
+ Assert.assertEquals(ImmutableSet.of("r3"), orders.get(1).getNames());
+ Assert.assertEquals(ImmutableSet.of("r4"), orders.get(2).getNames());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/test/java/org/apache/twill/yarn/YarnTestSuite.java
----------------------------------------------------------------------
diff --git a/yarn/src/test/java/org/apache/twill/yarn/YarnTestSuite.java b/yarn/src/test/java/org/apache/twill/yarn/YarnTestSuite.java
new file mode 100644
index 0000000..b55d620
--- /dev/null
+++ b/yarn/src/test/java/org/apache/twill/yarn/YarnTestSuite.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.yarn;
+
+import org.apache.twill.api.TwillRunner;
+import org.apache.twill.api.TwillRunnerService;
+import org.apache.twill.filesystem.LocalLocationFactory;
+import org.apache.twill.internal.zookeeper.InMemoryZKServer;
+import org.apache.twill.internal.yarn.YarnUtils;
+import com.google.common.collect.Iterables;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.MiniYARNCluster;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.rules.TemporaryFolder;
+import org.junit.runner.RunWith;
+import org.junit.runners.Suite;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Test suite for all tests with mini yarn cluster.
+ */
+@RunWith(Suite.class)
+@Suite.SuiteClasses({
+ EchoServerTestRun.class,
+ ResourceReportTestRun.class,
+ TaskCompletedTestRun.class,
+ DistributeShellTestRun.class,
+ LocalFileTestRun.class,
+ FailureRestartTestRun.class,
+ ProvisionTimeoutTestRun.class
+ })
+public class YarnTestSuite {
+ private static final Logger LOG = LoggerFactory.getLogger(YarnTestSuite.class);
+
+ @ClassRule
+ public static TemporaryFolder tmpFolder = new TemporaryFolder();
+
+ private static InMemoryZKServer zkServer;
+ private static MiniYARNCluster cluster;
+ private static TwillRunnerService runnerService;
+ private static YarnConfiguration config;
+
+ @BeforeClass
+ public static final void init() throws IOException {
+ // Starts Zookeeper
+ zkServer = InMemoryZKServer.builder().build();
+ zkServer.startAndWait();
+
+ // Start YARN mini cluster
+ config = new YarnConfiguration(new Configuration());
+
+ if (YarnUtils.isHadoop20()) {
+ config.set("yarn.resourcemanager.scheduler.class",
+ "org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler");
+ } else {
+ config.set("yarn.resourcemanager.scheduler.class",
+ "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler");
+ config.set("yarn.scheduler.capacity.resource-calculator",
+ "org.apache.hadoop.yarn.util.resource.DominantResourceCalculator");
+ }
+ config.set("yarn.minicluster.fixed.ports", "true");
+ config.set("yarn.nodemanager.vmem-pmem-ratio", "20.1");
+ config.set("yarn.nodemanager.vmem-check-enabled", "false");
+ config.set("yarn.scheduler.minimum-allocation-mb", "128");
+ config.set("yarn.nodemanager.delete.debug-delay-sec", "3600");
+
+ cluster = new MiniYARNCluster("test-cluster", 1, 1, 1);
+ cluster.init(config);
+ cluster.start();
+
+ runnerService = createTwillRunnerService();
+ runnerService.startAndWait();
+ }
+
+ @AfterClass
+ public static final void finish() {
+ runnerService.stopAndWait();
+ cluster.stop();
+ zkServer.stopAndWait();
+ }
+
+ public static final TwillRunner getTwillRunner() {
+ return runnerService;
+ }
+
+ /**
+ * Creates an unstarted instance of {@link org.apache.twill.api.TwillRunnerService}.
+ */
+ public static final TwillRunnerService createTwillRunnerService() throws IOException {
+ return new YarnTwillRunnerService(config, zkServer.getConnectionStr() + "/twill",
+ new LocalLocationFactory(tmpFolder.newFolder()));
+ }
+
+ public static final <T> boolean waitForSize(Iterable<T> iterable, int count, int limit) throws InterruptedException {
+ int trial = 0;
+ int size = Iterables.size(iterable);
+ while (size != count && trial < limit) {
+ LOG.info("Waiting for {} size {} == {}", iterable, size, count);
+ TimeUnit.SECONDS.sleep(1);
+ trial++;
+ size = Iterables.size(iterable);
+ }
+ return trial < limit;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/test/resources/header.txt
----------------------------------------------------------------------
diff --git a/yarn/src/test/resources/header.txt b/yarn/src/test/resources/header.txt
new file mode 100644
index 0000000..b6e25e6
--- /dev/null
+++ b/yarn/src/test/resources/header.txt
@@ -0,0 +1 @@
+Local file header
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/test/resources/logback-test.xml
----------------------------------------------------------------------
diff --git a/yarn/src/test/resources/logback-test.xml b/yarn/src/test/resources/logback-test.xml
new file mode 100644
index 0000000..2615cb4
--- /dev/null
+++ b/yarn/src/test/resources/logback-test.xml
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!-- Default logback configuration for twill library -->
+<configuration>
+ <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
+ <encoder>
+ <pattern>%d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n</pattern>
+ </encoder>
+ </appender>
+
+ <logger name="org.apache.twill" level="DEBUG" />
+
+ <root level="WARN">
+ <appender-ref ref="STDOUT"/>
+ </root>
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/zookeeper/pom.xml
----------------------------------------------------------------------
diff --git a/zookeeper/pom.xml b/zookeeper/pom.xml
new file mode 100644
index 0000000..109eb8f
--- /dev/null
+++ b/zookeeper/pom.xml
@@ -0,0 +1,67 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>twill-parent</artifactId>
+ <groupId>org.apache.twill</groupId>
+ <version>1.3.0-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>twill-zookeeper</artifactId>
+ <name>Twill ZooKeeper client library</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>twill-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.zookeeper</groupId>
+ <artifactId>zookeeper</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>ch.qos.logback</groupId>
+ <artifactId>logback-classic</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/BasicNodeChildren.java
----------------------------------------------------------------------
diff --git a/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/BasicNodeChildren.java b/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/BasicNodeChildren.java
new file mode 100644
index 0000000..9e4f55f
--- /dev/null
+++ b/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/BasicNodeChildren.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.zookeeper;
+
+import org.apache.twill.zookeeper.NodeChildren;
+import com.google.common.base.Objects;
+import org.apache.zookeeper.data.Stat;
+
+import java.util.List;
+
+/**
+ *
+ */
+final class BasicNodeChildren implements NodeChildren {
+
+ private final Stat stat;
+ private final List<String> children;
+
+ BasicNodeChildren(List<String> children, Stat stat) {
+ this.stat = stat;
+ this.children = children;
+ }
+
+ @Override
+ public Stat getStat() {
+ return stat;
+ }
+
+ @Override
+ public List<String> getChildren() {
+ return children;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || !(o instanceof NodeChildren)) {
+ return false;
+ }
+
+ NodeChildren that = (NodeChildren) o;
+ return stat.equals(that.getStat()) && children.equals(that.getChildren());
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(children, stat);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/BasicNodeData.java
----------------------------------------------------------------------
diff --git a/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/BasicNodeData.java b/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/BasicNodeData.java
new file mode 100644
index 0000000..98a3a66
--- /dev/null
+++ b/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/BasicNodeData.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.zookeeper;
+
+import org.apache.twill.zookeeper.NodeData;
+import com.google.common.base.Objects;
+import org.apache.zookeeper.data.Stat;
+
+import java.util.Arrays;
+
+/**
+ * A straightforward implementation for {@link NodeData}.
+ */
+final class BasicNodeData implements NodeData {
+
+ private final byte[] data;
+ private final Stat stat;
+
+ BasicNodeData(byte[] data, Stat stat) {
+ this.data = data;
+ this.stat = stat;
+ }
+
+ @Override
+ public Stat getStat() {
+ return stat;
+ }
+
+ @Override
+ public byte[] getData() {
+ return data;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || !(o instanceof NodeData)) {
+ return false;
+ }
+
+ BasicNodeData that = (BasicNodeData) o;
+
+ return stat.equals(that.getStat()) && Arrays.equals(data, that.getData());
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(data, stat);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/DefaultZKClientService.java
----------------------------------------------------------------------
diff --git a/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/DefaultZKClientService.java b/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/DefaultZKClientService.java
new file mode 100644
index 0000000..c52fb08
--- /dev/null
+++ b/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/DefaultZKClientService.java
@@ -0,0 +1,525 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.zookeeper;
+
+import org.apache.twill.common.Threads;
+import org.apache.twill.zookeeper.NodeChildren;
+import org.apache.twill.zookeeper.NodeData;
+import org.apache.twill.zookeeper.OperationFuture;
+import org.apache.twill.zookeeper.ZKClientService;
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Supplier;
+import com.google.common.util.concurrent.AbstractService;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.Service;
+import org.apache.zookeeper.AsyncCallback;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooDefs;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * The base implementation of {@link ZKClientService}.
+ */
+public final class DefaultZKClientService implements ZKClientService {
+
+ private static final Logger LOG = LoggerFactory.getLogger(DefaultZKClientService.class);
+
+ private final String zkStr;
+ private final int sessionTimeout;
+ private final List<Watcher> connectionWatchers;
+ private final AtomicReference<ZooKeeper> zooKeeper;
+ private final Function<String, List<ACL>> aclMapper;
+ private final Service serviceDelegate;
+ private ExecutorService eventExecutor;
+
+ public DefaultZKClientService(String zkStr, int sessionTimeout, Watcher connectionWatcher) {
+ this.zkStr = zkStr;
+ this.sessionTimeout = sessionTimeout;
+ this.connectionWatchers = new CopyOnWriteArrayList<Watcher>();
+ addConnectionWatcher(connectionWatcher);
+
+ this.zooKeeper = new AtomicReference<ZooKeeper>();
+
+ // TODO (terence): Add ACL
+ aclMapper = new Function<String, List<ACL>>() {
+ @Override
+ public List<ACL> apply(String input) {
+ return ZooDefs.Ids.OPEN_ACL_UNSAFE;
+ }
+ };
+ serviceDelegate = new ServiceDelegate();
+ }
+
+ @Override
+ public Long getSessionId() {
+ ZooKeeper zk = zooKeeper.get();
+ return zk == null ? null : zk.getSessionId();
+ }
+
+ @Override
+ public String getConnectString() {
+ return zkStr;
+ }
+
+ @Override
+ public void addConnectionWatcher(Watcher watcher) {
+ if (watcher != null) {
+ connectionWatchers.add(wrapWatcher(watcher));
+ }
+ }
+
+ @Override
+ public OperationFuture<String> create(String path, byte[] data, CreateMode createMode) {
+ return create(path, data, createMode, true);
+ }
+
+ @Override
+ public OperationFuture<String> create(String path, @Nullable byte[] data,
+ CreateMode createMode, boolean createParent) {
+ return doCreate(path, data, createMode, createParent, false);
+ }
+
+ private OperationFuture<String> doCreate(final String path,
+ @Nullable final byte[] data,
+ final CreateMode createMode,
+ final boolean createParent,
+ final boolean ignoreNodeExists) {
+ final SettableOperationFuture<String> createFuture = SettableOperationFuture.create(path, eventExecutor);
+ getZooKeeper().create(path, data, aclMapper.apply(path), createMode, Callbacks.STRING, createFuture);
+ if (!createParent) {
+ return createFuture;
+ }
+
+ // If create parent is request, return a different future
+ final SettableOperationFuture<String> result = SettableOperationFuture.create(path, eventExecutor);
+ // Watch for changes in the original future
+ Futures.addCallback(createFuture, new FutureCallback<String>() {
+ @Override
+ public void onSuccess(String path) {
+ // Propagate if creation was successful
+ result.set(path);
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ // See if the failure can be handled
+ if (updateFailureResult(t, result, path, ignoreNodeExists)) {
+ return;
+ }
+ // Create the parent node
+ String parentPath = getParent(path);
+ if (parentPath.isEmpty()) {
+ result.setException(t);
+ return;
+ }
+ // Watch for parent creation complete
+ Futures.addCallback(
+ doCreate(parentPath, null, CreateMode.PERSISTENT, createParent, true), new FutureCallback<String>() {
+ @Override
+ public void onSuccess(String parentPath) {
+ // Create the requested path again
+ Futures.addCallback(
+ doCreate(path, data, createMode, false, ignoreNodeExists), new FutureCallback<String>() {
+ @Override
+ public void onSuccess(String pathResult) {
+ result.set(pathResult);
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ // handle the failure
+ updateFailureResult(t, result, path, ignoreNodeExists);
+ }
+ });
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ result.setException(t);
+ }
+ });
+ }
+
+ /**
+ * Updates the result future based on the given {@link Throwable}.
+ * @param t Cause of the failure
+ * @param result Future to be updated
+ * @param path Request path for the operation
+ * @return {@code true} if it is a failure, {@code false} otherwise.
+ */
+ private boolean updateFailureResult(Throwable t, SettableOperationFuture<String> result,
+ String path, boolean ignoreNodeExists) {
+ // Propagate if there is error
+ if (!(t instanceof KeeperException)) {
+ result.setException(t);
+ return true;
+ }
+ KeeperException.Code code = ((KeeperException) t).code();
+ // Node already exists, simply return success if it allows for ignoring node exists (for parent node creation).
+ if (ignoreNodeExists && code == KeeperException.Code.NODEEXISTS) {
+ // The requested path could be used because it only applies to non-sequential node
+ result.set(path);
+ return false;
+ }
+ if (code != KeeperException.Code.NONODE) {
+ result.setException(t);
+ return true;
+ }
+ return false;
+ }
+
+ /**
+ * Gets the parent of the given path.
+ * @param path Path for computing its parent
+ * @return Parent of the given path, or empty string if the given path is the root path already.
+ */
+ private String getParent(String path) {
+ String parentPath = path.substring(0, path.lastIndexOf('/'));
+ return (parentPath.isEmpty() && !"/".equals(path)) ? "/" : parentPath;
+ }
+ });
+
+ return result;
+ }
+
+ @Override
+ public OperationFuture<Stat> exists(String path) {
+ return exists(path, null);
+ }
+
+ @Override
+ public OperationFuture<Stat> exists(String path, Watcher watcher) {
+ SettableOperationFuture<Stat> result = SettableOperationFuture.create(path, eventExecutor);
+ getZooKeeper().exists(path, wrapWatcher(watcher), Callbacks.STAT_NONODE, result);
+ return result;
+ }
+
+ @Override
+ public OperationFuture<NodeChildren> getChildren(String path) {
+ return getChildren(path, null);
+ }
+
+ @Override
+ public OperationFuture<NodeChildren> getChildren(String path, Watcher watcher) {
+ SettableOperationFuture<NodeChildren> result = SettableOperationFuture.create(path, eventExecutor);
+ getZooKeeper().getChildren(path, wrapWatcher(watcher), Callbacks.CHILDREN, result);
+ return result;
+ }
+
+ @Override
+ public OperationFuture<NodeData> getData(String path) {
+ return getData(path, null);
+ }
+
+ @Override
+ public OperationFuture<NodeData> getData(String path, Watcher watcher) {
+ SettableOperationFuture<NodeData> result = SettableOperationFuture.create(path, eventExecutor);
+ getZooKeeper().getData(path, wrapWatcher(watcher), Callbacks.DATA, result);
+
+ return result;
+ }
+
+ @Override
+ public OperationFuture<Stat> setData(String path, byte[] data) {
+ return setData(path, data, -1);
+ }
+
+ @Override
+ public OperationFuture<Stat> setData(String dataPath, byte[] data, int version) {
+ SettableOperationFuture<Stat> result = SettableOperationFuture.create(dataPath, eventExecutor);
+ getZooKeeper().setData(dataPath, data, version, Callbacks.STAT, result);
+ return result;
+ }
+
+ @Override
+ public OperationFuture<String> delete(String path) {
+ return delete(path, -1);
+ }
+
+ @Override
+ public OperationFuture<String> delete(String deletePath, int version) {
+ SettableOperationFuture<String> result = SettableOperationFuture.create(deletePath, eventExecutor);
+ getZooKeeper().delete(deletePath, version, Callbacks.VOID, result);
+ return result;
+ }
+
+ @Override
+ public Supplier<ZooKeeper> getZooKeeperSupplier() {
+ return new Supplier<ZooKeeper>() {
+ @Override
+ public ZooKeeper get() {
+ return getZooKeeper();
+ }
+ };
+ }
+
+ @Override
+ public ListenableFuture<State> start() {
+ return serviceDelegate.start();
+ }
+
+ @Override
+ public State startAndWait() {
+ return serviceDelegate.startAndWait();
+ }
+
+ @Override
+ public boolean isRunning() {
+ return serviceDelegate.isRunning();
+ }
+
+ @Override
+ public State state() {
+ return serviceDelegate.state();
+ }
+
+ @Override
+ public ListenableFuture<State> stop() {
+ return serviceDelegate.stop();
+ }
+
+ @Override
+ public State stopAndWait() {
+ return serviceDelegate.stopAndWait();
+ }
+
+ @Override
+ public void addListener(Listener listener, Executor executor) {
+ serviceDelegate.addListener(listener, executor);
+ }
+
+ /**
+ * @return Current {@link ZooKeeper} client.
+ */
+ private ZooKeeper getZooKeeper() {
+ ZooKeeper zk = zooKeeper.get();
+ Preconditions.checkArgument(zk != null, "Not connected to zooKeeper.");
+ return zk;
+ }
+
+ /**
+ * Wraps the given watcher to be called from the event executor.
+ * @param watcher Watcher to be wrapped
+ * @return The wrapped Watcher
+ */
+ private Watcher wrapWatcher(final Watcher watcher) {
+ if (watcher == null) {
+ return null;
+ }
+ return new Watcher() {
+ @Override
+ public void process(final WatchedEvent event) {
+ eventExecutor.execute(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ watcher.process(event);
+ } catch (Throwable t) {
+ LOG.error("Watcher throws exception.", t);
+ }
+ }
+ });
+ }
+ };
+ }
+
+ private final class ServiceDelegate extends AbstractService implements Watcher {
+
+ @Override
+ protected void doStart() {
+ // A single thread executor
+ eventExecutor = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>(),
+ Threads.createDaemonThreadFactory("zk-client-EventThread")) {
+ @Override
+ protected void terminated() {
+ super.terminated();
+ notifyStopped();
+ }
+ };
+
+ try {
+ zooKeeper.set(new ZooKeeper(zkStr, sessionTimeout, this));
+ } catch (IOException e) {
+ notifyFailed(e);
+ }
+ }
+
+ @Override
+ protected void doStop() {
+ ZooKeeper zk = zooKeeper.getAndSet(null);
+ if (zk != null) {
+ try {
+ zk.close();
+ } catch (InterruptedException e) {
+ notifyFailed(e);
+ } finally {
+ eventExecutor.shutdown();
+ }
+ }
+ }
+
+ @Override
+ public void process(WatchedEvent event) {
+ try {
+ if (event.getState() == Event.KeeperState.SyncConnected && state() == State.STARTING) {
+ LOG.info("Connected to ZooKeeper: " + zkStr);
+ notifyStarted();
+ return;
+ }
+ if (event.getState() == Event.KeeperState.Expired) {
+ LOG.info("ZooKeeper session expired: " + zkStr);
+
+ // When connection expired, simply reconnect again
+ Thread t = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ zooKeeper.set(new ZooKeeper(zkStr, sessionTimeout, ServiceDelegate.this));
+ } catch (IOException e) {
+ zooKeeper.set(null);
+ notifyFailed(e);
+ }
+ }
+ }, "zk-reconnect");
+ t.setDaemon(true);
+ t.start();
+ }
+ } finally {
+ if (event.getType() == Event.EventType.None && !connectionWatchers.isEmpty()) {
+ for (Watcher connectionWatcher : connectionWatchers) {
+ connectionWatcher.process(event);
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * Collection of generic callbacks that simply reflect results into OperationFuture.
+ */
+ private static final class Callbacks {
+ static final AsyncCallback.StringCallback STRING = new AsyncCallback.StringCallback() {
+ @Override
+ @SuppressWarnings("unchecked")
+ public void processResult(int rc, String path, Object ctx, String name) {
+ SettableOperationFuture<String> result = (SettableOperationFuture<String>) ctx;
+ KeeperException.Code code = KeeperException.Code.get(rc);
+ if (code == KeeperException.Code.OK) {
+ result.set((name == null || name.isEmpty()) ? path : name);
+ return;
+ }
+ result.setException(KeeperException.create(code, result.getRequestPath()));
+ }
+ };
+
+ static final AsyncCallback.StatCallback STAT = new AsyncCallback.StatCallback() {
+ @Override
+ @SuppressWarnings("unchecked")
+ public void processResult(int rc, String path, Object ctx, Stat stat) {
+ SettableOperationFuture<Stat> result = (SettableOperationFuture<Stat>) ctx;
+ KeeperException.Code code = KeeperException.Code.get(rc);
+ if (code == KeeperException.Code.OK) {
+ result.set(stat);
+ return;
+ }
+ result.setException(KeeperException.create(code, result.getRequestPath()));
+ }
+ };
+
+ /**
+ * A stat callback that treats NONODE as success.
+ */
+ static final AsyncCallback.StatCallback STAT_NONODE = new AsyncCallback.StatCallback() {
+ @Override
+ @SuppressWarnings("unchecked")
+ public void processResult(int rc, String path, Object ctx, Stat stat) {
+ SettableOperationFuture<Stat> result = (SettableOperationFuture<Stat>) ctx;
+ KeeperException.Code code = KeeperException.Code.get(rc);
+ if (code == KeeperException.Code.OK || code == KeeperException.Code.NONODE) {
+ result.set(stat);
+ return;
+ }
+ result.setException(KeeperException.create(code, result.getRequestPath()));
+ }
+ };
+
+ static final AsyncCallback.Children2Callback CHILDREN = new AsyncCallback.Children2Callback() {
+ @Override
+ @SuppressWarnings("unchecked")
+ public void processResult(int rc, String path, Object ctx, List<String> children, Stat stat) {
+ SettableOperationFuture<NodeChildren> result = (SettableOperationFuture<NodeChildren>) ctx;
+ KeeperException.Code code = KeeperException.Code.get(rc);
+ if (code == KeeperException.Code.OK) {
+ result.set(new BasicNodeChildren(children, stat));
+ return;
+ }
+ result.setException(KeeperException.create(code, result.getRequestPath()));
+ }
+ };
+
+ static final AsyncCallback.DataCallback DATA = new AsyncCallback.DataCallback() {
+ @Override
+ @SuppressWarnings("unchecked")
+ public void processResult(int rc, String path, Object ctx, byte[] data, Stat stat) {
+ SettableOperationFuture<NodeData> result = (SettableOperationFuture<NodeData>) ctx;
+ KeeperException.Code code = KeeperException.Code.get(rc);
+ if (code == KeeperException.Code.OK) {
+ result.set(new BasicNodeData(data, stat));
+ return;
+ }
+ result.setException(KeeperException.create(code, result.getRequestPath()));
+ }
+ };
+
+ static final AsyncCallback.VoidCallback VOID = new AsyncCallback.VoidCallback() {
+ @Override
+ @SuppressWarnings("unchecked")
+ public void processResult(int rc, String path, Object ctx) {
+ SettableOperationFuture<String> result = (SettableOperationFuture<String>) ctx;
+ KeeperException.Code code = KeeperException.Code.get(rc);
+ if (code == KeeperException.Code.OK) {
+ result.set(result.getRequestPath());
+ return;
+ }
+ // Otherwise, it is an error
+ result.setException(KeeperException.create(code, result.getRequestPath()));
+ }
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/FailureRetryZKClient.java
----------------------------------------------------------------------
diff --git a/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/FailureRetryZKClient.java b/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/FailureRetryZKClient.java
new file mode 100644
index 0000000..65ceadb
--- /dev/null
+++ b/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/FailureRetryZKClient.java
@@ -0,0 +1,240 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.zookeeper;
+
+import org.apache.twill.common.Threads;
+import org.apache.twill.zookeeper.ForwardingZKClient;
+import org.apache.twill.zookeeper.NodeChildren;
+import org.apache.twill.zookeeper.NodeData;
+import org.apache.twill.zookeeper.OperationFuture;
+import org.apache.twill.zookeeper.RetryStrategy;
+import org.apache.twill.zookeeper.RetryStrategy.OperationType;
+import org.apache.twill.zookeeper.ZKClient;
+import com.google.common.base.Supplier;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.data.Stat;
+
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * A {@link ZKClient} that will invoke {@link RetryStrategy} on operation failure.
+ * This {@link ZKClient} works by delegating calls to another {@link ZKClient}
+ * and listen for the result. If the result is a failure, and is
+ * {@link RetryUtils#canRetry(org.apache.zookeeper.KeeperException.Code) retryable}, the given {@link RetryStrategy}
+ * will be called to determine the next retry time, or give up, depending on the value returned by the strategy.
+ */
+public final class FailureRetryZKClient extends ForwardingZKClient {
+
+ private static final ScheduledExecutorService SCHEDULER = Executors.newSingleThreadScheduledExecutor(
+ Threads.createDaemonThreadFactory("retry-zkclient"));
+ private final RetryStrategy retryStrategy;
+
+ public FailureRetryZKClient(ZKClient delegate, RetryStrategy retryStrategy) {
+ super(delegate);
+ this.retryStrategy = retryStrategy;
+ }
+
+ @Override
+ public OperationFuture<String> create(String path, byte[] data, CreateMode createMode) {
+ return create(path, data, createMode, true);
+ }
+
+ @Override
+ public OperationFuture<String> create(final String path, final byte[] data,
+ final CreateMode createMode, final boolean createParent) {
+
+ // No retry for any SEQUENTIAL node, as some algorithms depends on only one sequential node being created.
+ if (createMode == CreateMode.PERSISTENT_SEQUENTIAL || createMode == CreateMode.EPHEMERAL_SEQUENTIAL) {
+ return super.create(path, data, createMode, createParent);
+ }
+
+ final SettableOperationFuture<String> result = SettableOperationFuture.create(path, Threads.SAME_THREAD_EXECUTOR);
+ Futures.addCallback(super.create(path, data, createMode, createParent),
+ new OperationFutureCallback<String>(OperationType.CREATE, System.currentTimeMillis(),
+ path, result, new Supplier<OperationFuture<String>>() {
+ @Override
+ public OperationFuture<String> get() {
+ return FailureRetryZKClient.super.create(path, data, createMode, createParent);
+ }
+ }));
+ return result;
+ }
+
+ @Override
+ public OperationFuture<Stat> exists(String path) {
+ return exists(path, null);
+ }
+
+ @Override
+ public OperationFuture<Stat> exists(final String path, final Watcher watcher) {
+ final SettableOperationFuture<Stat> result = SettableOperationFuture.create(path, Threads.SAME_THREAD_EXECUTOR);
+ Futures.addCallback(super.exists(path, watcher),
+ new OperationFutureCallback<Stat>(OperationType.EXISTS, System.currentTimeMillis(),
+ path, result, new Supplier<OperationFuture<Stat>>() {
+ @Override
+ public OperationFuture<Stat> get() {
+ return FailureRetryZKClient.super.exists(path, watcher);
+ }
+ }));
+ return result;
+ }
+
+ @Override
+ public OperationFuture<NodeChildren> getChildren(String path) {
+ return getChildren(path, null);
+ }
+
+ @Override
+ public OperationFuture<NodeChildren> getChildren(final String path, final Watcher watcher) {
+ final SettableOperationFuture<NodeChildren> result = SettableOperationFuture.create(path,
+ Threads.SAME_THREAD_EXECUTOR);
+ Futures.addCallback(super.getChildren(path, watcher),
+ new OperationFutureCallback<NodeChildren>(OperationType.GET_CHILDREN,
+ System.currentTimeMillis(), path, result,
+ new Supplier<OperationFuture<NodeChildren>>() {
+ @Override
+ public OperationFuture<NodeChildren> get() {
+ return FailureRetryZKClient.super.getChildren(path, watcher);
+ }
+ }));
+ return result;
+ }
+
+ @Override
+ public OperationFuture<NodeData> getData(String path) {
+ return getData(path, null);
+ }
+
+ @Override
+ public OperationFuture<NodeData> getData(final String path, final Watcher watcher) {
+ final SettableOperationFuture<NodeData> result = SettableOperationFuture.create(path, Threads.SAME_THREAD_EXECUTOR);
+ Futures.addCallback(super.getData(path, watcher),
+ new OperationFutureCallback<NodeData>(OperationType.GET_DATA, System.currentTimeMillis(),
+ path, result, new Supplier<OperationFuture<NodeData>>() {
+ @Override
+ public OperationFuture<NodeData> get() {
+ return FailureRetryZKClient.super.getData(path, watcher);
+ }
+ }));
+ return result;
+ }
+
+ @Override
+ public OperationFuture<Stat> setData(String path, byte[] data) {
+ return setData(path, data, -1);
+ }
+
+ @Override
+ public OperationFuture<Stat> setData(final String dataPath, final byte[] data, final int version) {
+ final SettableOperationFuture<Stat> result = SettableOperationFuture.create(dataPath, Threads.SAME_THREAD_EXECUTOR);
+ Futures.addCallback(super.setData(dataPath, data, version),
+ new OperationFutureCallback<Stat>(OperationType.SET_DATA, System.currentTimeMillis(),
+ dataPath, result, new Supplier<OperationFuture<Stat>>() {
+ @Override
+ public OperationFuture<Stat> get() {
+ return FailureRetryZKClient.super.setData(dataPath, data, version);
+ }
+ }));
+ return result;
+ }
+
+ @Override
+ public OperationFuture<String> delete(String path) {
+ return delete(path, -1);
+ }
+
+ @Override
+ public OperationFuture<String> delete(final String deletePath, final int version) {
+ final SettableOperationFuture<String> result = SettableOperationFuture.create(deletePath,
+ Threads.SAME_THREAD_EXECUTOR);
+ Futures.addCallback(super.delete(deletePath, version),
+ new OperationFutureCallback<String>(OperationType.DELETE, System.currentTimeMillis(),
+ deletePath, result, new Supplier<OperationFuture<String>>
+ () {
+ @Override
+ public OperationFuture<String> get() {
+ return FailureRetryZKClient.super.delete(deletePath, version);
+ }
+ }));
+ return result;
+ }
+
+ /**
+ * Callback to watch for operation result and trigger retry if necessary.
+ * @param <V> Type of operation result.
+ */
+ private final class OperationFutureCallback<V> implements FutureCallback<V> {
+
+ private final OperationType type;
+ private final long startTime;
+ private final String path;
+ private final SettableOperationFuture<V> result;
+ private final Supplier<OperationFuture<V>> retryAction;
+ private final AtomicInteger failureCount;
+
+ private OperationFutureCallback(OperationType type, long startTime, String path,
+ SettableOperationFuture<V> result, Supplier<OperationFuture<V>> retryAction) {
+ this.type = type;
+ this.startTime = startTime;
+ this.path = path;
+ this.result = result;
+ this.retryAction = retryAction;
+ this.failureCount = new AtomicInteger(0);
+ }
+
+ @Override
+ public void onSuccess(V result) {
+ this.result.set(result);
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ if (!doRetry(t)) {
+ result.setException(t);
+ }
+ }
+
+ private boolean doRetry(Throwable t) {
+ if (!RetryUtils.canRetry(t)) {
+ return false;
+ }
+
+ // Determine the relay delay
+ long nextRetry = retryStrategy.nextRetry(failureCount.incrementAndGet(), startTime, type, path);
+ if (nextRetry < 0) {
+ return false;
+ }
+
+ // Schedule the retry.
+ SCHEDULER.schedule(new Runnable() {
+ @Override
+ public void run() {
+ Futures.addCallback(retryAction.get(), OperationFutureCallback.this);
+ }
+ }, nextRetry, TimeUnit.MILLISECONDS);
+
+ return true;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/InMemoryZKServer.java
----------------------------------------------------------------------
diff --git a/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/InMemoryZKServer.java b/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/InMemoryZKServer.java
new file mode 100644
index 0000000..c4eed59
--- /dev/null
+++ b/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/InMemoryZKServer.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.zookeeper;
+
+import com.google.common.base.Preconditions;
+import com.google.common.base.Throwables;
+import com.google.common.io.Files;
+import com.google.common.util.concurrent.AbstractIdleService;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.Service;
+import org.apache.zookeeper.server.ServerCnxnFactory;
+import org.apache.zookeeper.server.ZooKeeperServer;
+import org.apache.zookeeper.server.persistence.FileTxnSnapLog;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.UnknownHostException;
+import java.util.concurrent.Executor;
+
+/**
+ *
+ */
+public final class InMemoryZKServer implements Service {
+
+ private static final Logger LOG = LoggerFactory.getLogger(InMemoryZKServer.class);
+
+ private final File dataDir;
+ private final int tickTime;
+ private final boolean autoClean;
+ private final int port;
+ private final Service delegateService = new AbstractIdleService() {
+ @Override
+ protected void startUp() throws Exception {
+ ZooKeeperServer zkServer = new ZooKeeperServer();
+ FileTxnSnapLog ftxn = new FileTxnSnapLog(dataDir, dataDir);
+ zkServer.setTxnLogFactory(ftxn);
+ zkServer.setTickTime(tickTime);
+
+ factory = ServerCnxnFactory.createFactory();
+ factory.configure(getAddress(port), -1);
+ factory.startup(zkServer);
+
+ LOG.info("In memory ZK started: " + getConnectionStr());
+ }
+
+ @Override
+ protected void shutDown() throws Exception {
+ try {
+ factory.shutdown();
+ } finally {
+ if (autoClean) {
+ cleanDir(dataDir);
+ }
+ }
+ }
+ };
+
+ private ServerCnxnFactory factory;
+
+ public static Builder builder() {
+ return new Builder();
+ }
+
+ private InMemoryZKServer(File dataDir, int tickTime, boolean autoClean, int port) {
+ if (dataDir == null) {
+ dataDir = Files.createTempDir();
+ autoClean = true;
+ } else {
+ Preconditions.checkArgument(dataDir.isDirectory() || dataDir.mkdirs() || dataDir.isDirectory());
+ }
+
+ this.dataDir = dataDir;
+ this.tickTime = tickTime;
+ this.autoClean = autoClean;
+ this.port = port;
+ }
+
+ public String getConnectionStr() {
+ InetSocketAddress addr = factory.getLocalAddress();
+ return String.format("%s:%d", addr.getHostName(), addr.getPort());
+ }
+
+ public InetSocketAddress getLocalAddress() {
+ return factory.getLocalAddress();
+ }
+
+ private InetSocketAddress getAddress(int port) {
+ try {
+// return new InetSocketAddress(InetAddress.getByAddress(new byte[] {127, 0, 0, 1}), port < 0 ? 0 : port);
+ return new InetSocketAddress(InetAddress.getLocalHost(), port < 0 ? 0 : port);
+ } catch (UnknownHostException e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ private void cleanDir(File dir) {
+ File[] files = dir.listFiles();
+ if (files == null) {
+ return;
+ }
+ for (File file : files) {
+ if (file.isDirectory()) {
+ cleanDir(file);
+ }
+ file.delete();
+ }
+ }
+
+ @Override
+ public ListenableFuture<State> start() {
+ return delegateService.start();
+ }
+
+ @Override
+ public State startAndWait() {
+ return delegateService.startAndWait();
+ }
+
+ @Override
+ public boolean isRunning() {
+ return delegateService.isRunning();
+ }
+
+ @Override
+ public State state() {
+ return delegateService.state();
+ }
+
+ @Override
+ public ListenableFuture<State> stop() {
+ return delegateService.stop();
+ }
+
+ @Override
+ public State stopAndWait() {
+ return delegateService.stopAndWait();
+ }
+
+ @Override
+ public void addListener(Listener listener, Executor executor) {
+ delegateService.addListener(listener, executor);
+ }
+
+ /**
+ * Builder for creating instance of {@link InMemoryZKServer}.
+ */
+ public static final class Builder {
+ private File dataDir;
+ private boolean autoCleanDataDir = false;
+ private int tickTime = ZooKeeperServer.DEFAULT_TICK_TIME;
+ private int port = -1;
+
+ public Builder setDataDir(File dataDir) {
+ this.dataDir = dataDir;
+ return this;
+ }
+
+ public Builder setAutoCleanDataDir(boolean auto) {
+ this.autoCleanDataDir = auto;
+ return this;
+ }
+
+ public Builder setTickTime(int tickTime) {
+ this.tickTime = tickTime;
+ return this;
+ }
+
+ public Builder setPort(int port) {
+ this.port = port;
+ return this;
+ }
+
+ public InMemoryZKServer build() {
+ return new InMemoryZKServer(dataDir, tickTime, autoCleanDataDir, port);
+ }
+
+ private Builder() {
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/KillZKSession.java
----------------------------------------------------------------------
diff --git a/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/KillZKSession.java b/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/KillZKSession.java
new file mode 100644
index 0000000..bc01f08
--- /dev/null
+++ b/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/KillZKSession.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.zookeeper;
+
+import com.google.common.base.Preconditions;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZooKeeper;
+
+import java.io.IOException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Utility class for killing ZK client to simulate failures during testing.
+ */
+public final class KillZKSession {
+
+ /**
+ * Utility classes should have a public constructor or a default constructor
+ * hence made it private.
+ */
+ private KillZKSession() {}
+
+ /**
+ * Kills a Zookeeper client to simulate failure scenarious during testing.
+ * Callee will provide the amount of time to wait before it's considered failure
+ * to kill a client.
+ *
+ * @param client that needs to be killed.
+ * @param connectionString of Quorum
+ * @param maxMs time in millisecond specifying the max time to kill a client.
+ * @throws IOException When there is IO error
+ * @throws InterruptedException When call has been interrupted.
+ */
+ public static void kill(ZooKeeper client, String connectionString,
+ int maxMs) throws IOException, InterruptedException {
+ final CountDownLatch latch = new CountDownLatch(1);
+ ZooKeeper zk = new ZooKeeper(connectionString, maxMs, new Watcher() {
+ @Override
+ public void process(WatchedEvent event) {
+ if (event.getState() == Event.KeeperState.SyncConnected) {
+ latch.countDown();
+ }
+ }
+ }, client.getSessionId(), client.getSessionPasswd());
+
+ try {
+ Preconditions.checkState(latch.await(maxMs, TimeUnit.MILLISECONDS), "Fail to kill ZK connection.");
+ } finally {
+ zk.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/NamespaceZKClient.java
----------------------------------------------------------------------
diff --git a/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/NamespaceZKClient.java b/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/NamespaceZKClient.java
new file mode 100644
index 0000000..1a82e4b
--- /dev/null
+++ b/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/NamespaceZKClient.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.zookeeper;
+
+import org.apache.twill.common.Threads;
+import org.apache.twill.zookeeper.ForwardingZKClient;
+import org.apache.twill.zookeeper.NodeChildren;
+import org.apache.twill.zookeeper.NodeData;
+import org.apache.twill.zookeeper.OperationFuture;
+import org.apache.twill.zookeeper.ZKClient;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.data.Stat;
+
+import javax.annotation.Nullable;
+
+/**
+ * A {@link ZKClient} that namespace every paths.
+ */
+public final class NamespaceZKClient extends ForwardingZKClient {
+ // This class extends from ForwardingZKClient but overrides every method is for letting the
+ // ZKClientServices delegate logic works.
+
+ private final String namespace;
+ private final ZKClient delegate;
+ private final String connectString;
+
+ public NamespaceZKClient(ZKClient delegate, String namespace) {
+ super(delegate);
+ this.namespace = namespace;
+ this.delegate = delegate;
+ this.connectString = delegate.getConnectString() + namespace;
+ }
+
+ @Override
+ public Long getSessionId() {
+ return delegate.getSessionId();
+ }
+
+ @Override
+ public String getConnectString() {
+ return connectString;
+ }
+
+ @Override
+ public void addConnectionWatcher(Watcher watcher) {
+ delegate.addConnectionWatcher(watcher);
+ }
+
+ @Override
+ public OperationFuture<String> create(String path, @Nullable byte[] data, CreateMode createMode) {
+ return relayPath(delegate.create(namespace + path, data, createMode), this.<String>createFuture(path));
+ }
+
+ @Override
+ public OperationFuture<String> create(String path, @Nullable byte[] data, CreateMode createMode,
+ boolean createParent) {
+ return relayPath(delegate.create(namespace + path, data, createMode, createParent),
+ this.<String>createFuture(path));
+ }
+
+ @Override
+ public OperationFuture<Stat> exists(String path) {
+ return relayFuture(delegate.exists(namespace + path), this.<Stat>createFuture(path));
+ }
+
+ @Override
+ public OperationFuture<Stat> exists(String path, @Nullable Watcher watcher) {
+ return relayFuture(delegate.exists(namespace + path, watcher), this.<Stat>createFuture(path));
+ }
+
+ @Override
+ public OperationFuture<NodeChildren> getChildren(String path) {
+ return relayFuture(delegate.getChildren(namespace + path), this.<NodeChildren>createFuture(path));
+ }
+
+ @Override
+ public OperationFuture<NodeChildren> getChildren(String path, @Nullable Watcher watcher) {
+ return relayFuture(delegate.getChildren(namespace + path, watcher), this.<NodeChildren>createFuture(path));
+ }
+
+ @Override
+ public OperationFuture<NodeData> getData(String path) {
+ return relayFuture(delegate.getData(namespace + path), this.<NodeData>createFuture(path));
+ }
+
+ @Override
+ public OperationFuture<NodeData> getData(String path, @Nullable Watcher watcher) {
+ return relayFuture(delegate.getData(namespace + path, watcher), this.<NodeData>createFuture(path));
+ }
+
+ @Override
+ public OperationFuture<Stat> setData(String path, byte[] data) {
+ return relayFuture(delegate.setData(namespace + path, data), this.<Stat>createFuture(path));
+ }
+
+ @Override
+ public OperationFuture<Stat> setData(String dataPath, byte[] data, int version) {
+ return relayFuture(delegate.setData(namespace + dataPath, data, version), this.<Stat>createFuture(dataPath));
+ }
+
+ @Override
+ public OperationFuture<String> delete(String path) {
+ return relayPath(delegate.delete(namespace + path), this.<String>createFuture(path));
+ }
+
+ @Override
+ public OperationFuture<String> delete(String deletePath, int version) {
+ return relayPath(delegate.delete(namespace + deletePath, version), this.<String>createFuture(deletePath));
+ }
+
+ private <V> SettableOperationFuture<V> createFuture(String path) {
+ return SettableOperationFuture.create(namespace + path, Threads.SAME_THREAD_EXECUTOR);
+ }
+
+ private <V> OperationFuture<V> relayFuture(final OperationFuture<V> from, final SettableOperationFuture<V> to) {
+ Futures.addCallback(from, new FutureCallback<V>() {
+ @Override
+ public void onSuccess(V result) {
+ to.set(result);
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ to.setException(t);
+ }
+ });
+ return to;
+ }
+
+ private OperationFuture<String> relayPath(final OperationFuture<String> from,
+ final SettableOperationFuture<String> to) {
+ from.addListener(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ String path = from.get();
+ to.set(path.substring(namespace.length()));
+ } catch (Exception e) {
+ to.setException(e.getCause());
+ }
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+ return to;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/RetryUtils.java
----------------------------------------------------------------------
diff --git a/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/RetryUtils.java b/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/RetryUtils.java
new file mode 100644
index 0000000..fb42491
--- /dev/null
+++ b/zookeeper/src/main/java/org/apache/twill/internal/zookeeper/RetryUtils.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.zookeeper;
+
+import org.apache.zookeeper.KeeperException;
+
+/**
+ * Utility class for help determining operation retry condition.
+ */
+final class RetryUtils {
+
+ /**
+ * Tells if a given operation error code can be retried or not.
+ * @param code The error code of the operation.
+ * @return {@code true} if the operation can be retried.
+ */
+ public static boolean canRetry(KeeperException.Code code) {
+ return (code == KeeperException.Code.CONNECTIONLOSS
+ || code == KeeperException.Code.OPERATIONTIMEOUT
+ || code == KeeperException.Code.SESSIONEXPIRED
+ || code == KeeperException.Code.SESSIONMOVED);
+ }
+
+ /**
+ * Tells if a given operation exception can be retried or not.
+ * @param t The exception raised by an operation.
+ * @return {@code true} if the operation can be retried.
+ */
+ public static boolean canRetry(Throwable t) {
+ return t instanceof KeeperException && canRetry(((KeeperException) t).code());
+ }
+
+ private RetryUtils() {
+ }
+}
[04/15] Initial import commit.
Posted by ch...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java b/yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
new file mode 100644
index 0000000..17425d4
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/yarn/YarnTwillPreparer.java
@@ -0,0 +1,600 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.yarn;
+
+import com.google.common.base.Charsets;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicates;
+import com.google.common.base.Supplier;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.ListMultimap;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Sets;
+import com.google.common.io.ByteStreams;
+import com.google.common.io.CharStreams;
+import com.google.common.io.OutputSupplier;
+import com.google.common.reflect.TypeToken;
+import com.google.gson.GsonBuilder;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.twill.api.EventHandlerSpecification;
+import org.apache.twill.api.LocalFile;
+import org.apache.twill.api.RunId;
+import org.apache.twill.api.RuntimeSpecification;
+import org.apache.twill.api.SecureStore;
+import org.apache.twill.api.TwillController;
+import org.apache.twill.api.TwillPreparer;
+import org.apache.twill.api.TwillSpecification;
+import org.apache.twill.api.logging.LogHandler;
+import org.apache.twill.filesystem.Location;
+import org.apache.twill.filesystem.LocationFactory;
+import org.apache.twill.internal.ApplicationBundler;
+import org.apache.twill.internal.Arguments;
+import org.apache.twill.internal.Configs;
+import org.apache.twill.internal.Constants;
+import org.apache.twill.internal.DefaultLocalFile;
+import org.apache.twill.internal.DefaultRuntimeSpecification;
+import org.apache.twill.internal.DefaultTwillSpecification;
+import org.apache.twill.internal.EnvKeys;
+import org.apache.twill.internal.LogOnlyEventHandler;
+import org.apache.twill.internal.ProcessController;
+import org.apache.twill.internal.ProcessLauncher;
+import org.apache.twill.internal.RunIds;
+import org.apache.twill.internal.appmaster.ApplicationMasterMain;
+import org.apache.twill.internal.container.TwillContainerMain;
+import org.apache.twill.internal.json.ArgumentsCodec;
+import org.apache.twill.internal.json.LocalFileCodec;
+import org.apache.twill.internal.json.TwillSpecificationAdapter;
+import org.apache.twill.internal.utils.Dependencies;
+import org.apache.twill.internal.utils.Paths;
+import org.apache.twill.internal.yarn.YarnAppClient;
+import org.apache.twill.internal.yarn.YarnApplicationReport;
+import org.apache.twill.internal.yarn.YarnUtils;
+import org.apache.twill.launcher.TwillLauncher;
+import org.apache.twill.zookeeper.ZKClient;
+import org.apache.twill.zookeeper.ZKClients;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.jar.JarEntry;
+import java.util.jar.JarOutputStream;
+
+/**
+ * Implementation for {@link TwillPreparer} to prepare and launch distributed application on Hadoop YARN.
+ */
+final class YarnTwillPreparer implements TwillPreparer {
+
+ private static final Logger LOG = LoggerFactory.getLogger(YarnTwillPreparer.class);
+ private static final String KAFKA_ARCHIVE = "kafka-0.7.2.tgz";
+
+ private final YarnConfiguration yarnConfig;
+ private final TwillSpecification twillSpec;
+ private final YarnAppClient yarnAppClient;
+ private final ZKClient zkClient;
+ private final LocationFactory locationFactory;
+ private final Supplier<String> jvmOpts;
+ private final YarnTwillControllerFactory controllerFactory;
+ private final RunId runId;
+
+ private final List<LogHandler> logHandlers = Lists.newArrayList();
+ private final List<String> arguments = Lists.newArrayList();
+ private final Set<Class<?>> dependencies = Sets.newIdentityHashSet();
+ private final List<URI> resources = Lists.newArrayList();
+ private final List<String> classPaths = Lists.newArrayList();
+ private final ListMultimap<String, String> runnableArgs = ArrayListMultimap.create();
+ private final Credentials credentials;
+ private final int reservedMemory;
+ private String user;
+
+ YarnTwillPreparer(YarnConfiguration yarnConfig, TwillSpecification twillSpec, YarnAppClient yarnAppClient,
+ ZKClient zkClient, LocationFactory locationFactory, Supplier<String> jvmOpts,
+ YarnTwillControllerFactory controllerFactory) {
+ this.yarnConfig = yarnConfig;
+ this.twillSpec = twillSpec;
+ this.yarnAppClient = yarnAppClient;
+ this.zkClient = ZKClients.namespace(zkClient, "/" + twillSpec.getName());
+ this.locationFactory = locationFactory;
+ this.jvmOpts = jvmOpts;
+ this.controllerFactory = controllerFactory;
+ this.runId = RunIds.generate();
+ this.credentials = createCredentials();
+ this.reservedMemory = yarnConfig.getInt(Configs.Keys.JAVA_RESERVED_MEMORY_MB,
+ Configs.Defaults.JAVA_RESERVED_MEMORY_MB);
+ this.user = System.getProperty("user.name");
+ }
+
+ @Override
+ public TwillPreparer addLogHandler(LogHandler handler) {
+ logHandlers.add(handler);
+ return this;
+ }
+
+ @Override
+ public TwillPreparer setUser(String user) {
+ this.user = user;
+ return this;
+ }
+
+ @Override
+ public TwillPreparer withApplicationArguments(String... args) {
+ return withApplicationArguments(ImmutableList.copyOf(args));
+ }
+
+ @Override
+ public TwillPreparer withApplicationArguments(Iterable<String> args) {
+ Iterables.addAll(arguments, args);
+ return this;
+ }
+
+ @Override
+ public TwillPreparer withArguments(String runnableName, String... args) {
+ return withArguments(runnableName, ImmutableList.copyOf(args));
+ }
+
+ @Override
+ public TwillPreparer withArguments(String runnableName, Iterable<String> args) {
+ runnableArgs.putAll(runnableName, args);
+ return this;
+ }
+
+ @Override
+ public TwillPreparer withDependencies(Class<?>... classes) {
+ return withDependencies(ImmutableList.copyOf(classes));
+ }
+
+ @Override
+ public TwillPreparer withDependencies(Iterable<Class<?>> classes) {
+ Iterables.addAll(dependencies, classes);
+ return this;
+ }
+
+ @Override
+ public TwillPreparer withResources(URI... resources) {
+ return withResources(ImmutableList.copyOf(resources));
+ }
+
+ @Override
+ public TwillPreparer withResources(Iterable<URI> resources) {
+ Iterables.addAll(this.resources, resources);
+ return this;
+ }
+
+ @Override
+ public TwillPreparer withClassPaths(String... classPaths) {
+ return withClassPaths(ImmutableList.copyOf(classPaths));
+ }
+
+ @Override
+ public TwillPreparer withClassPaths(Iterable<String> classPaths) {
+ Iterables.addAll(this.classPaths, classPaths);
+ return this;
+ }
+
+ @Override
+ public TwillPreparer addSecureStore(SecureStore secureStore) {
+ Object store = secureStore.getStore();
+ Preconditions.checkArgument(store instanceof Credentials, "Only Hadoop Credentials is supported.");
+ this.credentials.mergeAll((Credentials) store);
+ return this;
+ }
+
+ @Override
+ public TwillController start() {
+ try {
+ final ProcessLauncher<ApplicationId> launcher = yarnAppClient.createLauncher(user, twillSpec);
+ final ApplicationId appId = launcher.getContainerInfo();
+
+ Callable<ProcessController<YarnApplicationReport>> submitTask =
+ new Callable<ProcessController<YarnApplicationReport>>() {
+ @Override
+ public ProcessController<YarnApplicationReport> call() throws Exception {
+ String fsUser = locationFactory.getHomeLocation().getName();
+
+ // Local files needed by AM
+ Map<String, LocalFile> localFiles = Maps.newHashMap();
+ // Local files declared by runnables
+ Multimap<String, LocalFile> runnableLocalFiles = HashMultimap.create();
+
+ String vmOpts = jvmOpts.get();
+
+ createAppMasterJar(createBundler(), localFiles);
+ createContainerJar(createBundler(), localFiles);
+ populateRunnableLocalFiles(twillSpec, runnableLocalFiles);
+ saveSpecification(twillSpec, runnableLocalFiles, localFiles);
+ saveLogback(localFiles);
+ saveLauncher(localFiles);
+ saveKafka(localFiles);
+ saveVmOptions(vmOpts, localFiles);
+ saveArguments(new Arguments(arguments, runnableArgs), localFiles);
+ saveLocalFiles(localFiles, ImmutableSet.of(Constants.Files.TWILL_SPEC,
+ Constants.Files.LOGBACK_TEMPLATE,
+ Constants.Files.CONTAINER_JAR,
+ Constants.Files.LAUNCHER_JAR,
+ Constants.Files.ARGUMENTS));
+
+ LOG.debug("Submit AM container spec: {}", appId);
+ // java -Djava.io.tmpdir=tmp -cp launcher.jar:$HADOOP_CONF_DIR -XmxMemory
+ // org.apache.twill.internal.TwillLauncher
+ // appMaster.jar
+ // org.apache.twill.internal.appmaster.ApplicationMasterMain
+ // false
+ return launcher.prepareLaunch(
+ ImmutableMap.<String, String>builder()
+ .put(EnvKeys.TWILL_FS_USER, fsUser)
+ .put(EnvKeys.TWILL_APP_DIR, getAppLocation().toURI().toASCIIString())
+ .put(EnvKeys.TWILL_ZK_CONNECT, zkClient.getConnectString())
+ .put(EnvKeys.TWILL_RUN_ID, runId.getId())
+ .put(EnvKeys.TWILL_RESERVED_MEMORY_MB, Integer.toString(reservedMemory))
+ .put(EnvKeys.TWILL_APP_NAME, twillSpec.getName()).build(),
+ localFiles.values(), credentials)
+ .noResources()
+ .noEnvironment()
+ .withCommands().add(
+ "java",
+ "-Djava.io.tmpdir=tmp",
+ "-Dyarn.appId=$" + EnvKeys.YARN_APP_ID_STR,
+ "-Dtwill.app=$" + EnvKeys.TWILL_APP_NAME,
+ "-cp", Constants.Files.LAUNCHER_JAR + ":$HADOOP_CONF_DIR",
+ "-Xmx" + (Constants.APP_MASTER_MEMORY_MB - Constants.APP_MASTER_RESERVED_MEMORY_MB) + "m",
+ vmOpts,
+ TwillLauncher.class.getName(),
+ Constants.Files.APP_MASTER_JAR,
+ ApplicationMasterMain.class.getName(),
+ Boolean.FALSE.toString())
+ .redirectOutput(Constants.STDOUT)
+ .redirectError(Constants.STDERR)
+ .launch();
+ }
+ };
+
+ YarnTwillController controller = controllerFactory.create(runId, logHandlers, submitTask);
+ controller.start();
+ return controller;
+ } catch (Exception e) {
+ LOG.error("Failed to submit application {}", twillSpec.getName(), e);
+ throw Throwables.propagate(e);
+ }
+ }
+
+ private Credentials createCredentials() {
+ Credentials credentials = new Credentials();
+
+ try {
+ credentials.addAll(UserGroupInformation.getCurrentUser().getCredentials());
+
+ List<Token<?>> tokens = YarnUtils.addDelegationTokens(yarnConfig, locationFactory, credentials);
+ for (Token<?> token : tokens) {
+ LOG.debug("Delegation token acquired for {}, {}", locationFactory.getHomeLocation().toURI(), token);
+ }
+ } catch (IOException e) {
+ LOG.warn("Failed to check for secure login type. Not gathering any delegation token.", e);
+ }
+ return credentials;
+ }
+
+ private ApplicationBundler createBundler() {
+ return new ApplicationBundler(ImmutableList.<String>of());
+ }
+
+ private LocalFile createLocalFile(String name, Location location) throws IOException {
+ return createLocalFile(name, location, false);
+ }
+
+ private LocalFile createLocalFile(String name, Location location, boolean archive) throws IOException {
+ return new DefaultLocalFile(name, location.toURI(), location.lastModified(), location.length(), archive, null);
+ }
+
+ private void createAppMasterJar(ApplicationBundler bundler, Map<String, LocalFile> localFiles) throws IOException {
+ try {
+ LOG.debug("Create and copy {}", Constants.Files.APP_MASTER_JAR);
+ Location location = createTempLocation(Constants.Files.APP_MASTER_JAR);
+
+ List<Class<?>> classes = Lists.newArrayList();
+ classes.add(ApplicationMasterMain.class);
+
+ // Stuck in the yarnAppClient class to make bundler being able to pickup the right yarn-client version
+ classes.add(yarnAppClient.getClass());
+
+ // Add the TwillRunnableEventHandler class
+ if (twillSpec.getEventHandler() != null) {
+ classes.add(getClassLoader().loadClass(twillSpec.getEventHandler().getClassName()));
+ }
+
+ bundler.createBundle(location, classes);
+ LOG.debug("Done {}", Constants.Files.APP_MASTER_JAR);
+
+ localFiles.put(Constants.Files.APP_MASTER_JAR, createLocalFile(Constants.Files.APP_MASTER_JAR, location));
+ } catch (ClassNotFoundException e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ private void createContainerJar(ApplicationBundler bundler, Map<String, LocalFile> localFiles) throws IOException {
+ try {
+ Set<Class<?>> classes = Sets.newIdentityHashSet();
+ classes.add(TwillContainerMain.class);
+ classes.addAll(dependencies);
+
+ ClassLoader classLoader = getClassLoader();
+ for (RuntimeSpecification spec : twillSpec.getRunnables().values()) {
+ classes.add(classLoader.loadClass(spec.getRunnableSpecification().getClassName()));
+ }
+
+ LOG.debug("Create and copy {}", Constants.Files.CONTAINER_JAR);
+ Location location = createTempLocation(Constants.Files.CONTAINER_JAR);
+ bundler.createBundle(location, classes, resources);
+ LOG.debug("Done {}", Constants.Files.CONTAINER_JAR);
+
+ localFiles.put(Constants.Files.CONTAINER_JAR, createLocalFile(Constants.Files.CONTAINER_JAR, location));
+
+ } catch (ClassNotFoundException e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ /**
+ * Based on the given {@link TwillSpecification}, upload LocalFiles to Yarn Cluster.
+ * @param twillSpec The {@link TwillSpecification} for populating resource.
+ * @param localFiles A Multimap to store runnable name to transformed LocalFiles.
+ * @throws IOException
+ */
+ private void populateRunnableLocalFiles(TwillSpecification twillSpec,
+ Multimap<String, LocalFile> localFiles) throws IOException {
+
+ LOG.debug("Populating Runnable LocalFiles");
+ for (Map.Entry<String, RuntimeSpecification> entry: twillSpec.getRunnables().entrySet()) {
+ String runnableName = entry.getKey();
+ for (LocalFile localFile : entry.getValue().getLocalFiles()) {
+ Location location;
+
+ URI uri = localFile.getURI();
+ if ("hdfs".equals(uri.getScheme())) {
+ // Assuming the location factory is HDFS one. If it is not, it will failed, which is the correct behavior.
+ location = locationFactory.create(uri);
+ } else {
+ URL url = uri.toURL();
+ LOG.debug("Create and copy {} : {}", runnableName, url);
+ // Preserves original suffix for expansion.
+ location = copyFromURL(url, createTempLocation(Paths.appendSuffix(url.getFile(), localFile.getName())));
+ LOG.debug("Done {} : {}", runnableName, url);
+ }
+
+ localFiles.put(runnableName,
+ new DefaultLocalFile(localFile.getName(), location.toURI(), location.lastModified(),
+ location.length(), localFile.isArchive(), localFile.getPattern()));
+ }
+ }
+ LOG.debug("Done Runnable LocalFiles");
+ }
+
+ private void saveSpecification(TwillSpecification spec, final Multimap<String, LocalFile> runnableLocalFiles,
+ Map<String, LocalFile> localFiles) throws IOException {
+ // Rewrite LocalFiles inside twillSpec
+ Map<String, RuntimeSpecification> runtimeSpec = Maps.transformEntries(
+ spec.getRunnables(), new Maps.EntryTransformer<String, RuntimeSpecification, RuntimeSpecification>() {
+ @Override
+ public RuntimeSpecification transformEntry(String key, RuntimeSpecification value) {
+ return new DefaultRuntimeSpecification(value.getName(), value.getRunnableSpecification(),
+ value.getResourceSpecification(), runnableLocalFiles.get(key));
+ }
+ });
+
+ // Serialize into a local temp file.
+ LOG.debug("Create and copy {}", Constants.Files.TWILL_SPEC);
+ Location location = createTempLocation(Constants.Files.TWILL_SPEC);
+ Writer writer = new OutputStreamWriter(location.getOutputStream(), Charsets.UTF_8);
+ try {
+ EventHandlerSpecification eventHandler = spec.getEventHandler();
+ if (eventHandler == null) {
+ eventHandler = new LogOnlyEventHandler().configure();
+ }
+
+ TwillSpecificationAdapter.create().toJson(
+ new DefaultTwillSpecification(spec.getName(), runtimeSpec, spec.getOrders(), eventHandler),
+ writer);
+ } finally {
+ writer.close();
+ }
+ LOG.debug("Done {}", Constants.Files.TWILL_SPEC);
+
+ localFiles.put(Constants.Files.TWILL_SPEC, createLocalFile(Constants.Files.TWILL_SPEC, location));
+ }
+
+ private void saveLogback(Map<String, LocalFile> localFiles) throws IOException {
+ LOG.debug("Create and copy {}", Constants.Files.LOGBACK_TEMPLATE);
+ Location location = copyFromURL(getClass().getClassLoader().getResource(Constants.Files.LOGBACK_TEMPLATE),
+ createTempLocation(Constants.Files.LOGBACK_TEMPLATE));
+ LOG.debug("Done {}", Constants.Files.LOGBACK_TEMPLATE);
+
+ localFiles.put(Constants.Files.LOGBACK_TEMPLATE, createLocalFile(Constants.Files.LOGBACK_TEMPLATE, location));
+ }
+
+ /**
+ * Creates the launcher.jar for launch the main application.
+ */
+ private void saveLauncher(Map<String, LocalFile> localFiles) throws URISyntaxException, IOException {
+
+ LOG.debug("Create and copy {}", Constants.Files.LAUNCHER_JAR);
+ Location location = createTempLocation(Constants.Files.LAUNCHER_JAR);
+
+ final String launcherName = TwillLauncher.class.getName();
+
+ // Create a jar file with the TwillLauncher optionally a json serialized classpath.json in it.
+ final JarOutputStream jarOut = new JarOutputStream(location.getOutputStream());
+ ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+ if (classLoader == null) {
+ classLoader = getClass().getClassLoader();
+ }
+ Dependencies.findClassDependencies(classLoader, new Dependencies.ClassAcceptor() {
+ @Override
+ public boolean accept(String className, URL classUrl, URL classPathUrl) {
+ Preconditions.checkArgument(className.startsWith(launcherName),
+ "Launcher jar should not have dependencies: %s", className);
+ try {
+ jarOut.putNextEntry(new JarEntry(className.replace('.', '/') + ".class"));
+ InputStream is = classUrl.openStream();
+ try {
+ ByteStreams.copy(is, jarOut);
+ } finally {
+ is.close();
+ }
+ } catch (IOException e) {
+ throw Throwables.propagate(e);
+ }
+ return true;
+ }
+ }, TwillLauncher.class.getName());
+
+ try {
+ if (!classPaths.isEmpty()) {
+ jarOut.putNextEntry(new JarEntry("classpath"));
+ jarOut.write(Joiner.on(':').join(classPaths).getBytes(Charsets.UTF_8));
+ }
+ } finally {
+ jarOut.close();
+ }
+ LOG.debug("Done {}", Constants.Files.LAUNCHER_JAR);
+
+ localFiles.put(Constants.Files.LAUNCHER_JAR, createLocalFile(Constants.Files.LAUNCHER_JAR, location));
+ }
+
+ private void saveKafka(Map<String, LocalFile> localFiles) throws IOException {
+ LOG.debug("Copy {}", Constants.Files.KAFKA);
+ Location location = copyFromURL(getClass().getClassLoader().getResource(KAFKA_ARCHIVE),
+ createTempLocation(Constants.Files.KAFKA));
+ LOG.debug("Done {}", Constants.Files.KAFKA);
+
+ localFiles.put(Constants.Files.KAFKA, createLocalFile(Constants.Files.KAFKA, location, true));
+ }
+
+ private void saveVmOptions(String opts, Map<String, LocalFile> localFiles) throws IOException {
+ if (opts.isEmpty()) {
+ // If no vm options, no need to localize the file.
+ return;
+ }
+ LOG.debug("Copy {}", Constants.Files.JVM_OPTIONS);
+ final Location location = createTempLocation(Constants.Files.JVM_OPTIONS);
+ CharStreams.write(opts, new OutputSupplier<Writer>() {
+ @Override
+ public Writer getOutput() throws IOException {
+ return new OutputStreamWriter(location.getOutputStream(), Charsets.UTF_8);
+ }
+ });
+ LOG.debug("Done {}", Constants.Files.JVM_OPTIONS);
+
+ localFiles.put(Constants.Files.JVM_OPTIONS, createLocalFile(Constants.Files.JVM_OPTIONS, location));
+ }
+
+ private void saveArguments(Arguments arguments, Map<String, LocalFile> localFiles) throws IOException {
+ LOG.debug("Create and copy {}", Constants.Files.ARGUMENTS);
+ final Location location = createTempLocation(Constants.Files.ARGUMENTS);
+ ArgumentsCodec.encode(arguments, new OutputSupplier<Writer>() {
+ @Override
+ public Writer getOutput() throws IOException {
+ return new OutputStreamWriter(location.getOutputStream(), Charsets.UTF_8);
+ }
+ });
+ LOG.debug("Done {}", Constants.Files.ARGUMENTS);
+
+ localFiles.put(Constants.Files.ARGUMENTS, createLocalFile(Constants.Files.ARGUMENTS, location));
+ }
+
+ /**
+ * Serializes the list of files that needs to localize from AM to Container.
+ */
+ private void saveLocalFiles(Map<String, LocalFile> localFiles, Set<String> includes) throws IOException {
+ Map<String, LocalFile> localize = ImmutableMap.copyOf(Maps.filterKeys(localFiles, Predicates.in(includes)));
+ LOG.debug("Create and copy {}", Constants.Files.LOCALIZE_FILES);
+ Location location = createTempLocation(Constants.Files.LOCALIZE_FILES);
+ Writer writer = new OutputStreamWriter(location.getOutputStream(), Charsets.UTF_8);
+ try {
+ new GsonBuilder().registerTypeAdapter(LocalFile.class, new LocalFileCodec())
+ .create().toJson(localize.values(), new TypeToken<List<LocalFile>>() {
+ }.getType(), writer);
+ } finally {
+ writer.close();
+ }
+ LOG.debug("Done {}", Constants.Files.LOCALIZE_FILES);
+ localFiles.put(Constants.Files.LOCALIZE_FILES, createLocalFile(Constants.Files.LOCALIZE_FILES, location));
+ }
+
+ private Location copyFromURL(URL url, Location target) throws IOException {
+ InputStream is = url.openStream();
+ try {
+ OutputStream os = new BufferedOutputStream(target.getOutputStream());
+ try {
+ ByteStreams.copy(is, os);
+ } finally {
+ os.close();
+ }
+ } finally {
+ is.close();
+ }
+ return target;
+ }
+
+ private Location createTempLocation(String fileName) {
+ String name;
+ String suffix = Paths.getExtension(fileName);
+
+ name = fileName.substring(0, fileName.length() - suffix.length() - 1);
+
+ try {
+ return getAppLocation().append(name).getTempFile('.' + suffix);
+ } catch (IOException e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ private Location getAppLocation() {
+ return locationFactory.create(String.format("/%s/%s", twillSpec.getName(), runId.getId()));
+ }
+
+ /**
+ * Returns the context ClassLoader if there is any, otherwise, returns ClassLoader of this class.
+ */
+ private ClassLoader getClassLoader() {
+ ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+ return classLoader == null ? getClass().getClassLoader() : classLoader;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java b/yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java
new file mode 100644
index 0000000..9335465
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/yarn/YarnTwillRunnerService.java
@@ -0,0 +1,583 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.yarn;
+
+import org.apache.twill.api.ResourceSpecification;
+import org.apache.twill.api.RunId;
+import org.apache.twill.api.SecureStore;
+import org.apache.twill.api.SecureStoreUpdater;
+import org.apache.twill.api.TwillApplication;
+import org.apache.twill.api.TwillController;
+import org.apache.twill.api.TwillPreparer;
+import org.apache.twill.api.TwillRunnable;
+import org.apache.twill.api.TwillRunnerService;
+import org.apache.twill.api.TwillSpecification;
+import org.apache.twill.api.logging.LogHandler;
+import org.apache.twill.common.Cancellable;
+import org.apache.twill.common.ServiceListenerAdapter;
+import org.apache.twill.common.Threads;
+import org.apache.twill.filesystem.HDFSLocationFactory;
+import org.apache.twill.filesystem.Location;
+import org.apache.twill.filesystem.LocationFactory;
+import org.apache.twill.internal.Constants;
+import org.apache.twill.internal.ProcessController;
+import org.apache.twill.internal.RunIds;
+import org.apache.twill.internal.SingleRunnableApplication;
+import org.apache.twill.internal.appmaster.ApplicationMasterLiveNodeData;
+import org.apache.twill.internal.yarn.VersionDetectYarnAppClientFactory;
+import org.apache.twill.internal.yarn.YarnAppClient;
+import org.apache.twill.internal.yarn.YarnApplicationReport;
+import org.apache.twill.internal.yarn.YarnUtils;
+import org.apache.twill.zookeeper.NodeChildren;
+import org.apache.twill.zookeeper.NodeData;
+import org.apache.twill.zookeeper.RetryStrategies;
+import org.apache.twill.zookeeper.ZKClient;
+import org.apache.twill.zookeeper.ZKClientService;
+import org.apache.twill.zookeeper.ZKClientServices;
+import org.apache.twill.zookeeper.ZKClients;
+import org.apache.twill.zookeeper.ZKOperations;
+import com.google.common.base.Charsets;
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Predicate;
+import com.google.common.base.Suppliers;
+import com.google.common.base.Throwables;
+import com.google.common.collect.HashBasedTable;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableTable;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Iterators;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Multimap;
+import com.google.common.collect.Sets;
+import com.google.common.collect.Table;
+import com.google.common.util.concurrent.AbstractIdleService;
+import com.google.common.util.concurrent.Callables;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.gson.Gson;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ScheduledFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+/**
+ * An implementation of {@link org.apache.twill.api.TwillRunnerService} that runs application on a YARN cluster.
+ */
+public final class YarnTwillRunnerService extends AbstractIdleService implements TwillRunnerService {
+
+ private static final Logger LOG = LoggerFactory.getLogger(YarnTwillRunnerService.class);
+
+ private static final int ZK_TIMEOUT = 10000;
+ private static final Function<String, RunId> STRING_TO_RUN_ID = new Function<String, RunId>() {
+ @Override
+ public RunId apply(String input) {
+ return RunIds.fromString(input);
+ }
+ };
+ private static final Function<YarnTwillController, TwillController> CAST_CONTROLLER =
+ new Function<YarnTwillController, TwillController>() {
+ @Override
+ public TwillController apply(YarnTwillController controller) {
+ return controller;
+ }
+ };
+
+ private final YarnConfiguration yarnConfig;
+ private final YarnAppClient yarnAppClient;
+ private final ZKClientService zkClientService;
+ private final LocationFactory locationFactory;
+ private final Table<String, RunId, YarnTwillController> controllers;
+ private ScheduledExecutorService secureStoreScheduler;
+
+ private Iterable<LiveInfo> liveInfos;
+ private Cancellable watchCancellable;
+ private volatile String jvmOptions = "";
+
+ public YarnTwillRunnerService(YarnConfiguration config, String zkConnect) {
+ this(config, zkConnect, new HDFSLocationFactory(getFileSystem(config), "/twill"));
+ }
+
+ public YarnTwillRunnerService(YarnConfiguration config, String zkConnect, LocationFactory locationFactory) {
+ this.yarnConfig = config;
+ this.yarnAppClient = new VersionDetectYarnAppClientFactory().create(config);
+ this.locationFactory = locationFactory;
+ this.zkClientService = getZKClientService(zkConnect);
+ this.controllers = HashBasedTable.create();
+ }
+
+ /**
+ * This methods sets the extra JVM options that will be passed to the java command line for every application
+ * started through this {@link YarnTwillRunnerService} instance. It only affects applications that are started
+ * after options is set.
+ *
+ * This is intended for advance usage. All options will be passed unchanged to the java command line. Invalid
+ * options could cause application not able to start.
+ *
+ * @param options extra JVM options.
+ */
+ public void setJVMOptions(String options) {
+ Preconditions.checkArgument(options != null, "JVM options cannot be null.");
+ this.jvmOptions = options;
+ }
+
+ @Override
+ public Cancellable scheduleSecureStoreUpdate(final SecureStoreUpdater updater,
+ long initialDelay, long delay, TimeUnit unit) {
+ if (!UserGroupInformation.isSecurityEnabled()) {
+ return new Cancellable() {
+ @Override
+ public void cancel() {
+ // No-op
+ }
+ };
+ }
+
+ synchronized (this) {
+ if (secureStoreScheduler == null) {
+ secureStoreScheduler = Executors.newSingleThreadScheduledExecutor(
+ Threads.createDaemonThreadFactory("secure-store-updater"));
+ }
+ }
+
+ final ScheduledFuture<?> future = secureStoreScheduler.scheduleWithFixedDelay(new Runnable() {
+ @Override
+ public void run() {
+ // Collects all <application, runId> pairs first
+ Multimap<String, RunId> liveApps = HashMultimap.create();
+ synchronized (YarnTwillRunnerService.this) {
+ for (Table.Cell<String, RunId, YarnTwillController> cell : controllers.cellSet()) {
+ liveApps.put(cell.getRowKey(), cell.getColumnKey());
+ }
+ }
+
+ // Collect all secure stores that needs to be updated.
+ Table<String, RunId, SecureStore> secureStores = HashBasedTable.create();
+ for (Map.Entry<String, RunId> entry : liveApps.entries()) {
+ try {
+ secureStores.put(entry.getKey(), entry.getValue(), updater.update(entry.getKey(), entry.getValue()));
+ } catch (Throwable t) {
+ LOG.warn("Exception thrown by SecureStoreUpdater {}", updater, t);
+ }
+ }
+
+ // Update secure stores.
+ updateSecureStores(secureStores);
+ }
+ }, initialDelay, delay, unit);
+
+ return new Cancellable() {
+ @Override
+ public void cancel() {
+ future.cancel(false);
+ }
+ };
+ }
+
+ @Override
+ public TwillPreparer prepare(TwillRunnable runnable) {
+ return prepare(runnable, ResourceSpecification.BASIC);
+ }
+
+ @Override
+ public TwillPreparer prepare(TwillRunnable runnable, ResourceSpecification resourceSpecification) {
+ return prepare(new SingleRunnableApplication(runnable, resourceSpecification));
+ }
+
+ @Override
+ public TwillPreparer prepare(TwillApplication application) {
+ Preconditions.checkState(isRunning(), "Service not start. Please call start() first.");
+ final TwillSpecification twillSpec = application.configure();
+ final String appName = twillSpec.getName();
+
+ return new YarnTwillPreparer(yarnConfig, twillSpec, yarnAppClient, zkClientService, locationFactory,
+ Suppliers.ofInstance(jvmOptions),
+ new YarnTwillControllerFactory() {
+ @Override
+ public YarnTwillController create(RunId runId, Iterable<LogHandler> logHandlers,
+ Callable<ProcessController<YarnApplicationReport>> startUp) {
+ ZKClient zkClient = ZKClients.namespace(zkClientService, "/" + appName);
+ YarnTwillController controller = listenController(new YarnTwillController(runId, zkClient,
+ logHandlers, startUp));
+ synchronized (YarnTwillRunnerService.this) {
+ Preconditions.checkArgument(!controllers.contains(appName, runId),
+ "Application %s with runId %s is already running.", appName, runId);
+ controllers.put(appName, runId, controller);
+ }
+ return controller;
+ }
+ });
+ }
+
+ @Override
+ public synchronized TwillController lookup(String applicationName, final RunId runId) {
+ return controllers.get(applicationName, runId);
+ }
+
+ @Override
+ public Iterable<TwillController> lookup(final String applicationName) {
+ return new Iterable<TwillController>() {
+ @Override
+ public Iterator<TwillController> iterator() {
+ synchronized (YarnTwillRunnerService.this) {
+ return Iterators.transform(ImmutableList.copyOf(controllers.row(applicationName).values()).iterator(),
+ CAST_CONTROLLER);
+ }
+ }
+ };
+ }
+
+ @Override
+ public Iterable<LiveInfo> lookupLive() {
+ return liveInfos;
+ }
+
+ @Override
+ protected void startUp() throws Exception {
+ yarnAppClient.startAndWait();
+ zkClientService.startAndWait();
+
+ // Create the root node, so that the namespace root would get created if it is missing
+ // If the exception is caused by node exists, then it's ok. Otherwise propagate the exception.
+ ZKOperations.ignoreError(zkClientService.create("/", null, CreateMode.PERSISTENT),
+ KeeperException.NodeExistsException.class, null).get();
+
+ watchCancellable = watchLiveApps();
+ liveInfos = createLiveInfos();
+
+ // Schedule an updater for updating HDFS delegation tokens
+ if (UserGroupInformation.isSecurityEnabled()) {
+ long delay = yarnConfig.getLong(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY,
+ DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_DEFAULT);
+ scheduleSecureStoreUpdate(new LocationSecureStoreUpdater(yarnConfig, locationFactory),
+ delay, delay, TimeUnit.MILLISECONDS);
+ }
+ }
+
+ @Override
+ protected void shutDown() throws Exception {
+ // Shutdown shouldn't stop any controllers, as stopping this client service should let the remote containers
+ // running. However, this assumes that this TwillRunnerService is a long running service and you only stop it
+ // when the JVM process is about to exit. Hence it is important that threads created in the controllers are
+ // daemon threads.
+ synchronized (this) {
+ if (secureStoreScheduler != null) {
+ secureStoreScheduler.shutdownNow();
+ }
+ }
+ watchCancellable.cancel();
+ zkClientService.stopAndWait();
+ yarnAppClient.stopAndWait();
+ }
+
+ private Cancellable watchLiveApps() {
+ final Map<String, Cancellable> watched = Maps.newConcurrentMap();
+
+ final AtomicBoolean cancelled = new AtomicBoolean(false);
+ // Watch child changes in the root, which gives all application names.
+ final Cancellable cancellable = ZKOperations.watchChildren(zkClientService, "/",
+ new ZKOperations.ChildrenCallback() {
+ @Override
+ public void updated(NodeChildren nodeChildren) {
+ if (cancelled.get()) {
+ return;
+ }
+
+ Set<String> apps = ImmutableSet.copyOf(nodeChildren.getChildren());
+
+ // For each for the application name, watch for ephemeral nodes under /instances.
+ for (final String appName : apps) {
+ if (watched.containsKey(appName)) {
+ continue;
+ }
+
+ final String instancePath = String.format("/%s/instances", appName);
+ watched.put(appName,
+ ZKOperations.watchChildren(zkClientService, instancePath, new ZKOperations.ChildrenCallback() {
+ @Override
+ public void updated(NodeChildren nodeChildren) {
+ if (cancelled.get()) {
+ return;
+ }
+ if (nodeChildren.getChildren().isEmpty()) { // No more child, means no live instances
+ Cancellable removed = watched.remove(appName);
+ if (removed != null) {
+ removed.cancel();
+ }
+ return;
+ }
+ synchronized (YarnTwillRunnerService.this) {
+ // For each of the children, which the node name is the runId,
+ // fetch the application Id and construct TwillController.
+ for (final RunId runId : Iterables.transform(nodeChildren.getChildren(), STRING_TO_RUN_ID)) {
+ if (controllers.contains(appName, runId)) {
+ continue;
+ }
+ updateController(appName, runId, cancelled);
+ }
+ }
+ }
+ }));
+ }
+
+ // Remove app watches for apps that are gone. Removal of controller from controllers table is done
+ // in the state listener attached to the twill controller.
+ for (String removeApp : Sets.difference(watched.keySet(), apps)) {
+ watched.remove(removeApp).cancel();
+ }
+ }
+ });
+ return new Cancellable() {
+ @Override
+ public void cancel() {
+ cancelled.set(true);
+ cancellable.cancel();
+ for (Cancellable c : watched.values()) {
+ c.cancel();
+ }
+ }
+ };
+ }
+
+ private YarnTwillController listenController(final YarnTwillController controller) {
+ controller.addListener(new ServiceListenerAdapter() {
+ @Override
+ public void terminated(State from) {
+ removeController();
+ }
+
+ @Override
+ public void failed(State from, Throwable failure) {
+ removeController();
+ }
+
+ private void removeController() {
+ synchronized (YarnTwillRunnerService.this) {
+ Iterables.removeIf(controllers.values(),
+ new Predicate<TwillController>() {
+ @Override
+ public boolean apply(TwillController input) {
+ return input == controller;
+ }
+ });
+ }
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+ return controller;
+ }
+
+ private ZKClientService getZKClientService(String zkConnect) {
+ return ZKClientServices.delegate(
+ ZKClients.reWatchOnExpire(
+ ZKClients.retryOnFailure(ZKClientService.Builder.of(zkConnect)
+ .setSessionTimeout(ZK_TIMEOUT)
+ .build(), RetryStrategies.exponentialDelay(100, 2000, TimeUnit.MILLISECONDS))));
+ }
+
+ private Iterable<LiveInfo> createLiveInfos() {
+ return new Iterable<LiveInfo>() {
+
+ @Override
+ public Iterator<LiveInfo> iterator() {
+ Map<String, Map<RunId, YarnTwillController>> controllerMap = ImmutableTable.copyOf(controllers).rowMap();
+ return Iterators.transform(controllerMap.entrySet().iterator(),
+ new Function<Map.Entry<String, Map<RunId, YarnTwillController>>, LiveInfo>() {
+ @Override
+ public LiveInfo apply(final Map.Entry<String, Map<RunId, YarnTwillController>> entry) {
+ return new LiveInfo() {
+ @Override
+ public String getApplicationName() {
+ return entry.getKey();
+ }
+
+ @Override
+ public Iterable<TwillController> getControllers() {
+ return Iterables.transform(entry.getValue().values(), CAST_CONTROLLER);
+ }
+ };
+ }
+ });
+ }
+ };
+ }
+
+ private void updateController(final String appName, final RunId runId, final AtomicBoolean cancelled) {
+ String instancePath = String.format("/%s/instances/%s", appName, runId.getId());
+
+ // Fetch the content node.
+ Futures.addCallback(zkClientService.getData(instancePath), new FutureCallback<NodeData>() {
+ @Override
+ public void onSuccess(NodeData result) {
+ if (cancelled.get()) {
+ return;
+ }
+ ApplicationId appId = getApplicationId(result);
+ if (appId == null) {
+ return;
+ }
+
+ synchronized (YarnTwillRunnerService.this) {
+ if (!controllers.contains(appName, runId)) {
+ ZKClient zkClient = ZKClients.namespace(zkClientService, "/" + appName);
+ YarnTwillController controller = listenController(
+ new YarnTwillController(runId, zkClient,
+ Callables.returning(yarnAppClient.createProcessController(appId))));
+ controllers.put(appName, runId, controller);
+ controller.start();
+ }
+ }
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ LOG.warn("Failed in fetching application instance node.", t);
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+ }
+
+
+ /**
+ * Decodes application ID stored inside the node data.
+ * @param nodeData The node data to decode from. If it is {@code null}, this method would return {@code null}.
+ * @return The ApplicationId or {@code null} if failed to decode.
+ */
+ private ApplicationId getApplicationId(NodeData nodeData) {
+ byte[] data = nodeData == null ? null : nodeData.getData();
+ if (data == null) {
+ return null;
+ }
+
+ Gson gson = new Gson();
+ JsonElement json = gson.fromJson(new String(data, Charsets.UTF_8), JsonElement.class);
+ if (!json.isJsonObject()) {
+ LOG.warn("Unable to decode live data node.");
+ return null;
+ }
+
+ JsonObject jsonObj = json.getAsJsonObject();
+ json = jsonObj.get("data");
+ if (!json.isJsonObject()) {
+ LOG.warn("Property data not found in live data node.");
+ return null;
+ }
+
+ try {
+ ApplicationMasterLiveNodeData amLiveNode = gson.fromJson(json, ApplicationMasterLiveNodeData.class);
+ return YarnUtils.createApplicationId(amLiveNode.getAppIdClusterTime(), amLiveNode.getAppId());
+ } catch (Exception e) {
+ LOG.warn("Failed to decode application live node data.", e);
+ return null;
+ }
+ }
+
+ private void updateSecureStores(Table<String, RunId, SecureStore> secureStores) {
+ for (Table.Cell<String, RunId, SecureStore> cell : secureStores.cellSet()) {
+ Object store = cell.getValue().getStore();
+ if (!(store instanceof Credentials)) {
+ LOG.warn("Only Hadoop Credentials is supported. Ignore update for {}.", cell);
+ continue;
+ }
+
+ Credentials credentials = (Credentials) store;
+ if (credentials.getAllTokens().isEmpty()) {
+ // Nothing to update.
+ continue;
+ }
+
+ try {
+ updateCredentials(cell.getRowKey(), cell.getColumnKey(), credentials);
+ synchronized (YarnTwillRunnerService.this) {
+ // Notify the application for secure store updates if it is still running.
+ YarnTwillController controller = controllers.get(cell.getRowKey(), cell.getColumnKey());
+ if (controller != null) {
+ controller.secureStoreUpdated();
+ }
+ }
+ } catch (Throwable t) {
+ LOG.warn("Failed to update secure store for {}.", cell, t);
+ }
+ }
+ }
+
+ private void updateCredentials(String application, RunId runId, Credentials updates) throws IOException {
+ Location credentialsLocation = locationFactory.create(String.format("/%s/%s/%s", application, runId.getId(),
+ Constants.Files.CREDENTIALS));
+ // Try to read the old credentials.
+ Credentials credentials = new Credentials();
+ if (credentialsLocation.exists()) {
+ DataInputStream is = new DataInputStream(new BufferedInputStream(credentialsLocation.getInputStream()));
+ try {
+ credentials.readTokenStorageStream(is);
+ } finally {
+ is.close();
+ }
+ }
+
+ // Overwrite with the updates.
+ credentials.addAll(updates);
+
+ // Overwrite the credentials.
+ Location tmpLocation = credentialsLocation.getTempFile(Constants.Files.CREDENTIALS);
+
+ // Save the credentials store with user-only permission.
+ DataOutputStream os = new DataOutputStream(new BufferedOutputStream(tmpLocation.getOutputStream("600")));
+ try {
+ credentials.writeTokenStorageToStream(os);
+ } finally {
+ os.close();
+ }
+
+ // Rename the tmp file into the credentials location
+ tmpLocation.renameTo(credentialsLocation);
+
+ LOG.debug("Secure store for {} {} saved to {}.", application, runId, credentialsLocation.toURI());
+ }
+
+ private static FileSystem getFileSystem(YarnConfiguration configuration) {
+ try {
+ return FileSystem.get(configuration);
+ } catch (IOException e) {
+ throw Throwables.propagate(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/yarn/package-info.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/yarn/package-info.java b/yarn/src/main/java/org/apache/twill/yarn/package-info.java
new file mode 100644
index 0000000..b3cbc5e
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/yarn/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Classes in this package implement the Twill API for Apache Hadoop YARN.
+ */
+package org.apache.twill.yarn;
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/resources/logback-template.xml
----------------------------------------------------------------------
diff --git a/yarn/src/main/resources/logback-template.xml b/yarn/src/main/resources/logback-template.xml
new file mode 100644
index 0000000..38cf6c8
--- /dev/null
+++ b/yarn/src/main/resources/logback-template.xml
@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="UTF-8"?>
+
+<!-- Default logback configuration for twill library -->
+<configuration>
+
+ <logger name="org.apache.hadoop" level="WARN" />
+ <logger name="org.apache.zookeeper" level="WARN" />
+
+ <root level="INFO" />
+
+</configuration>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/test/java/org/apache/twill/yarn/BuggyServer.java
----------------------------------------------------------------------
diff --git a/yarn/src/test/java/org/apache/twill/yarn/BuggyServer.java b/yarn/src/test/java/org/apache/twill/yarn/BuggyServer.java
new file mode 100644
index 0000000..bb1a583
--- /dev/null
+++ b/yarn/src/test/java/org/apache/twill/yarn/BuggyServer.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.yarn;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.PrintWriter;
+
+/**
+ * Server for testing that will die if you give it a 0.
+ */
+public final class BuggyServer extends SocketServer {
+
+ private static final Logger LOG = LoggerFactory.getLogger(BuggyServer.class);
+
+ @Override
+ public void handleRequest(BufferedReader reader, PrintWriter writer) throws IOException {
+ String line = reader.readLine();
+ LOG.info("Received: " + line + " going to divide by it");
+ Integer toDivide = Integer.valueOf(line);
+ writer.println(Integer.toString(100 / toDivide));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/test/java/org/apache/twill/yarn/DistributeShellTestRun.java
----------------------------------------------------------------------
diff --git a/yarn/src/test/java/org/apache/twill/yarn/DistributeShellTestRun.java b/yarn/src/test/java/org/apache/twill/yarn/DistributeShellTestRun.java
new file mode 100644
index 0000000..1054ec9
--- /dev/null
+++ b/yarn/src/test/java/org/apache/twill/yarn/DistributeShellTestRun.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.yarn;
+
+import org.apache.twill.api.TwillController;
+import org.apache.twill.api.TwillRunner;
+import org.apache.twill.api.logging.PrinterLogHandler;
+import org.apache.twill.common.ServiceListenerAdapter;
+import org.apache.twill.common.Threads;
+import com.google.common.util.concurrent.Service;
+import org.junit.Assert;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import java.io.PrintWriter;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * This test is executed by {@link YarnTestSuite}.
+ */
+public class DistributeShellTestRun {
+
+ @Ignore
+ @Test
+ public void testDistributedShell() throws InterruptedException {
+ TwillRunner twillRunner = YarnTestSuite.getTwillRunner();
+
+ TwillController controller = twillRunner.prepare(new DistributedShell("pwd", "ls -al"))
+ .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out)))
+ .start();
+
+ final CountDownLatch stopLatch = new CountDownLatch(1);
+ controller.addListener(new ServiceListenerAdapter() {
+
+ @Override
+ public void terminated(Service.State from) {
+ stopLatch.countDown();
+ }
+
+ @Override
+ public void failed(Service.State from, Throwable failure) {
+ stopLatch.countDown();
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+
+ Assert.assertTrue(stopLatch.await(10, TimeUnit.SECONDS));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/test/java/org/apache/twill/yarn/DistributedShell.java
----------------------------------------------------------------------
diff --git a/yarn/src/test/java/org/apache/twill/yarn/DistributedShell.java b/yarn/src/test/java/org/apache/twill/yarn/DistributedShell.java
new file mode 100644
index 0000000..c89371c
--- /dev/null
+++ b/yarn/src/test/java/org/apache/twill/yarn/DistributedShell.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.yarn;
+
+import org.apache.twill.api.AbstractTwillRunnable;
+import com.google.common.base.Charsets;
+import com.google.common.base.Joiner;
+import com.google.common.base.Splitter;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+
+/**
+ *
+ */
+public final class DistributedShell extends AbstractTwillRunnable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(DistributedShell.class);
+
+ public DistributedShell(String...commands) {
+ super(ImmutableMap.of("cmds", Joiner.on(';').join(commands)));
+ }
+
+ @Override
+ public void run() {
+ for (String cmd : Splitter.on(';').split(getArgument("cmds"))) {
+ try {
+ Process process = new ProcessBuilder(ImmutableList.copyOf(Splitter.on(' ').split(cmd)))
+ .redirectErrorStream(true).start();
+ BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream(), Charsets.US_ASCII));
+ try {
+ String line = reader.readLine();
+ while (line != null) {
+ LOG.info(line);
+ line = reader.readLine();
+ }
+ } finally {
+ reader.close();
+ }
+ } catch (IOException e) {
+ LOG.error("Fail to execute command " + cmd, e);
+ }
+ }
+ }
+
+ @Override
+ public void stop() {
+ // No-op
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/test/java/org/apache/twill/yarn/EchoServer.java
----------------------------------------------------------------------
diff --git a/yarn/src/test/java/org/apache/twill/yarn/EchoServer.java b/yarn/src/test/java/org/apache/twill/yarn/EchoServer.java
new file mode 100644
index 0000000..6b77e66
--- /dev/null
+++ b/yarn/src/test/java/org/apache/twill/yarn/EchoServer.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.yarn;
+
+import org.apache.twill.api.Command;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.PrintWriter;
+
+/**
+ * Test server that echoes back what it receives.
+ */
+public final class EchoServer extends SocketServer {
+
+ private static final Logger LOG = LoggerFactory.getLogger(EchoServer.class);
+
+ @Override
+ public void handleRequest(BufferedReader reader, PrintWriter writer) throws IOException {
+ String line = reader.readLine();
+ LOG.info("Received: " + line);
+ if (line != null) {
+ writer.println(line);
+ }
+ }
+
+ @Override
+ public void handleCommand(Command command) throws Exception {
+ LOG.info("Command received: " + command + " " + getContext().getInstanceCount());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/test/java/org/apache/twill/yarn/EchoServerTestRun.java
----------------------------------------------------------------------
diff --git a/yarn/src/test/java/org/apache/twill/yarn/EchoServerTestRun.java b/yarn/src/test/java/org/apache/twill/yarn/EchoServerTestRun.java
new file mode 100644
index 0000000..d868eef
--- /dev/null
+++ b/yarn/src/test/java/org/apache/twill/yarn/EchoServerTestRun.java
@@ -0,0 +1,138 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.yarn;
+
+import org.apache.twill.api.ResourceSpecification;
+import org.apache.twill.api.TwillController;
+import org.apache.twill.api.TwillRunner;
+import org.apache.twill.api.TwillRunnerService;
+import org.apache.twill.api.logging.PrinterLogHandler;
+import org.apache.twill.common.ServiceListenerAdapter;
+import org.apache.twill.common.Threads;
+import org.apache.twill.discovery.Discoverable;
+import com.google.common.base.Charsets;
+import com.google.common.io.LineReader;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.net.Socket;
+import java.net.URISyntaxException;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+/**
+ * Using echo server to test various behavior of YarnTwillService.
+ * This test is executed by {@link YarnTestSuite}.
+ */
+public class EchoServerTestRun {
+
+ private static final Logger LOG = LoggerFactory.getLogger(EchoServerTestRun.class);
+
+ @Test
+ public void testEchoServer() throws InterruptedException, ExecutionException, IOException,
+ URISyntaxException, TimeoutException {
+ TwillRunner runner = YarnTestSuite.getTwillRunner();
+
+ TwillController controller = runner.prepare(new EchoServer(),
+ ResourceSpecification.Builder.with()
+ .setVirtualCores(1)
+ .setMemory(1, ResourceSpecification.SizeUnit.GIGA)
+ .setInstances(2)
+ .build())
+ .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
+ .withApplicationArguments("echo")
+ .withArguments("EchoServer", "echo2")
+ .start();
+
+ final CountDownLatch running = new CountDownLatch(1);
+ controller.addListener(new ServiceListenerAdapter() {
+ @Override
+ public void running() {
+ running.countDown();
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+
+ Assert.assertTrue(running.await(30, TimeUnit.SECONDS));
+
+ Iterable<Discoverable> echoServices = controller.discoverService("echo");
+ Assert.assertTrue(YarnTestSuite.waitForSize(echoServices, 2, 60));
+
+ for (Discoverable discoverable : echoServices) {
+ String msg = "Hello: " + discoverable.getSocketAddress();
+
+ Socket socket = new Socket(discoverable.getSocketAddress().getAddress(),
+ discoverable.getSocketAddress().getPort());
+ try {
+ PrintWriter writer = new PrintWriter(new OutputStreamWriter(socket.getOutputStream(), Charsets.UTF_8), true);
+ LineReader reader = new LineReader(new InputStreamReader(socket.getInputStream(), Charsets.UTF_8));
+
+ writer.println(msg);
+ Assert.assertEquals(msg, reader.readLine());
+ } finally {
+ socket.close();
+ }
+ }
+
+ // Increase number of instances
+ controller.changeInstances("EchoServer", 3);
+ Assert.assertTrue(YarnTestSuite.waitForSize(echoServices, 3, 60));
+
+ echoServices = controller.discoverService("echo2");
+
+ // Decrease number of instances
+ controller.changeInstances("EchoServer", 1);
+ Assert.assertTrue(YarnTestSuite.waitForSize(echoServices, 1, 60));
+
+ // Increase number of instances again
+ controller.changeInstances("EchoServer", 2);
+ Assert.assertTrue(YarnTestSuite.waitForSize(echoServices, 2, 60));
+
+ // Make sure still only one app is running
+ Iterable<TwillRunner.LiveInfo> apps = runner.lookupLive();
+ Assert.assertTrue(YarnTestSuite.waitForSize(apps, 1, 60));
+
+ // Creates a new runner service to check it can regain control over running app.
+ TwillRunnerService runnerService = YarnTestSuite.createTwillRunnerService();
+ runnerService.startAndWait();
+
+ try {
+ Iterable <TwillController> controllers = runnerService.lookup("EchoServer");
+ Assert.assertTrue(YarnTestSuite.waitForSize(controllers, 1, 60));
+
+ for (TwillController c : controllers) {
+ LOG.info("Stopping application: " + c.getRunId());
+ c.stop().get(30, TimeUnit.SECONDS);
+ }
+
+ Assert.assertTrue(YarnTestSuite.waitForSize(apps, 0, 60));
+ } finally {
+ runnerService.stopAndWait();
+ }
+
+ // Sleep a bit before exiting.
+ TimeUnit.SECONDS.sleep(2);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/test/java/org/apache/twill/yarn/EnvironmentEchoServer.java
----------------------------------------------------------------------
diff --git a/yarn/src/test/java/org/apache/twill/yarn/EnvironmentEchoServer.java b/yarn/src/test/java/org/apache/twill/yarn/EnvironmentEchoServer.java
new file mode 100644
index 0000000..4be2472
--- /dev/null
+++ b/yarn/src/test/java/org/apache/twill/yarn/EnvironmentEchoServer.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.yarn;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.PrintWriter;
+
+/**
+ * Test server that returns back the value of the env key sent in. Used to check env for
+ * runnables is correctly set.
+ */
+public class EnvironmentEchoServer extends SocketServer {
+
+ @Override
+ public void handleRequest(BufferedReader reader, PrintWriter writer) throws IOException {
+ String envKey = reader.readLine();
+ writer.println(System.getenv(envKey));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/test/java/org/apache/twill/yarn/FailureRestartTestRun.java
----------------------------------------------------------------------
diff --git a/yarn/src/test/java/org/apache/twill/yarn/FailureRestartTestRun.java b/yarn/src/test/java/org/apache/twill/yarn/FailureRestartTestRun.java
new file mode 100644
index 0000000..b3d3933
--- /dev/null
+++ b/yarn/src/test/java/org/apache/twill/yarn/FailureRestartTestRun.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.yarn;
+
+import org.apache.twill.api.Command;
+import org.apache.twill.api.ResourceSpecification;
+import org.apache.twill.api.TwillController;
+import org.apache.twill.api.TwillRunner;
+import org.apache.twill.api.logging.PrinterLogHandler;
+import org.apache.twill.discovery.Discoverable;
+import com.google.common.base.Charsets;
+import com.google.common.collect.Sets;
+import com.google.common.io.LineReader;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+
+/**
+ *
+ */
+public class FailureRestartTestRun {
+
+ @Test
+ public void testFailureRestart() throws Exception {
+ TwillRunner runner = YarnTestSuite.getTwillRunner();
+
+ ResourceSpecification resource = ResourceSpecification.Builder.with()
+ .setVirtualCores(1)
+ .setMemory(512, ResourceSpecification.SizeUnit.MEGA)
+ .setInstances(2)
+ .build();
+ TwillController controller = runner.prepare(new FailureRunnable(), resource)
+ .withApplicationArguments("failure")
+ .withArguments(FailureRunnable.class.getSimpleName(), "failure2")
+ .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
+ .start();
+
+ Iterable<Discoverable> discoverables = controller.discoverService("failure");
+ Assert.assertTrue(YarnTestSuite.waitForSize(discoverables, 2, 60));
+
+ // Make sure we see the right instance IDs
+ Assert.assertEquals(Sets.newHashSet(0, 1), getInstances(discoverables));
+
+ // Kill server with instanceId = 0
+ controller.sendCommand(FailureRunnable.class.getSimpleName(), Command.Builder.of("kill0").build());
+
+ // Do a shot sleep, make sure the runnable is killed.
+ TimeUnit.SECONDS.sleep(5);
+
+ Assert.assertTrue(YarnTestSuite.waitForSize(discoverables, 2, 60));
+ // Make sure we see the right instance IDs
+ Assert.assertEquals(Sets.newHashSet(0, 1), getInstances(discoverables));
+
+ controller.stopAndWait();
+ }
+
+ private Set<Integer> getInstances(Iterable<Discoverable> discoverables) throws IOException {
+ Set<Integer> instances = Sets.newHashSet();
+ for (Discoverable discoverable : discoverables) {
+ InetSocketAddress socketAddress = discoverable.getSocketAddress();
+ Socket socket = new Socket(socketAddress.getAddress(), socketAddress.getPort());
+ try {
+ PrintWriter writer = new PrintWriter(new OutputStreamWriter(socket.getOutputStream(), Charsets.UTF_8), true);
+ LineReader reader = new LineReader(new InputStreamReader(socket.getInputStream(), Charsets.UTF_8));
+
+ String msg = "Failure";
+ writer.println(msg);
+
+ String line = reader.readLine();
+ Assert.assertTrue(line.endsWith(msg));
+ instances.add(Integer.parseInt(line.substring(0, line.length() - msg.length())));
+ } finally {
+ socket.close();
+ }
+ }
+ return instances;
+ }
+
+
+ public static final class FailureRunnable extends SocketServer {
+
+ private volatile boolean killed;
+
+ @Override
+ public void run() {
+ killed = false;
+ super.run();
+ if (killed) {
+ throw new RuntimeException("Exception");
+ }
+ }
+
+ @Override
+ public void handleCommand(Command command) throws Exception {
+ if (command.getCommand().equals("kill" + getContext().getInstanceId())) {
+ killed = true;
+ running = false;
+ serverSocket.close();
+ }
+ }
+
+ @Override
+ public void handleRequest(BufferedReader reader, PrintWriter writer) throws IOException {
+ String line = reader.readLine();
+ writer.println(getContext().getInstanceId() + line);
+ writer.flush();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/test/java/org/apache/twill/yarn/LocalFileTestRun.java
----------------------------------------------------------------------
diff --git a/yarn/src/test/java/org/apache/twill/yarn/LocalFileTestRun.java b/yarn/src/test/java/org/apache/twill/yarn/LocalFileTestRun.java
new file mode 100644
index 0000000..de2c74c
--- /dev/null
+++ b/yarn/src/test/java/org/apache/twill/yarn/LocalFileTestRun.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.yarn;
+
+import org.apache.twill.api.TwillApplication;
+import org.apache.twill.api.TwillController;
+import org.apache.twill.api.TwillRunner;
+import org.apache.twill.api.TwillSpecification;
+import org.apache.twill.api.logging.PrinterLogHandler;
+import org.apache.twill.discovery.Discoverable;
+import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
+import com.google.common.io.ByteStreams;
+import com.google.common.io.Files;
+import com.google.common.io.LineReader;
+import org.junit.Assert;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.util.concurrent.TimeUnit;
+import java.util.jar.JarEntry;
+import java.util.jar.JarOutputStream;
+
+/**
+ * Test for local file transfer.
+ */
+public class LocalFileTestRun {
+
+ @ClassRule
+ public static TemporaryFolder tmpFolder = new TemporaryFolder();
+
+ @Test
+ public void testLocalFile() throws Exception {
+ String header = Files.readFirstLine(new File(getClass().getClassLoader().getResource("header.txt").toURI()),
+ Charsets.UTF_8);
+
+ TwillRunner runner = YarnTestSuite.getTwillRunner();
+ if (runner instanceof YarnTwillRunnerService) {
+ ((YarnTwillRunnerService) runner).setJVMOptions("-verbose:gc -Xloggc:gc.log -XX:+PrintGCDetails");
+ }
+
+ TwillController controller = runner.prepare(new LocalFileApplication())
+ .withApplicationArguments("local")
+ .withArguments("LocalFileSocketServer", "local2")
+ .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
+ .start();
+
+ if (runner instanceof YarnTwillRunnerService) {
+ ((YarnTwillRunnerService) runner).setJVMOptions("");
+ }
+
+ Iterable<Discoverable> discoverables = controller.discoverService("local");
+ Assert.assertTrue(YarnTestSuite.waitForSize(discoverables, 1, 60));
+
+ InetSocketAddress socketAddress = discoverables.iterator().next().getSocketAddress();
+ Socket socket = new Socket(socketAddress.getAddress(), socketAddress.getPort());
+ try {
+ PrintWriter writer = new PrintWriter(new OutputStreamWriter(socket.getOutputStream(), Charsets.UTF_8), true);
+ LineReader reader = new LineReader(new InputStreamReader(socket.getInputStream(), Charsets.UTF_8));
+
+ String msg = "Local file test";
+ writer.println(msg);
+ Assert.assertEquals(header, reader.readLine());
+ Assert.assertEquals(msg, reader.readLine());
+ } finally {
+ socket.close();
+ }
+
+ controller.stopAndWait();
+
+ Assert.assertTrue(YarnTestSuite.waitForSize(discoverables, 0, 60));
+
+ TimeUnit.SECONDS.sleep(2);
+ }
+
+ public static final class LocalFileApplication implements TwillApplication {
+
+ private final File headerFile;
+
+ public LocalFileApplication() throws Exception {
+ // Create a jar file that contains the header.txt file inside.
+ headerFile = tmpFolder.newFile("header.jar");
+ JarOutputStream os = new JarOutputStream(new FileOutputStream(headerFile));
+ try {
+ os.putNextEntry(new JarEntry("header.txt"));
+ ByteStreams.copy(getClass().getClassLoader().getResourceAsStream("header.txt"), os);
+ } finally {
+ os.close();
+ }
+ }
+
+ @Override
+ public TwillSpecification configure() {
+ return TwillSpecification.Builder.with()
+ .setName("LocalFileApp")
+ .withRunnable()
+ .add(new LocalFileSocketServer())
+ .withLocalFiles()
+ .add("header", headerFile, true).apply()
+ .anyOrder()
+ .build();
+ }
+ }
+
+ public static final class LocalFileSocketServer extends SocketServer {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LocalFileSocketServer.class);
+
+ @Override
+ public void handleRequest(BufferedReader reader, PrintWriter writer) throws IOException {
+ // Verify there is a gc.log file locally
+ Preconditions.checkState(new File("gc.log").exists());
+
+ LOG.info("handleRequest");
+ String header = Files.toString(new File("header/header.txt"), Charsets.UTF_8);
+ writer.write(header);
+ writer.println(reader.readLine());
+ LOG.info("Flushed response");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/test/java/org/apache/twill/yarn/ProvisionTimeoutTestRun.java
----------------------------------------------------------------------
diff --git a/yarn/src/test/java/org/apache/twill/yarn/ProvisionTimeoutTestRun.java b/yarn/src/test/java/org/apache/twill/yarn/ProvisionTimeoutTestRun.java
new file mode 100644
index 0000000..0598ef1
--- /dev/null
+++ b/yarn/src/test/java/org/apache/twill/yarn/ProvisionTimeoutTestRun.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.yarn;
+
+import org.apache.twill.api.AbstractTwillRunnable;
+import org.apache.twill.api.EventHandler;
+import org.apache.twill.api.EventHandlerContext;
+import org.apache.twill.api.ResourceSpecification;
+import org.apache.twill.api.TwillApplication;
+import org.apache.twill.api.TwillController;
+import org.apache.twill.api.TwillRunner;
+import org.apache.twill.api.TwillSpecification;
+import org.apache.twill.api.logging.PrinterLogHandler;
+import org.apache.twill.common.Services;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableMap;
+import org.junit.Test;
+
+import java.io.PrintWriter;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+/**
+ *
+ */
+public class ProvisionTimeoutTestRun {
+
+ @Test
+ public void testProvisionTimeout() throws InterruptedException, ExecutionException, TimeoutException {
+ TwillRunner runner = YarnTestSuite.getTwillRunner();
+
+ TwillController controller = runner.prepare(new TimeoutApplication())
+ .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
+ .start();
+
+ // The provision should failed in 30 seconds after AM started, which AM could took a while to start.
+ // Hence we give 90 seconds max time here.
+ try {
+ Services.getCompletionFuture(controller).get(90, TimeUnit.SECONDS);
+ } finally {
+ // If it timeout, kill the app as cleanup.
+ controller.kill();
+ }
+ }
+
+ public static final class Handler extends EventHandler {
+
+ private boolean abort;
+
+ @Override
+ protected Map<String, String> getConfigs() {
+ return ImmutableMap.of("abort", "true");
+ }
+
+ @Override
+ public void initialize(EventHandlerContext context) {
+ this.abort = Boolean.parseBoolean(context.getSpecification().getConfigs().get("abort"));
+ }
+
+ @Override
+ public TimeoutAction launchTimeout(Iterable<TimeoutEvent> timeoutEvents) {
+ if (abort) {
+ return TimeoutAction.abort();
+ } else {
+ return TimeoutAction.recheck(10, TimeUnit.SECONDS);
+ }
+ }
+ }
+
+ public static final class TimeoutApplication implements TwillApplication {
+
+ @Override
+ public TwillSpecification configure() {
+ return TwillSpecification.Builder.with()
+ .setName("TimeoutApplication")
+ .withRunnable()
+ .add(new TimeoutRunnable(),
+ ResourceSpecification.Builder.with()
+ .setVirtualCores(1)
+ .setMemory(8, ResourceSpecification.SizeUnit.GIGA).build())
+ .noLocalFiles()
+ .anyOrder()
+ .withEventHandler(new Handler())
+ .build();
+ }
+ }
+
+ /**
+ * A runnable that do nothing, as it's not expected to get provisioned.
+ */
+ public static final class TimeoutRunnable extends AbstractTwillRunnable {
+
+ private final CountDownLatch latch = new CountDownLatch(1);
+
+ @Override
+ public void stop() {
+ latch.countDown();
+ }
+
+ @Override
+ public void run() {
+ // Simply block here
+ try {
+ latch.await();
+ } catch (InterruptedException e) {
+ throw Throwables.propagate(e);
+ }
+ }
+ }
+}
[12/15] Initial import commit.
Posted by ch...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/TwillContainerLauncher.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/TwillContainerLauncher.java b/core/src/main/java/org/apache/twill/internal/TwillContainerLauncher.java
new file mode 100644
index 0000000..63f8732
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/TwillContainerLauncher.java
@@ -0,0 +1,181 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal;
+
+import org.apache.twill.api.LocalFile;
+import org.apache.twill.api.RunId;
+import org.apache.twill.api.RuntimeSpecification;
+import org.apache.twill.filesystem.Location;
+import org.apache.twill.internal.state.Message;
+import org.apache.twill.internal.state.StateNode;
+import org.apache.twill.launcher.TwillLauncher;
+import org.apache.twill.zookeeper.NodeData;
+import org.apache.twill.zookeeper.ZKClient;
+import com.google.common.util.concurrent.ListenableFuture;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+/**
+ * This class helps launching a container.
+ */
+public final class TwillContainerLauncher {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TwillContainerLauncher.class);
+
+ private static final double HEAP_MIN_RATIO = 0.7d;
+
+ private final RuntimeSpecification runtimeSpec;
+ private final ProcessLauncher.PrepareLaunchContext launchContext;
+ private final ZKClient zkClient;
+ private final int instanceCount;
+ private final String jvmOpts;
+ private final int reservedMemory;
+ private final Location secureStoreLocation;
+
+ public TwillContainerLauncher(RuntimeSpecification runtimeSpec, ProcessLauncher.PrepareLaunchContext launchContext,
+ ZKClient zkClient, int instanceCount, String jvmOpts, int reservedMemory,
+ Location secureStoreLocation) {
+ this.runtimeSpec = runtimeSpec;
+ this.launchContext = launchContext;
+ this.zkClient = zkClient;
+ this.instanceCount = instanceCount;
+ this.jvmOpts = jvmOpts;
+ this.reservedMemory = reservedMemory;
+ this.secureStoreLocation = secureStoreLocation;
+ }
+
+ public TwillContainerController start(RunId runId, int instanceId, Class<?> mainClass, String classPath) {
+ ProcessLauncher.PrepareLaunchContext.AfterResources afterResources = null;
+ ProcessLauncher.PrepareLaunchContext.ResourcesAdder resourcesAdder = null;
+
+ // Adds all file to be localized to container
+ if (!runtimeSpec.getLocalFiles().isEmpty()) {
+ resourcesAdder = launchContext.withResources();
+
+ for (LocalFile localFile : runtimeSpec.getLocalFiles()) {
+ afterResources = resourcesAdder.add(localFile);
+ }
+ }
+
+ // Optionally localize secure store.
+ try {
+ if (secureStoreLocation != null && secureStoreLocation.exists()) {
+ if (resourcesAdder == null) {
+ resourcesAdder = launchContext.withResources();
+ }
+ afterResources = resourcesAdder.add(new DefaultLocalFile(Constants.Files.CREDENTIALS,
+ secureStoreLocation.toURI(),
+ secureStoreLocation.lastModified(),
+ secureStoreLocation.length(), false, null));
+ }
+ } catch (IOException e) {
+ LOG.warn("Failed to launch container with secure store {}.", secureStoreLocation.toURI());
+ }
+
+ if (afterResources == null) {
+ afterResources = launchContext.noResources();
+ }
+
+ int memory = runtimeSpec.getResourceSpecification().getMemorySize();
+ if (((double) (memory - reservedMemory) / memory) >= HEAP_MIN_RATIO) {
+ // Reduce -Xmx by the reserved memory size.
+ memory = runtimeSpec.getResourceSpecification().getMemorySize() - reservedMemory;
+ } else {
+ // If it is a small VM, just discount it by the min ratio.
+ memory = (int) Math.ceil(memory * HEAP_MIN_RATIO);
+ }
+
+ // Currently no reporting is supported for runnable containers
+ ProcessController<Void> processController = afterResources
+ .withEnvironment()
+ .add(EnvKeys.TWILL_RUN_ID, runId.getId())
+ .add(EnvKeys.TWILL_RUNNABLE_NAME, runtimeSpec.getName())
+ .add(EnvKeys.TWILL_INSTANCE_ID, Integer.toString(instanceId))
+ .add(EnvKeys.TWILL_INSTANCE_COUNT, Integer.toString(instanceCount))
+ .withCommands()
+ .add("java",
+ "-Djava.io.tmpdir=tmp",
+ "-Dyarn.container=$" + EnvKeys.YARN_CONTAINER_ID,
+ "-Dtwill.runnable=$" + EnvKeys.TWILL_APP_NAME + ".$" + EnvKeys.TWILL_RUNNABLE_NAME,
+ "-cp", Constants.Files.LAUNCHER_JAR + ":" + classPath,
+ "-Xmx" + memory + "m",
+ jvmOpts,
+ TwillLauncher.class.getName(),
+ Constants.Files.CONTAINER_JAR,
+ mainClass.getName(),
+ Boolean.TRUE.toString())
+ .redirectOutput(Constants.STDOUT).redirectError(Constants.STDERR)
+ .launch();
+
+ TwillContainerControllerImpl controller = new TwillContainerControllerImpl(zkClient, runId, processController);
+ controller.start();
+ return controller;
+ }
+
+ private static final class TwillContainerControllerImpl extends AbstractZKServiceController
+ implements TwillContainerController {
+
+ private final ProcessController<Void> processController;
+
+ protected TwillContainerControllerImpl(ZKClient zkClient, RunId runId,
+ ProcessController<Void> processController) {
+ super(runId, zkClient);
+ this.processController = processController;
+ }
+
+ @Override
+ protected void doStartUp() {
+ // No-op
+ }
+
+ @Override
+ protected void doShutDown() {
+ // No-op
+ }
+
+ @Override
+ protected void instanceNodeUpdated(NodeData nodeData) {
+ // No-op
+ }
+
+ @Override
+ protected void stateNodeUpdated(StateNode stateNode) {
+ // No-op
+ }
+
+ @Override
+ public ListenableFuture<Message> sendMessage(Message message) {
+ return sendMessage(message, message);
+ }
+
+ @Override
+ public synchronized void completed(int exitStatus) {
+ if (exitStatus != 0) { // If a container terminated with exit code != 0, treat it as error
+// fireStateChange(new StateNode(State.FAILED, new StackTraceElement[0]));
+ }
+ forceShutDown();
+ }
+
+ @Override
+ public void kill() {
+ processController.cancel();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/ZKMessages.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/ZKMessages.java b/core/src/main/java/org/apache/twill/internal/ZKMessages.java
new file mode 100644
index 0000000..03575dd
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/ZKMessages.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal;
+
+import org.apache.twill.internal.state.Message;
+import org.apache.twill.internal.state.MessageCodec;
+import org.apache.twill.zookeeper.ZKClient;
+import org.apache.twill.zookeeper.ZKOperations;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.SettableFuture;
+import org.apache.zookeeper.CreateMode;
+
+/**
+ *
+ */
+public final class ZKMessages {
+
+ /**
+ * Creates a message node in zookeeper. The message node created is a PERSISTENT_SEQUENTIAL node.
+ *
+ * @param zkClient The ZooKeeper client for interacting with ZooKeeper.
+ * @param messagePathPrefix ZooKeeper path prefix for the message node.
+ * @param message The {@link Message} object for the content of the message node.
+ * @param completionResult Object to set to the result future when the message is processed.
+ * @param <V> Type of the completion result.
+ * @return A {@link ListenableFuture} that will be completed when the message is consumed, which indicated
+ * by deletion of the node. If there is exception during the process, it will be reflected
+ * to the future returned.
+ */
+ public static <V> ListenableFuture<V> sendMessage(final ZKClient zkClient, String messagePathPrefix,
+ Message message, final V completionResult) {
+ SettableFuture<V> result = SettableFuture.create();
+ sendMessage(zkClient, messagePathPrefix, message, result, completionResult);
+ return result;
+ }
+
+ /**
+ * Creates a message node in zookeeper. The message node created is a PERSISTENT_SEQUENTIAL node.
+ *
+ * @param zkClient The ZooKeeper client for interacting with ZooKeeper.
+ * @param messagePathPrefix ZooKeeper path prefix for the message node.
+ * @param message The {@link Message} object for the content of the message node.
+ * @param completion A {@link SettableFuture} to reflect the result of message process completion.
+ * @param completionResult Object to set to the result future when the message is processed.
+ * @param <V> Type of the completion result.
+ */
+ public static <V> void sendMessage(final ZKClient zkClient, String messagePathPrefix, Message message,
+ final SettableFuture<V> completion, final V completionResult) {
+
+ // Creates a message and watch for its deletion for completion.
+ Futures.addCallback(zkClient.create(messagePathPrefix, MessageCodec.encode(message),
+ CreateMode.PERSISTENT_SEQUENTIAL), new FutureCallback<String>() {
+ @Override
+ public void onSuccess(String path) {
+ Futures.addCallback(ZKOperations.watchDeleted(zkClient, path), new FutureCallback<String>() {
+ @Override
+ public void onSuccess(String result) {
+ completion.set(completionResult);
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ completion.setException(t);
+ }
+ });
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ completion.setException(t);
+ }
+ });
+ }
+
+ private ZKMessages() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/ZKServiceDecorator.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/ZKServiceDecorator.java b/core/src/main/java/org/apache/twill/internal/ZKServiceDecorator.java
new file mode 100644
index 0000000..7313d33
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/ZKServiceDecorator.java
@@ -0,0 +1,482 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal;
+
+import org.apache.twill.api.RunId;
+import org.apache.twill.api.ServiceController;
+import org.apache.twill.common.ServiceListenerAdapter;
+import org.apache.twill.common.Threads;
+import org.apache.twill.internal.json.StackTraceElementCodec;
+import org.apache.twill.internal.json.StateNodeCodec;
+import org.apache.twill.internal.state.Message;
+import org.apache.twill.internal.state.MessageCallback;
+import org.apache.twill.internal.state.MessageCodec;
+import org.apache.twill.internal.state.StateNode;
+import org.apache.twill.internal.state.SystemMessages;
+import org.apache.twill.zookeeper.NodeChildren;
+import org.apache.twill.zookeeper.NodeData;
+import org.apache.twill.zookeeper.OperationFuture;
+import org.apache.twill.zookeeper.ZKClient;
+import org.apache.twill.zookeeper.ZKOperations;
+import com.google.common.base.Charsets;
+import com.google.common.base.Supplier;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.util.concurrent.AbstractService;
+import com.google.common.util.concurrent.AsyncFunction;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.common.util.concurrent.Service;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import javax.annotation.Nullable;
+import java.util.Collections;
+import java.util.List;
+import java.util.concurrent.Executor;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/**
+ * A {@link Service} decorator that wrap another {@link Service} with the service states reflected
+ * to ZooKeeper.
+ */
+public final class ZKServiceDecorator extends AbstractService {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ZKServiceDecorator.class);
+
+ private final ZKClient zkClient;
+ private final RunId id;
+ private final Supplier<? extends JsonElement> liveNodeData;
+ private final Service decoratedService;
+ private final MessageCallbackCaller messageCallback;
+ private ExecutorService callbackExecutor;
+
+
+ public ZKServiceDecorator(ZKClient zkClient, RunId id, Supplier<? extends JsonElement> liveNodeData,
+ Service decoratedService) {
+ this(zkClient, id, liveNodeData, decoratedService, null);
+ }
+
+ /**
+ * Creates a ZKServiceDecorator.
+ * @param zkClient ZooKeeper client
+ * @param id The run id of the service
+ * @param liveNodeData A supplier for providing information writing to live node.
+ * @param decoratedService The Service for monitoring state changes
+ * @param finalizer An optional Runnable to run when this decorator terminated.
+ */
+ public ZKServiceDecorator(ZKClient zkClient, RunId id, Supplier <? extends JsonElement> liveNodeData,
+ Service decoratedService, @Nullable Runnable finalizer) {
+ this.zkClient = zkClient;
+ this.id = id;
+ this.liveNodeData = liveNodeData;
+ this.decoratedService = decoratedService;
+ if (decoratedService instanceof MessageCallback) {
+ this.messageCallback = new MessageCallbackCaller((MessageCallback) decoratedService, zkClient);
+ } else {
+ this.messageCallback = new MessageCallbackCaller(zkClient);
+ }
+ if (finalizer != null) {
+ addFinalizer(finalizer);
+ }
+ }
+
+ /**
+ * Deletes the given ZK path recursively and create the path again.
+ */
+ private ListenableFuture<String> deleteAndCreate(final String path, final byte[] data, final CreateMode mode) {
+ return Futures.transform(ZKOperations.ignoreError(ZKOperations.recursiveDelete(zkClient, path),
+ KeeperException.NoNodeException.class, null),
+ new AsyncFunction<String, String>() {
+ @Override
+ public ListenableFuture<String> apply(String input) throws Exception {
+ return zkClient.create(path, data, mode);
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+ }
+
+ @Override
+ protected void doStart() {
+ callbackExecutor = Executors.newSingleThreadExecutor(Threads.createDaemonThreadFactory("message-callback"));
+ Futures.addCallback(createLiveNode(), new FutureCallback<String>() {
+ @Override
+ public void onSuccess(String result) {
+ // Create nodes for states and messaging
+ StateNode stateNode = new StateNode(ServiceController.State.STARTING);
+
+ final ListenableFuture<List<String>> createFuture = Futures.allAsList(
+ deleteAndCreate(getZKPath("messages"), null, CreateMode.PERSISTENT),
+ deleteAndCreate(getZKPath("state"), encodeStateNode(stateNode), CreateMode.PERSISTENT)
+ );
+
+ createFuture.addListener(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ createFuture.get();
+ // Starts the decorated service
+ decoratedService.addListener(createListener(), Threads.SAME_THREAD_EXECUTOR);
+ decoratedService.start();
+ } catch (Exception e) {
+ notifyFailed(e);
+ }
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ notifyFailed(t);
+ }
+ });
+ }
+
+ @Override
+ protected void doStop() {
+ // Stops the decorated service
+ decoratedService.stop();
+ callbackExecutor.shutdownNow();
+ }
+
+ private void addFinalizer(final Runnable finalizer) {
+ addListener(new ServiceListenerAdapter() {
+ @Override
+ public void terminated(State from) {
+ try {
+ finalizer.run();
+ } catch (Throwable t) {
+ LOG.warn("Exception when running finalizer.", t);
+ }
+ }
+
+ @Override
+ public void failed(State from, Throwable failure) {
+ try {
+ finalizer.run();
+ } catch (Throwable t) {
+ LOG.warn("Exception when running finalizer.", t);
+ }
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+ }
+
+ private OperationFuture<String> createLiveNode() {
+ String liveNode = getLiveNodePath();
+ LOG.info("Create live node {}{}", zkClient.getConnectString(), liveNode);
+
+ JsonObject content = new JsonObject();
+ content.add("data", liveNodeData.get());
+ return ZKOperations.ignoreError(zkClient.create(liveNode, encodeJson(content), CreateMode.EPHEMERAL),
+ KeeperException.NodeExistsException.class, liveNode);
+ }
+
+ private OperationFuture<String> removeLiveNode() {
+ String liveNode = getLiveNodePath();
+ LOG.info("Remove live node {}{}", zkClient.getConnectString(), liveNode);
+ return ZKOperations.ignoreError(zkClient.delete(liveNode), KeeperException.NoNodeException.class, liveNode);
+ }
+
+ private OperationFuture<String> removeServiceNode() {
+ String serviceNode = String.format("/%s", id.getId());
+ LOG.info("Remove service node {}{}", zkClient.getConnectString(), serviceNode);
+ return ZKOperations.recursiveDelete(zkClient, serviceNode);
+ }
+
+ private void watchMessages() {
+ final String messagesPath = getZKPath("messages");
+ Futures.addCallback(zkClient.getChildren(messagesPath, new Watcher() {
+ @Override
+ public void process(WatchedEvent event) {
+ // TODO: Do we need to deal with other type of events?
+ if (event.getType() == Event.EventType.NodeChildrenChanged && decoratedService.isRunning()) {
+ watchMessages();
+ }
+ }
+ }), new FutureCallback<NodeChildren>() {
+ @Override
+ public void onSuccess(NodeChildren result) {
+ // Sort by the name, which is the messageId. Assumption is that message ids is ordered by time.
+ List<String> messages = Lists.newArrayList(result.getChildren());
+ Collections.sort(messages);
+ for (String messageId : messages) {
+ processMessage(messagesPath + "/" + messageId, messageId);
+ }
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ // TODO: what could be done besides just logging?
+ LOG.error("Failed to watch messages.", t);
+ }
+ });
+ }
+
+ private void processMessage(final String path, final String messageId) {
+ Futures.addCallback(zkClient.getData(path), new FutureCallback<NodeData>() {
+ @Override
+ public void onSuccess(NodeData result) {
+ Message message = MessageCodec.decode(result.getData());
+ if (message == null) {
+ LOG.error("Failed to decode message for " + messageId + " in " + path);
+ listenFailure(zkClient.delete(path, result.getStat().getVersion()));
+ return;
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Message received from " + path + ": " + new String(MessageCodec.encode(message), Charsets.UTF_8));
+ }
+ if (handleStopMessage(message, getDeleteSupplier(path, result.getStat().getVersion()))) {
+ return;
+ }
+ messageCallback.onReceived(callbackExecutor, path, result.getStat().getVersion(), messageId, message);
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ LOG.error("Failed to fetch message content.", t);
+ }
+ });
+ }
+
+ private <V> boolean handleStopMessage(Message message, final Supplier<OperationFuture<V>> postHandleSupplier) {
+ if (message.getType() == Message.Type.SYSTEM && SystemMessages.STOP_COMMAND.equals(message.getCommand())) {
+ callbackExecutor.execute(new Runnable() {
+ @Override
+ public void run() {
+ decoratedService.stop().addListener(new Runnable() {
+
+ @Override
+ public void run() {
+ stopServiceOnComplete(postHandleSupplier.get(), ZKServiceDecorator.this);
+ }
+ }, MoreExecutors.sameThreadExecutor());
+ }
+ });
+ return true;
+ }
+ return false;
+ }
+
+
+ private Supplier<OperationFuture<String>> getDeleteSupplier(final String path, final int version) {
+ return new Supplier<OperationFuture<String>>() {
+ @Override
+ public OperationFuture<String> get() {
+ return zkClient.delete(path, version);
+ }
+ };
+ }
+
+ private Listener createListener() {
+ return new DecoratedServiceListener();
+ }
+
+ private <V> byte[] encode(V data, Class<? extends V> clz) {
+ return new GsonBuilder().registerTypeAdapter(StateNode.class, new StateNodeCodec())
+ .registerTypeAdapter(StackTraceElement.class, new StackTraceElementCodec())
+ .create()
+ .toJson(data, clz).getBytes(Charsets.UTF_8);
+ }
+
+ private byte[] encodeStateNode(StateNode stateNode) {
+ return encode(stateNode, StateNode.class);
+ }
+
+ private <V extends JsonElement> byte[] encodeJson(V json) {
+ return new Gson().toJson(json).getBytes(Charsets.UTF_8);
+ }
+
+ private String getZKPath(String path) {
+ return String.format("/%s/%s", id, path);
+ }
+
+ private String getLiveNodePath() {
+ return "/instances/" + id;
+ }
+
+ private static <V> OperationFuture<V> listenFailure(final OperationFuture<V> operationFuture) {
+ operationFuture.addListener(new Runnable() {
+
+ @Override
+ public void run() {
+ try {
+ if (!operationFuture.isCancelled()) {
+ operationFuture.get();
+ }
+ } catch (Exception e) {
+ // TODO: what could be done besides just logging?
+ LOG.error("Operation execution failed for " + operationFuture.getRequestPath(), e);
+ }
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+ return operationFuture;
+ }
+
+ private static final class MessageCallbackCaller {
+ private final MessageCallback callback;
+ private final ZKClient zkClient;
+
+ private MessageCallbackCaller(ZKClient zkClient) {
+ this(null, zkClient);
+ }
+
+ private MessageCallbackCaller(MessageCallback callback, ZKClient zkClient) {
+ this.callback = callback;
+ this.zkClient = zkClient;
+ }
+
+ public void onReceived(Executor executor, final String path,
+ final int version, final String id, final Message message) {
+ if (callback == null) {
+ // Simply delete the message
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Ignoring incoming message from " + path + ": " + message);
+ }
+ listenFailure(zkClient.delete(path, version));
+ return;
+ }
+
+ executor.execute(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ // Message process is synchronous for now. Making it async needs more thoughts about race conditions.
+ // The executor is the callbackExecutor which is a single thread executor.
+ callback.onReceived(id, message).get();
+ } catch (Throwable t) {
+ LOG.error("Exception when processing message: {}, {}, {}", id, message, path, t);
+ } finally {
+ listenFailure(zkClient.delete(path, version));
+ }
+ }
+ });
+ }
+ }
+
+ private final class DecoratedServiceListener implements Listener {
+ private volatile boolean zkFailure = false;
+
+ @Override
+ public void starting() {
+ LOG.info("Starting: " + id);
+ saveState(ServiceController.State.STARTING);
+ }
+
+ @Override
+ public void running() {
+ LOG.info("Running: " + id);
+ notifyStarted();
+ watchMessages();
+ saveState(ServiceController.State.RUNNING);
+ }
+
+ @Override
+ public void stopping(State from) {
+ LOG.info("Stopping: " + id);
+ saveState(ServiceController.State.STOPPING);
+ }
+
+ @Override
+ public void terminated(State from) {
+ LOG.info("Terminated: " + from + " " + id);
+ if (zkFailure) {
+ return;
+ }
+
+ ImmutableList<OperationFuture<String>> futures = ImmutableList.of(removeLiveNode(), removeServiceNode());
+ final ListenableFuture<List<String>> future = Futures.allAsList(futures);
+ Futures.successfulAsList(futures).addListener(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ future.get();
+ LOG.info("Service and state node removed");
+ notifyStopped();
+ } catch (Exception e) {
+ LOG.warn("Failed to remove ZK nodes.", e);
+ notifyFailed(e);
+ }
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+ }
+
+ @Override
+ public void failed(State from, final Throwable failure) {
+ LOG.info("Failed: {} {}.", from, id, failure);
+ if (zkFailure) {
+ return;
+ }
+
+ ImmutableList<OperationFuture<String>> futures = ImmutableList.of(removeLiveNode(), removeServiceNode());
+ Futures.successfulAsList(futures).addListener(new Runnable() {
+ @Override
+ public void run() {
+ LOG.info("Service and state node removed");
+ notifyFailed(failure);
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+ }
+
+ private void saveState(ServiceController.State state) {
+ if (zkFailure) {
+ return;
+ }
+ StateNode stateNode = new StateNode(state);
+ stopOnFailure(zkClient.setData(getZKPath("state"), encodeStateNode(stateNode)));
+ }
+
+ private <V> void stopOnFailure(final OperationFuture<V> future) {
+ future.addListener(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ future.get();
+ } catch (final Exception e) {
+ LOG.error("ZK operation failed", e);
+ zkFailure = true;
+ decoratedService.stop().addListener(new Runnable() {
+ @Override
+ public void run() {
+ notifyFailed(e);
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+ }
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+ }
+ }
+
+ private <V> ListenableFuture<State> stopServiceOnComplete(ListenableFuture <V> future, final Service service) {
+ return Futures.transform(future, new AsyncFunction<V, State>() {
+ @Override
+ public ListenableFuture<State> apply(V input) throws Exception {
+ return service.stop();
+ }
+ });
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/json/ArgumentsCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/json/ArgumentsCodec.java b/core/src/main/java/org/apache/twill/internal/json/ArgumentsCodec.java
new file mode 100644
index 0000000..07d4c1d
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/json/ArgumentsCodec.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.json;
+
+import org.apache.twill.internal.Arguments;
+import com.google.common.collect.ImmutableMultimap;
+import com.google.common.io.InputSupplier;
+import com.google.common.io.OutputSupplier;
+import com.google.common.reflect.TypeToken;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonSerializationContext;
+import com.google.gson.JsonSerializer;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.io.Writer;
+import java.lang.reflect.Type;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+
+/**
+ *
+ */
+public final class ArgumentsCodec implements JsonSerializer<Arguments>, JsonDeserializer<Arguments> {
+
+ private static final Gson GSON = new GsonBuilder().registerTypeAdapter(Arguments.class, new ArgumentsCodec())
+ .create();
+
+ public static void encode(Arguments arguments, OutputSupplier<? extends Writer> writerSupplier) throws IOException {
+ Writer writer = writerSupplier.getOutput();
+ try {
+ GSON.toJson(arguments, writer);
+ } finally {
+ writer.close();
+ }
+ }
+
+
+ public static Arguments decode(InputSupplier<? extends Reader> readerSupplier) throws IOException {
+ Reader reader = readerSupplier.getInput();
+ try {
+ return GSON.fromJson(reader, Arguments.class);
+ } finally {
+ reader.close();
+ }
+ }
+
+ @Override
+ public JsonElement serialize(Arguments src, Type typeOfSrc,
+ JsonSerializationContext context) {
+ JsonObject json = new JsonObject();
+ json.add("arguments", context.serialize(src.getArguments()));
+ json.add("runnableArguments", context.serialize(src.getRunnableArguments().asMap()));
+
+ return json;
+ }
+
+ @Override
+ public Arguments deserialize(JsonElement json, Type typeOfT,
+ JsonDeserializationContext context) throws JsonParseException {
+ JsonObject jsonObj = json.getAsJsonObject();
+ List<String> arguments = context.deserialize(jsonObj.get("arguments"), new TypeToken<List<String>>() {}.getType());
+ Map<String, Collection<String>> args = context.deserialize(jsonObj.get("runnableArguments"),
+ new TypeToken<Map<String, Collection<String>>>(){
+ }.getType());
+
+ ImmutableMultimap.Builder<String, String> builder = ImmutableMultimap.builder();
+ for (Map.Entry<String, Collection<String>> entry : args.entrySet()) {
+ builder.putAll(entry.getKey(), entry.getValue());
+ }
+ return new Arguments(arguments, builder.build());
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/json/JsonUtils.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/json/JsonUtils.java b/core/src/main/java/org/apache/twill/internal/json/JsonUtils.java
new file mode 100644
index 0000000..9556ad8
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/json/JsonUtils.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.json;
+
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+
+/**
+ * Collections of helper functions for json codec.
+ */
+public final class JsonUtils {
+
+ private JsonUtils() {
+ }
+
+ /**
+ * Returns a String representation of the given property.
+ */
+ public static String getAsString(JsonObject json, String property) {
+ JsonElement jsonElement = json.get(property);
+ if (jsonElement.isJsonNull()) {
+ return null;
+ }
+ if (jsonElement.isJsonPrimitive()) {
+ return jsonElement.getAsString();
+ }
+ return jsonElement.toString();
+ }
+
+ /**
+ * Returns a long representation of the given property.
+ */
+ public static long getAsLong(JsonObject json, String property, long defaultValue) {
+ try {
+ return json.get(property).getAsLong();
+ } catch (Exception e) {
+ return defaultValue;
+ }
+ }
+
+ /**
+ * Returns a long representation of the given property.
+ */
+ public static int getAsInt(JsonObject json, String property, int defaultValue) {
+ try {
+ return json.get(property).getAsInt();
+ } catch (Exception e) {
+ return defaultValue;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/json/LocalFileCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/json/LocalFileCodec.java b/core/src/main/java/org/apache/twill/internal/json/LocalFileCodec.java
new file mode 100644
index 0000000..680a36c
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/json/LocalFileCodec.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.json;
+
+import org.apache.twill.api.LocalFile;
+import org.apache.twill.internal.DefaultLocalFile;
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonSerializationContext;
+import com.google.gson.JsonSerializer;
+
+import java.lang.reflect.Type;
+import java.net.URI;
+
+/**
+ *
+ */
+public final class LocalFileCodec implements JsonSerializer<LocalFile>, JsonDeserializer<LocalFile> {
+
+ @Override
+ public JsonElement serialize(LocalFile src, Type typeOfSrc, JsonSerializationContext context) {
+ JsonObject json = new JsonObject();
+
+ json.addProperty("name", src.getName());
+ json.addProperty("uri", src.getURI().toASCIIString());
+ json.addProperty("lastModified", src.getLastModified());
+ json.addProperty("size", src.getSize());
+ json.addProperty("archive", src.isArchive());
+ json.addProperty("pattern", src.getPattern());
+
+ return json;
+ }
+
+ @Override
+ public LocalFile deserialize(JsonElement json, Type typeOfT,
+ JsonDeserializationContext context) throws JsonParseException {
+ JsonObject jsonObj = json.getAsJsonObject();
+
+ String name = jsonObj.get("name").getAsString();
+ URI uri = URI.create(jsonObj.get("uri").getAsString());
+ long lastModified = jsonObj.get("lastModified").getAsLong();
+ long size = jsonObj.get("size").getAsLong();
+ boolean archive = jsonObj.get("archive").getAsBoolean();
+ JsonElement pattern = jsonObj.get("pattern");
+
+ return new DefaultLocalFile(name, uri, lastModified, size,
+ archive, (pattern == null || pattern.isJsonNull()) ? null : pattern.getAsString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/json/ResourceReportAdapter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/json/ResourceReportAdapter.java b/core/src/main/java/org/apache/twill/internal/json/ResourceReportAdapter.java
new file mode 100644
index 0000000..e473fe7
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/json/ResourceReportAdapter.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.json;
+
+import org.apache.twill.api.ResourceReport;
+import org.apache.twill.api.TwillRunResources;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+
+import java.io.Reader;
+import java.io.Writer;
+
+/**
+ * This class provides utility to help encode/decode {@link ResourceReport} to/from Json.
+ */
+public final class ResourceReportAdapter {
+
+ private final Gson gson;
+
+ public static ResourceReportAdapter create() {
+ return new ResourceReportAdapter();
+ }
+
+ private ResourceReportAdapter() {
+ gson = new GsonBuilder()
+ .serializeNulls()
+ .registerTypeAdapter(TwillRunResources.class, new TwillRunResourcesCodec())
+ .registerTypeAdapter(ResourceReport.class, new ResourceReportCodec())
+ .create();
+ }
+
+ public String toJson(ResourceReport report) {
+ return gson.toJson(report, ResourceReport.class);
+ }
+
+ public void toJson(ResourceReport report, Writer writer) {
+ gson.toJson(report, ResourceReport.class, writer);
+ }
+
+ public ResourceReport fromJson(String json) {
+ return gson.fromJson(json, ResourceReport.class);
+ }
+
+ public ResourceReport fromJson(Reader reader) {
+ return gson.fromJson(reader, ResourceReport.class);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/json/ResourceReportCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/json/ResourceReportCodec.java b/core/src/main/java/org/apache/twill/internal/json/ResourceReportCodec.java
new file mode 100644
index 0000000..884d889
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/json/ResourceReportCodec.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.json;
+
+import org.apache.twill.api.ResourceReport;
+import org.apache.twill.api.TwillRunResources;
+import org.apache.twill.internal.DefaultResourceReport;
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonSerializationContext;
+import com.google.gson.JsonSerializer;
+import com.google.gson.reflect.TypeToken;
+
+import java.lang.reflect.Type;
+import java.util.Collection;
+import java.util.Map;
+
+/**
+ * Codec for serializing and deserializing a {@link ResourceReport} object using json.
+ */
+public final class ResourceReportCodec implements JsonSerializer<ResourceReport>,
+ JsonDeserializer<ResourceReport> {
+
+ @Override
+ public JsonElement serialize(ResourceReport src, Type typeOfSrc, JsonSerializationContext context) {
+ JsonObject json = new JsonObject();
+
+ json.addProperty("appMasterId", src.getApplicationId());
+ json.add("appMasterResources", context.serialize(
+ src.getAppMasterResources(), new TypeToken<TwillRunResources>(){}.getType()));
+ json.add("runnableResources", context.serialize(
+ src.getResources(), new TypeToken<Map<String, Collection<TwillRunResources>>>(){}.getType()));
+
+ return json;
+ }
+
+ @Override
+ public ResourceReport deserialize(JsonElement json, Type typeOfT,
+ JsonDeserializationContext context) throws JsonParseException {
+ JsonObject jsonObj = json.getAsJsonObject();
+ String appMasterId = jsonObj.get("appMasterId").getAsString();
+ TwillRunResources masterResources = context.deserialize(
+ jsonObj.get("appMasterResources"), TwillRunResources.class);
+ Map<String, Collection<TwillRunResources>> resources = context.deserialize(
+ jsonObj.get("runnableResources"), new TypeToken<Map<String, Collection<TwillRunResources>>>(){}.getType());
+
+ return new DefaultResourceReport(appMasterId, masterResources, resources);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/json/ResourceSpecificationCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/json/ResourceSpecificationCodec.java b/core/src/main/java/org/apache/twill/internal/json/ResourceSpecificationCodec.java
new file mode 100644
index 0000000..bea73c4
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/json/ResourceSpecificationCodec.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.json;
+
+import org.apache.twill.api.ResourceSpecification;
+import org.apache.twill.internal.DefaultResourceSpecification;
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonSerializationContext;
+import com.google.gson.JsonSerializer;
+
+import java.lang.reflect.Type;
+
+/**
+ *
+ */
+final class ResourceSpecificationCodec implements JsonSerializer<ResourceSpecification>,
+ JsonDeserializer<ResourceSpecification> {
+
+ @Override
+ public JsonElement serialize(ResourceSpecification src, Type typeOfSrc, JsonSerializationContext context) {
+ JsonObject json = new JsonObject();
+
+ json.addProperty("cores", src.getVirtualCores());
+ json.addProperty("memorySize", src.getMemorySize());
+ json.addProperty("instances", src.getInstances());
+ json.addProperty("uplink", src.getUplink());
+ json.addProperty("downlink", src.getDownlink());
+
+ return json;
+ }
+
+ @Override
+ public ResourceSpecification deserialize(JsonElement json, Type typeOfT,
+ JsonDeserializationContext context) throws JsonParseException {
+ JsonObject jsonObj = json.getAsJsonObject();
+ return new DefaultResourceSpecification(jsonObj.get("cores").getAsInt(),
+ jsonObj.get("memorySize").getAsInt(),
+ jsonObj.get("instances").getAsInt(),
+ jsonObj.get("uplink").getAsInt(),
+ jsonObj.get("downlink").getAsInt());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/json/RuntimeSpecificationCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/json/RuntimeSpecificationCodec.java b/core/src/main/java/org/apache/twill/internal/json/RuntimeSpecificationCodec.java
new file mode 100644
index 0000000..867f4a8
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/json/RuntimeSpecificationCodec.java
@@ -0,0 +1,69 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.json;
+
+import org.apache.twill.api.LocalFile;
+import org.apache.twill.api.ResourceSpecification;
+import org.apache.twill.api.RuntimeSpecification;
+import org.apache.twill.api.TwillRunnableSpecification;
+import org.apache.twill.internal.DefaultRuntimeSpecification;
+import com.google.common.reflect.TypeToken;
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonSerializationContext;
+import com.google.gson.JsonSerializer;
+
+import java.lang.reflect.Type;
+import java.util.Collection;
+
+/**
+ *
+ */
+final class RuntimeSpecificationCodec implements JsonSerializer<RuntimeSpecification>,
+ JsonDeserializer<RuntimeSpecification> {
+
+ @Override
+ public JsonElement serialize(RuntimeSpecification src, Type typeOfSrc, JsonSerializationContext context) {
+ JsonObject json = new JsonObject();
+ json.addProperty("name", src.getName());
+ json.add("runnable", context.serialize(src.getRunnableSpecification(), TwillRunnableSpecification.class));
+ json.add("resources", context.serialize(src.getResourceSpecification(), ResourceSpecification.class));
+ json.add("files", context.serialize(src.getLocalFiles(), new TypeToken<Collection<LocalFile>>(){}.getType()));
+
+ return json;
+ }
+
+ @Override
+ public RuntimeSpecification deserialize(JsonElement json, Type typeOfT,
+ JsonDeserializationContext context) throws JsonParseException {
+ JsonObject jsonObj = json.getAsJsonObject();
+
+ String name = jsonObj.get("name").getAsString();
+ TwillRunnableSpecification runnable = context.deserialize(jsonObj.get("runnable"),
+ TwillRunnableSpecification.class);
+ ResourceSpecification resources = context.deserialize(jsonObj.get("resources"),
+ ResourceSpecification.class);
+ Collection<LocalFile> files = context.deserialize(jsonObj.get("files"),
+ new TypeToken<Collection<LocalFile>>(){}.getType());
+
+ return new DefaultRuntimeSpecification(name, runnable, resources, files);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/json/StackTraceElementCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/json/StackTraceElementCodec.java b/core/src/main/java/org/apache/twill/internal/json/StackTraceElementCodec.java
new file mode 100644
index 0000000..9a57b46
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/json/StackTraceElementCodec.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.json;
+
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonSerializationContext;
+import com.google.gson.JsonSerializer;
+
+import java.lang.reflect.Type;
+
+/**
+ *
+ */
+public final class StackTraceElementCodec implements JsonSerializer<StackTraceElement>,
+ JsonDeserializer<StackTraceElement> {
+
+ @Override
+ public StackTraceElement deserialize(JsonElement json, Type typeOfT,
+ JsonDeserializationContext context) throws JsonParseException {
+ JsonObject jsonObj = json.getAsJsonObject();
+ return new StackTraceElement(JsonUtils.getAsString(jsonObj, "className"),
+ JsonUtils.getAsString(jsonObj, "method"),
+ JsonUtils.getAsString(jsonObj, "file"),
+ JsonUtils.getAsInt(jsonObj, "line", -1));
+ }
+
+ @Override
+ public JsonElement serialize(StackTraceElement src, Type typeOfSrc, JsonSerializationContext context) {
+ JsonObject jsonObj = new JsonObject();
+ jsonObj.addProperty("className", src.getClassName());
+ jsonObj.addProperty("method", src.getMethodName());
+ jsonObj.addProperty("file", src.getFileName());
+ jsonObj.addProperty("line", src.getLineNumber());
+
+ return jsonObj;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/json/StateNodeCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/json/StateNodeCodec.java b/core/src/main/java/org/apache/twill/internal/json/StateNodeCodec.java
new file mode 100644
index 0000000..c1e9d1c
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/json/StateNodeCodec.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.json;
+
+import org.apache.twill.api.ServiceController;
+import org.apache.twill.internal.state.StateNode;
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonSerializationContext;
+import com.google.gson.JsonSerializer;
+
+import java.lang.reflect.Type;
+
+/**
+ *
+ */
+public final class StateNodeCodec implements JsonSerializer<StateNode>, JsonDeserializer<StateNode> {
+
+ @Override
+ public StateNode deserialize(JsonElement json, Type typeOfT,
+ JsonDeserializationContext context) throws JsonParseException {
+ JsonObject jsonObj = json.getAsJsonObject();
+ ServiceController.State state = ServiceController.State.valueOf(jsonObj.get("state").getAsString());
+ String errorMessage = jsonObj.has("errorMessage") ? jsonObj.get("errorMessage").getAsString() : null;
+
+ return new StateNode(state, errorMessage,
+ context.<StackTraceElement[]>deserialize(jsonObj.get("stackTraces"), StackTraceElement[].class));
+ }
+
+ @Override
+ public JsonElement serialize(StateNode src, Type typeOfSrc, JsonSerializationContext context) {
+ JsonObject jsonObj = new JsonObject();
+ jsonObj.addProperty("state", src.getState().name());
+ if (src.getErrorMessage() != null) {
+ jsonObj.addProperty("errorMessage", src.getErrorMessage());
+ }
+ if (src.getStackTraces() != null) {
+ jsonObj.add("stackTraces", context.serialize(src.getStackTraces(), StackTraceElement[].class));
+ }
+ return jsonObj;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/json/TwillRunResourcesCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/json/TwillRunResourcesCodec.java b/core/src/main/java/org/apache/twill/internal/json/TwillRunResourcesCodec.java
new file mode 100644
index 0000000..8951173
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/json/TwillRunResourcesCodec.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.json;
+
+import org.apache.twill.api.TwillRunResources;
+import org.apache.twill.internal.DefaultTwillRunResources;
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonSerializationContext;
+import com.google.gson.JsonSerializer;
+
+import java.lang.reflect.Type;
+
+/**
+ * Codec for serializing and deserializing a {@link org.apache.twill.api.TwillRunResources} object using json.
+ */
+public final class TwillRunResourcesCodec implements JsonSerializer<TwillRunResources>,
+ JsonDeserializer<TwillRunResources> {
+
+ @Override
+ public JsonElement serialize(TwillRunResources src, Type typeOfSrc, JsonSerializationContext context) {
+ JsonObject json = new JsonObject();
+
+ json.addProperty("containerId", src.getContainerId());
+ json.addProperty("instanceId", src.getInstanceId());
+ json.addProperty("host", src.getHost());
+ json.addProperty("memoryMB", src.getMemoryMB());
+ json.addProperty("virtualCores", src.getVirtualCores());
+
+ return json;
+ }
+
+ @Override
+ public TwillRunResources deserialize(JsonElement json, Type typeOfT,
+ JsonDeserializationContext context) throws JsonParseException {
+ JsonObject jsonObj = json.getAsJsonObject();
+ return new DefaultTwillRunResources(jsonObj.get("instanceId").getAsInt(),
+ jsonObj.get("containerId").getAsString(),
+ jsonObj.get("virtualCores").getAsInt(),
+ jsonObj.get("memoryMB").getAsInt(),
+ jsonObj.get("host").getAsString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/json/TwillRunnableSpecificationCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/json/TwillRunnableSpecificationCodec.java b/core/src/main/java/org/apache/twill/internal/json/TwillRunnableSpecificationCodec.java
new file mode 100644
index 0000000..f37c1e8
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/json/TwillRunnableSpecificationCodec.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.json;
+
+import org.apache.twill.api.TwillRunnableSpecification;
+import org.apache.twill.internal.DefaultTwillRunnableSpecification;
+import com.google.common.reflect.TypeToken;
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonSerializationContext;
+import com.google.gson.JsonSerializer;
+
+import java.lang.reflect.Type;
+import java.util.Map;
+
+/**
+ *
+ */
+final class TwillRunnableSpecificationCodec implements JsonSerializer<TwillRunnableSpecification>,
+ JsonDeserializer<TwillRunnableSpecification> {
+
+ @Override
+ public JsonElement serialize(TwillRunnableSpecification src, Type typeOfSrc, JsonSerializationContext context) {
+ JsonObject json = new JsonObject();
+
+ json.addProperty("classname", src.getClassName());
+ json.addProperty("name", src.getName());
+ json.add("arguments", context.serialize(src.getConfigs(), new TypeToken<Map<String, String>>(){}.getType()));
+
+ return json;
+ }
+
+ @Override
+ public TwillRunnableSpecification deserialize(JsonElement json, Type typeOfT,
+ JsonDeserializationContext context) throws JsonParseException {
+ JsonObject jsonObj = json.getAsJsonObject();
+
+ String className = jsonObj.get("classname").getAsString();
+ String name = jsonObj.get("name").getAsString();
+ Map<String, String> arguments = context.deserialize(jsonObj.get("arguments"),
+ new TypeToken<Map<String, String>>(){}.getType());
+
+ return new DefaultTwillRunnableSpecification(className, name, arguments);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/json/TwillSpecificationAdapter.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/json/TwillSpecificationAdapter.java b/core/src/main/java/org/apache/twill/internal/json/TwillSpecificationAdapter.java
new file mode 100644
index 0000000..67c15a2
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/json/TwillSpecificationAdapter.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.json;
+
+import com.google.common.base.Charsets;
+import com.google.common.collect.Maps;
+import com.google.common.io.Files;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.TypeAdapter;
+import com.google.gson.TypeAdapterFactory;
+import com.google.gson.reflect.TypeToken;
+import com.google.gson.stream.JsonReader;
+import com.google.gson.stream.JsonToken;
+import com.google.gson.stream.JsonWriter;
+import org.apache.twill.api.EventHandlerSpecification;
+import org.apache.twill.api.LocalFile;
+import org.apache.twill.api.ResourceSpecification;
+import org.apache.twill.api.RuntimeSpecification;
+import org.apache.twill.api.TwillRunnableSpecification;
+import org.apache.twill.api.TwillSpecification;
+import org.apache.twill.internal.json.TwillSpecificationCodec.EventHandlerSpecificationCoder;
+import org.apache.twill.internal.json.TwillSpecificationCodec.TwillSpecificationOrderCoder;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.Reader;
+import java.io.Writer;
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Type;
+import java.util.Map;
+
+/**
+ *
+ */
+public final class TwillSpecificationAdapter {
+
+ private final Gson gson;
+
+ public static TwillSpecificationAdapter create() {
+ return new TwillSpecificationAdapter();
+ }
+
+ private TwillSpecificationAdapter() {
+ gson = new GsonBuilder()
+ .serializeNulls()
+ .registerTypeAdapter(TwillSpecification.class, new TwillSpecificationCodec())
+ .registerTypeAdapter(TwillSpecification.Order.class, new TwillSpecificationOrderCoder())
+ .registerTypeAdapter(EventHandlerSpecification.class, new EventHandlerSpecificationCoder())
+ .registerTypeAdapter(RuntimeSpecification.class, new RuntimeSpecificationCodec())
+ .registerTypeAdapter(TwillRunnableSpecification.class, new TwillRunnableSpecificationCodec())
+ .registerTypeAdapter(ResourceSpecification.class, new ResourceSpecificationCodec())
+ .registerTypeAdapter(LocalFile.class, new LocalFileCodec())
+ .registerTypeAdapterFactory(new TwillSpecificationTypeAdapterFactory())
+ .create();
+ }
+
+ public String toJson(TwillSpecification spec) {
+ return gson.toJson(spec, TwillSpecification.class);
+ }
+
+ public void toJson(TwillSpecification spec, Writer writer) {
+ gson.toJson(spec, TwillSpecification.class, writer);
+ }
+
+ public void toJson(TwillSpecification spec, File file) throws IOException {
+ Writer writer = Files.newWriter(file, Charsets.UTF_8);
+ try {
+ toJson(spec, writer);
+ } finally {
+ writer.close();
+ }
+ }
+
+ public TwillSpecification fromJson(String json) {
+ return gson.fromJson(json, TwillSpecification.class);
+ }
+
+ public TwillSpecification fromJson(Reader reader) {
+ return gson.fromJson(reader, TwillSpecification.class);
+ }
+
+ public TwillSpecification fromJson(File file) throws IOException {
+ Reader reader = Files.newReader(file, Charsets.UTF_8);
+ try {
+ return fromJson(reader);
+ } finally {
+ reader.close();
+ }
+ }
+
+ // This is to get around gson ignoring of inner class
+ private static final class TwillSpecificationTypeAdapterFactory implements TypeAdapterFactory {
+
+ @Override
+ public <T> TypeAdapter<T> create(Gson gson, TypeToken<T> type) {
+ Class<?> rawType = type.getRawType();
+ if (!Map.class.isAssignableFrom(rawType)) {
+ return null;
+ }
+ Type[] typeArgs = ((ParameterizedType) type.getType()).getActualTypeArguments();
+ TypeToken<?> keyType = TypeToken.get(typeArgs[0]);
+ TypeToken<?> valueType = TypeToken.get(typeArgs[1]);
+ if (keyType.getRawType() != String.class) {
+ return null;
+ }
+ return (TypeAdapter<T>) mapAdapter(gson, valueType);
+ }
+
+ private <V> TypeAdapter<Map<String, V>> mapAdapter(Gson gson, TypeToken<V> valueType) {
+ final TypeAdapter<V> valueAdapter = gson.getAdapter(valueType);
+
+ return new TypeAdapter<Map<String, V>>() {
+ @Override
+ public void write(JsonWriter writer, Map<String, V> map) throws IOException {
+ if (map == null) {
+ writer.nullValue();
+ return;
+ }
+ writer.beginObject();
+ for (Map.Entry<String, V> entry : map.entrySet()) {
+ writer.name(entry.getKey());
+ valueAdapter.write(writer, entry.getValue());
+ }
+ writer.endObject();
+ }
+
+ @Override
+ public Map<String, V> read(JsonReader reader) throws IOException {
+ if (reader.peek() == JsonToken.NULL) {
+ reader.nextNull();
+ return null;
+ }
+ if (reader.peek() != JsonToken.BEGIN_OBJECT) {
+ return null;
+ }
+ Map<String, V> map = Maps.newHashMap();
+ reader.beginObject();
+ while (reader.peek() != JsonToken.END_OBJECT) {
+ map.put(reader.nextName(), valueAdapter.read(reader));
+ }
+ reader.endObject();
+ return map;
+ }
+ };
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/json/TwillSpecificationCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/json/TwillSpecificationCodec.java b/core/src/main/java/org/apache/twill/internal/json/TwillSpecificationCodec.java
new file mode 100644
index 0000000..5d88350
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/json/TwillSpecificationCodec.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.json;
+
+import org.apache.twill.api.EventHandlerSpecification;
+import org.apache.twill.api.RuntimeSpecification;
+import org.apache.twill.api.TwillSpecification;
+import org.apache.twill.internal.DefaultEventHandlerSpecification;
+import org.apache.twill.internal.DefaultTwillSpecification;
+import com.google.common.reflect.TypeToken;
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonSerializationContext;
+import com.google.gson.JsonSerializer;
+
+import java.lang.reflect.Type;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * An implementation of gson serializer/deserializer {@link org.apache.twill.api.TwillSpecification}.
+ */
+final class TwillSpecificationCodec implements JsonSerializer<TwillSpecification>,
+ JsonDeserializer<TwillSpecification> {
+
+ @Override
+ public JsonElement serialize(TwillSpecification src, Type typeOfSrc, JsonSerializationContext context) {
+ JsonObject json = new JsonObject();
+ json.addProperty("name", src.getName());
+ json.add("runnables", context.serialize(src.getRunnables(),
+ new TypeToken<Map<String, RuntimeSpecification>>(){}.getType()));
+ json.add("orders", context.serialize(src.getOrders(),
+ new TypeToken<List<TwillSpecification.Order>>(){}.getType()));
+ EventHandlerSpecification eventHandler = src.getEventHandler();
+ if (eventHandler != null) {
+ json.add("handler", context.serialize(eventHandler, EventHandlerSpecification.class));
+ }
+
+ return json;
+ }
+
+ @Override
+ public TwillSpecification deserialize(JsonElement json, Type typeOfT,
+ JsonDeserializationContext context) throws JsonParseException {
+ JsonObject jsonObj = json.getAsJsonObject();
+
+ String name = jsonObj.get("name").getAsString();
+ Map<String, RuntimeSpecification> runnables = context.deserialize(
+ jsonObj.get("runnables"), new TypeToken<Map<String, RuntimeSpecification>>(){}.getType());
+ List<TwillSpecification.Order> orders = context.deserialize(
+ jsonObj.get("orders"), new TypeToken<List<TwillSpecification.Order>>(){}.getType());
+
+ JsonElement handler = jsonObj.get("handler");
+ EventHandlerSpecification eventHandler = null;
+ if (handler != null && !handler.isJsonNull()) {
+ eventHandler = context.deserialize(handler, EventHandlerSpecification.class);
+ }
+
+ return new DefaultTwillSpecification(name, runnables, orders, eventHandler);
+ }
+
+ static final class TwillSpecificationOrderCoder implements JsonSerializer<TwillSpecification.Order>,
+ JsonDeserializer<TwillSpecification.Order> {
+
+ @Override
+ public JsonElement serialize(TwillSpecification.Order src, Type typeOfSrc, JsonSerializationContext context) {
+ JsonObject json = new JsonObject();
+ json.add("names", context.serialize(src.getNames(), new TypeToken<Set<String>>(){}.getType()));
+ json.addProperty("type", src.getType().name());
+ return json;
+ }
+
+ @Override
+ public TwillSpecification.Order deserialize(JsonElement json, Type typeOfT,
+ JsonDeserializationContext context) throws JsonParseException {
+ JsonObject jsonObj = json.getAsJsonObject();
+
+ Set<String> names = context.deserialize(jsonObj.get("names"), new TypeToken<Set<String>>(){}.getType());
+ TwillSpecification.Order.Type type = TwillSpecification.Order.Type.valueOf(jsonObj.get("type").getAsString());
+
+ return new DefaultTwillSpecification.DefaultOrder(names, type);
+ }
+ }
+
+ static final class EventHandlerSpecificationCoder implements JsonSerializer<EventHandlerSpecification>,
+ JsonDeserializer<EventHandlerSpecification> {
+
+ @Override
+ public JsonElement serialize(EventHandlerSpecification src, Type typeOfSrc, JsonSerializationContext context) {
+ JsonObject json = new JsonObject();
+ json.addProperty("classname", src.getClassName());
+ json.add("configs", context.serialize(src.getConfigs(), new TypeToken<Map<String, String>>(){}.getType()));
+ return json;
+ }
+
+ @Override
+ public EventHandlerSpecification deserialize(JsonElement json, Type typeOfT,
+ JsonDeserializationContext context) throws JsonParseException {
+ JsonObject jsonObj = json.getAsJsonObject();
+ String className = jsonObj.get("classname").getAsString();
+ Map<String, String> configs = context.deserialize(jsonObj.get("configs"),
+ new TypeToken<Map<String, String>>() {
+ }.getType());
+
+ return new DefaultEventHandlerSpecification(className, configs);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/kafka/EmbeddedKafkaServer.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/kafka/EmbeddedKafkaServer.java b/core/src/main/java/org/apache/twill/internal/kafka/EmbeddedKafkaServer.java
new file mode 100644
index 0000000..14dfc70
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/kafka/EmbeddedKafkaServer.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.kafka;
+
+import com.google.common.base.Throwables;
+import com.google.common.collect.Lists;
+import com.google.common.util.concurrent.AbstractIdleService;
+
+import java.io.File;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.util.List;
+import java.util.Properties;
+
+/**
+ *
+ */
+public final class EmbeddedKafkaServer extends AbstractIdleService {
+
+ private static final String KAFAK_CONFIG_CLASS = "kafka.server.KafkaConfig";
+ private static final String KAFKA_SERVER_CLASS = "kafka.server.KafkaServerStartable";
+
+ private final Object server;
+
+ public EmbeddedKafkaServer(File kafkaDir, Properties properties) {
+ this(createClassLoader(kafkaDir), properties);
+ }
+
+ public EmbeddedKafkaServer(ClassLoader classLoader, Properties properties) {
+ try {
+ Class<?> configClass = classLoader.loadClass(KAFAK_CONFIG_CLASS);
+ Object config = configClass.getConstructor(Properties.class).newInstance(properties);
+
+ Class<?> serverClass = classLoader.loadClass(KAFKA_SERVER_CLASS);
+ server = serverClass.getConstructor(configClass).newInstance(config);
+ } catch (Exception e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ protected void startUp() throws Exception {
+ server.getClass().getMethod("startup").invoke(server);
+ }
+
+ @Override
+ protected void shutDown() throws Exception {
+ server.getClass().getMethod("shutdown").invoke(server);
+ server.getClass().getMethod("awaitShutdown").invoke(server);
+ }
+
+ private static ClassLoader createClassLoader(File kafkaDir) {
+ ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader();
+ ClassLoader thisClassLoader = EmbeddedKafkaServer.class.getClassLoader();
+ ClassLoader parent = contextClassLoader != null
+ ? contextClassLoader
+ : thisClassLoader != null
+ ? thisClassLoader : ClassLoader.getSystemClassLoader();
+
+ return new URLClassLoader(findJars(kafkaDir, Lists.<URL>newArrayList()).toArray(new URL[0]), parent);
+ }
+
+ private static List<URL> findJars(File dir, List<URL> urls) {
+ try {
+ for (File file : dir.listFiles()) {
+ if (file.isDirectory()) {
+ findJars(file, urls);
+ } else if (file.getName().endsWith(".jar")) {
+ urls.add(file.toURI().toURL());
+ }
+ }
+ return urls;
+ } catch (MalformedURLException e) {
+ throw Throwables.propagate(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/kafka/client/AbstractCompressedMessageSetEncoder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/kafka/client/AbstractCompressedMessageSetEncoder.java b/core/src/main/java/org/apache/twill/internal/kafka/client/AbstractCompressedMessageSetEncoder.java
new file mode 100644
index 0000000..a9c3381
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/kafka/client/AbstractCompressedMessageSetEncoder.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.kafka.client;
+
+import com.google.common.base.Throwables;
+import org.jboss.netty.buffer.ChannelBuffer;
+import org.jboss.netty.buffer.ChannelBufferOutputStream;
+import org.jboss.netty.buffer.ChannelBuffers;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+/**
+ * A base implementation of {@link MessageSetEncoder} that do message compression.
+ */
+abstract class AbstractCompressedMessageSetEncoder extends AbstractMessageSetEncoder {
+
+ private final Compression compression;
+ private ChannelBufferOutputStream os;
+ private OutputStream compressedOutput;
+
+
+ protected AbstractCompressedMessageSetEncoder(Compression compression) {
+ this.compression = compression;
+ try {
+ this.os = new ChannelBufferOutputStream(ChannelBuffers.dynamicBuffer());
+ this.compressedOutput = createCompressedStream(os);
+ } catch (IOException e) {
+ // Should never happen
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ public final MessageSetEncoder add(ChannelBuffer payload) {
+ try {
+ ChannelBuffer encoded = encodePayload(payload);
+ encoded.readBytes(compressedOutput, encoded.readableBytes());
+ } catch (IOException e) {
+ throw Throwables.propagate(e);
+ }
+ return this;
+
+ }
+
+ @Override
+ public final ChannelBuffer finish() {
+ try {
+ compressedOutput.close();
+ ChannelBuffer buf = prefixLength(encodePayload(os.buffer(), compression));
+ compressedOutput = createCompressedStream(os);
+ os.buffer().clear();
+
+ return buf;
+
+ } catch (IOException e) {
+ throw Throwables.propagate(e);
+ }
+
+ }
+
+ protected abstract OutputStream createCompressedStream(OutputStream os) throws IOException;
+}
[10/15] Initial import commit.
Posted by ch...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/logging/KafkaTwillRunnable.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/logging/KafkaTwillRunnable.java b/core/src/main/java/org/apache/twill/internal/logging/KafkaTwillRunnable.java
new file mode 100644
index 0000000..c1695de
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/logging/KafkaTwillRunnable.java
@@ -0,0 +1,122 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.logging;
+
+import org.apache.twill.api.Command;
+import org.apache.twill.api.TwillContext;
+import org.apache.twill.api.TwillRunnable;
+import org.apache.twill.api.TwillRunnableSpecification;
+import org.apache.twill.internal.EnvKeys;
+import org.apache.twill.internal.kafka.EmbeddedKafkaServer;
+import org.apache.twill.internal.utils.Networks;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.util.Map;
+import java.util.Properties;
+import java.util.concurrent.CountDownLatch;
+
+/**
+ * A {@link org.apache.twill.api.TwillRunnable} for managing Kafka server.
+ */
+public final class KafkaTwillRunnable implements TwillRunnable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(KafkaTwillRunnable.class);
+
+ private final String kafkaDir;
+ private EmbeddedKafkaServer server;
+ private CountDownLatch stopLatch;
+
+ public KafkaTwillRunnable(String kafkaDir) {
+ this.kafkaDir = kafkaDir;
+ }
+
+ @Override
+ public TwillRunnableSpecification configure() {
+ return TwillRunnableSpecification.Builder.with()
+ .setName("kafka")
+ .withConfigs(ImmutableMap.of("kafkaDir", kafkaDir))
+ .build();
+ }
+
+ @Override
+ public void initialize(TwillContext context) {
+ Map<String, String> args = context.getSpecification().getConfigs();
+ String zkConnectStr = System.getenv(EnvKeys.TWILL_LOG_KAFKA_ZK);
+ stopLatch = new CountDownLatch(1);
+
+ try {
+ server = new EmbeddedKafkaServer(new File(args.get("kafkaDir")), generateKafkaConfig(zkConnectStr));
+ server.startAndWait();
+ } catch (Exception e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ public void handleCommand(Command command) throws Exception {
+ }
+
+ @Override
+ public void stop() {
+ stopLatch.countDown();
+ }
+
+ @Override
+ public void destroy() {
+ server.stopAndWait();
+ }
+
+ @Override
+ public void run() {
+ try {
+ stopLatch.await();
+ } catch (InterruptedException e) {
+ LOG.info("Running thread interrupted, shutting down kafka server.", e);
+ }
+ }
+
+ private Properties generateKafkaConfig(String zkConnectStr) {
+ int port = Networks.getRandomPort();
+ Preconditions.checkState(port > 0, "Failed to get random port.");
+
+ Properties prop = new Properties();
+ prop.setProperty("log.dir", new File("kafka-logs").getAbsolutePath());
+ prop.setProperty("zk.connect", zkConnectStr);
+ prop.setProperty("num.threads", "8");
+ prop.setProperty("port", Integer.toString(port));
+ prop.setProperty("log.flush.interval", "10000");
+ prop.setProperty("max.socket.request.bytes", "104857600");
+ prop.setProperty("log.cleanup.interval.mins", "1");
+ prop.setProperty("log.default.flush.scheduler.interval.ms", "1000");
+ prop.setProperty("zk.connectiontimeout.ms", "1000000");
+ prop.setProperty("socket.receive.buffer", "1048576");
+ prop.setProperty("enable.zookeeper", "true");
+ prop.setProperty("log.retention.hours", "168");
+ prop.setProperty("brokerid", "0");
+ prop.setProperty("socket.send.buffer", "1048576");
+ prop.setProperty("num.partitions", "1");
+ prop.setProperty("log.file.size", "536870912");
+ prop.setProperty("log.default.flush.interval.ms", "1000");
+ return prop;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/logging/LogEntryDecoder.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/logging/LogEntryDecoder.java b/core/src/main/java/org/apache/twill/internal/logging/LogEntryDecoder.java
new file mode 100644
index 0000000..dc11666
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/logging/LogEntryDecoder.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.logging;
+
+import org.apache.twill.api.logging.LogEntry;
+import org.apache.twill.internal.json.JsonUtils;
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+
+import java.lang.reflect.Type;
+
+/**
+ * A {@link com.google.gson.Gson} decoder for {@link LogEntry}.
+ */
+public final class LogEntryDecoder implements JsonDeserializer<LogEntry> {
+
+ @Override
+ public LogEntry deserialize(JsonElement json, Type typeOfT,
+ JsonDeserializationContext context) throws JsonParseException {
+ if (!json.isJsonObject()) {
+ return null;
+ }
+ JsonObject jsonObj = json.getAsJsonObject();
+
+ final String name = JsonUtils.getAsString(jsonObj, "name");
+ final String host = JsonUtils.getAsString(jsonObj, "host");
+ final long timestamp = JsonUtils.getAsLong(jsonObj, "timestamp", 0);
+ LogEntry.Level l;
+ try {
+ l = LogEntry.Level.valueOf(JsonUtils.getAsString(jsonObj, "level"));
+ } catch (Exception e) {
+ l = LogEntry.Level.FATAL;
+ }
+ final LogEntry.Level logLevel = l;
+ final String className = JsonUtils.getAsString(jsonObj, "className");
+ final String method = JsonUtils.getAsString(jsonObj, "method");
+ final String file = JsonUtils.getAsString(jsonObj, "file");
+ final String line = JsonUtils.getAsString(jsonObj, "line");
+ final String thread = JsonUtils.getAsString(jsonObj, "thread");
+ final String message = JsonUtils.getAsString(jsonObj, "message");
+
+ final StackTraceElement[] stackTraces = context.deserialize(jsonObj.get("stackTraces").getAsJsonArray(),
+ StackTraceElement[].class);
+
+ return new LogEntry() {
+ @Override
+ public String getLoggerName() {
+ return name;
+ }
+
+ @Override
+ public String getHost() {
+ return host;
+ }
+
+ @Override
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ @Override
+ public Level getLogLevel() {
+ return logLevel;
+ }
+
+ @Override
+ public String getSourceClassName() {
+ return className;
+ }
+
+ @Override
+ public String getSourceMethodName() {
+ return method;
+ }
+
+ @Override
+ public String getFileName() {
+ return file;
+ }
+
+ @Override
+ public int getLineNumber() {
+ if (line.equals("?")) {
+ return -1;
+ } else {
+ return Integer.parseInt(line);
+ }
+ }
+
+ @Override
+ public String getThreadName() {
+ return thread;
+ }
+
+ @Override
+ public String getMessage() {
+ return message;
+ }
+
+ @Override
+ public StackTraceElement[] getStackTraces() {
+ return stackTraces;
+ }
+ };
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/logging/Loggings.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/logging/Loggings.java b/core/src/main/java/org/apache/twill/internal/logging/Loggings.java
new file mode 100644
index 0000000..9baed63
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/logging/Loggings.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.logging;
+
+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.LoggerContext;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.Appender;
+import org.slf4j.ILoggerFactory;
+import org.slf4j.LoggerFactory;
+
+/**
+ *
+ */
+public final class Loggings {
+
+ public static void forceFlush() {
+ ILoggerFactory loggerFactory = LoggerFactory.getILoggerFactory();
+
+ if (loggerFactory instanceof LoggerContext) {
+ Appender<ILoggingEvent> appender = ((LoggerContext) loggerFactory).getLogger(Logger.ROOT_LOGGER_NAME)
+ .getAppender("KAFKA");
+ if (appender != null && appender instanceof KafkaAppender) {
+ ((KafkaAppender) appender).forceFlush();
+ }
+ }
+ }
+
+ private Loggings() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/package-info.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/package-info.java b/core/src/main/java/org/apache/twill/internal/package-info.java
new file mode 100644
index 0000000..a8459e0
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This package provides internal classes for Twill.
+ */
+package org.apache.twill.internal;
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/state/Message.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/state/Message.java b/core/src/main/java/org/apache/twill/internal/state/Message.java
new file mode 100644
index 0000000..6c3e719
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/state/Message.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.state;
+
+import org.apache.twill.api.Command;
+
+/**
+ *
+ */
+public interface Message {
+
+ /**
+ * Type of message.
+ */
+ enum Type {
+ SYSTEM,
+ USER
+ }
+
+ /**
+ * Scope of the message.
+ */
+ enum Scope {
+ APPLICATION,
+ ALL_RUNNABLE,
+ RUNNABLE
+ }
+
+ Type getType();
+
+ Scope getScope();
+
+ /**
+ * @return the name of the target runnable if scope is {@link Scope#RUNNABLE} or {@code null} otherwise.
+ */
+ String getRunnableName();
+
+ Command getCommand();
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/state/MessageCallback.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/state/MessageCallback.java b/core/src/main/java/org/apache/twill/internal/state/MessageCallback.java
new file mode 100644
index 0000000..f94eaa3
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/state/MessageCallback.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.state;
+
+import com.google.common.util.concurrent.ListenableFuture;
+
+/**
+ *
+ */
+public interface MessageCallback {
+
+ /**
+ * Called when a message is received.
+ * @param message Message being received.
+ * @return A {@link ListenableFuture} that would be completed when message processing is completed or failed.
+ * The result of the future should be the input message Id if succeeded.
+ */
+ ListenableFuture<String> onReceived(String messageId, Message message);
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/state/MessageCodec.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/state/MessageCodec.java b/core/src/main/java/org/apache/twill/internal/state/MessageCodec.java
new file mode 100644
index 0000000..176f620
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/state/MessageCodec.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.state;
+
+import org.apache.twill.api.Command;
+import com.google.common.base.Charsets;
+import com.google.common.reflect.TypeToken;
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.JsonDeserializationContext;
+import com.google.gson.JsonDeserializer;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import com.google.gson.JsonParseException;
+import com.google.gson.JsonSerializationContext;
+import com.google.gson.JsonSerializer;
+
+import java.lang.reflect.Type;
+import java.util.Map;
+
+/**
+ *
+ */
+public final class MessageCodec {
+
+ private static final Type OPTIONS_TYPE = new TypeToken<Map<String, String>>() {}.getType();
+ private static final Gson GSON = new GsonBuilder()
+ .registerTypeAdapter(Message.class, new MessageAdapter())
+ .registerTypeAdapter(Command.class, new CommandAdapter())
+ .create();
+
+ /**
+ * Decodes a {@link Message} from the given byte array.
+ * @param bytes byte array to be decoded
+ * @return Message decoded or {@code null} if fails to decode.
+ */
+ public static Message decode(byte[] bytes) {
+ if (bytes == null) {
+ return null;
+ }
+ String content = new String(bytes, Charsets.UTF_8);
+ return GSON.fromJson(content, Message.class);
+ }
+
+ /**
+ * Encodes a {@link Message} into byte array. Revserse of {@link #decode(byte[])} method.
+ * @param message Message to be encoded
+ * @return byte array representing the encoded message.
+ */
+ public static byte[] encode(Message message) {
+ return GSON.toJson(message, Message.class).getBytes(Charsets.UTF_8);
+ }
+
+ /**
+ * Gson codec for {@link Message} object.
+ */
+ private static final class MessageAdapter implements JsonSerializer<Message>, JsonDeserializer<Message> {
+
+ @Override
+ public Message deserialize(JsonElement json, Type typeOfT,
+ JsonDeserializationContext context) throws JsonParseException {
+ JsonObject jsonObj = json.getAsJsonObject();
+
+ Message.Type type = Message.Type.valueOf(jsonObj.get("type").getAsString());
+ Message.Scope scope = Message.Scope.valueOf(jsonObj.get("scope").getAsString());
+ JsonElement name = jsonObj.get("runnableName");
+ String runnableName = (name == null || name.isJsonNull()) ? null : name.getAsString();
+ Command command = context.deserialize(jsonObj.get("command"), Command.class);
+
+ return new SimpleMessage(type, scope, runnableName, command);
+ }
+
+ @Override
+ public JsonElement serialize(Message message, Type typeOfSrc, JsonSerializationContext context) {
+ JsonObject jsonObj = new JsonObject();
+ jsonObj.addProperty("type", message.getType().name());
+ jsonObj.addProperty("scope", message.getScope().name());
+ jsonObj.addProperty("runnableName", message.getRunnableName());
+ jsonObj.add("command", context.serialize(message.getCommand(), Command.class));
+
+ return jsonObj;
+ }
+ }
+
+ /**
+ * Gson codec for {@link Command} object.
+ */
+ private static final class CommandAdapter implements JsonSerializer<Command>, JsonDeserializer<Command> {
+
+ @Override
+ public Command deserialize(JsonElement json, Type typeOfT,
+ JsonDeserializationContext context) throws JsonParseException {
+ JsonObject jsonObj = json.getAsJsonObject();
+ return Command.Builder.of(jsonObj.get("command").getAsString())
+ .addOptions(context.<Map<String, String>>deserialize(jsonObj.get("options"), OPTIONS_TYPE))
+ .build();
+ }
+
+ @Override
+ public JsonElement serialize(Command command, Type typeOfSrc, JsonSerializationContext context) {
+ JsonObject jsonObj = new JsonObject();
+ jsonObj.addProperty("command", command.getCommand());
+ jsonObj.add("options", context.serialize(command.getOptions(), OPTIONS_TYPE));
+ return jsonObj;
+ }
+ }
+
+ private MessageCodec() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/state/Messages.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/state/Messages.java b/core/src/main/java/org/apache/twill/internal/state/Messages.java
new file mode 100644
index 0000000..9783d62
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/state/Messages.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.state;
+
+import org.apache.twill.api.Command;
+
+/**
+ * Factory class for creating instances of {@link Message}.
+ */
+public final class Messages {
+
+ /**
+ * Creates a {@link Message.Type#USER} type {@link Message} that sends the giving {@link Command} to a
+ * particular runnable.
+ *
+ * @param runnableName Name of the runnable.
+ * @param command The user command to send.
+ * @return A new instance of {@link Message}.
+ */
+ public static Message createForRunnable(String runnableName, Command command) {
+ return new SimpleMessage(Message.Type.USER, Message.Scope.RUNNABLE, runnableName, command);
+ }
+
+ /**
+ * Creates a {@link Message.Type#USER} type {@link Message} that sends the giving {@link Command} to all
+ * runnables.
+ *
+ * @param command The user command to send.
+ * @return A new instance of {@link Message}.
+ */
+ public static Message createForAll(Command command) {
+ return new SimpleMessage(Message.Type.USER, Message.Scope.ALL_RUNNABLE, null, command);
+ }
+
+ private Messages() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/state/SimpleMessage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/state/SimpleMessage.java b/core/src/main/java/org/apache/twill/internal/state/SimpleMessage.java
new file mode 100644
index 0000000..e146e56
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/state/SimpleMessage.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.state;
+
+import org.apache.twill.api.Command;
+import com.google.common.base.Objects;
+
+/**
+ *
+ */
+final class SimpleMessage implements Message {
+
+ private final Type type;
+ private final Scope scope;
+ private final String runnableName;
+ private final Command command;
+
+ SimpleMessage(Type type, Scope scope, String runnableName, Command command) {
+ this.type = type;
+ this.scope = scope;
+ this.runnableName = runnableName;
+ this.command = command;
+ }
+
+ @Override
+ public Type getType() {
+ return type;
+ }
+
+ @Override
+ public Scope getScope() {
+ return scope;
+ }
+
+ @Override
+ public String getRunnableName() {
+ return runnableName;
+ }
+
+ @Override
+ public Command getCommand() {
+ return command;
+ }
+
+ @Override
+ public String toString() {
+ return Objects.toStringHelper(Message.class)
+ .add("type", type)
+ .add("scope", scope)
+ .add("runnable", runnableName)
+ .add("command", command)
+ .toString();
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(type, scope, runnableName, command);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == this) {
+ return true;
+ }
+ if (!(obj instanceof Message)) {
+ return false;
+ }
+ Message other = (Message) obj;
+ return type == other.getType()
+ && scope == other.getScope()
+ && Objects.equal(runnableName, other.getRunnableName())
+ && Objects.equal(command, other.getCommand());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/state/StateNode.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/state/StateNode.java b/core/src/main/java/org/apache/twill/internal/state/StateNode.java
new file mode 100644
index 0000000..d66f8a2
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/state/StateNode.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.state;
+
+import org.apache.twill.api.ServiceController;
+import com.google.common.util.concurrent.Service;
+
+/**
+ *
+ */
+public final class StateNode {
+
+ private final ServiceController.State state;
+ private final String errorMessage;
+ private final StackTraceElement[] stackTraces;
+
+ /**
+ * Constructs a StateNode with the given state.
+ */
+ public StateNode(ServiceController.State state) {
+ this(state, null, null);
+ }
+
+ /**
+ * Constructs a StateNode with {@link ServiceController.State#FAILED} caused by the given error.
+ */
+ public StateNode(Throwable error) {
+ this(Service.State.FAILED, error.getMessage(), error.getStackTrace());
+ }
+
+ /**
+ * Constructs a StateNode with the given state, error and stacktraces.
+ * This constructor should only be used by the StateNodeCodec.
+ */
+ public StateNode(ServiceController.State state, String errorMessage, StackTraceElement[] stackTraces) {
+ this.state = state;
+ this.errorMessage = errorMessage;
+ this.stackTraces = stackTraces;
+ }
+
+ public ServiceController.State getState() {
+ return state;
+ }
+
+ public String getErrorMessage() {
+ return errorMessage;
+ }
+
+ public StackTraceElement[] getStackTraces() {
+ return stackTraces;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder("state=").append(state);
+
+ if (errorMessage != null) {
+ builder.append("\n").append("error=").append(errorMessage);
+ }
+ if (stackTraces != null) {
+ builder.append("\n");
+ for (StackTraceElement stackTrace : stackTraces) {
+ builder.append("\tat ").append(stackTrace.toString()).append("\n");
+ }
+ }
+ return builder.toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/state/SystemMessages.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/state/SystemMessages.java b/core/src/main/java/org/apache/twill/internal/state/SystemMessages.java
new file mode 100644
index 0000000..9877121
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/state/SystemMessages.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.state;
+
+import org.apache.twill.api.Command;
+import com.google.common.base.Preconditions;
+
+/**
+ * Collection of predefined system messages.
+ */
+public final class SystemMessages {
+
+ public static final Command STOP_COMMAND = Command.Builder.of("stop").build();
+ public static final Message SECURE_STORE_UPDATED = new SimpleMessage(
+ Message.Type.SYSTEM, Message.Scope.APPLICATION, null, Command.Builder.of("secureStoreUpdated").build());
+
+ public static Message stopApplication() {
+ return new SimpleMessage(Message.Type.SYSTEM, Message.Scope.APPLICATION, null, STOP_COMMAND);
+ }
+
+ public static Message stopRunnable(String runnableName) {
+ return new SimpleMessage(Message.Type.SYSTEM, Message.Scope.RUNNABLE, runnableName, STOP_COMMAND);
+ }
+
+ public static Message setInstances(String runnableName, int instances) {
+ Preconditions.checkArgument(instances > 0, "Instances should be > 0.");
+ return new SimpleMessage(Message.Type.SYSTEM, Message.Scope.RUNNABLE, runnableName,
+ Command.Builder.of("instances").addOption("count", Integer.toString(instances)).build());
+ }
+
+ private SystemMessages() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/utils/Dependencies.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/utils/Dependencies.java b/core/src/main/java/org/apache/twill/internal/utils/Dependencies.java
new file mode 100644
index 0000000..015b9f5
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/utils/Dependencies.java
@@ -0,0 +1,323 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.utils;
+
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import com.google.common.io.ByteStreams;
+import org.objectweb.asm.AnnotationVisitor;
+import org.objectweb.asm.ClassReader;
+import org.objectweb.asm.ClassVisitor;
+import org.objectweb.asm.FieldVisitor;
+import org.objectweb.asm.Label;
+import org.objectweb.asm.MethodVisitor;
+import org.objectweb.asm.Opcodes;
+import org.objectweb.asm.Type;
+import org.objectweb.asm.signature.SignatureReader;
+import org.objectweb.asm.signature.SignatureVisitor;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URL;
+import java.util.Queue;
+import java.util.Set;
+
+/**
+ * Utility class to help find out class dependencies.
+ */
+public final class Dependencies {
+
+ /**
+ * Represents a callback for accepting a class during dependency traversal.
+ */
+ public interface ClassAcceptor {
+ /**
+ * Invoked when a class is being found as a dependency.
+ *
+ * @param className Name of the class.
+ * @param classUrl URL for the class resource.
+ * @param classPathUrl URL for the class path resource that contains the class resource.
+ * If the URL protocol is {@code file}, it would be the path to root package.
+ * If the URL protocol is {@code jar}, it would be the jar file.
+ * @return true keep finding dependencies on the given class.
+ */
+ boolean accept(String className, URL classUrl, URL classPathUrl);
+ }
+
+ public static void findClassDependencies(ClassLoader classLoader,
+ ClassAcceptor acceptor,
+ String...classesToResolve) throws IOException {
+ findClassDependencies(classLoader, acceptor, ImmutableList.copyOf(classesToResolve));
+ }
+
+ /**
+ * Finds the class dependencies of the given class.
+ * @param classLoader ClassLoader for finding class bytecode.
+ * @param acceptor Predicate to accept a found class and its bytecode.
+ * @param classesToResolve Classes for looking for dependencies.
+ * @throws IOException Thrown where there is error when loading in class bytecode.
+ */
+ public static void findClassDependencies(ClassLoader classLoader,
+ ClassAcceptor acceptor,
+ Iterable<String> classesToResolve) throws IOException {
+
+ final Set<String> seenClasses = Sets.newHashSet(classesToResolve);
+ final Queue<String> classes = Lists.newLinkedList(classesToResolve);
+
+ // Breadth-first-search classes dependencies.
+ while (!classes.isEmpty()) {
+ String className = classes.remove();
+ URL classUrl = getClassURL(className, classLoader);
+ if (classUrl == null) {
+ continue;
+ }
+
+ // Call the accept to see if it accept the current class.
+ if (!acceptor.accept(className, classUrl, getClassPathURL(className, classUrl))) {
+ continue;
+ }
+
+ InputStream is = classUrl.openStream();
+ try {
+ // Visit the bytecode to lookup classes that the visiting class is depended on.
+ new ClassReader(ByteStreams.toByteArray(is)).accept(new DependencyClassVisitor(new DependencyAcceptor() {
+ @Override
+ public void accept(String className) {
+ // See if the class is accepted
+ if (seenClasses.add(className)) {
+ classes.add(className);
+ }
+ }
+ }), ClassReader.SKIP_DEBUG + ClassReader.SKIP_FRAMES);
+ } finally {
+ is.close();
+ }
+ }
+ }
+
+ /**
+ * Returns the URL for loading the class bytecode of the given class, or null if it is not found or if it is
+ * a system class.
+ */
+ private static URL getClassURL(String className, ClassLoader classLoader) {
+ String resourceName = className.replace('.', '/') + ".class";
+ return classLoader.getResource(resourceName);
+ }
+
+ private static URL getClassPathURL(String className, URL classUrl) {
+ try {
+ if ("file".equals(classUrl.getProtocol())) {
+ String path = classUrl.getFile();
+ // Compute the directory container the class.
+ int endIdx = path.length() - className.length() - ".class".length();
+ if (endIdx > 1) {
+ // If it is not the root directory, return the end index to remove the trailing '/'.
+ endIdx--;
+ }
+ return new URL("file", "", -1, path.substring(0, endIdx));
+ }
+ if ("jar".equals(classUrl.getProtocol())) {
+ String path = classUrl.getFile();
+ return URI.create(path.substring(0, path.indexOf("!/"))).toURL();
+ }
+ } catch (MalformedURLException e) {
+ throw Throwables.propagate(e);
+ }
+ throw new IllegalStateException("Unsupported class URL: " + classUrl);
+ }
+
+ /**
+ * A private interface for accepting a dependent class that is found during bytecode inspection.
+ */
+ private interface DependencyAcceptor {
+ void accept(String className);
+ }
+
+ /**
+ * ASM ClassVisitor for extracting classes dependencies.
+ */
+ private static final class DependencyClassVisitor extends ClassVisitor {
+
+ private final SignatureVisitor signatureVisitor;
+ private final DependencyAcceptor acceptor;
+
+ public DependencyClassVisitor(DependencyAcceptor acceptor) {
+ super(Opcodes.ASM4);
+ this.acceptor = acceptor;
+ this.signatureVisitor = new SignatureVisitor(Opcodes.ASM4) {
+ private String currentClass;
+
+ @Override
+ public void visitClassType(String name) {
+ currentClass = name;
+ addClass(name);
+ }
+
+ @Override
+ public void visitInnerClassType(String name) {
+ addClass(currentClass + "$" + name);
+ }
+ };
+ }
+
+ @Override
+ public void visit(int version, int access, String name, String signature, String superName, String[] interfaces) {
+ addClass(name);
+
+ if (signature != null) {
+ new SignatureReader(signature).accept(signatureVisitor);
+ } else {
+ addClass(superName);
+ addClasses(interfaces);
+ }
+ }
+
+ @Override
+ public void visitOuterClass(String owner, String name, String desc) {
+ addClass(owner);
+ }
+
+ @Override
+ public AnnotationVisitor visitAnnotation(String desc, boolean visible) {
+ addType(Type.getType(desc));
+ return null;
+ }
+
+ @Override
+ public void visitInnerClass(String name, String outerName, String innerName, int access) {
+ addClass(name);
+ }
+
+ @Override
+ public FieldVisitor visitField(int access, String name, String desc, String signature, Object value) {
+ if (signature != null) {
+ new SignatureReader(signature).acceptType(signatureVisitor);
+ } else {
+ addType(Type.getType(desc));
+ }
+
+ return new FieldVisitor(Opcodes.ASM4) {
+ @Override
+ public AnnotationVisitor visitAnnotation(String desc, boolean visible) {
+ addType(Type.getType(desc));
+ return null;
+ }
+ };
+ }
+
+ @Override
+ public MethodVisitor visitMethod(int access, String name, String desc, String signature, String[] exceptions) {
+ if (signature != null) {
+ new SignatureReader(signature).accept(signatureVisitor);
+ } else {
+ addMethod(desc);
+ }
+ addClasses(exceptions);
+
+ return new MethodVisitor(Opcodes.ASM4) {
+ @Override
+ public AnnotationVisitor visitAnnotation(String desc, boolean visible) {
+ addType(Type.getType(desc));
+ return null;
+ }
+
+ @Override
+ public AnnotationVisitor visitParameterAnnotation(int parameter, String desc, boolean visible) {
+ addType(Type.getType(desc));
+ return null;
+ }
+
+ @Override
+ public void visitTypeInsn(int opcode, String type) {
+ addType(Type.getObjectType(type));
+ }
+
+ @Override
+ public void visitFieldInsn(int opcode, String owner, String name, String desc) {
+ addType(Type.getObjectType(owner));
+ addType(Type.getType(desc));
+ }
+
+ @Override
+ public void visitMethodInsn(int opcode, String owner, String name, String desc) {
+ addType(Type.getObjectType(owner));
+ addMethod(desc);
+ }
+
+ @Override
+ public void visitLdcInsn(Object cst) {
+ if (cst instanceof Type) {
+ addType((Type) cst);
+ }
+ }
+
+ @Override
+ public void visitMultiANewArrayInsn(String desc, int dims) {
+ addType(Type.getType(desc));
+ }
+
+ @Override
+ public void visitLocalVariable(String name, String desc, String signature, Label start, Label end, int index) {
+ if (signature != null) {
+ new SignatureReader(signature).acceptType(signatureVisitor);
+ } else {
+ addType(Type.getType(desc));
+ }
+ }
+ };
+ }
+
+ private void addClass(String internalName) {
+ if (internalName == null || internalName.startsWith("java/")) {
+ return;
+ }
+ acceptor.accept(Type.getObjectType(internalName).getClassName());
+ }
+
+ private void addClasses(String[] classes) {
+ if (classes != null) {
+ for (String clz : classes) {
+ addClass(clz);
+ }
+ }
+ }
+
+ private void addType(Type type) {
+ if (type.getSort() == Type.ARRAY) {
+ type = type.getElementType();
+ }
+ if (type.getSort() == Type.OBJECT) {
+ addClass(type.getInternalName());
+ }
+ }
+
+ private void addMethod(String desc) {
+ addType(Type.getReturnType(desc));
+ for (Type type : Type.getArgumentTypes(desc)) {
+ addType(type);
+ }
+ }
+ }
+
+ private Dependencies() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/utils/Instances.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/utils/Instances.java b/core/src/main/java/org/apache/twill/internal/utils/Instances.java
new file mode 100644
index 0000000..28bfce9
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/utils/Instances.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.utils;
+
+import com.google.common.base.Defaults;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Throwables;
+import com.google.common.reflect.TypeToken;
+
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Field;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.lang.reflect.Modifier;
+
+/**
+ * Utility class to help instantiate object instance from class.
+ */
+public final class Instances {
+
+ private static final Object UNSAFE;
+ private static final Method UNSAFE_NEW_INSTANCE;
+
+ static {
+ Object unsafe;
+ Method newInstance;
+ try {
+ Class<?> clz = Class.forName("sun.misc.Unsafe");
+ Field f = clz.getDeclaredField("theUnsafe");
+ f.setAccessible(true);
+ unsafe = f.get(null);
+
+ newInstance = clz.getMethod("allocateInstance", Class.class);
+ } catch (Exception e) {
+ unsafe = null;
+ newInstance = null;
+ }
+ UNSAFE = unsafe;
+ UNSAFE_NEW_INSTANCE = newInstance;
+ }
+
+ /**
+ * Creates a new instance of the given class. It will use the default constructor if it is presents.
+ * Otherwise it will try to use {@link sun.misc.Unsafe#allocateInstance(Class)} to create the instance.
+ * @param clz Class of object to be instantiated.
+ * @param <T> Type of the class
+ * @return An instance of type {@code <T>}
+ */
+ @SuppressWarnings("unchecked")
+ public static <T> T newInstance(Class<T> clz) {
+ try {
+ try {
+ Constructor<T> cons = clz.getDeclaredConstructor();
+ if (!cons.isAccessible()) {
+ cons.setAccessible(true);
+ }
+ return cons.newInstance();
+ } catch (Exception e) {
+ // Try to use Unsafe
+ Preconditions.checkState(UNSAFE != null, "Fail to instantiate with Unsafe.");
+ return unsafeCreate(clz);
+ }
+ } catch (Exception e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+
+ /**
+ * Creates an instance of the given using Unsafe. It also initialize all fields into default values.
+ */
+ private static <T> T unsafeCreate(Class<T> clz) throws InvocationTargetException, IllegalAccessException {
+ T instance = (T) UNSAFE_NEW_INSTANCE.invoke(UNSAFE, clz);
+
+ for (TypeToken<?> type : TypeToken.of(clz).getTypes().classes()) {
+ if (Object.class.equals(type.getRawType())) {
+ break;
+ }
+ for (Field field : type.getRawType().getDeclaredFields()) {
+ if (Modifier.isStatic(field.getModifiers())) {
+ continue;
+ }
+ if (!field.isAccessible()) {
+ field.setAccessible(true);
+ }
+ field.set(instance, Defaults.defaultValue(field.getType()));
+ }
+ }
+
+ return instance;
+ }
+
+
+ private Instances() {
+ // Protect instantiation of this class
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/utils/Networks.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/utils/Networks.java b/core/src/main/java/org/apache/twill/internal/utils/Networks.java
new file mode 100644
index 0000000..8e7d736
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/utils/Networks.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.utils;
+
+import java.io.IOException;
+import java.net.ServerSocket;
+
+/**
+ *
+ */
+public final class Networks {
+
+ /**
+ * Find a random free port in localhost for binding.
+ * @return A port number or -1 for failure.
+ */
+ public static int getRandomPort() {
+ try {
+ ServerSocket socket = new ServerSocket(0);
+ try {
+ return socket.getLocalPort();
+ } finally {
+ socket.close();
+ }
+ } catch (IOException e) {
+ return -1;
+ }
+ }
+
+ private Networks() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/internal/utils/Paths.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/internal/utils/Paths.java b/core/src/main/java/org/apache/twill/internal/utils/Paths.java
new file mode 100644
index 0000000..aeee09f
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/internal/utils/Paths.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.utils;
+
+import com.google.common.io.Files;
+
+/**
+ *
+ */
+public final class Paths {
+
+
+ public static String appendSuffix(String extractFrom, String appendTo) {
+ String suffix = getExtension(extractFrom);
+ if (!suffix.isEmpty()) {
+ return appendTo + '.' + suffix;
+ }
+ return appendTo;
+ }
+
+ public static String getExtension(String path) {
+ if (path.endsWith(".tar.gz")) {
+ return "tar.gz";
+ }
+
+ return Files.getFileExtension(path);
+ }
+
+ private Paths() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/kafka/client/FetchException.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/kafka/client/FetchException.java b/core/src/main/java/org/apache/twill/kafka/client/FetchException.java
new file mode 100644
index 0000000..acccf04
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/kafka/client/FetchException.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.kafka.client;
+
+/**
+ *
+ */
+public final class FetchException extends RuntimeException {
+
+ private final ErrorCode errorCode;
+
+ public FetchException(String message, ErrorCode errorCode) {
+ super(message);
+ this.errorCode = errorCode;
+ }
+
+ public ErrorCode getErrorCode() {
+ return errorCode;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("%s. Error code: %s", super.toString(), errorCode);
+ }
+
+ public enum ErrorCode {
+ UNKNOWN(-1),
+ OK(0),
+ OFFSET_OUT_OF_RANGE(1),
+ INVALID_MESSAGE(2),
+ WRONG_PARTITION(3),
+ INVALID_FETCH_SIZE(4);
+
+ private final int code;
+
+ ErrorCode(int code) {
+ this.code = code;
+ }
+
+ public int getCode() {
+ return code;
+ }
+
+ public static ErrorCode fromCode(int code) {
+ switch (code) {
+ case -1:
+ return UNKNOWN;
+ case 0:
+ return OK;
+ case 1:
+ return OFFSET_OUT_OF_RANGE;
+ case 2:
+ return INVALID_MESSAGE;
+ case 3:
+ return WRONG_PARTITION;
+ case 4:
+ return INVALID_FETCH_SIZE;
+ }
+ throw new IllegalArgumentException("Unknown error code");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/kafka/client/FetchedMessage.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/kafka/client/FetchedMessage.java b/core/src/main/java/org/apache/twill/kafka/client/FetchedMessage.java
new file mode 100644
index 0000000..65e140f
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/kafka/client/FetchedMessage.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.kafka.client;
+
+import java.nio.ByteBuffer;
+
+/**
+ * Represents a message fetched from kafka broker.
+ */
+public interface FetchedMessage {
+
+ /**
+ * Returns the message offset.
+ */
+ long getOffset();
+
+ /**
+ * Returns the message payload.
+ */
+ ByteBuffer getBuffer();
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/kafka/client/KafkaClient.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/kafka/client/KafkaClient.java b/core/src/main/java/org/apache/twill/kafka/client/KafkaClient.java
new file mode 100644
index 0000000..496195b
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/kafka/client/KafkaClient.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.kafka.client;
+
+import org.apache.twill.internal.kafka.client.Compression;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.Service;
+
+import java.util.Iterator;
+
+/**
+ * This interface provides methods for interacting with kafka broker. It also
+ * extends from {@link Service} for lifecycle management. The {@link #start()} method
+ * must be called prior to other methods in this class. When instance of this class
+ * is not needed, call {@link #stop()}} to release any resources that it holds.
+ */
+public interface KafkaClient extends Service {
+
+ PreparePublish preparePublish(String topic, Compression compression);
+
+ Iterator<FetchedMessage> consume(String topic, int partition, long offset, int maxSize);
+
+ /**
+ * Fetches offset from the given topic and partition.
+ * @param topic Topic to fetch from.
+ * @param partition Partition to fetch from.
+ * @param time The first offset of every segment file for a given partition with a modified time less than time.
+ * {@code -1} for latest offset, {@code -2} for earliest offset.
+ * @param maxOffsets Maximum number of offsets to fetch.
+ * @return A Future that carry the result as an array of offsets in descending order.
+ * The size of the result array would not be larger than maxOffsets. If there is any error during the fetch,
+ * the exception will be carried in the exception.
+ */
+ ListenableFuture<long[]> getOffset(String topic, int partition, long time, int maxOffsets);
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/kafka/client/PreparePublish.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/kafka/client/PreparePublish.java b/core/src/main/java/org/apache/twill/kafka/client/PreparePublish.java
new file mode 100644
index 0000000..5db4abb
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/kafka/client/PreparePublish.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.kafka.client;
+
+import com.google.common.util.concurrent.ListenableFuture;
+
+import java.nio.ByteBuffer;
+
+/**
+ * This interface is for preparing to publish a set of messages to kafka.
+ */
+public interface PreparePublish {
+
+ PreparePublish add(byte[] payload, Object partitionKey);
+
+ PreparePublish add(ByteBuffer payload, Object partitionKey);
+
+ ListenableFuture<?> publish();
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/kafka/client/package-info.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/kafka/client/package-info.java b/core/src/main/java/org/apache/twill/kafka/client/package-info.java
new file mode 100644
index 0000000..ea3bf20
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/kafka/client/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This package provides a pure java Kafka client interface.
+ */
+package org.apache.twill.kafka.client;
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/java/org/apache/twill/launcher/TwillLauncher.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/twill/launcher/TwillLauncher.java b/core/src/main/java/org/apache/twill/launcher/TwillLauncher.java
new file mode 100644
index 0000000..2c8c1ef
--- /dev/null
+++ b/core/src/main/java/org/apache/twill/launcher/TwillLauncher.java
@@ -0,0 +1,236 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.launcher;
+
+import java.io.BufferedOutputStream;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.OutputStream;
+import java.lang.reflect.Method;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.net.URLClassLoader;
+import java.nio.charset.Charset;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.jar.JarEntry;
+import java.util.jar.JarInputStream;
+
+/**
+ * A launcher for application from a archive jar.
+ * This class should have no dependencies on any library except the J2SE one.
+ * This class should not import any thing except java.*
+ */
+public final class TwillLauncher {
+
+ private static final int TEMP_DIR_ATTEMPTS = 20;
+
+ /**
+ * Main method to unpackage a jar and run the mainClass.main() method.
+ * @param args args[0] is the path to jar file, args[1] is the class name of the mainClass.
+ * The rest of args will be passed the mainClass unmodified.
+ */
+ public static void main(String[] args) throws Exception {
+ if (args.length < 3) {
+ System.out.println("Usage: java " + TwillLauncher.class.getName() + " [jarFile] [mainClass] [use_classpath]");
+ return;
+ }
+
+ File file = new File(args[0]);
+ final File targetDir = createTempDir("twill.launcher");
+
+ Runtime.getRuntime().addShutdownHook(new Thread() {
+ @Override
+ public void run() {
+ System.out.println("Cleanup directory " + targetDir);
+ deleteDir(targetDir);
+ }
+ });
+
+ System.out.println("UnJar " + file + " to " + targetDir);
+ unJar(file, targetDir);
+
+ // Create ClassLoader
+ URLClassLoader classLoader = createClassLoader(targetDir, Boolean.parseBoolean(args[2]));
+ Thread.currentThread().setContextClassLoader(classLoader);
+
+ System.out.println("Launch class with classpath: " + Arrays.toString(classLoader.getURLs()));
+
+ Class<?> mainClass = classLoader.loadClass(args[1]);
+ Method mainMethod = mainClass.getMethod("main", String[].class);
+ String[] arguments = Arrays.copyOfRange(args, 3, args.length);
+ System.out.println("Launching main: " + mainMethod + " " + Arrays.toString(arguments));
+ mainMethod.invoke(mainClass, new Object[]{arguments});
+ System.out.println("Main class completed.");
+
+ System.out.println("Launcher completed");
+ }
+
+ /**
+ * This method is copied from Guava Files.createTempDir().
+ */
+ private static File createTempDir(String prefix) throws IOException {
+ File baseDir = new File(System.getProperty("java.io.tmpdir"));
+ if (!baseDir.isDirectory() && !baseDir.mkdirs()) {
+ throw new IOException("Tmp directory not exists: " + baseDir.getAbsolutePath());
+ }
+
+ String baseName = prefix + "-" + System.currentTimeMillis() + "-";
+
+ for (int counter = 0; counter < TEMP_DIR_ATTEMPTS; counter++) {
+ File tempDir = new File(baseDir, baseName + counter);
+ if (tempDir.mkdir()) {
+ return tempDir;
+ }
+ }
+ throw new IOException("Failed to create directory within "
+ + TEMP_DIR_ATTEMPTS + " attempts (tried "
+ + baseName + "0 to " + baseName + (TEMP_DIR_ATTEMPTS - 1) + ')');
+ }
+
+ private static void unJar(File jarFile, File targetDir) throws IOException {
+ JarInputStream jarInput = new JarInputStream(new FileInputStream(jarFile));
+ try {
+ JarEntry jarEntry = jarInput.getNextJarEntry();
+ while (jarEntry != null) {
+ File target = new File(targetDir, jarEntry.getName());
+ if (jarEntry.isDirectory()) {
+ target.mkdirs();
+ } else {
+ target.getParentFile().mkdirs();
+ copy(jarInput, target);
+ }
+ jarEntry = jarInput.getNextJarEntry();
+ }
+ } finally {
+ jarInput.close();
+ }
+ }
+
+ private static void copy(InputStream is, File file) throws IOException {
+ byte[] buf = new byte[8192];
+ OutputStream os = new BufferedOutputStream(new FileOutputStream(file));
+ try {
+ int len = is.read(buf);
+ while (len != -1) {
+ os.write(buf, 0, len);
+ len = is.read(buf);
+ }
+ } finally {
+ os.close();
+ }
+ }
+
+ private static URLClassLoader createClassLoader(File dir, boolean useClassPath) {
+ try {
+ List<URL> urls = new ArrayList<URL>();
+ urls.add(dir.toURI().toURL());
+ urls.add(new File(dir, "classes").toURI().toURL());
+ urls.add(new File(dir, "resources").toURI().toURL());
+
+ File libDir = new File(dir, "lib");
+ File[] files = libDir.listFiles();
+ if (files != null) {
+ for (File file : files) {
+ if (file.getName().endsWith(".jar")) {
+ urls.add(file.toURI().toURL());
+ }
+ }
+ }
+
+ if (useClassPath) {
+ InputStream is = ClassLoader.getSystemResourceAsStream("classpath");
+ if (is != null) {
+ try {
+ BufferedReader reader = new BufferedReader(new InputStreamReader(is, Charset.forName("UTF-8")));
+ String line = reader.readLine();
+ if (line != null) {
+ for (String path : line.split(":")) {
+ urls.addAll(getClassPaths(path));
+ }
+ }
+ } finally {
+ is.close();
+ }
+ }
+ }
+
+ return new URLClassLoader(urls.toArray(new URL[0]));
+
+ } catch (Exception e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ private static Collection<URL> getClassPaths(String path) throws MalformedURLException {
+ String classpath = expand(path);
+ if (classpath.endsWith("/*")) {
+ // Grab all .jar files
+ File dir = new File(classpath.substring(0, classpath.length() - 2));
+ File[] files = dir.listFiles();
+ if (files == null || files.length == 0) {
+ return singleItem(dir.toURI().toURL());
+ }
+
+ List<URL> result = new ArrayList<URL>(files.length);
+ for (File file : files) {
+ if (file.getName().endsWith(".jar")) {
+ result.add(file.toURI().toURL());
+ }
+ }
+ return result;
+ } else {
+ return singleItem(new File(classpath).toURI().toURL());
+ }
+ }
+
+ private static Collection<URL> singleItem(URL url) {
+ List<URL> result = new ArrayList<URL>(1);
+ result.add(url);
+ return result;
+ }
+
+ private static String expand(String value) {
+ String result = value;
+ for (Map.Entry<String, String> entry : System.getenv().entrySet()) {
+ result = result.replace("$" + entry.getKey(), entry.getValue());
+ result = result.replace("${" + entry.getKey() + "}", entry.getValue());
+ }
+ return result;
+ }
+
+ private static void deleteDir(File dir) {
+ File[] files = dir.listFiles();
+ if (files == null || files.length == 0) {
+ dir.delete();
+ return;
+ }
+ for (File file : files) {
+ deleteDir(file);
+ }
+ dir.delete();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/main/resources/kafka-0.7.2.tgz
----------------------------------------------------------------------
diff --git a/core/src/main/resources/kafka-0.7.2.tgz b/core/src/main/resources/kafka-0.7.2.tgz
new file mode 100644
index 0000000..24178d9
Binary files /dev/null and b/core/src/main/resources/kafka-0.7.2.tgz differ
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/test/java/org/apache/twill/internal/ControllerTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/twill/internal/ControllerTest.java b/core/src/test/java/org/apache/twill/internal/ControllerTest.java
new file mode 100644
index 0000000..382dc95
--- /dev/null
+++ b/core/src/test/java/org/apache/twill/internal/ControllerTest.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal;
+
+import org.apache.twill.api.Command;
+import org.apache.twill.api.ResourceReport;
+import org.apache.twill.api.RunId;
+import org.apache.twill.api.ServiceController;
+import org.apache.twill.api.TwillController;
+import org.apache.twill.api.logging.LogHandler;
+import org.apache.twill.common.ServiceListenerAdapter;
+import org.apache.twill.common.Threads;
+import org.apache.twill.internal.state.StateNode;
+import org.apache.twill.internal.zookeeper.InMemoryZKServer;
+import org.apache.twill.zookeeper.NodeData;
+import org.apache.twill.zookeeper.ZKClient;
+import org.apache.twill.zookeeper.ZKClientService;
+import com.google.common.base.Suppliers;
+import com.google.common.collect.ImmutableList;
+import com.google.common.util.concurrent.AbstractIdleService;
+import com.google.common.util.concurrent.Service;
+import com.google.gson.JsonObject;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+/**
+ *
+ */
+public class ControllerTest {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ControllerTest.class);
+
+ @Test
+ public void testController() throws ExecutionException, InterruptedException, TimeoutException {
+ InMemoryZKServer zkServer = InMemoryZKServer.builder().build();
+ zkServer.startAndWait();
+
+ LOG.info("ZKServer: " + zkServer.getConnectionStr());
+
+ try {
+ RunId runId = RunIds.generate();
+ ZKClientService zkClientService = ZKClientService.Builder.of(zkServer.getConnectionStr()).build();
+ zkClientService.startAndWait();
+
+ Service service = createService(zkClientService, runId);
+ service.startAndWait();
+
+ TwillController controller = getController(zkClientService, runId);
+ controller.sendCommand(Command.Builder.of("test").build()).get(2, TimeUnit.SECONDS);
+ controller.stop().get(2, TimeUnit.SECONDS);
+
+ Assert.assertEquals(ServiceController.State.TERMINATED, controller.state());
+
+ final CountDownLatch terminateLatch = new CountDownLatch(1);
+ service.addListener(new ServiceListenerAdapter() {
+ @Override
+ public void terminated(Service.State from) {
+ terminateLatch.countDown();
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+
+ Assert.assertTrue(service.state() == Service.State.TERMINATED || terminateLatch.await(2, TimeUnit.SECONDS));
+
+ zkClientService.stopAndWait();
+
+ } finally {
+ zkServer.stopAndWait();
+ }
+ }
+
+ // Test controller created before service starts.
+ @Test
+ public void testControllerBefore() throws InterruptedException {
+ InMemoryZKServer zkServer = InMemoryZKServer.builder().build();
+ zkServer.startAndWait();
+
+ LOG.info("ZKServer: " + zkServer.getConnectionStr());
+ try {
+ RunId runId = RunIds.generate();
+ ZKClientService zkClientService = ZKClientService.Builder.of(zkServer.getConnectionStr()).build();
+ zkClientService.startAndWait();
+
+ final CountDownLatch runLatch = new CountDownLatch(1);
+ final CountDownLatch stopLatch = new CountDownLatch(1);
+ TwillController controller = getController(zkClientService, runId);
+ controller.addListener(new ServiceListenerAdapter() {
+ @Override
+ public void running() {
+ runLatch.countDown();
+ }
+
+ @Override
+ public void terminated(Service.State from) {
+ stopLatch.countDown();
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+
+ Service service = createService(zkClientService, runId);
+ service.start();
+
+ Assert.assertTrue(runLatch.await(2, TimeUnit.SECONDS));
+ Assert.assertFalse(stopLatch.await(2, TimeUnit.SECONDS));
+
+ service.stop();
+
+ Assert.assertTrue(stopLatch.await(2, TimeUnit.SECONDS));
+
+ } finally {
+ zkServer.stopAndWait();
+ }
+ }
+
+ // Test controller listener receive first state change without state transition from service
+ @Test
+ public void testControllerListener() throws InterruptedException {
+ InMemoryZKServer zkServer = InMemoryZKServer.builder().build();
+ zkServer.startAndWait();
+
+ LOG.info("ZKServer: " + zkServer.getConnectionStr());
+ try {
+ RunId runId = RunIds.generate();
+ ZKClientService zkClientService = ZKClientService.Builder.of(zkServer.getConnectionStr()).build();
+ zkClientService.startAndWait();
+
+ Service service = createService(zkClientService, runId);
+ service.startAndWait();
+
+ final CountDownLatch runLatch = new CountDownLatch(1);
+ TwillController controller = getController(zkClientService, runId);
+ controller.addListener(new ServiceListenerAdapter() {
+ @Override
+ public void running() {
+ runLatch.countDown();
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+
+ Assert.assertTrue(runLatch.await(2, TimeUnit.SECONDS));
+
+ service.stopAndWait();
+
+ zkClientService.stopAndWait();
+ } finally {
+ zkServer.stopAndWait();
+ }
+ }
+
+ private Service createService(ZKClient zkClient, RunId runId) {
+ return new ZKServiceDecorator(
+ zkClient, runId, Suppliers.ofInstance(new JsonObject()), new AbstractIdleService() {
+
+ @Override
+ protected void startUp() throws Exception {
+ LOG.info("Start");
+ }
+
+ @Override
+ protected void shutDown() throws Exception {
+ LOG.info("Stop");
+ }
+ });
+ }
+
+ private TwillController getController(ZKClient zkClient, RunId runId) {
+ TwillController controller = new AbstractTwillController(runId, zkClient, ImmutableList.<LogHandler>of()) {
+
+ @Override
+ public void kill() {
+ // No-op
+ }
+
+ @Override
+ protected void instanceNodeUpdated(NodeData nodeData) {
+ // No-op
+ }
+
+ @Override
+ protected void stateNodeUpdated(StateNode stateNode) {
+ // No-op
+ }
+
+ @Override
+ public ResourceReport getResourceReport() {
+ return null;
+ }
+ };
+ controller.startAndWait();
+ return controller;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/test/java/org/apache/twill/internal/state/MessageCodecTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/twill/internal/state/MessageCodecTest.java b/core/src/test/java/org/apache/twill/internal/state/MessageCodecTest.java
new file mode 100644
index 0000000..d267cf8
--- /dev/null
+++ b/core/src/test/java/org/apache/twill/internal/state/MessageCodecTest.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.state;
+
+import org.apache.twill.api.Command;
+import com.google.common.collect.ImmutableMap;
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Map;
+
+/**
+ *
+ */
+public class MessageCodecTest {
+
+ @Test
+ public void testCodec() {
+ Message message = MessageCodec.decode(MessageCodec.encode(new Message() {
+
+ @Override
+ public Type getType() {
+ return Type.SYSTEM;
+ }
+
+ @Override
+ public Scope getScope() {
+ return Scope.APPLICATION;
+ }
+
+ @Override
+ public String getRunnableName() {
+ return null;
+ }
+
+ @Override
+ public Command getCommand() {
+ return new Command() {
+ @Override
+ public String getCommand() {
+ return "stop";
+ }
+
+ @Override
+ public Map<String, String> getOptions() {
+ return ImmutableMap.of("timeout", "1", "timeoutUnit", "SECONDS");
+ }
+ };
+ }
+ }));
+
+ Assert.assertEquals(Message.Type.SYSTEM, message.getType());
+ Assert.assertEquals(Message.Scope.APPLICATION, message.getScope());
+ Assert.assertNull(message.getRunnableName());
+ Assert.assertEquals("stop", message.getCommand().getCommand());
+ Assert.assertEquals(ImmutableMap.of("timeout", "1", "timeoutUnit", "SECONDS"), message.getCommand().getOptions());
+ }
+
+ @Test
+ public void testFailureDecode() {
+ Assert.assertNull(MessageCodec.decode("".getBytes()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/core/src/test/java/org/apache/twill/internal/state/ZKServiceDecoratorTest.java
----------------------------------------------------------------------
diff --git a/core/src/test/java/org/apache/twill/internal/state/ZKServiceDecoratorTest.java b/core/src/test/java/org/apache/twill/internal/state/ZKServiceDecoratorTest.java
new file mode 100644
index 0000000..47d8562
--- /dev/null
+++ b/core/src/test/java/org/apache/twill/internal/state/ZKServiceDecoratorTest.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.state;
+
+import org.apache.twill.api.RunId;
+import org.apache.twill.internal.RunIds;
+import org.apache.twill.internal.ZKServiceDecorator;
+import org.apache.twill.internal.zookeeper.InMemoryZKServer;
+import org.apache.twill.zookeeper.NodeData;
+import org.apache.twill.zookeeper.ZKClientService;
+import org.apache.twill.zookeeper.ZKClients;
+import com.google.common.base.Charsets;
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Suppliers;
+import com.google.common.util.concurrent.AbstractIdleService;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.Service;
+import com.google.gson.Gson;
+import com.google.gson.JsonElement;
+import com.google.gson.JsonObject;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.data.Stat;
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ *
+ */
+public class ZKServiceDecoratorTest {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ZKServiceDecoratorTest.class);
+
+ @Test
+ public void testStateTransition() throws InterruptedException, ExecutionException, TimeoutException {
+ InMemoryZKServer zkServer = InMemoryZKServer.builder().build();
+ zkServer.startAndWait();
+
+ try {
+ final String namespace = Joiner.on('/').join("/twill", RunIds.generate(), "runnables", "Runner1");
+
+ final ZKClientService zkClient = ZKClientService.Builder.of(zkServer.getConnectionStr()).build();
+ zkClient.startAndWait();
+ zkClient.create(namespace, null, CreateMode.PERSISTENT).get();
+
+ try {
+ JsonObject content = new JsonObject();
+ content.addProperty("containerId", "container-123");
+ content.addProperty("host", "localhost");
+
+ RunId runId = RunIds.generate();
+ final Semaphore semaphore = new Semaphore(0);
+ ZKServiceDecorator service = new ZKServiceDecorator(ZKClients.namespace(zkClient, namespace),
+ runId, Suppliers.ofInstance(content),
+ new AbstractIdleService() {
+ @Override
+ protected void startUp() throws Exception {
+ Preconditions.checkArgument(semaphore.tryAcquire(5, TimeUnit.SECONDS), "Fail to start");
+ }
+
+ @Override
+ protected void shutDown() throws Exception {
+ Preconditions.checkArgument(semaphore.tryAcquire(5, TimeUnit.SECONDS), "Fail to stop");
+ }
+ });
+
+ final String runnablePath = namespace + "/" + runId.getId();
+ final AtomicReference<String> stateMatch = new AtomicReference<String>("STARTING");
+ watchDataChange(zkClient, runnablePath + "/state", semaphore, stateMatch);
+ Assert.assertEquals(Service.State.RUNNING, service.start().get(5, TimeUnit.SECONDS));
+
+ stateMatch.set("STOPPING");
+ Assert.assertEquals(Service.State.TERMINATED, service.stop().get(5, TimeUnit.SECONDS));
+
+ } finally {
+ zkClient.stopAndWait();
+ }
+ } finally {
+ zkServer.stopAndWait();
+ }
+ }
+
+ private void watchDataChange(final ZKClientService zkClient, final String path,
+ final Semaphore semaphore, final AtomicReference<String> stateMatch) {
+ Futures.addCallback(zkClient.getData(path, new Watcher() {
+ @Override
+ public void process(WatchedEvent event) {
+ if (event.getType() == Event.EventType.NodeDataChanged) {
+ watchDataChange(zkClient, path, semaphore, stateMatch);
+ }
+ }
+ }), new FutureCallback<NodeData>() {
+ @Override
+ public void onSuccess(NodeData result) {
+ String content = new String(result.getData(), Charsets.UTF_8);
+ JsonObject json = new Gson().fromJson(content, JsonElement.class).getAsJsonObject();
+ if (stateMatch.get().equals(json.get("state").getAsString())) {
+ semaphore.release();
+ }
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ exists();
+ }
+
+ private void exists() {
+ Futures.addCallback(zkClient.exists(path, new Watcher() {
+ @Override
+ public void process(WatchedEvent event) {
+ if (event.getType() == Event.EventType.NodeCreated) {
+ watchDataChange(zkClient, path, semaphore, stateMatch);
+ }
+ }
+ }), new FutureCallback<Stat>() {
+ @Override
+ public void onSuccess(Stat result) {
+ if (result != null) {
+ watchDataChange(zkClient, path, semaphore, stateMatch);
+ }
+ }
+
+ @Override
+ public void onFailure(Throwable t) {
+ LOG.error(t.getMessage(), t);
+ }
+ });
+ }
+ });
+ }
+}
[05/15] Initial import commit.
Posted by ch...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/internal/container/TwillContainerMain.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/internal/container/TwillContainerMain.java b/yarn/src/main/java/org/apache/twill/internal/container/TwillContainerMain.java
new file mode 100644
index 0000000..bbd6c10
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/internal/container/TwillContainerMain.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.container;
+
+import org.apache.twill.api.LocalFile;
+import org.apache.twill.api.RunId;
+import org.apache.twill.api.RuntimeSpecification;
+import org.apache.twill.api.TwillRunnableSpecification;
+import org.apache.twill.api.TwillSpecification;
+import org.apache.twill.discovery.DiscoveryService;
+import org.apache.twill.discovery.ZKDiscoveryService;
+import org.apache.twill.internal.Arguments;
+import org.apache.twill.internal.BasicTwillContext;
+import org.apache.twill.internal.Constants;
+import org.apache.twill.internal.ContainerInfo;
+import org.apache.twill.internal.EnvContainerInfo;
+import org.apache.twill.internal.EnvKeys;
+import org.apache.twill.internal.RunIds;
+import org.apache.twill.internal.ServiceMain;
+import org.apache.twill.internal.json.ArgumentsCodec;
+import org.apache.twill.internal.json.TwillSpecificationAdapter;
+import org.apache.twill.zookeeper.RetryStrategies;
+import org.apache.twill.zookeeper.ZKClient;
+import org.apache.twill.zookeeper.ZKClientService;
+import org.apache.twill.zookeeper.ZKClientServices;
+import org.apache.twill.zookeeper.ZKClients;
+import com.google.common.base.Charsets;
+import com.google.common.base.Preconditions;
+import com.google.common.io.Files;
+import com.google.common.util.concurrent.Service;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.DataInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.Reader;
+import java.util.concurrent.TimeUnit;
+
+/**
+ *
+ */
+public final class TwillContainerMain extends ServiceMain {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TwillContainerMain.class);
+
+ /**
+ * Main method for launching a {@link TwillContainerService} which runs
+ * a {@link org.apache.twill.api.TwillRunnable}.
+ */
+ public static void main(final String[] args) throws Exception {
+ // Try to load the secure store from localized file, which AM requested RM to localize it for this container.
+ loadSecureStore();
+
+ String zkConnectStr = System.getenv(EnvKeys.TWILL_ZK_CONNECT);
+ File twillSpecFile = new File(Constants.Files.TWILL_SPEC);
+ RunId appRunId = RunIds.fromString(System.getenv(EnvKeys.TWILL_APP_RUN_ID));
+ RunId runId = RunIds.fromString(System.getenv(EnvKeys.TWILL_RUN_ID));
+ String runnableName = System.getenv(EnvKeys.TWILL_RUNNABLE_NAME);
+ int instanceId = Integer.parseInt(System.getenv(EnvKeys.TWILL_INSTANCE_ID));
+ int instanceCount = Integer.parseInt(System.getenv(EnvKeys.TWILL_INSTANCE_COUNT));
+
+ ZKClientService zkClientService = ZKClientServices.delegate(
+ ZKClients.reWatchOnExpire(
+ ZKClients.retryOnFailure(ZKClientService.Builder.of(zkConnectStr).build(),
+ RetryStrategies.fixDelay(1, TimeUnit.SECONDS))));
+
+ DiscoveryService discoveryService = new ZKDiscoveryService(zkClientService);
+
+ TwillSpecification twillSpec = loadTwillSpec(twillSpecFile);
+ renameLocalFiles(twillSpec.getRunnables().get(runnableName));
+
+ TwillRunnableSpecification runnableSpec = twillSpec.getRunnables().get(runnableName).getRunnableSpecification();
+ ContainerInfo containerInfo = new EnvContainerInfo();
+ Arguments arguments = decodeArgs();
+ BasicTwillContext context = new BasicTwillContext(
+ runId, appRunId, containerInfo.getHost(),
+ arguments.getRunnableArguments().get(runnableName).toArray(new String[0]),
+ arguments.getArguments().toArray(new String[0]),
+ runnableSpec, instanceId, discoveryService, instanceCount,
+ containerInfo.getMemoryMB(), containerInfo.getVirtualCores()
+ );
+
+ Configuration conf = new YarnConfiguration(new HdfsConfiguration(new Configuration()));
+ Service service = new TwillContainerService(context, containerInfo,
+ getContainerZKClient(zkClientService, appRunId, runnableName),
+ runId, runnableSpec, getClassLoader(),
+ createAppLocation(conf));
+ new TwillContainerMain().doMain(zkClientService, service);
+ }
+
+ private static void loadSecureStore() throws IOException {
+ if (!UserGroupInformation.isSecurityEnabled()) {
+ return;
+ }
+
+ File file = new File(Constants.Files.CREDENTIALS);
+ if (file.exists()) {
+ Credentials credentials = new Credentials();
+ DataInputStream input = new DataInputStream(new FileInputStream(file));
+ try {
+ credentials.readTokenStorageStream(input);
+ } finally {
+ input.close();
+ }
+
+ UserGroupInformation.getCurrentUser().addCredentials(credentials);
+ LOG.info("Secure store updated from {}", file);
+ }
+ }
+
+ private static void renameLocalFiles(RuntimeSpecification runtimeSpec) {
+ for (LocalFile file : runtimeSpec.getLocalFiles()) {
+ if (file.isArchive()) {
+ String path = file.getURI().toString();
+ String name = file.getName() + (path.endsWith(".tar.gz") ? ".tar.gz" : path.substring(path.lastIndexOf('.')));
+ Preconditions.checkState(new File(name).renameTo(new File(file.getName())),
+ "Fail to rename file from %s to %s.",
+ name, file.getName());
+ }
+ }
+ }
+
+ private static ZKClient getContainerZKClient(ZKClient zkClient, RunId appRunId, String runnableName) {
+ return ZKClients.namespace(zkClient, String.format("/%s/runnables/%s", appRunId, runnableName));
+ }
+
+ /**
+ * Returns the ClassLoader for the runnable.
+ */
+ private static ClassLoader getClassLoader() {
+ ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+ if (classLoader == null) {
+ return ClassLoader.getSystemClassLoader();
+ }
+ return classLoader;
+ }
+
+ private static TwillSpecification loadTwillSpec(File specFile) throws IOException {
+ Reader reader = Files.newReader(specFile, Charsets.UTF_8);
+ try {
+ return TwillSpecificationAdapter.create().fromJson(reader);
+ } finally {
+ reader.close();
+ }
+ }
+
+ private static Arguments decodeArgs() throws IOException {
+ return ArgumentsCodec.decode(Files.newReaderSupplier(new File(Constants.Files.ARGUMENTS), Charsets.UTF_8));
+ }
+
+ @Override
+ protected String getHostname() {
+ return System.getenv(EnvKeys.YARN_CONTAINER_HOST);
+ }
+
+ @Override
+ protected String getKafkaZKConnect() {
+ return System.getenv(EnvKeys.TWILL_LOG_KAFKA_ZK);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/internal/container/TwillContainerService.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/internal/container/TwillContainerService.java b/yarn/src/main/java/org/apache/twill/internal/container/TwillContainerService.java
new file mode 100644
index 0000000..f5bc1f2
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/internal/container/TwillContainerService.java
@@ -0,0 +1,168 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.container;
+
+import org.apache.twill.api.Command;
+import org.apache.twill.api.RunId;
+import org.apache.twill.api.TwillRunnable;
+import org.apache.twill.api.TwillRunnableSpecification;
+import org.apache.twill.common.Threads;
+import org.apache.twill.filesystem.Location;
+import org.apache.twill.internal.AbstractTwillService;
+import org.apache.twill.internal.AbstractTwillService;
+import org.apache.twill.internal.BasicTwillContext;
+import org.apache.twill.internal.ContainerInfo;
+import org.apache.twill.internal.ContainerLiveNodeData;
+import org.apache.twill.internal.ZKServiceDecorator;
+import org.apache.twill.internal.logging.Loggings;
+import org.apache.twill.internal.state.Message;
+import org.apache.twill.internal.state.MessageCallback;
+import org.apache.twill.internal.utils.Instances;
+import org.apache.twill.zookeeper.ZKClient;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Supplier;
+import com.google.common.util.concurrent.AbstractExecutionThreadService;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.Service;
+import com.google.common.util.concurrent.SettableFuture;
+import com.google.gson.Gson;
+import com.google.gson.JsonElement;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+
+/**
+ * This class act as a yarn container and run a {@link org.apache.twill.api.TwillRunnable}.
+ */
+public final class TwillContainerService extends AbstractTwillService {
+
+ private static final Logger LOG = LoggerFactory.getLogger(TwillContainerService.class);
+
+ private final TwillRunnableSpecification specification;
+ private final ClassLoader classLoader;
+ private final ContainerLiveNodeData containerLiveNode;
+ private final BasicTwillContext context;
+ private final ZKServiceDecorator serviceDelegate;
+ private ExecutorService commandExecutor;
+ private TwillRunnable runnable;
+
+ public TwillContainerService(BasicTwillContext context, ContainerInfo containerInfo, ZKClient zkClient,
+ RunId runId, TwillRunnableSpecification specification, ClassLoader classLoader,
+ Location applicationLocation) {
+ super(applicationLocation);
+
+ this.specification = specification;
+ this.classLoader = classLoader;
+ this.serviceDelegate = new ZKServiceDecorator(zkClient, runId, createLiveNodeSupplier(), new ServiceDelegate());
+ this.context = context;
+ this.containerLiveNode = new ContainerLiveNodeData(containerInfo.getId(),
+ containerInfo.getHost().getCanonicalHostName());
+ }
+
+ private ListenableFuture<String> processMessage(final String messageId, final Message message) {
+ LOG.debug("Message received: {} {}.", messageId, message);
+
+ if (handleSecureStoreUpdate(message)) {
+ return Futures.immediateFuture(messageId);
+ }
+
+ final SettableFuture<String> result = SettableFuture.create();
+ Command command = message.getCommand();
+ if (message.getType() == Message.Type.SYSTEM
+ && "instances".equals(command.getCommand()) && command.getOptions().containsKey("count")) {
+ context.setInstanceCount(Integer.parseInt(command.getOptions().get("count")));
+ }
+
+ commandExecutor.execute(new Runnable() {
+
+ @Override
+ public void run() {
+ try {
+ runnable.handleCommand(message.getCommand());
+ result.set(messageId);
+ } catch (Exception e) {
+ result.setException(e);
+ }
+ }
+ });
+ return result;
+ }
+
+ private Supplier<? extends JsonElement> createLiveNodeSupplier() {
+ return new Supplier<JsonElement>() {
+ @Override
+ public JsonElement get() {
+ return new Gson().toJsonTree(containerLiveNode);
+ }
+ };
+ }
+
+ @Override
+ protected Service getServiceDelegate() {
+ return serviceDelegate;
+ }
+
+ private final class ServiceDelegate extends AbstractExecutionThreadService implements MessageCallback {
+
+ @Override
+ protected void startUp() throws Exception {
+ commandExecutor = Executors.newSingleThreadExecutor(
+ Threads.createDaemonThreadFactory("runnable-command-executor"));
+
+ Class<?> runnableClass = classLoader.loadClass(specification.getClassName());
+ Preconditions.checkArgument(TwillRunnable.class.isAssignableFrom(runnableClass),
+ "Class %s is not instance of TwillRunnable.", specification.getClassName());
+
+ runnable = Instances.newInstance((Class<TwillRunnable>) runnableClass);
+ runnable.initialize(context);
+ }
+
+ @Override
+ protected void triggerShutdown() {
+ try {
+ runnable.stop();
+ } catch (Throwable t) {
+ LOG.error("Exception when stopping runnable.", t);
+ }
+ }
+
+ @Override
+ protected void shutDown() throws Exception {
+ commandExecutor.shutdownNow();
+ runnable.destroy();
+ Loggings.forceFlush();
+ }
+
+ @Override
+ protected void run() throws Exception {
+ runnable.run();
+ }
+
+ @Override
+ public ListenableFuture<String> onReceived(String messageId, Message message) {
+ if (state() == State.RUNNING) {
+ // Only process message if the service is still alive
+ return processMessage(messageId, message);
+ }
+ return Futures.immediateFuture(messageId);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/internal/yarn/AbstractYarnProcessLauncher.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/internal/yarn/AbstractYarnProcessLauncher.java b/yarn/src/main/java/org/apache/twill/internal/yarn/AbstractYarnProcessLauncher.java
new file mode 100644
index 0000000..b810854
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/internal/yarn/AbstractYarnProcessLauncher.java
@@ -0,0 +1,220 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn;
+
+import org.apache.twill.api.LocalFile;
+import org.apache.twill.internal.ProcessController;
+import org.apache.twill.internal.ProcessLauncher;
+import org.apache.twill.internal.utils.Paths;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Abstract class to help creating different types of process launcher that process on yarn.
+ *
+ * @param <T> Type of the object that contains information about the container that the process is going to launch.
+ */
+public abstract class AbstractYarnProcessLauncher<T> implements ProcessLauncher<T> {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AbstractYarnProcessLauncher.class);
+
+ private final T containerInfo;
+
+ protected AbstractYarnProcessLauncher(T containerInfo) {
+ this.containerInfo = containerInfo;
+ }
+
+ @Override
+ public T getContainerInfo() {
+ return containerInfo;
+ }
+
+ @Override
+ public <C> PrepareLaunchContext prepareLaunch(Map<String, String> environments,
+ Iterable<LocalFile> resources, C credentials) {
+ if (credentials != null) {
+ Preconditions.checkArgument(credentials instanceof Credentials, "Credentials should be of type %s",
+ Credentials.class.getName());
+ }
+ return new PrepareLaunchContextImpl(environments, resources, (Credentials) credentials);
+ }
+
+ /**
+ * Tells whether to append suffix to localize resource name for archive file type. Default is true.
+ */
+ protected boolean useArchiveSuffix() {
+ return true;
+ }
+
+ /**
+ * For children class to override to perform actual process launching.
+ */
+ protected abstract <R> ProcessController<R> doLaunch(YarnLaunchContext launchContext);
+
+ /**
+ * Implementation for the {@link PrepareLaunchContext}.
+ */
+ private final class PrepareLaunchContextImpl implements PrepareLaunchContext {
+
+ private final Credentials credentials;
+ private final YarnLaunchContext launchContext;
+ private final Map<String, YarnLocalResource> localResources;
+ private final Map<String, String> environment;
+ private final List<String> commands;
+
+ private PrepareLaunchContextImpl(Map<String, String> env, Iterable<LocalFile> localFiles, Credentials credentials) {
+ this.credentials = credentials;
+ this.launchContext = YarnUtils.createLaunchContext();
+ this.localResources = Maps.newHashMap();
+ this.environment = Maps.newHashMap(env);
+ this.commands = Lists.newLinkedList();
+
+ for (LocalFile localFile : localFiles) {
+ addLocalFile(localFile);
+ }
+ }
+
+ private void addLocalFile(LocalFile localFile) {
+ String name = localFile.getName();
+ // Always append the file extension as the resource name so that archive expansion by Yarn could work.
+ // Renaming would happen by the Container Launcher.
+ if (localFile.isArchive() && useArchiveSuffix()) {
+ String path = localFile.getURI().toString();
+ String suffix = Paths.getExtension(path);
+ if (!suffix.isEmpty()) {
+ name += '.' + suffix;
+ }
+ }
+ localResources.put(name, YarnUtils.createLocalResource(localFile));
+ }
+
+ @Override
+ public ResourcesAdder withResources() {
+ return new MoreResourcesImpl();
+ }
+
+ @Override
+ public AfterResources noResources() {
+ return new MoreResourcesImpl();
+ }
+
+ private final class MoreResourcesImpl implements MoreResources {
+
+ @Override
+ public MoreResources add(LocalFile localFile) {
+ addLocalFile(localFile);
+ return this;
+ }
+
+ @Override
+ public EnvironmentAdder withEnvironment() {
+ return finish();
+ }
+
+ @Override
+ public AfterEnvironment noEnvironment() {
+ return finish();
+ }
+
+ private MoreEnvironmentImpl finish() {
+ launchContext.setLocalResources(localResources);
+ return new MoreEnvironmentImpl();
+ }
+ }
+
+ private final class MoreEnvironmentImpl implements MoreEnvironment {
+
+ @Override
+ public CommandAdder withCommands() {
+ launchContext.setEnvironment(environment);
+ return new MoreCommandImpl();
+ }
+
+ @Override
+ public <V> MoreEnvironment add(String key, V value) {
+ environment.put(key, value.toString());
+ return this;
+ }
+ }
+
+ private final class MoreCommandImpl implements MoreCommand, StdOutSetter, StdErrSetter {
+
+ private final StringBuilder commandBuilder = new StringBuilder();
+
+ @Override
+ public StdOutSetter add(String cmd, String... args) {
+ commandBuilder.append(cmd);
+ for (String arg : args) {
+ commandBuilder.append(' ').append(arg);
+ }
+ return this;
+ }
+
+ @Override
+ public <R> ProcessController<R> launch() {
+ if (credentials != null && !credentials.getAllTokens().isEmpty()) {
+ for (Token<?> token : credentials.getAllTokens()) {
+ LOG.info("Launch with delegation token {}", token);
+ }
+ launchContext.setCredentials(credentials);
+ }
+ launchContext.setCommands(commands);
+ return doLaunch(launchContext);
+ }
+
+ @Override
+ public MoreCommand redirectError(String stderr) {
+ redirect(2, stderr);
+ return noError();
+ }
+
+ @Override
+ public MoreCommand noError() {
+ commands.add(commandBuilder.toString());
+ commandBuilder.setLength(0);
+ return this;
+ }
+
+ @Override
+ public StdErrSetter redirectOutput(String stdout) {
+ redirect(1, stdout);
+ return this;
+ }
+
+ @Override
+ public StdErrSetter noOutput() {
+ return this;
+ }
+
+ private void redirect(int type, String out) {
+ commandBuilder.append(' ')
+ .append(type).append('>')
+ .append(ApplicationConstants.LOG_DIR_EXPANSION_VAR).append('/').append(out);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/internal/yarn/VersionDetectYarnAMClientFactory.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/internal/yarn/VersionDetectYarnAMClientFactory.java b/yarn/src/main/java/org/apache/twill/internal/yarn/VersionDetectYarnAMClientFactory.java
new file mode 100644
index 0000000..6f47b6c
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/internal/yarn/VersionDetectYarnAMClientFactory.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn;
+
+import com.google.common.base.Throwables;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ *
+ */
+public final class VersionDetectYarnAMClientFactory implements YarnAMClientFactory {
+
+ private final Configuration conf;
+
+ public VersionDetectYarnAMClientFactory(Configuration conf) {
+ this.conf = conf;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public YarnAMClient create() {
+ try {
+ Class<YarnAMClient> clz;
+ if (YarnUtils.isHadoop20()) {
+ // Uses hadoop-2.0 class
+ String clzName = getClass().getPackage().getName() + ".Hadoop20YarnAMClient";
+ clz = (Class<YarnAMClient>) Class.forName(clzName);
+ } else {
+ // Uses hadoop-2.1 class
+ String clzName = getClass().getPackage().getName() + ".Hadoop21YarnAMClient";
+ clz = (Class<YarnAMClient>) Class.forName(clzName);
+ }
+
+ return clz.getConstructor(Configuration.class).newInstance(conf);
+
+ } catch (Exception e) {
+ throw Throwables.propagate(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/internal/yarn/VersionDetectYarnAppClientFactory.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/internal/yarn/VersionDetectYarnAppClientFactory.java b/yarn/src/main/java/org/apache/twill/internal/yarn/VersionDetectYarnAppClientFactory.java
new file mode 100644
index 0000000..f9db959
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/internal/yarn/VersionDetectYarnAppClientFactory.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn;
+
+import com.google.common.base.Throwables;
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ *
+ */
+public final class VersionDetectYarnAppClientFactory implements YarnAppClientFactory {
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public YarnAppClient create(Configuration configuration) {
+ try {
+ Class<YarnAppClient> clz;
+
+ if (YarnUtils.isHadoop20()) {
+ // Uses hadoop-2.0 class.
+ String clzName = getClass().getPackage().getName() + ".Hadoop20YarnAppClient";
+ clz = (Class<YarnAppClient>) Class.forName(clzName);
+ } else {
+ // Uses hadoop-2.1 class
+ String clzName = getClass().getPackage().getName() + ".Hadoop21YarnAppClient";
+ clz = (Class<YarnAppClient>) Class.forName(clzName);
+ }
+
+ return clz.getConstructor(Configuration.class).newInstance(configuration);
+
+ } catch (Exception e) {
+ throw Throwables.propagate(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/internal/yarn/YarnAMClient.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/internal/yarn/YarnAMClient.java b/yarn/src/main/java/org/apache/twill/internal/yarn/YarnAMClient.java
new file mode 100644
index 0000000..83ba6a8
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/internal/yarn/YarnAMClient.java
@@ -0,0 +1,117 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn;
+
+import org.apache.twill.internal.ProcessLauncher;
+import com.google.common.collect.Sets;
+import com.google.common.util.concurrent.Service;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.Records;
+
+import java.net.InetSocketAddress;
+import java.net.URL;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * This interface provides abstraction for AM to interacts with YARN to abstract out YARN version specific
+ * code, making multi-version compatibility easier.
+ */
+public interface YarnAMClient extends Service {
+
+ /**
+ * Builder for creating a container request.
+ */
+ abstract class ContainerRequestBuilder {
+
+ protected final Resource capability;
+ protected final int count;
+ protected final Set<String> hosts = Sets.newHashSet();
+ protected final Set<String> racks = Sets.newHashSet();
+ protected final Priority priority = Records.newRecord(Priority.class);
+
+ protected ContainerRequestBuilder(Resource capability, int count) {
+ this.capability = capability;
+ this.count = count;
+ }
+
+ public ContainerRequestBuilder addHosts(String firstHost, String...moreHosts) {
+ return add(hosts, firstHost, moreHosts);
+ }
+
+ public ContainerRequestBuilder addRacks(String firstRack, String...moreRacks) {
+ return add(racks, firstRack, moreRacks);
+ }
+
+ public ContainerRequestBuilder setPriority(int prio) {
+ priority.setPriority(prio);
+ return this;
+ }
+
+ /**
+ * Adds a container request. Returns an unique ID for the request.
+ */
+ public abstract String apply();
+
+ private <T> ContainerRequestBuilder add(Collection<T> collection, T first, T... more) {
+ collection.add(first);
+ Collections.addAll(collection, more);
+ return this;
+ }
+ }
+
+ ContainerId getContainerId();
+
+ String getHost();
+
+ /**
+ * Sets the tracker address and tracker url. This method should be called before calling {@link #start()}.
+ */
+ void setTracker(InetSocketAddress trackerAddr, URL trackerUrl);
+
+ /**
+ * Callback for allocate call.
+ */
+ // TODO: Move AM heartbeat logic into this interface so AM only needs to handle callback.
+ interface AllocateHandler {
+ void acquired(List<ProcessLauncher<YarnContainerInfo>> launchers);
+
+ void completed(List<YarnContainerStatus> completed);
+ }
+
+ void allocate(float progress, AllocateHandler handler) throws Exception;
+
+ ContainerRequestBuilder addContainerRequest(Resource capability);
+
+ ContainerRequestBuilder addContainerRequest(Resource capability, int count);
+
+ /**
+ * Notify a container request is fulfilled.
+ *
+ * Note: This method is needed to workaround a seemingly bug from AMRMClient implementation in YARN that if
+ * a container is requested after a previous container was acquired (with the same capability), multiple containers
+ * will get allocated instead of one.
+ *
+ * @param id The ID returned by {@link YarnAMClient.ContainerRequestBuilder#apply()}.
+ */
+ void completeContainerRequest(String id);
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/internal/yarn/YarnAMClientFactory.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/internal/yarn/YarnAMClientFactory.java b/yarn/src/main/java/org/apache/twill/internal/yarn/YarnAMClientFactory.java
new file mode 100644
index 0000000..b2a1194
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/internal/yarn/YarnAMClientFactory.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn;
+
+/**
+ *
+ */
+public interface YarnAMClientFactory {
+
+ YarnAMClient create();
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/internal/yarn/YarnAppClient.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/internal/yarn/YarnAppClient.java b/yarn/src/main/java/org/apache/twill/internal/yarn/YarnAppClient.java
new file mode 100644
index 0000000..71a9e68
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/internal/yarn/YarnAppClient.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn;
+
+import org.apache.twill.api.TwillSpecification;
+import org.apache.twill.internal.ProcessController;
+import org.apache.twill.internal.ProcessLauncher;
+import com.google.common.util.concurrent.Service;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+/**
+ * Interface for launching Yarn application from client.
+ */
+public interface YarnAppClient extends Service {
+
+ /**
+ * Creates a {@link ProcessLauncher} for launching the application represented by the given spec.
+ */
+ ProcessLauncher<ApplicationId> createLauncher(TwillSpecification twillSpec) throws Exception;
+
+ /**
+ * Creates a {@link ProcessLauncher} for launching application with the given user and spec.
+ *
+ * @deprecated This method will get removed.
+ */
+ @Deprecated
+ ProcessLauncher<ApplicationId> createLauncher(String user, TwillSpecification twillSpec) throws Exception;
+
+ ProcessController<YarnApplicationReport> createProcessController(ApplicationId appId);
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/internal/yarn/YarnAppClientFactory.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/internal/yarn/YarnAppClientFactory.java b/yarn/src/main/java/org/apache/twill/internal/yarn/YarnAppClientFactory.java
new file mode 100644
index 0000000..70cecad
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/internal/yarn/YarnAppClientFactory.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn;
+
+import org.apache.hadoop.conf.Configuration;
+
+/**
+ *
+ */
+public interface YarnAppClientFactory {
+
+ YarnAppClient create(Configuration configuration);
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/internal/yarn/YarnApplicationReport.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/internal/yarn/YarnApplicationReport.java b/yarn/src/main/java/org/apache/twill/internal/yarn/YarnApplicationReport.java
new file mode 100644
index 0000000..4dbb1d1
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/internal/yarn/YarnApplicationReport.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn;
+
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+
+/**
+ * This interface is for adapting differences in ApplicationReport in different Hadoop version.
+ */
+public interface YarnApplicationReport {
+
+ /**
+ * Get the <code>ApplicationId</code> of the application.
+ * @return <code>ApplicationId</code> of the application
+ */
+ ApplicationId getApplicationId();
+
+ /**
+ * Get the <code>ApplicationAttemptId</code> of the current
+ * attempt of the application
+ * @return <code>ApplicationAttemptId</code> of the attempt
+ */
+ ApplicationAttemptId getCurrentApplicationAttemptId();
+
+ /**
+ * Get the <em>queue</em> to which the application was submitted.
+ * @return <em>queue</em> to which the application was submitted
+ */
+ String getQueue();
+
+ /**
+ * Get the user-defined <em>name</em> of the application.
+ * @return <em>name</em> of the application
+ */
+ String getName();
+
+ /**
+ * Get the <em>host</em> on which the <code>ApplicationMaster</code>
+ * is running.
+ * @return <em>host</em> on which the <code>ApplicationMaster</code>
+ * is running
+ */
+ String getHost();
+
+ /**
+ * Get the <em>RPC port</em> of the <code>ApplicationMaster</code>.
+ * @return <em>RPC port</em> of the <code>ApplicationMaster</code>
+ */
+ int getRpcPort();
+
+
+ /**
+ * Get the <code>YarnApplicationState</code> of the application.
+ * @return <code>YarnApplicationState</code> of the application
+ */
+ YarnApplicationState getYarnApplicationState();
+
+
+ /**
+ * Get the <em>diagnositic information</em> of the application in case of
+ * errors.
+ * @return <em>diagnositic information</em> of the application in case
+ * of errors
+ */
+ String getDiagnostics();
+
+
+ /**
+ * Get the <em>tracking url</em> for the application.
+ * @return <em>tracking url</em> for the application
+ */
+ String getTrackingUrl();
+
+
+ /**
+ * Get the original not-proxied <em>tracking url</em> for the application.
+ * This is intended to only be used by the proxy itself.
+ * @return the original not-proxied <em>tracking url</em> for the application
+ */
+ String getOriginalTrackingUrl();
+
+ /**
+ * Get the <em>start time</em> of the application.
+ * @return <em>start time</em> of the application
+ */
+ long getStartTime();
+
+
+ /**
+ * Get the <em>finish time</em> of the application.
+ * @return <em>finish time</em> of the application
+ */
+ long getFinishTime();
+
+
+ /**
+ * Get the <em>final finish status</em> of the application.
+ * @return <em>final finish status</em> of the application
+ */
+ FinalApplicationStatus getFinalApplicationStatus();
+
+ /**
+ * Retrieve the structure containing the job resources for this application
+ * @return the job resources structure for this application
+ */
+ ApplicationResourceUsageReport getApplicationResourceUsageReport();
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/internal/yarn/YarnContainerInfo.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/internal/yarn/YarnContainerInfo.java b/yarn/src/main/java/org/apache/twill/internal/yarn/YarnContainerInfo.java
new file mode 100644
index 0000000..e806da7
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/internal/yarn/YarnContainerInfo.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn;
+
+import org.apache.twill.internal.ContainerInfo;
+
+/**
+ *
+ */
+public interface YarnContainerInfo extends ContainerInfo {
+
+ <T> T getContainer();
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/internal/yarn/YarnContainerStatus.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/internal/yarn/YarnContainerStatus.java b/yarn/src/main/java/org/apache/twill/internal/yarn/YarnContainerStatus.java
new file mode 100644
index 0000000..57e712c
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/internal/yarn/YarnContainerStatus.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn;
+
+import org.apache.hadoop.yarn.api.records.ContainerState;
+
+/**
+ * This interface is for adapting differences in ContainerStatus between Hadoop 2.0 and 2.1
+ */
+public interface YarnContainerStatus {
+
+ String getContainerId();
+
+ ContainerState getState();
+
+ int getExitStatus();
+
+ String getDiagnostics();
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/internal/yarn/YarnLaunchContext.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/internal/yarn/YarnLaunchContext.java b/yarn/src/main/java/org/apache/twill/internal/yarn/YarnLaunchContext.java
new file mode 100644
index 0000000..984a1be
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/internal/yarn/YarnLaunchContext.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn;
+
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * This interface is for adapting ContainerLaunchContext in different Hadoop version
+ */
+public interface YarnLaunchContext {
+
+ <T> T getLaunchContext();
+
+ void setCredentials(Credentials credentials);
+
+ void setLocalResources(Map<String, YarnLocalResource> localResources);
+
+ void setServiceData(Map<String, ByteBuffer> serviceData);
+
+ Map<String, String> getEnvironment();
+
+ void setEnvironment(Map<String, String> environment);
+
+ List<String> getCommands();
+
+ void setCommands(List<String> commands);
+
+ void setApplicationACLs(Map<ApplicationAccessType, String> acls);
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/internal/yarn/YarnLocalResource.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/internal/yarn/YarnLocalResource.java b/yarn/src/main/java/org/apache/twill/internal/yarn/YarnLocalResource.java
new file mode 100644
index 0000000..9bfc224
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/internal/yarn/YarnLocalResource.java
@@ -0,0 +1,115 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn;
+
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.URL;
+
+/**
+ * A adapter interface for the LocalResource class/interface in different Hadoop version.
+ */
+public interface YarnLocalResource {
+
+ /**
+ * Returns the actual LocalResource object in Yarn.
+ */
+ <T> T getLocalResource();
+
+ /**
+ * Get the <em>location</em> of the resource to be localized.
+ * @return <em>location</em> of the resource to be localized
+ */
+ URL getResource();
+
+ /**
+ * Set <em>location</em> of the resource to be localized.
+ * @param resource <em>location</em> of the resource to be localized
+ */
+ void setResource(URL resource);
+
+ /**
+ * Get the <em>size</em> of the resource to be localized.
+ * @return <em>size</em> of the resource to be localized
+ */
+ long getSize();
+
+ /**
+ * Set the <em>size</em> of the resource to be localized.
+ * @param size <em>size</em> of the resource to be localized
+ */
+ void setSize(long size);
+
+ /**
+ * Get the original <em>timestamp</em> of the resource to be localized, used
+ * for verification.
+ * @return <em>timestamp</em> of the resource to be localized
+ */
+ long getTimestamp();
+
+ /**
+ * Set the <em>timestamp</em> of the resource to be localized, used
+ * for verification.
+ * @param timestamp <em>timestamp</em> of the resource to be localized
+ */
+ void setTimestamp(long timestamp);
+
+ /**
+ * Get the <code>LocalResourceType</code> of the resource to be localized.
+ * @return <code>LocalResourceType</code> of the resource to be localized
+ */
+ LocalResourceType getType();
+
+ /**
+ * Set the <code>LocalResourceType</code> of the resource to be localized.
+ * @param type <code>LocalResourceType</code> of the resource to be localized
+ */
+ void setType(LocalResourceType type);
+
+ /**
+ * Get the <code>LocalResourceVisibility</code> of the resource to be
+ * localized.
+ * @return <code>LocalResourceVisibility</code> of the resource to be
+ * localized
+ */
+ LocalResourceVisibility getVisibility();
+
+ /**
+ * Set the <code>LocalResourceVisibility</code> of the resource to be
+ * localized.
+ * @param visibility <code>LocalResourceVisibility</code> of the resource to be
+ * localized
+ */
+ void setVisibility(LocalResourceVisibility visibility);
+
+ /**
+ * Get the <em>pattern</em> that should be used to extract entries from the
+ * archive (only used when type is <code>PATTERN</code>).
+ * @return <em>pattern</em> that should be used to extract entries from the
+ * archive.
+ */
+ String getPattern();
+
+ /**
+ * Set the <em>pattern</em> that should be used to extract entries from the
+ * archive (only used when type is <code>PATTERN</code>).
+ * @param pattern <em>pattern</em> that should be used to extract entries
+ * from the archive.
+ */
+ void setPattern(String pattern);
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/internal/yarn/YarnNMClient.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/internal/yarn/YarnNMClient.java b/yarn/src/main/java/org/apache/twill/internal/yarn/YarnNMClient.java
new file mode 100644
index 0000000..d863c91
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/internal/yarn/YarnNMClient.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn;
+
+import org.apache.twill.common.Cancellable;
+
+/**
+ * Abstraction for dealing with API differences in different hadoop yarn version
+ */
+public interface YarnNMClient {
+
+ /**
+ * Starts a process based on the given launch context.
+ *
+ * @param containerInfo The containerInfo that the new process will launch in.
+ * @param launchContext Contains information about the process going to start.
+ * @return A {@link Cancellable} that when {@link Cancellable#cancel()}} is invoked,
+ * it will try to shutdown the process.
+ *
+ */
+ Cancellable start(YarnContainerInfo containerInfo, YarnLaunchContext launchContext);
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/internal/yarn/YarnUtils.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/internal/yarn/YarnUtils.java b/yarn/src/main/java/org/apache/twill/internal/yarn/YarnUtils.java
new file mode 100644
index 0000000..4f7597b
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/internal/yarn/YarnUtils.java
@@ -0,0 +1,279 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal.yarn;
+
+import org.apache.twill.api.LocalFile;
+import org.apache.twill.filesystem.ForwardingLocationFactory;
+import org.apache.twill.filesystem.HDFSLocationFactory;
+import org.apache.twill.filesystem.LocationFactory;
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Maps;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.io.DataInputByteBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.LocalResourceType;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.util.Records;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.net.InetSocketAddress;
+import java.nio.ByteBuffer;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicReference;
+
+/**
+ * Collection of helper methods to simplify YARN calls.
+ */
+public class YarnUtils {
+
+ private static final Logger LOG = LoggerFactory.getLogger(YarnUtils.class);
+ private static final AtomicReference<Boolean> HADOOP_20 = new AtomicReference<Boolean>();
+
+ public static YarnLocalResource createLocalResource(LocalFile localFile) {
+ Preconditions.checkArgument(localFile.getLastModified() >= 0, "Last modified time should be >= 0.");
+ Preconditions.checkArgument(localFile.getSize() >= 0, "File size should be >= 0.");
+
+ YarnLocalResource resource = createAdapter(YarnLocalResource.class);
+ resource.setVisibility(LocalResourceVisibility.APPLICATION);
+ resource.setResource(ConverterUtils.getYarnUrlFromURI(localFile.getURI()));
+ resource.setTimestamp(localFile.getLastModified());
+ resource.setSize(localFile.getSize());
+ return setLocalResourceType(resource, localFile);
+ }
+
+ public static YarnLaunchContext createLaunchContext() {
+ return createAdapter(YarnLaunchContext.class);
+ }
+
+ // temporary workaround since older versions of hadoop don't have the getVirtualCores method.
+ public static int getVirtualCores(Resource resource) {
+ try {
+ Method getVirtualCores = Resource.class.getMethod("getVirtualCores");
+ return (Integer) getVirtualCores.invoke(resource);
+ } catch (Exception e) {
+ return 0;
+ }
+ }
+
+ /**
+ * Temporary workaround since older versions of hadoop don't have the setCores method.
+ *
+ * @param resource
+ * @param cores
+ * @return true if virtual cores was set, false if not.
+ */
+ public static boolean setVirtualCores(Resource resource, int cores) {
+ try {
+ Method setVirtualCores = Resource.class.getMethod("setVirtualCores", int.class);
+ setVirtualCores.invoke(resource, cores);
+ } catch (Exception e) {
+ // It's ok to ignore this exception, as it's using older version of API.
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Creates {@link ApplicationId} from the given cluster timestamp and id.
+ */
+ public static ApplicationId createApplicationId(long timestamp, int id) {
+ try {
+ try {
+ // For Hadoop-2.1
+ Method method = ApplicationId.class.getMethod("newInstance", long.class, int.class);
+ return (ApplicationId) method.invoke(null, timestamp, id);
+ } catch (NoSuchMethodException e) {
+ // Try with Hadoop-2.0 way
+ ApplicationId appId = Records.newRecord(ApplicationId.class);
+
+ Method setClusterTimestamp = ApplicationId.class.getMethod("setClusterTimestamp", long.class);
+ Method setId = ApplicationId.class.getMethod("setId", int.class);
+
+ setClusterTimestamp.invoke(appId, timestamp);
+ setId.invoke(appId, id);
+
+ return appId;
+ }
+ } catch (Exception e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ /**
+ * Helper method to get delegation tokens for the given LocationFactory.
+ * @param config The hadoop configuration.
+ * @param locationFactory The LocationFactory for generating tokens.
+ * @param credentials Credentials for storing tokens acquired.
+ * @return List of delegation Tokens acquired.
+ */
+ public static List<Token<?>> addDelegationTokens(Configuration config,
+ LocationFactory locationFactory,
+ Credentials credentials) throws IOException {
+ if (!UserGroupInformation.isSecurityEnabled()) {
+ LOG.debug("Security is not enabled");
+ return ImmutableList.of();
+ }
+
+ FileSystem fileSystem = getFileSystem(locationFactory);
+
+ if (fileSystem == null) {
+ LOG.debug("LocationFactory is not HDFS");
+ return ImmutableList.of();
+ }
+
+ String renewer = getYarnTokenRenewer(config);
+
+ Token<?>[] tokens = fileSystem.addDelegationTokens(renewer, credentials);
+ return tokens == null ? ImmutableList.<Token<?>>of() : ImmutableList.copyOf(tokens);
+ }
+
+ public static ByteBuffer encodeCredentials(Credentials credentials) {
+ try {
+ DataOutputBuffer out = new DataOutputBuffer();
+ credentials.writeTokenStorageToStream(out);
+ return ByteBuffer.wrap(out.getData(), 0, out.getLength());
+ } catch (IOException e) {
+ // Shouldn't throw
+ LOG.error("Failed to encode Credentials.", e);
+ throw Throwables.propagate(e);
+ }
+ }
+
+ /**
+ * Decodes {@link Credentials} from the given buffer.
+ * If the buffer is null or empty, it returns an empty Credentials.
+ */
+ public static Credentials decodeCredentials(ByteBuffer buffer) throws IOException {
+ Credentials credentials = new Credentials();
+ if (buffer != null && buffer.hasRemaining()) {
+ DataInputByteBuffer in = new DataInputByteBuffer();
+ in.reset(buffer);
+ credentials.readTokenStorageStream(in);
+ }
+ return credentials;
+ }
+
+ public static String getYarnTokenRenewer(Configuration config) throws IOException {
+ String rmHost = getRMAddress(config).getHostName();
+ String renewer = SecurityUtil.getServerPrincipal(config.get(YarnConfiguration.RM_PRINCIPAL), rmHost);
+
+ if (renewer == null || renewer.length() == 0) {
+ throw new IOException("No Kerberos principal for Yarn RM to use as renewer");
+ }
+
+ return renewer;
+ }
+
+ public static InetSocketAddress getRMAddress(Configuration config) {
+ return config.getSocketAddr(YarnConfiguration.RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_PORT);
+ }
+
+ /**
+ * Returns true if Hadoop-2.0 classes are in the classpath.
+ */
+ public static boolean isHadoop20() {
+ Boolean hadoop20 = HADOOP_20.get();
+ if (hadoop20 != null) {
+ return hadoop20;
+ }
+ try {
+ Class.forName("org.apache.hadoop.yarn.client.api.NMClient");
+ HADOOP_20.set(false);
+ return false;
+ } catch (ClassNotFoundException e) {
+ HADOOP_20.set(true);
+ return true;
+ }
+ }
+
+ /**
+ * Helper method to create adapter class for bridging between Hadoop 2.0 and 2.1
+ */
+ private static <T> T createAdapter(Class<T> clz) {
+ String className = clz.getPackage().getName();
+
+ if (isHadoop20()) {
+ className += ".Hadoop20" + clz.getSimpleName();
+ } else {
+ className += ".Hadoop21" + clz.getSimpleName();
+ }
+
+ try {
+ return (T) Class.forName(className).newInstance();
+ } catch (Exception e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ private static YarnLocalResource setLocalResourceType(YarnLocalResource localResource, LocalFile localFile) {
+ if (localFile.isArchive()) {
+ if (localFile.getPattern() == null) {
+ localResource.setType(LocalResourceType.ARCHIVE);
+ } else {
+ localResource.setType(LocalResourceType.PATTERN);
+ localResource.setPattern(localFile.getPattern());
+ }
+ } else {
+ localResource.setType(LocalResourceType.FILE);
+ }
+ return localResource;
+ }
+
+ private static <T> Map<String, T> transformResource(Map<String, YarnLocalResource> from) {
+ return Maps.transformValues(from, new Function<YarnLocalResource, T>() {
+ @Override
+ public T apply(YarnLocalResource resource) {
+ return resource.getLocalResource();
+ }
+ });
+ }
+
+ /**
+ * Gets the Hadoop FileSystem from LocationFactory.
+ */
+ private static FileSystem getFileSystem(LocationFactory locationFactory) {
+ if (locationFactory instanceof HDFSLocationFactory) {
+ return ((HDFSLocationFactory) locationFactory).getFileSystem();
+ }
+ if (locationFactory instanceof ForwardingLocationFactory) {
+ return getFileSystem(((ForwardingLocationFactory) locationFactory).getDelegate());
+ }
+ return null;
+ }
+
+ private YarnUtils() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/internal/yarn/package-info.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/internal/yarn/package-info.java b/yarn/src/main/java/org/apache/twill/internal/yarn/package-info.java
new file mode 100644
index 0000000..d6ec9f7
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/internal/yarn/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This package contains class for interacting with Yarn.
+ */
+package org.apache.twill.internal.yarn;
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/yarn/LocationSecureStoreUpdater.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/yarn/LocationSecureStoreUpdater.java b/yarn/src/main/java/org/apache/twill/yarn/LocationSecureStoreUpdater.java
new file mode 100644
index 0000000..4d20c9c
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/yarn/LocationSecureStoreUpdater.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.yarn;
+
+import org.apache.twill.api.RunId;
+import org.apache.twill.api.SecureStore;
+import org.apache.twill.api.SecureStoreUpdater;
+import org.apache.twill.filesystem.LocationFactory;
+import org.apache.twill.internal.yarn.YarnUtils;
+import com.google.common.base.Throwables;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.Credentials;
+
+import java.io.IOException;
+
+/**
+ * Package private class for updating location related secure store.
+ */
+final class LocationSecureStoreUpdater implements SecureStoreUpdater {
+
+ private final Configuration configuration;
+ private final LocationFactory locationFactory;
+
+ LocationSecureStoreUpdater(Configuration configuration, LocationFactory locationFactory) {
+ this.configuration = configuration;
+ this.locationFactory = locationFactory;
+ }
+
+ @Override
+ public SecureStore update(String application, RunId runId) {
+ try {
+ Credentials credentials = new Credentials();
+ YarnUtils.addDelegationTokens(configuration, locationFactory, credentials);
+ return YarnSecureStore.create(credentials);
+ } catch (IOException e) {
+ throw Throwables.propagate(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/yarn/ResourceReportClient.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/yarn/ResourceReportClient.java b/yarn/src/main/java/org/apache/twill/yarn/ResourceReportClient.java
new file mode 100644
index 0000000..2974c3f
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/yarn/ResourceReportClient.java
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.yarn;
+
+import org.apache.twill.api.ResourceReport;
+import org.apache.twill.internal.json.ResourceReportAdapter;
+import com.google.common.base.Charsets;
+import com.google.common.io.Closeables;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.net.URL;
+
+/**
+ * Package private class to get {@link ResourceReport} from the application master.
+ */
+final class ResourceReportClient {
+ private static final Logger LOG = LoggerFactory.getLogger(ResourceReportClient.class);
+
+ private final ResourceReportAdapter reportAdapter;
+ private final URL resourceUrl;
+
+ ResourceReportClient(URL resourceUrl) {
+ this.resourceUrl = resourceUrl;
+ this.reportAdapter = ResourceReportAdapter.create();
+ }
+
+ /**
+ * Returns the resource usage of the application fetched from the resource endpoint URL.
+ * @return A {@link ResourceReport} or {@code null} if failed to fetch the report.
+ */
+ public ResourceReport get() {
+ try {
+ Reader reader = new BufferedReader(new InputStreamReader(resourceUrl.openStream(), Charsets.UTF_8));
+ try {
+ return reportAdapter.fromJson(reader);
+ } finally {
+ Closeables.closeQuietly(reader);
+ }
+ } catch (Exception e) {
+ LOG.error("Exception getting resource report from {}.", resourceUrl, e);
+ return null;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/yarn/YarnSecureStore.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/yarn/YarnSecureStore.java b/yarn/src/main/java/org/apache/twill/yarn/YarnSecureStore.java
new file mode 100644
index 0000000..e6f461a
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/yarn/YarnSecureStore.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.yarn;
+
+import org.apache.twill.api.SecureStore;
+import org.apache.hadoop.security.Credentials;
+
+/**
+ * A {@link SecureStore} for hadoop credentials.
+ */
+public final class YarnSecureStore implements SecureStore {
+
+ private final Credentials credentials;
+
+ public static SecureStore create(Credentials credentials) {
+ return new YarnSecureStore(credentials);
+ }
+
+ private YarnSecureStore(Credentials credentials) {
+ this.credentials = credentials;
+ }
+
+ @Override
+ public Credentials getStore() {
+ return credentials;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/yarn/YarnTwillController.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/yarn/YarnTwillController.java b/yarn/src/main/java/org/apache/twill/yarn/YarnTwillController.java
new file mode 100644
index 0000000..4c240fb
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/yarn/YarnTwillController.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.yarn;
+
+import org.apache.twill.api.ResourceReport;
+import org.apache.twill.api.RunId;
+import org.apache.twill.api.TwillController;
+import org.apache.twill.api.logging.LogHandler;
+import org.apache.twill.internal.AbstractTwillController;
+import org.apache.twill.internal.Constants;
+import org.apache.twill.internal.ProcessController;
+import org.apache.twill.internal.appmaster.TrackerService;
+import org.apache.twill.internal.state.StateNode;
+import org.apache.twill.internal.state.SystemMessages;
+import org.apache.twill.internal.yarn.YarnApplicationReport;
+import org.apache.twill.zookeeper.NodeData;
+import org.apache.twill.zookeeper.ZKClient;
+import com.google.common.base.Throwables;
+import com.google.common.collect.ImmutableList;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.Uninterruptibles;
+import org.apache.commons.lang.time.StopWatch;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URL;
+import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A {@link org.apache.twill.api.TwillController} that controllers application running on Hadoop YARN.
+ */
+final class YarnTwillController extends AbstractTwillController implements TwillController {
+
+ private static final Logger LOG = LoggerFactory.getLogger(YarnTwillController.class);
+
+ private final Callable<ProcessController<YarnApplicationReport>> startUp;
+ private ProcessController<YarnApplicationReport> processController;
+ private ResourceReportClient resourcesClient;
+
+ /**
+ * Creates an instance without any {@link LogHandler}.
+ */
+ YarnTwillController(RunId runId, ZKClient zkClient, Callable<ProcessController<YarnApplicationReport>> startUp) {
+ this(runId, zkClient, ImmutableList.<LogHandler>of(), startUp);
+ }
+
+ YarnTwillController(RunId runId, ZKClient zkClient, Iterable<LogHandler> logHandlers,
+ Callable<ProcessController<YarnApplicationReport>> startUp) {
+ super(runId, zkClient, logHandlers);
+ this.startUp = startUp;
+ }
+
+
+ /**
+ * Sends a message to application to notify the secure store has be updated.
+ */
+ ListenableFuture<Void> secureStoreUpdated() {
+ return sendMessage(SystemMessages.SECURE_STORE_UPDATED, null);
+ }
+
+ @Override
+ protected void doStartUp() {
+ super.doStartUp();
+
+ // Submit and poll the status of the yarn application
+ try {
+ processController = startUp.call();
+
+ YarnApplicationReport report = processController.getReport();
+ LOG.debug("Application {} submit", report.getApplicationId());
+
+ YarnApplicationState state = report.getYarnApplicationState();
+ StopWatch stopWatch = new StopWatch();
+ stopWatch.start();
+ stopWatch.split();
+ long maxTime = TimeUnit.MILLISECONDS.convert(Constants.APPLICATION_MAX_START_SECONDS, TimeUnit.SECONDS);
+
+ LOG.info("Checking yarn application status");
+ while (!hasRun(state) && stopWatch.getSplitTime() < maxTime) {
+ report = processController.getReport();
+ state = report.getYarnApplicationState();
+ LOG.debug("Yarn application status: {}", state);
+ TimeUnit.SECONDS.sleep(1);
+ stopWatch.split();
+ }
+ LOG.info("Yarn application is in state {}", state);
+ if (state != YarnApplicationState.RUNNING) {
+ LOG.info("Yarn application is not in running state. Shutting down controller.",
+ Constants.APPLICATION_MAX_START_SECONDS);
+ forceShutDown();
+ } else {
+ try {
+ URL resourceUrl = URI.create(String.format("http://%s:%d", report.getHost(), report.getRpcPort()))
+ .resolve(TrackerService.PATH).toURL();
+ resourcesClient = new ResourceReportClient(resourceUrl);
+ } catch (IOException e) {
+ resourcesClient = null;
+ }
+ }
+ } catch (Exception e) {
+ throw Throwables.propagate(e);
+ }
+ }
+
+ @Override
+ protected void doShutDown() {
+ if (processController == null) {
+ LOG.warn("No process controller for application that is not submitted.");
+ return;
+ }
+
+ // Wait for the stop message being processed
+ try {
+ Uninterruptibles.getUninterruptibly(getStopMessageFuture(),
+ Constants.APPLICATION_MAX_STOP_SECONDS, TimeUnit.SECONDS);
+ } catch (Exception e) {
+ LOG.error("Failed to wait for stop message being processed.", e);
+ // Kill the application through yarn
+ kill();
+ }
+
+ // Poll application status from yarn
+ try {
+ StopWatch stopWatch = new StopWatch();
+ stopWatch.start();
+ stopWatch.split();
+ long maxTime = TimeUnit.MILLISECONDS.convert(Constants.APPLICATION_MAX_STOP_SECONDS, TimeUnit.SECONDS);
+
+ YarnApplicationReport report = processController.getReport();
+ FinalApplicationStatus finalStatus = report.getFinalApplicationStatus();
+ while (finalStatus == FinalApplicationStatus.UNDEFINED && stopWatch.getSplitTime() < maxTime) {
+ LOG.debug("Yarn application final status for {} {}", report.getApplicationId(), finalStatus);
+ TimeUnit.SECONDS.sleep(1);
+ stopWatch.split();
+ finalStatus = processController.getReport().getFinalApplicationStatus();
+ }
+ LOG.debug("Yarn application final status is {}", finalStatus);
+
+ // Application not finished after max stop time, kill the application
+ if (finalStatus == FinalApplicationStatus.UNDEFINED) {
+ kill();
+ }
+ } catch (Exception e) {
+ LOG.warn("Exception while waiting for application report: {}", e.getMessage(), e);
+ kill();
+ }
+
+ super.doShutDown();
+ }
+
+ @Override
+ public void kill() {
+ if (processController != null) {
+ YarnApplicationReport report = processController.getReport();
+ LOG.info("Killing application {}", report.getApplicationId());
+ processController.cancel();
+ } else {
+ LOG.warn("No process controller for application that is not submitted.");
+ }
+ }
+
+ @Override
+ protected void instanceNodeUpdated(NodeData nodeData) {
+
+ }
+
+ @Override
+ protected void stateNodeUpdated(StateNode stateNode) {
+
+ }
+
+ private boolean hasRun(YarnApplicationState state) {
+ switch (state) {
+ case RUNNING:
+ case FINISHED:
+ case FAILED:
+ case KILLED:
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public ResourceReport getResourceReport() {
+ // in case the user calls this before starting, return null
+ return (resourcesClient == null) ? null : resourcesClient.get();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/yarn/src/main/java/org/apache/twill/yarn/YarnTwillControllerFactory.java
----------------------------------------------------------------------
diff --git a/yarn/src/main/java/org/apache/twill/yarn/YarnTwillControllerFactory.java b/yarn/src/main/java/org/apache/twill/yarn/YarnTwillControllerFactory.java
new file mode 100644
index 0000000..11c2ae6
--- /dev/null
+++ b/yarn/src/main/java/org/apache/twill/yarn/YarnTwillControllerFactory.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.yarn;
+
+import org.apache.twill.api.RunId;
+import org.apache.twill.api.logging.LogHandler;
+import org.apache.twill.internal.ProcessController;
+import org.apache.twill.internal.yarn.YarnApplicationReport;
+
+import java.util.concurrent.Callable;
+
+/**
+ * Factory for creating {@link YarnTwillController}.
+ */
+interface YarnTwillControllerFactory {
+
+ YarnTwillController create(RunId runId, Iterable<LogHandler> logHandlers,
+ Callable<ProcessController<YarnApplicationReport>> startUp);
+}
[15/15] git commit: Initial import commit.
Posted by ch...@apache.org.
Initial import commit.
Project: http://git-wip-us.apache.org/repos/asf/incubator-twill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-twill/commit/1925ffaf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-twill/tree/1925ffaf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-twill/diff/1925ffaf
Branch: refs/heads/master
Commit: 1925ffafc30e8f35d08c9666d22dba323f1fcac0
Parents:
Author: Terence Yim <te...@continuuity.com>
Authored: Wed Nov 20 17:18:51 2013 -0800
Committer: Terence Yim <te...@continuuity.com>
Committed: Thu Nov 21 13:45:28 2013 -0800
----------------------------------------------------------------------
.gitignore | 32 +
LICENSE | 203 +++++
NOTICE | 31 +
api/pom.xml | 54 ++
.../apache/twill/api/AbstractTwillRunnable.java | 75 ++
.../main/java/org/apache/twill/api/Command.java | 114 +++
.../java/org/apache/twill/api/EventHandler.java | 146 ++++
.../apache/twill/api/EventHandlerContext.java | 26 +
.../twill/api/EventHandlerSpecification.java | 30 +
.../java/org/apache/twill/api/LocalFile.java | 46 ++
.../org/apache/twill/api/ResourceReport.java | 56 ++
.../apache/twill/api/ResourceSpecification.java | 152 ++++
.../main/java/org/apache/twill/api/RunId.java | 26 +
.../apache/twill/api/RuntimeSpecification.java | 34 +
.../java/org/apache/twill/api/SecureStore.java | 26 +
.../apache/twill/api/SecureStoreUpdater.java | 33 +
.../org/apache/twill/api/ServiceAnnouncer.java | 33 +
.../org/apache/twill/api/ServiceController.java | 70 ++
.../org/apache/twill/api/TwillApplication.java | 30 +
.../java/org/apache/twill/api/TwillContext.java | 76 ++
.../org/apache/twill/api/TwillController.java | 61 ++
.../org/apache/twill/api/TwillPreparer.java | 146 ++++
.../org/apache/twill/api/TwillRunResources.java | 51 ++
.../org/apache/twill/api/TwillRunnable.java | 57 ++
.../twill/api/TwillRunnableSpecification.java | 76 ++
.../java/org/apache/twill/api/TwillRunner.java | 107 +++
.../apache/twill/api/TwillRunnerService.java | 29 +
.../apache/twill/api/TwillSpecification.java | 327 ++++++++
.../org/apache/twill/api/logging/LogEntry.java | 58 ++
.../apache/twill/api/logging/LogHandler.java | 26 +
.../twill/api/logging/PrinterLogHandler.java | 101 +++
.../apache/twill/api/logging/package-info.java | 22 +
.../java/org/apache/twill/api/package-info.java | 21 +
.../DefaultEventHandlerSpecification.java | 57 ++
.../apache/twill/internal/DefaultLocalFile.java | 76 ++
.../twill/internal/DefaultResourceReport.java | 123 +++
.../internal/DefaultResourceSpecification.java | 70 ++
.../internal/DefaultRuntimeSpecification.java | 67 ++
.../internal/DefaultTwillRunResources.java | 106 +++
.../DefaultTwillRunnableSpecification.java | 60 ++
.../internal/DefaultTwillSpecification.java | 103 +++
.../java/org/apache/twill/internal/RunIds.java | 76 ++
.../org/apache/twill/internal/package-info.java | 22 +
common/pom.xml | 51 ++
.../org/apache/twill/common/Cancellable.java | 29 +
.../twill/common/ServiceListenerAdapter.java | 50 ++
.../java/org/apache/twill/common/Services.java | 140 ++++
.../java/org/apache/twill/common/Threads.java | 52 ++
.../filesystem/ForwardingLocationFactory.java | 34 +
.../apache/twill/filesystem/LocalLocation.java | 205 +++++
.../twill/filesystem/LocalLocationFactory.java | 58 ++
.../org/apache/twill/filesystem/Location.java | 154 ++++
.../twill/filesystem/LocationFactories.java | 67 ++
.../twill/filesystem/LocationFactory.java | 46 ++
.../org/apache/twill/common/ServicesTest.java | 106 +++
.../twill/filesystem/LocalLocationTest.java | 64 ++
core/pom.xml | 89 +++
.../AbstractExecutionServiceController.java | 207 +++++
.../twill/internal/AbstractTwillController.java | 180 +++++
.../internal/AbstractZKServiceController.java | 314 ++++++++
.../twill/internal/ApplicationBundler.java | 339 ++++++++
.../org/apache/twill/internal/Arguments.java | 46 ++
.../twill/internal/BasicTwillContext.java | 131 +++
.../java/org/apache/twill/internal/Configs.java | 45 ++
.../org/apache/twill/internal/Constants.java | 64 ++
.../apache/twill/internal/ContainerInfo.java | 36 +
.../twill/internal/ContainerLiveNodeData.java | 40 +
.../apache/twill/internal/EnvContainerInfo.java | 65 ++
.../java/org/apache/twill/internal/EnvKeys.java | 59 ++
.../apache/twill/internal/ListenerExecutor.java | 134 ++++
.../twill/internal/LogOnlyEventHandler.java | 43 +
.../twill/internal/ProcessController.java | 35 +
.../apache/twill/internal/ProcessLauncher.java | 94 +++
.../internal/SingleRunnableApplication.java | 49 ++
.../internal/TwillContainerController.java | 36 +
.../twill/internal/TwillContainerLauncher.java | 181 +++++
.../org/apache/twill/internal/ZKMessages.java | 94 +++
.../twill/internal/ZKServiceDecorator.java | 482 +++++++++++
.../twill/internal/json/ArgumentsCodec.java | 95 +++
.../apache/twill/internal/json/JsonUtils.java | 66 ++
.../twill/internal/json/LocalFileCodec.java | 67 ++
.../internal/json/ResourceReportAdapter.java | 62 ++
.../internal/json/ResourceReportCodec.java | 67 ++
.../json/ResourceSpecificationCodec.java | 61 ++
.../json/RuntimeSpecificationCodec.java | 69 ++
.../internal/json/StackTraceElementCodec.java | 56 ++
.../twill/internal/json/StateNodeCodec.java | 60 ++
.../internal/json/TwillRunResourcesCodec.java | 61 ++
.../json/TwillRunnableSpecificationCodec.java | 63 ++
.../json/TwillSpecificationAdapter.java | 163 ++++
.../internal/json/TwillSpecificationCodec.java | 127 +++
.../internal/kafka/EmbeddedKafkaServer.java | 93 +++
.../AbstractCompressedMessageSetEncoder.java | 78 ++
.../kafka/client/AbstractMessageSetEncoder.java | 79 ++
.../kafka/client/BasicFetchedMessage.java | 46 ++
.../twill/internal/kafka/client/Bufferer.java | 61 ++
.../internal/kafka/client/Compression.java | 49 ++
.../internal/kafka/client/ConnectionPool.java | 125 +++
.../kafka/client/GZipMessageSetEncoder.java | 37 +
.../kafka/client/IdentityMessageSetEncoder.java | 42 +
.../internal/kafka/client/KafkaBrokerCache.java | 326 ++++++++
.../internal/kafka/client/KafkaRequest.java | 91 +++
.../kafka/client/KafkaRequestEncoder.java | 60 ++
.../kafka/client/KafkaRequestSender.java | 26 +
.../internal/kafka/client/KafkaResponse.java | 49 ++
.../kafka/client/KafkaResponseDispatcher.java | 63 ++
.../kafka/client/KafkaResponseHandler.java | 51 ++
.../internal/kafka/client/MessageFetcher.java | 243 ++++++
.../kafka/client/MessageSetEncoder.java | 31 +
.../internal/kafka/client/ResponseHandler.java | 33 +
.../kafka/client/SimpleKafkaClient.java | 304 +++++++
.../kafka/client/SnappyMessageSetEncoder.java | 38 +
.../internal/kafka/client/TopicBroker.java | 48 ++
.../internal/kafka/client/package-info.java | 21 +
.../twill/internal/logging/KafkaAppender.java | 303 +++++++
.../internal/logging/KafkaTwillRunnable.java | 122 +++
.../twill/internal/logging/LogEntryDecoder.java | 124 +++
.../apache/twill/internal/logging/Loggings.java | 46 ++
.../org/apache/twill/internal/package-info.java | 21 +
.../apache/twill/internal/state/Message.java | 54 ++
.../twill/internal/state/MessageCallback.java | 34 +
.../twill/internal/state/MessageCodec.java | 125 +++
.../apache/twill/internal/state/Messages.java | 52 ++
.../twill/internal/state/SimpleMessage.java | 89 +++
.../apache/twill/internal/state/StateNode.java | 84 ++
.../twill/internal/state/SystemMessages.java | 48 ++
.../twill/internal/utils/Dependencies.java | 323 ++++++++
.../apache/twill/internal/utils/Instances.java | 112 +++
.../apache/twill/internal/utils/Networks.java | 47 ++
.../org/apache/twill/internal/utils/Paths.java | 46 ++
.../twill/kafka/client/FetchException.java | 77 ++
.../twill/kafka/client/FetchedMessage.java | 36 +
.../apache/twill/kafka/client/KafkaClient.java | 50 ++
.../twill/kafka/client/PreparePublish.java | 34 +
.../apache/twill/kafka/client/package-info.java | 21 +
.../apache/twill/launcher/TwillLauncher.java | 236 ++++++
core/src/main/resources/kafka-0.7.2.tgz | Bin 0 -> 8811693 bytes
.../apache/twill/internal/ControllerTest.java | 211 +++++
.../twill/internal/state/MessageCodecTest.java | 78 ++
.../internal/state/ZKServiceDecoratorTest.java | 157 ++++
.../internal/utils/ApplicationBundlerTest.java | 113 +++
.../apache/twill/kafka/client/KafkaTest.java | 220 +++++
core/src/test/resources/logback-test.xml | 18 +
discovery-api/pom.xml | 39 +
.../apache/twill/discovery/Discoverable.java | 37 +
.../twill/discovery/DiscoveryService.java | 35 +
.../twill/discovery/DiscoveryServiceClient.java | 34 +
discovery-core/pom.xml | 52 ++
.../twill/discovery/DiscoverableWrapper.java | 69 ++
.../discovery/InMemoryDiscoveryService.java | 73 ++
.../twill/discovery/ZKDiscoveryService.java | 511 ++++++++++++
.../apache/twill/discovery/package-info.java | 21 +
.../discovery/InMemoryDiscoveryServiceTest.java | 67 ++
.../twill/discovery/ZKDiscoveryServiceTest.java | 253 ++++++
.../src/test/resources/logback-test.xml | 17 +
pom.xml | 556 +++++++++++++
yarn/pom.xml | 127 +++
.../internal/yarn/Hadoop20YarnAMClient.java | 213 +++++
.../internal/yarn/Hadoop20YarnAppClient.java | 197 +++++
.../yarn/Hadoop20YarnApplicationReport.java | 107 +++
.../yarn/Hadoop20YarnContainerInfo.java | 70 ++
.../yarn/Hadoop20YarnContainerStatus.java | 53 ++
.../yarn/Hadoop20YarnLaunchContext.java | 99 +++
.../yarn/Hadoop20YarnLocalResource.java | 101 +++
.../internal/yarn/Hadoop20YarnNMClient.java | 121 +++
.../twill/internal/yarn/ports/AMRMClient.java | 149 ++++
.../internal/yarn/ports/AMRMClientImpl.java | 412 ++++++++++
.../internal/yarn/ports/AllocationResponse.java | 38 +
.../yarn/ports/AllocationResponses.java | 111 +++
.../internal/yarn/Hadoop21YarnAMClient.java | 207 +++++
.../internal/yarn/Hadoop21YarnAppClient.java | 177 ++++
.../yarn/Hadoop21YarnApplicationReport.java | 107 +++
.../yarn/Hadoop21YarnContainerInfo.java | 70 ++
.../yarn/Hadoop21YarnContainerStatus.java | 53 ++
.../yarn/Hadoop21YarnLaunchContext.java | 99 +++
.../yarn/Hadoop21YarnLocalResource.java | 101 +++
.../internal/yarn/Hadoop21YarnNMClient.java | 99 +++
.../apache/twill/filesystem/HDFSLocation.java | 193 +++++
.../twill/filesystem/HDFSLocationFactory.java | 95 +++
.../apache/twill/filesystem/package-info.java | 21 +
.../twill/internal/AbstractTwillService.java | 141 ++++
.../org/apache/twill/internal/ServiceMain.java | 201 +++++
.../ApplicationMasterLiveNodeData.java | 46 ++
.../appmaster/ApplicationMasterMain.java | 85 ++
.../ApplicationMasterProcessLauncher.java | 73 ++
.../appmaster/ApplicationMasterService.java | 799 +++++++++++++++++++
.../appmaster/ApplicationSubmitter.java | 31 +
.../appmaster/BasicEventHandlerContext.java | 38 +
.../internal/appmaster/ExpectedContainers.java | 82 ++
.../appmaster/LoggerContextListenerAdapter.java | 56 ++
.../internal/appmaster/ProvisionRequest.java | 52 ++
.../appmaster/RunnableContainerRequest.java | 58 ++
.../appmaster/RunnableProcessLauncher.java | 93 +++
.../internal/appmaster/RunningContainers.java | 427 ++++++++++
.../internal/appmaster/TrackerService.java | 222 ++++++
.../twill/internal/appmaster/package-info.java | 21 +
.../internal/container/TwillContainerMain.java | 182 +++++
.../container/TwillContainerService.java | 168 ++++
.../yarn/AbstractYarnProcessLauncher.java | 220 +++++
.../yarn/VersionDetectYarnAMClientFactory.java | 55 ++
.../yarn/VersionDetectYarnAppClientFactory.java | 50 ++
.../twill/internal/yarn/YarnAMClient.java | 117 +++
.../internal/yarn/YarnAMClientFactory.java | 26 +
.../twill/internal/yarn/YarnAppClient.java | 45 ++
.../internal/yarn/YarnAppClientFactory.java | 28 +
.../internal/yarn/YarnApplicationReport.java | 126 +++
.../twill/internal/yarn/YarnContainerInfo.java | 28 +
.../internal/yarn/YarnContainerStatus.java | 34 +
.../twill/internal/yarn/YarnLaunchContext.java | 49 ++
.../twill/internal/yarn/YarnLocalResource.java | 115 +++
.../twill/internal/yarn/YarnNMClient.java | 37 +
.../apache/twill/internal/yarn/YarnUtils.java | 279 +++++++
.../twill/internal/yarn/package-info.java | 21 +
.../twill/yarn/LocationSecureStoreUpdater.java | 54 ++
.../apache/twill/yarn/ResourceReportClient.java | 63 ++
.../org/apache/twill/yarn/YarnSecureStore.java | 42 +
.../apache/twill/yarn/YarnTwillController.java | 208 +++++
.../twill/yarn/YarnTwillControllerFactory.java | 34 +
.../apache/twill/yarn/YarnTwillPreparer.java | 600 ++++++++++++++
.../twill/yarn/YarnTwillRunnerService.java | 583 ++++++++++++++
.../org/apache/twill/yarn/package-info.java | 21 +
yarn/src/main/resources/logback-template.xml | 11 +
.../java/org/apache/twill/yarn/BuggyServer.java | 41 +
.../twill/yarn/DistributeShellTestRun.java | 64 ++
.../org/apache/twill/yarn/DistributedShell.java | 70 ++
.../java/org/apache/twill/yarn/EchoServer.java | 48 ++
.../apache/twill/yarn/EchoServerTestRun.java | 138 ++++
.../twill/yarn/EnvironmentEchoServer.java | 35 +
.../twill/yarn/FailureRestartTestRun.java | 133 +++
.../org/apache/twill/yarn/LocalFileTestRun.java | 148 ++++
.../twill/yarn/ProvisionTimeoutTestRun.java | 128 +++
.../twill/yarn/ResourceReportTestRun.java | 268 +++++++
.../org/apache/twill/yarn/SocketServer.java | 133 +++
.../apache/twill/yarn/TaskCompletedTestRun.java | 93 +++
.../twill/yarn/TwillSpecificationTest.java | 87 ++
.../org/apache/twill/yarn/YarnTestSuite.java | 127 +++
yarn/src/test/resources/header.txt | 1 +
yarn/src/test/resources/logback-test.xml | 17 +
zookeeper/pom.xml | 67 ++
.../internal/zookeeper/BasicNodeChildren.java | 66 ++
.../twill/internal/zookeeper/BasicNodeData.java | 67 ++
.../zookeeper/DefaultZKClientService.java | 525 ++++++++++++
.../zookeeper/FailureRetryZKClient.java | 240 ++++++
.../internal/zookeeper/InMemoryZKServer.java | 198 +++++
.../twill/internal/zookeeper/KillZKSession.java | 69 ++
.../internal/zookeeper/NamespaceZKClient.java | 163 ++++
.../twill/internal/zookeeper/RetryUtils.java | 50 ++
.../zookeeper/RewatchOnExpireWatcher.java | 207 +++++
.../zookeeper/RewatchOnExpireZKClient.java | 95 +++
.../zookeeper/SettableOperationFuture.java | 68 ++
.../twill/internal/zookeeper/package-info.java | 22 +
.../twill/zookeeper/ForwardingZKClient.java | 116 +++
.../zookeeper/ForwardingZKClientService.java | 78 ++
.../apache/twill/zookeeper/NodeChildren.java | 38 +
.../org/apache/twill/zookeeper/NodeData.java | 39 +
.../apache/twill/zookeeper/OperationFuture.java | 33 +
.../apache/twill/zookeeper/RetryStrategies.java | 117 +++
.../apache/twill/zookeeper/RetryStrategy.java | 48 ++
.../org/apache/twill/zookeeper/ZKClient.java | 161 ++++
.../apache/twill/zookeeper/ZKClientService.java | 96 +++
.../twill/zookeeper/ZKClientServices.java | 145 ++++
.../org/apache/twill/zookeeper/ZKClients.java | 61 ++
.../apache/twill/zookeeper/ZKOperations.java | 355 ++++++++
.../apache/twill/zookeeper/package-info.java | 22 +
.../twill/zookeeper/RetryStrategyTest.java | 94 +++
.../apache/twill/zookeeper/ZKClientTest.java | 254 ++++++
.../twill/zookeeper/ZKOperationsTest.java | 63 ++
zookeeper/src/test/resources/logback-test.xml | 17 +
268 files changed, 28356 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/.gitignore
----------------------------------------------------------------------
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..7aff00d
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,32 @@
+*.class
+.*.swp
+
+# Package Files #
+*.jar
+*.war
+*.ear
+
+# Intellij Files & Dir #
+*.iml
+*.ipr
+*.iws
+atlassian-ide-plugin.xml
+out/
+.DS_Store
+lib/
+.idea
+data/
+
+# Gradle Files & Dir #
+build/
+.gradle/
+.stickyStorage
+.build/
+target/
+
+# Node log
+npm-*.log
+logs/
+
+# Singlenode run files.
+data/
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/LICENSE
----------------------------------------------------------------------
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..6b0b127
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,203 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/NOTICE
----------------------------------------------------------------------
diff --git a/NOTICE b/NOTICE
new file mode 100644
index 0000000..df3ad5e
--- /dev/null
+++ b/NOTICE
@@ -0,0 +1,31 @@
+This product includes software developed by The Apache Software
+Foundation (http://www.apache.org/).
+
+In addition, this product includes software developed by:
+
+Guava (https://code.google.com/p/guava-libraries) is a Google Core Libraries.
+Licensed under the Apache License 2.0.
+
+Netty (http://netty.io) is a an asynchronous event-driven network application framework.
+Licensed under the Apache License 2.0.
+
+Findbugs (https://code.google.com/p/findbugs) is a defect detection tool for Java.
+Licensed under the GNU Lesser GPL
+
+Gson (https://code.google.com/p/google-gson) is a Java library for converting Java Objects into JSON.
+Licensed under the Apache License 2.0.
+
+Snappy-java (https://code.google.com/p/snappy-java) is a Java library for compression.
+Licensed under the Apache License 2.0.
+
+SLF4J (http://www.slf4j.org/) is a logging library for Java.
+Licensed under the MIT License.
+
+Logback (http://logback.qos.ch) is a logging library for Java.
+Dual licensed under EPL/LGPL. We use it under LGPL.
+
+ASM (http://asm.ow2.org) is a bytecode manipulation library for Java.
+Licensed under the BSD License.
+
+JUnit (http://www.junit.org/) included under the Common Public License v1.0. See
+the full text here: http://junit.sourceforge.net/cpl-v10.html
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/pom.xml
----------------------------------------------------------------------
diff --git a/api/pom.xml b/api/pom.xml
new file mode 100644
index 0000000..ea731f2
--- /dev/null
+++ b/api/pom.xml
@@ -0,0 +1,54 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.twill</groupId>
+ <artifactId>twill-parent</artifactId>
+ <version>1.3.0-SNAPSHOT</version>
+ </parent>
+
+ <artifactId>twill-api</artifactId>
+ <packaging>jar</packaging>
+ <name>Twill API</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>twill-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
+ <artifactId>twill-discovery-api</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/api/AbstractTwillRunnable.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/api/AbstractTwillRunnable.java b/api/src/main/java/org/apache/twill/api/AbstractTwillRunnable.java
new file mode 100644
index 0000000..67cec0a
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/api/AbstractTwillRunnable.java
@@ -0,0 +1,75 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.api;
+
+import com.google.common.collect.ImmutableMap;
+
+import java.util.Map;
+
+/**
+ * This abstract class provides default implementation of the {@link TwillRunnable}.
+ */
+public abstract class AbstractTwillRunnable implements TwillRunnable {
+
+ private Map<String, String> args;
+ private TwillContext context;
+
+ protected AbstractTwillRunnable() {
+ this.args = ImmutableMap.of();
+ }
+
+ protected AbstractTwillRunnable(Map<String, String> args) {
+ this.args = ImmutableMap.copyOf(args);
+ }
+
+ @Override
+ public TwillRunnableSpecification configure() {
+ return TwillRunnableSpecification.Builder.with()
+ .setName(getClass().getSimpleName())
+ .withConfigs(args)
+ .build();
+ }
+
+ @Override
+ public void initialize(TwillContext context) {
+ this.context = context;
+ this.args = context.getSpecification().getConfigs();
+ }
+
+ @Override
+ public void handleCommand(org.apache.twill.api.Command command) throws Exception {
+ // No-op by default. Left for children class to override.
+ }
+
+ @Override
+ public void destroy() {
+ // No-op by default. Left for children class to override.
+ }
+
+ protected Map<String, String> getArguments() {
+ return args;
+ }
+
+ protected String getArgument(String key) {
+ return args.get(key);
+ }
+
+ protected TwillContext getContext() {
+ return context;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/api/Command.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/api/Command.java b/api/src/main/java/org/apache/twill/api/Command.java
new file mode 100644
index 0000000..b23b3a8
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/api/Command.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.api;
+
+import com.google.common.base.Objects;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableMap;
+
+import java.util.Map;
+
+/**
+ * Represents command objects.
+ */
+public interface Command {
+
+ String getCommand();
+
+ Map<String, String> getOptions();
+
+ /**
+ * Builder for creating {@link Command} object.
+ */
+ static final class Builder {
+
+ private final String command;
+ private final ImmutableMap.Builder<String, String> options = ImmutableMap.builder();
+
+ public static Builder of(String command) {
+ Preconditions.checkArgument(command != null, "Command cannot be null.");
+ return new Builder(command);
+ }
+
+ public Builder addOption(String key, String value) {
+ options.put(key, value);
+ return this;
+ }
+
+ public Builder addOptions(Map<String, String> map) {
+ options.putAll(map);
+ return this;
+ }
+
+ public Command build() {
+ return new SimpleCommand(command, options.build());
+ }
+
+ private Builder(String command) {
+ this.command = command;
+ }
+
+ /**
+ * Simple implementation of {@link org.apache.twill.api.Command}.
+ */
+ private static final class SimpleCommand implements Command {
+ private final String command;
+ private final Map<String, String> options;
+
+ SimpleCommand(String command, Map<String, String> options) {
+ this.command = command;
+ this.options = options;
+ }
+
+ @Override
+ public String getCommand() {
+ return command;
+ }
+
+ @Override
+ public Map<String, String> getOptions() {
+ return options;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(command, options);
+ }
+
+ @Override
+ public String toString() {
+ return Objects.toStringHelper(Command.class)
+ .add("command", command)
+ .add("options", options)
+ .toString();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == this) {
+ return true;
+ }
+ if (!(obj instanceof Command)) {
+ return false;
+ }
+ Command other = (Command) obj;
+ return command.equals(other.getCommand()) && options.equals(other.getOptions());
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/api/EventHandler.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/api/EventHandler.java b/api/src/main/java/org/apache/twill/api/EventHandler.java
new file mode 100644
index 0000000..ede5b65
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/api/EventHandler.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.api;
+
+import com.google.common.collect.ImmutableMap;
+
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * A callback handler for acting on application events related to {@link TwillRunnable} lifecycle events.
+ */
+public abstract class EventHandler {
+
+ protected EventHandlerContext context;
+
+ /**
+ * Represents action to act upon runnable launch timeout.
+ */
+ public static final class TimeoutAction {
+
+ // Next timeout in milliseconds.
+ private final long timeout;
+
+ /**
+ * Creates a {@link TimeoutAction} to indicate aborting the application.
+ */
+ public static TimeoutAction abort() {
+ return new TimeoutAction(-1);
+ }
+
+ /**
+ * Creates a {@link TimeoutAction} to indicate recheck again after the given time has passed.
+ * @param elapse Time to elapse before checking for the timeout again.
+ * @param unit Unit of the elapse time.
+ */
+ public static TimeoutAction recheck(long elapse, TimeUnit unit) {
+ return new TimeoutAction(TimeUnit.MILLISECONDS.convert(elapse, unit));
+ }
+
+ private TimeoutAction(long timeout) {
+ this.timeout = timeout;
+ }
+
+ /**
+ * Returns timeout in milliseconds or {@code -1} if to abort the application.
+ */
+ public long getTimeout() {
+ return timeout;
+ }
+ }
+
+ /**
+ * This class holds information about a launch timeout event.
+ */
+ public static final class TimeoutEvent {
+ private final String runnableName;
+ private final int expectedInstances;
+ private final int actualInstances;
+ private final long requestTime;
+
+ public TimeoutEvent(String runnableName, int expectedInstances, int actualInstances, long requestTime) {
+ this.runnableName = runnableName;
+ this.expectedInstances = expectedInstances;
+ this.actualInstances = actualInstances;
+ this.requestTime = requestTime;
+ }
+
+ public String getRunnableName() {
+ return runnableName;
+ }
+
+ public int getExpectedInstances() {
+ return expectedInstances;
+ }
+
+ public int getActualInstances() {
+ return actualInstances;
+ }
+
+ public long getRequestTime() {
+ return requestTime;
+ }
+ }
+
+ /**
+ * Returns an {@link EventHandlerSpecification} for configuring this handler class.
+ */
+ public EventHandlerSpecification configure() {
+ return new EventHandlerSpecification() {
+ @Override
+ public String getClassName() {
+ return EventHandler.this.getClass().getName();
+ }
+
+ @Override
+ public Map<String, String> getConfigs() {
+ return EventHandler.this.getConfigs();
+ }
+ };
+ }
+
+ /**
+ * Invoked by the application to initialize this EventHandler instance.
+ * @param context
+ */
+ public void initialize(EventHandlerContext context) {
+ this.context = context;
+ }
+
+ /**
+ * Invoked by the application when shutting down.
+ */
+ public void destroy() {
+ // No-op
+ }
+
+ /**
+ * Invoked when the number of expected instances doesn't match with number of actual instances.
+ * @param timeoutEvents An Iterable of {@link TimeoutEvent} that contains information about runnable launch timeout.
+ * @return A {@link TimeoutAction} to govern action to act.
+ */
+ public abstract TimeoutAction launchTimeout(Iterable<TimeoutEvent> timeoutEvents);
+
+ /**
+ * Returns set of configurations available at runtime for access.
+ */
+ protected Map<String, String> getConfigs() {
+ return ImmutableMap.of();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/api/EventHandlerContext.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/api/EventHandlerContext.java b/api/src/main/java/org/apache/twill/api/EventHandlerContext.java
new file mode 100644
index 0000000..8e58af6
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/api/EventHandlerContext.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.api;
+
+/**
+ * Represents runtime context for {@link EventHandler}.
+ */
+public interface EventHandlerContext {
+
+ EventHandlerSpecification getSpecification();
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/api/EventHandlerSpecification.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/api/EventHandlerSpecification.java b/api/src/main/java/org/apache/twill/api/EventHandlerSpecification.java
new file mode 100644
index 0000000..190f222
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/api/EventHandlerSpecification.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.api;
+
+import java.util.Map;
+
+/**
+ * Specification for {@link EventHandler}.
+ */
+public interface EventHandlerSpecification {
+
+ String getClassName();
+
+ Map<String, String> getConfigs();
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/api/LocalFile.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/api/LocalFile.java b/api/src/main/java/org/apache/twill/api/LocalFile.java
new file mode 100644
index 0000000..df35a3b
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/api/LocalFile.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.api;
+
+import javax.annotation.Nullable;
+import java.net.URI;
+
+/**
+ * This interface represents a local file that will be available for the container running a {@link TwillRunnable}.
+ */
+public interface LocalFile {
+
+ String getName();
+
+ URI getURI();
+
+ /**
+ * Returns the the last modified time of the file or {@code -1} if unknown.
+ */
+ long getLastModified();
+
+ /**
+ * Returns the size of the file or {@code -1} if unknown.
+ */
+ long getSize();
+
+ boolean isArchive();
+
+ @Nullable
+ String getPattern();
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/api/ResourceReport.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/api/ResourceReport.java b/api/src/main/java/org/apache/twill/api/ResourceReport.java
new file mode 100644
index 0000000..0d63378
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/api/ResourceReport.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.api;
+
+import java.util.Collection;
+import java.util.Map;
+
+/**
+ * This interface provides a snapshot of the resources an application is using
+ * broken down by each runnable.
+ */
+public interface ResourceReport {
+ /**
+ * Get all the run resources being used by all instances of the specified runnable.
+ *
+ * @param runnableName the runnable name.
+ * @return resources being used by all instances of the runnable.
+ */
+ public Collection<TwillRunResources> getRunnableResources(String runnableName);
+
+ /**
+ * Get all the run resources being used across all runnables.
+ *
+ * @return all run resources used by all instances of all runnables.
+ */
+ public Map<String, Collection<TwillRunResources>> getResources();
+
+ /**
+ * Get the resources application master is using.
+ *
+ * @return resources being used by the application master.
+ */
+ public TwillRunResources getAppMasterResources();
+
+ /**
+ * Get the id of the application master.
+ *
+ * @return id of the application master.
+ */
+ public String getApplicationId();
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/api/ResourceSpecification.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/api/ResourceSpecification.java b/api/src/main/java/org/apache/twill/api/ResourceSpecification.java
new file mode 100644
index 0000000..b40682f
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/api/ResourceSpecification.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.api;
+
+import org.apache.twill.internal.DefaultResourceSpecification;
+
+/**
+ * This interface provides specifications for resource requirements including set and get methods for number of cores, amount of memory, and number of instances.
+ */
+public interface ResourceSpecification {
+
+ final ResourceSpecification BASIC = Builder.with().setVirtualCores(1).setMemory(512, SizeUnit.MEGA).build();
+
+ /**
+ * Unit for specifying memory size.
+ */
+ enum SizeUnit {
+ MEGA(1),
+ GIGA(1024);
+
+ private final int multiplier;
+
+ private SizeUnit(int multiplier) {
+ this.multiplier = multiplier;
+ }
+ }
+
+ /**
+ * Returns the number of virtual CPU cores. DEPRECATED, use getVirtualCores instead.
+ * @return Number of virtual CPU cores.
+ */
+ @Deprecated
+ int getCores();
+
+ /**
+ * Returns the number of virtual CPU cores.
+ * @return Number of virtual CPU cores.
+ */
+ int getVirtualCores();
+
+ /**
+ * Returns the memory size in MB.
+ * @return Memory size
+ */
+ int getMemorySize();
+
+ /**
+ * Returns the uplink bandwidth in Mbps.
+ * @return Uplink bandwidth or -1 representing unlimited bandwidth.
+ */
+ int getUplink();
+
+ /**
+ * Returns the downlink bandwidth in Mbps.
+ * @return Downlink bandwidth or -1 representing unlimited bandwidth.
+ */
+ int getDownlink();
+
+ /**
+ * Returns number of execution instances.
+ * @return Number of execution instances.
+ */
+ int getInstances();
+
+ /**
+ * Builder for creating {@link ResourceSpecification}.
+ */
+ static final class Builder {
+
+ private int cores;
+ private int memory;
+ private int uplink = -1;
+ private int downlink = -1;
+ private int instances = 1;
+
+ public static CoreSetter with() {
+ return new Builder().new CoreSetter();
+ }
+
+ public final class CoreSetter {
+ @Deprecated
+ public MemorySetter setCores(int cores) {
+ Builder.this.cores = cores;
+ return new MemorySetter();
+ }
+
+ public MemorySetter setVirtualCores(int cores) {
+ Builder.this.cores = cores;
+ return new MemorySetter();
+ }
+ }
+
+ public final class MemorySetter {
+ public AfterMemory setMemory(int size, SizeUnit unit) {
+ Builder.this.memory = size * unit.multiplier;
+ return new AfterMemory();
+ }
+ }
+
+ public final class AfterMemory extends Build {
+ public AfterInstances setInstances(int instances) {
+ Builder.this.instances = instances;
+ return new AfterInstances();
+ }
+ }
+
+ public final class AfterInstances extends Build {
+ public AfterUplink setUplink(int uplink, SizeUnit unit) {
+ Builder.this.uplink = uplink * unit.multiplier;
+ return new AfterUplink();
+ }
+ }
+
+ public final class AfterUplink extends Build {
+ public AfterDownlink setDownlink(int downlink, SizeUnit unit) {
+ Builder.this.downlink = downlink * unit.multiplier;
+ return new AfterDownlink();
+ }
+ }
+
+ public final class AfterDownlink extends Build {
+
+ @Override
+ public ResourceSpecification build() {
+ return super.build();
+ }
+ }
+
+ public abstract class Build {
+ public ResourceSpecification build() {
+ return new DefaultResourceSpecification(cores, memory, instances, uplink, downlink);
+ }
+ }
+
+ private Builder() {}
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/api/RunId.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/api/RunId.java b/api/src/main/java/org/apache/twill/api/RunId.java
new file mode 100644
index 0000000..7f3c4fe
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/api/RunId.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.api;
+
+/**
+ * Represents the unique ID of a particular execution.
+ */
+public interface RunId {
+
+ String getId();
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/api/RuntimeSpecification.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/api/RuntimeSpecification.java b/api/src/main/java/org/apache/twill/api/RuntimeSpecification.java
new file mode 100644
index 0000000..99e11a4
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/api/RuntimeSpecification.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.api;
+
+import java.util.Collection;
+
+/**
+ * Specifications for runtime requirements.
+ */
+public interface RuntimeSpecification {
+
+ String getName();
+
+ TwillRunnableSpecification getRunnableSpecification();
+
+ ResourceSpecification getResourceSpecification();
+
+ Collection<LocalFile> getLocalFiles();
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/api/SecureStore.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/api/SecureStore.java b/api/src/main/java/org/apache/twill/api/SecureStore.java
new file mode 100644
index 0000000..707a152
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/api/SecureStore.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.api;
+
+/**
+ * Represents storage of secure tokens.
+ */
+public interface SecureStore {
+
+ <T> T getStore();
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/api/SecureStoreUpdater.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/api/SecureStoreUpdater.java b/api/src/main/java/org/apache/twill/api/SecureStoreUpdater.java
new file mode 100644
index 0000000..5912247
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/api/SecureStoreUpdater.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.api;
+
+/**
+ * Represents class capable of creating update of {@link SecureStore} for live applications.
+ */
+public interface SecureStoreUpdater {
+
+ /**
+ * Invoked when an update to SecureStore is needed.
+ *
+ * @param application The name of the application.
+ * @param runId The runId of the live application.
+ * @return A new {@link SecureStore}.
+ */
+ SecureStore update(String application, RunId runId);
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/api/ServiceAnnouncer.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/api/ServiceAnnouncer.java b/api/src/main/java/org/apache/twill/api/ServiceAnnouncer.java
new file mode 100644
index 0000000..d8e4358
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/api/ServiceAnnouncer.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.api;
+
+import org.apache.twill.common.Cancellable;
+
+/**
+ * This interface provides a way to announce the availability of a service.
+ */
+public interface ServiceAnnouncer {
+
+ /**
+ * Registers an endpoint that could be discovered by external party.
+ * @param serviceName Name of the endpoint
+ * @param port Port of the endpoint.
+ */
+ Cancellable announce(String serviceName, int port);
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/api/ServiceController.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/api/ServiceController.java b/api/src/main/java/org/apache/twill/api/ServiceController.java
new file mode 100644
index 0000000..0ea64f9
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/api/ServiceController.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.api;
+
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.Service;
+
+import java.util.concurrent.Executor;
+
+/**
+ * This interface is for controlling a remote running service.
+ */
+public interface ServiceController extends Service {
+
+ /**
+ * Returns the {@link RunId} of the running application.
+ */
+ RunId getRunId();
+
+ /**
+ * Sends a user command to the running application.
+ * @param command The command to send.
+ * @return A {@link ListenableFuture} that will be completed when the command is successfully processed
+ * by the target application.
+ */
+ ListenableFuture<Command> sendCommand(Command command);
+
+ /**
+ * Sends a user command to the given runnable of the running application.
+ * @param runnableName Name of the {@link TwillRunnable}.
+ * @param command The command to send.
+ * @return A {@link ListenableFuture} that will be completed when the command is successfully processed
+ * by the target runnable.
+ */
+ ListenableFuture<Command> sendCommand(String runnableName, Command command);
+
+ /**
+ * Requests to forcefully kill a running service.
+ */
+ void kill();
+
+ /**
+ * Registers a {@link Listener} to be {@linkplain Executor#execute executed} on the given
+ * executor. The listener will have the corresponding transition method called whenever the
+ * service changes state. When added, the current state of the service will be reflected through
+ * callback to the listener. Methods on the listener is guaranteed to be called no more than once.
+ *
+ * @param listener the listener to run when the service changes state is complete
+ * @param executor the executor in which the the listeners callback methods will be run. For fast,
+ * lightweight listeners that would be safe to execute in any thread, consider
+ * {@link com.google.common.util.concurrent.MoreExecutors#sameThreadExecutor}.
+ */
+ @Override
+ void addListener(Listener listener, Executor executor);
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/api/TwillApplication.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/api/TwillApplication.java b/api/src/main/java/org/apache/twill/api/TwillApplication.java
new file mode 100644
index 0000000..b49e7a7
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/api/TwillApplication.java
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.api;
+
+/**
+ * Represents a application that can be launched by Twill.
+ */
+public interface TwillApplication {
+
+ /**
+ * Invoked when launching the application on the client side.
+ * @return A {@link TwillSpecification} specifying properties about this application.
+ */
+ TwillSpecification configure();
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/api/TwillContext.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/api/TwillContext.java b/api/src/main/java/org/apache/twill/api/TwillContext.java
new file mode 100644
index 0000000..b4ddb6e
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/api/TwillContext.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.api;
+
+import java.net.InetAddress;
+
+/**
+ * Represents the runtime context of a {@link TwillRunnable}.
+ */
+public interface TwillContext extends ServiceAnnouncer {
+
+ /**
+ * Returns the {@link RunId} of this running instance of {@link TwillRunnable}.
+ */
+ RunId getRunId();
+
+ /**
+ * Returns the {@link RunId} of this running application.
+ */
+ RunId getApplicationRunId();
+
+ /**
+ * Returns the number of running instances assigned for this {@link TwillRunnable}.
+ */
+ int getInstanceCount();
+
+ /**
+ * Returns the hostname that the runnable is running on.
+ */
+ InetAddress getHost();
+
+ /**
+ * Returns the runtime arguments that are passed to the {@link TwillRunnable}.
+ */
+ String[] getArguments();
+
+ /**
+ * Returns the runtime arguments that are passed to the {@link TwillApplication}.
+ */
+ String[] getApplicationArguments();
+
+ /**
+ * Returns the {@link TwillRunnableSpecification} that was created by {@link TwillRunnable#configure()}.
+ */
+ TwillRunnableSpecification getSpecification();
+
+ /**
+ * Returns an integer id from 0 to (instanceCount - 1).
+ */
+ int getInstanceId();
+
+ /**
+ * Returns the number of virtual cores the runnable is allowed to use.
+ */
+ int getVirtualCores();
+
+ /**
+ * Returns the amount of memory in MB the runnable is allowed to use.
+ */
+ int getMaxMemoryMB();
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/api/TwillController.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/api/TwillController.java b/api/src/main/java/org/apache/twill/api/TwillController.java
new file mode 100644
index 0000000..f31d3f9
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/api/TwillController.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.api;
+
+import org.apache.twill.api.logging.LogHandler;
+import org.apache.twill.discovery.Discoverable;
+import com.google.common.util.concurrent.ListenableFuture;
+
+/**
+ * For controlling a running application.
+ */
+public interface TwillController extends ServiceController {
+
+ /**
+ * Adds a {@link LogHandler} for receiving application log.
+ * @param handler The handler to add.
+ */
+ void addLogHandler(LogHandler handler);
+
+ /**
+ * Discovers the set of {@link Discoverable} endpoints that provides service for the given service name.
+ * @param serviceName Name of the service to discovery.
+ * @return An {@link Iterable} that gives set of latest {@link Discoverable} every time when
+ * {@link Iterable#iterator()}} is invoked.
+ */
+ Iterable<Discoverable> discoverService(String serviceName);
+
+
+ /**
+ * Changes the number of running instances of a given runnable.
+ *
+ * @param runnable The name of the runnable.
+ * @param newCount Number of instances for the given runnable.
+ * @return A {@link ListenableFuture} that will be completed when the number running instances has been
+ * successfully changed. The future will carry the new count as the result. If there is any error
+ * while changing instances, it'll be reflected in the future.
+ */
+ ListenableFuture<Integer> changeInstances(String runnable, int newCount);
+
+ /**
+ * Get a snapshot of the resources used by the application, broken down by each runnable.
+ *
+ * @return A {@link ResourceReport} containing information about resources used by the application.
+ */
+ ResourceReport getResourceReport();
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/api/TwillPreparer.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/api/TwillPreparer.java b/api/src/main/java/org/apache/twill/api/TwillPreparer.java
new file mode 100644
index 0000000..b2a3ce2
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/api/TwillPreparer.java
@@ -0,0 +1,146 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.api;
+
+import org.apache.twill.api.logging.LogHandler;
+
+import java.net.URI;
+
+/**
+ * This interface exposes methods to set up the Twill runtime environment and start a Twill application.
+ */
+public interface TwillPreparer {
+
+ /**
+ * Adds a {@link LogHandler} for receiving an application log.
+ * @param handler The {@link LogHandler}.
+ * @return This {@link TwillPreparer}.
+ */
+ TwillPreparer addLogHandler(LogHandler handler);
+
+ /**
+ * Sets the user name that runs the application. Default value is get from {@code "user.name"} by calling
+ * {@link System#getProperty(String)}.
+ * @param user User name
+ * @return This {@link TwillPreparer}.
+ *
+ * @deprecated This method will be removed in future version.
+ */
+ @Deprecated
+ TwillPreparer setUser(String user);
+
+ /**
+ * Sets the list of arguments that will be passed to the application. The arguments can be retrieved
+ * from {@link TwillContext#getApplicationArguments()}.
+ *
+ * @param args Array of arguments.
+ * @return This {@link TwillPreparer}.
+ */
+ TwillPreparer withApplicationArguments(String... args);
+
+ /**
+ * Sets the list of arguments that will be passed to the application. The arguments can be retrieved
+ * from {@link TwillContext#getApplicationArguments()}.
+ *
+ * @param args Iterable of arguments.
+ * @return This {@link TwillPreparer}.
+ */
+ TwillPreparer withApplicationArguments(Iterable<String> args);
+
+ /**
+ * Sets the list of arguments that will be passed to the {@link TwillRunnable} identified by the given name.
+ * The arguments can be retrieved from {@link TwillContext#getArguments()}.
+ *
+ * @param runnableName Name of the {@link TwillRunnable}.
+ * @param args Array of arguments.
+ * @return This {@link TwillPreparer}.
+ */
+ TwillPreparer withArguments(String runnableName, String...args);
+
+ /**
+ * Sets the list of arguments that will be passed to the {@link TwillRunnable} identified by the given name.
+ * The arguments can be retrieved from {@link TwillContext#getArguments()}.
+ *
+ * @param runnableName Name of the {@link TwillRunnable}.
+ * @param args Iterable of arguments.
+ * @return This {@link TwillPreparer}.
+ */
+ TwillPreparer withArguments(String runnableName, Iterable<String> args);
+
+ /**
+ * Adds extra classes that the application is dependent on and is not traceable from the application itself.
+ * @see #withDependencies(Iterable)
+ * @return This {@link TwillPreparer}.
+ */
+ TwillPreparer withDependencies(Class<?>...classes);
+
+ /**
+ * Adds extra classes that the application is dependent on and is not traceable from the application itself.
+ * E.g. Class name used in {@link Class#forName(String)}.
+ * @param classes set of classes to add to dependency list for generating the deployment jar.
+ * @return This {@link TwillPreparer}.
+ */
+ TwillPreparer withDependencies(Iterable<Class<?>> classes);
+
+ /**
+ * Adds resources that will be available through the ClassLoader of the {@link TwillRunnable runnables}.
+ * @see #withResources(Iterable)
+ * @return This {@link TwillPreparer}.
+ */
+ TwillPreparer withResources(URI...resources);
+
+ /**
+ * Adds resources that will be available through the ClassLoader of the {@link TwillRunnable runnables}.
+ * Useful for adding extra resource files or libraries that are not traceable from the application itself.
+ * If the URI is a jar file, classes inside would be loadable by the ClassLoader. If the URI is a directory,
+ * everything underneath would be available.
+ *
+ * @param resources Set of URI to the resources.
+ * @return This {@link TwillPreparer}.
+ */
+ TwillPreparer withResources(Iterable<URI> resources);
+
+ /**
+ * Adds the set of paths to the classpath on the target machine for all runnables.
+ * @see #withClassPaths(Iterable)
+ * @return This {@link TwillPreparer}
+ */
+ TwillPreparer withClassPaths(String... classPaths);
+
+ /**
+ * Adds the set of paths to the classpath on the target machine for all runnables.
+ * Note that the paths would be just added without verification.
+ * @param classPaths Set of classpaths
+ * @return This {@link TwillPreparer}
+ */
+ TwillPreparer withClassPaths(Iterable<String> classPaths);
+
+ /**
+ * Adds security credentials for the runtime environment to gives application access to resources.
+ *
+ * @param secureStore Contains security token available for the runtime environment.
+ * @return This {@link TwillPreparer}.
+ */
+ TwillPreparer addSecureStore(SecureStore secureStore);
+
+ /**
+ * Starts the application.
+ * @return A {@link TwillController} for controlling the running application.
+ */
+ TwillController start();
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/api/TwillRunResources.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/api/TwillRunResources.java b/api/src/main/java/org/apache/twill/api/TwillRunResources.java
new file mode 100644
index 0000000..4c3d2e7
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/api/TwillRunResources.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.api;
+
+/**
+ * Information about the container the {@link TwillRunnable}
+ * is running in.
+ */
+public interface TwillRunResources {
+
+ /**
+ * @return instance id of the runnable.
+ */
+ int getInstanceId();
+
+ /**
+ * @return number of cores the runnable is allowed to use. YARN must be at least v2.1.0 and
+ * it must be configured to use cgroups in order for this to be a reflection of truth.
+ */
+ int getVirtualCores();
+
+ /**
+ * @return amount of memory in MB the runnable is allowed to use.
+ */
+ int getMemoryMB();
+
+ /**
+ * @return the host the runnable is running on.
+ */
+ String getHost();
+
+ /**
+ * @return id of the container the runnable is running in.
+ */
+ String getContainerId();
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/api/TwillRunnable.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/api/TwillRunnable.java b/api/src/main/java/org/apache/twill/api/TwillRunnable.java
new file mode 100644
index 0000000..4350bfb
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/api/TwillRunnable.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.api;
+
+/**
+ * The {@link TwillRunnable} interface should be implemented by any
+ * class whose instances are intended to be executed in a Twill cluster.
+ */
+public interface TwillRunnable extends Runnable {
+
+ /**
+ * Called at submission time. Executed on the client side.
+ * @return A {@link TwillRunnableSpecification} built by {@link TwillRunnableSpecification.Builder}.
+ */
+ TwillRunnableSpecification configure();
+
+ /**
+ * Called when the container process starts. Executed in container machine.
+ * @param context Contains information about the runtime context.
+ */
+ void initialize(TwillContext context);
+
+ /**
+ * Called when a command is received. A normal return denotes the command has been processed successfully, otherwise
+ * {@link Exception} should be thrown.
+ * @param command Contains details of the command.
+ * @throws Exception
+ */
+ void handleCommand(Command command) throws Exception;
+
+ /**
+ * Requests to stop the running service.
+ */
+ void stop();
+
+ /**
+ * Called when the {@link TwillRunnable#run()} completed. Useful for doing
+ * resource cleanup. This method would only get called if the call to {@link #initialize(TwillContext)} was
+ * successful.
+ */
+ void destroy();
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/api/TwillRunnableSpecification.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/api/TwillRunnableSpecification.java b/api/src/main/java/org/apache/twill/api/TwillRunnableSpecification.java
new file mode 100644
index 0000000..bbcc5d7
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/api/TwillRunnableSpecification.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.api;
+
+import org.apache.twill.internal.DefaultTwillRunnableSpecification;
+import com.google.common.collect.ImmutableMap;
+
+import java.util.Map;
+
+/**
+ * Represents a specification of a {@link TwillRunnable}.
+ */
+public interface TwillRunnableSpecification {
+
+ String getClassName();
+
+ String getName();
+
+ Map<String, String> getConfigs();
+
+ /**
+ * Builder for constructing {@link TwillRunnableSpecification}.
+ */
+ static final class Builder {
+
+ private String name;
+ private Map<String, String> args;
+
+ public static NameSetter with() {
+ return new Builder().new NameSetter();
+ }
+
+ public final class NameSetter {
+ public AfterName setName(String name) {
+ Builder.this.name = name;
+ return new AfterName();
+ }
+ }
+
+ public final class AfterName {
+ public AfterConfigs withConfigs(Map<String, String> args) {
+ Builder.this.args = args;
+ return new AfterConfigs();
+ }
+
+ public AfterConfigs noConfigs() {
+ Builder.this.args = ImmutableMap.of();
+ return new AfterConfigs();
+ }
+ }
+
+ public final class AfterConfigs {
+ public TwillRunnableSpecification build() {
+ return new DefaultTwillRunnableSpecification(null, name, args);
+ }
+ }
+
+ private Builder() {
+ }
+ }
+}
[14/15] Initial import commit.
Posted by ch...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/api/TwillRunner.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/api/TwillRunner.java b/api/src/main/java/org/apache/twill/api/TwillRunner.java
new file mode 100644
index 0000000..0393a85
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/api/TwillRunner.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.api;
+
+import org.apache.twill.common.Cancellable;
+
+import java.util.concurrent.TimeUnit;
+
+/**
+ * This interface prepares execution of {@link TwillRunnable} and {@link TwillApplication}.
+ */
+public interface TwillRunner {
+
+ /**
+ * Interface to represents information of a live application.
+ */
+ interface LiveInfo {
+
+ /**
+ * Returns name of the application.
+ * @return Application name as a {@link String}.
+ */
+ String getApplicationName();
+
+ /**
+ * Returns {@link TwillController}s for all live instances of the application.
+ * @return An {@link Iterable} of {@link TwillController}.
+ */
+ Iterable<TwillController> getControllers();
+ }
+
+ /**
+ * Prepares to run the given {@link TwillRunnable} with {@link ResourceSpecification#BASIC} resource specification.
+ * @param runnable The runnable to run through Twill when {@link TwillPreparer#start()} is called.
+ * @return A {@link TwillPreparer} for setting up runtime options.
+ */
+ TwillPreparer prepare(TwillRunnable runnable);
+
+ /**
+ * Prepares to run the given {@link TwillRunnable} with the given resource specification.
+ * @param runnable The runnable to run through Twill when {@link TwillPreparer#start()} is called.
+ * @param resourceSpecification The resource specification for running the runnable.
+ * @return A {@link TwillPreparer} for setting up runtime options.
+ */
+ TwillPreparer prepare(TwillRunnable runnable, ResourceSpecification resourceSpecification);
+
+ /**
+ * Prepares to run the given {@link TwillApplication} as specified by the application.
+ * @param application The application to run through Twill when {@link TwillPreparer#start()} is called.
+ * @return A {@link TwillPreparer} for setting up runtime options.
+ */
+ TwillPreparer prepare(TwillApplication application);
+
+ /**
+ * Gets a {@link TwillController} for the given application and runId.
+ * @param applicationName Name of the application.
+ * @param runId The runId of the running application.
+ * @return A {@link TwillController} to interact with the application or null if no such runId is found.
+ */
+ TwillController lookup(String applicationName, RunId runId);
+
+ /**
+ * Gets an {@link Iterable} of {@link TwillController} for all running instances of the given application.
+ * @param applicationName Name of the application.
+ * @return A live {@link Iterable} that gives the latest {@link TwillController} set for all running
+ * instances of the application when {@link Iterable#iterator()} is invoked.
+ */
+ Iterable<TwillController> lookup(String applicationName);
+
+ /**
+ * Gets an {@link Iterable} of {@link LiveInfo}.
+ * @return A live {@link Iterable} that gives the latest information on the set of applications that
+ * have running instances when {@link Iterable#iterator()}} is invoked.
+ */
+ Iterable<LiveInfo> lookupLive();
+
+ /**
+ * Schedules a periodic update of SecureStore. The first call to the given {@link SecureStoreUpdater} will be made
+ * after {@code initialDelay}, and subsequently with the given {@code delay} between completion of one update
+ * and starting of the next. If exception is thrown on call
+ * {@link SecureStoreUpdater#update(String, RunId)}, the exception will only get logged
+ * and won't suppress the next update call.
+ *
+ * @param updater A {@link SecureStoreUpdater} for creating new SecureStore.
+ * @param initialDelay Delay before the first call to update method.
+ * @param delay Delay between completion of one update call to the next one.
+ * @param unit time unit for the initialDelay and delay.
+ * @return A {@link Cancellable} for cancelling the scheduled update.
+ */
+ Cancellable scheduleSecureStoreUpdate(final SecureStoreUpdater updater,
+ long initialDelay, long delay, TimeUnit unit);
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/api/TwillRunnerService.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/api/TwillRunnerService.java b/api/src/main/java/org/apache/twill/api/TwillRunnerService.java
new file mode 100644
index 0000000..76ec136
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/api/TwillRunnerService.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.api;
+
+import com.google.common.util.concurrent.Service;
+
+/**
+ * A {@link TwillRunner} that extends {@link Service} to provide lifecycle management functions.
+ * The {@link #start()} method needs to be called before calling any other method of this interface.
+ * When done with this service, call {@link #stop()} to release any resources that it holds.
+ */
+public interface TwillRunnerService extends TwillRunner, Service {
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/api/TwillSpecification.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/api/TwillSpecification.java b/api/src/main/java/org/apache/twill/api/TwillSpecification.java
new file mode 100644
index 0000000..00d171d
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/api/TwillSpecification.java
@@ -0,0 +1,327 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.api;
+
+import org.apache.twill.internal.DefaultLocalFile;
+import org.apache.twill.internal.DefaultRuntimeSpecification;
+import org.apache.twill.internal.DefaultTwillRunnableSpecification;
+import org.apache.twill.internal.DefaultTwillSpecification;
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+
+import javax.annotation.Nullable;
+import java.io.File;
+import java.net.URI;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Represents specification of a {@link TwillApplication}.
+ */
+public interface TwillSpecification {
+
+ /**
+ * Defines execution order.
+ */
+ interface Order {
+
+ enum Type {
+ STARTED,
+ COMPLETED
+ }
+
+ /**
+ * @return Set of {@link TwillRunnable} name that belongs to this order.
+ */
+ Set<String> getNames();
+
+ Type getType();
+ }
+
+ /**
+ * @return Name of the application.
+ */
+ String getName();
+
+ /**
+ * @return A map from {@link TwillRunnable} name to {@link RuntimeSpecification}.
+ */
+ Map<String, RuntimeSpecification> getRunnables();
+
+ /**
+ * @return Returns a list of runnable names that should be executed in the given order.
+ */
+ List<Order> getOrders();
+
+ /**
+ * @return The {@link EventHandlerSpecification} for the {@link EventHandler} to be used for this application,
+ * or {@code null} if no event handler has been provided.
+ */
+ @Nullable
+ EventHandlerSpecification getEventHandler();
+
+ /**
+ * Builder for constructing instance of {@link TwillSpecification}.
+ */
+ static final class Builder {
+
+ private String name;
+ private Map<String, RuntimeSpecification> runnables = Maps.newHashMap();
+ private List<Order> orders = Lists.newArrayList();
+ private EventHandlerSpecification eventHandler;
+
+ public static NameSetter with() {
+ return new Builder().new NameSetter();
+ }
+
+ public final class NameSetter {
+ public AfterName setName(String name) {
+ Builder.this.name = name;
+ return new AfterName();
+ }
+ }
+
+ public final class AfterName {
+ public MoreRunnable withRunnable() {
+ return new RunnableSetter();
+ }
+ }
+
+ public interface MoreRunnable {
+ RuntimeSpecificationAdder add(TwillRunnable runnable);
+
+ RuntimeSpecificationAdder add(TwillRunnable runnable, ResourceSpecification resourceSpec);
+
+ /**
+ * Adds a {@link TwillRunnable} with {@link ResourceSpecification#BASIC} resource specification.
+ * @param name Name of runnable
+ * @param runnable {@link TwillRunnable} to be run
+ * @return instance of {@link RuntimeSpecificationAdder}
+ */
+ RuntimeSpecificationAdder add(String name, TwillRunnable runnable);
+
+ RuntimeSpecificationAdder add(String name, TwillRunnable runnable, ResourceSpecification resourceSpec);
+ }
+
+ public interface AfterRunnable {
+ FirstOrder withOrder();
+
+ AfterOrder anyOrder();
+ }
+
+ public final class RunnableSetter implements MoreRunnable, AfterRunnable {
+
+ @Override
+ public RuntimeSpecificationAdder add(TwillRunnable runnable) {
+ return add(runnable.configure().getName(), runnable);
+ }
+
+ @Override
+ public RuntimeSpecificationAdder add(TwillRunnable runnable, ResourceSpecification resourceSpec) {
+ return add(runnable.configure().getName(), runnable, resourceSpec);
+ }
+
+ @Override
+ public RuntimeSpecificationAdder add(String name, TwillRunnable runnable) {
+ return add(name, runnable, ResourceSpecification.BASIC);
+ }
+
+ @Override
+ public RuntimeSpecificationAdder add(String name, TwillRunnable runnable,
+ final ResourceSpecification resourceSpec) {
+ final TwillRunnableSpecification spec = new DefaultTwillRunnableSpecification(
+ runnable.getClass().getName(), name, runnable.configure().getConfigs());
+ return new RuntimeSpecificationAdder(new Function<Collection<LocalFile>, RunnableSetter>() {
+ @Override
+ public RunnableSetter apply(Collection<LocalFile> files) {
+ runnables.put(spec.getName(), new DefaultRuntimeSpecification(spec.getName(), spec, resourceSpec, files));
+ return RunnableSetter.this;
+ }
+ });
+ }
+
+ @Override
+ public FirstOrder withOrder() {
+ return new OrderSetter();
+ }
+
+ @Override
+ public AfterOrder anyOrder() {
+ return new OrderSetter();
+ }
+ }
+
+ /**
+ * For setting runtime specific settings.
+ */
+ public final class RuntimeSpecificationAdder {
+
+ private final Function<Collection<LocalFile>, RunnableSetter> completer;
+
+ RuntimeSpecificationAdder(Function<Collection<LocalFile>, RunnableSetter> completer) {
+ this.completer = completer;
+ }
+
+ public LocalFileAdder withLocalFiles() {
+ return new MoreFile(completer);
+ }
+
+ public RunnableSetter noLocalFiles() {
+ return completer.apply(ImmutableList.<LocalFile>of());
+ }
+ }
+
+ public interface LocalFileAdder {
+ MoreFile add(String name, File file);
+
+ MoreFile add(String name, URI uri);
+
+ MoreFile add(String name, File file, boolean archive);
+
+ MoreFile add(String name, URI uri, boolean archive);
+
+ MoreFile add(String name, File file, String pattern);
+
+ MoreFile add(String name, URI uri, String pattern);
+ }
+
+ public final class MoreFile implements LocalFileAdder {
+
+ private final Function<Collection<LocalFile>, RunnableSetter> completer;
+ private final List<LocalFile> files = Lists.newArrayList();
+
+ public MoreFile(Function<Collection<LocalFile>, RunnableSetter> completer) {
+ this.completer = completer;
+ }
+
+ @Override
+ public MoreFile add(String name, File file) {
+ return add(name, file, false);
+ }
+
+ @Override
+ public MoreFile add(String name, URI uri) {
+ return add(name, uri, false);
+ }
+
+ @Override
+ public MoreFile add(String name, File file, boolean archive) {
+ return add(name, file.toURI(), archive);
+ }
+
+ @Override
+ public MoreFile add(String name, URI uri, boolean archive) {
+ files.add(new DefaultLocalFile(name, uri, -1, -1, archive, null));
+ return this;
+ }
+
+ @Override
+ public MoreFile add(String name, File file, String pattern) {
+ return add(name, file.toURI(), pattern);
+ }
+
+ @Override
+ public MoreFile add(String name, URI uri, String pattern) {
+ files.add(new DefaultLocalFile(name, uri, -1, -1, true, pattern));
+ return this;
+ }
+
+ public RunnableSetter apply() {
+ return completer.apply(files);
+ }
+ }
+
+ public interface FirstOrder {
+ NextOrder begin(String name, String...names);
+ }
+
+ public interface NextOrder extends AfterOrder {
+ NextOrder nextWhenStarted(String name, String...names);
+
+ NextOrder nextWhenCompleted(String name, String...names);
+ }
+
+ public interface AfterOrder {
+ AfterOrder withEventHandler(EventHandler handler);
+
+ TwillSpecification build();
+ }
+
+ public final class OrderSetter implements FirstOrder, NextOrder {
+ @Override
+ public NextOrder begin(String name, String... names) {
+ addOrder(Order.Type.STARTED, name, names);
+ return this;
+ }
+
+ @Override
+ public NextOrder nextWhenStarted(String name, String... names) {
+ addOrder(Order.Type.STARTED, name, names);
+ return this;
+ }
+
+ @Override
+ public NextOrder nextWhenCompleted(String name, String... names) {
+ addOrder(Order.Type.COMPLETED, name, names);
+ return this;
+ }
+
+ @Override
+ public AfterOrder withEventHandler(EventHandler handler) {
+ eventHandler = handler.configure();
+ return this;
+ }
+
+ @Override
+ public TwillSpecification build() {
+ // Set to track with runnable hasn't been assigned an order.
+ Set<String> runnableNames = Sets.newHashSet(runnables.keySet());
+ for (Order order : orders) {
+ runnableNames.removeAll(order.getNames());
+ }
+
+ // For all unordered runnables, add it to the end of orders list
+ orders.add(new DefaultTwillSpecification.DefaultOrder(runnableNames, Order.Type.STARTED));
+
+ return new DefaultTwillSpecification(name, runnables, orders, eventHandler);
+ }
+
+ private void addOrder(final Order.Type type, String name, String...names) {
+ Preconditions.checkArgument(name != null, "Name cannot be null.");
+ Preconditions.checkArgument(runnables.containsKey(name), "Runnable not exists.");
+
+ Set<String> runnableNames = Sets.newHashSet(name);
+ for (String runnableName : names) {
+ Preconditions.checkArgument(name != null, "Name cannot be null.");
+ Preconditions.checkArgument(runnables.containsKey(name), "Runnable not exists.");
+ runnableNames.add(runnableName);
+ }
+
+ orders.add(new DefaultTwillSpecification.DefaultOrder(runnableNames, type));
+ }
+ }
+
+ private Builder() {}
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/api/logging/LogEntry.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/api/logging/LogEntry.java b/api/src/main/java/org/apache/twill/api/logging/LogEntry.java
new file mode 100644
index 0000000..4995328
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/api/logging/LogEntry.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.api.logging;
+
+/**
+ * Represents a log entry emitted by application.
+ */
+public interface LogEntry {
+
+ /**
+ * Log level.
+ */
+ enum Level {
+ FATAL,
+ ERROR,
+ WARN,
+ INFO,
+ DEBUG,
+ TRACE
+ }
+
+ String getLoggerName();
+
+ String getHost();
+
+ long getTimestamp();
+
+ Level getLogLevel();
+
+ String getSourceClassName();
+
+ String getSourceMethodName();
+
+ String getFileName();
+
+ int getLineNumber();
+
+ String getThreadName();
+
+ String getMessage();
+
+ StackTraceElement[] getStackTraces();
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/api/logging/LogHandler.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/api/logging/LogHandler.java b/api/src/main/java/org/apache/twill/api/logging/LogHandler.java
new file mode 100644
index 0000000..afded19
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/api/logging/LogHandler.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.api.logging;
+
+/**
+ *
+ */
+public interface LogHandler {
+
+ void onLog(LogEntry logEntry);
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/api/logging/PrinterLogHandler.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/api/logging/PrinterLogHandler.java b/api/src/main/java/org/apache/twill/api/logging/PrinterLogHandler.java
new file mode 100644
index 0000000..71a2bca
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/api/logging/PrinterLogHandler.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.api.logging;
+
+import com.google.common.base.Splitter;
+
+import java.io.PrintWriter;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.Formatter;
+import java.util.TimeZone;
+
+/**
+ * A {@link LogHandler} that prints the {@link LogEntry} through a {@link PrintWriter}.
+ */
+public final class PrinterLogHandler implements LogHandler {
+
+ private static final ThreadLocal<DateFormat> DATE_FORMAT = new ThreadLocal<DateFormat>() {
+ @Override
+ protected DateFormat initialValue() {
+ DateFormat format = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss,SSS'Z'");
+ format.setTimeZone(TimeZone.getTimeZone("UTC"));
+ return format;
+ }
+ };
+
+ private final PrintWriter writer;
+ private final Formatter formatter;
+
+ /**
+ * Creates a {@link PrinterLogHandler} which has {@link LogEntry} written to the given {@link PrintWriter}.
+ * @param writer The write that log entries will write to.
+ */
+ public PrinterLogHandler(PrintWriter writer) {
+ this.writer = writer;
+ this.formatter = new Formatter(writer);
+ }
+
+ @Override
+ public void onLog(LogEntry logEntry) {
+ String utc = timestampToUTC(logEntry.getTimestamp());
+
+ formatter.format("%s %-5s %s [%s] [%s] %s:%s(%s:%d) - %s\n",
+ utc,
+ logEntry.getLogLevel().name(),
+ getShortenLoggerName(logEntry.getLoggerName()),
+ logEntry.getHost(),
+ logEntry.getThreadName(),
+ getSimpleClassName(logEntry.getSourceClassName()),
+ logEntry.getSourceMethodName(),
+ logEntry.getFileName(),
+ logEntry.getLineNumber(),
+ logEntry.getMessage());
+ formatter.flush();
+
+ StackTraceElement[] stackTraces = logEntry.getStackTraces();
+ if (stackTraces != null) {
+ for (StackTraceElement stackTrace : stackTraces) {
+ writer.append("\tat ").append(stackTrace.toString());
+ writer.println();
+ }
+ writer.flush();
+ }
+ }
+
+ private String timestampToUTC(long timestamp) {
+ return DATE_FORMAT.get().format(new Date(timestamp));
+ }
+
+ private String getShortenLoggerName(String loggerName) {
+ StringBuilder builder = new StringBuilder();
+ String previous = null;
+ for (String part : Splitter.on('.').split(loggerName)) {
+ if (previous != null) {
+ builder.append(previous.charAt(0)).append('.');
+ }
+ previous = part;
+ }
+ return builder.append(previous).toString();
+ }
+
+ private String getSimpleClassName(String className) {
+ return className.substring(className.lastIndexOf('.') + 1);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/api/logging/package-info.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/api/logging/package-info.java b/api/src/main/java/org/apache/twill/api/logging/package-info.java
new file mode 100644
index 0000000..e325c18
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/api/logging/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This package contains class for handling logging events.
+ */
+package org.apache.twill.api.logging;
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/api/package-info.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/api/package-info.java b/api/src/main/java/org/apache/twill/api/package-info.java
new file mode 100644
index 0000000..5d9df6b
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/api/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * Classes in this package provides core functionality of the Twill library.
+ */
+package org.apache.twill.api;
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/internal/DefaultEventHandlerSpecification.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/internal/DefaultEventHandlerSpecification.java b/api/src/main/java/org/apache/twill/internal/DefaultEventHandlerSpecification.java
new file mode 100644
index 0000000..df21400
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/internal/DefaultEventHandlerSpecification.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal;
+
+import org.apache.twill.api.EventHandlerSpecification;
+import org.apache.twill.api.EventHandler;
+import org.apache.twill.api.EventHandlerSpecification;
+import org.apache.twill.api.EventHandlerSpecification;
+import com.google.common.collect.ImmutableMap;
+import org.apache.twill.api.EventHandlerSpecification;
+
+import java.util.Map;
+
+/**
+ *
+ */
+public class DefaultEventHandlerSpecification implements EventHandlerSpecification {
+
+ private final String className;
+ private final Map<String, String> configs;
+
+ public DefaultEventHandlerSpecification(String className, Map<String, String> configs) {
+ this.className = className;
+ this.configs = configs;
+ }
+
+ public DefaultEventHandlerSpecification(EventHandler eventHandler) {
+ EventHandlerSpecification spec = eventHandler.configure();
+ this.className = eventHandler.getClass().getName();
+ this.configs = ImmutableMap.copyOf(spec.getConfigs());
+ }
+
+ @Override
+ public String getClassName() {
+ return className;
+ }
+
+ @Override
+ public Map<String, String> getConfigs() {
+ return configs;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/internal/DefaultLocalFile.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/internal/DefaultLocalFile.java b/api/src/main/java/org/apache/twill/internal/DefaultLocalFile.java
new file mode 100644
index 0000000..e43c0c0
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/internal/DefaultLocalFile.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal;
+
+import org.apache.twill.api.LocalFile;
+
+import javax.annotation.Nullable;
+import java.net.URI;
+
+/**
+ * A straightforward implementation of {@link LocalFile}.
+ */
+public final class DefaultLocalFile implements LocalFile {
+
+ private final String name;
+ private final URI uri;
+ private final long lastModified;
+ private final long size;
+ private final boolean archive;
+ private final String pattern;
+
+ public DefaultLocalFile(String name, URI uri, long lastModified,
+ long size, boolean archive, @Nullable String pattern) {
+ this.name = name;
+ this.uri = uri;
+ this.lastModified = lastModified;
+ this.size = size;
+ this.archive = archive;
+ this.pattern = pattern;
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public URI getURI() {
+ return uri;
+ }
+
+ @Override
+ public long getLastModified() {
+ return lastModified;
+ }
+
+ @Override
+ public long getSize() {
+ return size;
+ }
+
+ @Override
+ public boolean isArchive() {
+ return archive;
+ }
+
+ @Override
+ public String getPattern() {
+ return pattern;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/internal/DefaultResourceReport.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/internal/DefaultResourceReport.java b/api/src/main/java/org/apache/twill/internal/DefaultResourceReport.java
new file mode 100644
index 0000000..c4c8a29
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/internal/DefaultResourceReport.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal;
+
+import org.apache.twill.api.ResourceReport;
+import org.apache.twill.api.TwillRunResources;
+import com.google.common.collect.HashMultimap;
+import com.google.common.collect.Multimaps;
+import com.google.common.collect.SetMultimap;
+
+import java.util.Collection;
+import java.util.Map;
+
+/**
+ * Implementation of {@link org.apache.twill.api.ResourceReport} with some
+ * additional methods for maintaining the report.
+ */
+public final class DefaultResourceReport implements ResourceReport {
+ private final SetMultimap<String, TwillRunResources> usedResources;
+ private final TwillRunResources appMasterResources;
+ private final String applicationId;
+
+ public DefaultResourceReport(String applicationId, TwillRunResources masterResources) {
+ this.applicationId = applicationId;
+ this.appMasterResources = masterResources;
+ this.usedResources = HashMultimap.create();
+ }
+
+ public DefaultResourceReport(String applicationId, TwillRunResources masterResources,
+ Map<String, Collection<TwillRunResources>> resources) {
+ this.applicationId = applicationId;
+ this.appMasterResources = masterResources;
+ this.usedResources = HashMultimap.create();
+ for (Map.Entry<String, Collection<TwillRunResources>> entry : resources.entrySet()) {
+ this.usedResources.putAll(entry.getKey(), entry.getValue());
+ }
+ }
+
+ /**
+ * Add resources used by an instance of the runnable.
+ *
+ * @param runnableName name of runnable.
+ * @param resources resources to add.
+ */
+ public void addRunResources(String runnableName, TwillRunResources resources) {
+ usedResources.put(runnableName, resources);
+ }
+
+ /**
+ * Remove the resource corresponding to the given runnable and container.
+ *
+ * @param runnableName name of runnable.
+ * @param containerId container id of the runnable.
+ */
+ public void removeRunnableResources(String runnableName, String containerId) {
+ TwillRunResources toRemove = null;
+ // could be faster if usedResources was a Table, but that makes returning the
+ // report a little more complex, and this does not need to be terribly fast.
+ for (TwillRunResources resources : usedResources.get(runnableName)) {
+ if (resources.getContainerId().equals(containerId)) {
+ toRemove = resources;
+ break;
+ }
+ }
+ usedResources.remove(runnableName, toRemove);
+ }
+
+ /**
+ * Get all the run resources being used by all instances of the specified runnable.
+ *
+ * @param runnableName the runnable name.
+ * @return resources being used by all instances of the runnable.
+ */
+ @Override
+ public Collection<TwillRunResources> getRunnableResources(String runnableName) {
+ return usedResources.get(runnableName);
+ }
+
+ /**
+ * Get all the run resources being used across all runnables.
+ *
+ * @return all run resources used by all instances of all runnables.
+ */
+ @Override
+ public Map<String, Collection<TwillRunResources>> getResources() {
+ return Multimaps.unmodifiableSetMultimap(usedResources).asMap();
+ }
+
+ /**
+ * Get the resources application master is using.
+ *
+ * @return resources being used by the application master.
+ */
+ @Override
+ public TwillRunResources getAppMasterResources() {
+ return appMasterResources;
+ }
+
+ /**
+ * Get the id of the application master.
+ *
+ * @return id of the application master.
+ */
+ @Override
+ public String getApplicationId() {
+ return applicationId;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/internal/DefaultResourceSpecification.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/internal/DefaultResourceSpecification.java b/api/src/main/java/org/apache/twill/internal/DefaultResourceSpecification.java
new file mode 100644
index 0000000..1327ce5
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/internal/DefaultResourceSpecification.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal;
+
+import org.apache.twill.api.ResourceSpecification;
+
+/**
+ * Straightforward implementation of {@link org.apache.twill.api.ResourceSpecification}.
+ */
+public final class DefaultResourceSpecification implements ResourceSpecification {
+ private final int virtualCores;
+ private final int memorySize;
+ private final int instances;
+ private final int uplink;
+ private final int downlink;
+
+ public DefaultResourceSpecification(int virtualCores, int memorySize, int instances, int uplink, int downlink) {
+ this.virtualCores = virtualCores;
+ this.memorySize = memorySize;
+ this.instances = instances;
+ this.uplink = uplink;
+ this.downlink = downlink;
+ }
+
+ @Deprecated
+ @Override
+ public int getCores() {
+ return virtualCores;
+ }
+
+ @Override
+ public int getVirtualCores() {
+ return virtualCores;
+ }
+
+ @Override
+ public int getMemorySize() {
+ return memorySize;
+ }
+
+ @Override
+ public int getInstances() {
+ return instances;
+ }
+
+ @Override
+ public int getUplink() {
+ return uplink;
+ }
+
+ @Override
+ public int getDownlink() {
+ return downlink;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/internal/DefaultRuntimeSpecification.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/internal/DefaultRuntimeSpecification.java b/api/src/main/java/org/apache/twill/internal/DefaultRuntimeSpecification.java
new file mode 100644
index 0000000..c4f496e
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/internal/DefaultRuntimeSpecification.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal;
+
+import org.apache.twill.api.LocalFile;
+import org.apache.twill.api.ResourceSpecification;
+import org.apache.twill.api.RuntimeSpecification;
+import org.apache.twill.api.TwillRunnableSpecification;
+import com.google.common.collect.ImmutableList;
+
+import java.util.Collection;
+
+/**
+ * Straightforward implementation of {@link RuntimeSpecification}.
+ */
+public final class DefaultRuntimeSpecification implements RuntimeSpecification {
+
+ private final String name;
+ private final TwillRunnableSpecification runnableSpec;
+ private final ResourceSpecification resourceSpec;
+ private final Collection<LocalFile> localFiles;
+
+ public DefaultRuntimeSpecification(String name,
+ TwillRunnableSpecification runnableSpec,
+ ResourceSpecification resourceSpec,
+ Collection<LocalFile> localFiles) {
+ this.name = name;
+ this.runnableSpec = runnableSpec;
+ this.resourceSpec = resourceSpec;
+ this.localFiles = ImmutableList.copyOf(localFiles);
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public TwillRunnableSpecification getRunnableSpecification() {
+ return runnableSpec;
+ }
+
+ @Override
+ public ResourceSpecification getResourceSpecification() {
+ return resourceSpec;
+ }
+
+ @Override
+ public Collection<LocalFile> getLocalFiles() {
+ return localFiles;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/internal/DefaultTwillRunResources.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/internal/DefaultTwillRunResources.java b/api/src/main/java/org/apache/twill/internal/DefaultTwillRunResources.java
new file mode 100644
index 0000000..bd8f8f5
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/internal/DefaultTwillRunResources.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal;
+
+import org.apache.twill.api.TwillRunResources;
+
+/**
+ * Straightforward implementation of {@link org.apache.twill.api.TwillRunResources}.
+ */
+public class DefaultTwillRunResources implements TwillRunResources {
+ private final String containerId;
+ private final int instanceId;
+ private final int virtualCores;
+ private final int memoryMB;
+ private final String host;
+
+ public DefaultTwillRunResources(int instanceId, String containerId,
+ int cores, int memoryMB, String host) {
+ this.instanceId = instanceId;
+ this.containerId = containerId;
+ this.virtualCores = cores;
+ this.memoryMB = memoryMB;
+ this.host = host;
+ }
+
+ /**
+ * @return instance id of the runnable.
+ */
+ @Override
+ public int getInstanceId() {
+ return instanceId;
+ }
+
+ /**
+ * @return id of the container the runnable is running in.
+ */
+ @Override
+ public String getContainerId() {
+ return containerId;
+ }
+
+ /**
+ * @return number of cores the runnable is allowed to use. YARN must be at least v2.1.0 and
+ * it must be configured to use cgroups in order for this to be a reflection of truth.
+ */
+ @Override
+ public int getVirtualCores() {
+ return virtualCores;
+ }
+
+ /**
+ * @return amount of memory in MB the runnable is allowed to use.
+ */
+ @Override
+ public int getMemoryMB() {
+ return memoryMB;
+ }
+
+ /**
+ * @return the host the runnable is running on.
+ */
+ @Override
+ public String getHost() {
+ return host;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof TwillRunResources)) {
+ return false;
+ }
+ TwillRunResources other = (TwillRunResources) o;
+ return (instanceId == other.getInstanceId()) &&
+ containerId.equals(other.getContainerId()) &&
+ host.equals(other.getHost()) &&
+ (virtualCores == other.getVirtualCores()) &&
+ (memoryMB == other.getMemoryMB());
+ }
+
+ @Override
+ public int hashCode() {
+ int hash = 17;
+ hash = 31 * hash + containerId.hashCode();
+ hash = 31 * hash + host.hashCode();
+ hash = 31 * hash + (int) (instanceId ^ (instanceId >>> 32));
+ hash = 31 * hash + (int) (virtualCores ^ (virtualCores >>> 32));
+ hash = 31 * hash + (int) (memoryMB ^ (memoryMB >>> 32));
+ return hash;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/internal/DefaultTwillRunnableSpecification.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/internal/DefaultTwillRunnableSpecification.java b/api/src/main/java/org/apache/twill/internal/DefaultTwillRunnableSpecification.java
new file mode 100644
index 0000000..14ea7f5
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/internal/DefaultTwillRunnableSpecification.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal;
+
+import org.apache.twill.api.TwillRunnableSpecification;
+import com.google.common.collect.ImmutableMap;
+
+import java.util.Map;
+
+/**
+ * Straightforward implementation of {@link org.apache.twill.api.TwillRunnableSpecification}.
+ */
+public final class DefaultTwillRunnableSpecification implements TwillRunnableSpecification {
+
+ private final String className;
+ private final String name;
+ private final Map<String, String> arguments;
+
+ public DefaultTwillRunnableSpecification(String className, String name, Map<String, String> arguments) {
+ this.className = className;
+ this.name = name;
+ this.arguments = ImmutableMap.copyOf(arguments);
+ }
+
+ public DefaultTwillRunnableSpecification(String className, TwillRunnableSpecification other) {
+ this.className = className;
+ this.name = other.getName();
+ this.arguments = ImmutableMap.copyOf(other.getConfigs());
+ }
+
+ @Override
+ public String getClassName() {
+ return className;
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public Map<String, String> getConfigs() {
+ return arguments;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/internal/DefaultTwillSpecification.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/internal/DefaultTwillSpecification.java b/api/src/main/java/org/apache/twill/internal/DefaultTwillSpecification.java
new file mode 100644
index 0000000..6bb2b15
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/internal/DefaultTwillSpecification.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal;
+
+import org.apache.twill.api.EventHandlerSpecification;
+import org.apache.twill.api.RuntimeSpecification;
+import org.apache.twill.api.TwillSpecification;
+import com.google.common.base.Objects;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+
+import javax.annotation.Nullable;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * Straightforward implementation of {@link org.apache.twill.api.TwillSpecification}.
+ */
+public final class DefaultTwillSpecification implements TwillSpecification {
+
+ private final String name;
+ private final Map<String, RuntimeSpecification> runnables;
+ private final List<Order> orders;
+ private final EventHandlerSpecification eventHandler;
+
+ public DefaultTwillSpecification(String name, Map<String, RuntimeSpecification> runnables,
+ List<Order> orders, EventHandlerSpecification eventHandler) {
+ this.name = name;
+ this.runnables = ImmutableMap.copyOf(runnables);
+ this.orders = ImmutableList.copyOf(orders);
+ this.eventHandler = eventHandler;
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public Map<String, RuntimeSpecification> getRunnables() {
+ return runnables;
+ }
+
+ @Override
+ public List<Order> getOrders() {
+ return orders;
+ }
+
+ @Nullable
+ @Override
+ public EventHandlerSpecification getEventHandler() {
+ return eventHandler;
+ }
+
+ /**
+ * Straightforward implementation of {@link Order}.
+ */
+ public static final class DefaultOrder implements Order {
+
+ private final Set<String> names;
+ private final Type type;
+
+ public DefaultOrder(Iterable<String> names, Type type) {
+ this.names = ImmutableSet.copyOf(names);
+ this.type = type;
+ }
+
+ @Override
+ public Set<String> getNames() {
+ return names;
+ }
+
+ @Override
+ public Type getType() {
+ return type;
+ }
+
+ @Override
+ public String toString() {
+ return Objects.toStringHelper(this)
+ .add("names", names)
+ .add("type", type)
+ .toString();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/internal/RunIds.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/internal/RunIds.java b/api/src/main/java/org/apache/twill/internal/RunIds.java
new file mode 100644
index 0000000..7249d81
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/internal/RunIds.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.internal;
+
+import org.apache.twill.api.RunId;
+import com.google.common.base.Preconditions;
+
+import java.util.UUID;
+
+/**
+ * Factory class for creating instance of {@link org.apache.twill.api.RunId}.
+ */
+public final class RunIds {
+
+ public static RunId generate() {
+ return new RunIdImpl(UUID.randomUUID().toString());
+ }
+
+ public static RunId fromString(String str) {
+ return new RunIdImpl(str);
+ }
+
+ private RunIds() {
+ }
+
+ private static final class RunIdImpl implements RunId {
+
+ final String id;
+
+ private RunIdImpl(String id) {
+ Preconditions.checkArgument(id != null, "RunId cannot be null.");
+ this.id = id;
+ }
+
+ @Override
+ public String getId() {
+ return id;
+ }
+
+ @Override
+ public String toString() {
+ return getId();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || !(other instanceof RunId)) {
+ return false;
+ }
+ return id.equals(((RunId)other).getId());
+ }
+
+ @Override
+ public int hashCode() {
+ return id.hashCode();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/api/src/main/java/org/apache/twill/internal/package-info.java
----------------------------------------------------------------------
diff --git a/api/src/main/java/org/apache/twill/internal/package-info.java b/api/src/main/java/org/apache/twill/internal/package-info.java
new file mode 100644
index 0000000..8af8362
--- /dev/null
+++ b/api/src/main/java/org/apache/twill/internal/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Internal classes for Twill API.
+ */
+package org.apache.twill.internal;
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/common/pom.xml
----------------------------------------------------------------------
diff --git a/common/pom.xml b/common/pom.xml
new file mode 100644
index 0000000..df93fd7
--- /dev/null
+++ b/common/pom.xml
@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <artifactId>twill-parent</artifactId>
+ <groupId>org.apache.twill</groupId>
+ <version>1.3.0-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>twill-common</artifactId>
+ <name>Twill common library</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>junit</groupId>
+ <artifactId>junit</artifactId>
+ </dependency>
+ </dependencies>
+</project>
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/common/src/main/java/org/apache/twill/common/Cancellable.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/twill/common/Cancellable.java b/common/src/main/java/org/apache/twill/common/Cancellable.java
new file mode 100644
index 0000000..08f22d3
--- /dev/null
+++ b/common/src/main/java/org/apache/twill/common/Cancellable.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.twill.common;
+
+/**
+ * Something, usually a task, that can be cancelled. Cancellation is performed by the cancel method.
+ */
+public interface Cancellable {
+ /**
+ * Attempts to cancel execution of this task.
+ */
+ void cancel();
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/common/src/main/java/org/apache/twill/common/ServiceListenerAdapter.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/twill/common/ServiceListenerAdapter.java b/common/src/main/java/org/apache/twill/common/ServiceListenerAdapter.java
new file mode 100644
index 0000000..527ba7d
--- /dev/null
+++ b/common/src/main/java/org/apache/twill/common/ServiceListenerAdapter.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.common;
+
+import com.google.common.util.concurrent.Service;
+
+/**
+ * An adapter for implementing {@link Service.Listener} with all method default to no-op.
+ */
+public abstract class ServiceListenerAdapter implements Service.Listener {
+ @Override
+ public void starting() {
+ // No-op
+ }
+
+ @Override
+ public void running() {
+ // No-op
+ }
+
+ @Override
+ public void stopping(Service.State from) {
+ // No-op
+ }
+
+ @Override
+ public void terminated(Service.State from) {
+ // No-op
+ }
+
+ @Override
+ public void failed(Service.State from, Throwable failure) {
+ // No-op
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/common/src/main/java/org/apache/twill/common/Services.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/twill/common/Services.java b/common/src/main/java/org/apache/twill/common/Services.java
new file mode 100644
index 0000000..7e294f0
--- /dev/null
+++ b/common/src/main/java/org/apache/twill/common/Services.java
@@ -0,0 +1,140 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.common;
+
+import com.google.common.collect.Lists;
+import com.google.common.util.concurrent.Futures;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.Service;
+import com.google.common.util.concurrent.SettableFuture;
+
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/**
+ * Utility methods for help dealing with {@link Service}.
+ */
+public final class Services {
+
+ /**
+ * Starts a list of {@link Service} one by one. Starting of next Service is triggered from the callback listener
+ * thread of the previous Service.
+ *
+ * @param firstService First service to start.
+ * @param moreServices The rest services to start.
+ * @return A {@link ListenableFuture} that will be completed when all services are started, with the
+ * result carries the completed {@link ListenableFuture} of each corresponding service in the
+ * same order as they are passed to this method.
+ */
+ public static ListenableFuture<List<ListenableFuture<Service.State>>> chainStart(Service firstService,
+ Service...moreServices) {
+ return doChain(true, firstService, moreServices);
+ }
+
+ /**
+ * Stops a list of {@link Service} one by one. It behaves the same as
+ * {@link #chainStart(com.google.common.util.concurrent.Service, com.google.common.util.concurrent.Service...)}
+ * except {@link com.google.common.util.concurrent.Service#stop()} is called instead of start.
+ *
+ * @param firstService First service to stop.
+ * @param moreServices The rest services to stop.
+ * @return A {@link ListenableFuture} that will be completed when all services are stopped.
+ * @see #chainStart(com.google.common.util.concurrent.Service, com.google.common.util.concurrent.Service...)
+ */
+ public static ListenableFuture<List<ListenableFuture<Service.State>>> chainStop(Service firstService,
+ Service...moreServices) {
+ return doChain(false, firstService, moreServices);
+ }
+
+ /**
+ * Returns a {@link ListenableFuture} that will be completed when the given service is stopped. If the service
+ * stopped due to error, the failure cause would be reflected in the future.
+ *
+ * @param service The {@link Service} to block on.
+ * @return A {@link ListenableFuture} that will be completed when the service is stopped.
+ */
+ public static ListenableFuture<Service.State> getCompletionFuture(Service service) {
+ final SettableFuture<Service.State> resultFuture = SettableFuture.create();
+
+ service.addListener(new ServiceListenerAdapter() {
+ @Override
+ public void terminated(Service.State from) {
+ resultFuture.set(Service.State.TERMINATED);
+ }
+
+ @Override
+ public void failed(Service.State from, Throwable failure) {
+ resultFuture.setException(failure);
+ }
+ }, Threads.SAME_THREAD_EXECUTOR);
+
+ Service.State state = service.state();
+ if (state == Service.State.TERMINATED) {
+ return Futures.immediateFuture(state);
+ } else if (state == Service.State.FAILED) {
+ return Futures.immediateFailedFuture(new IllegalStateException("Service failed with unknown exception."));
+ }
+
+ return resultFuture;
+ }
+
+ /**
+ * Performs the actual logic of chain Service start/stop.
+ */
+ private static ListenableFuture<List<ListenableFuture<Service.State>>> doChain(boolean doStart,
+ Service firstService,
+ Service...moreServices) {
+ SettableFuture<List<ListenableFuture<Service.State>>> resultFuture = SettableFuture.create();
+ List<ListenableFuture<Service.State>> result = Lists.newArrayListWithCapacity(moreServices.length + 1);
+
+ ListenableFuture<Service.State> future = doStart ? firstService.start() : firstService.stop();
+ future.addListener(createChainListener(future, moreServices, new AtomicInteger(0), result, resultFuture, doStart),
+ Threads.SAME_THREAD_EXECUTOR);
+ return resultFuture;
+ }
+
+ /**
+ * Returns a {@link Runnable} that can be used as a {@link ListenableFuture} listener to trigger
+ * further service action or completing the result future. Used by
+ * {@link #doChain(boolean, com.google.common.util.concurrent.Service, com.google.common.util.concurrent.Service...)}
+ */
+ private static Runnable createChainListener(final ListenableFuture<Service.State> future, final Service[] services,
+ final AtomicInteger idx,
+ final List<ListenableFuture<Service.State>> result,
+ final SettableFuture<List<ListenableFuture<Service.State>>> resultFuture,
+ final boolean doStart) {
+ return new Runnable() {
+
+ @Override
+ public void run() {
+ result.add(future);
+ int nextIdx = idx.getAndIncrement();
+ if (nextIdx == services.length) {
+ resultFuture.set(result);
+ return;
+ }
+ ListenableFuture<Service.State> actionFuture = doStart ? services[nextIdx].start() : services[nextIdx].stop();
+ actionFuture.addListener(createChainListener(actionFuture, services, idx, result, resultFuture, doStart),
+ Threads.SAME_THREAD_EXECUTOR);
+ }
+ };
+ }
+
+ private Services() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/common/src/main/java/org/apache/twill/common/Threads.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/twill/common/Threads.java b/common/src/main/java/org/apache/twill/common/Threads.java
new file mode 100644
index 0000000..e33a677
--- /dev/null
+++ b/common/src/main/java/org/apache/twill/common/Threads.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.common;
+
+import com.google.common.util.concurrent.MoreExecutors;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+import java.util.concurrent.Executor;
+import java.util.concurrent.ThreadFactory;
+
+/**
+ *
+ */
+public final class Threads {
+
+ /**
+ * A executor that execute task from the submitter thread.
+ */
+ public static final Executor SAME_THREAD_EXECUTOR = MoreExecutors.sameThreadExecutor();
+
+ /**
+ * Handy method to create {@link ThreadFactory} that creates daemon threads with the given name format.
+ *
+ * @param nameFormat Name format for the thread names
+ * @return A {@link ThreadFactory}.
+ * @see ThreadFactoryBuilder
+ */
+ public static ThreadFactory createDaemonThreadFactory(String nameFormat) {
+ return new ThreadFactoryBuilder()
+ .setDaemon(true)
+ .setNameFormat(nameFormat)
+ .build();
+ }
+
+ private Threads() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/common/src/main/java/org/apache/twill/filesystem/ForwardingLocationFactory.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/twill/filesystem/ForwardingLocationFactory.java b/common/src/main/java/org/apache/twill/filesystem/ForwardingLocationFactory.java
new file mode 100644
index 0000000..d25ea20
--- /dev/null
+++ b/common/src/main/java/org/apache/twill/filesystem/ForwardingLocationFactory.java
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.filesystem;
+
+/**
+ *
+ */
+public abstract class ForwardingLocationFactory implements LocationFactory {
+
+ private final LocationFactory delegate;
+
+ protected ForwardingLocationFactory(LocationFactory delegate) {
+ this.delegate = delegate;
+ }
+
+ public LocationFactory getDelegate() {
+ return delegate;
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/common/src/main/java/org/apache/twill/filesystem/LocalLocation.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/twill/filesystem/LocalLocation.java b/common/src/main/java/org/apache/twill/filesystem/LocalLocation.java
new file mode 100644
index 0000000..d107eac
--- /dev/null
+++ b/common/src/main/java/org/apache/twill/filesystem/LocalLocation.java
@@ -0,0 +1,205 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.filesystem;
+
+import com.google.common.collect.Lists;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.util.Collections;
+import java.util.Deque;
+import java.util.UUID;
+
+/**
+ * A concrete implementation of {@link Location} for the Local filesystem.
+ */
+final class LocalLocation implements Location {
+ private final File file;
+
+ /**
+ * Constructs a LocalLocation.
+ *
+ * @param file to the file.
+ */
+ LocalLocation(File file) {
+ this.file = file;
+ }
+
+ /**
+ * Checks if the this location exists on local file system.
+ *
+ * @return true if found; false otherwise.
+ * @throws java.io.IOException
+ */
+ @Override
+ public boolean exists() throws IOException {
+ return file.exists();
+ }
+
+ /**
+ * @return An {@link java.io.InputStream} for this location on local filesystem.
+ * @throws IOException
+ */
+ @Override
+ public InputStream getInputStream() throws IOException {
+ File parent = file.getParentFile();
+ if (!parent.exists()) {
+ parent.mkdirs();
+ }
+ return new FileInputStream(file);
+ }
+
+ /**
+ * @return An {@link java.io.OutputStream} for this location on local filesystem.
+ * @throws IOException
+ */
+ @Override
+ public OutputStream getOutputStream() throws IOException {
+ File parent = file.getParentFile();
+ if (!parent.exists()) {
+ parent.mkdirs();
+ }
+ return new FileOutputStream(file);
+ }
+
+ /**
+ * Local location doesn't supports permission. It's the same as calling {@link #getOutputStream()}.
+ */
+ @Override
+ public OutputStream getOutputStream(String permission) throws IOException {
+ return getOutputStream();
+ }
+
+ /**
+ * @return Returns the name of the file or directory denoteed by this abstract pathname.
+ */
+ @Override
+ public String getName() {
+ return file.getName();
+ }
+
+ @Override
+ public boolean createNew() throws IOException {
+ return file.createNewFile();
+ }
+
+ /**
+ * Appends the child to the current {@link Location} on local filesystem.
+ * <p>
+ * Returns a new instance of Location.
+ * </p>
+ *
+ * @param child to be appended to this location.
+ * @return A new instance of {@link Location}
+ * @throws IOException
+ */
+ @Override
+ public Location append(String child) throws IOException {
+ return new LocalLocation(new File(file, child));
+ }
+
+ @Override
+ public Location getTempFile(String suffix) throws IOException {
+ return new LocalLocation(
+ new File(file.getAbsolutePath() + "." + UUID.randomUUID() + (suffix == null ? TEMP_FILE_SUFFIX : suffix)));
+ }
+
+ /**
+ * @return A {@link URI} for this location on local filesystem.
+ */
+ @Override
+ public URI toURI() {
+ return file.toURI();
+ }
+
+ /**
+ * Deletes the file or directory denoted by this abstract pathname. If this
+ * pathname denotes a directory, then the directory must be empty in order
+ * to be deleted.
+ *
+ * @return true if and only if the file or directory is successfully delete; false otherwise.
+ */
+ @Override
+ public boolean delete() throws IOException {
+ return file.delete();
+ }
+
+ @Override
+ public boolean delete(boolean recursive) throws IOException {
+ if (!recursive) {
+ return delete();
+ }
+
+ Deque<File> stack = Lists.newLinkedList();
+ stack.add(file);
+ while (!stack.isEmpty()) {
+ File f = stack.peekLast();
+ File[] files = f.listFiles();
+
+ if (files != null && files.length != 0) {
+ Collections.addAll(stack, files);
+ } else {
+ if (!f.delete()) {
+ return false;
+ }
+ stack.pollLast();
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public Location renameTo(Location destination) throws IOException {
+ // destination will always be of the same type as this location
+ boolean success = file.renameTo(((LocalLocation) destination).file);
+ if (success) {
+ return new LocalLocation(((LocalLocation) destination).file);
+ } else {
+ return null;
+ }
+ }
+
+ /**
+ * Creates the directory named by this abstract pathname, including any necessary
+ * but nonexistent parent directories.
+ *
+ * @return true if and only if the renaming succeeded; false otherwise
+ */
+ @Override
+ public boolean mkdirs() throws IOException {
+ return file.mkdirs();
+ }
+
+ /**
+ * @return Length of file.
+ */
+ @Override
+ public long length() throws IOException {
+ return file.length();
+ }
+
+ @Override
+ public long lastModified() {
+ return file.lastModified();
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/common/src/main/java/org/apache/twill/filesystem/LocalLocationFactory.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/twill/filesystem/LocalLocationFactory.java b/common/src/main/java/org/apache/twill/filesystem/LocalLocationFactory.java
new file mode 100644
index 0000000..f44cd87
--- /dev/null
+++ b/common/src/main/java/org/apache/twill/filesystem/LocalLocationFactory.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.filesystem;
+
+import java.io.File;
+import java.net.URI;
+
+/**
+ * A {@link LocationFactory} for creating local file {@link Location}.
+ */
+public final class LocalLocationFactory implements LocationFactory {
+
+ private final File basePath;
+
+ /**
+ * Constructs a LocalLocationFactory that Location created will be relative to system root.
+ */
+ public LocalLocationFactory() {
+ this(new File("/"));
+ }
+
+ public LocalLocationFactory(File basePath) {
+ this.basePath = basePath;
+ }
+
+ @Override
+ public Location create(String path) {
+ return new LocalLocation(new File(basePath, path));
+ }
+
+ @Override
+ public Location create(URI uri) {
+ if (uri.isAbsolute()) {
+ return new LocalLocation(new File(uri));
+ }
+ return new LocalLocation(new File(basePath, uri.getPath()));
+ }
+
+ @Override
+ public Location getHomeLocation() {
+ return new LocalLocation(new File(System.getProperty("user.home")));
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-twill/blob/1925ffaf/common/src/main/java/org/apache/twill/filesystem/Location.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/twill/filesystem/Location.java b/common/src/main/java/org/apache/twill/filesystem/Location.java
new file mode 100644
index 0000000..dee9546
--- /dev/null
+++ b/common/src/main/java/org/apache/twill/filesystem/Location.java
@@ -0,0 +1,154 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.twill.filesystem;
+
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+
+/**
+ * This interface defines the location and operations of a resource on the filesystem.
+ * <p>
+ * {@link Location} is agnostic to the type of file system the resource is on.
+ * </p>
+ */
+public interface Location {
+ /**
+ * Suffix added to every temp file name generated with {@link #getTempFile(String)}.
+ */
+ static final String TEMP_FILE_SUFFIX = ".tmp";
+
+ /**
+ * Checks if the this location exists.
+ *
+ * @return true if found; false otherwise.
+ * @throws IOException
+ */
+ boolean exists() throws IOException;
+
+ /**
+ * @return Returns the name of the file or directory denoteed by this abstract pathname.
+ */
+ String getName();
+
+ /**
+ * Atomically creates a new, empty file named by this abstract pathname if and only if a file with this name
+ * does not yet exist.
+ * @return {@code true} if the file is successfully create, {@code false} otherwise.
+ * @throws IOException
+ */
+ boolean createNew() throws IOException;
+
+ /**
+ * @return An {@link java.io.InputStream} for this location.
+ * @throws IOException
+ */
+ InputStream getInputStream() throws IOException;
+
+ /**
+ * @return An {@link java.io.OutputStream} for this location.
+ * @throws IOException
+ */
+ OutputStream getOutputStream() throws IOException;
+
+ /**
+ * Creates an {@link OutputStream} for this location with the given permission. The actual permission supported
+ * depends on implementation.
+ *
+ * @param permission A POSIX permission string.
+ * @return An {@link OutputStream} for writing to this location.
+ * @throws IOException If failed to create the {@link OutputStream}.
+ */
+ OutputStream getOutputStream(String permission) throws IOException;
+
+ /**
+ * Appends the child to the current {@link Location}.
+ * <p>
+ * Returns a new instance of Location.
+ * </p>
+ *
+ * @param child to be appended to this location.
+ * @return A new instance of {@link Location}
+ * @throws IOException
+ */
+ Location append(String child) throws IOException;
+
+ /**
+ * Returns unique location for temporary file to be placed near this location.
+ * Allows all temp files to follow same pattern for easier management of them.
+ * @param suffix part of the file name to include in the temp file name
+ * @return location of the temp file
+ * @throws IOException
+ */
+ Location getTempFile(String suffix) throws IOException;
+
+ /**
+ * @return A {@link java.net.URI} for this location.
+ */
+ URI toURI();
+
+ /**
+ * Deletes the file or directory denoted by this abstract pathname. If this
+ * pathname denotes a directory, then the directory must be empty in order
+ * to be deleted.
+ *
+ * @return true if and only if the file or directory is successfully delete; false otherwise.
+ */
+ boolean delete() throws IOException;
+
+ /**
+ * Deletes the file or directory denoted by this abstract pathname. If this
+ * pathname denotes a directory and {@code recursive} is {@code true}, then content of the
+ * directory will be deleted recursively, otherwise the directory must be empty in order to be deleted.
+ * Note that when calling this method with {@code recursive = true} for a directory, any
+ * failure during deletion will have some entries inside the directory being deleted while some are not.
+ *
+ * @param recursive Indicate if recursively delete a directory. Ignored if the pathname represents a file.
+ * @return true if and only if the file or directory is successfully delete; false otherwise.
+ */
+ boolean delete(boolean recursive) throws IOException;
+
+ /**
+ * Moves the file or directory denoted by this abstract pathname.
+ *
+ * @param destination destination location
+ * @return new location if and only if the file or directory is successfully moved; null otherwise.
+ */
+ @Nullable
+ Location renameTo(Location destination) throws IOException;
+
+ /**
+ * Creates the directory named by this abstract pathname, including any necessary
+ * but nonexistent parent directories.
+ *
+ * @return true if and only if the renaming succeeded; false otherwise
+ */
+ boolean mkdirs() throws IOException;
+
+ /**
+ * @return Length of file.
+ */
+ long length() throws IOException;
+
+ /**
+ * @return Last modified time of file.
+ */
+ long lastModified() throws IOException;
+}