You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@hama.apache.org by Tommaso Teofili <to...@gmail.com> on 2015/04/09 14:51:06 UTC
Re: svn commit: r1672307 - in /hama/trunk: pom.xml
yarn/src/main/java/org/apache/hama/bsp/YARNBSPJobClient.java yarn/src/main/java/org/apache/hama/bsp/YarnSerializePrinting.java
thanks for having take care of that yourself Edward, I just didn't have
time to do it myself yet.
Regards,
Tommaso
2015-04-09 14:15 GMT+02:00 <ed...@apache.org>:
> Author: edwardyoon
> Date: Thu Apr 9 12:15:13 2015
> New Revision: 1672307
>
> URL: http://svn.apache.org/r1672307
> Log:
> Fix YarnSerializePrinting and pom file for jdk8
>
> Modified:
> hama/trunk/pom.xml
> hama/trunk/yarn/src/main/java/org/apache/hama/bsp/YARNBSPJobClient.java
>
> hama/trunk/yarn/src/main/java/org/apache/hama/bsp/YarnSerializePrinting.java
>
> Modified: hama/trunk/pom.xml
> URL:
> http://svn.apache.org/viewvc/hama/trunk/pom.xml?rev=1672307&r1=1672306&r2=1672307&view=diff
>
> ==============================================================================
> --- hama/trunk/pom.xml (original)
> +++ hama/trunk/pom.xml Thu Apr 9 12:15:13 2015
> @@ -126,6 +126,24 @@
>
> <profiles>
> <profile>
> + <id>doclint-java8-disable</id>
> + <activation>
> + <jdk>[1.8,)</jdk>
> + </activation>
> + <build>
> + <plugins>
> + <plugin>
> + <groupId>org.apache.maven.plugins</groupId>
> + <artifactId>maven-javadoc-plugin</artifactId>
> + <configuration>
> + <additionalparam>-Xdoclint:none</additionalparam>
> + </configuration>
> + </plugin>
> + </plugins>
> + </build>
> + </profile>
> +
> + <profile>
> <id>hadoop1</id>
>
> <modules>
> @@ -476,7 +494,6 @@
> <configuration>
> <aggregate>true</aggregate>
> <outputDirectory>docs/apidocs</outputDirectory>
> - <additionalparam>-Xdoclint:none</additionalparam>
> </configuration>
> <executions>
> <execution>
>
> Modified:
> hama/trunk/yarn/src/main/java/org/apache/hama/bsp/YARNBSPJobClient.java
> URL:
> http://svn.apache.org/viewvc/hama/trunk/yarn/src/main/java/org/apache/hama/bsp/YARNBSPJobClient.java?rev=1672307&r1=1672306&r2=1672307&view=diff
>
> ==============================================================================
> ---
> hama/trunk/yarn/src/main/java/org/apache/hama/bsp/YARNBSPJobClient.java
> (original)
> +++
> hama/trunk/yarn/src/main/java/org/apache/hama/bsp/YARNBSPJobClient.java Thu
> Apr 9 12:15:13 2015
> @@ -424,7 +424,6 @@ public class YARNBSPJobClient extends BS
> return files;
> }
>
> -
> private void addToLocalResources(FileSystem fs, String fileSrcPath,
> String fileDstPath, String fileName, Map<String, LocalResource>
> localResources)
> throws IOException {
>
> Modified:
> hama/trunk/yarn/src/main/java/org/apache/hama/bsp/YarnSerializePrinting.java
> URL:
> http://svn.apache.org/viewvc/hama/trunk/yarn/src/main/java/org/apache/hama/bsp/YarnSerializePrinting.java?rev=1672307&r1=1672306&r2=1672307&view=diff
>
> ==============================================================================
> ---
> hama/trunk/yarn/src/main/java/org/apache/hama/bsp/YarnSerializePrinting.java
> (original)
> +++
> hama/trunk/yarn/src/main/java/org/apache/hama/bsp/YarnSerializePrinting.java
> Thu Apr 9 12:15:13 2015
> @@ -21,39 +21,62 @@ import java.io.IOException;
>
> import org.apache.commons.logging.Log;
> import org.apache.commons.logging.LogFactory;
> +import org.apache.hadoop.fs.FSDataInputStream;
> +import org.apache.hadoop.fs.FileStatus;
> +import org.apache.hadoop.fs.FileSystem;
> +import org.apache.hadoop.fs.Path;
> +import org.apache.hadoop.io.IntWritable;
> +import org.apache.hadoop.io.IOUtils;
> import org.apache.hadoop.io.NullWritable;
> -import org.apache.hama.Constants;
> +import org.apache.hadoop.io.Text;
> import org.apache.hama.HamaConfiguration;
> import org.apache.hama.bsp.sync.SyncException;
>
> public class YarnSerializePrinting {
>
> + public static Path OUTPUT_PATH = new Path("/tmp/serialout");
> +
> public static class HelloBSP extends
> - BSP<NullWritable, NullWritable, NullWritable, NullWritable,
> NullWritable> {
> + BSP<NullWritable, NullWritable, IntWritable, Text, NullWritable> {
> public static final Log LOG = LogFactory.getLog(HelloBSP.class);
> - private final static int PRINT_INTERVAL = 1000;
> private int num;
>
> @Override
> public void bsp(
> - BSPPeer<NullWritable, NullWritable, NullWritable, NullWritable,
> NullWritable> bspPeer)
> + BSPPeer<NullWritable, NullWritable, IntWritable, Text,
> NullWritable> bspPeer)
> throws IOException, SyncException, InterruptedException {
> num = bspPeer.getConfiguration().getInt("bsp.peers.num", 1);
> - LOG.info(bspPeer.getAllPeerNames());
> + IntWritable peerNum = new IntWritable();
> + Text txt = new Text();
> int i = 0;
> for (String otherPeer : bspPeer.getAllPeerNames()) {
> String peerName = bspPeer.getPeerName();
> if (peerName.equals(otherPeer)) {
> - LOG.info("Hello BSP from " + (i + 1) + " of " + num + ": " +
> peerName);
> + peerNum.set(i);
> + txt.set("Hello BSP from " + (i + 1) + " of " + num + ": " +
> peerName);
> + bspPeer.write(null, txt);
> }
>
> - Thread.sleep(PRINT_INTERVAL);
> bspPeer.sync();
> i++;
> }
> }
> }
>
> + static void printOutput(HamaConfiguration conf) throws IOException {
> + FileSystem fs = FileSystem.get(conf);
> + FileStatus[] files = fs.listStatus(OUTPUT_PATH);
> + for (FileStatus file : files) {
> + if (file.getLen() > 0) {
> + FSDataInputStream in = fs.open(file.getPath());
> + IOUtils.copyBytes(in, System.out, conf, false);
> + in.close();
> + }
> + }
> +
> + //fs.delete(OUTPUT_PATH, true);
> + }
> +
> public static void main(String[] args) throws IOException,
> InterruptedException, ClassNotFoundException {
> HamaConfiguration conf = new HamaConfiguration();
> @@ -63,9 +86,17 @@ public class YarnSerializePrinting {
> job.setJarByClass(HelloBSP.class);
> job.setJobName("Serialize Printing");
> job.setInputFormat(NullInputFormat.class);
> - job.setOutputFormat(NullOutputFormat.class);
> + job.setOutputFormat(TextOutputFormat.class);
> + job.setOutputKeyClass(IntWritable.class);
> + job.setOutputValueClass(Text.class);
> + job.setOutputPath(OUTPUT_PATH);
> job.setMemoryUsedPerTaskInMb(100);
> job.setNumBspTask(4);
> +
> + long startTime = System.currentTimeMillis();
> job.waitForCompletion(true);
> + printOutput(conf);
> + System.out.println("Job Finished in "
> + + (System.currentTimeMillis() - startTime) / 1000.0 + "
> seconds");
> }
> }
>
>
>
Re: svn commit: r1672307 - in /hama/trunk: pom.xml
yarn/src/main/java/org/apache/hama/bsp/YARNBSPJobClient.java yarn/src/main/java/org/apache/hama/bsp/YarnSerializePrinting.java
Posted by "Edward J. Yoon" <ed...@apache.org>.
No problem. I didn't test on jdk8 machine yet too.
On Thu, Apr 9, 2015 at 9:51 PM, Tommaso Teofili
<to...@gmail.com> wrote:
> thanks for having take care of that yourself Edward, I just didn't have
> time to do it myself yet.
>
> Regards,
> Tommaso
>
> 2015-04-09 14:15 GMT+02:00 <ed...@apache.org>:
>
>> Author: edwardyoon
>> Date: Thu Apr 9 12:15:13 2015
>> New Revision: 1672307
>>
>> URL: http://svn.apache.org/r1672307
>> Log:
>> Fix YarnSerializePrinting and pom file for jdk8
>>
>> Modified:
>> hama/trunk/pom.xml
>> hama/trunk/yarn/src/main/java/org/apache/hama/bsp/YARNBSPJobClient.java
>>
>> hama/trunk/yarn/src/main/java/org/apache/hama/bsp/YarnSerializePrinting.java
>>
>> Modified: hama/trunk/pom.xml
>> URL:
>> http://svn.apache.org/viewvc/hama/trunk/pom.xml?rev=1672307&r1=1672306&r2=1672307&view=diff
>>
>> ==============================================================================
>> --- hama/trunk/pom.xml (original)
>> +++ hama/trunk/pom.xml Thu Apr 9 12:15:13 2015
>> @@ -126,6 +126,24 @@
>>
>> <profiles>
>> <profile>
>> + <id>doclint-java8-disable</id>
>> + <activation>
>> + <jdk>[1.8,)</jdk>
>> + </activation>
>> + <build>
>> + <plugins>
>> + <plugin>
>> + <groupId>org.apache.maven.plugins</groupId>
>> + <artifactId>maven-javadoc-plugin</artifactId>
>> + <configuration>
>> + <additionalparam>-Xdoclint:none</additionalparam>
>> + </configuration>
>> + </plugin>
>> + </plugins>
>> + </build>
>> + </profile>
>> +
>> + <profile>
>> <id>hadoop1</id>
>>
>> <modules>
>> @@ -476,7 +494,6 @@
>> <configuration>
>> <aggregate>true</aggregate>
>> <outputDirectory>docs/apidocs</outputDirectory>
>> - <additionalparam>-Xdoclint:none</additionalparam>
>> </configuration>
>> <executions>
>> <execution>
>>
>> Modified:
>> hama/trunk/yarn/src/main/java/org/apache/hama/bsp/YARNBSPJobClient.java
>> URL:
>> http://svn.apache.org/viewvc/hama/trunk/yarn/src/main/java/org/apache/hama/bsp/YARNBSPJobClient.java?rev=1672307&r1=1672306&r2=1672307&view=diff
>>
>> ==============================================================================
>> ---
>> hama/trunk/yarn/src/main/java/org/apache/hama/bsp/YARNBSPJobClient.java
>> (original)
>> +++
>> hama/trunk/yarn/src/main/java/org/apache/hama/bsp/YARNBSPJobClient.java Thu
>> Apr 9 12:15:13 2015
>> @@ -424,7 +424,6 @@ public class YARNBSPJobClient extends BS
>> return files;
>> }
>>
>> -
>> private void addToLocalResources(FileSystem fs, String fileSrcPath,
>> String fileDstPath, String fileName, Map<String, LocalResource>
>> localResources)
>> throws IOException {
>>
>> Modified:
>> hama/trunk/yarn/src/main/java/org/apache/hama/bsp/YarnSerializePrinting.java
>> URL:
>> http://svn.apache.org/viewvc/hama/trunk/yarn/src/main/java/org/apache/hama/bsp/YarnSerializePrinting.java?rev=1672307&r1=1672306&r2=1672307&view=diff
>>
>> ==============================================================================
>> ---
>> hama/trunk/yarn/src/main/java/org/apache/hama/bsp/YarnSerializePrinting.java
>> (original)
>> +++
>> hama/trunk/yarn/src/main/java/org/apache/hama/bsp/YarnSerializePrinting.java
>> Thu Apr 9 12:15:13 2015
>> @@ -21,39 +21,62 @@ import java.io.IOException;
>>
>> import org.apache.commons.logging.Log;
>> import org.apache.commons.logging.LogFactory;
>> +import org.apache.hadoop.fs.FSDataInputStream;
>> +import org.apache.hadoop.fs.FileStatus;
>> +import org.apache.hadoop.fs.FileSystem;
>> +import org.apache.hadoop.fs.Path;
>> +import org.apache.hadoop.io.IntWritable;
>> +import org.apache.hadoop.io.IOUtils;
>> import org.apache.hadoop.io.NullWritable;
>> -import org.apache.hama.Constants;
>> +import org.apache.hadoop.io.Text;
>> import org.apache.hama.HamaConfiguration;
>> import org.apache.hama.bsp.sync.SyncException;
>>
>> public class YarnSerializePrinting {
>>
>> + public static Path OUTPUT_PATH = new Path("/tmp/serialout");
>> +
>> public static class HelloBSP extends
>> - BSP<NullWritable, NullWritable, NullWritable, NullWritable,
>> NullWritable> {
>> + BSP<NullWritable, NullWritable, IntWritable, Text, NullWritable> {
>> public static final Log LOG = LogFactory.getLog(HelloBSP.class);
>> - private final static int PRINT_INTERVAL = 1000;
>> private int num;
>>
>> @Override
>> public void bsp(
>> - BSPPeer<NullWritable, NullWritable, NullWritable, NullWritable,
>> NullWritable> bspPeer)
>> + BSPPeer<NullWritable, NullWritable, IntWritable, Text,
>> NullWritable> bspPeer)
>> throws IOException, SyncException, InterruptedException {
>> num = bspPeer.getConfiguration().getInt("bsp.peers.num", 1);
>> - LOG.info(bspPeer.getAllPeerNames());
>> + IntWritable peerNum = new IntWritable();
>> + Text txt = new Text();
>> int i = 0;
>> for (String otherPeer : bspPeer.getAllPeerNames()) {
>> String peerName = bspPeer.getPeerName();
>> if (peerName.equals(otherPeer)) {
>> - LOG.info("Hello BSP from " + (i + 1) + " of " + num + ": " +
>> peerName);
>> + peerNum.set(i);
>> + txt.set("Hello BSP from " + (i + 1) + " of " + num + ": " +
>> peerName);
>> + bspPeer.write(null, txt);
>> }
>>
>> - Thread.sleep(PRINT_INTERVAL);
>> bspPeer.sync();
>> i++;
>> }
>> }
>> }
>>
>> + static void printOutput(HamaConfiguration conf) throws IOException {
>> + FileSystem fs = FileSystem.get(conf);
>> + FileStatus[] files = fs.listStatus(OUTPUT_PATH);
>> + for (FileStatus file : files) {
>> + if (file.getLen() > 0) {
>> + FSDataInputStream in = fs.open(file.getPath());
>> + IOUtils.copyBytes(in, System.out, conf, false);
>> + in.close();
>> + }
>> + }
>> +
>> + //fs.delete(OUTPUT_PATH, true);
>> + }
>> +
>> public static void main(String[] args) throws IOException,
>> InterruptedException, ClassNotFoundException {
>> HamaConfiguration conf = new HamaConfiguration();
>> @@ -63,9 +86,17 @@ public class YarnSerializePrinting {
>> job.setJarByClass(HelloBSP.class);
>> job.setJobName("Serialize Printing");
>> job.setInputFormat(NullInputFormat.class);
>> - job.setOutputFormat(NullOutputFormat.class);
>> + job.setOutputFormat(TextOutputFormat.class);
>> + job.setOutputKeyClass(IntWritable.class);
>> + job.setOutputValueClass(Text.class);
>> + job.setOutputPath(OUTPUT_PATH);
>> job.setMemoryUsedPerTaskInMb(100);
>> job.setNumBspTask(4);
>> +
>> + long startTime = System.currentTimeMillis();
>> job.waitForCompletion(true);
>> + printOutput(conf);
>> + System.out.println("Job Finished in "
>> + + (System.currentTimeMillis() - startTime) / 1000.0 + "
>> seconds");
>> }
>> }
>>
>>
>>
--
Best Regards, Edward J. Yoon