You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by sh...@apache.org on 2011/09/28 05:11:41 UTC
svn commit: r1176694 - in /hadoop/common/branches/branch-0.22/mapreduce: ./
ivy/ src/contrib/capacity-scheduler/ src/contrib/fairscheduler/
src/contrib/gridmix/ src/contrib/mrunit/ src/contrib/mumak/
src/contrib/raid/ src/contrib/streaming/ src/java/or...
Author: shv
Date: Wed Sep 28 03:11:40 2011
New Revision: 1176694
URL: http://svn.apache.org/viewvc?rev=1176694&view=rev
Log:
MAPREDUCE-3039. Upgrade to Avro 1.5.3. Contributed by Joep Rottinghuis.
Modified:
hadoop/common/branches/branch-0.22/mapreduce/CHANGES.txt
hadoop/common/branches/branch-0.22/mapreduce/build.xml
hadoop/common/branches/branch-0.22/mapreduce/ivy.xml
hadoop/common/branches/branch-0.22/mapreduce/ivy/libraries.properties
hadoop/common/branches/branch-0.22/mapreduce/src/contrib/capacity-scheduler/ivy.xml
hadoop/common/branches/branch-0.22/mapreduce/src/contrib/fairscheduler/ivy.xml
hadoop/common/branches/branch-0.22/mapreduce/src/contrib/gridmix/ivy.xml
hadoop/common/branches/branch-0.22/mapreduce/src/contrib/mrunit/ivy.xml
hadoop/common/branches/branch-0.22/mapreduce/src/contrib/mumak/ivy.xml
hadoop/common/branches/branch-0.22/mapreduce/src/contrib/raid/ivy.xml
hadoop/common/branches/branch-0.22/mapreduce/src/contrib/streaming/ivy.xml
hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java
hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java
Modified: hadoop/common/branches/branch-0.22/mapreduce/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/CHANGES.txt?rev=1176694&r1=1176693&r2=1176694&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/mapreduce/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.22/mapreduce/CHANGES.txt Wed Sep 28 03:11:40 2011
@@ -223,6 +223,8 @@ Release 0.22.0 - Unreleased
HADOOP-7106. Reorganize project SVN layout to "unsplit" the projects.
(todd, nigel)
+ MAPREDUCE-3039. Upgrade to Avro 1.5.3. (Joep Rottinghuis via shv)
+
OPTIMIZATIONS
MAPREDUCE-1354. Enhancements to JobTracker for better performance and
Modified: hadoop/common/branches/branch-0.22/mapreduce/build.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/build.xml?rev=1176694&r1=1176693&r2=1176694&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/mapreduce/build.xml (original)
+++ hadoop/common/branches/branch-0.22/mapreduce/build.xml Wed Sep 28 03:11:40 2011
@@ -339,7 +339,7 @@
<target name="avro-generate" depends="init">
<mkdir dir="${build.src}/org/apache/hadoop/mapreduce/jobhistory"/>
- <taskdef name="protocol" classname="org.apache.avro.specific.ProtocolTask">
+ <taskdef name="protocol" classname="org.apache.avro.compiler.specific.ProtocolTask">
<classpath refid="classpath" />
</taskdef>
<protocol destdir="${build.src}">
Modified: hadoop/common/branches/branch-0.22/mapreduce/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/ivy.xml?rev=1176694&r1=1176693&r2=1176694&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/mapreduce/ivy.xml (original)
+++ hadoop/common/branches/branch-0.22/mapreduce/ivy.xml Wed Sep 28 03:11:40 2011
@@ -97,12 +97,18 @@
<dependency org="org.apache.lucene" name="lucene-core"
rev="${lucene-core.version}" conf="javadoc->default"/>
- <dependency org="org.apache.hadoop" name="avro" rev="${avro.version}"
- conf="common->default">
+ <dependency org="org.apache.avro" name="avro"
+ rev="${avro.version}" conf="common->default">
<exclude module="ant"/>
<exclude module="jetty"/>
- <exclude module="slf4j-simple"/>
- </dependency>
+ <exclude module="slf4j-api"/>
+ </dependency>
+ <dependency org="org.apache.avro" name="avro-compiler"
+ rev="${avro.version}" conf="common->default">
+ <exclude module="ant"/>
+ <exclude module="jetty"/>
+ <exclude module="slf4j-api"/>
+ </dependency>
<dependency org="org.mockito" name="mockito-all" rev="${mockito-all.version}"
conf="test->default"/>
<dependency org="org.mortbay.jetty" name="jetty-servlet-tester" rev="${jetty.version}"
Modified: hadoop/common/branches/branch-0.22/mapreduce/ivy/libraries.properties
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/ivy/libraries.properties?rev=1176694&r1=1176693&r2=1176694&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/mapreduce/ivy/libraries.properties (original)
+++ hadoop/common/branches/branch-0.22/mapreduce/ivy/libraries.properties Wed Sep 28 03:11:40 2011
@@ -20,7 +20,7 @@ ant-task.version=2.0.10
#This property has to be updated synchronously with aop.xml
aspectj.version=1.6.5
-avro.version=1.3.2
+avro.version=1.5.3
checkstyle.version=4.2
Modified: hadoop/common/branches/branch-0.22/mapreduce/src/contrib/capacity-scheduler/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/contrib/capacity-scheduler/ivy.xml?rev=1176694&r1=1176693&r2=1176694&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/mapreduce/src/contrib/capacity-scheduler/ivy.xml (original)
+++ hadoop/common/branches/branch-0.22/mapreduce/src/contrib/capacity-scheduler/ivy.xml Wed Sep 28 03:11:40 2011
@@ -69,11 +69,5 @@
rev="${servlet-api-2.5.version}" conf="common->master"/>
<dependency org="commons-httpclient" name="commons-httpclient"
rev="${commons-httpclient.version}" conf="common->master"/>
- <dependency org="org.apache.hadoop" name="avro"
- rev="${avro.version}" conf="common->default">
- <exclude module="ant"/>
- <exclude module="jetty"/>
- <exclude module="slf4j-simple"/>
- </dependency>
</dependencies>
</ivy-module>
Modified: hadoop/common/branches/branch-0.22/mapreduce/src/contrib/fairscheduler/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/contrib/fairscheduler/ivy.xml?rev=1176694&r1=1176693&r2=1176694&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/mapreduce/src/contrib/fairscheduler/ivy.xml (original)
+++ hadoop/common/branches/branch-0.22/mapreduce/src/contrib/fairscheduler/ivy.xml Wed Sep 28 03:11:40 2011
@@ -64,26 +64,10 @@
name="junit"
rev="${junit.version}"
conf="common->default"/>
- <dependency org="org.apache.hadoop"
- name="avro"
- rev="${avro.version}"
- conf="common->default">
- <exclude module="ant"/>
- <exclude module="jetty"/>
- <exclude module="slf4j-simple"/>
- </dependency>
<dependency org="org.codehaus.jackson"
name="jackson-mapper-asl"
rev="${jackson.version}"
conf="common->default"/>
- <dependency org="com.thoughtworks.paranamer"
- name="paranamer"
- rev="${paranamer.version}"
- conf="common->default"/>
- <dependency org="com.thoughtworks.paranamer"
- name="paranamer-ant"
- rev="${paranamer.version}"
- conf="common->default"/>
<dependency org="org.mortbay.jetty"
name="jetty-util"
rev="${jetty-util.version}"
Modified: hadoop/common/branches/branch-0.22/mapreduce/src/contrib/gridmix/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/contrib/gridmix/ivy.xml?rev=1176694&r1=1176693&r2=1176694&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/mapreduce/src/contrib/gridmix/ivy.xml (original)
+++ hadoop/common/branches/branch-0.22/mapreduce/src/contrib/gridmix/ivy.xml Wed Sep 28 03:11:40 2011
@@ -102,14 +102,6 @@
name="commons-cli"
rev="${commons-cli.version}"
conf="common->default"/>
- <dependency org="org.apache.hadoop"
- name="avro"
- rev="${avro.version}"
- conf="common->default">
- <exclude module="ant"/>
- <exclude module="jetty"/>
- <exclude module="slf4j-simple"/>
- </dependency>
<dependency org="org.codehaus.jackson"
name="jackson-mapper-asl"
rev="${jackson.version}"
@@ -118,9 +110,5 @@
name="jackson-core-asl"
rev="${jackson.version}"
conf="common->default"/>
- <dependency org="com.thoughtworks.paranamer"
- name="paranamer"
- rev="${paranamer.version}"
- conf="common->default"/>
</dependencies>
</ivy-module>
Modified: hadoop/common/branches/branch-0.22/mapreduce/src/contrib/mrunit/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/contrib/mrunit/ivy.xml?rev=1176694&r1=1176693&r2=1176694&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/mapreduce/src/contrib/mrunit/ivy.xml (original)
+++ hadoop/common/branches/branch-0.22/mapreduce/src/contrib/mrunit/ivy.xml Wed Sep 28 03:11:40 2011
@@ -45,11 +45,6 @@
<dependency org="commons-logging" name="commons-logging" rev="${commons-logging.version}" conf="common->default"/>
<dependency org="log4j" name="log4j" rev="${log4j.version}" conf="common->master"/>
<dependency org="junit" name="junit" rev="${junit.version}" conf="common->master"/>
- <dependency org="org.apache.hadoop" name="avro" rev="${avro.version}" conf="common->default">
- <exclude module="ant"/>
- <exclude module="jetty"/>
- <exclude module="slf4j-simple"/>
- </dependency>
</dependencies>
</ivy-module>
Modified: hadoop/common/branches/branch-0.22/mapreduce/src/contrib/mumak/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/contrib/mumak/ivy.xml?rev=1176694&r1=1176693&r2=1176694&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/mapreduce/src/contrib/mumak/ivy.xml (original)
+++ hadoop/common/branches/branch-0.22/mapreduce/src/contrib/mumak/ivy.xml Wed Sep 28 03:11:40 2011
@@ -114,13 +114,5 @@
name="commons-cli"
rev="${commons-cli.version}"
conf="common->default"/>
- <dependency org="org.apache.hadoop"
- name="avro"
- rev="${avro.version}"
- conf="common->default">
- <exclude module="ant"/>
- <exclude module="jetty"/>
- <exclude module="slf4j-simple"/>
- </dependency>
</dependencies>
</ivy-module>
Modified: hadoop/common/branches/branch-0.22/mapreduce/src/contrib/raid/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/contrib/raid/ivy.xml?rev=1176694&r1=1176693&r2=1176694&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/mapreduce/src/contrib/raid/ivy.xml (original)
+++ hadoop/common/branches/branch-0.22/mapreduce/src/contrib/raid/ivy.xml Wed Sep 28 03:11:40 2011
@@ -102,14 +102,6 @@
name="commons-cli"
rev="${commons-cli.version}"
conf="common->default"/>
- <dependency org="org.apache.hadoop"
- name="avro"
- rev="${avro.version}"
- conf="common->default">
- <exclude module="ant"/>
- <exclude module="jetty"/>
- <exclude module="slf4j-simple"/>
- </dependency>
<dependency org="org.codehaus.jackson"
name="jackson-mapper-asl"
rev="${jackson.version}"
Modified: hadoop/common/branches/branch-0.22/mapreduce/src/contrib/streaming/ivy.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/contrib/streaming/ivy.xml?rev=1176694&r1=1176693&r2=1176694&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/mapreduce/src/contrib/streaming/ivy.xml (original)
+++ hadoop/common/branches/branch-0.22/mapreduce/src/contrib/streaming/ivy.xml Wed Sep 28 03:11:40 2011
@@ -68,12 +68,6 @@
rev="${commons-httpclient.version}" conf="common->default"/>
<dependency org="log4j" name="log4j"
rev="${log4j.version}" conf="common->master"/>
- <dependency org="org.apache.hadoop" name="avro"
- rev="${avro.version}" conf="common->default">
- <exclude module="ant"/>
- <exclude module="jetty"/>
- <exclude module="slf4j-simple"/>
- </dependency>
<dependency org="org.slf4j" name="slf4j-api"
rev="${slf4j-api.version}" conf="common->master"/>
</dependencies>
Modified: hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java?rev=1176694&r1=1176693&r2=1176694&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java (original)
+++ hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java Wed Sep 28 03:11:40 2011
@@ -20,10 +20,14 @@ package org.apache.hadoop.mapreduce.jobh
import java.io.Closeable;
import java.io.DataInputStream;
-import java.io.IOException;
import java.io.EOFException;
-import java.io.StringBufferInputStream;
+import java.io.IOException;
+import org.apache.avro.Schema;
+import org.apache.avro.io.DatumReader;
+import org.apache.avro.io.Decoder;
+import org.apache.avro.io.DecoderFactory;
+import org.apache.avro.specific.SpecificDatumReader;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.FileSystem;
@@ -32,13 +36,6 @@ import org.apache.hadoop.mapreduce.Count
import org.apache.hadoop.mapreduce.CounterGroup;
import org.apache.hadoop.mapreduce.Counters;
-import org.apache.avro.Schema;
-import org.apache.avro.AvroRuntimeException;
-import org.apache.avro.io.Decoder;
-import org.apache.avro.io.JsonDecoder;
-import org.apache.avro.io.DatumReader;
-import org.apache.avro.specific.SpecificDatumReader;
-
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class EventReader implements Closeable {
@@ -46,7 +43,7 @@ public class EventReader implements Clos
private Schema schema;
private DataInputStream in;
private Decoder decoder;
- private DatumReader reader;
+ private DatumReader<Event> reader;
/**
* Create a new Event Reader
@@ -73,8 +70,8 @@ public class EventReader implements Clos
}
this.schema = Schema.parse(in.readLine());
- this.reader = new SpecificDatumReader(schema);
- this.decoder = new JsonDecoder(schema, in);
+ this.reader = new SpecificDatumReader<Event>(schema);
+ this.decoder = DecoderFactory.get().jsonDecoder(schema, in);
}
/**
@@ -82,11 +79,10 @@ public class EventReader implements Clos
* @return the next event
* @throws IOException
*/
- @SuppressWarnings("unchecked")
public HistoryEvent getNextEvent() throws IOException {
Event wrapper;
try {
- wrapper = (Event)reader.read(null, decoder);
+ wrapper = reader.read(null, decoder);
} catch (EOFException e) { // at EOF
return null;
}
Modified: hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java?rev=1176694&r1=1176693&r2=1176694&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java (original)
+++ hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java Wed Sep 28 03:11:40 2011
@@ -19,20 +19,18 @@
package org.apache.hadoop.mapreduce.jobhistory;
import java.io.IOException;
-import java.util.Iterator;
-
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.mapreduce.Counter;
-import org.apache.hadoop.mapreduce.CounterGroup;
-import org.apache.hadoop.mapreduce.Counters;
import org.apache.avro.Schema;
-import org.apache.avro.io.Encoder;
-import org.apache.avro.io.JsonEncoder;
+import org.apache.avro.generic.GenericData;
import org.apache.avro.io.DatumWriter;
+import org.apache.avro.io.Encoder;
+import org.apache.avro.io.EncoderFactory;
import org.apache.avro.specific.SpecificDatumWriter;
-import org.apache.avro.generic.GenericData;
import org.apache.avro.util.Utf8;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.mapreduce.CounterGroup;
+import org.apache.hadoop.mapreduce.Counters;
/**
* Event Writer is an utility class used to write events to the underlying
@@ -54,7 +52,7 @@ class EventWriter {
out.writeBytes("\n");
out.writeBytes(Event.SCHEMA$.toString());
out.writeBytes("\n");
- this.encoder = new JsonEncoder(Event.SCHEMA$, out);
+ this.encoder = EncoderFactory.get().jsonEncoder(Event.SCHEMA$, out);
}
synchronized void write(HistoryEvent event) throws IOException {
Modified: hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java?rev=1176694&r1=1176693&r2=1176694&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java (original)
+++ hadoop/common/branches/branch-0.22/mapreduce/src/java/org/apache/hadoop/mapreduce/jobhistory/JobSubmittedEvent.java Wed Sep 28 03:11:40 2011
@@ -57,7 +57,7 @@ public class JobSubmittedEvent implement
datum.userName = new Utf8(userName);
datum.submitTime = submitTime;
datum.jobConfPath = new Utf8(jobConfPath);
- Map<Utf8, Utf8> jobAcls = new HashMap<Utf8, Utf8>();
+ Map<CharSequence, CharSequence> jobAcls = new HashMap<CharSequence, CharSequence>();
for (Entry<JobACL, AccessControlList> entry : jobACLs.entrySet()) {
jobAcls.put(new Utf8(entry.getKey().getAclName()), new Utf8(
entry.getValue().getAclString()));