You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hcatalog-commits@incubator.apache.org by tr...@apache.org on 2012/10/03 17:30:01 UTC
svn commit: r1393629 - in /incubator/hcatalog/trunk: ./
hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/
server-extensions/ shims/ shims/src/20/java/org/apache/hcatalog/shims/
shims/src/23/java/org/apache/hcatalog/shims/ src/java/org/apache/...
Author: travis
Date: Wed Oct 3 17:30:01 2012
New Revision: 1393629
URL: http://svn.apache.org/viewvc?rev=1393629&view=rev
Log:
HCATALOG-522 Make HCatHadoopShims.commitJob() generic
Modified:
incubator/hcatalog/trunk/CHANGES.txt
incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatStorer.java
incubator/hcatalog/trunk/ivy.xml
incubator/hcatalog/trunk/server-extensions/pom.xml
incubator/hcatalog/trunk/shims/ivy.xml
incubator/hcatalog/trunk/shims/src/20/java/org/apache/hcatalog/shims/HCatHadoopShims20S.java
incubator/hcatalog/trunk/shims/src/23/java/org/apache/hcatalog/shims/HCatHadoopShims23.java
incubator/hcatalog/trunk/src/java/org/apache/hcatalog/shims/HCatHadoopShims.java
Modified: incubator/hcatalog/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/CHANGES.txt?rev=1393629&r1=1393628&r2=1393629&view=diff
==============================================================================
--- incubator/hcatalog/trunk/CHANGES.txt (original)
+++ incubator/hcatalog/trunk/CHANGES.txt Wed Oct 3 17:30:01 2012
@@ -119,6 +119,8 @@ Trunk (unreleased changes)
OPTIMIZATIONS
BUG FIXES
+ HCAT-522 Make HCatHadoopShims.commitJob() generic (traviscrawford)
+
HCAT-451 Partitions are created even when Jobs are aborted (avandana)
HCAT-513 Data Store onto HCatalog table fails for dynamic partitioning as the temporary directory gets deleted by the completed map tasks (amalakar via toffer)
Modified: incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatStorer.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatStorer.java?rev=1393629&r1=1393628&r2=1393629&view=diff
==============================================================================
--- incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatStorer.java (original)
+++ incubator/hcatalog/trunk/hcatalog-pig-adapter/src/main/java/org/apache/hcatalog/pig/HCatStorer.java Wed Oct 3 17:30:01 2012
@@ -156,7 +156,7 @@ public class HCatStorer extends HCatBase
@Override
public void storeSchema(ResourceSchema schema, String arg1, Job job) throws IOException {
- HCatHadoopShims.Instance.get().commitJob(getOutputFormat(), schema, arg1, job);
+ HCatHadoopShims.Instance.get().commitJob(getOutputFormat(), job);
}
@Override
Modified: incubator/hcatalog/trunk/ivy.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/ivy.xml?rev=1393629&r1=1393628&r2=1393629&view=diff
==============================================================================
--- incubator/hcatalog/trunk/ivy.xml (original)
+++ incubator/hcatalog/trunk/ivy.xml Wed Oct 3 17:30:01 2012
@@ -36,7 +36,7 @@
<dependencies>
<!-- hadoop20 dependencies -->
- <dependency org="org.apache.pig" name="pig" rev="${pig.version}" conf="default->*"/>
+ <dependency org="org.apache.pig" name="pig" rev="${pig.version}" conf="test->*"/>
<dependency org="org.apache.hadoop" name="hadoop-tools" rev="${hadoop20.version}" conf="default->*"/>
<dependency org="javax.management.j2ee" name="management-api" rev="${javax-mgmt.version}"/>
<dependency org="org.codehaus.jackson" name="jackson-mapper-asl" rev="${jackson.version}"/>
Modified: incubator/hcatalog/trunk/server-extensions/pom.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/server-extensions/pom.xml?rev=1393629&r1=1393628&r2=1393629&view=diff
==============================================================================
--- incubator/hcatalog/trunk/server-extensions/pom.xml (original)
+++ incubator/hcatalog/trunk/server-extensions/pom.xml Wed Oct 3 17:30:01 2012
@@ -79,5 +79,11 @@
<version>${hadoop20.version}</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.pig</groupId>
+ <artifactId>pig</artifactId>
+ <version>${pig.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
Modified: incubator/hcatalog/trunk/shims/ivy.xml
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/shims/ivy.xml?rev=1393629&r1=1393628&r2=1393629&view=diff
==============================================================================
--- incubator/hcatalog/trunk/shims/ivy.xml (original)
+++ incubator/hcatalog/trunk/shims/ivy.xml Wed Oct 3 17:30:01 2012
@@ -11,8 +11,6 @@
<conf name="hadoop23" visibility="private"/>
</configurations>
<dependencies>
- <dependency org="org.apache.pig" name="pig" rev="${pig.version}" conf="hadoop20->*;hadoop23->*"/>
-
<!-- Hadoop 0.20 shim dependencies. Used for building 0.20 shims. -->
<dependency org="org.apache.hadoop" name="hadoop-tools" rev="${hadoop20.version}" conf="hadoop20->*">
<exclude org="commons-daemon" module="commons-daemon"/>
Modified: incubator/hcatalog/trunk/shims/src/20/java/org/apache/hcatalog/shims/HCatHadoopShims20S.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/shims/src/20/java/org/apache/hcatalog/shims/HCatHadoopShims20S.java?rev=1393629&r1=1393628&r2=1393629&view=diff
==============================================================================
--- incubator/hcatalog/trunk/shims/src/20/java/org/apache/hcatalog/shims/HCatHadoopShims20S.java (original)
+++ incubator/hcatalog/trunk/shims/src/20/java/org/apache/hcatalog/shims/HCatHadoopShims20S.java Wed Oct 3 17:30:01 2012
@@ -33,7 +33,6 @@ import org.apache.hadoop.mapreduce.TaskA
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.util.Progressable;
-import org.apache.pig.ResourceSchema;
public class HCatHadoopShims20S implements HCatHadoopShims {
@Override
@@ -90,8 +89,7 @@ public class HCatHadoopShims20S implemen
}
@Override
- public void commitJob(OutputFormat outputFormat, ResourceSchema schema,
- String arg1, Job job) throws IOException {
+ public void commitJob(OutputFormat outputFormat, Job job) throws IOException {
if( job.getConfiguration().get("mapred.job.tracker", "").equalsIgnoreCase("local") ) {
try {
//In local mode, mapreduce will not call OutputCommitter.cleanupJob.
Modified: incubator/hcatalog/trunk/shims/src/23/java/org/apache/hcatalog/shims/HCatHadoopShims23.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/shims/src/23/java/org/apache/hcatalog/shims/HCatHadoopShims23.java?rev=1393629&r1=1393628&r2=1393629&view=diff
==============================================================================
--- incubator/hcatalog/trunk/shims/src/23/java/org/apache/hcatalog/shims/HCatHadoopShims23.java (original)
+++ incubator/hcatalog/trunk/shims/src/23/java/org/apache/hcatalog/shims/HCatHadoopShims23.java Wed Oct 3 17:30:01 2012
@@ -32,7 +32,6 @@ import org.apache.hadoop.mapreduce.TaskI
import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.task.JobContextImpl;
import org.apache.hadoop.util.Progressable;
-import org.apache.pig.ResourceSchema;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.net.NetUtils;
@@ -88,8 +87,7 @@ public class HCatHadoopShims23 implement
}
@Override
- public void commitJob(OutputFormat outputFormat, ResourceSchema schema,
- String arg1, Job job) throws IOException {
+ public void commitJob(OutputFormat outputFormat, Job job) throws IOException {
// Do nothing as this was fixed by MAPREDUCE-1447.
}
Modified: incubator/hcatalog/trunk/src/java/org/apache/hcatalog/shims/HCatHadoopShims.java
URL: http://svn.apache.org/viewvc/incubator/hcatalog/trunk/src/java/org/apache/hcatalog/shims/HCatHadoopShims.java?rev=1393629&r1=1393628&r2=1393629&view=diff
==============================================================================
--- incubator/hcatalog/trunk/src/java/org/apache/hcatalog/shims/HCatHadoopShims.java (original)
+++ incubator/hcatalog/trunk/src/java/org/apache/hcatalog/shims/HCatHadoopShims.java Wed Oct 3 17:30:01 2012
@@ -30,7 +30,6 @@ import org.apache.hadoop.mapreduce.Outpu
import org.apache.hadoop.mapreduce.TaskAttemptID;
import org.apache.hadoop.mapreduce.TaskID;
import org.apache.hadoop.util.Progressable;
-import org.apache.pig.ResourceSchema;
/**
* Shim layer to abstract differences between Hadoop 0.20 and 0.23
@@ -81,8 +80,7 @@ public interface HCatHadoopShims {
public org.apache.hadoop.mapred.JobContext createJobContext(JobConf conf, JobID jobId, Progressable progressable);
- public void commitJob(OutputFormat outputFormat, ResourceSchema schema,
- String arg1, Job job) throws IOException;
+ public void commitJob(OutputFormat outputFormat, Job job) throws IOException;
public void abortJob(OutputFormat outputFormat, Job job) throws IOException;