You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by br...@apache.org on 2014/11/24 04:10:18 UTC
svn commit: r1641310 - in /hive/trunk:
ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/
shims/0.20/src/main/java/org/apache/hadoop/hive/shims/
shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/
shims/0.23/src/main/java/org/apache/hadoop/hive/s...
Author: brock
Date: Mon Nov 24 03:10:17 2014
New Revision: 1641310
URL: http://svn.apache.org/r1641310
Log:
HIVE-8944 - TestCompactor fails with IncompatibleClassChangeError (Alan Gates via Brock)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
hive/trunk/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
hive/trunk/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
hive/trunk/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java?rev=1641310&r1=1641309&r2=1641310&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactorMR.java Mon Nov 24 03:10:17 2014
@@ -37,6 +37,7 @@ import org.apache.hadoop.hive.ql.io.Acid
import org.apache.hadoop.hive.ql.io.RecordIdentifier;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
@@ -697,9 +698,10 @@ public class CompactorMR {
@Override
public void commitJob(JobContext context) throws IOException {
- Path tmpLocation = new Path(context.getJobConf().get(TMP_LOCATION));
- Path finalLocation = new Path(context.getJobConf().get(FINAL_LOCATION));
- FileSystem fs = tmpLocation.getFileSystem(context.getJobConf());
+ JobConf conf = ShimLoader.getHadoopShims().getJobConf(context);
+ Path tmpLocation = new Path(conf.get(TMP_LOCATION));
+ Path finalLocation = new Path(conf.get(FINAL_LOCATION));
+ FileSystem fs = tmpLocation.getFileSystem(conf);
LOG.debug("Moving contents of " + tmpLocation.toString() + " to " +
finalLocation.toString());
@@ -713,8 +715,9 @@ public class CompactorMR {
@Override
public void abortJob(JobContext context, int status) throws IOException {
- Path tmpLocation = new Path(context.getJobConf().get(TMP_LOCATION));
- FileSystem fs = tmpLocation.getFileSystem(context.getJobConf());
+ JobConf conf = ShimLoader.getHadoopShims().getJobConf(context);
+ Path tmpLocation = new Path(conf.get(TMP_LOCATION));
+ FileSystem fs = tmpLocation.getFileSystem(conf);
LOG.debug("Removing " + tmpLocation.toString());
fs.delete(tmpLocation, true);
}
Modified: hive/trunk/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java?rev=1641310&r1=1641309&r2=1641310&view=diff
==============================================================================
--- hive/trunk/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java (original)
+++ hive/trunk/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java Mon Nov 24 03:10:17 2014
@@ -889,6 +889,11 @@ public class Hadoop20Shims implements Ha
}
@Override
+ public JobConf getJobConf(JobContext context) {
+ return context.getJobConf();
+ }
+
+ @Override
public FileSystem getNonCachedFileSystem(URI uri, Configuration conf) throws IOException {
boolean origDisableHDFSCache =
conf.getBoolean("fs." + uri.getScheme() + ".impl.disable.cache", false);
Modified: hive/trunk/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java?rev=1641310&r1=1641309&r2=1641310&view=diff
==============================================================================
--- hive/trunk/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java (original)
+++ hive/trunk/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java Mon Nov 24 03:10:17 2014
@@ -512,6 +512,11 @@ public class Hadoop20SShims extends Hado
}
@Override
+ public JobConf getJobConf(org.apache.hadoop.mapred.JobContext context) {
+ return context.getJobConf();
+ }
+
+ @Override
public FileSystem getNonCachedFileSystem(URI uri, Configuration conf) throws IOException {
boolean origDisableHDFSCache =
conf.getBoolean("fs." + uri.getScheme() + ".impl.disable.cache", false);
Modified: hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java?rev=1641310&r1=1641309&r2=1641310&view=diff
==============================================================================
--- hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java (original)
+++ hive/trunk/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java Mon Nov 24 03:10:17 2014
@@ -774,6 +774,11 @@ public class Hadoop23Shims extends Hadoo
}
@Override
+ public JobConf getJobConf(org.apache.hadoop.mapred.JobContext context) {
+ return context.getJobConf();
+ }
+
+ @Override
public FileSystem getNonCachedFileSystem(URI uri, Configuration conf) throws IOException {
return FileSystem.newInstance(uri, conf);
}
Modified: hive/trunk/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java?rev=1641310&r1=1641309&r2=1641310&view=diff
==============================================================================
--- hive/trunk/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java (original)
+++ hive/trunk/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java Mon Nov 24 03:10:17 2014
@@ -708,6 +708,13 @@ public interface HadoopShims {
*/
public Configuration getConfiguration(JobContext context);
+ /**
+ * Get job conf from the old style JobContext.
+ * @param context job context
+ * @return job conf
+ */
+ public JobConf getJobConf(org.apache.hadoop.mapred.JobContext context);
+
public FileSystem getNonCachedFileSystem(URI uri, Configuration conf) throws IOException;
public void getMergedCredentials(JobConf jobConf) throws IOException;