You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2014/05/20 17:12:11 UTC
svn commit: r1596285 - in /hive/trunk:
ql/src/java/org/apache/hadoop/hive/ql/exec/
ql/src/java/org/apache/hadoop/hive/ql/io/
serde/src/java/org/apache/hadoop/hive/serde2/lazy/
serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/
Author: hashutosh
Date: Tue May 20 15:12:11 2014
New Revision: 1596285
URL: http://svn.apache.org/r1596285
Log:
HIVE-5342 : Remove pre hadoop-0.20.0 related codes (Jason Dere via Ashutosh Chauhan)
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SkewJoinHandler.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/SymlinkTextInputFormat.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SkewJoinHandler.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SkewJoinHandler.java?rev=1596285&r1=1596284&r2=1596285&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SkewJoinHandler.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/SkewJoinHandler.java Tue May 20 15:12:11 2014
@@ -317,8 +317,6 @@ public class SkewJoinHandler {
Path outPath = getOperatorOutputPath(specPath);
Path finalPath = getOperatorFinalPath(specPath);
FileSystem fs = outPath.getFileSystem(hconf);
- // for local file system in Hadoop-0.17.2.1, it will throw IOException when
- // file not existing.
try {
if (!fs.rename(outPath, finalPath)) {
throw new IOException("Unable to rename output to: " + finalPath);
@@ -327,11 +325,6 @@ public class SkewJoinHandler {
if (!ignoreNonExisting) {
throw e;
}
- } catch (IOException e) {
- if (!fs.exists(outPath) && ignoreNonExisting) {
- return;
- }
- throw e;
}
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1596285&r1=1596284&r2=1596285&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Tue May 20 15:12:11 2014
@@ -1431,10 +1431,6 @@ public final class Utilities {
}
String file = path.makeQualified(fs).toString();
- // For compatibility with hadoop 0.17, change file:/a/b/c to file:///a/b/c
- if (StringUtils.startsWith(file, "file:/") && !StringUtils.startsWith(file, "file:///")) {
- file = "file:///" + file.substring("file:/".length());
- }
return file;
}
@@ -1936,6 +1932,26 @@ public final class Utilities {
}
/**
+ * Create a URL from a string representing a path to a local file.
+ * The path string can be just a path, or can start with file:/, file:///
+ * @param onestr path string
+ * @return
+ */
+ private static URL urlFromPathString(String onestr) {
+ URL oneurl = null;
+ try {
+ if (StringUtils.indexOf(onestr, "file:/") == 0) {
+ oneurl = new URL(onestr);
+ } else {
+ oneurl = new File(onestr).toURL();
+ }
+ } catch (Exception err) {
+ LOG.error("Bad URL " + onestr + ", ignoring path");
+ }
+ return oneurl;
+ }
+
+ /**
* Add new elements to the classpath.
*
* @param newPaths
@@ -1953,13 +1969,8 @@ public final class Utilities {
curPath = newPath;
for (String onestr : newPaths) {
- // special processing for hadoop-17. file:// needs to be removed
- if (StringUtils.indexOf(onestr, "file://") == 0) {
- onestr = StringUtils.substring(onestr, 7);
- }
-
- URL oneurl = (new File(onestr)).toURL();
- if (!curPath.contains(oneurl)) {
+ URL oneurl = urlFromPathString(onestr);
+ if (oneurl != null && !curPath.contains(oneurl)) {
curPath.add(oneurl);
}
}
@@ -1979,13 +1990,10 @@ public final class Utilities {
Set<URL> newPath = new HashSet<URL>(Arrays.asList(loader.getURLs()));
for (String onestr : pathsToRemove) {
- // special processing for hadoop-17. file:// needs to be removed
- if (StringUtils.indexOf(onestr, "file://") == 0) {
- onestr = StringUtils.substring(onestr, 7);
+ URL oneurl = urlFromPathString(onestr);
+ if (oneurl != null) {
+ newPath.remove(oneurl);
}
-
- URL oneurl = (new File(onestr)).toURL();
- newPath.remove(oneurl);
}
loader = new URLClassLoader(newPath.toArray(new URL[0]));
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/SymlinkTextInputFormat.java
URL: http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/SymlinkTextInputFormat.java?rev=1596285&r1=1596284&r2=1596285&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/SymlinkTextInputFormat.java (original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/SymlinkTextInputFormat.java Tue May 20 15:12:11 2014
@@ -213,13 +213,6 @@ public class SymlinkTextInputFormat exte
}
}
- /**
- * For backward compatibility with hadoop 0.17.
- */
- public void validateInput(JobConf job) throws IOException {
- // do nothing
- }
-
@Override
public ContentSummary getContentSummary(Path p, JobConf job)
throws IOException {
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java?rev=1596285&r1=1596284&r2=1596285&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySimpleSerDe.java Tue May 20 15:12:11 2014
@@ -47,7 +47,7 @@ import org.apache.hadoop.hive.serde2.typ
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
-import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.BinaryComparable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
@@ -334,19 +334,9 @@ public class LazySimpleSerDe extends Abs
if (byteArrayRef == null) {
byteArrayRef = new ByteArrayRef();
}
- if (field instanceof BytesWritable) {
- BytesWritable b = (BytesWritable) field;
- // For backward-compatibility with hadoop 0.17
- byteArrayRef.setData(b.getBytes());
- cachedLazyStruct.init(byteArrayRef, 0, b.getLength());
- } else if (field instanceof Text) {
- Text t = (Text) field;
- byteArrayRef.setData(t.getBytes());
- cachedLazyStruct.init(byteArrayRef, 0, t.getLength());
- } else {
- throw new SerDeException(getClass().toString()
- + ": expects either BytesWritable or Text object!");
- }
+ BinaryComparable b = (BinaryComparable) field;
+ byteArrayRef.setData(b.getBytes());
+ cachedLazyStruct.init(byteArrayRef, 0, b.getLength());
lastOperationSerialize = false;
lastOperationDeserialize = true;
return cachedLazyStruct;
Modified: hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java
URL: http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java?rev=1596285&r1=1596284&r2=1596285&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java (original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazybinary/LazyBinarySerDe.java Tue May 20 15:12:11 2014
@@ -162,25 +162,12 @@ public class LazyBinarySerDe extends Abs
if (byteArrayRef == null) {
byteArrayRef = new ByteArrayRef();
}
- if (field instanceof BinaryComparable) {
- BinaryComparable b = (BinaryComparable) field;
- if (b.getLength() == 0) {
- return null;
- }
- // For backward-compatibility with hadoop 0.17
- byteArrayRef.setData(b.getBytes());
- cachedLazyBinaryStruct.init(byteArrayRef, 0, b.getLength());
- } else if (field instanceof Text) {
- Text t = (Text) field;
- if (t.getLength() == 0) {
- return null;
- }
- byteArrayRef.setData(t.getBytes());
- cachedLazyBinaryStruct.init(byteArrayRef, 0, t.getLength());
- } else {
- throw new SerDeException(getClass().toString()
- + ": expects either BinaryComparable or Text object!");
+ BinaryComparable b = (BinaryComparable) field;
+ if (b.getLength() == 0) {
+ return null;
}
+ byteArrayRef.setData(b.getBytes());
+ cachedLazyBinaryStruct.init(byteArrayRef, 0, b.getLength());
lastOperationSerialize = false;
lastOperationDeserialize = true;
return cachedLazyBinaryStruct;