You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by cw...@apache.org on 2012/09/10 00:23:53 UTC
svn commit: r1382600 - in /hive/trunk/shims/src:
0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
Author: cws
Date: Sun Sep 9 22:23:53 2012
New Revision: 1382600
URL: http://svn.apache.org/viewvc?rev=1382600&view=rev
Log:
HIVE-3387. Meta data file size exceeds limit (Navis Ryu via cws)
Modified:
hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
Modified: hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java?rev=1382600&r1=1382599&r2=1382600&view=diff
==============================================================================
--- hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java (original)
+++ hive/trunk/shims/src/0.20/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java Sun Sep 9 22:23:53 2012
@@ -28,7 +28,10 @@ import java.net.URL;
import java.security.PrivilegedActionException;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import javax.security.auth.Subject;
import javax.security.auth.login.LoginException;
@@ -160,10 +163,17 @@ public class Hadoop20Shims implements Ha
}
public InputSplitShim(CombineFileSplit old) throws IOException {
- super(old);
+ super(old.getJob(), old.getPaths(), old.getStartOffsets(),
+ old.getLengths(), dedup(old.getLocations()));
_isShrinked = false;
}
+ private static String[] dedup(String[] locations) {
+ Set<String> dedup = new HashSet<String>();
+ Collections.addAll(dedup, locations);
+ return dedup.toArray(new String[dedup.size()]);
+ }
+
@Override
public void shrinkSplit(long length) {
_isShrinked = true;
Modified: hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java
URL: http://svn.apache.org/viewvc/hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java?rev=1382600&r1=1382599&r2=1382600&view=diff
==============================================================================
--- hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java (original)
+++ hive/trunk/shims/src/common-secure/java/org/apache/hadoop/hive/shims/HadoopShimsSecure.java Sun Sep 9 22:23:53 2012
@@ -25,7 +25,10 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -165,10 +168,17 @@ public abstract class HadoopShimsSecure
}
public InputSplitShim(CombineFileSplit old) throws IOException {
- super(old);
+ super(old.getJob(), old.getPaths(), old.getStartOffsets(),
+ old.getLengths(), dedup(old.getLocations()));
_isShrinked = false;
}
+ private static String[] dedup(String[] locations) {
+ Set<String> dedup = new HashSet<String>();
+ Collections.addAll(dedup, locations);
+ return dedup.toArray(new String[dedup.size()]);
+ }
+
@Override
public void shrinkSplit(long length) {
_isShrinked = true;