You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by gu...@apache.org on 2014/09/20 01:26:49 UTC
svn commit: r1626364 [2/2] - in /hive/branches/cbo: ./
accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/mr/
accumulo-handler/src/java/org/apache/hadoop/hive/accumulo/serde/
itests/util/src/main/java/org/apache/hadoop/hive/ql/security/ itests/u...
Modified: hive/branches/cbo/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java?rev=1626364&r1=1626363&r2=1626364&view=diff
==============================================================================
--- hive/branches/cbo/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java (original)
+++ hive/branches/cbo/shims/0.20/src/main/java/org/apache/hadoop/hive/shims/Hadoop20Shims.java Fri Sep 19 23:26:48 2014
@@ -37,6 +37,7 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.TreeMap;
import javax.security.auth.Subject;
import javax.security.auth.login.LoginException;
@@ -652,6 +653,17 @@ public class Hadoop20Shims implements Ha
}
@Override
+ public TreeMap<Long, BlockLocation> getLocationsWithOffset(FileSystem fs,
+ FileStatus status) throws IOException {
+ TreeMap<Long, BlockLocation> offsetBlockMap = new TreeMap<Long, BlockLocation>();
+ BlockLocation[] locations = getLocations(fs, status);
+ for (BlockLocation location : locations) {
+ offsetBlockMap.put(location.getOffset(), location);
+ }
+ return offsetBlockMap;
+ }
+
+ @Override
public void hflush(FSDataOutputStream stream) throws IOException {
stream.sync();
}
Modified: hive/branches/cbo/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java?rev=1626364&r1=1626363&r2=1626364&view=diff
==============================================================================
--- hive/branches/cbo/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java (original)
+++ hive/branches/cbo/shims/0.20S/src/main/java/org/apache/hadoop/hive/shims/Hadoop20SShims.java Fri Sep 19 23:26:48 2014
@@ -27,6 +27,7 @@ import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.TreeMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.filecache.DistributedCache;
@@ -403,6 +404,17 @@ public class Hadoop20SShims extends Hado
}
@Override
+ public TreeMap<Long, BlockLocation> getLocationsWithOffset(FileSystem fs,
+ FileStatus status) throws IOException {
+ TreeMap<Long, BlockLocation> offsetBlockMap = new TreeMap<Long, BlockLocation>();
+ BlockLocation[] locations = getLocations(fs, status);
+ for (BlockLocation location : locations) {
+ offsetBlockMap.put(location.getOffset(), location);
+ }
+ return offsetBlockMap;
+ }
+
+ @Override
public void hflush(FSDataOutputStream stream) throws IOException {
stream.sync();
}
Modified: hive/branches/cbo/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java?rev=1626364&r1=1626363&r2=1626364&view=diff
==============================================================================
--- hive/branches/cbo/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java (original)
+++ hive/branches/cbo/shims/0.23/src/main/java/org/apache/hadoop/hive/shims/Hadoop23Shims.java Fri Sep 19 23:26:48 2014
@@ -29,6 +29,7 @@ import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.TreeMap;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
@@ -511,6 +512,17 @@ public class Hadoop23Shims extends Hadoo
}
@Override
+ public TreeMap<Long, BlockLocation> getLocationsWithOffset(FileSystem fs,
+ FileStatus status) throws IOException {
+ TreeMap<Long, BlockLocation> offsetBlockMap = new TreeMap<Long, BlockLocation>();
+ BlockLocation[] locations = getLocations(fs, status);
+ for (BlockLocation location : locations) {
+ offsetBlockMap.put(location.getOffset(), location);
+ }
+ return offsetBlockMap;
+ }
+
+ @Override
public void hflush(FSDataOutputStream stream) throws IOException {
stream.hflush();
}
Modified: hive/branches/cbo/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java
URL: http://svn.apache.org/viewvc/hive/branches/cbo/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java?rev=1626364&r1=1626363&r2=1626364&view=diff
==============================================================================
--- hive/branches/cbo/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java (original)
+++ hive/branches/cbo/shims/common/src/main/java/org/apache/hadoop/hive/shims/HadoopShims.java Fri Sep 19 23:26:48 2014
@@ -30,6 +30,7 @@ import java.security.PrivilegedException
import java.util.Comparator;
import java.util.List;
import java.util.Map;
+import java.util.TreeMap;
import javax.security.auth.login.LoginException;
@@ -477,6 +478,19 @@ public interface HadoopShims {
FileStatus status) throws IOException;
/**
+ * For the block locations returned by getLocations() convert them into a Treemap
+ * <Offset,blockLocation> by iterating over the list of blockLocation.
+ * Using TreeMap from offset to blockLocation, makes it O(logn) to get a particular
+ * block based upon offset.
+ * @param fs the file system
+ * @param status the file information
+ * @return TreeMap<Long, BlockLocation>
+ * @throws IOException
+ */
+ TreeMap<Long, BlockLocation> getLocationsWithOffset(FileSystem fs,
+ FileStatus status) throws IOException;
+
+ /**
* Flush and make visible to other users the changes to the given stream.
* @param stream the stream to hflush.
* @throws IOException