You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2017/11/29 06:41:51 UTC
[2/7] impala git commit: IMPALA-6053: Fix exception when storadeIds
don't match hosts
IMPALA-6053: Fix exception when storadeIds don't match hosts
This commit fixes an issue where an IllegalStateException is thrown if
there is a mismatch between the number of storageIDs and the number of
host locations of a file block, causing the metadata load of a table to
abort. With this fix, the storadeIDs are ignored if they don't match the
number of hosts of a block, allowing table loading to proceed. That
change will also cause remote reads during table scans for the
blocks for which the mismatch was detected.
Testing:
No additional tests were added as this error was triggered on an EMC
Isilon system v8.0.
Change-Id: Ia3d685208dce7a1cbe94a33b8ac9aeb7c8a3f391
Reviewed-on: http://gerrit.cloudera.org:8080/8668
Reviewed-by: Bharath Vissapragada <bh...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/0588309e
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/0588309e
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/0588309e
Branch: refs/heads/master
Commit: 0588309ed6938fd7c280a241be01d0b69b6357d9
Parents: 4f11bed
Author: Dimitris Tsirogiannis <dt...@cloudera.com>
Authored: Tue Nov 28 11:02:17 2017 -0800
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Tue Nov 28 23:40:38 2017 +0000
----------------------------------------------------------------------
.../apache/impala/catalog/HdfsPartition.java | 27 ++++++++------------
1 file changed, 11 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/0588309e/fe/src/main/java/org/apache/impala/catalog/HdfsPartition.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/HdfsPartition.java b/fe/src/main/java/org/apache/impala/catalog/HdfsPartition.java
index 35309c5..e78ce92 100644
--- a/fe/src/main/java/org/apache/impala/catalog/HdfsPartition.java
+++ b/fe/src/main/java/org/apache/impala/catalog/HdfsPartition.java
@@ -20,7 +20,6 @@ package org.apache.impala.catalog;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Collection;
-import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -68,7 +67,6 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.common.primitives.Ints;
-import com.google.common.primitives.Shorts;
import com.google.flatbuffers.FlatBufferBuilder;
/**
@@ -308,10 +306,10 @@ public class HdfsPartition implements Comparable<HdfsPartition> {
fbb.addShort(replicaIdx);
}
int fbReplicaHostIdxOffset = fbb.endVector();
-
- // disk ids
short[] diskIds = createDiskIds(loc, numUnknownDiskIds);
- Preconditions.checkState(diskIds.length != 0);
+ Preconditions.checkState(diskIds.length == loc.getNames().length,
+ "Mismatch detected between number of diskIDs and block locations for block: " +
+ loc.toString());
int fbDiskIdsOffset = FbFileBlock.createDiskIdsVector(fbb, diskIds);
FbFileBlock.startFbFileBlock(fbb);
FbFileBlock.addOffset(fbb, loc.getOffset());
@@ -345,20 +343,17 @@ public class HdfsPartition implements Comparable<HdfsPartition> {
* disk ids and populates 'numUnknownDiskIds' with the number of unknown disk ids.
*/
private static short[] createDiskIds(BlockLocation location,
- Reference<Long> numUnknownDiskIds) {
+ Reference<Long> numUnknownDiskIds) throws IOException {
long unknownDiskIdCount = 0;
String[] storageIds = location.getStorageIds();
- String[] hosts;
- try {
- hosts = location.getHosts();
- } catch (IOException e) {
- LOG.error("Couldn't get hosts for block: " + location.toString(), e);
- return new short[0];
- }
+ String[] hosts = location.getHosts();
if (storageIds.length != hosts.length) {
- LOG.error("Number of storage IDs and number of hosts for block: " + location
- .toString() + " mismatch. Skipping disk ID loading for this block.");
- return Shorts.toArray(Collections.<Short>emptyList());
+ if (LOG.isTraceEnabled()) {
+ LOG.trace(String.format("Number of storage IDs and number of hosts for block " +
+ "%s mismatch (storageIDs:hosts) %d:%d. Skipping disk ID loading for this " +
+ "block.", location.toString(), storageIds.length, hosts.length));
+ }
+ storageIds = new String[hosts.length];
}
short[] diskIDs = new short[storageIds.length];
for (int i = 0; i < storageIds.length; ++i) {