You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by mi...@apache.org on 2017/06/27 12:27:49 UTC
[1/6] incubator-impala git commit: IMPALA-5551: Fix
AggregationNode::Close() when Prepare() fails
Repository: incubator-impala
Updated Branches:
refs/heads/master 67b63f37e -> 8a77fff9a
IMPALA-5551: Fix AggregationNode::Close() when Prepare() fails
A recent change moved the initialization of output_tuple_desc_
to the constructor of AggregationNode. This changes the order
in which tuple_pool_ and output_tuple_desc_ are initialized.
The code in AggregationNode::Close() made the assumption that
tuple_pool_ cannot be NULL (although without a DCHECK) if
output_tuple_desc_ is not NULL. This no longer holds in the
new code.
This change makes AggregationNode::Close() checks tuple_pool_
to see if it's NULL before using it.
Change-Id: Ia0f8cce7cf6e3183d3a5e145c2fcfa50f36c77e0
Reviewed-on: http://gerrit.cloudera.org:8080/7254
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/384e864f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/384e864f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/384e864f
Branch: refs/heads/master
Commit: 384e864fa19cc004e66c4139508cf44a41e0479c
Parents: 67b63f3
Author: Michael Ho <kw...@cloudera.com>
Authored: Wed Jun 21 09:01:41 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Mon Jun 26 23:14:07 2017 +0000
----------------------------------------------------------------------
be/src/exec/aggregation-node.cc | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/384e864f/be/src/exec/aggregation-node.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/aggregation-node.cc b/be/src/exec/aggregation-node.cc
index 17820ff..13f7dd3 100644
--- a/be/src/exec/aggregation-node.cc
+++ b/be/src/exec/aggregation-node.cc
@@ -107,6 +107,7 @@ Status AggregationNode::Init(const TPlanNode& tnode, RuntimeState* state) {
}
Status AggregationNode::Prepare(RuntimeState* state) {
+ DCHECK(output_iterator_.AtEnd());
SCOPED_TIMER(runtime_profile_->total_time_counter());
RETURN_IF_ERROR(ExecNode::Prepare(state));
@@ -268,7 +269,8 @@ void AggregationNode::Close(RuntimeState* state) {
// but we don't actually need the result, so allocate a single dummy tuple to avoid
// accumulating memory.
Tuple* dummy_dst = nullptr;
- if (needs_finalize_ && output_tuple_desc_ != nullptr) {
+ // 'tuple_pool_' can be NULL if Prepare() failed.
+ if (needs_finalize_ && tuple_pool_.get() != nullptr) {
dummy_dst = Tuple::Create(output_tuple_desc_->byte_size(), tuple_pool_.get());
}
while (!output_iterator_.AtEnd()) {
[4/6] incubator-impala git commit: IMPALA-5431: Remove redundant path
exists checks during table load
Posted by mi...@apache.org.
IMPALA-5431: Remove redundant path exists checks during table load
There are multiple places that do an exists() check on a path and then
perform some subsequent action on it. This pattern results in two
RPCs to the NN (one for the exists() check and one for the subsequent
action). We can avoid the exists() check in these cases since most HDFS
methods on paths throw a FileNotFoundException if the path does not
exist. This can save an RPC to NN and improve the metadata loading time.
Testing: Enough tests already cover this code path. This patch
passed core and exhaustive tests.
Metadata benchmark shows decent increase in perf numbers, for ex:
100K-PARTITIONS-1M-FILES-CUSTOM-05-QUERY-AFTER-INV -20.51%
80-PARTITIONS-250K-FILES-S3-03-RECOVER -20.58%
80-PARTITIONS-250K-FILES-11-DROP-PARTITION -22.13%
80-PARTITIONS-250K-FILES-S3-08-ADD-PARTITION -22.38%
80-PARTITIONS-250K-FILES-S3-12-DROP -23.69%
100K-PARTITIONS-1M-FILES-CUSTOM-11-REFRESH-PARTITION -23.91%
100K-PARTITIONS-1M-FILES-CUSTOM-10-REFRESH-AFTER-ADD-PARTITION -26.04%
100K-PARTITIONS-1M-FILES-CUSTOM-07-REFRESH -26.38%
80-PARTITIONS-250K-FILES-S3-02-CREATE -36.47%
100K-PARTITIONS-1M-FILES-CUSTOM-12-QUERY-PARTITIONS -58.72%
80-PARTITIONS-250K-FILES-S3-01-DROP -95.33%
80-PARTITIONS-250K-FILES-01-DROP -95.93%
Change-Id: Id10ecf64ea2eda2d0f9299c0aa371933eca22281
Reviewed-on: http://gerrit.cloudera.org:8080/7095
Reviewed-by: Bharath Vissapragada <bh...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/d9fc9be0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/d9fc9be0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/d9fc9be0
Branch: refs/heads/master
Commit: d9fc9be02195605a63365d6171b75e95e646dab7
Parents: 466808a
Author: Bharath Vissapragada <bh...@cloudera.com>
Authored: Mon Jun 5 16:04:46 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Tue Jun 27 03:40:52 2017 +0000
----------------------------------------------------------------------
.../org/apache/impala/catalog/HdfsTable.java | 68 +++++++++-----------
.../apache/impala/common/FileSystemUtil.java | 35 +++++++++-
2 files changed, 64 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/d9fc9be0/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
index bbdbd16..6aca8dc 100644
--- a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java
@@ -17,6 +17,7 @@
package org.apache.impala.catalog;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
@@ -262,9 +263,8 @@ public class HdfsTable extends Table {
private void loadBlockMetadata(Path dirPath,
HashMap<Path, List<HdfsPartition>> partsByPath) {
try {
- FileSystem fs = dirPath.getFileSystem(CONF);
// No need to load blocks for empty partitions list.
- if (partsByPath.size() == 0 || !fs.exists(dirPath)) return;
+ if (partsByPath.size() == 0) return;
if (LOG.isTraceEnabled()) {
LOG.trace("Loading block md for " + name_ + " directory " + dirPath.toString());
}
@@ -291,6 +291,7 @@ public class HdfsTable extends Table {
}
}
+ FileSystem fs = dirPath.getFileSystem(CONF);
// For file systems that do not support BlockLocation API, we manually synthesize
// block location metadata based on file formats.
if (!FileSystemUtil.supportsStorageIds(fs)) {
@@ -298,8 +299,10 @@ public class HdfsTable extends Table {
return;
}
+ RemoteIterator<LocatedFileStatus> fileStatusIter =
+ FileSystemUtil.listFiles(fs, dirPath, true);
+ if (fileStatusIter == null) return;
Reference<Long> numUnknownDiskIds = new Reference<Long>(Long.valueOf(0));
- RemoteIterator<LocatedFileStatus> fileStatusIter = fs.listFiles(dirPath, true);
while (fileStatusIter.hasNext()) {
LocatedFileStatus fileStatus = fileStatusIter.next();
if (!FileSystemUtil.isValidDataFile(fileStatus)) continue;
@@ -360,7 +363,9 @@ public class HdfsTable extends Table {
*/
private void synthesizeBlockMetadata(FileSystem fs, Path dirPath, HashMap<Path,
List<HdfsPartition>> partsByPath) throws IOException {
- RemoteIterator<LocatedFileStatus> fileStatusIter = fs.listFiles(dirPath, true);
+ RemoteIterator<LocatedFileStatus> fileStatusIter =
+ FileSystemUtil.listFiles(fs, dirPath, true);
+ if (fileStatusIter == null) return;
while (fileStatusIter.hasNext()) {
LocatedFileStatus fileStatus = fileStatusIter.next();
if (!FileSystemUtil.isValidDataFile(fileStatus)) continue;
@@ -644,9 +649,7 @@ public class HdfsTable extends Table {
if (isMarkedCached_) part.markCached();
addPartition(part);
FileSystem fs = tblLocation.getFileSystem(CONF);
- if (fs.exists(tblLocation)) {
- accessLevel_ = getAvailableAccessLevel(fs, tblLocation);
- }
+ accessLevel_ = getAvailableAccessLevel(fs, tblLocation);
} else {
for (org.apache.hadoop.hive.metastore.api.Partition msPartition: msPartitions) {
HdfsPartition partition = createPartition(msPartition.getSd(), msPartition);
@@ -704,10 +707,6 @@ public class HdfsTable extends Table {
Preconditions.checkNotNull(partDir);
try {
FileSystem fs = partDir.getFileSystem(CONF);
- if (!fs.exists(partDir)) {
- partition.setFileDescriptors(new ArrayList<FileDescriptor>());
- return;
- }
if (!FileSystemUtil.supportsStorageIds(fs)) {
synthesizeBlockMetadata(fs, partition);
return;
@@ -722,21 +721,22 @@ public class HdfsTable extends Table {
// Iterate through the current snapshot of the partition directory listing to
// figure out files that were newly added/modified.
List<FileDescriptor> newFileDescs = Lists.newArrayList();
- long newPartSizeBytes = 0;
- for (FileStatus fileStatus : fs.listStatus(partDir)) {
- if (!FileSystemUtil.isValidDataFile(fileStatus)) continue;
- String fileName = fileStatus.getPath().getName().toString();
- FileDescriptor fd = fileDescsByName.get(fileName);
- if (fd == null || partition.isMarkedCached() ||
- fd.getFileLength() != fileStatus.getLen() ||
- fd.getModificationTime() != fileStatus.getModificationTime()) {
- BlockLocation[] locations =
- fs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
- fd = FileDescriptor.create(fileStatus, locations, fs, hostIndex_,
- new Reference<Long>(Long.valueOf(0)));
+ FileStatus[] pathStatus = FileSystemUtil.listStatus(fs, partDir);
+ if (pathStatus != null) {
+ for (FileStatus fileStatus: pathStatus) {
+ if (!FileSystemUtil.isValidDataFile(fileStatus)) continue;
+ String fileName = fileStatus.getPath().getName().toString();
+ FileDescriptor fd = fileDescsByName.get(fileName);
+ if (fd == null || partition.isMarkedCached() ||
+ fd.getFileLength() != fileStatus.getLen() ||
+ fd.getModificationTime() != fileStatus.getModificationTime()) {
+ BlockLocation[] locations =
+ fs.getFileBlockLocations(fileStatus, 0, fileStatus.getLen());
+ fd = FileDescriptor.create(fileStatus, locations, fs, hostIndex_,
+ new Reference<Long>(Long.valueOf(0)));
+ }
+ newFileDescs.add(fd);
}
- newFileDescs.add(fd);
- newPartSizeBytes += fileStatus.getLen();
}
partition.setFileDescriptors(newFileDescs);
} catch(IOException e) {
@@ -800,7 +800,7 @@ public class HdfsTable extends Table {
FsPermissionChecker permissionChecker = FsPermissionChecker.getInstance();
while (location != null) {
- if (fs.exists(location)) {
+ try {
FsPermissionChecker.Permissions perms =
permissionChecker.getPermissions(fs, location);
if (perms.canReadAndWrite()) {
@@ -811,8 +811,9 @@ public class HdfsTable extends Table {
return TAccessLevel.WRITE_ONLY;
}
return TAccessLevel.NONE;
+ } catch (FileNotFoundException e) {
+ location = location.getParent();
}
- location = location.getParent();
}
// Should never get here.
Preconditions.checkNotNull(location, "Error: no path ancestor exists");
@@ -1105,9 +1106,7 @@ public class HdfsTable extends Table {
if (msTbl.getPartitionKeysSize() == 0) {
Path location = new Path(hdfsBaseDir_);
FileSystem fs = location.getFileSystem(CONF);
- if (fs.exists(location)) {
- accessLevel_ = getAvailableAccessLevel(fs, location);
- }
+ accessLevel_ = getAvailableAccessLevel(fs, location);
}
setMetaStoreTable(msTbl);
}
@@ -1482,9 +1481,6 @@ public class HdfsTable extends Table {
HdfsPartition partition) throws Exception {
Preconditions.checkNotNull(storageDescriptor);
Preconditions.checkNotNull(partition);
- Path partDirPath = new Path(storageDescriptor.getLocation());
- FileSystem fs = partDirPath.getFileSystem(CONF);
- if (!fs.exists(partDirPath)) return;
refreshFileMetadata(partition);
}
@@ -1661,9 +1657,6 @@ public class HdfsTable extends Table {
HashSet<List<LiteralExpr>> existingPartitions,
List<List<String>> partitionsNotInHms) throws IOException {
FileSystem fs = path.getFileSystem(CONF);
- // Check whether the base directory exists.
- if (!fs.exists(path)) return;
-
List<String> partitionValues = Lists.newArrayList();
List<LiteralExpr> partitionExprs = Lists.newArrayList();
getAllPartitionsNotInHms(path, partitionKeys, 0, fs, partitionValues,
@@ -1701,7 +1694,8 @@ public class HdfsTable extends Table {
return;
}
- FileStatus[] statuses = fs.listStatus(path);
+ FileStatus[] statuses = FileSystemUtil.listStatus(fs, path);
+ if (statuses == null) return;
for (FileStatus status: statuses) {
if (!status.isDirectory()) continue;
Pair<String, LiteralExpr> keyValues =
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/d9fc9be0/fe/src/main/java/org/apache/impala/common/FileSystemUtil.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/common/FileSystemUtil.java b/fe/src/main/java/org/apache/impala/common/FileSystemUtil.java
index 9ae4269..2841993 100644
--- a/fe/src/main/java/org/apache/impala/common/FileSystemUtil.java
+++ b/fe/src/main/java/org/apache/impala/common/FileSystemUtil.java
@@ -29,14 +29,17 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.LocalFileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.adl.AdlFileSystem;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.client.HdfsAdmin;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
import org.apache.impala.catalog.HdfsCompression;
-import org.apache.log4j.Logger;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.base.Objects;
import com.google.common.base.Preconditions;
@@ -46,7 +49,7 @@ import com.google.common.base.Preconditions;
*/
public class FileSystemUtil {
private static final Configuration CONF = new Configuration();
- private static final Logger LOG = Logger.getLogger(FileSystemUtil.class);
+ private static final Logger LOG = LoggerFactory.getLogger(FileSystemUtil.class);
/**
* Performs a non-recursive delete of all visible (non-hidden) files in a given
@@ -483,4 +486,32 @@ public class FileSystemUtil {
FileSystemUtil.isS3AFileSystem(path) ||
FileSystemUtil.isADLFileSystem(path));
}
+
+ /**
+ * Wrapper around FileSystem.listStatus() that specifically handles the case when
+ * the path does not exist. This helps simplify the caller code in cases where
+ * the file does not exist and also saves an RPC as the caller need not do a separate
+ * exists check for the path. Returns null if the path does not exist.
+ */
+ public static FileStatus[] listStatus(FileSystem fs, Path p) throws IOException {
+ try {
+ return fs.listStatus(p);
+ } catch (FileNotFoundException e) {
+ if (LOG.isWarnEnabled()) LOG.warn("Path does not exist: " + p.toString(), e);
+ return null;
+ }
+ }
+
+ /**
+ * Wrapper around FileSystem.listFiles(), similar to the listStatus() wrapper above.
+ */
+ public static RemoteIterator<LocatedFileStatus> listFiles(FileSystem fs, Path p,
+ boolean recursive) throws IOException {
+ try {
+ return fs.listFiles(p, recursive);
+ } catch (FileNotFoundException e) {
+ if (LOG.isWarnEnabled()) LOG.warn("Path does not exist: " + p.toString(), e);
+ return null;
+ }
+ }
}
[2/6] incubator-impala git commit: IMPALA-4029: [DOCS] Include known
issue for 'flat buffers' change
Posted by mi...@apache.org.
IMPALA-4029: [DOCS] Include known issue for 'flat buffers' change
Change-Id: I65495748984268763b2690c193488a338409e13e
Reviewed-on: http://gerrit.cloudera.org:8080/7236
Reviewed-by: John Russell <jr...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/91dbeb3c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/91dbeb3c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/91dbeb3c
Branch: refs/heads/master
Commit: 91dbeb3c061292c832b069bdf07fb76533332f23
Parents: 384e864
Author: John Russell <jr...@cloudera.com>
Authored: Tue Jun 20 14:27:19 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Tue Jun 27 00:17:03 2017 +0000
----------------------------------------------------------------------
docs/topics/impala_known_issues.xml | 26 ++++++++++++++++++++++++++
1 file changed, 26 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/91dbeb3c/docs/topics/impala_known_issues.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_known_issues.xml b/docs/topics/impala_known_issues.xml
index 75c9d2d..c3f1e51 100644
--- a/docs/topics/impala_known_issues.xml
+++ b/docs/topics/impala_known_issues.xml
@@ -564,6 +564,32 @@ https://issues.apache.org/jira/browse/IMPALA-2144 - Don't have
</conbody>
+ <concept id="flatbuffers_mem_usage">
+ <title>Memory usage when compact_catalog_topic flag enabled</title>
+ <conbody>
+ <p>
+ The efficiency improvement from <xref keyref="IMPALA-4029">IMPALA-4029</xref>
+ can cause an increase in size of the updates to Impala catalog metadata
+ that are broadcast to the <cmdname>impalad</cmdname> daemons
+ by the <cmdname>statestored</cmdname> daemon.
+ The increase in catalog update topic size results in higher CPU and network
+ utilization. By default, the increase in topic size is about 5-7%. If the
+ <codeph>compact_catalog_topic</codeph> flag is used, the
+ size increase is more substantial, with a topic size approximately twice as
+ large as in previous versions.
+ </p>
+ <p><b>Bug:</b> <xref keyref="IMPALA-5500">IMPALA-5500</xref></p>
+ <p><b>Severity:</b> Medium</p>
+ <p>
+ <b>Workaround:</b> Consider leaving the <codeph>compact_catalog_topic</codeph>
+ configuration setting at its default value of <codeph>false</codeph> until
+ this issue is resolved.
+ </p>
+ <p><b>Resolution:</b> A fix is in the pipeline. Check the status of
+ <xref keyref="IMPALA-5500">IMPALA-5500</xref> for the release where the fix is available.</p>
+ </conbody>
+ </concept>
+
<concept id="drop_table_purge_s3a">
<title>DROP TABLE PURGE on S3A table may not delete externally written files</title>
<conbody>
[6/6] incubator-impala git commit: IMPALA-5571: Fix authorization
tests
Posted by mi...@apache.org.
IMPALA-5571: Fix authorization tests
Issue:
With the fix for IMPALA-5549 the catalog cache of an impalad may end up
with stale authorization metadata in the presence of GRANT/REVOKE
statements that modify multiple privileges.
Fix:
Revert the behavior of GRANT/REVOKE privilege DDL statements.
Change-Id: Icad893a10fdeb7a721264e69413014603dec6339
Reviewed-on: http://gerrit.cloudera.org:8080/7299
Reviewed-by: Dimitris Tsirogiannis <dt...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/8a77fff9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/8a77fff9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/8a77fff9
Branch: refs/heads/master
Commit: 8a77fff9a8175a6d0939d195ac622ed099988a37
Parents: 4a3ef9c
Author: Dimitris Tsirogiannis <dt...@cloudera.com>
Authored: Mon Jun 26 15:16:48 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Tue Jun 27 07:39:44 2017 +0000
----------------------------------------------------------------------
.../java/org/apache/impala/service/CatalogOpExecutor.java | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/8a77fff9/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
index 6716338..675ced5 100644
--- a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
+++ b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
@@ -2906,7 +2906,11 @@ public class CatalogOpExecutor {
updatedPrivs.add(catalogObject);
}
- if (!updatedPrivs.isEmpty()) {
+ // TODO: Currently we only support sending back 1 catalog object in a "direct DDL"
+ // response. If multiple privileges have been updated, just send back the
+ // catalog version so subscribers can wait for the statestore heartbeat that
+ // contains all updates (see IMPALA-5571).
+ if (updatedPrivs.size() == 1) {
// If this is a REVOKE statement with hasGrantOpt, only the GRANT OPTION is revoked
// from the privilege.
if (grantRevokePrivParams.isIs_grant() ||
@@ -2915,6 +2919,8 @@ public class CatalogOpExecutor {
} else {
resp.result.setRemoved_catalog_objects(updatedPrivs);
}
+ resp.result.setVersion(updatedPrivs.get(0).getCatalog_version());
+ } else if (updatedPrivs.size() > 1) {
resp.result.setVersion(
updatedPrivs.get(updatedPrivs.size() - 1).getCatalog_version());
}
[3/6] incubator-impala git commit: IMPALA-5579: Fix
IndexOutOfBoundsException in GetTables metadata request
Posted by mi...@apache.org.
IMPALA-5579: Fix IndexOutOfBoundsException in GetTables metadata request
Issue:
GetTables metadata request may result in an IndexOutOfBoundsException
while accessing table comments fields if one or more tables can't be
retrieved from the catalog cache.
Fix:
Only add tables that can be retrieved from the catalog cache (i.e. no
TableLoadingException is thrown) in the result of a GetTables request.
Change-Id: Ic933b9950d58e6f880d4078faf1a69ee7c6adbf1
Reviewed-on: http://gerrit.cloudera.org:8080/7296
Reviewed-by: Dimitris Tsirogiannis <dt...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/466808ae
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/466808ae
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/466808ae
Branch: refs/heads/master
Commit: 466808aede88df3bb6b6f8faff39ab5db8103955
Parents: 91dbeb3
Author: Dimitris Tsirogiannis <dt...@cloudera.com>
Authored: Mon Jun 26 11:12:43 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Tue Jun 27 02:36:46 2017 +0000
----------------------------------------------------------------------
.../main/java/org/apache/impala/service/MetadataOp.java | 2 +-
.../test/java/org/apache/impala/service/FrontendTest.java | 10 ++++++++++
2 files changed, 11 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/466808ae/fe/src/main/java/org/apache/impala/service/MetadataOp.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/service/MetadataOp.java b/fe/src/main/java/org/apache/impala/service/MetadataOp.java
index 3bf7534..6883bac 100644
--- a/fe/src/main/java/org/apache/impala/service/MetadataOp.java
+++ b/fe/src/main/java/org/apache/impala/service/MetadataOp.java
@@ -272,7 +272,6 @@ public class MetadataOp {
List<List<Column>> tablesColumnsList = Lists.newArrayList();
List<String> tableComments = Lists.newArrayList();
for (String tabName: fe.getTableNames(db.getName(), tablePatternMatcher, user)) {
- tableList.add(tabName);
Table table = null;
try {
table = catalog.getTable(db.getName(), tabName);
@@ -291,6 +290,7 @@ public class MetadataOp {
comment = table.getMetaStoreTable().getParameters().get("comment");
columns.addAll(fe.getColumns(table, columnPatternMatcher, user));
}
+ tableList.add(tabName);
tablesColumnsList.add(columns);
tableComments.add(Strings.nullToEmpty(comment));
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/466808ae/fe/src/test/java/org/apache/impala/service/FrontendTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/service/FrontendTest.java b/fe/src/test/java/org/apache/impala/service/FrontendTest.java
index c3faa0a..d303315 100644
--- a/fe/src/test/java/org/apache/impala/service/FrontendTest.java
+++ b/fe/src/test/java/org/apache/impala/service/FrontendTest.java
@@ -207,6 +207,16 @@ public class FrontendTest extends FrontendTestBase {
assertEquals("", row.colVals.get(4).string_val);
}
}
+
+ // Make sure tables that can't be loaded don't result in errors in the GetTables
+ // request (see IMPALA-5579)
+ req = new TMetadataOpRequest();
+ req.opcode = TMetadataOpcode.GET_TABLES;
+ req.get_tables_req = new TGetTablesReq();
+ req.get_tables_req.setSchemaName("functional");
+ req.get_tables_req.setTableName("hive_index_tbl");
+ resp = execMetadataOp(req);
+ assertEquals(0, resp.rows.size());
}
@Test
[5/6] incubator-impala git commit: IMPALA-5554: sorter DCHECK on null
column
Posted by mi...@apache.org.
IMPALA-5554: sorter DCHECK on null column
The bug was in the DCHECK. The DCHECK is intended to make sure that a
tuple's string data didn't get split across blocks. The logic assumed
that if the second-or-later string column was in the next-block, that
the strings were split between blocks. However, that assumption is
invalid if there are NULL strings, which do not belong in any block.
The fix for the DCHECK (which is still useful) is to count the number
of non-NULL strings and make sure that no non-NULL strings were split
between blocks.
Testing:
Added a test that reproduces the crash.
Change-Id: I7a8dee982501008efff5b5abc192cfb5e6544a90
Reviewed-on: http://gerrit.cloudera.org:8080/7295
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/4a3ef9c7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/4a3ef9c7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/4a3ef9c7
Branch: refs/heads/master
Commit: 4a3ef9c7731a45c028a36ef75d3a6a10bd4bf4cd
Parents: d9fc9be
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Sun Jun 25 15:37:33 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Tue Jun 27 05:36:11 2017 +0000
----------------------------------------------------------------------
be/src/runtime/sorter.cc | 6 +++-
.../QueryTest/single-node-large-sorts.test | 36 ++++++++++++++++++--
2 files changed, 39 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4a3ef9c7/be/src/runtime/sorter.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/sorter.cc b/be/src/runtime/sorter.cc
index b8182e2..6760373 100644
--- a/be/src/runtime/sorter.cc
+++ b/be/src/runtime/sorter.cc
@@ -1006,9 +1006,11 @@ bool Sorter::Run::ConvertOffsetsToPtrs(Tuple* tuple) {
var_len_blocks_[var_len_blocks_index_]->buffer();
const vector<SlotDescriptor*>& string_slots = sort_tuple_desc_->string_slots();
+ int num_non_null_string_slots = 0;
for (int i = 0; i < string_slots.size(); ++i) {
SlotDescriptor* slot_desc = string_slots[i];
if (tuple->IsNull(slot_desc->null_indicator_offset())) continue;
+ ++num_non_null_string_slots;
DCHECK(slot_desc->type().IsVarLenStringType());
StringValue* value = reinterpret_cast<StringValue*>(
@@ -1026,7 +1028,9 @@ bool Sorter::Run::ConvertOffsetsToPtrs(Tuple* tuple) {
DCHECK_LE(block_index, var_len_blocks_.size());
DCHECK_EQ(block_index, var_len_blocks_index_ + 1);
DCHECK_EQ(block_offset, 0); // The data is the first thing in the next block.
- DCHECK_EQ(i, 0); // Var len data for tuple shouldn't be split across blocks.
+ // This must be the first slot with var len data for the tuple. Var len data
+ // for tuple shouldn't be split across blocks.
+ DCHECK_EQ(num_non_null_string_slots, 1);
return false;
}
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/4a3ef9c7/testdata/workloads/functional-query/queries/QueryTest/single-node-large-sorts.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/single-node-large-sorts.test b/testdata/workloads/functional-query/queries/QueryTest/single-node-large-sorts.test
index 8000ad3..74b7eee 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/single-node-large-sorts.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/single-node-large-sorts.test
@@ -29,6 +29,38 @@ STRING,STRING,STRING
'Faaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa','',''
'Faaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa','',''
---- RUNTIME_PROFILE
-row_regex: .* TotalMergesPerformed: [^0] .*
-row_regex: .* SpilledRuns: [^0] .*
+row_regex: .* TotalMergesPerformed: [^0].*
+row_regex: .* SpilledRuns: [^0].*
+====
+---- QUERY
+# Regression test for IMPALA-5554: first string column in sort tuple is null
+# on boundary of spilled block. Test does two sorts with a NULL and non-NULL
+# string column in both potential orders.
+set max_block_mgr_memory=50m;
+select *
+from (
+ select *, first_value(col) over (order by sort_col) fv
+ from (
+ select concat(l_linestatus, repeat('a', 63)) sort_col, if(l_returnflag = 'foo', l_returnflag, NULL) col
+ from tpch_parquet.lineitem limit 100000
+ union all
+ select if(l_returnflag = 'foo', l_returnflag, NULL) sort_col, concat(l_linestatus, repeat('a', 63)) col
+ from tpch_parquet.lineitem) q limit 100000
+ ) q2
+limit 10
+---- TYPES
+STRING,STRING,STRING
+---- RESULTS
+'Faaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa','NULL','NULL'
+'Faaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa','NULL','NULL'
+'Faaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa','NULL','NULL'
+'Faaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa','NULL','NULL'
+'Faaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa','NULL','NULL'
+'Faaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa','NULL','NULL'
+'Faaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa','NULL','NULL'
+'Faaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa','NULL','NULL'
+'Faaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa','NULL','NULL'
+'Faaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa','NULL','NULL'
+---- RUNTIME_PROFILE
+row_regex: .* SpilledRuns: [^0].*
====