You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by to...@apache.org on 2018/08/07 18:10:06 UTC
[3/5] impala git commit: IMPALA-7276 (re-fix). Re-fix CTAS and INSERT
for LocalCatalog
IMPALA-7276 (re-fix). Re-fix CTAS and INSERT for LocalCatalog
A recent commit merged and accidentally added a new downcast to
HdfsTable instead of FeFsTable. It also removed the implementation of
one necessary function in LocalFsTable.
This small fix re-fixes those two points so that CTAS works again.
Tested with a simple CTAS query from the shell.
Change-Id: I30bfd9a02c0a8d4e0f793ed84fd1693a44f6e9ee
Reviewed-on: http://gerrit.cloudera.org:8080/11141
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Vuk Ercegovac <ve...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/58191d54
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/58191d54
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/58191d54
Branch: refs/heads/master
Commit: 58191d546ae20eced50c955bd0051c775960bdc1
Parents: b822200
Author: Todd Lipcon <to...@cloudera.com>
Authored: Mon Aug 6 23:16:09 2018 -0700
Committer: Todd Lipcon <to...@apache.org>
Committed: Tue Aug 7 17:38:04 2018 +0000
----------------------------------------------------------------------
.../org/apache/impala/catalog/local/LocalFsTable.java | 10 ++++++++--
.../java/org/apache/impala/planner/HdfsTableSink.java | 3 +--
2 files changed, 9 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/58191d54/fe/src/main/java/org/apache/impala/catalog/local/LocalFsTable.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/local/LocalFsTable.java b/fe/src/main/java/org/apache/impala/catalog/local/LocalFsTable.java
index 7753faa..29c9aaa 100644
--- a/fe/src/main/java/org/apache/impala/catalog/local/LocalFsTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/local/LocalFsTable.java
@@ -179,8 +179,14 @@ public class LocalFsTable extends LocalTable implements FeFsTable {
@Override
public Set<HdfsFileFormat> getFileFormats() {
- // Needed by HdfsTableSink.
- throw new UnsupportedOperationException("TODO: implement me");
+ // TODO(todd): can we avoid loading all partitions here? this is called
+ // for any INSERT query, even if the partition is specified.
+ Collection<? extends FeFsPartition> parts = FeCatalogUtils.loadAllPartitions(this);
+ // In the case that we have no partitions added to the table yet, it's
+ // important to add the "prototype" partition as a fallback.
+ Iterable<FeFsPartition> partitionsToConsider = Iterables.concat(
+ parts, Collections.singleton(createPrototypePartition()));
+ return FeCatalogUtils.getFileFormats(partitionsToConsider);
}
@Override
http://git-wip-us.apache.org/repos/asf/impala/blob/58191d54/fe/src/main/java/org/apache/impala/planner/HdfsTableSink.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/planner/HdfsTableSink.java b/fe/src/main/java/org/apache/impala/planner/HdfsTableSink.java
index 48e7c62..03e1348 100644
--- a/fe/src/main/java/org/apache/impala/planner/HdfsTableSink.java
+++ b/fe/src/main/java/org/apache/impala/planner/HdfsTableSink.java
@@ -25,7 +25,6 @@ import org.apache.impala.analysis.Expr;
import org.apache.impala.catalog.FeFsTable;
import org.apache.impala.catalog.FeTable;
import org.apache.impala.catalog.HdfsFileFormat;
-import org.apache.impala.catalog.HdfsTable;
import org.apache.impala.thrift.TDataSink;
import org.apache.impala.thrift.TDataSinkType;
import org.apache.impala.thrift.TExplainLevel;
@@ -90,7 +89,7 @@ public class HdfsTableSink extends TableSink {
numPartitionsPerInstance = DEFAULT_NUM_PARTITIONS;
}
- HdfsTable table = (HdfsTable) targetTable_;
+ FeFsTable table = (FeFsTable) targetTable_;
// TODO: Estimate the memory requirements more accurately by partition type.
Set<HdfsFileFormat> formats = table.getFileFormats();
long perPartitionMemReq = getPerPartitionMemReq(formats);