You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2019/08/08 01:06:33 UTC
[impala] 01/03: IMPALA-8600: Fix test_acid_compaction
This is an automated email from the ASF dual-hosted git repository.
tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit df7730f7fea948409958db2dfdce2c52b0333c2f
Author: Gabor Kaszab <ga...@cloudera.com>
AuthorDate: Wed Aug 7 11:18:20 2019 +0200
IMPALA-8600: Fix test_acid_compaction
Apparently Hive doesn't update the writeIds on a transactional table
after compaction. This breaks an assumption made in the REFRESH table
logic that only does an actual refresh when the HMS writeId is
different than the one cached locally. As a result the non-partitioned
ACID tables aren't refreshed when a REFRESH table is invoked in Impala
right after a major compaction.
Change-Id: I58b79f8864b31e18eca818032ad5a9af954913f6
Reviewed-on: http://gerrit.cloudera.org:8080/14027
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../apache/impala/service/CatalogOpExecutor.java | 38 +++++-----------------
1 file changed, 9 insertions(+), 29 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
index 0177c8d..7ad8b36 100644
--- a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
+++ b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
@@ -3612,40 +3612,20 @@ public class CatalogOpExecutor {
"Load triggered by " + cmdString);
if (tbl != null) {
if (isTableLoadedInCatalog) {
- boolean isTransactional = AcidUtils.isTransactionalTable(
- tbl.getMetaStoreTable().getParameters());
if (req.isSetPartition_spec()) {
+ boolean isTransactional = AcidUtils.isTransactionalTable(
+ tbl.getMetaStoreTable().getParameters());
Preconditions.checkArgument(!isTransactional);
updatedThriftTable = catalog_.reloadPartition(tbl,
req.getPartition_spec(), cmdString);
} else {
- if (isTransactional) {
- org.apache.hadoop.hive.metastore.api.Table hmsTbl =
- getTableFromMetaStore(tblName);
- if (hmsTbl == null) {
- throw new TableNotFoundException("Table not found: " +
- tblName.toString());
- }
- HdfsTable hdfsTable = (HdfsTable)tbl;
- if (!hdfsTable.isPartitioned() &&
- MetastoreShim.getWriteIdFromMSTable(tbl.getMetaStoreTable()) ==
- MetastoreShim.getWriteIdFromMSTable(hmsTbl)) {
- // No need to refresh the table if the local writeId equals to the
- // latest writeId from HMS and the table is not partitioned.
- LOG.debug("Skip reloading table " + tblName.toString() +
- " because it has the latest writeId locally");
- resp.getResult().setStatus(new TStatus(TErrorCode.OK,
- new ArrayList<String>()));
- return resp;
- }
- // TODO IMPALA-8809: Optimisation for partitioned tables:
- // 1: Reload the whole table if schema change happened. Identify
- // such scenario by checking Table.TBL_PROP_LAST_DDL_TIME property.
- // Note, table level writeId is not updated by HMS for partitioned
- // ACID tables, there is a Jira to cover this: HIVE-22062.
- // 2: If no need for a full table reload then fetch partition level
- // writeIds and reload only the ones that changed.
- }
+ // TODO IMPALA-8809: Optimisation for partitioned tables:
+ // 1: Reload the whole table if schema change happened. Identify
+ // such scenario by checking Table.TBL_PROP_LAST_DDL_TIME property.
+ // Note, table level writeId is not updated by HMS for partitioned
+ // ACID tables, there is a Jira to cover this: HIVE-22062.
+ // 2: If no need for a full table reload then fetch partition level
+ // writeIds and reload only the ones that changed.
updatedThriftTable = catalog_.reloadTable(tbl, cmdString);
}
} else {