You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2020/06/18 21:41:53 UTC
[impala] 02/03: IMPALA-7533: Cache partitions across table versions
in LocalCatalog
This is an automated email from the ASF dual-hosted git repository.
tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit dc5046487ae66b37a2fdca19e55b9daf8bf410c7
Author: stiga-huang <hu...@gmail.com>
AuthorDate: Mon Jun 15 18:01:12 2020 +0800
IMPALA-7533: Cache partitions across table versions in LocalCatalog
In LocalCatlaog cache, partition metadata is cached with a composed key
of table name, table version and partition id. Whenever the table
version bumps, e.g. due to comments being changed, all the cached
partition values will be unreachable. Following queries have to reload
the partitions and cache them with the new table version. Actually, the
partition id is an unique id across the whole catalog. It's sufficient
to identify the partition. However, there are no partition level
invalidations if the partition is modified in-place in catalogd. So we
have to include the table version and depend on it.
After IMPALA-9778, there are no in-place modifications on partition
metadata. We can safely reuse partition meta across table versions in
LocalCatalog cache. This patch removes the table name and version in the
partition cache key. So metadata of unchanged partitions can be reused
when table version bumps.
Tests:
- Add tests in test_local_catalog.py to verify the partition metadata
is reused based on profile metrics.
Change-Id: I512f735b596bc51d553e6d395d108f49727619ed
Reviewed-on: http://gerrit.cloudera.org:8080/16081
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Vihang Karajgaonkar <vi...@cloudera.com>
---
.../impala/catalog/local/CatalogdMetaProvider.java | 36 +++++---------
.../org/apache/impala/catalog/CatalogTest.java | 13 ++---
.../impala/catalog/local/LocalCatalogTest.java | 5 +-
tests/custom_cluster/test_local_catalog.py | 55 ++++++++++++++++++++++
4 files changed, 72 insertions(+), 37 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/catalog/local/CatalogdMetaProvider.java b/fe/src/main/java/org/apache/impala/catalog/local/CatalogdMetaProvider.java
index 02195ad..9cf4f2e 100644
--- a/fe/src/main/java/org/apache/impala/catalog/local/CatalogdMetaProvider.java
+++ b/fe/src/main/java/org/apache/impala/catalog/local/CatalogdMetaProvider.java
@@ -892,7 +892,7 @@ public class CatalogdMetaProvider implements MetaProvider {
refImpl, hostIndex, missingRefs);
refToMeta.putAll(fromCatalogd);
// Write back to the cache.
- storePartitionsInCache(refImpl, hostIndex, fromCatalogd);
+ storePartitionsInCache(hostIndex, fromCatalogd);
}
sw.stop();
addStatsToProfile(PARTITIONS_STATS_CATEGORY, numHits, numMisses, sw);
@@ -991,7 +991,7 @@ public class CatalogdMetaProvider implements MetaProvider {
partitionRefs.size());
for (PartitionRef ref: partitionRefs) {
PartitionRefImpl prefImpl = (PartitionRefImpl)ref;
- PartitionCacheKey cacheKey = new PartitionCacheKey(table, prefImpl.getId());
+ PartitionCacheKey cacheKey = new PartitionCacheKey(prefImpl.getId());
PartitionMetadataImpl val = (PartitionMetadataImpl)getIfPresent(cacheKey);
if (val == null) continue;
@@ -1007,12 +1007,12 @@ public class CatalogdMetaProvider implements MetaProvider {
* Write back the partitions in 'metas' into the cache. The file descriptors in these
* partitions must be relative to the 'hostIndex'.
*/
- private void storePartitionsInCache(TableMetaRefImpl table,
- ListMap<TNetworkAddress> hostIndex, Map<PartitionRef, PartitionMetadata> metas) {
+ private void storePartitionsInCache(ListMap<TNetworkAddress> hostIndex,
+ Map<PartitionRef, PartitionMetadata> metas) {
for (Map.Entry<PartitionRef, PartitionMetadata> e: metas.entrySet()) {
PartitionRefImpl prefImpl = (PartitionRefImpl)e.getKey();
PartitionMetadataImpl metaImpl = (PartitionMetadataImpl)e.getValue();
- PartitionCacheKey cacheKey = new PartitionCacheKey(table, prefImpl.getId());
+ PartitionCacheKey cacheKey = new PartitionCacheKey(prefImpl.getId());
PartitionMetadataImpl cacheVal = metaImpl.cloneRelativeToHostIndex(hostIndex,
cacheHostIndex_);
cache_.put(cacheKey, cacheVal);
@@ -1593,9 +1593,7 @@ public class CatalogdMetaProvider implements MetaProvider {
@Override
public boolean equals(Object obj) {
- if (obj == null || !(obj instanceof ColStatsCacheKey)) {
- return false;
- }
+ if (!(obj instanceof ColStatsCacheKey)) return false;
ColStatsCacheKey other = (ColStatsCacheKey)obj;
return super.equals(obj) && colName_.equals(other.colName_);
}
@@ -1615,34 +1613,26 @@ public class CatalogdMetaProvider implements MetaProvider {
/**
* Key for caching information about a single partition.
*
- * TODO(todd): currently this inherits from VersionedTableCacheKey. This means that, if
- * a table's version number changes, all of its partitions must be reloaded. However,
- * since partition IDs are globally unique within a catalogd instance, we could
- * optimize this to just key based on the partition ID. However, currently, there are
- * some cases where partitions are mutated in place rather than replaced with a new ID.
- * We need to eliminate those or add a partition sequence number before we can make
- * this optimization.
+ * Values for these keys are 'PartitionMetadataImpl' objects.
*/
- private static class PartitionCacheKey extends VersionedTableCacheKey {
+ @Immutable
+ private static class PartitionCacheKey {
private final long partId_;
- PartitionCacheKey(TableMetaRefImpl table, long partId) {
- super(table);
+ PartitionCacheKey(long partId) {
partId_ = partId;
}
@Override
public int hashCode() {
- return Objects.hashCode(super.hashCode(), partId_);
+ return Objects.hashCode(partId_, getClass());
}
@Override
public boolean equals(Object obj) {
- if (obj == null || !(obj instanceof PartitionCacheKey)) {
- return false;
- }
+ if (!(obj instanceof PartitionCacheKey)) return false;
PartitionCacheKey other = (PartitionCacheKey)obj;
- return super.equals(obj) && partId_ == other.partId_;
+ return partId_ == other.partId_;
}
}
diff --git a/fe/src/test/java/org/apache/impala/catalog/CatalogTest.java b/fe/src/test/java/org/apache/impala/catalog/CatalogTest.java
index 9a2ca3b..762107b 100644
--- a/fe/src/test/java/org/apache/impala/catalog/CatalogTest.java
+++ b/fe/src/test/java/org/apache/impala/catalog/CatalogTest.java
@@ -440,7 +440,7 @@ public class CatalogTest {
public void TestPartitions() throws CatalogException {
HdfsTable table =
(HdfsTable) catalog_.getOrLoadTable("functional", "AllTypes", "test", null);
- checkAllTypesPartitioning(table, true);
+ checkAllTypesPartitioning(table);
}
/**
@@ -502,8 +502,7 @@ public class CatalogTest {
assertEquals(0, foreignKeys.size());
}
- public static void checkAllTypesPartitioning(FeFsTable table,
- boolean checkFileDescriptors) {
+ public static void checkAllTypesPartitioning(FeFsTable table) {
assertEquals(24, table.getPartitionIds().size());
assertEquals(24, table.getPartitions().size());
Collection<? extends FeFsPartition> partitions =
@@ -529,13 +528,7 @@ public class CatalogTest {
assertTrue(key2 >= 1 && key2 <= 12);
months.add(key1 * 100 + key2);
-
- if (checkFileDescriptors) {
- // TODO(todd): once LocalCatalog supports file descriptors,
- // no need for this boolean anymore.
- assertEquals(p.getFileDescriptors().size(), 1);
- }
-
+ assertEquals(p.getFileDescriptors().size(), 1);
uniqueSds.add(p.getInputFormatDescriptor());
}
assertEquals(months.size(), 24);
diff --git a/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java b/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java
index b5f44c0..5ecb9f8 100644
--- a/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java
+++ b/fe/src/test/java/org/apache/impala/catalog/local/LocalCatalogTest.java
@@ -159,9 +159,7 @@ public class LocalCatalogTest {
@Test
public void testPartitioning() throws Exception {
FeFsTable t = (FeFsTable) catalog_.getTable("functional", "alltypes");
- // TODO(todd): once we support file descriptors in LocalCatalog,
- // run the full test.
- CatalogTest.checkAllTypesPartitioning(t, /*checkFileDescriptors=*/false);
+ CatalogTest.checkAllTypesPartitioning(t);
}
/**
@@ -251,7 +249,6 @@ public class LocalCatalogTest {
assertTrue(t.getHostIndex().size() > 0);
}
-
@Test
public void testLoadFileDescriptorsUnpartitioned() throws Exception {
FeFsTable t = (FeFsTable) catalog_.getTable("tpch", "region");
diff --git a/tests/custom_cluster/test_local_catalog.py b/tests/custom_cluster/test_local_catalog.py
index 7f04328..4e1caec 100644
--- a/tests/custom_cluster/test_local_catalog.py
+++ b/tests/custom_cluster/test_local_catalog.py
@@ -20,6 +20,7 @@
import pytest
import Queue
import random
+import re
import threading
import time
@@ -469,3 +470,57 @@ class TestFullAcid(CustomClusterTestSuite):
res = self.execute_query("select id from functional_orc_def.alltypestiny")
res.data.sort()
assert res.data == ['0', '1', '2', '3', '4', '5', '6', '7']
+
+
+class TestReusePartitionMetadata(CustomClusterTestSuite):
+ @pytest.mark.execute_serially
+ @CustomClusterTestSuite.with_args(
+ impalad_args="--use_local_catalog=true",
+ catalogd_args="--catalog_topic_mode=minimal")
+ def test_reuse_partition_meta(self, unique_database):
+ """
+ Test that unchanged partition metadata can be shared across table versions.
+ """
+ self.execute_query(
+ "create table %s.alltypes like functional.alltypes" % unique_database)
+ self.execute_query("insert into %s.alltypes partition(year, month) "
+ "select * from functional.alltypes" % unique_database)
+ # Make sure the table is unloaded either in catalogd or coordinator.
+ self.execute_query("invalidate metadata %s.alltypes" % unique_database)
+ # First time: misses all(24) partitions.
+ self.check_missing_partitions(unique_database, 24)
+ # Second time: hits all(24) partitions.
+ self.check_missing_partitions(unique_database, 0)
+
+ # Alter comment on the table. Partition metadata should be reusable.
+ self.execute_query(
+ "comment on table %s.alltypes is null" % unique_database)
+ self.check_missing_partitions(unique_database, 0)
+
+ # Refresh one partition. Although table version bumps, metadata cache of other
+ # partitions should be reusable.
+ self.execute_query(
+ "refresh %s.alltypes partition(year=2009, month=1)" % unique_database)
+ self.check_missing_partitions(unique_database, 1)
+
+ # Drop one partition. Although table version bumps, metadata cache of existing
+ # partitions should be reusable.
+ self.execute_query(
+ "alter table %s.alltypes drop partition(year=2009, month=1)" % unique_database)
+ self.check_missing_partitions(unique_database, 0)
+
+ # Add back one partition. The partition meta is loaded in catalogd but not the
+ # coordinator. So we still miss its meta. For other partitions, we can reuse them.
+ self.execute_query(
+ "insert into %s.alltypes partition(year=2009, month=1) "
+ "select 0,true,0,0,0,0,0,0,'a','a',NULL" % unique_database)
+ self.check_missing_partitions(unique_database, 1)
+
+ def check_missing_partitions(self, unique_database, partition_misses):
+ """Helper method for checking number of missing partitions while selecting
+ all partitions of the alltypes table"""
+ ret = self.execute_query_expect_success(
+ self.client, "explain select count(*) from %s.alltypes" % unique_database)
+ match = re.search(r"CatalogFetch.Partitions.Misses: (\d+)", ret.runtime_profile)
+ assert len(match.groups()) == 1
+ assert match.group(1) == str(partition_misses)