You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by st...@apache.org on 2022/11/03 01:19:16 UTC
[impala] 02/02: IMPALA-11682: Add tests for minor compacted insert only ACID tables
This is an automated email from the ASF dual-hosted git repository.
stigahuang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit a983a347a77af74e1a9bd6156d12a020d6b4df6d
Author: Csaba Ringhofer <cs...@cloudera.com>
AuthorDate: Mon Oct 24 20:15:50 2022 +0200
IMPALA-11682: Add tests for minor compacted insert only ACID tables
Only test changes. Minor compacted delta dirs are supported in
Impala since IMPALA-9512, but at that time Hive supported minor
compaction only on full ACID tables. Since that time Hive added
support for minor compacting insert only/MM tables (HIVE-22610).
Change-Id: I7159283f3658f2119d38bd3393729535edd0a76f
Reviewed-on: http://gerrit.cloudera.org:8080/19164
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../impala/catalog/FileMetadataLoaderTest.java | 36 +++++++++++++++------
.../functional/functional_schema_template.sql | 37 +++++++++++++++++++++-
.../datasets/functional/schema_constraints.csv | 3 ++
.../functional-query/queries/QueryTest/acid.test | 25 +++++++++++++++
4 files changed, 91 insertions(+), 10 deletions(-)
diff --git a/fe/src/test/java/org/apache/impala/catalog/FileMetadataLoaderTest.java b/fe/src/test/java/org/apache/impala/catalog/FileMetadataLoaderTest.java
index a92595464..b0ad5f768 100644
--- a/fe/src/test/java/org/apache/impala/catalog/FileMetadataLoaderTest.java
+++ b/fe/src/test/java/org/apache/impala/catalog/FileMetadataLoaderTest.java
@@ -146,22 +146,40 @@ public class FileMetadataLoaderTest extends FrontendTestBase {
relPaths.get(0));
}
- @Test
- public void testAcidMinorCompactionLoading() throws IOException, CatalogException {
- //TODO(IMPALA-9042): Remove "throws CatalogException"
+ private FileMetadataLoader getLoaderForAcidTable(
+ String validWriteIdString, String path, HdfsFileFormat format)
+ throws IOException, CatalogException {
ListMap<TNetworkAddress> hostIndex = new ListMap<>();
- ValidWriteIdList writeIds = MetastoreShim.getValidWriteIdListFromString(
- "functional_orc_def.complextypestbl_minor_compacted:10:10::");
- Path tablePath = new Path("hdfs://localhost:20500/test-warehouse/managed/" +
- "functional_orc_def.db/" +
- "complextypestbl_minor_compacted_orc_def/");
+ ValidWriteIdList writeIds =
+ MetastoreShim.getValidWriteIdListFromString(validWriteIdString);
+ Path tablePath = new Path(path);
FileMetadataLoader fml = new FileMetadataLoader(tablePath, /* recursive=*/true,
/* oldFds = */ Collections.emptyList(), hostIndex, new ValidReadTxnList(""),
- writeIds, HdfsFileFormat.ORC);
+ writeIds, format);
fml.load();
+ return fml;
+ }
+
+ @Test
+ public void testAcidMinorCompactionLoading() throws IOException, CatalogException {
+ //TODO(IMPALA-9042): Remove "throws CatalogException"
+ FileMetadataLoader fml = getLoaderForAcidTable(
+ "functional_orc_def.complextypestbl_minor_compacted:10:10::",
+ "hdfs://localhost:20500/test-warehouse/managed/functional_orc_def.db/" +
+ "complextypestbl_minor_compacted_orc_def/",
+ HdfsFileFormat.ORC);
// Only load the compacted file.
assertEquals(1, fml.getStats().loadedFiles);
assertEquals(8, fml.getStats().filesSupersededByAcidState);
+
+ fml = getLoaderForAcidTable(
+ "functional_parquet.insert_only_minor_compacted:6:6::",
+ "hdfs://localhost:20500/test-warehouse/managed/functional_parquet.db/" +
+ "insert_only_minor_compacted_parquet/",
+ HdfsFileFormat.PARQUET);
+ // Only load files after compaction.
+ assertEquals(3, fml.getStats().loadedFiles);
+ assertEquals(2, fml.getStats().filesSupersededByAcidState);
}
@Test
diff --git a/testdata/datasets/functional/functional_schema_template.sql b/testdata/datasets/functional/functional_schema_template.sql
index 4f1ff5cd0..d2a0941bd 100644
--- a/testdata/datasets/functional/functional_schema_template.sql
+++ b/testdata/datasets/functional/functional_schema_template.sql
@@ -3741,5 +3741,40 @@ values (
map(cast("key1" as binary), 1, cast("key2" as binary), 2),
map(1, cast("value1" as binary), 2, cast("value2" as binary)),
named_struct("i", 0, "b", cast("member" as binary))
- )
+ );
+====
+---- DATASET
+functional
+---- BASE_TABLE_NAME
+insert_only_minor_compacted
+---- COLUMNS
+id bigint
+---- DEPENDENT_LOAD_HIVE
+INSERT INTO TABLE {db_name}{db_suffix}.{table_name} VALUES (1);
+INSERT INTO TABLE {db_name}{db_suffix}.{table_name} VALUES (2);
+ALTER TABLE {db_name}{db_suffix}.{table_name} compact 'minor' AND WAIT;
+INSERT INTO TABLE {db_name}{db_suffix}.{table_name} VALUES (3);
+INSERT INTO TABLE {db_name}{db_suffix}.{table_name} VALUES (4);
+---- TABLE_PROPERTIES
+transactional=true
+transactional_properties=insert_only
+====
+---- DATASET
+functional
+---- BASE_TABLE_NAME
+insert_only_major_and_minor_compacted
+---- COLUMNS
+id bigint
+---- DEPENDENT_LOAD_HIVE
+INSERT INTO TABLE {db_name}{db_suffix}.{table_name} VALUES (1);
+INSERT INTO TABLE {db_name}{db_suffix}.{table_name} VALUES (2);
+ALTER TABLE {db_name}{db_suffix}.{table_name} compact 'major' AND WAIT;
+INSERT INTO TABLE {db_name}{db_suffix}.{table_name} VALUES (3);
+INSERT INTO TABLE {db_name}{db_suffix}.{table_name} VALUES (4);
+ALTER TABLE {db_name}{db_suffix}.{table_name} compact 'minor' AND WAIT;
+INSERT INTO TABLE {db_name}{db_suffix}.{table_name} VALUES (5);
+INSERT INTO TABLE {db_name}{db_suffix}.{table_name} VALUES (6);
+---- TABLE_PROPERTIES
+transactional=true
+transactional_properties=insert_only
====
diff --git a/testdata/datasets/functional/schema_constraints.csv b/testdata/datasets/functional/schema_constraints.csv
index 7b3a265be..55e0d17d1 100644
--- a/testdata/datasets/functional/schema_constraints.csv
+++ b/testdata/datasets/functional/schema_constraints.csv
@@ -352,3 +352,6 @@ table_name:complextypes_maps_view, constraint:restrict_to, table_format:orc/def/
# 'alltypestiny_negative' only used in ORC tests.
table_name:alltypestiny_negative, constraint:restrict_to, table_format:orc/def/block
+
+table_name:insert_only_minor_compacted, constraint:restrict_to, table_format:parquet/none/none
+table_name:insert_only_major_and_minor_compacted, constraint:restrict_to, table_format:parquet/none/none
diff --git a/testdata/workloads/functional-query/queries/QueryTest/acid.test b/testdata/workloads/functional-query/queries/QueryTest/acid.test
index 1b27ee404..1553c9f3c 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/acid.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/acid.test
@@ -136,3 +136,28 @@ from functional_orc_def.complextypestbl_minor_compacted;
---- TYPES
BIGINT,BIGINT
====
+---- QUERY
+# Test that Impala sees the compacted delta dir in a minor-compacted insert-only table.
+show files in functional_parquet.insert_only_minor_compacted;
+---- LABELS
+Path,Size,Partition
+---- RESULTS
+row_regex:'$NAMENODE/test-warehouse/managed/functional_parquet.db/insert_only_minor_compacted_parquet/delta_0000001_0000002_v\d+/000000_0','.+B',''
+row_regex:'$NAMENODE/test-warehouse/managed/functional_parquet.db/insert_only_minor_compacted_parquet/delta_0000003_0000003_0000/000000_0','.+B',''
+row_regex:'$NAMENODE/test-warehouse/managed/functional_parquet.db/insert_only_minor_compacted_parquet/delta_0000004_0000004_0000/000000_0','.+B',''
+---- TYPES
+STRING,STRING,STRING
+====
+---- QUERY
+# Test that Impala sees the compacted delta dir in a first major then minor-compacted insert-only table.
+show files in functional_parquet.insert_only_major_and_minor_compacted;
+---- LABELS
+Path,Size,Partition
+---- RESULTS
+row_regex:'$NAMENODE/test-warehouse/managed/functional_parquet.db/insert_only_major_and_minor_compacted_parquet/base_0000002_v\d+/000000_0','.+B',''
+row_regex:'$NAMENODE/test-warehouse/managed/functional_parquet.db/insert_only_major_and_minor_compacted_parquet/delta_0000003_0000004_v\d+/000000_0','.+B',''
+row_regex:'$NAMENODE/test-warehouse/managed/functional_parquet.db/insert_only_major_and_minor_compacted_parquet/delta_0000005_0000005_0000/000000_0','.+B',''
+row_regex:'$NAMENODE/test-warehouse/managed/functional_parquet.db/insert_only_major_and_minor_compacted_parquet/delta_0000006_0000006_0000/000000_0','.+B',''
+---- TYPES
+STRING,STRING,STRING
+====