You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by dk...@apache.org on 2022/07/15 10:59:47 UTC
[hive] branch master updated: HIVE-26374: Query based compaction fails for tables with CDT and columns with Reserved Keywords (Chiran Ravani, reviewed by Denys Kuzmenko, Sai Hemanth Gantasala)
This is an automated email from the ASF dual-hosted git repository.
dkuzmenko pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new e3751ab5453 HIVE-26374: Query based compaction fails for tables with CDT and columns with Reserved Keywords (Chiran Ravani, reviewed by Denys Kuzmenko, Sai Hemanth Gantasala)
e3751ab5453 is described below
commit e3751ab545370f9b252d0b4a07bc315037541a95
Author: Chiran Ravani <ch...@gmail.com>
AuthorDate: Fri Jul 15 06:59:37 2022 -0400
HIVE-26374: Query based compaction fails for tables with CDT and columns with Reserved Keywords (Chiran Ravani, reviewed by Denys Kuzmenko, Sai Hemanth Gantasala)
Closes #3419
---
.../ql/txn/compactor/TestCrudCompactorOnTez.java | 65 ++++++++++++++++++++++
.../apache/hadoop/hive/ql/exec/DDLPlanUtils.java | 2 +-
.../ql/txn/compactor/CompactionQueryBuilder.java | 15 +++--
3 files changed, 73 insertions(+), 9 deletions(-)
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java
index ece4d5290e1..cffb58bc41c 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/txn/compactor/TestCrudCompactorOnTez.java
@@ -2351,4 +2351,69 @@ public class TestCrudCompactorOnTez extends CompactorOnTezTest {
Assert.assertEquals("base_10000000_v0000009", baseName);
}
+ @Test
+ public void testCompactionShouldNotFailOnStructField() throws Exception {
+ conf.setBoolVar(HiveConf.ConfVars.COMPACTOR_CRUD_QUERY_BASED, true);
+ String dbName = "default";
+ String tblName = "compaction_hive_26374";
+
+ TxnStore txnHandler = TxnUtils.getTxnStore(conf);
+ TestDataProvider testDP = new TestDataProvider();
+
+ // Create test table
+ executeStatementOnDriver("drop table if exists " + tblName, driver);
+ executeStatementOnDriver("CREATE TABLE " + tblName + "(col1 array<struct<arr_col1:int, `timestamp`:string>>)" +
+ "STORED AS ORC TBLPROPERTIES('transactional'='true')", driver);
+
+ // Insert test data into test table
+ executeStatementOnDriver("INSERT INTO TABLE " + tblName +
+ " SELECT ARRAY(NAMED_STRUCT('arr_col1',1,'timestamp','2022-07-05 21:51:20.371'))",driver);
+ executeStatementOnDriver("INSERT INTO TABLE " + tblName +
+ " SELECT ARRAY(NAMED_STRUCT('arr_col1',2,'timestamp','2022-07-05 21:51:20.371'))",driver);
+
+ // Find the location of the table
+ IMetaStoreClient msClient = new HiveMetaStoreClient(conf);
+ Table table = msClient.getTable(dbName, tblName);
+ FileSystem fs = FileSystem.get(conf);
+ // Verify deltas (delta_0000001_0000001_0000, delta_0000002_0000002_0000) are present
+ Assert.assertEquals("Delta directories does not match before compaction",
+ Arrays.asList("delta_0000001_0000001_0000", "delta_0000002_0000002_0000"),
+ CompactorTestUtil.getBaseOrDeltaNames(fs, AcidUtils.deltaFileFilter, table, null));
+
+ // Get all data before compaction is run
+ List<String> expectedData = testDP.getAllData(tblName);
+
+ //Do a compaction directly and wait for it to finish
+ CompactionRequest rqst = new CompactionRequest(dbName, tblName, CompactionType.MAJOR);
+ CompactionResponse resp = txnHandler.compact(rqst);
+ runWorker(conf);
+
+ CompactorTestUtil.runCleaner(conf);
+
+ //Check if the compaction succeed
+ ShowCompactResponse rsp = txnHandler.showCompact(new ShowCompactRequest());
+ List<ShowCompactResponseElement> compacts = rsp.getCompacts();
+ Assert.assertEquals("Expecting 1 rows and found " + compacts.size(), 1, compacts.size());
+ Assert.assertEquals("Expecting compaction state 'succeeded' and found:" + compacts.get(0).getState(),
+ "succeeded", compacts.get(0).getState());
+ // Should contain only one base directory now
+ FileStatus[] status = fs.listStatus(new Path(table.getSd().getLocation()));
+ int inputFileCount = 0;
+ for(FileStatus file: status) {
+ inputFileCount++;
+ }
+ Assert.assertEquals("Expecting 1 file and found "+ inputFileCount, 1, inputFileCount);
+
+ // Check bucket file name
+ List<String> baseDir = CompactorTestUtil.getBaseOrDeltaNames(fs, AcidUtils.baseFileFilter, table, null);
+ List<String> expectedBucketFiles = Arrays.asList("bucket_00000");
+ Assert.assertEquals("Bucket names are not matching after compaction", expectedBucketFiles,
+ CompactorTestUtil
+ .getBucketFileNames(fs, table, null, baseDir.get(0)));
+
+ // Verify all contents
+ List<String> actualData = testDP.getAllData(tblName);
+ Assert.assertEquals(expectedData, actualData);
+ }
+
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java
index 80561179242..62248ff4f71 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLPlanUtils.java
@@ -837,7 +837,7 @@ public class DDLPlanUtils {
/**
* Struct fields are identifiers, need to be put between ``.
*/
- private String formatType(TypeInfo typeInfo) {
+ public static String formatType(TypeInfo typeInfo) {
switch (typeInfo.getCategory()) {
case PRIMITIVE:
return typeInfo.getTypeName();
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactionQueryBuilder.java b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactionQueryBuilder.java
index d9aeeeed8d6..8a297533888 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactionQueryBuilder.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/txn/compactor/CompactionQueryBuilder.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hive.ql.io.AcidUtils;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.util.DirectionUtils;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hive.common.util.HiveStringUtils;
import org.slf4j.Logger;
@@ -43,6 +44,7 @@ import org.slf4j.LoggerFactory;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
@@ -418,16 +420,13 @@ class CompactionQueryBuilder {
+ "`currentTransaction` bigint, `row` struct<");
}
List<FieldSchema> cols = sourceTab.getSd().getCols();
- boolean isFirst = true;
+ List<String> columnDescs = new ArrayList<>();
for (FieldSchema col : cols) {
- if (!isFirst) {
- query.append(", ");
- }
- isFirst = false;
- query.append("`").append(col.getName()).append("` ");
- query.append(crud ? ":" : "");
- query.append(col.getType());
+ String columnType = DDLPlanUtils.formatType(TypeInfoUtils.getTypeInfoFromTypeString(col.getType()));
+ String columnDesc = "`" + col.getName() + "` " + (crud ? ":" : "") + columnType;
+ columnDescs.add(columnDesc);
}
+ query.append(StringUtils.join(',',columnDescs));
query.append(crud ? ">" : "");
query.append(") ");
}