You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by to...@apache.org on 2019/05/22 18:18:25 UTC
[impala] 04/04: IMPALA-8435. Prohibit operations on transactional
table.
This is an automated email from the ASF dual-hosted git repository.
todd pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 67015004cfd16b78cf5b4019c29a6f1301b37fcc
Author: arorasudhanshu <su...@cloudera.com>
AuthorDate: Fri May 3 15:04:09 2019 -0700
IMPALA-8435. Prohibit operations on transactional table.
Copied some code from Hive to identify if the table is transactional,
insert only table.
Also modified code to prohibit write operations on insert only table.
That code will be reverted once we add support for write operations on
insert only table.
Testing Done:
- Added a new test in AnalyzerTest
Change-Id: I740dc4ce0dbbc0c2e042b01832e606cc1ac4132a
Reviewed-on: http://gerrit.cloudera.org:8080/13311
Tested-by: Todd Lipcon <to...@apache.org>
Reviewed-by: Sudhanshu Arora <su...@cloudera.com>
Reviewed-by: Todd Lipcon <to...@apache.org>
---
.../org/apache/impala/analysis/AlterTableStmt.java | 1 +
.../java/org/apache/impala/analysis/Analyzer.java | 40 ++++++++++
.../org/apache/impala/analysis/BaseTableRef.java | 1 +
.../apache/impala/analysis/ComputeStatsStmt.java | 3 +
.../impala/analysis/CreateTableLikeStmt.java | 3 +
.../impala/analysis/DropTableOrViewStmt.java | 1 +
.../org/apache/impala/analysis/InsertStmt.java | 2 +
.../org/apache/impala/analysis/LoadDataStmt.java | 1 +
.../java/org/apache/impala/analysis/TableDef.java | 2 +
.../org/apache/impala/analysis/TruncateStmt.java | 1 +
.../java/org/apache/impala/util/AcidUtils.java | 64 ++++++++++++++++
.../org/apache/impala/analysis/AnalyzerTest.java | 85 ++++++++++++++++++++++
testdata/bin/generate-schema-statements.py | 62 ++++++++++++----
testdata/datasets/README | 8 ++
.../functional/functional_schema_template.sql | 30 ++++++++
.../datasets/functional/schema_constraints.csv | 7 ++
16 files changed, 296 insertions(+), 15 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/analysis/AlterTableStmt.java b/fe/src/main/java/org/apache/impala/analysis/AlterTableStmt.java
index 6304678..1a31195 100644
--- a/fe/src/main/java/org/apache/impala/analysis/AlterTableStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/AlterTableStmt.java
@@ -90,6 +90,7 @@ public abstract class AlterTableStmt extends StatementBase {
}
Preconditions.checkState(tableRef instanceof BaseTableRef);
table_ = tableRef.getTable();
+ analyzer.ensureTableNotTransactional(table_);
if (table_ instanceof FeDataSourceTable
&& !(this instanceof AlterTableSetColumnStats)) {
throw new AnalysisException(String.format(
diff --git a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
index d54d02e..d20c54e 100644
--- a/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
+++ b/fe/src/main/java/org/apache/impala/analysis/Analyzer.java
@@ -78,6 +78,7 @@ import org.apache.impala.thrift.TLineageGraph;
import org.apache.impala.thrift.TNetworkAddress;
import org.apache.impala.thrift.TQueryCtx;
import org.apache.impala.thrift.TQueryOptions;
+import org.apache.impala.util.AcidUtils;
import org.apache.impala.util.DisjointSet;
import org.apache.impala.util.Graph.RandomAccessibleGraph;
import org.apache.impala.util.Graph.SccCondensedGraph;
@@ -134,6 +135,12 @@ public class Analyzer {
"Data source does not exist: ";
public final static String DATA_SRC_ALREADY_EXISTS_ERROR_MSG =
"Data source already exists: ";
+ private static final String INSERT_ONLY_ACID_TABLE_SUPPORTED_ERROR_MSG =
+ "Table %s not supported. Transactional (ACID) tables are " +
+ "only supported when they are configured as insert_only.";
+ private static final String TRANSACTIONAL_TABLE_NOT_SUPPORTED =
+ "Table %s not supported. Transactional (ACID) tables are " +
+ "only supported for read.";
private final static Logger LOG = LoggerFactory.getLogger(Analyzer.class);
@@ -187,6 +194,39 @@ public class Analyzer {
public void setHasWithClause() { hasWithClause_ = true; }
public boolean hasWithClause() { return hasWithClause_; }
+
+ /**
+ * @param tblProperties Table properties that are used to check transactional nature
+ * @param tableName Table name to be reported in exception message
+ * @throws AnalysisException If table is full acid table.
+ */
+ public static void ensureTableNotFullAcid(Map<String, String> tblProperties,
+ String tableName)
+ throws AnalysisException {
+ if (AcidUtils.isFullAcidTable(tblProperties)) {
+ throw new AnalysisException(String.format(
+ INSERT_ONLY_ACID_TABLE_SUPPORTED_ERROR_MSG, tableName));
+ }
+ }
+
+ /**
+ * @param table Table whose properties need to be checked.
+ * @throws AnalysisException If table is full acid table.
+ */
+ public static void ensureTableNotFullAcid(FeTable table)
+ throws AnalysisException {
+ ensureTableNotFullAcid(table.getMetaStoreTable().getParameters(),
+ table.getFullName());
+ }
+
+ public static void ensureTableNotTransactional(FeTable table)
+ throws AnalysisException {
+ if (AcidUtils.isTransactionalTable(table.getMetaStoreTable().getParameters())) {
+ throw new AnalysisException(String.format(TRANSACTIONAL_TABLE_NOT_SUPPORTED,
+ table.getFullName()));
+ }
+ }
+
// State shared between all objects of an Analyzer tree. We use LinkedHashMap and
// LinkedHashSet where applicable to preserve the iteration order and make the class
// behave identical across different implementations of the JVM.
diff --git a/fe/src/main/java/org/apache/impala/analysis/BaseTableRef.java b/fe/src/main/java/org/apache/impala/analysis/BaseTableRef.java
index 7b10fdc..ea031a6 100644
--- a/fe/src/main/java/org/apache/impala/analysis/BaseTableRef.java
+++ b/fe/src/main/java/org/apache/impala/analysis/BaseTableRef.java
@@ -64,6 +64,7 @@ public class BaseTableRef extends TableRef {
requireGrantOption_);
desc_ = analyzer.registerTableRef(this);
isAnalyzed_ = true;
+ analyzer.ensureTableNotFullAcid(getTable());
analyzeTableSample(analyzer);
analyzeHints(analyzer);
analyzeJoin(analyzer);
diff --git a/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java b/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java
index f0844bb..a23df47 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ComputeStatsStmt.java
@@ -360,6 +360,9 @@ public class ComputeStatsStmt extends StatementBase {
"COMPUTE STATS not supported for nested collection: %s", tableName_));
}
table_ = analyzer.getTable(tableName_, Privilege.ALTER, Privilege.SELECT);
+ // Adding the check here instead of tableRef.analyze because tableRef is
+ // used at multiple places and will even disallow select.
+ analyzer.ensureTableNotTransactional(table_);
if (!(table_ instanceof FeFsTable)) {
if (partitionSet_ != null) {
diff --git a/fe/src/main/java/org/apache/impala/analysis/CreateTableLikeStmt.java b/fe/src/main/java/org/apache/impala/analysis/CreateTableLikeStmt.java
index e465ee7..4eec464 100644
--- a/fe/src/main/java/org/apache/impala/analysis/CreateTableLikeStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/CreateTableLikeStmt.java
@@ -164,6 +164,9 @@ public class CreateTableLikeStmt extends StatementBase {
// Make sure the source table exists and the user has permission to access it.
FeTable srcTable = analyzer.getTable(srcTableName_, Privilege.VIEW_METADATA);
+
+ analyzer.ensureTableNotFullAcid(srcTable);
+
if (KuduTable.isKuduTable(srcTable.getMetaStoreTable())) {
throw new AnalysisException("Cloning a Kudu table using CREATE TABLE LIKE is " +
"not supported.");
diff --git a/fe/src/main/java/org/apache/impala/analysis/DropTableOrViewStmt.java b/fe/src/main/java/org/apache/impala/analysis/DropTableOrViewStmt.java
index 364444e..3619d9e 100644
--- a/fe/src/main/java/org/apache/impala/analysis/DropTableOrViewStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/DropTableOrViewStmt.java
@@ -103,6 +103,7 @@ public class DropTableOrViewStmt extends StatementBase {
FeTable table = analyzer.getTable(tableName_, /* add access event */ true,
/* add column-level privilege */ false, Privilege.DROP);
Preconditions.checkNotNull(table);
+ analyzer.ensureTableNotTransactional(table);
if (table instanceof FeView && dropTable_) {
throw new AnalysisException(String.format(
"DROP TABLE not allowed on a view: %s.%s", dbName_, getTbl()));
diff --git a/fe/src/main/java/org/apache/impala/analysis/InsertStmt.java b/fe/src/main/java/org/apache/impala/analysis/InsertStmt.java
index 1dcc99f..e0bd6fb 100644
--- a/fe/src/main/java/org/apache/impala/analysis/InsertStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/InsertStmt.java
@@ -425,6 +425,8 @@ public class InsertStmt extends StatementBase {
.build());
}
+ analyzer.ensureTableNotTransactional(table_);
+
// We do not support (in|up)serting into views.
if (table_ instanceof FeView) {
throw new AnalysisException(
diff --git a/fe/src/main/java/org/apache/impala/analysis/LoadDataStmt.java b/fe/src/main/java/org/apache/impala/analysis/LoadDataStmt.java
index 85e9557..31614ca 100644
--- a/fe/src/main/java/org/apache/impala/analysis/LoadDataStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/LoadDataStmt.java
@@ -109,6 +109,7 @@ public class LoadDataStmt extends StatementBase {
throw new AnalysisException("LOAD DATA only supported for HDFS tables: " +
dbName_ + "." + getTbl());
}
+ analyzer.ensureTableNotTransactional(table);
// Analyze the partition spec, if one was specified.
if (partitionSpec_ != null) {
diff --git a/fe/src/main/java/org/apache/impala/analysis/TableDef.java b/fe/src/main/java/org/apache/impala/analysis/TableDef.java
index e9e5399..b37864a 100644
--- a/fe/src/main/java/org/apache/impala/analysis/TableDef.java
+++ b/fe/src/main/java/org/apache/impala/analysis/TableDef.java
@@ -222,6 +222,8 @@ class TableDef {
Preconditions.checkState(tableName_ != null && !tableName_.isEmpty());
fqTableName_ = analyzer.getFqTableName(getTblName());
fqTableName_.analyze();
+ // Disallow creation of full ACID table.
+ analyzer.ensureTableNotFullAcid(options_.tblProperties, fqTableName_.toString());
analyzeColumnDefs(analyzer);
analyzePrimaryKeys();
diff --git a/fe/src/main/java/org/apache/impala/analysis/TruncateStmt.java b/fe/src/main/java/org/apache/impala/analysis/TruncateStmt.java
index 2c78259..64a1e99 100644
--- a/fe/src/main/java/org/apache/impala/analysis/TruncateStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/TruncateStmt.java
@@ -67,6 +67,7 @@ public class TruncateStmt extends StatementBase {
throw new AnalysisException(String.format(
"TRUNCATE TABLE not supported on non-HDFS table: %s", table_.getFullName()));
}
+ analyzer.ensureTableNotTransactional(table_);
}
@Override
diff --git a/fe/src/main/java/org/apache/impala/util/AcidUtils.java b/fe/src/main/java/org/apache/impala/util/AcidUtils.java
new file mode 100644
index 0000000..8bffca4
--- /dev/null
+++ b/fe/src/main/java/org/apache/impala/util/AcidUtils.java
@@ -0,0 +1,64 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+package org.apache.impala.util;
+
+import com.google.common.base.Preconditions;
+
+import java.util.Map;
+
+/**
+ * Contains utility functions for working with Acid tables.
+ * <p>
+ * The code is mostly copy pasted from Hive. Ideally we should use the
+ * the code directly from Hive.
+ * </p>
+ */
+public class AcidUtils {
+ // Constant also defined in TransactionalValidationListener
+ public static final String INSERTONLY_TRANSACTIONAL_PROPERTY = "insert_only";
+ // Constant also defined in hive_metastoreConstants
+ public static final String TABLE_IS_TRANSACTIONAL = "transactional";
+ public static final String TABLE_TRANSACTIONAL_PROPERTIES = "transactional_properties";
+
+
+ // The code is same as what exists in AcidUtils.java in hive-exec.
+ // Ideally we should move the AcidUtils code from hive-exec into
+ // hive-standalone-metastore or some other jar and use it here.
+ private static boolean isInsertOnlyTable(Map<String, String> props) {
+ Preconditions.checkNotNull(props);
+ if (!isTransactionalTable(props)) {
+ return false;
+ }
+ String transactionalProp = props.get(TABLE_TRANSACTIONAL_PROPERTIES);
+ return transactionalProp != null && INSERTONLY_TRANSACTIONAL_PROPERTY.
+ equalsIgnoreCase(transactionalProp);
+ }
+
+ public static boolean isTransactionalTable(Map<String, String> props) {
+ Preconditions.checkNotNull(props);
+ String tableIsTransactional = props.get(TABLE_IS_TRANSACTIONAL);
+ if (tableIsTransactional == null) {
+ tableIsTransactional = props.get(TABLE_IS_TRANSACTIONAL.toUpperCase());
+ }
+ return tableIsTransactional != null && tableIsTransactional.equalsIgnoreCase("true");
+ }
+
+ public static boolean isFullAcidTable(Map<String, String> props) {
+ return isTransactionalTable(props) && !isInsertOnlyTable(props);
+ }
+
+}
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzerTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzerTest.java
index 874578c..4177bf7 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzerTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzerTest.java
@@ -28,8 +28,11 @@ import org.apache.impala.catalog.ScalarType;
import org.apache.impala.catalog.Type;
import org.apache.impala.common.AnalysisException;
import org.apache.impala.common.FrontendTestBase;
+import org.apache.impala.compat.MetastoreShim;
import org.apache.impala.util.FunctionUtils;
import org.junit.Assert;
+import org.junit.Assume;
+import org.junit.Ignore;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -530,6 +533,88 @@ public class AnalyzerTest extends FrontendTestBase {
"Table does not exist: default.doesnt_exist");
}
+ @Test
+ public void TestAnalyzeTransactional() {
+ Assume.assumeTrue(MetastoreShim.getMajorVersion() > 2);
+ String errorMsg =
+ "Table functional_orc_def.full_transactional_table not supported. Transactional (ACID)" +
+ " tables are only supported when they are configured as insert_only.";
+
+ String insertOnlyErrorMsg =
+ "Table functional.insert_only_transactional_table not supported. " +
+ "Transactional (ACID) tables are only supported for read.";
+
+ String insertOnlyErrorForFullMsg =
+ "Table functional_orc_def.full_transactional_table not supported. " +
+ "Transactional (ACID) tables are only supported for read.";
+
+ AnalysisError(
+ "create table test as select * from functional_orc_def.full_transactional_table",
+ errorMsg);
+ AnalyzesOk(
+ "create table test as select * from functional.insert_only_transactional_table");
+
+ AnalysisError(
+ "create table test like functional_orc_def.full_transactional_table",
+ errorMsg);
+ AnalyzesOk("create table test like functional.insert_only_transactional_table");
+
+ AnalysisError(
+ "insert into test select * from functional_orc_def.full_transactional_table",
+ errorMsg);
+ AnalyzesOk("insert into functional.testtbl select *,'test',1 " +
+ "from functional.insert_only_transactional_table");
+
+ AnalysisError(
+ "insert into functional.insert_only_transactional_table select * " +
+ "from functional.insert_only_transactional_table",
+ insertOnlyErrorMsg);
+
+ AnalysisError(
+ "compute stats functional_orc_def.full_transactional_table",
+ errorMsg);
+ AnalysisError(
+ "compute stats functional.insert_only_transactional_table",
+ insertOnlyErrorMsg);
+
+ AnalysisError(
+ "select * from functional_orc_def.full_transactional_table",
+ errorMsg);
+ AnalyzesOk("select * from functional.insert_only_transactional_table");
+
+ AnalysisError(
+ "drop table functional_orc_def.full_transactional_table",
+ insertOnlyErrorForFullMsg);
+ AnalysisError("drop table functional.insert_only_transactional_table",
+ insertOnlyErrorMsg);
+
+ AnalysisError(
+ "truncate table functional_orc_def.full_transactional_table",
+ insertOnlyErrorForFullMsg);
+ AnalysisError("truncate table functional.insert_only_transactional_table",
+ insertOnlyErrorMsg);
+
+ AnalysisError(
+ "alter table functional_orc_def.full_transactional_table " +
+ "add columns (col2 string)",
+ errorMsg);
+ AnalysisError(
+ "alter table functional.insert_only_transactional_table " +
+ "add columns (col2 string)",
+ insertOnlyErrorMsg);
+
+ AnalysisError(
+ "drop stats functional_orc_def.full_transactional_table",
+ errorMsg);
+ AnalyzesOk("drop stats functional.insert_only_transactional_table");
+
+ AnalyzesOk("describe functional.insert_only_transactional_table");
+ AnalyzesOk("describe functional_orc_def.full_transactional_table");
+
+ AnalyzesOk("show column stats functional_orc_def.full_transactional_table");
+ AnalyzesOk("show column stats functional.insert_only_transactional_table");
+ }
+
private Function createFunction(boolean hasVarArgs, Type... args) {
return new Function(new FunctionName("test"), args, Type.INVALID, hasVarArgs);
}
diff --git a/testdata/bin/generate-schema-statements.py b/testdata/bin/generate-schema-statements.py
index 95df878..f4905ae 100755
--- a/testdata/bin/generate-schema-statements.py
+++ b/testdata/bin/generate-schema-statements.py
@@ -244,8 +244,34 @@ def build_create_statement(table_template, table_name, db_name, db_suffix,
hdfs_location=hdfs_location)
return create_stmt
+
+def parse_table_properties(file_format, table_properties):
+ """
+ Read the properties specified in the TABLE_PROPERTIES section.
+ The table properties can be restricted to a file format or are applicable
+ for all formats.
+ For specific format the syntax is <fileformat>:<key>=<val>
+ """
+ tblproperties = {}
+ TABLE_PROPERTY_RE = re.compile(
+ # Optional '<data-format>:' prefix, capturing just the 'data-format' part.
+ r'(?:(\w+):)?' +
+ # Required key=value, capturing the key and value
+ r'(.+?)=(.*)')
+ for table_property in filter(None, table_properties.split("\n")):
+ m = TABLE_PROPERTY_RE.match(table_property)
+ if not m:
+ raise Exception("Invalid table property line: {0}", format(table_property))
+ only_format, key, val = m.groups()
+ if only_format is not None and only_format != file_format:
+ continue
+ tblproperties[key] = val
+
+ return tblproperties
+
+
def build_table_template(file_format, columns, partition_columns, row_format,
- avro_schema_dir, table_name, table_properties):
+ avro_schema_dir, table_name, tblproperties):
if file_format == 'hbase':
return build_hbase_create_stmt_in_hive(columns, partition_columns, table_name)
@@ -262,9 +288,8 @@ def build_table_template(file_format, columns, partition_columns, row_format,
file_format_string = "STORED AS {file_format}"
tblproperties_clause = "TBLPROPERTIES (\n{0}\n)"
- tblproperties = {}
- external = "EXTERNAL"
+ external = "" if is_transactional(tblproperties) else "EXTERNAL"
if file_format == 'avro':
# TODO Is this flag ever used?
@@ -286,15 +311,6 @@ def build_table_template(file_format, columns, partition_columns, row_format,
# Kudu's test tables are managed.
external = ""
- # Read the properties specified in the TABLE_PROPERTIES section. When the specified
- # properties have the same key as a default property, the value for the specified
- # property is used.
- if table_properties:
- for table_property in table_properties.split("\n"):
- format_prop = table_property.split(":")
- if format_prop[0] == file_format:
- key_val = format_prop[1].split("=")
- tblproperties[key_val[0]] = key_val[1]
all_tblproperties = []
for key, value in tblproperties.iteritems():
@@ -564,10 +580,17 @@ def generate_statements(output_name, test_vectors, sections,
print 'Skipping table: %s.%s, table is not in specified table list' % (db, table_name)
continue
+ # Check Hive version requirement, if present.
+ if section['HIVE_MAJOR_VERSION'] and \
+ section['HIVE_MAJOR_VERSION'].strip() != \
+ os.environ['IMPALA_HIVE_MAJOR_VERSION'].strip():
+ print "Skipping table '{0}.{1}': wrong Hive major version".format(db, table_name)
+ continue
+
if table_format in schema_only_constraints and \
table_name.lower() not in schema_only_constraints[table_format]:
print ('Skipping table: %s.%s, \'only\' constraint for format did not '
- 'include this table.') % (db, table_name)
+ 'include this table.') % (db, table_name)
continue
if schema_include_constraints[table_name.lower()] and \
@@ -636,12 +659,16 @@ def generate_statements(output_name, test_vectors, sections,
if file_format not in IMPALA_SUPPORTED_INSERT_FORMATS:
create_file_format = 'text'
+ tblproperties = parse_table_properties(create_file_format, table_properties)
+
output = impala_create
if create_hive or file_format == 'hbase':
output = hive_output
elif codec == 'lzo':
# Impala CREATE TABLE doesn't allow INPUTFORMAT.
output = hive_output
+ elif is_transactional(tblproperties):
+ output = hive_output
# TODO: Currently, Kudu does not support partitioned tables via Impala.
# If a CREATE_KUDU section was provided, assume it handles the partition columns
@@ -667,7 +694,7 @@ def generate_statements(output_name, test_vectors, sections,
avro_schema_dir = "%s/%s" % (AVRO_SCHEMA_DIR, data_set)
table_template = build_table_template(
create_file_format, columns, partition_columns,
- row_format, avro_schema_dir, table_name, table_properties)
+ row_format, avro_schema_dir, table_name, tblproperties)
# Write Avro schema to local file
if file_format == 'avro':
if not os.path.exists(avro_schema_dir):
@@ -768,12 +795,17 @@ def generate_statements(output_name, test_vectors, sections,
hbase_post_load.write_to_file('post-load-' + output_name + '-hbase-generated.sql')
impala_invalidate.write_to_file("invalidate-" + output_name + "-impala-generated.sql")
+
+def is_transactional(table_properties):
+ return table_properties.get('transactional', "").lower() == 'true'
+
+
def parse_schema_template_file(file_name):
VALID_SECTION_NAMES = ['DATASET', 'BASE_TABLE_NAME', 'COLUMNS', 'PARTITION_COLUMNS',
'ROW_FORMAT', 'CREATE', 'CREATE_HIVE', 'CREATE_KUDU',
'DEPENDENT_LOAD', 'DEPENDENT_LOAD_KUDU', 'DEPENDENT_LOAD_HIVE',
'LOAD', 'ALTER', 'HBASE_COLUMN_FAMILIES',
- 'TABLE_PROPERTIES', 'HBASE_REGION_SPLITS']
+ 'TABLE_PROPERTIES', 'HBASE_REGION_SPLITS', 'HIVE_MAJOR_VERSION']
return parse_test_file(file_name, VALID_SECTION_NAMES, skip_unknown_sections=False)
if __name__ == "__main__":
diff --git a/testdata/datasets/README b/testdata/datasets/README
index bf12a56..75de1b4 100644
--- a/testdata/datasets/README
+++ b/testdata/datasets/README
@@ -74,3 +74,11 @@ The schema template SQL files have the following format:
DEPENDENT_LOAD_HIVE
Statements to be executed during the "dependent load" phase. These statements
are run after the initial (base table) load is complete.
+
+ HIVE_MAJOR_VERSION
+ The required major version of Hive for this table. If the major version
+ of Hive at runtime does not exactly match the version specified in this section,
+ the table will be skipped.
+
+ NOTE: this is not a _minimum_ version -- if HIVE_MAJOR_VERSION specifies '2',
+ the table will _not_ be loaded/created on Hive 3.
diff --git a/testdata/datasets/functional/functional_schema_template.sql b/testdata/datasets/functional/functional_schema_template.sql
index 187f478..3de8363 100644
--- a/testdata/datasets/functional/functional_schema_template.sql
+++ b/testdata/datasets/functional/functional_schema_template.sql
@@ -2122,6 +2122,36 @@ hadoop fs -put -f ${IMPALA_HOME}/testdata/data/table_missing_columns.csv /test-w
---- DATASET
functional
---- BASE_TABLE_NAME
+insert_only_transactional_table
+---- HIVE_MAJOR_VERSION
+3
+---- CREATE_HIVE
+---- COLUMNS
+col1 int
+---- TABLE_PROPERTIES
+transactional=true
+transactional_properties=insert_only
+---- LOAD
+-- TODO(todd) we need an empty load section with a comment in it here.
+-- This works around some "logic" in generate-schema-statements.py that
+-- says that, if a table has no LOAD section, it shouldn't be in non-text
+-- formats.
+====
+---- DATASET
+functional
+---- BASE_TABLE_NAME
+full_transactional_table
+---- HIVE_MAJOR_VERSION
+3
+---- CREATE_HIVE
+CREATE TABLE IF NOT EXISTS {db_name}{db_suffix}.{table_name} (
+ col1 int)
+STORED AS ORC
+TBLPROPERTIES('transactional'='true');
+====
+---- DATASET
+functional
+---- BASE_TABLE_NAME
testescape_16_lf
---- CREATE
CREATE EXTERNAL TABLE IF NOT EXISTS {db_name}{db_suffix}.{table_name} (
diff --git a/testdata/datasets/functional/schema_constraints.csv b/testdata/datasets/functional/schema_constraints.csv
index 379826c..d66e60d 100644
--- a/testdata/datasets/functional/schema_constraints.csv
+++ b/testdata/datasets/functional/schema_constraints.csv
@@ -241,3 +241,10 @@ table_name:date_tbl_error, constraint:restrict_to, table_format:text/bzip/block
table_name:date_tbl_error, constraint:restrict_to, table_format:text/gzip/block
table_name:date_tbl_error, constraint:restrict_to, table_format:text/snap/block
table_name:insert_date_tbl, constraint:restrict_to, table_format:hbase/none/none
+
+# Full transactional table is only supported for ORC
+table_name:full_transactional_table, constraint:restrict_to, table_format:orc/def/block
+
+# Insert-only transactional tables only work for file-format based tables
+table_name:insert_only_transactional_table, constraint:exclude, table_format:hbase/none/none
+table_name:insert_only_transactional_table, constraint:exclude, table_format:kudu/none/none