You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2019/09/28 18:14:49 UTC
[impala] branch master updated: IMPALA-8198: DATE: Read from avro.
This is an automated email from the ASF dual-hosted git repository.
tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new 27fa27e IMPALA-8198: DATE: Read from avro.
27fa27e is described below
commit 27fa27e808d74db3e07be419dd0657c4448534e5
Author: Attila Jeges <at...@cloudera.com>
AuthorDate: Tue Jun 25 14:36:26 2019 +0200
IMPALA-8198: DATE: Read from avro.
This change is a follow-up to IMPALA-7368 and adds support for DATE
type to the avro scanner.
Similarly to parquet, avro uses DATE logical type for dates. DATE
logical type annotates an INT32 that stores the number of days since
the unix epoch, 1 January 1970.
This representation introduces an avro interoperability issue between
Impala and older versions of Hive:
- Before version 3.1, Hive used Julian calendar to represent dates
up to 1582-10-05 and Gregorian calendar for dates starting with
1582-10-15. Dates between 1582-10-05 and 1582-10-15 were lost.
- Impala uses proleptic Gregorian calendar, extending the Gregorian
calendar backward to dates preceding its official introduction in
1582-10-15.
This means that pre-1582-10-15 dates written to an avro table by Hive
will be read back incorrectly by Impala.
Note that Hive 3.1 switched to proleptic Gregorian calendar too, so
for Hive 3.1+ this is no longer an issue.
Dependency changes:
- BE uses avro 1.7.4-p5 from native-toolchain.
Change-Id: I7a9d5b93a22cf3a00244037e187f8c145cacc959
Reviewed-on: http://gerrit.cloudera.org:8080/13944
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/codegen/gen_ir_descriptions.py | 2 +
be/src/exec/hdfs-avro-scanner-ir.cc | 21 ++++
be/src/exec/hdfs-avro-scanner-test.cc | 64 +++++++++++
be/src/exec/hdfs-avro-scanner.cc | 28 ++++-
be/src/exec/hdfs-avro-scanner.h | 2 +
be/src/util/avro-util.cc | 4 +
common/thrift/generate_error_codes.py | 3 +
.../org/apache/impala/catalog/HdfsFileFormat.java | 2 +-
.../apache/impala/planner/SingleNodePlanner.java | 2 +-
.../apache/impala/util/AvroSchemaConverter.java | 5 +-
.../org/apache/impala/util/AvroSchemaParser.java | 1 +
.../catalog/CatalogObjectToFromThriftTest.java | 4 +-
testdata/avro_schema_resolution/create_table.sql | 1 +
testdata/avro_schema_resolution/file_schema2.avsc | 1 +
testdata/avro_schema_resolution/records2.avro | Bin 560 -> 618 bytes
testdata/avro_schema_resolution/records2.json | 1 +
testdata/bad_avro_snap/README | 16 +++
.../bad_avro_snap/hive2_pre_gregorian_date.avro | Bin 0 -> 219 bytes
.../bad_avro_snap/hive3_pre_gregorian_date.avro | Bin 0 -> 276 bytes
testdata/bad_avro_snap/out_of_range_date.avro | Bin 0 -> 220 bytes
testdata/bin/generate-schema-statements.py | 2 +
testdata/data/README | 9 +-
testdata/data/date_tbl.avro | Bin 207 -> 218 bytes
testdata/data/date_tbl.orc | Bin 0 -> 201 bytes
.../functional/functional_schema_template.sql | 30 +++++
.../datasets/functional/schema_constraints.csv | 6 +-
.../queries/DataErrorsTest/avro-errors.test | 43 +++++++
.../queries/QueryTest/avro-schema-changes.test | 11 +-
.../queries/QueryTest/avro-schema-resolution.test | 6 +-
.../queries/QueryTest/avro_date.test | 125 +++++++++++++++++++++
.../queries/QueryTest/date-fileformat-support.test | 41 +++++--
.../queries/QueryTest/date-partitioning.test | 31 -----
.../QueryTest/hive2-pre-gregorian-date.test | 2 +-
.../queries/QueryTest/out-of-range-date.test | 2 +
tests/query_test/test_date_queries.py | 44 ++++++--
35 files changed, 434 insertions(+), 75 deletions(-)
diff --git a/be/src/codegen/gen_ir_descriptions.py b/be/src/codegen/gen_ir_descriptions.py
index 1b369d9..5063015 100755
--- a/be/src/codegen/gen_ir_descriptions.py
+++ b/be/src/codegen/gen_ir_descriptions.py
@@ -157,6 +157,8 @@ ir_functions = [
"_ZN6impala15HdfsAvroScanner13ReadUnionTypeEiPPhS1_Pb"],
["READ_AVRO_BOOLEAN",
"_ZN6impala15HdfsAvroScanner15ReadAvroBooleanENS_13PrimitiveTypeEPPhS2_bPvPNS_7MemPoolE"],
+ ["READ_AVRO_DATE",
+ "_ZN6impala15HdfsAvroScanner12ReadAvroDateENS_13PrimitiveTypeEPPhS2_bPvPNS_7MemPoolE"],
["READ_AVRO_INT32",
"_ZN6impala15HdfsAvroScanner13ReadAvroInt32ENS_13PrimitiveTypeEPPhS2_bPvPNS_7MemPoolE"],
["READ_AVRO_INT64",
diff --git a/be/src/exec/hdfs-avro-scanner-ir.cc b/be/src/exec/hdfs-avro-scanner-ir.cc
index 8eba43e..abfc984 100644
--- a/be/src/exec/hdfs-avro-scanner-ir.cc
+++ b/be/src/exec/hdfs-avro-scanner-ir.cc
@@ -20,7 +20,9 @@
#include "exec/hdfs-avro-scanner.h"
#include "exec/read-write-util.h"
+#include "runtime/date-value.h"
#include "runtime/mem-tracker.h"
+#include "runtime/runtime-state.h"
#include "runtime/string-value.inline.h"
using namespace impala;
@@ -100,6 +102,25 @@ bool HdfsAvroScanner::ReadAvroBoolean(PrimitiveType type, uint8_t** data,
return true;
}
+bool HdfsAvroScanner::ReadAvroDate(PrimitiveType type, uint8_t** data, uint8_t* data_end,
+ bool write_slot, void* slot, MemPool* pool) {
+ ReadWriteUtil::ZIntResult r = ReadWriteUtil::ReadZInt(data, data_end);
+ if (UNLIKELY(!r.ok)) {
+ SetStatusCorruptData(TErrorCode::SCANNER_INVALID_INT);
+ return false;
+ }
+ if (write_slot) {
+ DCHECK_EQ(type, TYPE_DATE);
+ DateValue dv(r.val);
+ if (UNLIKELY(!dv.IsValid())) {
+ SetStatusInvalidValue(TErrorCode::AVRO_INVALID_DATE, r.val);
+ return false;
+ }
+ *reinterpret_cast<DateValue*>(slot) = dv;
+ }
+ return true;
+}
+
bool HdfsAvroScanner::ReadAvroInt32(PrimitiveType type, uint8_t** data, uint8_t* data_end,
bool write_slot, void* slot, MemPool* pool) {
ReadWriteUtil::ZIntResult r = ReadWriteUtil::ReadZInt(data, data_end);
diff --git a/be/src/exec/hdfs-avro-scanner-test.cc b/be/src/exec/hdfs-avro-scanner-test.cc
index def42e4..cea8d06 100644
--- a/be/src/exec/hdfs-avro-scanner-test.cc
+++ b/be/src/exec/hdfs-avro-scanner-test.cc
@@ -21,6 +21,7 @@
#include <limits.h>
#include "exec/read-write-util.h"
+#include "runtime/date-value.h"
#include "runtime/decimal-value.inline.h"
#include "runtime/runtime-state.h"
#include "runtime/string-value.inline.h"
@@ -113,6 +114,12 @@ class HdfsAvroScannerTest : public testing::Test {
expected_double, expected_encoded_len, expected_error);
}
+ void TestReadAvroDate(uint8_t* data, int64_t data_len, const DateValue& expected_val,
+ int expected_encoded_len, TErrorCode::type expected_error = TErrorCode::OK) {
+ TestReadAvroType(&HdfsAvroScanner::ReadAvroDate, TYPE_DATE, data, data_len,
+ expected_val, expected_encoded_len, expected_error);
+ }
+
void TestReadAvroInt64(uint8_t* data, int64_t data_len, int64_t expected_val,
int expected_encoded_len, TErrorCode::type expected_error = TErrorCode::OK) {
TestReadAvroType(&HdfsAvroScanner::ReadAvroInt64, TYPE_BIGINT, data, data_len,
@@ -271,6 +278,63 @@ TEST_F(HdfsAvroScannerTest, Int32Test) {
// TODO: we don't handle invalid values (e.g. overflow) (IMPALA-3659)
}
+TEST_F(HdfsAvroScannerTest, DateTest) {
+ uint8_t data[100];
+ data[0] = 1; // decodes to -1 which corresponds to 1969-12-31.
+ TestReadAvroDate(data, 1, DateValue(1969, 12, 31), 1);
+ TestReadAvroDate(data, 10, DateValue(1969, 12, 31), 1);
+
+ // Empty data.
+ DateValue invalid_dv;
+ TestReadAvroDate(data, 0, invalid_dv, -1, TErrorCode::SCANNER_INVALID_INT);
+
+ data[0] = 2; // decodes to 1 which corresponds to 1970-01-02.
+ TestReadAvroDate(data, 1, DateValue(1970, 1, 2), 1);
+ TestReadAvroDate(data, 10, DateValue(1970, 1, 2), 1);
+ TestReadAvroDate(data, 0, invalid_dv, -1, TErrorCode::SCANNER_INVALID_INT);
+
+ data[0] = 0x80; // decodes to 64 which corresponds to 1970-03-06.
+ data[1] = 0x01;
+ TestReadAvroDate(data, 2, DateValue(1970, 3, 6), 2);
+ TestReadAvroDate(data, 10, DateValue(1970, 3, 6), 2);
+ TestReadAvroDate(data, 0, invalid_dv, -1, TErrorCode::SCANNER_INVALID_INT);
+ TestReadAvroDate(data, 1, invalid_dv, -1, TErrorCode::SCANNER_INVALID_INT);
+
+ // Test upper limit: 9999-12-31.
+ const int32_t MAX_DATE_DAYS_SINCE_EPOCH = 2932896;
+ int len = ReadWriteUtil::PutZInt(MAX_DATE_DAYS_SINCE_EPOCH, data);
+ TestReadAvroDate(data, len, DateValue(9999, 12 ,31), len);
+ TestReadAvroDate(data, len + 1, DateValue(9999, 12, 31), len);
+ TestReadAvroDate(data, len - 1, invalid_dv, -1, TErrorCode::SCANNER_INVALID_INT);
+
+ len = ReadWriteUtil::PutZInt(MAX_DATE_DAYS_SINCE_EPOCH + 1, data);
+ TestReadAvroDate(data, len, invalid_dv, -1, TErrorCode::AVRO_INVALID_DATE);
+ TestReadAvroDate(data, len + 1, invalid_dv, -1, TErrorCode::AVRO_INVALID_DATE);
+ TestReadAvroDate(data, len - 1, invalid_dv, -1, TErrorCode::SCANNER_INVALID_INT);
+
+ len = ReadWriteUtil::PutZInt(INT_MAX, data);
+ TestReadAvroDate(data, len, invalid_dv, -1, TErrorCode::AVRO_INVALID_DATE);
+ TestReadAvroDate(data, len + 1, invalid_dv, -1, TErrorCode::AVRO_INVALID_DATE);
+ TestReadAvroDate(data, len - 1, invalid_dv, -1, TErrorCode::SCANNER_INVALID_INT);
+
+ // Test lower limit: 0000-01-01.
+ const int32_t MIN_DATE_DAYS_SINCE_EPOCH = -719528;
+ len = ReadWriteUtil::PutZInt(MIN_DATE_DAYS_SINCE_EPOCH, data);
+ TestReadAvroDate(data, len, DateValue(0, 1, 1), len);
+ TestReadAvroDate(data, len + 1, DateValue(0, 1, 1), len);
+ TestReadAvroDate(data, len - 1, invalid_dv, -1, TErrorCode::SCANNER_INVALID_INT);
+
+ len = ReadWriteUtil::PutZInt(MIN_DATE_DAYS_SINCE_EPOCH - 1, data);
+ TestReadAvroDate(data, len, invalid_dv, -1, TErrorCode::AVRO_INVALID_DATE);
+ TestReadAvroDate(data, len + 1, invalid_dv, -1, TErrorCode::AVRO_INVALID_DATE);
+ TestReadAvroDate(data, len - 1, invalid_dv, -1, TErrorCode::SCANNER_INVALID_INT);
+
+ len = ReadWriteUtil::PutZInt(INT_MIN, data);
+ TestReadAvroDate(data, len, invalid_dv, -1, TErrorCode::AVRO_INVALID_DATE);
+ TestReadAvroDate(data, len + 1, invalid_dv, -1, TErrorCode::AVRO_INVALID_DATE);
+ TestReadAvroDate(data, len - 1, invalid_dv, -1, TErrorCode::SCANNER_INVALID_INT);
+}
+
TEST_F(HdfsAvroScannerTest, Int64Test) {
uint8_t data[100];
data[0] = 1; // decodes to -1
diff --git a/be/src/exec/hdfs-avro-scanner.cc b/be/src/exec/hdfs-avro-scanner.cc
index 967227c..8c82675 100644
--- a/be/src/exec/hdfs-avro-scanner.cc
+++ b/be/src/exec/hdfs-avro-scanner.cc
@@ -308,7 +308,8 @@ Status HdfsAvroScanner::WriteDefaultValue(
RawValue::Write(&v, avro_header_->template_tuple, slot_desc, nullptr);
break;
}
- case AVRO_INT32: {
+ case AVRO_INT32:
+ case AVRO_DATE: {
RETURN_IF_ERROR(VerifyTypesMatch(slot_desc, default_value));
int32_t v;
if (avro_int32_get(default_value, &v)) DCHECK(false);
@@ -431,8 +432,10 @@ bool HdfsAvroScanner::VerifyTypesMatch(
return true;
case TYPE_STRING: return reader_type.IsStringType();
case TYPE_INT:
+ case TYPE_DATE:
switch(reader_type.type) {
case TYPE_INT:
+ case TYPE_DATE:
// Type promotion
case TYPE_BIGINT:
case TYPE_FLOAT:
@@ -617,8 +620,13 @@ bool HdfsAvroScanner::MaterializeTuple(const AvroSchemaElement& record_schema,
case AVRO_BOOLEAN:
success = ReadAvroBoolean(slot_type, data, data_end, write_slot, slot, pool);
break;
+ case AVRO_DATE:
case AVRO_INT32:
- success = ReadAvroInt32(slot_type, data, data_end, write_slot, slot, pool);
+ if (slot_type == TYPE_DATE) {
+ success = ReadAvroDate(slot_type, data, data_end, write_slot, slot, pool);
+ } else {
+ success = ReadAvroInt32(slot_type, data, data_end, write_slot, slot, pool);
+ }
break;
case AVRO_INT64:
success = ReadAvroInt64(slot_type, data, data_end, write_slot, slot, pool);
@@ -689,7 +697,8 @@ void HdfsAvroScanner::SetStatusValueOverflow(TErrorCode::type error_code, int64_
if (TestInfo::is_test()) {
parse_status_ = Status(error_code, "test file", len, limit, 123);
} else {
- parse_status_ = Status(error_code, stream_->filename(), len, limit, stream_->file_offset());
+ parse_status_ = Status(error_code, stream_->filename(), len, limit,
+ stream_->file_offset());
}
}
@@ -1012,8 +1021,19 @@ Status HdfsAvroScanner::CodegenReadScalar(const AvroSchemaElement& element,
case AVRO_BOOLEAN:
read_field_fn = codegen->GetFunction(IRFunction::READ_AVRO_BOOLEAN, false);
break;
+ case AVRO_DATE:
+ if (slot_desc != nullptr && slot_desc->type().type == TYPE_INT) {
+ read_field_fn = codegen->GetFunction(IRFunction::READ_AVRO_INT32, false);
+ } else {
+ read_field_fn = codegen->GetFunction(IRFunction::READ_AVRO_DATE, false);
+ }
+ break;
case AVRO_INT32:
- read_field_fn = codegen->GetFunction(IRFunction::READ_AVRO_INT32, false);
+ if (slot_desc != nullptr && slot_desc->type().type == TYPE_DATE) {
+ read_field_fn = codegen->GetFunction(IRFunction::READ_AVRO_DATE, false);
+ } else {
+ read_field_fn = codegen->GetFunction(IRFunction::READ_AVRO_INT32, false);
+ }
break;
case AVRO_INT64:
read_field_fn = codegen->GetFunction(IRFunction::READ_AVRO_INT64, false);
diff --git a/be/src/exec/hdfs-avro-scanner.h b/be/src/exec/hdfs-avro-scanner.h
index f7b1c31..f88adcf 100644
--- a/be/src/exec/hdfs-avro-scanner.h
+++ b/be/src/exec/hdfs-avro-scanner.h
@@ -267,6 +267,8 @@ class HdfsAvroScanner : public BaseSequenceScanner {
///
bool ReadAvroBoolean(PrimitiveType type, uint8_t** data, uint8_t* data_end,
bool write_slot, void* slot, MemPool* pool);
+ bool ReadAvroDate(PrimitiveType type, uint8_t** data, uint8_t* data_end,
+ bool write_slot, void* slot, MemPool* pool);
bool ReadAvroInt32(PrimitiveType type, uint8_t** data, uint8_t* data_end,
bool write_slot, void* slot, MemPool* pool);
bool ReadAvroInt64(PrimitiveType type, uint8_t** data, uint8_t* data_end,
diff --git a/be/src/util/avro-util.cc b/be/src/util/avro-util.cc
index f03242c..5b4d6e0 100644
--- a/be/src/util/avro-util.cc
+++ b/be/src/util/avro-util.cc
@@ -31,6 +31,7 @@ bool IsSupportedAvroType(const avro_schema_t& schema) {
case AVRO_STRING:
case AVRO_BYTES:
case AVRO_INT32:
+ case AVRO_DATE:
case AVRO_INT64:
case AVRO_FLOAT:
case AVRO_DOUBLE:
@@ -113,6 +114,9 @@ Status AvroSchemaToColumnType(
case AVRO_INT32:
*column_type = TYPE_INT;
return Status::OK();
+ case AVRO_DATE:
+ *column_type = TYPE_DATE;
+ return Status::OK();
case AVRO_INT64:
*column_type = TYPE_BIGINT;
return Status::OK();
diff --git a/common/thrift/generate_error_codes.py b/common/thrift/generate_error_codes.py
index 8dd5b06..5a73e9c 100755
--- a/common/thrift/generate_error_codes.py
+++ b/common/thrift/generate_error_codes.py
@@ -435,6 +435,9 @@ error_codes = (
("MAX_STATEMENT_LENGTH_EXCEEDED", 143, "Statement length of $0 bytes exceeds the "
"maximum statement length ($1 bytes)"),
+
+ ("AVRO_INVALID_DATE", 144, "Avro file '$0' is corrupt: out of range date value $1 "
+ "at offset $2. The valid date range is -719528..2932896 (0000-01-01..9999-12-31)."),
)
import sys
diff --git a/fe/src/main/java/org/apache/impala/catalog/HdfsFileFormat.java b/fe/src/main/java/org/apache/impala/catalog/HdfsFileFormat.java
index 2cdbc30..47d9773 100644
--- a/fe/src/main/java/org/apache/impala/catalog/HdfsFileFormat.java
+++ b/fe/src/main/java/org/apache/impala/catalog/HdfsFileFormat.java
@@ -58,7 +58,7 @@ public enum HdfsFileFormat {
AVRO("org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat",
"org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat",
"org.apache.hadoop.hive.serde2.avro.AvroSerDe",
- false, false, false),
+ false, false, true),
PARQUET("org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat",
"org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat",
"org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe",
diff --git a/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java b/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java
index 3265daf..3a3dba3 100644
--- a/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java
+++ b/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java
@@ -1340,7 +1340,7 @@ public class SingleNodePlanner {
/**
* Looks for a filesystem-based partition in 'partitions' with no DATE support and
* returns the first one it finds. Right now, scanning DATE values is only supported for
- * TEXT and PARQUET fileformats.
+ * TEXT, PARQUET and AVRO fileformats.
*
* Returns null otherwise.
*/
diff --git a/fe/src/main/java/org/apache/impala/util/AvroSchemaConverter.java b/fe/src/main/java/org/apache/impala/util/AvroSchemaConverter.java
index 084b789..773c3f1 100644
--- a/fe/src/main/java/org/apache/impala/util/AvroSchemaConverter.java
+++ b/fe/src/main/java/org/apache/impala/util/AvroSchemaConverter.java
@@ -54,11 +54,12 @@ public class AvroSchemaConverter {
private static final String DEFAULT_SCHEMA_NAME = "baseRecord";
private static final String RECORD_NAME_PREFIX = "record_";
- // Constants for Avro logical types, in particular, for DECIMAL.
+ // Constants for Avro logical types, in particular, for DECIMAL and DATE.
private static final String AVRO_LOGICAL_TYPE = "logicalType";
private static final String PRECISION_PROP_NAME = "precision";
private static final String SCALE_PROP_NAME = "scale";
private static final String AVRO_DECIMAL_TYPE = "decimal";
+ private static final String AVRO_DATE_TYPE = "date";
// Used to generate unique record names as required by Avro.
private int recordCounter_ = 0;
@@ -168,7 +169,7 @@ public class AvroSchemaConverter {
private Schema createDateSchema() {
Schema dateSchema = Schema.create(Schema.Type.INT);
- dateSchema.addProp(AVRO_LOGICAL_TYPE, "date");
+ dateSchema.addProp(AVRO_LOGICAL_TYPE, AVRO_DATE_TYPE);
return dateSchema;
}
diff --git a/fe/src/main/java/org/apache/impala/util/AvroSchemaParser.java b/fe/src/main/java/org/apache/impala/util/AvroSchemaParser.java
index 8129dff..92992f0 100644
--- a/fe/src/main/java/org/apache/impala/util/AvroSchemaParser.java
+++ b/fe/src/main/java/org/apache/impala/util/AvroSchemaParser.java
@@ -108,6 +108,7 @@ public class AvroSchemaParser {
Schema.Type type = schema.getType();
if (type == INT) {
String logicalType = schema.getProp("logicalType");
+ // DATE is stored in Avro as INT.
if (logicalType != null && logicalType.equalsIgnoreCase("date")) return Type.DATE;
}
if (avroToImpalaPrimitiveTypeMap_.containsKey(type)) {
diff --git a/fe/src/test/java/org/apache/impala/catalog/CatalogObjectToFromThriftTest.java b/fe/src/test/java/org/apache/impala/catalog/CatalogObjectToFromThriftTest.java
index 995f3ba..e6c47b6 100644
--- a/fe/src/test/java/org/apache/impala/catalog/CatalogObjectToFromThriftTest.java
+++ b/fe/src/test/java/org/apache/impala/catalog/CatalogObjectToFromThriftTest.java
@@ -135,14 +135,14 @@ public class CatalogObjectToFromThriftTest {
TTable thriftTable = getThriftTable(table);
Assert.assertEquals(thriftTable.tbl_name, "schema_resolution_test");
Assert.assertTrue(thriftTable.isSetTable_type());
- Assert.assertEquals(thriftTable.getColumns().size(), 8);
+ Assert.assertEquals(thriftTable.getColumns().size(), 9);
Assert.assertEquals(thriftTable.getClustering_columns().size(), 0);
Assert.assertEquals(thriftTable.getTable_type(), TTableType.HDFS_TABLE);
// Now try to load the thrift struct.
Table newTable = Table.fromThrift(catalog_.getDb("functional_avro_snap"),
thriftTable);
- Assert.assertEquals(newTable.getColumns().size(), 8);
+ Assert.assertEquals(newTable.getColumns().size(), 9);
// The table schema does not match the Avro schema - it has only 2 columns.
Assert.assertEquals(newTable.getMetaStoreTable().getSd().getCols().size(), 2);
diff --git a/testdata/avro_schema_resolution/create_table.sql b/testdata/avro_schema_resolution/create_table.sql
index b1425e1..da4e67b 100644
--- a/testdata/avro_schema_resolution/create_table.sql
+++ b/testdata/avro_schema_resolution/create_table.sql
@@ -36,6 +36,7 @@ WITH SERDEPROPERTIES ('avro.schema.literal'='{
{"name":"long1", "type":"long", "default": 1},
{"name":"float1", "type":"float", "default": 1.0},
{"name":"double1", "type":"double", "default": 1.0},
+ {"name":"date1", "type": {"type": "int", "logicalType": "date"}, "default": 1},
{"name":"string1", "type":"string", "default": "default string"},
{"name":"string2", "type": ["string", "null"], "default": ""},
{"name":"string3", "type": ["null", "string"], "default": null}
diff --git a/testdata/avro_schema_resolution/file_schema2.avsc b/testdata/avro_schema_resolution/file_schema2.avsc
index 63db949..877a324 100644
--- a/testdata/avro_schema_resolution/file_schema2.avsc
+++ b/testdata/avro_schema_resolution/file_schema2.avsc
@@ -4,6 +4,7 @@
"fields": [
{"name":"boolean2", "type":"boolean"},
{"name":"string1", "type":"string"},
+ {"name":"date1", "type":"int", "comment": "type promotion"},
{"name":"double1", "type":"int", "comment": "type promotion"},
{"name":"float1", "type":"int", "comment": "type promotion"},
{"name":"long1", "type":"int", "comment": "type promotion"},
diff --git a/testdata/avro_schema_resolution/records2.avro b/testdata/avro_schema_resolution/records2.avro
index 7617d6b..eac79fa 100644
Binary files a/testdata/avro_schema_resolution/records2.avro and b/testdata/avro_schema_resolution/records2.avro differ
diff --git a/testdata/avro_schema_resolution/records2.json b/testdata/avro_schema_resolution/records2.json
index 0625f20..41e39a6 100644
--- a/testdata/avro_schema_resolution/records2.json
+++ b/testdata/avro_schema_resolution/records2.json
@@ -1,5 +1,6 @@
{"boolean2": false,
"string1": "serialized string",
+ "date1": 2,
"double1": 2,
"float1": 2,
"long1": 2,
diff --git a/testdata/bad_avro_snap/README b/testdata/bad_avro_snap/README
index 71eb398..f7e55ec 100644
--- a/testdata/bad_avro_snap/README
+++ b/testdata/bad_avro_snap/README
@@ -26,3 +26,19 @@ Created by editing the schema of a valid Avro file with vim.
invalid_decimal_schema.avro: two columns, name STRING and value DECIMAL(5,7).
The DECIMAL value is invalid.
+
+Date Data
+---------
+out_of_range_date.avro:
+Small avro table with a single DATE column. Contains 5 values, two of which are out of
+range.
+
+hive2_pre_gregorian_date.avro:
+Small avro table with one DATE column, created by Hive 2.1.1.
+Used to demonstrate avro interoperability issues between Hive and Impala for dates
+before the introduction of Gregorian calendar in 1582-10-15.
+
+hive3_pre_gregorian_date.avro:
+Small avro table with one DATE column, created by Hive 3.1.0.
+Used to demonstrate that pre-Gregorian avro dates written by Hive3 are read back correctly
+by Impala.
diff --git a/testdata/bad_avro_snap/hive2_pre_gregorian_date.avro b/testdata/bad_avro_snap/hive2_pre_gregorian_date.avro
new file mode 100644
index 0000000..9880ceb
Binary files /dev/null and b/testdata/bad_avro_snap/hive2_pre_gregorian_date.avro differ
diff --git a/testdata/bad_avro_snap/hive3_pre_gregorian_date.avro b/testdata/bad_avro_snap/hive3_pre_gregorian_date.avro
new file mode 100644
index 0000000..87ed996
Binary files /dev/null and b/testdata/bad_avro_snap/hive3_pre_gregorian_date.avro differ
diff --git a/testdata/bad_avro_snap/out_of_range_date.avro b/testdata/bad_avro_snap/out_of_range_date.avro
new file mode 100644
index 0000000..87ae277
Binary files /dev/null and b/testdata/bad_avro_snap/out_of_range_date.avro differ
diff --git a/testdata/bin/generate-schema-statements.py b/testdata/bin/generate-schema-statements.py
index f4905ae..64f8cd5 100755
--- a/testdata/bin/generate-schema-statements.py
+++ b/testdata/bin/generate-schema-statements.py
@@ -404,6 +404,8 @@ def avro_schema(columns):
hive_type = column_spec.split()[1].upper()
if hive_type.startswith('CHAR(') or hive_type.startswith('VARCHAR('):
type = 'string'
+ elif hive_type == 'DATE':
+ type = {"type": "int", "logicalType": "date"}
else:
type = HIVE_TO_AVRO_TYPE_MAP[hive_type]
diff --git a/testdata/data/README b/testdata/data/README
index 4e5f92c..8f946d0 100644
--- a/testdata/data/README
+++ b/testdata/data/README
@@ -298,11 +298,14 @@ decimal_rtf_tiny_tbl.txt:
Small table with specific decimal values picked from decimal_rtf_tbl.txt so that
min-max filter based pruning can be tested with decimal types on Kudu.
+date_tbl.orc
+Small orc table with one DATE column, created by Hive.
+
date_tbl.avro
-Small table with one DATE column, created by Hive.
+Small avro table with one DATE column, created by Hive.
date_tbl.parquet
-Small table with one DATE column, created by Parquet MR.
+Small parquet table with one DATE column, created by Parquet MR.
out_of_range_date.parquet:
Generated with a hacked version of Impala parquet writer.
@@ -319,7 +322,7 @@ and should be read as NULL by Impala:
10000-01-01 (invalid - date too large)
hive2_pre_gregorian.parquet:
-Small table with one DATE column, created by Hive 2.1.1.
+Small parquet table with one DATE column, created by Hive 2.1.1.
Used to demonstrate parquet interoperability issues between Hive and Impala for dates
before the introduction of Gregorian calendar in 1582-10-15.
diff --git a/testdata/data/date_tbl.avro b/testdata/data/date_tbl.avro
index 62ae117..a5b30de 100644
Binary files a/testdata/data/date_tbl.avro and b/testdata/data/date_tbl.avro differ
diff --git a/testdata/data/date_tbl.orc b/testdata/data/date_tbl.orc
new file mode 100644
index 0000000..c0f9243
Binary files /dev/null and b/testdata/data/date_tbl.orc differ
diff --git a/testdata/datasets/functional/functional_schema_template.sql b/testdata/datasets/functional/functional_schema_template.sql
index d460eaa..c98270e 100644
--- a/testdata/datasets/functional/functional_schema_template.sql
+++ b/testdata/datasets/functional/functional_schema_template.sql
@@ -1605,6 +1605,36 @@ LOAD DATA LOCAL INPATH '{impala_home}/testdata/bad_avro_snap/invalid_decimal_sch
OVERWRITE INTO TABLE {db_name}{db_suffix}.{table_name};
====
---- DATASET
+functional
+---- BASE_TABLE_NAME
+bad_avro_date_out_of_range
+---- COLUMNS
+d DATE
+---- DEPENDENT_LOAD_HIVE
+LOAD DATA LOCAL INPATH '{impala_home}/testdata/bad_avro_snap/out_of_range_date.avro'
+OVERWRITE INTO TABLE {db_name}{db_suffix}.{table_name};
+====
+---- DATASET
+functional
+---- BASE_TABLE_NAME
+hive2_bad_avro_date_pre_gregorian
+---- COLUMNS
+d DATE
+---- DEPENDENT_LOAD_HIVE
+LOAD DATA LOCAL INPATH '{impala_home}/testdata/bad_avro_snap/hive2_pre_gregorian_date.avro'
+OVERWRITE INTO TABLE {db_name}{db_suffix}.{table_name};
+====
+---- DATASET
+functional
+---- BASE_TABLE_NAME
+hive3_avro_date_pre_gregorian
+---- COLUMNS
+d DATE
+---- DEPENDENT_LOAD_HIVE
+LOAD DATA LOCAL INPATH '{impala_home}/testdata/bad_avro_snap/hive3_pre_gregorian_date.avro'
+OVERWRITE INTO TABLE {db_name}{db_suffix}.{table_name};
+====
+---- DATASET
-- IMPALA-694: uses data file produced by parquet-mr version 1.2.5-cdh4.5.0
-- (can't use LOAD DATA LOCAL with Impala so copied in create-load-data.sh)
functional
diff --git a/testdata/datasets/functional/schema_constraints.csv b/testdata/datasets/functional/schema_constraints.csv
index 75185f5..ab778a3 100644
--- a/testdata/datasets/functional/schema_constraints.csv
+++ b/testdata/datasets/functional/schema_constraints.csv
@@ -39,6 +39,9 @@ table_name:bad_seq_snap, constraint:restrict_to, table_format:seq/snap/block
table_name:bad_avro_snap_strings, constraint:restrict_to, table_format:avro/snap/block
table_name:bad_avro_snap_floats, constraint:restrict_to, table_format:avro/snap/block
table_name:bad_avro_decimal_schema, constraint:restrict_to, table_format:avro/snap/block
+table_name:bad_avro_date_out_of_range, constraint:restrict_to, table_format:avro/snap/block
+table_name:hive2_bad_avro_date_pre_gregorian, constraint:restrict_to, table_format:avro/snap/block
+table_name:hive3_avro_date_pre_gregorian, constraint:restrict_to, table_format:avro/snap/block
table_name:bad_parquet, constraint:restrict_to, table_format:parquet/none/none
table_name:bad_parquet_strings_negative_len, constraint:restrict_to, table_format:parquet/none/none
table_name:bad_parquet_strings_out_of_bounds, constraint:restrict_to, table_format:parquet/none/none
@@ -226,9 +229,10 @@ table_name:table_with_header_insert, constraint:restrict_to, table_format:text/g
# property, so we test parquet format as well.
table_name:table_with_header_insert, constraint:restrict_to, table_format:parquet/none/none
-# IMPALA-7368/IMPALA-7370 adds DATE support for text, hbase and parquet.
+# IMPALA-7368/IMPALA-7370/IMPALA-8198 adds DATE support for text, hbase, parquet and avro.
# Other file-formats will be introduced later.
table_name:date_tbl, constraint:restrict_to, table_format:parquet/none/none
+table_name:date_tbl, constraint:restrict_to, table_format:avro/snap/block
table_name:date_tbl, constraint:restrict_to, table_format:hbase/none/none
table_name:date_tbl, constraint:restrict_to, table_format:text/none/none
table_name:date_tbl, constraint:restrict_to, table_format:text/lzo/block
diff --git a/testdata/workloads/functional-query/queries/DataErrorsTest/avro-errors.test b/testdata/workloads/functional-query/queries/DataErrorsTest/avro-errors.test
index 3f62b2e..8eed333 100644
--- a/testdata/workloads/functional-query/queries/DataErrorsTest/avro-errors.test
+++ b/testdata/workloads/functional-query/queries/DataErrorsTest/avro-errors.test
@@ -31,3 +31,46 @@ string,decimal
---- ERRORS
Column 'value': invalid Avro decimal type with precision = '5' scale = '7'
====
+---- QUERY
+# Out of range DATE values in an avro table.
+# Note that when the avro scanner encounters an out of range DATE error a warning is
+# emitted and no result rows are returned. This behavior differs from how the parquet
+# scanner works but it is consistent with how avro scanner handles similar errors.
+select * from bad_avro_date_out_of_range;
+---- RESULTS
+---- TYPES
+DATE
+---- ERRORS
+Problem parsing file $NAMENODE/test-warehouse/bad_avro_date_out_of_range_avro_snap/out_of_range_date.avro at 204
+Avro file '$NAMENODE/test-warehouse/bad_avro_date_out_of_range_avro_snap/out_of_range_date.avro' is corrupt: out of range date value -719530 at offset 204. The valid date range is -719528..2932896 (0000-01-01..9999-12-31).
+====
+---- QUERY
+# Query an avro table created by Hive 2.1.1 containig the following dates:
+# 1582-10-04, 1582-10-05, 1582-10-06, 1582-10-15, 1582-10-16.
+# Impala will incorrectly read back the dates that precede the introduction of Gregorian
+# calendar (1582-10-15).
+select * from hive2_bad_avro_date_pre_gregorian;
+---- TYPES
+DATE
+---- RESULTS
+1582-10-14
+1582-10-15
+1582-10-16
+1582-10-15
+1582-10-16
+====
+---- QUERY
+# Query an avro table created by Hive 3.1.0 containig the following dates:
+# 1582-10-04, 1582-10-05, 1582-10-06, 1582-10-15, 1582-10-16.
+# Impala will correctly read back all the dates, including those that precede the
+# introduction of Gregorian calendar (1582-10-15).
+select * from hive3_avro_date_pre_gregorian;
+---- TYPES
+DATE
+---- RESULTS
+1582-10-04
+1582-10-05
+1582-10-06
+1582-10-15
+1582-10-16
+====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/avro-schema-changes.test b/testdata/workloads/functional-query/queries/QueryTest/avro-schema-changes.test
index 14f0549..1575246 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/avro-schema-changes.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/avro-schema-changes.test
@@ -93,17 +93,18 @@ float_col float,
double_col double,
timestamp_col timestamp,
decimal_col decimal(2,0),
-string_col string)
+string_col string,
+date_col date)
====
---- QUERY
# Every new column just added should have NULL filled
select * from avro_alter_table_add_new_column
---- RESULTS
-'aaaaaaa','bbbbbbb',NULL,NULL,NULL,NULL,NULL,'NULL',NULL,'NULL'
-'ccccc','dddd',NULL,NULL,NULL,NULL,NULL,'NULL',NULL,'NULL'
-'eeeeeeee','f',NULL,NULL,NULL,NULL,NULL,'NULL',NULL,'NULL'
+'aaaaaaa','bbbbbbb',NULL,NULL,NULL,NULL,NULL,'NULL',NULL,'NULL',NULL
+'ccccc','dddd',NULL,NULL,NULL,NULL,NULL,'NULL',NULL,'NULL',NULL
+'eeeeeeee','f',NULL,NULL,NULL,NULL,NULL,'NULL',NULL,'NULL',NULL
---- TYPES
-string, string, boolean, int, bigint, float, double, string, decimal, string
+string, string, boolean, int, bigint, float, double, string, decimal, string, date
====
---- QUERY
# IMPALA-3776: Create an Avro table, add a column to the Avro schema and make sure
diff --git a/testdata/workloads/functional-query/queries/QueryTest/avro-schema-resolution.test b/testdata/workloads/functional-query/queries/QueryTest/avro-schema-resolution.test
index c640fa6..726a570 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/avro-schema-resolution.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/avro-schema-resolution.test
@@ -3,10 +3,10 @@
# see testdata/avro_schema_resolution
select * from functional_avro_snap.schema_resolution_test
---- TYPES
-boolean, int, long, float, double, string, string, string
+boolean, int, long, float, double, date, string, string, string
---- RESULTS
-true,1,1,1,1,'default string','','NULL'
-false,2,2,2,2,'serialized string','','NULL'
+true,1,1,1,1,1970-01-02,'default string','','NULL'
+false,2,2,2,2,1970-01-03,'serialized string','','NULL'
====
---- QUERY
# IMPALA-1136: Tests that Impala can read Hive-created Avro tables that have
diff --git a/testdata/workloads/functional-query/queries/QueryTest/avro_date.test b/testdata/workloads/functional-query/queries/QueryTest/avro_date.test
new file mode 100644
index 0000000..64cef54
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/avro_date.test
@@ -0,0 +1,125 @@
+====
+---- HIVE_MAJOR_VERSION
+2
+---- QUERY
+# TODO: Once DATE type is supported across all fileformats, move this test to
+# hdfs-scan-node.test.
+select id_col, date_part, date_col from date_tbl;
+---- RESULTS
+0,0001-01-01,0000-12-30
+1,0001-01-01,0001-12-29
+2,0001-01-01,0001-12-30
+3,0001-01-01,1400-01-08
+4,0001-01-01,2017-11-28
+5,0001-01-01,9999-12-31
+6,0001-01-01,NULL
+10,1399-06-27,2017-11-28
+11,1399-06-27,NULL
+12,1399-06-27,2018-12-31
+20,2017-11-27,0001-06-19
+21,2017-11-27,0001-06-20
+22,2017-11-27,0001-06-21
+23,2017-11-27,0001-06-22
+24,2017-11-27,0001-06-23
+25,2017-11-27,0001-06-24
+26,2017-11-27,0001-06-25
+27,2017-11-27,0001-06-26
+28,2017-11-27,0001-06-27
+29,2017-11-27,2017-11-28
+30,9999-12-31,9999-12-01
+31,9999-12-31,9999-12-31
+---- TYPES
+INT,DATE,DATE
+====
+---- HIVE_MAJOR_VERSION
+3
+---- QUERY
+select id_col, date_part, date_col from date_tbl;
+---- RESULTS
+0,0001-01-01,0001-01-01
+1,0001-01-01,0001-12-31
+2,0001-01-01,0002-01-01
+3,0001-01-01,1399-12-31
+4,0001-01-01,2017-11-28
+5,0001-01-01,9999-12-31
+6,0001-01-01,NULL
+10,1399-06-27,2017-11-28
+11,1399-06-27,NULL
+12,1399-06-27,2018-12-31
+20,2017-11-27,0001-06-21
+21,2017-11-27,0001-06-22
+22,2017-11-27,0001-06-23
+23,2017-11-27,0001-06-24
+24,2017-11-27,0001-06-25
+25,2017-11-27,0001-06-26
+26,2017-11-27,0001-06-27
+27,2017-11-27,0001-06-28
+28,2017-11-27,0001-06-29
+29,2017-11-27,2017-11-28
+30,9999-12-31,9999-12-01
+31,9999-12-31,9999-12-31
+---- TYPES
+INT,DATE,DATE
+====
+---- QUERY
+select date_part, count(date_col) from date_tbl group by date_part;
+---- RESULTS
+2017-11-27,10
+1399-06-27,2
+9999-12-31,2
+0001-01-01,6
+---- TYPES
+DATE, BIGINT
+====
+---- HIVE_MAJOR_VERSION
+2
+---- QUERY
+select min(date_part), max(date_part), min(date_col), max(date_col) from date_tbl;
+---- RESULTS
+0001-01-01,9999-12-31,0000-12-30,9999-12-31
+---- TYPES
+DATE, DATE, DATE, DATE
+====
+---- HIVE_MAJOR_VERSION
+3
+---- QUERY
+select min(date_part), max(date_part), min(date_col), max(date_col) from date_tbl;
+---- RESULTS
+0001-01-01,9999-12-31,0001-01-01,9999-12-31
+---- TYPES
+DATE, DATE, DATE, DATE
+====
+---- HIVE_MAJOR_VERSION
+2
+---- QUERY
+select date_part, min(date_col), max(date_col) from date_tbl group by date_part;
+---- RESULTS
+2017-11-27,0001-06-19,2017-11-28
+1399-06-27,2017-11-28,2018-12-31
+9999-12-31,9999-12-01,9999-12-31
+0001-01-01,0000-12-30,9999-12-31
+---- TYPES
+DATE, DATE, DATE
+====
+---- HIVE_MAJOR_VERSION
+3
+---- QUERY
+select date_part, min(date_col), max(date_col) from date_tbl group by date_part;
+---- RESULTS
+2017-11-27,0001-06-21,2017-11-28
+1399-06-27,2017-11-28,2018-12-31
+9999-12-31,9999-12-01,9999-12-31
+0001-01-01,0001-01-01,9999-12-31
+---- TYPES
+DATE, DATE, DATE
+====
+---- QUERY
+select date_part, count(*) from date_tbl group by date_part;
+---- RESULTS
+2017-11-27,10
+1399-06-27,3
+9999-12-31,2
+0001-01-01,7
+---- TYPES
+DATE, BIGINT
+====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/date-fileformat-support.test b/testdata/workloads/functional-query/queries/QueryTest/date-fileformat-support.test
index 4793a44..a8a6778 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/date-fileformat-support.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/date-fileformat-support.test
@@ -20,13 +20,24 @@ select * from $DATABASE.parquet_date_tbl;
DATE
====
---- QUERY
-# Querying avro table is not supported
+# Querying avro table is supported.
select * from $DATABASE.avro_date_tbl;
+---- RESULTS
+1970-01-01
+1970-01-02
+1224-05-19
+8543-11-21
+---- TYPES
+DATE
+====
+---- QUERY
+# Querying orc table is not supported.
+select * from $DATABASE.orc_date_tbl;
---- CATCH
-NotImplementedException: Scanning DATE values in table '$DATABASE.avro_date_tbl' is not supported for fileformat AVRO
+NotImplementedException: Scanning DATE values in table '$DATABASE.orc_date_tbl' is not supported for fileformat ORC
====
---- QUERY
-# Inserting text partitions to $DATABASE.date_tbl is OK
+# Inserting text partitions to $DATABASE.date_tbl is OK.
insert into $DATABASE.date_tbl partition (date_part)
select date_col, date_part from functional.date_tbl;
---- RESULTS
@@ -36,23 +47,27 @@ date_part=2017-11-27/: 10
date_part=9999-12-31/: 2
====
---- QUERY
-# Inserting into parquet partition is supported
+# Inserting into parquet partition is supported.
insert into $DATABASE.date_tbl partition(date_part='1899-12-31')
select date_col from functional_parquet.date_tbl where date_part = '1399-06-27';
---- RESULTS
date_part=1899-12-31/: 3
====
---- QUERY
-# Querying all the partitions fails because of the one avro partition
+# Adding orc partition works even though Impala cannot scan/write DATE values in orc.
+# Querying all the partitions fails because of the one orc partition.
+alter table $DATABASE.date_tbl add partition (date_part='2099-12-31')
+location '$NAMENODE/test-warehouse/$DATABASE.db/orc_date_tbl';
+alter table $DATABASE.date_tbl partition (date_part='2099-12-31') set fileformat orc;
select * from $DATABASE.date_tbl;
---- CATCH
-NotImplementedException: Scanning DATE values in table '$DATABASE.date_tbl' is not supported for fileformat AVRO
+NotImplementedException: Scanning DATE values in table '$DATABASE.date_tbl' is not supported for fileformat ORC
====
---- QUERY
-# Querying text and parquet partitions is OK
+# Querying text, parquet and avro partitions is OK.
select date_part, date_col
from $DATABASE.date_tbl
-where date_part != '1999-12-31';
+where date_part != '2099-12-31';
---- RESULTS
0001-01-01,0001-01-01
0001-01-01,0001-12-31
@@ -85,12 +100,16 @@ where date_part != '1999-12-31';
1899-12-31,2017-11-28
1899-12-31,NULL
1899-12-31,2018-12-31
+1999-12-31,1970-01-01
+1999-12-31,1970-01-02
+1999-12-31,1224-05-19
+1999-12-31,8543-11-21
---- TYPES
DATE,DATE
====
---- QUERY
-# Querying avro partition fails.
-select date_part, date_col from $DATABASE.date_tbl where date_part='1999-12-31';
+# Querying the orc partition separately fails.
+select date_part, date_col from $DATABASE.date_tbl where date_part='2099-12-31';
---- CATCH
-NotImplementedException: Scanning DATE values in table '$DATABASE.date_tbl' is not supported for fileformat AVRO
+NotImplementedException: Scanning DATE values in table '$DATABASE.date_tbl' is not supported for fileformat ORC
====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/date-partitioning.test b/testdata/workloads/functional-query/queries/QueryTest/date-partitioning.test
index 1be81f7..3574281 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/date-partitioning.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/date-partitioning.test
@@ -121,34 +121,3 @@ select p, c from $DATABASE.dtbl;
---- TYPES
DATE,DATE
====
----- QUERY
-# Create a parquet table partitioned by DATE. It has no non-partitioning DATE columns.
-create table $DATABASE.ptbl (v int, s string) partitioned by (p date) stored as parquet;
-insert into $DATABASE.ptbl partition (p='0000-01-01') values (0, 'zero'), (1, 'one'), (2, 'two');
----- RESULTS
-p=0000-01-01/: 3
-====
----- QUERY
-insert into $DATABASE.ptbl partition (p='1400-6-25') values (3, 'three'), (4, 'four');
----- RESULTS
-p=1400-06-25/: 2
-====
----- QUERY
-insert into $DATABASE.ptbl partition (p='9999-11-1') values (5, 'five'), (6, 'six');
----- RESULTS
-p=9999-11-01/: 2
-====
----- QUERY
-# Test that querying a parquet table works if it has no non-partitioning DATE columns.
-select p, v, s from $DATABASE.ptbl;
----- RESULTS
-0000-01-01,0,'zero'
-0000-01-01,1,'one'
-0000-01-01,2,'two'
-1400-06-25,3,'three'
-1400-06-25,4,'four'
-9999-11-01,5,'five'
-9999-11-01,6,'six'
----- TYPES
-DATE,INT,STRING
-====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/hive2-pre-gregorian-date.test b/testdata/workloads/functional-query/queries/QueryTest/hive2-pre-gregorian-date.test
index ed9d1ff..97b6031 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/hive2-pre-gregorian-date.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/hive2-pre-gregorian-date.test
@@ -1,6 +1,6 @@
====
---- QUERY
-# Query a table created by Hive 2.1.1 containig ithe following dates:
+# Query a parquet table created by Hive 2.1.1 containig the following dates:
# 1582-10-04, 1582-10-05, 1582-10-06, 1582-10-15, 1582-10-16.
# Impala will incorrectly read back the dates that precede the introduction of Gregorian
# calendar (1582-10-15).
diff --git a/testdata/workloads/functional-query/queries/QueryTest/out-of-range-date.test b/testdata/workloads/functional-query/queries/QueryTest/out-of-range-date.test
index b406242..9dd23f6 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/out-of-range-date.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/out-of-range-date.test
@@ -1,5 +1,6 @@
====
---- QUERY
+# Out of range DATE values in a parquet table.
set abort_on_error=0;
SELECT * FROM $DATABASE.out_of_range_date;
---- TYPES
@@ -18,6 +19,7 @@ NULL
Parquet file '$NAMENODE/test-warehouse/$DATABASE.db/out_of_range_date/out_of_range_date.parquet' column 'v' contains an out of range date. The valid date range is 0000-01-01..9999-12-31. (1 of 2 similar)
====
---- QUERY
+# Out of range DATE values in a parquet table.
set abort_on_error=1;
SELECT * FROM $DATABASE.out_of_range_date;
---- CATCH
diff --git a/tests/query_test/test_date_queries.py b/tests/query_test/test_date_queries.py
index cc1e2c5..99788b3 100644
--- a/tests/query_test/test_date_queries.py
+++ b/tests/query_test/test_date_queries.py
@@ -39,9 +39,11 @@ class TestDateQueries(ImpalaTestSuite):
'batch_size': [0, 1],
'disable_codegen': ['false', 'true'],
'disable_codegen_rows_threshold': [0]}))
- # DATE type is only supported for text & parquet fileformat on HDFS and HBASE.
+ # DATE type is only supported for text, parquet and avro fileformat on HDFS and HBASE.
cls.ImpalaTestMatrix.add_constraint(lambda v:
- v.get_value('table_format').file_format in ('text', 'hbase', 'parquet'))
+ v.get_value('table_format').file_format in ('text', 'hbase', 'parquet')
+ or (v.get_value('table_format').file_format == 'avro'
+ and v.get_value('table_format').compression_codec == 'snap'))
# Run these queries through both beeswax and HS2 to get coverage of date returned
# via both protocols.
@@ -49,7 +51,15 @@ class TestDateQueries(ImpalaTestSuite):
cls.ImpalaTestMatrix.add_constraint(hs2_parquet_constraint)
def test_queries(self, vector):
- self.run_test_case('QueryTest/date', vector)
+ if vector.get_value('table_format').file_format == 'avro':
+ # Avro date test queries are in a separate test file.
+ # - Hive2 uses Julian Calendar for writing dates before 1582-10-15, whereas Impala
+ # uses proleptic Gregorian Calendar. This affects the results Impala gets when
+ # querying avro tables written by Hive2.
+ # - Hive3 on the other hand uses proleptic Gregorian Calendar to write dates.
+ self.run_test_case('QueryTest/avro_date', vector)
+ else:
+ self.run_test_case('QueryTest/date', vector)
def test_partitioning(self, vector, unique_database):
""" Test partitioning by DATE. """
@@ -64,43 +74,57 @@ class TestDateQueries(ImpalaTestSuite):
@SkipIfADLS.qualified_path
@SkipIfLocal.qualified_path
def test_fileformat_support(self, vector, unique_database):
- """ Test that scanning and writing DATE is supported for text tables only."""
+ """ Test that scanning and writing DATE is supported for text and parquet tables.
+ Test that scanning DATE is supported for avro tables as well.
+ """
# This test specifies databases and locations explicitly. No need to execute it for
# anything other than text fileformat on HDFS.
if vector.get_value('table_format').file_format != 'text':
pytest.skip()
- # Parquet table with date column
+
+ # Parquet table with date column.
TABLE_NAME = "parquet_date_tbl"
CREATE_SQL = "CREATE TABLE {0}.{1} (date_col DATE) STORED AS PARQUET".format(
unique_database, TABLE_NAME)
create_table_and_copy_files(self.client, CREATE_SQL, unique_database, TABLE_NAME,
["/testdata/data/date_tbl.parquet"])
- # Avro table with date column
+ # Avro table with date column.
TABLE_NAME = "avro_date_tbl"
CREATE_SQL = "CREATE TABLE {0}.{1} (date_col DATE) STORED AS AVRO".format(
unique_database, TABLE_NAME)
create_table_and_copy_files(self.client, CREATE_SQL, unique_database, TABLE_NAME,
["/testdata/data/date_tbl.avro"])
- # Partitioned table with parquet and avro partitions
+ # Orc table with date column.
+ TABLE_NAME = "orc_date_tbl"
+ CREATE_SQL = "CREATE TABLE {0}.{1} (date_col DATE) STORED AS ORC".format(
+ unique_database, TABLE_NAME)
+ create_table_and_copy_files(self.client, CREATE_SQL, unique_database, TABLE_NAME,
+ ["/testdata/data/date_tbl.orc"])
+
+ # Partitioned table with parquet and avro partitions.
TABLE_NAME = "date_tbl"
CREATE_SQL = """CREATE TABLE {0}.{1} (date_col DATE)
PARTITIONED BY (date_part DATE)""".format(unique_database, TABLE_NAME)
self.client.execute(CREATE_SQL)
- # Add partitions
+ # Add partitions.
ADD_PART_SQL = """ALTER TABLE {0}.{1} ADD PARTITION (date_part='1899-12-31')
LOCATION '/test-warehouse/{0}.db/parquet_date_tbl'
PARTITION (date_part='1999-12-31')
LOCATION '/test-warehouse/{0}.db/avro_date_tbl'
""".format(unique_database, TABLE_NAME)
self.client.execute(ADD_PART_SQL)
- # Parquet fileformat
+ # Parquet fileformat.
SET_PART_FF_SQL = """ALTER TABLE {0}.{1} PARTITION (date_part='1899-12-31')
SET FILEFORMAT PARQUET""".format(unique_database, TABLE_NAME)
self.client.execute(SET_PART_FF_SQL)
- # Avro fileformat
+ # Avro fileformat.
SET_PART_FF_SQL = """ALTER TABLE {0}.{1} PARTITION (date_part='1999-12-31')
SET FILEFORMAT AVRO""".format(unique_database, TABLE_NAME)
self.client.execute(SET_PART_FF_SQL)
+ # After adding the avro partition, metadata has to be invalidated, otherwise querying
+ # the table will fail with stale metadata error.
+ self.client.execute("INVALIDATE METADATA {0}.{1}".format(unique_database, TABLE_NAME))
+
# Test scanning/writing tables with different fileformats.
self.run_test_case('QueryTest/date-fileformat-support', vector,
use_db=unique_database)