You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2018/06/13 15:48:43 UTC

[05/10] impala git commit: IMPALA-7417: Remove DCHECKs with unnecessary constraint on dictionary encoding bit width

IMPALA-7417: Remove DCHECKs with unnecessary constraint on dictionary encoding bit width

Reading dictionary encoded Parquet data pages where the bit width is
larger than the encoded type's size (e.g. coding 8 bit TINYINT with
16 bit dictionary indices) led to DCHECK error in debug builds.
Impala does not create such parquet files (an N bit type can have
maximum 2^N distinct values, so N bit dictionary indices are enough
for a dictionary that contains every possible value), but the Parquet
standard does not forbid to do so.

These DCHECKs were probably introduced by a copy paste error (similar
checks exist in the non-dictionary encoded bit reader functions,
where they are valid).

Testing:
- a new test is added to check that these data pages can be decoded
  correctly

Change-Id: I9ff3b00cbcab09dec11b3607d7d9a9c2c0025e1a
Reviewed-on: http://gerrit.cloudera.org:8080/10683
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>


Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/873053fb
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/873053fb
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/873053fb

Branch: refs/heads/2.x
Commit: 873053fb81ae24e03bcd03aed3dea2ded55e5e5d
Parents: 11554a1
Author: Csaba Ringhofer <cs...@cloudera.com>
Authored: Mon Jun 11 16:53:04 2018 +0200
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Wed Jun 13 03:10:18 2018 +0000

----------------------------------------------------------------------
 be/src/util/bit-packing.inline.h                   |   2 --
 be/src/util/bit-stream-utils.inline.h              |   1 -
 testdata/data/README                               |   5 +++++
 .../dict_encoding_with_large_bit_width.parquet     | Bin 0 -> 390 bytes
 tests/query_test/test_scanners.py                  |  16 ++++++++++++++++
 5 files changed, 21 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/impala/blob/873053fb/be/src/util/bit-packing.inline.h
----------------------------------------------------------------------
diff --git a/be/src/util/bit-packing.inline.h b/be/src/util/bit-packing.inline.h
index 4b2bb33..9a43644 100644
--- a/be/src/util/bit-packing.inline.h
+++ b/be/src/util/bit-packing.inline.h
@@ -240,7 +240,6 @@ const uint8_t* BitPacking::UnpackAndDecode32Values(const uint8_t* __restrict__ i
     OutType* __restrict__ out, bool* __restrict__ decode_error) {
   static_assert(BIT_WIDTH >= 0, "BIT_WIDTH too low");
   static_assert(BIT_WIDTH <= 32, "BIT_WIDTH > 32");
-  DCHECK_LE(BIT_WIDTH, sizeof(OutType) * CHAR_BIT) << "BIT_WIDTH too high for output";
   constexpr int BYTES_TO_READ = BitUtil::RoundUpNumBytes(32 * BIT_WIDTH);
   DCHECK_GE(in_bytes, BYTES_TO_READ);
   // TODO: this could be optimised further by using SIMD instructions.
@@ -305,7 +304,6 @@ const uint8_t* BitPacking::UnpackAndDecodeUpTo31Values(const uint8_t* __restrict
       OutType* __restrict__ out, bool* __restrict__ decode_error) {
   static_assert(BIT_WIDTH >= 0, "BIT_WIDTH too low");
   static_assert(BIT_WIDTH <= 32, "BIT_WIDTH > 32");
-  DCHECK_LE(BIT_WIDTH, sizeof(OutType) * CHAR_BIT) << "BIT_WIDTH too high for output";
   constexpr int MAX_BATCH_SIZE = 31;
   const int BYTES_TO_READ = BitUtil::RoundUpNumBytes(num_values * BIT_WIDTH);
   DCHECK_GE(in_bytes, BYTES_TO_READ);

http://git-wip-us.apache.org/repos/asf/impala/blob/873053fb/be/src/util/bit-stream-utils.inline.h
----------------------------------------------------------------------
diff --git a/be/src/util/bit-stream-utils.inline.h b/be/src/util/bit-stream-utils.inline.h
index b85b252..371a81d 100644
--- a/be/src/util/bit-stream-utils.inline.h
+++ b/be/src/util/bit-stream-utils.inline.h
@@ -108,7 +108,6 @@ inline int BatchedBitReader::UnpackAndDecodeBatch(
   DCHECK(buffer_pos_ != nullptr);
   DCHECK_GE(bit_width, 0);
   DCHECK_LE(bit_width, MAX_BITWIDTH);
-  DCHECK_LE(bit_width, sizeof(T) * 8);
   DCHECK_GE(num_values, 0);
 
   const uint8_t* new_buffer_pos;

http://git-wip-us.apache.org/repos/asf/impala/blob/873053fb/testdata/data/README
----------------------------------------------------------------------
diff --git a/testdata/data/README b/testdata/data/README
index ac260ef..fec0e82 100644
--- a/testdata/data/README
+++ b/testdata/data/README
@@ -174,3 +174,8 @@ Parquet v1 file with RLE encoded boolean column "b" and int column "i".
 Created for IMPALA-6324, generated with modified parquet-mr. Contains 279 rows,
 139 with value false, and 140 with value true. "i" is always 1 if "b" is True
 and always 0 if "b" is false.
+
+dict_encoding_with_large_bit_width.parquet:
+Parquet file with a single TINYINT column "i" with 33 rows. Created by a modified
+Impala to use 9 bit dictionary indices for encoding. Reading this file used to lead
+to DCHECK errors (IMPALA-7147).

http://git-wip-us.apache.org/repos/asf/impala/blob/873053fb/testdata/data/dict_encoding_with_large_bit_width.parquet
----------------------------------------------------------------------
diff --git a/testdata/data/dict_encoding_with_large_bit_width.parquet b/testdata/data/dict_encoding_with_large_bit_width.parquet
new file mode 100644
index 0000000..f12bb3b
Binary files /dev/null and b/testdata/data/dict_encoding_with_large_bit_width.parquet differ

http://git-wip-us.apache.org/repos/asf/impala/blob/873053fb/tests/query_test/test_scanners.py
----------------------------------------------------------------------
diff --git a/tests/query_test/test_scanners.py b/tests/query_test/test_scanners.py
index 194149e..3c1ade7 100644
--- a/tests/query_test/test_scanners.py
+++ b/tests/query_test/test_scanners.py
@@ -713,6 +713,22 @@ class TestParquet(ImpalaTestSuite):
     self.run_test_case(
         'QueryTest/parquet-rle-encoded-bool', vector, unique_database)
 
+  def test_dict_encoding_with_large_bit_width(self, vector, unique_database):
+    """IMPALA-7147: Test that Impala can decode dictionary encoded pages where the
+       dictionary index bit width is larger than the encoded byte's bit width.
+    """
+    TABLE_NAME = "dict_encoding_with_large_bit_width"
+    self.client.execute("CREATE TABLE {0}.{1} (i tinyint) STORED AS PARQUET".format(
+        unique_database, TABLE_NAME))
+    table_loc = get_fs_path(
+        "/test-warehouse/{0}.db/{1}".format(unique_database, TABLE_NAME))
+    check_call(['hdfs', 'dfs', '-copyFromLocal', os.environ['IMPALA_HOME'] +
+        "/testdata/data/{0}.parquet".format(TABLE_NAME), table_loc])
+
+    result = self.execute_query(
+        "select * from {0}.{1}".format(unique_database, TABLE_NAME))
+    assert(len(result.data) == 33)
+
 # We use various scan range lengths to exercise corner cases in the HDFS scanner more
 # thoroughly. In particular, it will exercise:
 # 1. default scan range