You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mm...@apache.org on 2018/08/08 07:38:14 UTC
[51/51] [partial] hive git commit: HIVE-20315: Vectorization: Fix
more NULL / Wrong Results issues and avoid unnecessary casts/conversions
(Matt McCline, reviewed by Teddy Choi)
HIVE-20315: Vectorization: Fix more NULL / Wrong Results issues and avoid unnecessary casts/conversions (Matt McCline, reviewed by Teddy Choi)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/470ba3e2
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/470ba3e2
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/470ba3e2
Branch: refs/heads/master
Commit: 470ba3e2835ef769f940d013acbe6c05d9208903
Parents: c0f63bf
Author: Matt McCline <mm...@hortonworks.com>
Authored: Wed Aug 8 02:37:03 2018 -0500
Committer: Matt McCline <mm...@hortonworks.com>
Committed: Wed Aug 8 02:37:03 2018 -0500
----------------------------------------------------------------------
.../ql/exec/vector/VectorColumnSetInfo.java | 8 +-
.../hive/ql/exec/vector/VectorCopyRow.java | 63 +-
.../ql/exec/vector/VectorGroupKeyHelper.java | 5 +-
.../exec/vector/VectorHashKeyWrapperBatch.java | 12 +-
.../exec/vector/VectorSMBMapJoinOperator.java | 2 +-
.../ql/exec/vector/VectorizationContext.java | 46 +-
.../expressions/CastStringGroupToString.java | 40 -
.../ql/exec/vector/expressions/VectorElt.java | 168 +-
.../VectorExpressionWriterFactory.java | 34 +
.../hive/ql/optimizer/physical/Vectorizer.java | 19 +-
.../vector/TestVectorHashKeyWrapperBatch.java | 6 +-
.../ql/exec/vector/TestVectorRowObject.java | 3 +-
.../hive/ql/exec/vector/TestVectorSerDeRow.java | 137 +-
.../ql/exec/vector/VectorRandomRowSource.java | 67 +-
.../hive/ql/exec/vector/VectorVerifyFast.java | 6 +-
.../aggregation/TestVectorAggregation.java | 9 +-
.../expressions/TestVectorArithmetic.java | 14 +-
.../vector/expressions/TestVectorBetweenIn.java | 38 +-
.../expressions/TestVectorCastStatement.java | 11 +-
.../expressions/TestVectorCoalesceElt.java | 87 +-
.../expressions/TestVectorDateAddSub.java | 10 +-
.../vector/expressions/TestVectorDateDiff.java | 9 +-
.../expressions/TestVectorFilterCompare.java | 12 +-
.../expressions/TestVectorIfStatement.java | 3 +-
.../vector/expressions/TestVectorIndex.java | 5 +-
.../vector/expressions/TestVectorNegative.java | 21 +-
.../exec/vector/expressions/TestVectorNull.java | 14 +-
.../expressions/TestVectorStringConcat.java | 3 +-
.../expressions/TestVectorStringUnary.java | 3 +-
.../expressions/TestVectorStructField.java | 370 ++
.../vector/expressions/TestVectorSubStr.java | 3 +-
.../expressions/TestVectorTimestampExtract.java | 3 +-
.../fast/TestVectorMapJoinFastRowHashMap.java | 101 +-
.../test/queries/clientpositive/perf/query1.q | 7 +-
.../test/queries/clientpositive/perf/query10.q | 7 +-
.../test/queries/clientpositive/perf/query11.q | 7 +-
.../test/queries/clientpositive/perf/query12.q | 7 +-
.../test/queries/clientpositive/perf/query13.q | 7 +-
.../test/queries/clientpositive/perf/query14.q | 7 +-
.../test/queries/clientpositive/perf/query15.q | 7 +-
.../test/queries/clientpositive/perf/query16.q | 7 +-
.../test/queries/clientpositive/perf/query17.q | 7 +-
.../test/queries/clientpositive/perf/query18.q | 7 +-
.../test/queries/clientpositive/perf/query19.q | 7 +-
.../test/queries/clientpositive/perf/query2.q | 7 +-
.../test/queries/clientpositive/perf/query20.q | 7 +-
.../test/queries/clientpositive/perf/query21.q | 7 +-
.../test/queries/clientpositive/perf/query22.q | 7 +-
.../test/queries/clientpositive/perf/query23.q | 7 +-
.../test/queries/clientpositive/perf/query24.q | 7 +-
.../test/queries/clientpositive/perf/query25.q | 7 +-
.../test/queries/clientpositive/perf/query26.q | 7 +-
.../test/queries/clientpositive/perf/query27.q | 7 +-
.../test/queries/clientpositive/perf/query28.q | 7 +-
.../test/queries/clientpositive/perf/query29.q | 7 +-
.../test/queries/clientpositive/perf/query3.q | 7 +-
.../test/queries/clientpositive/perf/query30.q | 7 +-
.../test/queries/clientpositive/perf/query31.q | 7 +-
.../test/queries/clientpositive/perf/query32.q | 7 +-
.../test/queries/clientpositive/perf/query33.q | 7 +-
.../test/queries/clientpositive/perf/query34.q | 7 +-
.../test/queries/clientpositive/perf/query35.q | 7 +-
.../test/queries/clientpositive/perf/query36.q | 7 +-
.../test/queries/clientpositive/perf/query37.q | 7 +-
.../test/queries/clientpositive/perf/query38.q | 7 +-
.../test/queries/clientpositive/perf/query39.q | 7 +-
.../test/queries/clientpositive/perf/query4.q | 7 +-
.../test/queries/clientpositive/perf/query40.q | 7 +-
.../test/queries/clientpositive/perf/query42.q | 7 +-
.../test/queries/clientpositive/perf/query43.q | 7 +-
.../test/queries/clientpositive/perf/query44.q | 7 +-
.../test/queries/clientpositive/perf/query45.q | 7 +-
.../test/queries/clientpositive/perf/query46.q | 7 +-
.../test/queries/clientpositive/perf/query47.q | 7 +-
.../test/queries/clientpositive/perf/query48.q | 7 +-
.../test/queries/clientpositive/perf/query49.q | 7 +-
.../test/queries/clientpositive/perf/query5.q | 7 +-
.../test/queries/clientpositive/perf/query50.q | 7 +-
.../test/queries/clientpositive/perf/query51.q | 7 +-
.../test/queries/clientpositive/perf/query52.q | 7 +-
.../test/queries/clientpositive/perf/query53.q | 7 +-
.../test/queries/clientpositive/perf/query54.q | 7 +-
.../test/queries/clientpositive/perf/query55.q | 7 +-
.../test/queries/clientpositive/perf/query56.q | 7 +-
.../test/queries/clientpositive/perf/query57.q | 7 +-
.../test/queries/clientpositive/perf/query58.q | 7 +-
.../test/queries/clientpositive/perf/query59.q | 7 +-
.../test/queries/clientpositive/perf/query6.q | 7 +-
.../test/queries/clientpositive/perf/query60.q | 7 +-
.../test/queries/clientpositive/perf/query61.q | 7 +-
.../test/queries/clientpositive/perf/query63.q | 7 +-
.../test/queries/clientpositive/perf/query64.q | 7 +-
.../test/queries/clientpositive/perf/query65.q | 7 +-
.../test/queries/clientpositive/perf/query66.q | 7 +-
.../test/queries/clientpositive/perf/query67.q | 7 +-
.../test/queries/clientpositive/perf/query68.q | 7 +-
.../test/queries/clientpositive/perf/query69.q | 7 +-
.../test/queries/clientpositive/perf/query7.q | 7 +-
.../test/queries/clientpositive/perf/query70.q | 7 +-
.../test/queries/clientpositive/perf/query71.q | 7 +-
.../test/queries/clientpositive/perf/query72.q | 7 +-
.../test/queries/clientpositive/perf/query73.q | 7 +-
.../test/queries/clientpositive/perf/query74.q | 7 +-
.../test/queries/clientpositive/perf/query75.q | 7 +-
.../test/queries/clientpositive/perf/query76.q | 7 +-
.../test/queries/clientpositive/perf/query77.q | 7 +-
.../test/queries/clientpositive/perf/query78.q | 7 +-
.../test/queries/clientpositive/perf/query79.q | 7 +-
.../test/queries/clientpositive/perf/query8.q | 7 +-
.../test/queries/clientpositive/perf/query80.q | 7 +-
.../test/queries/clientpositive/perf/query81.q | 7 +-
.../test/queries/clientpositive/perf/query82.q | 7 +-
.../test/queries/clientpositive/perf/query83.q | 7 +-
.../test/queries/clientpositive/perf/query84.q | 7 +-
.../test/queries/clientpositive/perf/query85.q | 7 +-
.../test/queries/clientpositive/perf/query86.q | 7 +-
.../test/queries/clientpositive/perf/query87.q | 7 +-
.../test/queries/clientpositive/perf/query88.q | 7 +-
.../test/queries/clientpositive/perf/query89.q | 7 +-
.../test/queries/clientpositive/perf/query9.q | 7 +-
.../test/queries/clientpositive/perf/query90.q | 7 +-
.../test/queries/clientpositive/perf/query91.q | 7 +-
.../test/queries/clientpositive/perf/query92.q | 7 +-
.../test/queries/clientpositive/perf/query93.q | 7 +-
.../test/queries/clientpositive/perf/query94.q | 7 +-
.../test/queries/clientpositive/perf/query95.q | 7 +-
.../test/queries/clientpositive/perf/query96.q | 7 +-
.../test/queries/clientpositive/perf/query97.q | 7 +-
.../test/queries/clientpositive/perf/query98.q | 7 +-
.../test/queries/clientpositive/perf/query99.q | 7 +-
.../clientpositive/query_result_fileformat.q | 4 +-
.../llap/convert_decimal64_to_decimal.q.out | 4 +-
.../llap/vector_binary_join_groupby.q.out | 7 +-
.../llap/vector_case_when_1.q.out | 8 +-
.../llap/vector_char_mapjoin1.q.out | 1 -
.../llap/vector_decimal_mapjoin.q.out | 6 +-
.../llap/vector_outer_reference_windowed.q.out | 8 +-
.../clientpositive/llap/vector_udf1.q.out | 18 +-
.../clientpositive/llap/vectorized_casts.q.out | 6 +-
.../llap/vectorized_mapjoin3.q.out | 27 +-
.../clientpositive/perf/spark/query1.q.out | 242 +-
.../clientpositive/perf/spark/query10.q.out | 346 +-
.../clientpositive/perf/spark/query11.q.out | 449 +-
.../clientpositive/perf/spark/query12.q.out | 145 +-
.../clientpositive/perf/spark/query13.q.out | 180 +-
.../clientpositive/perf/spark/query15.q.out | 149 +-
.../clientpositive/perf/spark/query16.q.out | 257 +-
.../clientpositive/perf/spark/query17.q.out | 260 +-
.../clientpositive/perf/spark/query18.q.out | 238 +-
.../clientpositive/perf/spark/query19.q.out | 203 +-
.../clientpositive/perf/spark/query2.q.out | 261 +-
.../clientpositive/perf/spark/query20.q.out | 145 +-
.../clientpositive/perf/spark/query21.q.out | 145 +-
.../clientpositive/perf/spark/query22.q.out | 150 +-
.../clientpositive/perf/spark/query23.q.out | 891 ++-
.../clientpositive/perf/spark/query24.q.out | 423 +-
.../clientpositive/perf/spark/query25.q.out | 251 +-
.../clientpositive/perf/spark/query26.q.out | 176 +-
.../clientpositive/perf/spark/query27.q.out | 176 +-
.../clientpositive/perf/spark/query28.q.out | 483 +-
.../clientpositive/perf/spark/query29.q.out | 253 +-
.../clientpositive/perf/spark/query3.q.out | 121 +-
.../clientpositive/perf/spark/query30.q.out | 300 +-
.../clientpositive/perf/spark/query31.q.out | 600 +-
.../clientpositive/perf/spark/query32.q.out | 189 +-
.../clientpositive/perf/spark/query33.q.out | 446 +-
.../clientpositive/perf/spark/query34.q.out | 170 +-
.../clientpositive/perf/spark/query35.q.out | 328 +-
.../clientpositive/perf/spark/query36.q.out | 175 +-
.../clientpositive/perf/spark/query37.q.out | 141 +-
.../clientpositive/perf/spark/query38.q.out | 383 +-
.../clientpositive/perf/spark/query39.q.out | 291 +-
.../clientpositive/perf/spark/query4.q.out | 659 +-
.../clientpositive/perf/spark/query40.q.out | 165 +-
.../clientpositive/perf/spark/query42.q.out | 126 +-
.../clientpositive/perf/spark/query43.q.out | 115 +-
.../clientpositive/perf/spark/query44.q.out | 309 +-
.../clientpositive/perf/spark/query45.q.out | 280 +-
.../clientpositive/perf/spark/query46.q.out | 225 +-
.../clientpositive/perf/spark/query47.q.out | 426 +-
.../clientpositive/perf/spark/query48.q.out | 153 +-
.../clientpositive/perf/spark/query49.q.out | 516 +-
.../clientpositive/perf/spark/query5.q.out | 443 +-
.../clientpositive/perf/spark/query50.q.out | 171 +-
.../clientpositive/perf/spark/query51.q.out | 158 +-
.../clientpositive/perf/spark/query52.q.out | 126 +-
.../clientpositive/perf/spark/query53.q.out | 130 +-
.../clientpositive/perf/spark/query54.q.out | 611 +-
.../clientpositive/perf/spark/query55.q.out | 125 +-
.../clientpositive/perf/spark/query56.q.out | 446 +-
.../clientpositive/perf/spark/query57.q.out | 426 +-
.../clientpositive/perf/spark/query58.q.out | 713 ++-
.../clientpositive/perf/spark/query59.q.out | 267 +-
.../clientpositive/perf/spark/query6.q.out | 374 +-
.../clientpositive/perf/spark/query60.q.out | 446 +-
.../clientpositive/perf/spark/query61.q.out | 402 +-
.../clientpositive/perf/spark/query63.q.out | 130 +-
.../clientpositive/perf/spark/query65.q.out | 247 +-
.../clientpositive/perf/spark/query66.q.out | 351 +-
.../clientpositive/perf/spark/query67.q.out | 178 +-
.../clientpositive/perf/spark/query68.q.out | 225 +-
.../clientpositive/perf/spark/query69.q.out | 351 +-
.../clientpositive/perf/spark/query7.q.out | 176 +-
.../clientpositive/perf/spark/query70.q.out | 274 +-
.../clientpositive/perf/spark/query71.q.out | 229 +-
.../clientpositive/perf/spark/query72.q.out | 327 +-
.../clientpositive/perf/spark/query73.q.out | 170 +-
.../clientpositive/perf/spark/query74.q.out | 453 +-
.../clientpositive/perf/spark/query75.q.out | 590 +-
.../clientpositive/perf/spark/query76.q.out | 279 +-
.../clientpositive/perf/spark/query77.q.out | 561 +-
.../clientpositive/perf/spark/query78.q.out | 303 +-
.../clientpositive/perf/spark/query79.q.out | 169 +-
.../clientpositive/perf/spark/query8.q.out | 315 +-
.../clientpositive/perf/spark/query80.q.out | 557 +-
.../clientpositive/perf/spark/query81.q.out | 305 +-
.../clientpositive/perf/spark/query82.q.out | 141 +-
.../clientpositive/perf/spark/query83.q.out | 560 +-
.../clientpositive/perf/spark/query84.q.out | 174 +-
.../clientpositive/perf/spark/query85.q.out | 254 +-
.../clientpositive/perf/spark/query86.q.out | 151 +-
.../clientpositive/perf/spark/query87.q.out | 439 +-
.../clientpositive/perf/spark/query88.q.out | 1010 ++-
.../clientpositive/perf/spark/query89.q.out | 177 +-
.../clientpositive/perf/spark/query9.q.out | 864 ++-
.../clientpositive/perf/spark/query90.q.out | 282 +-
.../clientpositive/perf/spark/query91.q.out | 222 +-
.../clientpositive/perf/spark/query92.q.out | 207 +-
.../clientpositive/perf/spark/query93.q.out | 119 +-
.../clientpositive/perf/spark/query94.q.out | 257 +-
.../clientpositive/perf/spark/query95.q.out | 338 +-
.../clientpositive/perf/spark/query96.q.out | 153 +-
.../clientpositive/perf/spark/query97.q.out | 178 +-
.../clientpositive/perf/spark/query98.q.out | 142 +-
.../clientpositive/perf/spark/query99.q.out | 186 +-
.../clientpositive/perf/tez/query1.q.out | 581 +-
.../clientpositive/perf/tez/query10.q.out | 914 ++-
.../clientpositive/perf/tez/query11.q.out | 1442 ++++-
.../clientpositive/perf/tez/query12.q.out | 474 +-
.../clientpositive/perf/tez/query13.q.out | 856 ++-
.../clientpositive/perf/tez/query14.q.out | 6055 ++++++++++++++----
.../clientpositive/perf/tez/query15.q.out | 484 +-
.../clientpositive/perf/tez/query16.q.out | 830 ++-
.../clientpositive/perf/tez/query17.q.out | 1099 +++-
.../clientpositive/perf/tez/query18.q.out | 853 ++-
.../clientpositive/perf/tez/query19.q.out | 824 ++-
.../clientpositive/perf/tez/query2.q.out | 800 ++-
.../clientpositive/perf/tez/query20.q.out | 474 +-
.../clientpositive/perf/tez/query21.q.out | 402 +-
.../clientpositive/perf/tez/query22.q.out | 400 +-
.../clientpositive/perf/tez/query23.q.out | 2316 +++++--
.../clientpositive/perf/tez/query24.q.out | 1588 ++++-
.../clientpositive/perf/tez/query25.q.out | 1051 ++-
.../clientpositive/perf/tez/query26.q.out | 761 ++-
.../clientpositive/perf/tez/query27.q.out | 768 ++-
.../clientpositive/perf/tez/query28.q.out | 979 ++-
.../clientpositive/perf/tez/query29.q.out | 1173 +++-
.../clientpositive/perf/tez/query3.q.out | 461 +-
.../clientpositive/perf/tez/query30.q.out | 707 +-
.../clientpositive/perf/tez/query31.q.out | 2153 +++++--
.../clientpositive/perf/tez/query32.q.out | 474 +-
.../clientpositive/perf/tez/query33.q.out | 1627 ++++-
.../clientpositive/perf/tez/query34.q.out | 752 ++-
.../clientpositive/perf/tez/query35.q.out | 1071 +++-
.../clientpositive/perf/tez/query36.q.out | 691 +-
.../clientpositive/perf/tez/query37.q.out | 515 +-
.../clientpositive/perf/tez/query38.q.out | 1043 ++-
.../clientpositive/perf/tez/query39.q.out | 657 +-
.../clientpositive/perf/tez/query4.q.out | 2146 +++++--
.../clientpositive/perf/tez/query40.q.out | 677 +-
.../clientpositive/perf/tez/query42.q.out | 470 +-
.../clientpositive/perf/tez/query43.q.out | 468 +-
.../clientpositive/perf/tez/query44.q.out | 650 +-
.../clientpositive/perf/tez/query45.q.out | 834 ++-
.../clientpositive/perf/tez/query46.q.out | 924 ++-
.../clientpositive/perf/tez/query47.q.out | 917 ++-
.../clientpositive/perf/tez/query48.q.out | 700 +-
.../clientpositive/perf/tez/query49.q.out | 1635 ++++-
.../clientpositive/perf/tez/query5.q.out | 1645 ++++-
.../clientpositive/perf/tez/query50.q.out | 776 ++-
.../clientpositive/perf/tez/query51.q.out | 535 +-
.../clientpositive/perf/tez/query52.q.out | 471 +-
.../clientpositive/perf/tez/query53.q.out | 629 +-
.../clientpositive/perf/tez/query54.q.out | 1683 ++++-
.../clientpositive/perf/tez/query55.q.out | 470 +-
.../clientpositive/perf/tez/query56.q.out | 1627 ++++-
.../clientpositive/perf/tez/query57.q.out | 917 ++-
.../clientpositive/perf/tez/query58.q.out | 1396 +++-
.../clientpositive/perf/tez/query59.q.out | 988 ++-
.../clientpositive/perf/tez/query6.q.out | 1079 +++-
.../clientpositive/perf/tez/query60.q.out | 1662 ++++-
.../clientpositive/perf/tez/query61.q.out | 1456 ++++-
.../clientpositive/perf/tez/query63.q.out | 629 +-
.../clientpositive/perf/tez/query64.q.out | 3721 ++++++++---
.../clientpositive/perf/tez/query65.q.out | 764 ++-
.../clientpositive/perf/tez/query66.q.out | 1277 +++-
.../clientpositive/perf/tez/query67.q.out | 588 +-
.../clientpositive/perf/tez/query68.q.out | 756 ++-
.../clientpositive/perf/tez/query69.q.out | 946 ++-
.../clientpositive/perf/tez/query7.q.out | 761 ++-
.../clientpositive/perf/tez/query70.q.out | 781 ++-
.../clientpositive/perf/tez/query71.q.out | 1067 ++-
.../clientpositive/perf/tez/query72.q.out | 1328 +++-
.../clientpositive/perf/tez/query73.q.out | 592 +-
.../clientpositive/perf/tez/query74.q.out | 1425 ++++-
.../clientpositive/perf/tez/query75.q.out | 2479 +++++--
.../clientpositive/perf/tez/query76.q.out | 1134 +++-
.../clientpositive/perf/tez/query77.q.out | 1434 ++++-
.../clientpositive/perf/tez/query78.q.out | 1030 ++-
.../clientpositive/perf/tez/query79.q.out | 753 ++-
.../clientpositive/perf/tez/query8.q.out | 938 ++-
.../clientpositive/perf/tez/query80.q.out | 2226 +++++--
.../clientpositive/perf/tez/query81.q.out | 719 ++-
.../clientpositive/perf/tez/query82.q.out | 515 +-
.../clientpositive/perf/tez/query83.q.out | 893 ++-
.../clientpositive/perf/tez/query84.q.out | 469 +-
.../clientpositive/perf/tez/query85.q.out | 803 ++-
.../clientpositive/perf/tez/query86.q.out | 493 +-
.../clientpositive/perf/tez/query87.q.out | 1216 +++-
.../clientpositive/perf/tez/query88.q.out | 2225 ++++---
.../clientpositive/perf/tez/query89.q.out | 693 +-
.../clientpositive/perf/tez/query9.q.out | 1770 ++++-
.../clientpositive/perf/tez/query90.q.out | 710 +-
.../clientpositive/perf/tez/query91.q.out | 605 +-
.../clientpositive/perf/tez/query92.q.out | 515 +-
.../clientpositive/perf/tez/query93.q.out | 464 +-
.../clientpositive/perf/tez/query94.q.out | 830 ++-
.../clientpositive/perf/tez/query95.q.out | 1076 +++-
.../clientpositive/perf/tez/query96.q.out | 432 +-
.../clientpositive/perf/tez/query97.q.out | 446 +-
.../clientpositive/perf/tez/query98.q.out | 465 +-
.../clientpositive/perf/tez/query99.q.out | 551 +-
.../query_result_fileformat.q.out | 76 +-
.../spark/vector_decimal_mapjoin.q.out | 6 +-
.../vector_binary_join_groupby.q.out | 2 +-
.../clientpositive/vector_case_when_1.q.out | 8 +-
.../clientpositive/vector_char_mapjoin1.q.out | 2 +-
.../clientpositive/vector_decimal_mapjoin.q.out | 6 +-
.../clientpositive/vectorized_casts.q.out | 6 +-
.../hadoop/hive/serde2/RandomTypeUtil.java | 29 +
340 files changed, 110242 insertions(+), 21365 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/470ba3e2/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnSetInfo.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnSetInfo.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnSetInfo.java
index 7758ac4..2f15749 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnSetInfo.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorColumnSetInfo.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.exec.vector;
import java.util.Arrays;
+import org.apache.hadoop.hive.common.type.DataTypePhysicalVariation;
import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.ColumnVector.Type;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -105,9 +106,12 @@ public class VectorColumnSetInfo {
}
- protected void addKey(TypeInfo typeInfo) throws HiveException {
+ protected void addKey(TypeInfo typeInfo, DataTypePhysicalVariation dataTypePhysicalVariation)
+ throws HiveException {
- Type columnVectorType = VectorizationContext.getColumnVectorTypeFromTypeInfo(typeInfo);
+ Type columnVectorType =
+ VectorizationContext.getColumnVectorTypeFromTypeInfo(
+ typeInfo, dataTypePhysicalVariation);
switch (columnVectorType) {
case LONG:
case DECIMAL_64:
http://git-wip-us.apache.org/repos/asf/hive/blob/470ba3e2/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java
index bedc12a..586d100 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorCopyRow.java
@@ -22,6 +22,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hadoop.hive.ql.exec.vector.ColumnVector.Type;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
@@ -170,14 +171,72 @@ public class VectorCopyRow {
private class DecimalCopyRow extends CopyRow {
+ private HiveDecimalWritable decimalWritableTemp;
+
DecimalCopyRow(int inColumnIndex, int outColumnIndex) {
super(inColumnIndex, outColumnIndex);
}
@Override
void copy(VectorizedRowBatch inBatch, int inBatchIndex, VectorizedRowBatch outBatch, int outBatchIndex) {
- DecimalColumnVector inColVector = (DecimalColumnVector) inBatch.cols[inColumnIndex];
- DecimalColumnVector outColVector = (DecimalColumnVector) outBatch.cols[outColumnIndex];
+
+ // FUTURE: Figure out how to avoid this runtime checking by working out
+ // DECIMAL_64 to DECIMAL and DECIMAL_64 to DECIMAL_64 up front...
+
+ ColumnVector inColVectorBase = inBatch.cols[inColumnIndex];
+ ColumnVector outColVectorBase = outBatch.cols[outColumnIndex];
+ if (inColVectorBase instanceof Decimal64ColumnVector) {
+ Decimal64ColumnVector inColVector = (Decimal64ColumnVector) inColVectorBase;
+
+ if (outColVectorBase instanceof Decimal64ColumnVector) {
+ Decimal64ColumnVector outColVector = (Decimal64ColumnVector) outColVectorBase;
+
+ if (inColVector.isRepeating) {
+ if (inColVector.noNulls || !inColVector.isNull[0]) {
+ outColVector.vector[outBatchIndex] = inColVector.vector[0];
+ outColVector.isNull[outBatchIndex] = false;
+ } else {
+ VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex);
+ }
+ } else {
+ if (inColVector.noNulls || !inColVector.isNull[inBatchIndex]) {
+ outColVector.vector[outBatchIndex] = inColVector.vector[inBatchIndex];
+ outColVector.isNull[outBatchIndex] = false;
+ } else {
+ VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex);
+ }
+ }
+ } else {
+ DecimalColumnVector outColVector = (DecimalColumnVector) outColVectorBase;
+
+ if (decimalWritableTemp == null) {
+ decimalWritableTemp = new HiveDecimalWritable(0);
+ }
+ if (inColVector.isRepeating) {
+ if (inColVector.noNulls || !inColVector.isNull[0]) {
+ decimalWritableTemp.deserialize64(
+ inColVector.vector[0], inColVector.scale);
+ outColVector.set(outBatchIndex, decimalWritableTemp);
+ outColVector.isNull[outBatchIndex] = false;
+ } else {
+ VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex);
+ }
+ } else {
+ if (inColVector.noNulls || !inColVector.isNull[inBatchIndex]) {
+ decimalWritableTemp.deserialize64(
+ inColVector.vector[inBatchIndex], inColVector.scale);
+ outColVector.set(outBatchIndex, decimalWritableTemp);
+ outColVector.isNull[outBatchIndex] = false;
+ } else {
+ VectorizedBatchUtil.setNullColIsNullValue(outColVector, outBatchIndex);
+ }
+ }
+ }
+ return;
+ }
+
+ DecimalColumnVector inColVector = (DecimalColumnVector) inColVectorBase;
+ DecimalColumnVector outColVector = (DecimalColumnVector) outColVectorBase;
if (inColVector.isRepeating) {
if (inColVector.noNulls || !inColVector.isNull[0]) {
http://git-wip-us.apache.org/repos/asf/hive/blob/470ba3e2/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupKeyHelper.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupKeyHelper.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupKeyHelper.java
index 82dc4a7..6f8822d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupKeyHelper.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupKeyHelper.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hive.ql.exec.vector;
import java.io.IOException;
+import org.apache.hadoop.hive.common.type.DataTypePhysicalVariation;
import org.apache.hadoop.hive.ql.exec.vector.ColumnVector.Type;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -49,7 +50,9 @@ public class VectorGroupKeyHelper extends VectorColumnSetInfo {
VectorExpression keyExpression = keyExpressions[i];
TypeInfo typeInfo = keyExpression.getOutputTypeInfo();
- addKey(typeInfo);
+ DataTypePhysicalVariation dataTypePhysicalVariation =
+ keyExpression.getOutputDataTypePhysicalVariation();
+ addKey(typeInfo, dataTypePhysicalVariation);
// The output of the key expression is the input column.
final int inputColumnNum = keyExpression.getOutputColumnNum();
http://git-wip-us.apache.org/repos/asf/hive/blob/470ba3e2/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java
index 689d3c3..fe504a6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorHashKeyWrapperBatch.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.ql.exec.vector;
+import org.apache.hadoop.hive.common.type.DataTypePhysicalVariation;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriter;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -877,10 +878,12 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
final int size = keyExpressions.length;
TypeInfo[] typeInfos = new TypeInfo[size];
+ DataTypePhysicalVariation[] dataTypePhysicalVariations = new DataTypePhysicalVariation[size];
for (int i = 0; i < size; i++) {
typeInfos[i] = keyExpressions[i].getOutputTypeInfo();
+ dataTypePhysicalVariations[i] = keyExpressions[i].getOutputDataTypePhysicalVariation();
}
- return compileKeyWrapperBatch(keyExpressions, typeInfos);
+ return compileKeyWrapperBatch(keyExpressions, typeInfos, dataTypePhysicalVariations);
}
/**
@@ -890,7 +893,7 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
* will be used to generate proper individual VectorKeyHashWrapper objects.
*/
public static VectorHashKeyWrapperBatch compileKeyWrapperBatch(VectorExpression[] keyExpressions,
- TypeInfo[] typeInfos)
+ TypeInfo[] typeInfos, DataTypePhysicalVariation[] dataTypePhysicalVariations)
throws HiveException {
VectorHashKeyWrapperBatch compiledKeyWrapperBatch = new VectorHashKeyWrapperBatch(keyExpressions.length);
compiledKeyWrapperBatch.keyExpressions = keyExpressions;
@@ -899,7 +902,7 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
// Inspect the output type of each key expression.
for(int i=0; i < typeInfos.length; ++i) {
- compiledKeyWrapperBatch.addKey(typeInfos[i]);
+ compiledKeyWrapperBatch.addKey(typeInfos[i], dataTypePhysicalVariations[i]);
}
compiledKeyWrapperBatch.finishAdding();
@@ -962,6 +965,7 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
switch (columnVectorType) {
case LONG:
+ case DECIMAL_64:
return keyOutputWriter.writeValue(
kw.getLongValue(columnTypeSpecificIndex));
case DOUBLE:
@@ -975,8 +979,6 @@ public class VectorHashKeyWrapperBatch extends VectorColumnSetInfo {
case DECIMAL:
return keyOutputWriter.writeValue(
kw.getDecimal(columnTypeSpecificIndex));
- case DECIMAL_64:
- throw new RuntimeException("Getting writable for DECIMAL_64 not supported");
case TIMESTAMP:
return keyOutputWriter.writeValue(
kw.getTimestamp(columnTypeSpecificIndex));
http://git-wip-us.apache.org/repos/asf/hive/blob/470ba3e2/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSMBMapJoinOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSMBMapJoinOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSMBMapJoinOperator.java
index 35f810f..879a7b4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSMBMapJoinOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorSMBMapJoinOperator.java
@@ -129,7 +129,7 @@ public class VectorSMBMapJoinOperator extends SMBMapJoinOperator
List<ExprNodeDesc> keyDesc = desc.getKeys().get(posBigTable);
keyExpressions = vContext.getVectorExpressions(keyDesc);
- keyOutputWriters = VectorExpressionWriterFactory.getExpressionWriters(keyDesc);
+ keyOutputWriters = VectorExpressionWriterFactory.getExpressionWriters(keyExpressions);
Map<Byte, List<ExprNodeDesc>> exprs = desc.getExprs();
bigTableValueExpressions = vContext.getVectorExpressions(exprs.get(posBigTable));
http://git-wip-us.apache.org/repos/asf/hive/blob/470ba3e2/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
index d6bfa7a..125bc65 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
@@ -1806,6 +1806,25 @@ public class VectorizationContext {
return vectorExpression;
}
+ public void wrapWithDecimal64ToDecimalConversions(VectorExpression[] vecExprs)
+ throws HiveException{
+ if (vecExprs == null) {
+ return;
+ }
+ final int size = vecExprs.length;
+ for (int i = 0; i < size; i++) {
+ VectorExpression vecExpr = vecExprs[i];
+ if (vecExpr.getOutputTypeInfo() instanceof DecimalTypeInfo) {
+ DataTypePhysicalVariation outputDataTypePhysicalVariation =
+ vecExpr.getOutputDataTypePhysicalVariation();
+ if (outputDataTypePhysicalVariation == DataTypePhysicalVariation.DECIMAL_64) {
+ vecExprs[i] =
+ wrapWithDecimal64ToDecimalConversion(vecExpr);
+ }
+ }
+ }
+ }
+
public VectorExpression wrapWithDecimal64ToDecimalConversion(VectorExpression inputExpression)
throws HiveException {
@@ -2854,7 +2873,11 @@ public class VectorizationContext {
} else if (isTimestampFamily(inputType)) {
return createVectorExpression(CastTimestampToString.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType);
} else if (isStringFamily(inputType)) {
- return createVectorExpression(CastStringGroupToString.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType);
+
+ // STRING and VARCHAR types require no conversion, so use a no-op.
+ // Also, CHAR is stored in BytesColumnVector with trimmed blank padding, so it also
+ // requires no conversion;
+ return getIdentityExpression(childExpr);
}
return null;
}
@@ -3074,8 +3097,27 @@ public class VectorizationContext {
List<ExprNodeDesc> castChildren = new ArrayList<ExprNodeDesc>();
boolean wereCastUdfs = false;
+ Category commonTypeCategory = commonType.getCategory();
for (ExprNodeDesc desc: childExpr.subList(1, 4)) {
- if (commonType.equals(desc.getTypeInfo())) {
+ TypeInfo childTypeInfo = desc.getTypeInfo();
+ Category childCategory = childTypeInfo.getCategory();
+
+ if (childCategory != commonTypeCategory) {
+ return null;
+ }
+ final boolean isNeedsCast;
+ if (commonTypeCategory == Category.PRIMITIVE) {
+
+ // Do not to strict TypeInfo comparisons for DECIMAL -- just compare the category.
+ // Otherwise, we generate unnecessary casts.
+ isNeedsCast =
+ ((PrimitiveTypeInfo) commonType).getPrimitiveCategory() !=
+ ((PrimitiveTypeInfo) childTypeInfo).getPrimitiveCategory();
+ } else {
+ isNeedsCast = !commonType.equals(desc.getTypeInfo());
+ }
+
+ if (!isNeedsCast) {
castChildren.add(desc);
} else {
GenericUDF castUdf = getGenericUDFForCast(commonType);
http://git-wip-us.apache.org/repos/asf/hive/blob/470ba3e2/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastStringGroupToString.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastStringGroupToString.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastStringGroupToString.java
deleted file mode 100644
index 8232e67..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/CastStringGroupToString.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.exec.vector.expressions;
-
-import org.apache.hadoop.hive.ql.exec.vector.BytesColumnVector;
-
-// cast string group to string (varchar to string, etc.)
-public class CastStringGroupToString extends StringUnaryUDFDirect {
-
- private static final long serialVersionUID = 1L;
-
- public CastStringGroupToString() {
- super();
- }
-
- public CastStringGroupToString(int inputColumn, int outputColumnNum) {
- super(inputColumn, outputColumnNum);
- }
-
- @Override
- protected void func(BytesColumnVector outV, byte[][] vector, int[] start, int[] length, int i) {
- outV.setVal(i, vector[i], start[i], length[i]);
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/470ba3e2/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorElt.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorElt.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorElt.java
index 00e529d..75e60eb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorElt.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorElt.java
@@ -66,53 +66,157 @@ public class VectorElt extends VectorExpression {
outputVector.init();
- outputVector.noNulls = false;
outputVector.isRepeating = false;
+ final int limit = inputColumns.length;
LongColumnVector inputIndexVector = (LongColumnVector) batch.cols[inputColumns[0]];
+ boolean[] inputIndexIsNull = inputIndexVector.isNull;
long[] indexVector = inputIndexVector.vector;
if (inputIndexVector.isRepeating) {
- int index = (int)indexVector[0];
- if (index > 0 && index < inputColumns.length) {
- BytesColumnVector cv = (BytesColumnVector) batch.cols[inputColumns[index]];
- if (cv.isRepeating) {
- outputVector.setElement(0, 0, cv);
- outputVector.isRepeating = true;
- } else if (batch.selectedInUse) {
- for (int j = 0; j != n; j++) {
- int i = sel[j];
- outputVector.setVal(i, cv.vector[0], cv.start[0], cv.length[0]);
+ if (inputIndexVector.noNulls || !inputIndexIsNull[0]) {
+ int repeatedIndex = (int) indexVector[0];
+ if (repeatedIndex > 0 && repeatedIndex < limit) {
+ BytesColumnVector cv = (BytesColumnVector) batch.cols[inputColumns[repeatedIndex]];
+ if (cv.isRepeating) {
+ outputVector.isNull[0] = false;
+ outputVector.setElement(0, 0, cv);
+ outputVector.isRepeating = true;
+ } else if (cv.noNulls) {
+ if (batch.selectedInUse) {
+ for (int j = 0; j != n; j++) {
+ int i = sel[j];
+ outputVector.isNull[i] = false;
+ outputVector.setVal(i, cv.vector[i], cv.start[i], cv.length[i]);
+ }
+ } else {
+ for (int i = 0; i != n; i++) {
+ outputVector.isNull[i] = false;
+ outputVector.setVal(i, cv.vector[i], cv.start[i], cv.length[i]);
+ }
+ }
+ } else {
+ if (batch.selectedInUse) {
+ for (int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (!cv.isNull[i]) {
+ outputVector.isNull[i] = false;
+ outputVector.setVal(i, cv.vector[i], cv.start[i], cv.length[i]);
+ } else {
+ outputVector.isNull[i] = true;
+ outputVector.noNulls = false;
+ }
+ }
+ } else {
+ for (int i = 0; i != n; i++) {
+ if (!cv.isNull[i]) {
+ outputVector.isNull[i] = false;
+ outputVector.setVal(i, cv.vector[i], cv.start[i], cv.length[i]);
+ } else {
+ outputVector.isNull[i] = true;
+ outputVector.noNulls = false;
+ }
+ }
+ }
}
} else {
- for (int i = 0; i != n; i++) {
- outputVector.setVal(i, cv.vector[0], cv.start[0], cv.length[0]);
- }
+ outputVector.isNull[0] = true;
+ outputVector.noNulls = false;
+ outputVector.isRepeating = true;
}
} else {
outputVector.isNull[0] = true;
+ outputVector.noNulls = false;
outputVector.isRepeating = true;
}
- } else if (batch.selectedInUse) {
- for (int j = 0; j != n; j++) {
- int i = sel[j];
- int index = (int)indexVector[i];
- if (index > 0 && index < inputColumns.length) {
- BytesColumnVector cv = (BytesColumnVector) batch.cols[inputColumns[index]];
- int cvi = cv.isRepeating ? 0 : i;
- outputVector.setVal(i, cv.vector[cvi], cv.start[cvi], cv.length[cvi]);
- } else {
- outputVector.isNull[i] = true;
+ return;
+ }
+
+ if (inputIndexVector.noNulls) {
+ if (batch.selectedInUse) {
+ for (int j = 0; j != n; j++) {
+ int i = sel[j];
+ int index = (int) indexVector[i];
+ if (index > 0 && index < limit) {
+ BytesColumnVector cv = (BytesColumnVector) batch.cols[inputColumns[index]];
+ int adjusted = cv.isRepeating ? 0 : i;
+ if (!cv.isNull[adjusted]) {
+ outputVector.isNull[i] = false;
+ outputVector.setVal(i, cv.vector[adjusted], cv.start[adjusted], cv.length[adjusted]);
+ } else {
+ outputVector.isNull[i] = true;
+ outputVector.noNulls = false;
+ }
+ } else {
+ outputVector.isNull[i] = true;
+ outputVector.noNulls = false;
+ }
+ }
+ } else {
+ for (int i = 0; i != n; i++) {
+ int index = (int) indexVector[i];
+ if (index > 0 && index < limit) {
+ BytesColumnVector cv = (BytesColumnVector) batch.cols[inputColumns[index]];
+ int adjusted = cv.isRepeating ? 0 : i;
+ if (!cv.isNull[adjusted]) {
+ outputVector.isNull[i] = false;
+ outputVector.setVal(i, cv.vector[adjusted], cv.start[adjusted], cv.length[adjusted]);
+ } else {
+ outputVector.isNull[i] = true;
+ outputVector.noNulls = false;
+ }
+ } else {
+ outputVector.isNull[i] = true;
+ outputVector.noNulls = false;
+ }
}
}
} else {
- for (int i = 0; i != n; i++) {
- int index = (int)indexVector[i];
- if (index > 0 && index < inputColumns.length) {
- BytesColumnVector cv = (BytesColumnVector) batch.cols[inputColumns[index]];
- int cvi = cv.isRepeating ? 0 : i;
- outputVector.setVal(i, cv.vector[cvi], cv.start[cvi], cv.length[cvi]);
- } else {
- outputVector.isNull[i] = true;
+ if (batch.selectedInUse) {
+ for (int j = 0; j != n; j++) {
+ int i = sel[j];
+ if (!inputIndexVector.isNull[i]) {
+ int index = (int) indexVector[i];
+ if (index > 0 && index < limit) {
+ BytesColumnVector cv = (BytesColumnVector) batch.cols[inputColumns[index]];
+ int adjusted = cv.isRepeating ? 0 : i;
+ if (cv.noNulls || !cv.isNull[adjusted]) {
+ outputVector.isNull[i] = false;
+ outputVector.setVal(i, cv.vector[adjusted], cv.start[adjusted], cv.length[adjusted]);
+ } else {
+ outputVector.isNull[i] = true;
+ outputVector.noNulls = false;
+ }
+ } else {
+ outputVector.isNull[i] = true;
+ outputVector.noNulls = false;
+ }
+ } else {
+ outputVector.isNull[i] = true;
+ outputVector.noNulls = false;
+ }
+ }
+ } else {
+ for (int i = 0; i != n; i++) {
+ if (!inputIndexVector.isNull[i]) {
+ int index = (int) indexVector[i];
+ if (index > 0 && index < limit) {
+ BytesColumnVector cv = (BytesColumnVector) batch.cols[inputColumns[index]];
+ int adjusted = cv.isRepeating ? 0 : i;
+ if (cv.noNulls || !cv.isNull[adjusted]) {
+ outputVector.isNull[i] = false;
+ outputVector.setVal(i, cv.vector[adjusted], cv.start[adjusted], cv.length[adjusted]);
+ } else {
+ outputVector.isNull[i] = true;
+ outputVector.noNulls = false;
+ }
+ } else {
+ outputVector.isNull[i] = true;
+ outputVector.noNulls = false;
+ }
+ } else {
+ outputVector.isNull[i] = true;
+ outputVector.noNulls = false;
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/470ba3e2/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java
index 6a87927..b78b97db 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/VectorExpressionWriterFactory.java
@@ -27,6 +27,7 @@ import java.util.Map;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.hive.common.type.DataTypePhysicalVariation;
import org.apache.hadoop.hive.common.type.HiveChar;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
@@ -63,10 +64,12 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableTimestamp
import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableShortObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.SettableStringObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.VoidObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.MapTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.ListTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
import org.apache.hadoop.hive.serde2.typeinfo.UnionTypeInfo;
import org.apache.hadoop.io.Text;
@@ -601,6 +604,24 @@ public final class VectorExpressionWriterFactory {
}
/**
+ * Compiles the appropriate vector expression writer based on an expression info (ExprNodeDesc)
+ */
+ public static VectorExpressionWriter genVectorExpressionWritable(VectorExpression vecExpr)
+ throws HiveException {
+ TypeInfo outputTypeInfo = vecExpr.getOutputTypeInfo();
+ DataTypePhysicalVariation outputDataTypePhysicalVariation =
+ vecExpr.getOutputDataTypePhysicalVariation();
+ if (outputTypeInfo instanceof DecimalTypeInfo &&
+ outputDataTypePhysicalVariation == DataTypePhysicalVariation.DECIMAL_64) {
+ outputTypeInfo = TypeInfoFactory.longTypeInfo;
+ }
+ ObjectInspector objectInspector =
+ TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(
+ outputTypeInfo);
+ return genVectorExpressionWritable(objectInspector);
+ }
+
+ /**
* Specialized writer for ListColumnVector. Will throw cast exception
* if the wrong vector column is used.
*/
@@ -1746,6 +1767,19 @@ public final class VectorExpressionWriterFactory {
}
/**
+ * Helper function to create an array of writers from a list of expression descriptors.
+ */
+ public static VectorExpressionWriter[] getExpressionWriters(VectorExpression[] vecExprs)
+ throws HiveException {
+ VectorExpressionWriter[] writers = new VectorExpressionWriter[vecExprs.length];
+ for(int i=0; i<writers.length; ++i) {
+ VectorExpression vecExpr = vecExprs[i];
+ writers[i] = genVectorExpressionWritable(vecExpr);
+ }
+ return writers;
+ }
+
+ /**
* A poor man Java closure. Works around the problem of having to return multiple objects
* from one function call.
*/
http://git-wip-us.apache.org/repos/asf/hive/blob/470ba3e2/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
index 9bb104d..a6a3417 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
@@ -3435,9 +3435,8 @@ public class Vectorizer implements PhysicalPlanResolver {
boolean outerJoinHasNoKeys = (!desc.isNoOuterJoin() && keyDesc.size() == 0);
- // For now, we don't support joins on or using DECIMAL_64.
- VectorExpression[] allBigTableKeyExpressions =
- vContext.getVectorExpressionsUpConvertDecimal64(keyDesc);
+ VectorExpression[] allBigTableKeyExpressions = vContext.getVectorExpressions(keyDesc);
+
final int allBigTableKeyExpressionsLength = allBigTableKeyExpressions.length;
boolean supportsKeyTypes = true; // Assume.
HashSet<String> notSupportedKeyTypes = new HashSet<String>();
@@ -3479,9 +3478,7 @@ public class Vectorizer implements PhysicalPlanResolver {
List<ExprNodeDesc> bigTableExprs = desc.getExprs().get(posBigTable);
- // For now, we don't support joins on or using DECIMAL_64.
- VectorExpression[] allBigTableValueExpressions =
- vContext.getVectorExpressionsUpConvertDecimal64(bigTableExprs);
+ VectorExpression[] allBigTableValueExpressions = vContext.getVectorExpressions(bigTableExprs);
boolean isFastHashTableEnabled =
HiveConf.getBoolVar(hiveConf,
@@ -4476,9 +4473,7 @@ public class Vectorizer implements PhysicalPlanResolver {
List<ExprNodeDesc> keysDesc = groupByDesc.getKeys();
- // For now, we don't support group by on DECIMAL_64 keys.
- VectorExpression[] vecKeyExpressions =
- vContext.getVectorExpressionsUpConvertDecimal64(keysDesc);
+ VectorExpression[] vecKeyExpressions = vContext.getVectorExpressions(keysDesc);
ArrayList<AggregationDesc> aggrDesc = groupByDesc.getAggregators();
final int size = aggrDesc.size();
@@ -5020,6 +5015,12 @@ public class Vectorizer implements PhysicalPlanResolver {
opClass = VectorMapJoinOuterFilteredOperator.class;
}
+ // Wrap any DECIMAL_64 expressions with Conversion.
+ vContext.wrapWithDecimal64ToDecimalConversions(
+ vectorMapJoinDesc.getAllBigTableKeyExpressions());
+ vContext.wrapWithDecimal64ToDecimalConversions(
+ vectorMapJoinDesc.getAllBigTableValueExpressions());
+
vectorOp = OperatorFactory.getVectorOperator(
opClass, op.getCompilationOpContext(), desc,
vContext, vectorMapJoinDesc);
http://git-wip-us.apache.org/repos/asf/hive/blob/470ba3e2/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorHashKeyWrapperBatch.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorHashKeyWrapperBatch.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorHashKeyWrapperBatch.java
index e349fbd..b2818ad 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorHashKeyWrapperBatch.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorHashKeyWrapperBatch.java
@@ -26,6 +26,7 @@ import static org.junit.Assert.assertTrue;
import java.sql.Timestamp;
import org.junit.Test;
+import org.apache.hadoop.hive.common.type.DataTypePhysicalVariation;
import org.apache.hadoop.hive.ql.exec.vector.expressions.IdentityExpression;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.exec.vector.util.FakeVectorRowBatchFromObjectIterables;
@@ -47,10 +48,13 @@ public class TestVectorHashKeyWrapperBatch {
new VectorExpression[] { new IdentityExpression(0) };
TypeInfo[] typeInfos =
new TypeInfo[] {TypeInfoFactory.timestampTypeInfo};
+ DataTypePhysicalVariation[] dataTypePhysicalVariations =
+ new DataTypePhysicalVariation[] {DataTypePhysicalVariation.NONE};
VectorHashKeyWrapperBatch vhkwb =
VectorHashKeyWrapperBatch.compileKeyWrapperBatch(
keyExpressions,
- typeInfos);
+ typeInfos,
+ dataTypePhysicalVariations);
VectorizedRowBatch batch = new VectorizedRowBatch(1);
batch.selectedInUse = false;
http://git-wip-us.apache.org/repos/asf/hive/blob/470ba3e2/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorRowObject.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorRowObject.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorRowObject.java
index d1efaec..1f2b171 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorRowObject.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorRowObject.java
@@ -58,7 +58,8 @@ public class TestVectorRowObject extends TestCase {
VectorRandomRowSource source = new VectorRandomRowSource();
- source.init(r, VectorRandomRowSource.SupportedTypes.ALL, 4);
+ source.init(r, VectorRandomRowSource.SupportedTypes.ALL, 4,
+ /* allowNulls */ true, /* isUnicodeOk */ true);
VectorizedRowBatchCtx batchContext = new VectorizedRowBatchCtx();
batchContext.init(source.rowStructObjectInspector(), emptyScratchTypeNames);
http://git-wip-us.apache.org/repos/asf/hive/blob/470ba3e2/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java
index 83cdb2d..8b1b612 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorSerDeRow.java
@@ -21,12 +21,15 @@ package org.apache.hadoop.hive.ql.exec.vector;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.List;
import java.util.Properties;
import java.util.Random;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.common.type.HiveChar;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.serde2.ByteStream.Output;
import org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe;
@@ -47,6 +50,9 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.UnionObject;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.fast.SerializeWrite;
+import org.apache.hadoop.hive.serde2.io.HiveCharWritable;
+import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable;
+import org.apache.hadoop.io.Text;
import junit.framework.TestCase;
@@ -147,7 +153,10 @@ public class TestVectorSerDeRow extends TestCase {
VectorRandomRowSource source = new VectorRandomRowSource();
- source.init(r, VectorRandomRowSource.SupportedTypes.ALL, 4, false);
+ // FUTURE: try NULLs and UNICODE.
+ source.init(
+ r, VectorRandomRowSource.SupportedTypes.ALL, 4,
+ /* allowNulls */ false, /* isUnicodeOk */ false);
VectorizedRowBatchCtx batchContext = new VectorizedRowBatchCtx();
batchContext.init(source.rowStructObjectInspector(), emptyScratchTypeNames);
@@ -206,8 +215,100 @@ public class TestVectorSerDeRow extends TestCase {
}
}
+ private String getDifferenceInfo(Object actualRow, Object expectedRow) {
+ if (actualRow instanceof List && expectedRow instanceof List) {
+ List<Object> actualList = (List) actualRow;
+ final int actualSize = actualList.size();
+ List<Object> expectedList = (List) expectedRow;
+ final int expectedSize = expectedList.size();
+ if (actualSize != expectedSize) {
+ return "Actual size " + actualSize + ", expected size " + expectedSize;
+ }
+ for (int i = 0; i < actualSize; i++) {
+ Object actualObject = actualList.get(i);
+ Object expecedObject = expectedList.get(i);
+ if (!actualObject.equals(expecedObject)) {
+ return "Column " + i + " is different";
+ }
+ }
+ } else {
+ if (!actualRow.equals(expectedRow)) {
+ return "Object is different";
+ }
+ }
+ return "Actual and expected row are the same";
+ }
+
+ private String getObjectDisplayString(Object object) {
+ StringBuilder sb = new StringBuilder();
+
+ if (object == null) {
+ sb.append("NULL");
+ } else if (object instanceof Text ||
+ object instanceof HiveChar || object instanceof HiveCharWritable ||
+ object instanceof HiveVarchar || object instanceof HiveVarcharWritable) {
+ final String string;
+ if (object instanceof Text) {
+ Text text = (Text) object;
+ string = text.toString();
+ } else if (object instanceof HiveChar) {
+ HiveChar hiveChar = (HiveChar) object;
+ string = hiveChar.getStrippedValue();
+ } else if (object instanceof HiveCharWritable) {
+ HiveChar hiveChar = ((HiveCharWritable) object).getHiveChar();
+ string = hiveChar.getStrippedValue();
+ } else if (object instanceof HiveVarchar) {
+ HiveVarchar hiveVarchar = (HiveVarchar) object;
+ string = hiveVarchar.getValue();
+ } else if (object instanceof HiveVarcharWritable) {
+ HiveVarchar hiveVarchar = ((HiveVarcharWritable) object).getHiveVarchar();
+ string = hiveVarchar.getValue();
+ } else {
+ throw new RuntimeException("Unexpected");
+ }
+
+ byte[] bytes = string.getBytes();
+ final int byteLength = bytes.length;
+
+ sb.append("'");
+ sb.append(string);
+ sb.append("' (byte length ");
+ sb.append(bytes.length);
+ sb.append(", string length ");
+ sb.append(string.length());
+ sb.append(", bytes ");
+ sb.append(VectorizedBatchUtil.displayBytes(bytes, 0, byteLength));
+ sb.append(")");
+ } else {
+ sb.append(object.toString());
+ }
+ return sb.toString();
+ }
+
+ private String getRowDisplayString(Object row) {
+ StringBuilder sb = new StringBuilder();
+ if (row instanceof List) {
+ List<Object> list = (List) row;
+ final int size = list.size();
+ boolean isFirst = true;
+ for (int i = 0; i < size; i++) {
+ if (isFirst) {
+ isFirst = false;
+ } else {
+ sb.append(", ");
+ }
+ Object object = list.get(i);
+ sb.append(getObjectDisplayString(object));
+ }
+ } else {
+ sb.append(getObjectDisplayString(row));
+ }
+ return sb.toString();
+ }
+
void examineBatch(VectorizedRowBatch batch, VectorExtractRow vectorExtractRow,
- TypeInfo[] typeInfos, Object[][] randomRows, int firstRandomRowIndex ) {
+ TypeInfo[] typeInfos, Object[][] randomRows, int firstRandomRowIndex,
+ String title) {
int rowSize = vectorExtractRow.getCount();
Object[] row = new Object[rowSize];
@@ -228,9 +329,15 @@ public class TestVectorSerDeRow extends TestCase {
" batch index " + i + " firstRandomRowIndex " + firstRandomRowIndex);
}
if (!rowObj.equals(expectedObj)) {
+ String actualValueString = getRowDisplayString(rowObj);
+ String expectedValueString = getRowDisplayString(expectedObj);
+ String differentInfoString = getDifferenceInfo(row, expectedObj);
fail("Row " + (firstRandomRowIndex + i) + " and column " + c + " mismatch (" +
- typeInfos[c].getCategory() + " actual value " + rowObj +
- " and expected value " + expectedObj + ")");
+ typeInfos[c].getCategory() + " actual value '" + actualValueString + "'" +
+ " and expected value '" + expectedValueString + "')" +
+ " difference info " + differentInfoString +
+ " typeInfos " + Arrays.toString(typeInfos) +
+ " title " + title);
}
}
}
@@ -283,19 +390,27 @@ public class TestVectorSerDeRow extends TestCase {
throws HiveException, IOException, SerDeException {
for (int i = 0; i < 20; i++) {
- innerTestVectorDeserializeRow(r, serializationType, alternate1, alternate2, useExternalBuffer);
+ innerTestVectorDeserializeRow(
+ r, i,serializationType, alternate1, alternate2, useExternalBuffer);
}
}
void innerTestVectorDeserializeRow(
- Random r, SerializationType serializationType,
+ Random r, int iteration,
+ SerializationType serializationType,
boolean alternate1, boolean alternate2, boolean useExternalBuffer)
throws HiveException, IOException, SerDeException {
+ String title = "serializationType: " + serializationType + ", iteration " + iteration;
+
String[] emptyScratchTypeNames = new String[0];
VectorRandomRowSource source = new VectorRandomRowSource();
- source.init(r, VectorRandomRowSource.SupportedTypes.ALL, 4, false);
+
+ // FUTURE: try NULLs and UNICODE.
+ source.init(
+ r, VectorRandomRowSource.SupportedTypes.ALL, 4,
+ /* allowNulls */ false, /* isUnicodeOk */ false);
VectorizedRowBatchCtx batchContext = new VectorizedRowBatchCtx();
batchContext.init(source.rowStructObjectInspector(), emptyScratchTypeNames);
@@ -426,13 +541,17 @@ public class TestVectorSerDeRow extends TestCase {
}
batch.size++;
if (batch.size == batch.DEFAULT_SIZE) {
- examineBatch(batch, vectorExtractRow, typeInfos, randomRows, firstRandomRowIndex);
+ examineBatch(
+ batch, vectorExtractRow, typeInfos, randomRows, firstRandomRowIndex,
+ title);
firstRandomRowIndex = i + 1;
batch.reset();
}
}
if (batch.size > 0) {
- examineBatch(batch, vectorExtractRow, typeInfos, randomRows, firstRandomRowIndex);
+ examineBatch(
+ batch, vectorExtractRow, typeInfos, randomRows, firstRandomRowIndex,
+ title);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/470ba3e2/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/VectorRandomRowSource.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/VectorRandomRowSource.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/VectorRandomRowSource.java
index af73ee6..b84273a 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/VectorRandomRowSource.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/VectorRandomRowSource.java
@@ -129,6 +129,7 @@ public class VectorRandomRowSource {
private String[] alphabets;
private boolean allowNull;
+ private boolean isUnicodeOk;
private boolean addEscapables;
private String needsEscapeStr;
@@ -289,26 +290,28 @@ public class VectorRandomRowSource {
ALL, PRIMITIVES, ALL_EXCEPT_MAP
}
- public void init(Random r, SupportedTypes supportedTypes, int maxComplexDepth) {
- init(r, supportedTypes, maxComplexDepth, true);
- }
-
- public void init(Random r, SupportedTypes supportedTypes, int maxComplexDepth, boolean allowNull) {
+ public void init(Random r, SupportedTypes supportedTypes, int maxComplexDepth, boolean allowNull,
+ boolean isUnicodeOk) {
this.r = r;
this.allowNull = allowNull;
+ this.isUnicodeOk = isUnicodeOk;
chooseSchema(supportedTypes, null, null, null, maxComplexDepth);
}
- public void init(Random r, Set<String> allowedTypeNameSet, int maxComplexDepth, boolean allowNull) {
+ public void init(Random r, Set<String> allowedTypeNameSet, int maxComplexDepth, boolean allowNull,
+ boolean isUnicodeOk) {
this.r = r;
this.allowNull = allowNull;
+ this.isUnicodeOk = isUnicodeOk;
chooseSchema(SupportedTypes.ALL, allowedTypeNameSet, null, null, maxComplexDepth);
}
public void initExplicitSchema(Random r, List<String> explicitTypeNameList, int maxComplexDepth,
- boolean allowNull, List<DataTypePhysicalVariation> explicitDataTypePhysicalVariationList) {
+ boolean allowNull, boolean isUnicodeOk,
+ List<DataTypePhysicalVariation> explicitDataTypePhysicalVariationList) {
this.r = r;
this.allowNull = allowNull;
+ this.isUnicodeOk = isUnicodeOk;
List<GenerationSpec> generationSpecList = new ArrayList<GenerationSpec>();
for (String explicitTypeName : explicitTypeNameList) {
@@ -324,9 +327,11 @@ public class VectorRandomRowSource {
}
public void initGenerationSpecSchema(Random r, List<GenerationSpec> generationSpecList, int maxComplexDepth,
- boolean allowNull, List<DataTypePhysicalVariation> explicitDataTypePhysicalVariationList) {
+ boolean allowNull, boolean isUnicodeOk,
+ List<DataTypePhysicalVariation> explicitDataTypePhysicalVariationList) {
this.r = r;
this.allowNull = allowNull;
+ this.isUnicodeOk = isUnicodeOk;
chooseSchema(
SupportedTypes.ALL, null, generationSpecList, explicitDataTypePhysicalVariationList,
maxComplexDepth);
@@ -1009,9 +1014,19 @@ public class VectorRandomRowSource {
PrimitiveTypeInfo[] primitiveTypeInfos,
DataTypePhysicalVariation[] dataTypePhysicalVariations) {
+ return randomPrimitiveRow(
+ columnCount, r, primitiveTypeInfos, dataTypePhysicalVariations, false);
+ }
+
+ public static Object[] randomPrimitiveRow(int columnCount, Random r,
+ PrimitiveTypeInfo[] primitiveTypeInfos,
+ DataTypePhysicalVariation[] dataTypePhysicalVariations, boolean isUnicodeOk) {
+
final Object row[] = new Object[columnCount];
for (int c = 0; c < columnCount; c++) {
- row[c] = randomPrimitiveObject(r, primitiveTypeInfos[c], dataTypePhysicalVariations[c]);
+ row[c] =
+ randomPrimitiveObject(
+ r, primitiveTypeInfos[c], dataTypePhysicalVariations[c], isUnicodeOk);
}
return row;
}
@@ -1624,11 +1639,11 @@ public class VectorRandomRowSource {
}
public static Object randomPrimitiveObject(Random r, PrimitiveTypeInfo primitiveTypeInfo) {
- return randomPrimitiveObject(r, primitiveTypeInfo, DataTypePhysicalVariation.NONE);
+ return randomPrimitiveObject(r, primitiveTypeInfo, DataTypePhysicalVariation.NONE, false);
}
public static Object randomPrimitiveObject(Random r, PrimitiveTypeInfo primitiveTypeInfo,
- DataTypePhysicalVariation dataTypePhysicalVariation) {
+ DataTypePhysicalVariation dataTypePhysicalVariation, boolean isUnicodeOk) {
switch (primitiveTypeInfo.getPrimitiveCategory()) {
case BOOLEAN:
@@ -1648,11 +1663,11 @@ public class VectorRandomRowSource {
case DOUBLE:
return Double.valueOf(r.nextDouble() * 10 - 5);
case STRING:
- return RandomTypeUtil.getRandString(r);
+ return getRandString(r, isUnicodeOk);
case CHAR:
- return getRandHiveChar(r, (CharTypeInfo) primitiveTypeInfo);
+ return getRandHiveChar(r, (CharTypeInfo) primitiveTypeInfo, isUnicodeOk);
case VARCHAR:
- return getRandHiveVarchar(r, (VarcharTypeInfo) primitiveTypeInfo);
+ return getRandHiveVarchar(r, (VarcharTypeInfo) primitiveTypeInfo, isUnicodeOk);
case BINARY:
return getRandBinary(r, 1 + r.nextInt(100));
case TIMESTAMP:
@@ -1682,22 +1697,30 @@ public class VectorRandomRowSource {
return RandomTypeUtil.getRandTimestamp(r).toString();
}
- public static HiveChar getRandHiveChar(Random r, CharTypeInfo charTypeInfo) {
+ public static String getRandString(Random r, boolean isUnicodeOk) {
+ return getRandString(r, r.nextInt(10), isUnicodeOk);
+ }
+
+ public static String getRandString(Random r, int length, boolean isUnicodeOk) {
+ return
+ !isUnicodeOk || r.nextBoolean() ?
+ RandomTypeUtil.getRandString(r, "abcdefghijklmnopqrstuvwxyz", length) :
+ RandomTypeUtil.getRandUnicodeString(r, length);
+ }
+
+ public static HiveChar getRandHiveChar(Random r, CharTypeInfo charTypeInfo, boolean isUnicodeOk) {
final int maxLength = 1 + r.nextInt(charTypeInfo.getLength());
- final String randomString = RandomTypeUtil.getRandString(r, "abcdefghijklmnopqrstuvwxyz", 100);
+ final String randomString = getRandString(r, 100, isUnicodeOk);
return new HiveChar(randomString, maxLength);
}
- public static HiveVarchar getRandHiveVarchar(Random r, VarcharTypeInfo varcharTypeInfo, String alphabet) {
+ public static HiveVarchar getRandHiveVarchar(Random r, VarcharTypeInfo varcharTypeInfo,
+ boolean isUnicodeOk) {
final int maxLength = 1 + r.nextInt(varcharTypeInfo.getLength());
- final String randomString = RandomTypeUtil.getRandString(r, alphabet, 100);
+ final String randomString = getRandString(r, 100, isUnicodeOk);
return new HiveVarchar(randomString, maxLength);
}
- public static HiveVarchar getRandHiveVarchar(Random r, VarcharTypeInfo varcharTypeInfo) {
- return getRandHiveVarchar(r, varcharTypeInfo, "abcdefghijklmnopqrstuvwxyz");
- }
-
public static byte[] getRandBinary(Random r, int len){
final byte[] bytes = new byte[len];
for (int j = 0; j < len; j++){
http://git-wip-us.apache.org/repos/asf/hive/blob/470ba3e2/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/VectorVerifyFast.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/VectorVerifyFast.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/VectorVerifyFast.java
index 458aae8..a0ba0e1 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/VectorVerifyFast.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/VectorVerifyFast.java
@@ -364,9 +364,9 @@ public class VectorVerifyFast {
case STRING:
{
Text value = (Text) object;
- byte[] stringBytes = value.getBytes();
- int stringLength = stringBytes.length;
- serializeWrite.writeString(stringBytes, 0, stringLength);
+ byte[] bytes = value.getBytes();
+ int byteLength = value.getLength();
+ serializeWrite.writeString(bytes, 0, byteLength);
}
break;
case CHAR:
http://git-wip-us.apache.org/repos/asf/hive/blob/470ba3e2/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/aggregation/TestVectorAggregation.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/aggregation/TestVectorAggregation.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/aggregation/TestVectorAggregation.java
index d4ed6b5..211eaa2 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/aggregation/TestVectorAggregation.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/aggregation/TestVectorAggregation.java
@@ -369,7 +369,8 @@ public class TestVectorAggregation extends AggregationBase {
VectorRandomRowSource mergeRowSource = new VectorRandomRowSource();
mergeRowSource.initGenerationSpecSchema(
- random, mergeAggrGenerationSpecList, /* maxComplexDepth */ 0, /* allowNull */ false,
+ random, mergeAggrGenerationSpecList, /* maxComplexDepth */ 0,
+ /* allowNull */ false, /* isUnicodeOk */ true,
mergeDataTypePhysicalVariationList);
Object[][] mergeRandomRows = mergeRowSource.randomRows(TEST_ROW_COUNT);
@@ -508,7 +509,8 @@ public class TestVectorAggregation extends AggregationBase {
boolean allowNull = !aggregationName.equals("bloom_filter");
partial1RowSource.initGenerationSpecSchema(
- random, dataAggrGenerationSpecList, /* maxComplexDepth */ 0, allowNull,
+ random, dataAggrGenerationSpecList, /* maxComplexDepth */ 0,
+ allowNull, /* isUnicodeOk */ true,
explicitDataTypePhysicalVariationList);
Object[][] partial1RandomRows = partial1RowSource.randomRows(TEST_ROW_COUNT);
@@ -604,7 +606,8 @@ public class TestVectorAggregation extends AggregationBase {
VectorRandomRowSource completeRowSource = new VectorRandomRowSource();
completeRowSource.initGenerationSpecSchema(
- random, dataAggrGenerationSpecList, /* maxComplexDepth */ 0, /* allowNull */ true,
+ random, dataAggrGenerationSpecList, /* maxComplexDepth */ 0,
+ /* allowNull */ true, /* isUnicodeOk */ true,
explicitDataTypePhysicalVariationList);
Object[][] completeRandomRows = completeRowSource.randomRows(TEST_ROW_COUNT);
http://git-wip-us.apache.org/repos/asf/hive/blob/470ba3e2/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorArithmetic.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorArithmetic.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorArithmetic.java
index 1b61071..1329d79 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorArithmetic.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorArithmetic.java
@@ -26,9 +26,7 @@ import java.util.List;
import java.util.Random;
import org.apache.hadoop.hive.common.type.DataTypePhysicalVariation;
-import org.apache.hadoop.hive.common.type.HiveChar;
import org.apache.hadoop.hive.common.type.HiveDecimal;
-import org.apache.hadoop.hive.common.type.HiveVarchar;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorFactory;
@@ -59,25 +57,16 @@ import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPMinus;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPMod;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPMultiply;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFOPPlus;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject;
-import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject;
-import org.apache.hadoop.hive.serde2.io.HiveCharWritable;
import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
-import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
-import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
-import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo;
-import org.apache.hadoop.hive.serde2.io.ShortWritable;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.LongWritable;
import junit.framework.Assert;
@@ -437,7 +426,8 @@ public class TestVectorArithmetic {
VectorRandomRowSource rowSource = new VectorRandomRowSource();
rowSource.initGenerationSpecSchema(
- random, generationSpecList, /* maxComplexDepth */ 0, /* allowNull */ true,
+ random, generationSpecList, /* maxComplexDepth */ 0,
+ /* allowNull */ true, /* isUnicodeOk */ true,
explicitDataTypePhysicalVariationList);
Object[][] randomRows = rowSource.randomRows(100000);
http://git-wip-us.apache.org/repos/asf/hive/blob/470ba3e2/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorBetweenIn.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorBetweenIn.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorBetweenIn.java
index 5b69bdf..16bb445 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorBetweenIn.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorBetweenIn.java
@@ -453,7 +453,8 @@ public class TestVectorBetweenIn {
VectorRandomRowSource rowSource = new VectorRandomRowSource();
rowSource.initGenerationSpecSchema(
- random, generationSpecList, /* maxComplexDepth */ 0, /* allowNull */ true,
+ random, generationSpecList, /* maxComplexDepth */ 0,
+ /* allowNull */ true, /* isUnicodeOk */ true,
explicitDataTypePhysicalVariationList);
List<String> columns = new ArrayList<String>();
@@ -575,7 +576,8 @@ public class TestVectorBetweenIn {
VectorRandomRowSource structRowSource = new VectorRandomRowSource();
structRowSource.initGenerationSpecSchema(
- random, structGenerationSpecList, /* maxComplexDepth */ 0, /* allowNull */ true,
+ random, structGenerationSpecList, /* maxComplexDepth */ 0,
+ /* allowNull */ true, /* isUnicodeOk */ true,
structExplicitDataTypePhysicalVariationList);
Object[][] structRandomRows = structRowSource.randomRows(100000);
@@ -597,7 +599,8 @@ public class TestVectorBetweenIn {
VectorRandomRowSource rowSource = new VectorRandomRowSource();
rowSource.initGenerationSpecSchema(
- random, generationSpecList, /* maxComplexDepth */ 0, /* allowNull */ true,
+ random, generationSpecList, /* maxComplexDepth */ 0,
+ /* allowNull */ true, /* isUnicodeOk */ true,
explicitDataTypePhysicalVariationList);
Object[][] randomRows = rowSource.randomRows(100000);
@@ -729,7 +732,7 @@ public class TestVectorBetweenIn {
continue;
}
case VECTOR_EXPRESSION:
- if (!doVectorCastTest(
+ if (!doVectorBetweenInTest(
typeInfo,
betweenInVariation,
compareList,
@@ -866,7 +869,7 @@ public class TestVectorBetweenIn {
}
}
- private boolean doVectorCastTest(TypeInfo typeInfo,
+ private boolean doVectorBetweenInTest(TypeInfo typeInfo,
BetweenInVariation betweenInVariation, List<Object> compareList,
List<String> columns, String[] columnNames,
TypeInfo[] typeInfos, DataTypePhysicalVariation[] dataTypePhysicalVariations,
@@ -899,13 +902,24 @@ public class TestVectorBetweenIn {
VectorExpressionDescriptor.Mode.PROJECTION));
vectorExpression.transientInit();
- if (betweenInTestMode == BetweenInTestMode.VECTOR_EXPRESSION &&
- vectorExpression instanceof VectorUDFAdaptor) {
- System.out.println(
- "*NO NATIVE VECTOR EXPRESSION* typeInfo " + typeInfo.toString() +
- " betweenInTestMode " + betweenInTestMode +
- " betweenInVariation " + betweenInVariation +
- " vectorExpression " + vectorExpression.toString());
+ if (betweenInTestMode == BetweenInTestMode.VECTOR_EXPRESSION) {
+ String vecExprString = vectorExpression.toString();
+ if (vectorExpression instanceof VectorUDFAdaptor) {
+ System.out.println(
+ "*NO NATIVE VECTOR EXPRESSION* typeInfo " + typeInfo.toString() +
+ " betweenInTestMode " + betweenInTestMode +
+ " betweenInVariation " + betweenInVariation +
+ " vectorExpression " + vecExprString);
+ } else if (dataTypePhysicalVariations[0] == DataTypePhysicalVariation.DECIMAL_64) {
+ final String nameToCheck = vectorExpression.getClass().getSimpleName();
+ if (!nameToCheck.contains("Decimal64")) {
+ System.out.println(
+ "*EXPECTED DECIMAL_64 VECTOR EXPRESSION* typeInfo " + typeInfo.toString() +
+ " betweenInTestMode " + betweenInTestMode +
+ " betweenInVariation " + betweenInVariation +
+ " vectorExpression " + vecExprString);
+ }
+ }
}
// System.out.println("*VECTOR EXPRESSION* " + vectorExpression.getClass().getSimpleName());
http://git-wip-us.apache.org/repos/asf/hive/blob/470ba3e2/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorCastStatement.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorCastStatement.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorCastStatement.java
index cc1415a..8a68506 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorCastStatement.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/expressions/TestVectorCastStatement.java
@@ -24,8 +24,6 @@ import java.util.List;
import java.util.Random;
import org.apache.hadoop.hive.common.type.DataTypePhysicalVariation;
-import org.apache.hadoop.hive.common.type.HiveChar;
-import org.apache.hadoop.hive.common.type.HiveVarchar;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluatorFactory;
@@ -48,23 +46,17 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFIf;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFWhen;
-import org.apache.hadoop.hive.serde2.io.DoubleWritable;
-import org.apache.hadoop.hive.serde2.io.HiveCharWritable;
import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
-import org.apache.hadoop.hive.serde2.io.HiveVarcharWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
-import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
-import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo;
import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.io.Text;
import junit.framework.Assert;
@@ -286,7 +278,8 @@ public class TestVectorCastStatement {
VectorRandomRowSource rowSource = new VectorRandomRowSource();
rowSource.initGenerationSpecSchema(
- random, generationSpecList, /* maxComplexDepth */ 0, /* allowNull */ true,
+ random, generationSpecList, /* maxComplexDepth */ 0,
+ /* allowNull */ true, /* isUnicodeOk */ true,
explicitDataTypePhysicalVariationList);
List<String> columns = new ArrayList<String>();