You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by om...@apache.org on 2014/03/26 18:11:33 UTC
svn commit: r1581945 [1/3] - in /hive/branches/branch-0.13: ./ data/files/
ql/src/java/org/apache/hadoop/hive/ql/io/sarg/
ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
Author: omalley
Date: Wed Mar 26 17:11:32 2014
New Revision: 1581945
URL: http://svn.apache.org/r1581945
Log:
HIVE-5567. Add better protection code for SARGs. (omalley)
Modified:
hive/branches/branch-0.13/ (props changed)
hive/branches/branch-0.13/data/files/orc_create_people.txt
hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java
hive/branches/branch-0.13/ql/src/test/queries/clientpositive/orc_analyze.q
hive/branches/branch-0.13/ql/src/test/queries/clientpositive/orc_create.q
hive/branches/branch-0.13/ql/src/test/results/clientpositive/orc_analyze.q.out
hive/branches/branch-0.13/ql/src/test/results/clientpositive/orc_create.q.out
Propchange: hive/branches/branch-0.13/
------------------------------------------------------------------------------
Merged /hive/trunk:r1581924
Modified: hive/branches/branch-0.13/data/files/orc_create_people.txt
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/data/files/orc_create_people.txt?rev=1581945&r1=1581944&r2=1581945&view=diff
==============================================================================
--- hive/branches/branch-0.13/data/files/orc_create_people.txt (original)
+++ hive/branches/branch-0.13/data/files/orc_create_people.txt Wed Mar 26 17:11:32 2014
@@ -1,100 +1,100 @@
-1CelesteBrowning959-3763 Nec, Av.Ca
-2RisaYangP.O. Box 292, 8229 Porttitor RoadOr
-3VenusSuttonAp #962-8021 Egestas Rd.Ca
-4GretchenHarrisonP.O. Box 636, 8734 Magna AvenueOr
-5LaniIrwinAp #441-5911 Iaculis, AveCa
-6VeraGeorge409-1555 Vel, AveOr
-7JessicaMalone286-9779 Aliquam RoadCa
-8AnnChapmanAp #504-3915 Placerat RoadOr
-9NigelBartlettAp #185-385 Diam StreetCa
-10AzaliaJennings5772 Diam St.Or
-11PrestonCannonAp #527-8769 Nunc AvenueCa
-12AllistairVasquez2562 Odio. St.Or
-13ReedHayes5190 Elit StreetCa
-14ElaineBarronP.O. Box 840, 8860 Sodales Av.Or
-15LydiaHoodP.O. Box 698, 5666 Semper RoadCa
-16VanceMaxwell298-3313 Malesuada RoadOr
-17KeikoDeleonP.O. Box 732, 5921 Massa. Av.Ca
-18DolanKaneAp #906-3606 Ut Rd.Or
-19MerrittPerkinsP.O. Box 228, 7090 Egestas StreetCa
-20CaseySalazar506-5065 Ut St.Or
-21SamsonNoel1370 Ultrices, RoadCa
-22ByronWalkerP.O. Box 386, 8324 Tellus AveOr
-23PiperSingletonAp #500-3561 Primis St.Ca
-24RiaMckinney3080 Dui Rd.Or
-25RahimStanley559-9016 Nascetur StreetCa
-26ChloeSteeleP.O. Box 766, 1628 Elit StreetOr
-27PalomaWardAp #390-3042 Ipsum Rd.Ca
-28RoaryShermanAp #409-6549 Metus St.Or
-29CalvinBuckner6378 Diam AvenueCa
-30CamilleGoodAp #113-8659 Suspendisse St.Or
-31SteelAyala5518 Justo St.Ca
-32JosiahGilbertAp #149-6651 At, Av.Or
-33HamiltonCruz4620 Tellus. AveCa
-34ScarletSantos586-1785 Velit. Av.Or
-35LewisMcintyre629-6419 Ac Rd.Ca
-36ArsenioMejiaP.O. Box 767, 8625 Justo Rd.Or
-37VelmaHaley1377 At Rd.Ca
-38TatumJennings829-7432 Posuere, RoadOr
-39BritanniEaton8811 Morbi StreetCa
-40AileenJacobsonP.O. Box 469, 2266 Dui, Rd.Or
-41KareemAyala2706 Ridiculus StreetCa
-42MaiteRush7592 Neque RoadOr
-43SigneVelasquezAp #868-3039 Eget St.Ca
-44ZoritaCamachoP.O. Box 651, 3340 Quis Av.Or
-45GlennaCurtis953-7965 Enim AveCa
-46QuinCortez4898 Ridiculus St.Or
-47TalonDaltonP.O. Box 408, 7597 Integer Rd.Ca
-48DarrylBlankenshipP.O. Box 771, 1471 Non Rd.Or
-49VernonReyesP.O. Box 971, 7009 Vulputate StreetCa
-50TallulahHeathP.O. Box 865, 3697 Dis AveOr
-51CiaranOlson2721 Et St.Ca
-52OrlandoWittP.O. Box 717, 1102 Nulla. Rd.Or
-53QuinnRiceAp #647-6627 Tristique AvenueCa
-54WyattPickettAp #128-3130 Vel, Rd.Or
-55EmeraldCopeland857-5119 Turpis Rd.Ca
-56JonasQuinnAp #441-7183 Ligula. StreetOr
-57WillaBerg6672 Velit AveCa
-58MalikLee998-9208 In StreetOr
-59CallieMedina1620 Dui. Rd.Ca
-60LukeMasonP.O. Box 143, 2070 Augue Rd.Or
-61ShafiraEstrada8824 Ante StreetCa
-62ElizabethRutledge315-6510 Sit St.Or
-63PandoraLevine357-3596 Nibh. AveCa
-64HilelPrince845-1229 Sociosqu Rd.Or
-65RinahTorresAp #492-9328 At St.Ca
-66YaelHobbsP.O. Box 477, 3896 In StreetOr
-67NevadaNashP.O. Box 251, 1914 Tincidunt RoadCa
-68MarnyHuffP.O. Box 818, 6086 Ultricies St.Or
-69KimberleyMilesAp #893-3685 In RoadCa
-70DuncanFullerAp #197-5216 Iaculis StreetOr
-71YardleyLeblancP.O. Box 938, 1278 Sit AveCa
-72HamishBrewerAp #854-781 Quisque St.Or
-73PetraMoon453-6609 Curabitur StreetCa
-74ReeseEstradaAp #382-3313 Malesuada St.Or
-75GageHiggins7443 Eu StreetCa
-76ZacheryCamachoAp #795-4143 Quam. St.Or
-77KellyGarnerP.O. Box 895, 2843 Cras Rd.Ca
-78HanaeCarr9440 Amet St.Or
-79AnnAlston884-7948 Dictum RoadCa
-80ChancellorCobbP.O. Box 889, 5978 Ac AvenueOr
-81DorothyHarrell6974 Tristique AveCa
-82VaughanLeon1610 Luctus Av.Or
-83WynneJimenez321-9171 Felis. AvenueCa
-84WillaMendoza489-182 Sed Av.Or
-85CamdenGoodwin4579 Ante St.Ca
-86IfeomaFrenchP.O. Box 160, 8769 Integer RoadOr
-87RamonaStrong1666 Ridiculus AvenueCa
-88BrettRamosAp #579-9879 Et, RoadOr
-89UllaGray595-7066 Malesuada RoadCa
-90KevynMccallP.O. Box 968, 1420 Aenean AvenueOr
-91GenevieveWilkins908 Turpis. StreetCa
-92ThaneOneil6766 Lectus St.Or
-93MarikoClineP.O. Box 329, 5375 Ac St.Ca
-94LaelMclean500-7010 Sit St.Or
-95WinifredHopperAp #140-8982 Velit AvenueCa
-96RafaelEnglandP.O. Box 405, 7857 Eget Av.Or
-97DanaCarter814-601 Purus. Av.Ca
-98JulietBattleAp #535-1965 Cursus St.Or
-99WynterVincent626-8492 Mollis AvenueCa
-100WangMitchell4023 Lacinia. AveOr
+1CelesteBrowning959-3763 Nec, Av.100.002011-03-12 15:20:00Ca
+2RisaYangP.O. Box 292, 8229 Porttitor Road200.002011-03-12 15:20:00Or
+3VenusSuttonAp #962-8021 Egestas Rd.300.002011-03-12 15:20:00Ca
+4GretchenHarrisonP.O. Box 636, 8734 Magna Avenue400.002011-03-12 15:20:00Or
+5LaniIrwinAp #441-5911 Iaculis, Ave500.002011-03-12 15:20:00Ca
+6VeraGeorge409-1555 Vel, Ave600.002011-03-12 15:20:00Or
+7JessicaMalone286-9779 Aliquam Road700.002011-03-12 15:20:00Ca
+8AnnChapmanAp #504-3915 Placerat Road800.002011-03-12 15:20:00Or
+9NigelBartlettAp #185-385 Diam Street900.002011-03-12 15:20:00Ca
+10AzaliaJennings5772 Diam St.100.002011-03-12 15:20:00Or
+11PrestonCannonAp #527-8769 Nunc Avenue100.002011-03-12 15:20:00Ca
+12AllistairVasquez2562 Odio. St.100.002011-03-12 15:20:00Or
+13ReedHayes5190 Elit Street100.002011-03-12 15:20:00Ca
+14ElaineBarronP.O. Box 840, 8860 Sodales Av.100.002011-03-12 15:20:00Or
+15LydiaHoodP.O. Box 698, 5666 Semper Road100.002011-03-12 15:20:00Ca
+16VanceMaxwell298-3313 Malesuada Road100.002011-03-12 15:20:00Or
+17KeikoDeleonP.O. Box 732, 5921 Massa. Av.100.002011-03-12 15:20:00Ca
+18DolanKaneAp #906-3606 Ut Rd.100.002011-03-12 15:20:00Or
+19MerrittPerkinsP.O. Box 228, 7090 Egestas Street100.002011-03-12 15:20:00Ca
+20CaseySalazar506-5065 Ut St.200.002011-03-12 15:20:00Or
+21SamsonNoel1370 Ultrices, Road200.002012-03-12 15:20:00Ca
+22ByronWalkerP.O. Box 386, 8324 Tellus Ave200.002012-03-12 15:20:00Or
+23PiperSingletonAp #500-3561 Primis St.200.002012-03-12 15:20:00Ca
+24RiaMckinney3080 Dui Rd.200.002012-03-12 15:20:00Or
+25RahimStanley559-9016 Nascetur Street200.002012-03-12 15:20:00Ca
+26ChloeSteeleP.O. Box 766, 1628 Elit Street200.002012-03-12 15:20:00Or
+27PalomaWardAp #390-3042 Ipsum Rd.200.002012-03-12 15:20:00Ca
+28RoaryShermanAp #409-6549 Metus St.200.002012-03-12 15:20:00Or
+29CalvinBuckner6378 Diam Avenue200.002012-03-12 15:20:00Ca
+30CamilleGoodAp #113-8659 Suspendisse St.300.002012-03-12 15:20:00Or
+31SteelAyala5518 Justo St.300.002012-03-12 15:20:00Ca
+32JosiahGilbertAp #149-6651 At, Av.300.002012-03-12 15:20:00Or
+33HamiltonCruz4620 Tellus. Ave300.002012-03-12 15:20:00Ca
+34ScarletSantos586-1785 Velit. Av.300.002012-03-12 15:20:00Or
+35LewisMcintyre629-6419 Ac Rd.300.002012-03-12 15:20:00Ca
+36ArsenioMejiaP.O. Box 767, 8625 Justo Rd.300.002012-03-12 15:20:00Or
+37VelmaHaley1377 At Rd.300.002012-03-12 15:20:00Ca
+38TatumJennings829-7432 Posuere, Road300.002012-03-12 15:20:00Or
+39BritanniEaton8811 Morbi Street300.002012-03-12 15:20:00Ca
+40AileenJacobsonP.O. Box 469, 2266 Dui, Rd.400.002012-03-12 15:20:00Or
+41KareemAyala2706 Ridiculus Street400.002013-03-12 15:20:00Ca
+42MaiteRush7592 Neque Road400.002013-03-12 15:20:00Or
+43SigneVelasquezAp #868-3039 Eget St.400.002013-03-12 15:20:00Ca
+44ZoritaCamachoP.O. Box 651, 3340 Quis Av.400.002013-03-12 15:20:00Or
+45GlennaCurtis953-7965 Enim Ave400.002013-03-12 15:20:00Ca
+46QuinCortez4898 Ridiculus St.400.002013-03-12 15:20:00Or
+47TalonDaltonP.O. Box 408, 7597 Integer Rd.400.002013-03-12 15:20:00Ca
+48DarrylBlankenshipP.O. Box 771, 1471 Non Rd.400.002013-03-12 15:20:00Or
+49VernonReyesP.O. Box 971, 7009 Vulputate Street400.002013-03-12 15:20:00Ca
+50TallulahHeathP.O. Box 865, 3697 Dis Ave500.002013-03-12 15:20:00Or
+51CiaranOlson2721 Et St.500.002013-03-12 15:20:00Ca
+52OrlandoWittP.O. Box 717, 1102 Nulla. Rd.500.002013-03-12 15:20:00Or
+53QuinnRiceAp #647-6627 Tristique Avenue500.002013-03-12 15:20:00Ca
+54WyattPickettAp #128-3130 Vel, Rd.500.002013-03-12 15:20:00Or
+55EmeraldCopeland857-5119 Turpis Rd.500.002013-03-12 15:20:00Ca
+56JonasQuinnAp #441-7183 Ligula. Street500.002013-03-12 15:20:00Or
+57WillaBerg6672 Velit Ave500.002013-03-12 15:20:00Ca
+58MalikLee998-9208 In Street500.002013-03-12 15:20:00Or
+59CallieMedina1620 Dui. Rd.500.002013-03-12 15:20:00Ca
+60LukeMasonP.O. Box 143, 2070 Augue Rd.600.002013-03-12 15:20:00Or
+61ShafiraEstrada8824 Ante Street600.002014-03-12 15:20:00Ca
+62ElizabethRutledge315-6510 Sit St.600.002014-03-12 15:20:00Or
+63PandoraLevine357-3596 Nibh. Ave600.002014-03-12 15:20:00Ca
+64HilelPrince845-1229 Sociosqu Rd.600.002014-03-12 15:20:00Or
+65RinahTorresAp #492-9328 At St.600.002014-03-12 15:20:00Ca
+66YaelHobbsP.O. Box 477, 3896 In Street600.002014-03-12 15:20:00Or
+67NevadaNashP.O. Box 251, 1914 Tincidunt Road600.002014-03-12 15:20:00Ca
+68MarnyHuffP.O. Box 818, 6086 Ultricies St.600.002014-03-12 15:20:00Or
+69KimberleyMilesAp #893-3685 In Road600.002014-03-12 15:20:00Ca
+70DuncanFullerAp #197-5216 Iaculis Street700.002014-03-12 15:20:00Or
+71YardleyLeblancP.O. Box 938, 1278 Sit Ave700.002014-03-12 15:20:00Ca
+72HamishBrewerAp #854-781 Quisque St.700.002014-03-12 15:20:00Or
+73PetraMoon453-6609 Curabitur Street700.002014-03-12 15:20:00Ca
+74ReeseEstradaAp #382-3313 Malesuada St.700.002014-03-12 15:20:00Or
+75GageHiggins7443 Eu Street700.002014-03-12 15:20:00Ca
+76ZacheryCamachoAp #795-4143 Quam. St.700.002014-03-12 15:20:00Or
+77KellyGarnerP.O. Box 895, 2843 Cras Rd.700.002014-03-12 15:20:00Ca
+78HanaeCarr9440 Amet St.700.002014-03-12 15:20:00Or
+79AnnAlston884-7948 Dictum Road700.002014-03-12 15:20:00Ca
+80ChancellorCobbP.O. Box 889, 5978 Ac Avenue800.002014-03-12 15:20:00Or
+81DorothyHarrell6974 Tristique Ave800.002010-03-12 15:20:00Ca
+82VaughanLeon1610 Luctus Av.800.002010-03-12 15:20:00Or
+83WynneJimenez321-9171 Felis. Avenue800.002010-03-12 15:20:00Ca
+84WillaMendoza489-182 Sed Av.800.002010-03-12 15:20:00Or
+85CamdenGoodwin4579 Ante St.800.002010-03-12 15:20:00Ca
+86IfeomaFrenchP.O. Box 160, 8769 Integer Road800.002010-03-12 15:20:00Or
+87RamonaStrong1666 Ridiculus Avenue800.002010-03-12 15:20:00Ca
+88BrettRamosAp #579-9879 Et, Road800.002010-03-12 15:20:00Or
+89UllaGray595-7066 Malesuada Road800.002010-03-12 15:20:00Ca
+90KevynMccallP.O. Box 968, 1420 Aenean Avenue900.002010-03-12 15:20:00Or
+91GenevieveWilkins908 Turpis. Street900.002010-03-12 15:20:00Ca
+92ThaneOneil6766 Lectus St.900.002010-03-12 15:20:00Or
+93MarikoClineP.O. Box 329, 5375 Ac St.900.002010-03-12 15:20:00Ca
+94LaelMclean500-7010 Sit St.900.002010-03-12 15:20:00Or
+95WinifredHopperAp #140-8982 Velit Avenue900.002010-03-12 15:20:00Ca
+96RafaelEnglandP.O. Box 405, 7857 Eget Av.900.002010-03-12 15:20:00Or
+97DanaCarter814-601 Purus. Av.900.002010-03-12 15:20:00Ca
+98JulietBattleAp #535-1965 Cursus St.900.002010-03-12 15:20:00Or
+99WynterVincent626-8492 Mollis Avenue900.002010-03-12 15:20:00Ca
+100WangMitchell4023 Lacinia. Ave100.002010-03-12 15:20:00Or
Modified: hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java?rev=1581945&r1=1581944&r2=1581945&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java (original)
+++ hive/branches/branch-0.13/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java Wed Mar 26 17:11:32 2014
@@ -383,6 +383,9 @@ final class SearchArgumentImpl implement
return new ExpressionTree(TruthValue.YES_NO_NULL);
}
PredicateLeaf.Type type = getType(expression.getChildren().get(variable));
+ if (type == null) {
+ return new ExpressionTree(TruthValue.YES_NO_NULL);
+ }
Object literal = null;
List<Object> literalList = null;
switch (operator) {
Modified: hive/branches/branch-0.13/ql/src/test/queries/clientpositive/orc_analyze.q
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/queries/clientpositive/orc_analyze.q?rev=1581945&r1=1581944&r2=1581945&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/test/queries/clientpositive/orc_analyze.q (original)
+++ hive/branches/branch-0.13/ql/src/test/queries/clientpositive/orc_analyze.q Wed Mar 26 17:11:32 2014
@@ -3,6 +3,8 @@ CREATE TABLE orc_create_people_staging (
first_name string,
last_name string,
address string,
+ salary decimal,
+ start_date timestamp,
state string);
LOAD DATA LOCAL INPATH '../../data/files/orc_create_people.txt' OVERWRITE INTO TABLE orc_create_people_staging;
@@ -17,6 +19,8 @@ CREATE TABLE orc_create_people (
first_name string,
last_name string,
address string,
+ salary decimal,
+ start_date timestamp,
state string)
STORED AS orc;
@@ -35,6 +39,8 @@ CREATE TABLE orc_create_people (
first_name string,
last_name string,
address string,
+ salary decimal,
+ start_date timestamp,
state string)
STORED AS orc;
@@ -51,7 +57,9 @@ CREATE TABLE orc_create_people (
id int,
first_name string,
last_name string,
- address string)
+ address string,
+ salary decimal,
+ start_date timestamp)
PARTITIONED BY (state string)
STORED AS orc;
@@ -71,7 +79,9 @@ CREATE TABLE orc_create_people (
id int,
first_name string,
last_name string,
- address string)
+ address string,
+ salary decimal,
+ start_date timestamp)
PARTITIONED BY (state string)
STORED AS orc;
@@ -90,7 +100,9 @@ CREATE TABLE orc_create_people (
id int,
first_name string,
last_name string,
- address string)
+ address string,
+ salary decimal,
+ start_date timestamp)
PARTITIONED BY (state string)
clustered by (first_name)
sorted by (last_name)
@@ -113,7 +125,9 @@ CREATE TABLE orc_create_people (
id int,
first_name string,
last_name string,
- address string)
+ address string,
+ salary decimal,
+ start_date timestamp)
PARTITIONED BY (state string)
clustered by (first_name)
sorted by (last_name)
@@ -136,16 +150,26 @@ CREATE TABLE orc_create_people (
id int,
first_name string,
last_name string,
- address string)
+ address string,
+ salary decimal,
+ start_date timestamp)
PARTITIONED BY (state string)
STORED AS orc;
INSERT OVERWRITE TABLE orc_create_people PARTITION (state)
SELECT * FROM orc_create_people_staging;
-ALTER TABLE orc_create_people ADD PARTITION(state="OH");
+-- set the table to text format
+ALTER TABLE orc_create_people SET SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe';
+ALTER TABLE orc_create_people SET FILEFORMAT TEXTFILE;
+
+-- load the text data into a new partition
LOAD DATA LOCAL INPATH '../../data/files/kv1.txt' OVERWRITE INTO TABLE orc_create_people PARTITION(state="OH");
+-- set the table back to orc
+ALTER TABLE orc_create_people SET SERDE 'org.apache.hadoop.hive.ql.io.orc.OrcSerde';
+ALTER TABLE orc_create_people SET FILEFORMAT ORC;
+
set hive.stats.autogather = true;
analyze table orc_create_people partition(state) compute statistics noscan;
Modified: hive/branches/branch-0.13/ql/src/test/queries/clientpositive/orc_create.q
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.13/ql/src/test/queries/clientpositive/orc_create.q?rev=1581945&r1=1581944&r2=1581945&view=diff
==============================================================================
--- hive/branches/branch-0.13/ql/src/test/queries/clientpositive/orc_create.q (original)
+++ hive/branches/branch-0.13/ql/src/test/queries/clientpositive/orc_create.q Wed Mar 26 17:11:32 2014
@@ -70,6 +70,8 @@ CREATE TABLE orc_create_people_staging (
first_name string,
last_name string,
address string,
+ salary decimal,
+ start_date timestamp,
state string);
LOAD DATA LOCAL INPATH '../../data/files/orc_create_people.txt'
@@ -79,7 +81,9 @@ CREATE TABLE orc_create_people (
id int,
first_name string,
last_name string,
- address string)
+ address string,
+ salary decimal,
+ start_date timestamp)
PARTITIONED BY (state string)
STORED AS orc;
@@ -99,7 +103,14 @@ SELECT COUNT(*) FROM orc_create_people w
SELECT COUNT(*) FROM orc_create_people where (id + 1) = 20;
SELECT COUNT(*) FROM orc_create_people where (id + 10) < 200;
SELECT COUNT(*) FROM orc_create_people where id < 30 or first_name = "Rafael";
-SELECT COUNT(*) FROM orc_create_people where length(substr(first_name, 1, 2)) <= 2 and last_name like '%';
+SELECT COUNT(*) FROM orc_create_people
+ where length(substr(first_name, 1, 2)) <= 2 and last_name like '%';
+SELECT COUNT(*) FROM orc_create_people where salary = 200.00;
+SELECT COUNT(*) FROM orc_create_people WHERE start_date IS NULL;
+SELECT COUNT(*) FROM orc_create_people WHERE YEAR(start_date) = 2014;
+
+-- test predicate push down with partition pruning
+SELECT COUNT(*) FROM orc_create_people where salary = 200.00 and state = 'Ca';
-- test predicate push down with no column projection
SELECT id, first_name, last_name, address