You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by th...@apache.org on 2013/10/04 09:55:39 UTC
svn commit: r1529098 - in /hive/branches/branch-0.12: data/files/
ql/src/java/org/apache/hadoop/hive/ql/io/orc/
ql/src/test/queries/clientpositive/ ql/src/test/results/clientpositive/
Author: thejas
Date: Fri Oct 4 07:55:39 2013
New Revision: 1529098
URL: http://svn.apache.org/r1529098
Log:
HIVE-5364 : NPE on some queries from partitioned orc table (Owen O'Malley via Gunther Hagleitner)
Added:
hive/branches/branch-0.12/data/files/orc_create_people.txt
Modified:
hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
hive/branches/branch-0.12/ql/src/test/queries/clientpositive/orc_create.q
hive/branches/branch-0.12/ql/src/test/results/clientpositive/orc_create.q.out
Added: hive/branches/branch-0.12/data/files/orc_create_people.txt
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/data/files/orc_create_people.txt?rev=1529098&view=auto
==============================================================================
--- hive/branches/branch-0.12/data/files/orc_create_people.txt (added)
+++ hive/branches/branch-0.12/data/files/orc_create_people.txt Fri Oct 4 07:55:39 2013
@@ -0,0 +1,100 @@
+1CelesteBrowning959-3763 Nec, Av.Ca
+2RisaYangP.O. Box 292, 8229 Porttitor RoadOr
+3VenusSuttonAp #962-8021 Egestas Rd.Ca
+4GretchenHarrisonP.O. Box 636, 8734 Magna AvenueOr
+5LaniIrwinAp #441-5911 Iaculis, AveCa
+6VeraGeorge409-1555 Vel, AveOr
+7JessicaMalone286-9779 Aliquam RoadCa
+8AnnChapmanAp #504-3915 Placerat RoadOr
+9NigelBartlettAp #185-385 Diam StreetCa
+10AzaliaJennings5772 Diam St.Or
+11PrestonCannonAp #527-8769 Nunc AvenueCa
+12AllistairVasquez2562 Odio. St.Or
+13ReedHayes5190 Elit StreetCa
+14ElaineBarronP.O. Box 840, 8860 Sodales Av.Or
+15LydiaHoodP.O. Box 698, 5666 Semper RoadCa
+16VanceMaxwell298-3313 Malesuada RoadOr
+17KeikoDeleonP.O. Box 732, 5921 Massa. Av.Ca
+18DolanKaneAp #906-3606 Ut Rd.Or
+19MerrittPerkinsP.O. Box 228, 7090 Egestas StreetCa
+20CaseySalazar506-5065 Ut St.Or
+21SamsonNoel1370 Ultrices, RoadCa
+22ByronWalkerP.O. Box 386, 8324 Tellus AveOr
+23PiperSingletonAp #500-3561 Primis St.Ca
+24RiaMckinney3080 Dui Rd.Or
+25RahimStanley559-9016 Nascetur StreetCa
+26ChloeSteeleP.O. Box 766, 1628 Elit StreetOr
+27PalomaWardAp #390-3042 Ipsum Rd.Ca
+28RoaryShermanAp #409-6549 Metus St.Or
+29CalvinBuckner6378 Diam AvenueCa
+30CamilleGoodAp #113-8659 Suspendisse St.Or
+31SteelAyala5518 Justo St.Ca
+32JosiahGilbertAp #149-6651 At, Av.Or
+33HamiltonCruz4620 Tellus. AveCa
+34ScarletSantos586-1785 Velit. Av.Or
+35LewisMcintyre629-6419 Ac Rd.Ca
+36ArsenioMejiaP.O. Box 767, 8625 Justo Rd.Or
+37VelmaHaley1377 At Rd.Ca
+38TatumJennings829-7432 Posuere, RoadOr
+39BritanniEaton8811 Morbi StreetCa
+40AileenJacobsonP.O. Box 469, 2266 Dui, Rd.Or
+41KareemAyala2706 Ridiculus StreetCa
+42MaiteRush7592 Neque RoadOr
+43SigneVelasquezAp #868-3039 Eget St.Ca
+44ZoritaCamachoP.O. Box 651, 3340 Quis Av.Or
+45GlennaCurtis953-7965 Enim AveCa
+46QuinCortez4898 Ridiculus St.Or
+47TalonDaltonP.O. Box 408, 7597 Integer Rd.Ca
+48DarrylBlankenshipP.O. Box 771, 1471 Non Rd.Or
+49VernonReyesP.O. Box 971, 7009 Vulputate StreetCa
+50TallulahHeathP.O. Box 865, 3697 Dis AveOr
+51CiaranOlson2721 Et St.Ca
+52OrlandoWittP.O. Box 717, 1102 Nulla. Rd.Or
+53QuinnRiceAp #647-6627 Tristique AvenueCa
+54WyattPickettAp #128-3130 Vel, Rd.Or
+55EmeraldCopeland857-5119 Turpis Rd.Ca
+56JonasQuinnAp #441-7183 Ligula. StreetOr
+57WillaBerg6672 Velit AveCa
+58MalikLee998-9208 In StreetOr
+59CallieMedina1620 Dui. Rd.Ca
+60LukeMasonP.O. Box 143, 2070 Augue Rd.Or
+61ShafiraEstrada8824 Ante StreetCa
+62ElizabethRutledge315-6510 Sit St.Or
+63PandoraLevine357-3596 Nibh. AveCa
+64HilelPrince845-1229 Sociosqu Rd.Or
+65RinahTorresAp #492-9328 At St.Ca
+66YaelHobbsP.O. Box 477, 3896 In StreetOr
+67NevadaNashP.O. Box 251, 1914 Tincidunt RoadCa
+68MarnyHuffP.O. Box 818, 6086 Ultricies St.Or
+69KimberleyMilesAp #893-3685 In RoadCa
+70DuncanFullerAp #197-5216 Iaculis StreetOr
+71YardleyLeblancP.O. Box 938, 1278 Sit AveCa
+72HamishBrewerAp #854-781 Quisque St.Or
+73PetraMoon453-6609 Curabitur StreetCa
+74ReeseEstradaAp #382-3313 Malesuada St.Or
+75GageHiggins7443 Eu StreetCa
+76ZacheryCamachoAp #795-4143 Quam. St.Or
+77KellyGarnerP.O. Box 895, 2843 Cras Rd.Ca
+78HanaeCarr9440 Amet St.Or
+79AnnAlston884-7948 Dictum RoadCa
+80ChancellorCobbP.O. Box 889, 5978 Ac AvenueOr
+81DorothyHarrell6974 Tristique AveCa
+82VaughanLeon1610 Luctus Av.Or
+83WynneJimenez321-9171 Felis. AvenueCa
+84WillaMendoza489-182 Sed Av.Or
+85CamdenGoodwin4579 Ante St.Ca
+86IfeomaFrenchP.O. Box 160, 8769 Integer RoadOr
+87RamonaStrong1666 Ridiculus AvenueCa
+88BrettRamosAp #579-9879 Et, RoadOr
+89UllaGray595-7066 Malesuada RoadCa
+90KevynMccallP.O. Box 968, 1420 Aenean AvenueOr
+91GenevieveWilkins908 Turpis. StreetCa
+92ThaneOneil6766 Lectus St.Or
+93MarikoClineP.O. Box 329, 5375 Ac St.Ca
+94LaelMclean500-7010 Sit St.Or
+95WinifredHopperAp #140-8982 Velit AvenueCa
+96RafaelEnglandP.O. Box 405, 7857 Eget Av.Or
+97DanaCarter814-601 Purus. Av.Ca
+98JulietBattleAp #535-1965 Cursus St.Or
+99WynterVincent626-8492 Mollis AvenueCa
+100WangMitchell4023 Lacinia. AveOr
Modified: hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java?rev=1529098&r1=1529097&r2=1529098&view=diff
==============================================================================
--- hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java (original)
+++ hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java Fri Oct 4 07:55:39 2013
@@ -88,7 +88,7 @@ public class OrcInputFormat extends Fil
String[] neededColumnNames = columnNamesString.split(",");
int i = 0;
for(int columnId: types.get(0).getSubtypesList()) {
- if (includeColumn[columnId]) {
+ if (includeColumn == null || includeColumn[columnId]) {
columnNames[columnId] = neededColumnNames[i++];
}
}
Modified: hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java?rev=1529098&r1=1529097&r2=1529098&view=diff
==============================================================================
--- hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java (original)
+++ hive/branches/branch-0.12/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java Fri Oct 4 07:55:39 2013
@@ -1706,13 +1706,18 @@ class RecordReaderImpl implements Record
TruthValue[] leafValues = new TruthValue[sargLeaves.size()];
for(int rowGroup=0; rowGroup < result.length; ++rowGroup) {
for(int pred=0; pred < leafValues.length; ++pred) {
- OrcProto.ColumnStatistics stats =
- indexes[filterColumns[pred]].getEntry(rowGroup).getStatistics();
- leafValues[pred] = evaluatePredicate(stats, sargLeaves.get(pred));
- if (LOG.isDebugEnabled()) {
- LOG.debug("Stats = " + stats);
- LOG.debug("Setting " + sargLeaves.get(pred) + " to " +
- leafValues[pred]);
+ if (filterColumns[pred] != -1) {
+ OrcProto.ColumnStatistics stats =
+ indexes[filterColumns[pred]].getEntry(rowGroup).getStatistics();
+ leafValues[pred] = evaluatePredicate(stats, sargLeaves.get(pred));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Stats = " + stats);
+ LOG.debug("Setting " + sargLeaves.get(pred) + " to " +
+ leafValues[pred]);
+ }
+ } else {
+ // the column is a virtual column
+ leafValues[pred] = TruthValue.YES_NO_NULL;
}
}
result[rowGroup] = sarg.evaluate(leafValues).isNotNeeded();
Modified: hive/branches/branch-0.12/ql/src/test/queries/clientpositive/orc_create.q
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/ql/src/test/queries/clientpositive/orc_create.q?rev=1529098&r1=1529097&r2=1529098&view=diff
==============================================================================
--- hive/branches/branch-0.12/ql/src/test/queries/clientpositive/orc_create.q (original)
+++ hive/branches/branch-0.12/ql/src/test/queries/clientpositive/orc_create.q Fri Oct 4 07:55:39 2013
@@ -1,6 +1,8 @@
DROP TABLE orc_create;
DROP TABLE orc_create_complex;
DROP TABLE orc_create_staging;
+DROP TABLE orc_create_people_staging;
+DROP TABLE orc_create_people;
CREATE TABLE orc_create_staging (
str STRING,
@@ -38,6 +40,8 @@ set hive.default.fileformat=orc;
CREATE TABLE orc_create (key INT, value STRING)
PARTITIONED BY (ds string);
+set hive.default.fileformat=text;
+
DESCRIBE FORMATTED orc_create;
CREATE TABLE orc_create_complex (
@@ -61,6 +65,39 @@ SELECT mp from orc_create_complex;
SELECT lst from orc_create_complex;
SELECT strct from orc_create_complex;
+CREATE TABLE orc_create_people_staging (
+ id int,
+ first_name string,
+ last_name string,
+ address string,
+ state string);
+
+LOAD DATA LOCAL INPATH '../data/files/orc_create_people.txt'
+ OVERWRITE INTO TABLE orc_create_people_staging;
+
+CREATE TABLE orc_create_people (
+ id int,
+ first_name string,
+ last_name string,
+ address string)
+PARTITIONED BY (state string)
+STORED AS orc;
+
+set hive.exec.dynamic.partition.mode=nonstrict;
+
+INSERT OVERWRITE TABLE orc_create_people PARTITION (state)
+ SELECT * FROM orc_create_people_staging;
+
+SET hive.optimize.index.filter=true;
+-- test predicate push down with partition pruning
+SELECT COUNT(*) FROM orc_create_people where id < 10 and state = 'Ca';
+
+-- test predicate push down with no column projection
+SELECT id, first_name, last_name, address
+ FROM orc_create_people WHERE id > 90;
+
DROP TABLE orc_create;
DROP TABLE orc_create_complex;
DROP TABLE orc_create_staging;
+DROP TABLE orc_create_people_staging;
+DROP TABLE orc_create_people;
Modified: hive/branches/branch-0.12/ql/src/test/results/clientpositive/orc_create.q.out
URL: http://svn.apache.org/viewvc/hive/branches/branch-0.12/ql/src/test/results/clientpositive/orc_create.q.out?rev=1529098&r1=1529097&r2=1529098&view=diff
==============================================================================
--- hive/branches/branch-0.12/ql/src/test/results/clientpositive/orc_create.q.out (original)
+++ hive/branches/branch-0.12/ql/src/test/results/clientpositive/orc_create.q.out Fri Oct 4 07:55:39 2013
@@ -10,6 +10,14 @@ PREHOOK: query: DROP TABLE orc_create_st
PREHOOK: type: DROPTABLE
POSTHOOK: query: DROP TABLE orc_create_staging
POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE orc_create_people_staging
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE orc_create_people_staging
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE orc_create_people
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE orc_create_people
+POSTHOOK: type: DROPTABLE
PREHOOK: query: CREATE TABLE orc_create_staging (
str STRING,
mp MAP<STRING,STRING>,
@@ -398,6 +406,144 @@ POSTHOOK: Lineage: orc_create_complex.st
{"a":"one","b":"two"}
{"a":"three","b":"four"}
{"a":"five","b":"six"}
+PREHOOK: query: CREATE TABLE orc_create_people_staging (
+ id int,
+ first_name string,
+ last_name string,
+ address string,
+ state string)
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE orc_create_people_staging (
+ id int,
+ first_name string,
+ last_name string,
+ address string,
+ state string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@orc_create_people_staging
+POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ]
+PREHOOK: query: LOAD DATA LOCAL INPATH '../data/files/orc_create_people.txt'
+ OVERWRITE INTO TABLE orc_create_people_staging
+PREHOOK: type: LOAD
+PREHOOK: Output: default@orc_create_people_staging
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../data/files/orc_create_people.txt'
+ OVERWRITE INTO TABLE orc_create_people_staging
+POSTHOOK: type: LOAD
+POSTHOOK: Output: default@orc_create_people_staging
+POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ]
+PREHOOK: query: CREATE TABLE orc_create_people (
+ id int,
+ first_name string,
+ last_name string,
+ address string)
+PARTITIONED BY (state string)
+STORED AS orc
+PREHOOK: type: CREATETABLE
+POSTHOOK: query: CREATE TABLE orc_create_people (
+ id int,
+ first_name string,
+ last_name string,
+ address string)
+PARTITIONED BY (state string)
+STORED AS orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: default@orc_create_people
+POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ]
+PREHOOK: query: INSERT OVERWRITE TABLE orc_create_people PARTITION (state)
+ SELECT * FROM orc_create_people_staging
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_create_people_staging
+PREHOOK: Output: default@orc_create_people
+POSTHOOK: query: INSERT OVERWRITE TABLE orc_create_people PARTITION (state)
+ SELECT * FROM orc_create_people_staging
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_create_people_staging
+POSTHOOK: Output: default@orc_create_people@state=Ca
+POSTHOOK: Output: default@orc_create_people@state=Or
+POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Ca).address SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:address, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Ca).first_name SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:first_name, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Ca).id SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:id, type:int, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Ca).last_name SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:last_name, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Or).address SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:address, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Or).first_name SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:first_name, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Or).id SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:id, type:int, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Or).last_name SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:last_name, type:string, comment:null), ]
+PREHOOK: query: -- test predicate push down with partition pruning
+SELECT COUNT(*) FROM orc_create_people where id < 10 and state = 'Ca'
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_create_people
+PREHOOK: Input: default@orc_create_people@state=Ca
+#### A masked pattern was here ####
+POSTHOOK: query: -- test predicate push down with partition pruning
+SELECT COUNT(*) FROM orc_create_people where id < 10 and state = 'Ca'
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_create_people
+POSTHOOK: Input: default@orc_create_people@state=Ca
+#### A masked pattern was here ####
+POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Ca).address SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:address, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Ca).first_name SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:first_name, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Ca).id SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:id, type:int, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Ca).last_name SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:last_name, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Or).address SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:address, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Or).first_name SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:first_name, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Or).id SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:id, type:int, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Or).last_name SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:last_name, type:string, comment:null), ]
+5
+PREHOOK: query: -- test predicate push down with no column projection
+SELECT id, first_name, last_name, address
+ FROM orc_create_people WHERE id > 90
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orc_create_people
+PREHOOK: Input: default@orc_create_people@state=Ca
+PREHOOK: Input: default@orc_create_people@state=Or
+#### A masked pattern was here ####
+POSTHOOK: query: -- test predicate push down with no column projection
+SELECT id, first_name, last_name, address
+ FROM orc_create_people WHERE id > 90
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orc_create_people
+POSTHOOK: Input: default@orc_create_people@state=Ca
+POSTHOOK: Input: default@orc_create_people@state=Or
+#### A masked pattern was here ####
+POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Ca).address SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:address, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Ca).first_name SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:first_name, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Ca).id SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:id, type:int, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Ca).last_name SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:last_name, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Or).address SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:address, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Or).first_name SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:first_name, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Or).id SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:id, type:int, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Or).last_name SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:last_name, type:string, comment:null), ]
+91 Genevieve Wilkins 908 Turpis. Street
+93 Mariko Cline P.O. Box 329, 5375 Ac St.
+95 Winifred Hopper Ap #140-8982 Velit Avenue
+97 Dana Carter 814-601 Purus. Av.
+99 Wynter Vincent 626-8492 Mollis Avenue
+92 Thane Oneil 6766 Lectus St.
+94 Lael Mclean 500-7010 Sit St.
+96 Rafael England P.O. Box 405, 7857 Eget Av.
+98 Juliet Battle Ap #535-1965 Cursus St.
+100 Wang Mitchell 4023 Lacinia. Ave
PREHOOK: query: DROP TABLE orc_create
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@orc_create
@@ -410,6 +556,14 @@ POSTHOOK: Lineage: orc_create_complex.ls
POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ]
POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str, type:string, comment:null), ]
POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Ca).address SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:address, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Ca).first_name SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:first_name, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Ca).id SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:id, type:int, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Ca).last_name SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:last_name, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Or).address SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:address, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Or).first_name SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:first_name, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Or).id SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:id, type:int, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Or).last_name SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:last_name, type:string, comment:null), ]
PREHOOK: query: DROP TABLE orc_create_complex
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@orc_create_complex
@@ -422,6 +576,14 @@ POSTHOOK: Lineage: orc_create_complex.ls
POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ]
POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str, type:string, comment:null), ]
POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Ca).address SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:address, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Ca).first_name SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:first_name, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Ca).id SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:id, type:int, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Ca).last_name SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:last_name, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Or).address SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:address, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Or).first_name SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:first_name, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Or).id SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:id, type:int, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Or).last_name SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:last_name, type:string, comment:null), ]
PREHOOK: query: DROP TABLE orc_create_staging
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@orc_create_staging
@@ -434,3 +596,51 @@ POSTHOOK: Lineage: orc_create_complex.ls
POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ]
POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str, type:string, comment:null), ]
POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Ca).address SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:address, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Ca).first_name SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:first_name, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Ca).id SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:id, type:int, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Ca).last_name SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:last_name, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Or).address SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:address, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Or).first_name SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:first_name, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Or).id SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:id, type:int, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Or).last_name SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:last_name, type:string, comment:null), ]
+PREHOOK: query: DROP TABLE orc_create_people_staging
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@orc_create_people_staging
+PREHOOK: Output: default@orc_create_people_staging
+POSTHOOK: query: DROP TABLE orc_create_people_staging
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@orc_create_people_staging
+POSTHOOK: Output: default@orc_create_people_staging
+POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Ca).address SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:address, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Ca).first_name SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:first_name, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Ca).id SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:id, type:int, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Ca).last_name SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:last_name, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Or).address SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:address, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Or).first_name SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:first_name, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Or).id SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:id, type:int, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Or).last_name SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:last_name, type:string, comment:null), ]
+PREHOOK: query: DROP TABLE orc_create_people
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@orc_create_people
+PREHOOK: Output: default@orc_create_people
+POSTHOOK: query: DROP TABLE orc_create_people
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@orc_create_people
+POSTHOOK: Output: default@orc_create_people
+POSTHOOK: Lineage: orc_create_complex.lst SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:lst, type:array<string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.mp SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:mp, type:map<string,string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.str SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:str, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_complex.strct SIMPLE [(orc_create_staging)orc_create_staging.FieldSchema(name:strct, type:struct<A:string,B:string>, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Ca).address SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:address, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Ca).first_name SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:first_name, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Ca).id SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:id, type:int, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Ca).last_name SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:last_name, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Or).address SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:address, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Or).first_name SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:first_name, type:string, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Or).id SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:id, type:int, comment:null), ]
+POSTHOOK: Lineage: orc_create_people PARTITION(state=Or).last_name SIMPLE [(orc_create_people_staging)orc_create_people_staging.FieldSchema(name:last_name, type:string, comment:null), ]