You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by am...@apache.org on 2013/05/27 07:27:16 UTC
svn commit: r1486517 [1/2] - in /hive/branches/HIVE-4115: ./
cli/src/java/org/apache/hadoop/hive/cli/
common/src/java/org/apache/hadoop/hive/conf/ conf/ data/conf/
eclipse-templates/ hcatalog/src/test/e2e/hcatalog/
hcatalog/src/test/e2e/hcatalog/resour...
Author: amareshwari
Date: Mon May 27 05:27:15 2013
New Revision: 1486517
URL: http://svn.apache.org/r1486517
Log:
Merging r1480540 through r1486516 into HIVE-4115
Added:
hive/branches/HIVE-4115/hcatalog/src/test/e2e/hcatalog/resource/
- copied from r1486516, hive/trunk/hcatalog/src/test/e2e/hcatalog/resource/
hive/branches/HIVE-4115/hcatalog/src/test/e2e/hcatalog/resource/default.res
- copied unchanged from r1486516, hive/trunk/hcatalog/src/test/e2e/hcatalog/resource/default.res
hive/branches/HIVE-4115/hcatalog/src/test/e2e/hcatalog/resource/windows.res
- copied unchanged from r1486516, hive/trunk/hcatalog/src/test/e2e/hcatalog/resource/windows.res
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorHead.java
- copied unchanged from r1486516, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorHead.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorRef.java
- copied unchanged from r1486516, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorRef.java
hive/branches/HIVE-4115/ql/src/test/queries/clientpositive/auto_join32.q
- copied unchanged from r1486516, hive/trunk/ql/src/test/queries/clientpositive/auto_join32.q
hive/branches/HIVE-4115/ql/src/test/queries/clientpositive/remote_script.q
- copied unchanged from r1486516, hive/trunk/ql/src/test/queries/clientpositive/remote_script.q
hive/branches/HIVE-4115/ql/src/test/results/clientpositive/auto_join32.q.out
- copied unchanged from r1486516, hive/trunk/ql/src/test/results/clientpositive/auto_join32.q.out
hive/branches/HIVE-4115/ql/src/test/results/clientpositive/remote_script.q.out
- copied unchanged from r1486516, hive/trunk/ql/src/test/results/clientpositive/remote_script.q.out
Modified:
hive/branches/HIVE-4115/ (props changed)
hive/branches/HIVE-4115/RELEASE_NOTES.txt
hive/branches/HIVE-4115/build-common.xml
hive/branches/HIVE-4115/build.xml
hive/branches/HIVE-4115/cli/src/java/org/apache/hadoop/hive/cli/CliSessionState.java
hive/branches/HIVE-4115/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/branches/HIVE-4115/conf/hive-default.xml.template
hive/branches/HIVE-4115/data/conf/hive-site.xml
hive/branches/HIVE-4115/eclipse-templates/.classpath
hive/branches/HIVE-4115/hcatalog/src/test/e2e/hcatalog/build.xml
hive/branches/HIVE-4115/hcatalog/src/test/e2e/hcatalog/tests/hcat.conf
hive/branches/HIVE-4115/jdbc/src/java/org/apache/hive/jdbc/HiveDriver.java
hive/branches/HIVE-4115/jdbc/src/test/org/apache/hive/jdbc/TestJdbcDriver2.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeColumnEvaluator.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeConstantEvaluator.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluator.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorFactory.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeFieldEvaluator.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeNullEvaluator.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AvgPartitionSizeBasedBigTableSelectorForAutoSMJ.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/optimizer/TableSizeBasedBigTableSelectorForAutoSMJ.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/CommonJoinTaskDispatcher.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/session/SessionState.java
hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDF.java
hive/branches/HIVE-4115/ql/src/test/org/apache/hadoop/hive/ql/QTestUtil.java
hive/branches/HIVE-4115/ql/src/test/queries/clientpositive/rcfile_default_format.q
hive/branches/HIVE-4115/ql/src/test/results/clientnegative/local_mapred_error_cache.q.out
hive/branches/HIVE-4115/ql/src/test/results/clientpositive/rcfile_default_format.q.out
hive/branches/HIVE-4115/service/src/java/org/apache/hive/service/auth/TUGIContainingProcessor.java
hive/branches/HIVE-4115/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java
Propchange: hive/branches/HIVE-4115/
------------------------------------------------------------------------------
Merged /hive/branches/branch-0.11:r1481120,1481344,1481346,1481348,1481352,1483872
Merged /hive/trunk:r1480540-1486516
Modified: hive/branches/HIVE-4115/RELEASE_NOTES.txt
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/RELEASE_NOTES.txt?rev=1486517&r1=1486516&r2=1486517&view=diff
==============================================================================
--- hive/branches/HIVE-4115/RELEASE_NOTES.txt (original)
+++ hive/branches/HIVE-4115/RELEASE_NOTES.txt Mon May 27 05:27:15 2013
@@ -1,3 +1,385 @@
+Release Notes - Hive - Version 0.11.0
+
+** Sub-task
+ * [HIVE-2340] - optimize orderby followed by a groupby
+ * [HIVE-4075] - TypeInfoFactory is not thread safe and is access by multiple threads
+ * [HIVE-4141] - InspectorFactories contains static HashMaps which can cause infinite loop
+ * [HIVE-4162] - disable TestBeeLineDriver
+ * [HIVE-4176] - disable TestBeeLineDriver in ptest util
+ * [HIVE-4238] - Integrate HCatalog site into Hive site
+ * [HIVE-4263] - Adjust build.xml package command to move all hcat jars and binaries into build
+ * [HIVE-4264] - Move HCatalog trunk code from trunk/hcatalog/historical to trunk/hcatalog
+ * [HIVE-4265] - HCatalog branches need to move out of trunk/hcatalog/historical
+ * [HIVE-4278] - HCat needs to get current Hive jars instead of pulling them from maven repo
+ * [HIVE-4325] - Merge HCat NOTICE file with Hive NOTICE file
+ * [HIVE-4326] - Clean up remaining items in hive/hcatalog/historical/trunk
+
+** Bug
+ * [HIVE-2264] - Hive server is SHUTTING DOWN when invalid queries beeing executed.
+ * [HIVE-2332] - If all of the parameters of distinct functions are exists in group by columns, query fails in runtime
+ * [HIVE-2689] - ObjectInspectorConverters cannot convert Void types to Array/Map/Struct types.
+ * [HIVE-2723] - should throw "Ambiguous column reference key" Exception in particular join condition
+ * [HIVE-2794] - Aggregations without grouping should return NULL when applied to partitioning column of a partitionless table
+ * [HIVE-2820] - Invalid tag is used for MapJoinProcessor
+ * [HIVE-2839] - Filters on outer join with mapjoin hint is not applied correctly
+ * [HIVE-3084] - Hive CI failing due to script_broken_pipe1.q
+ * [HIVE-3140] - Comment indenting is broken for "describe" in CLI
+ * [HIVE-3179] - HBase Handler doesn't handle NULLs properly
+ * [HIVE-3197] - Hive compile errors under Java 7 (JDBC 4.1)
+ * [HIVE-3297] - change hive.auto.convert.join's default value to true
+ * [HIVE-3300] - LOAD DATA INPATH fails if a hdfs file with same name is added to table
+ * [HIVE-3308] - Mixing avro and snappy gives null values
+ * [HIVE-3348] - semi-colon in comments in .q file does not work
+ * [HIVE-3381] - Result of outer join is not valid
+ * [HIVE-3384] - HIVE JDBC module won't compile under JDK1.7 as new methods added in JDBC specification
+ * [HIVE-3403] - user should not specify mapjoin to perform sort-merge bucketed join
+ * [HIVE-3428] - Fix log4j configuration errors when running hive on hadoop23
+ * [HIVE-3446] - PrimitiveObjectInspector doesn't handle timestamps properly
+ * [HIVE-3464] - Merging join tree may reorder joins which could be invalid
+ * [HIVE-3490] - Implement * or a.* for arguments to UDFs
+ * [HIVE-3528] - Avro SerDe doesn't handle serializing Nullable types that require access to a Schema
+ * [HIVE-3537] - release locks at the end of move tasks
+ * [HIVE-3582] - NPE in union processing followed by lateral view followed by 2 group bys
+ * [HIVE-3594] - When Group by Partition Column Type is Timestamp or STRING Which Format contains "HH:MM:SS", It will occur URISyntaxException
+ * [HIVE-3622] - reflect udf cannot find method which has arguments of primitive types and String, Binary, Timestamp types mixed
+ * [HIVE-3631] - script_pipe.q fails when using JDK7
+ * [HIVE-3645] - RCFileWriter does not implement the right function to support Federation
+ * [HIVE-3648] - HiveMetaStoreFsImpl is not compatible with hadoop viewfs
+ * [HIVE-3665] - Allow URIs without port to be specified in metatool
+ * [HIVE-3697] - External JAR files on HDFS can lead to race condition with hive.downloaded.resources.dir
+ * [HIVE-3698] - enhanceModel.notRequired is incorrectly determined
+ * [HIVE-3699] - Multiple insert overwrite into multiple tables query stores same results in all tables
+ * [HIVE-3702] - Renaming table changes table location scheme/authority
+ * [HIVE-3703] - Hive Query Explain Plan JSON not being created properly
+ * [HIVE-3714] - Patch: Hive's ivy internal resolvers need to use sourceforge for sqlline
+ * [HIVE-3717] - Hive won't compile with -Dhadoop.mr.rev=20S
+ * [HIVE-3728] - make optimizing multi-group by configurable
+ * [HIVE-3729] - Error in groupSetExpression rule in Hive grammar
+ * [HIVE-3735] - PTest doesn't work due to hive snapshot version upgrade to 11
+ * [HIVE-3741] - Driver.validateConfVariables() should perform more validations
+ * [HIVE-3747] - Provide hive operation name for hookContext
+ * [HIVE-3750] - JDBCStatsPublisher fails when ID length exceeds length of ID column
+ * [HIVE-3757] - union_remove_9.q fails in trunk (hadoop 23)
+ * [HIVE-3760] - TestNegativeMinimrCliDriver_mapreduce_stack_trace.q fails on hadoop-1
+ * [HIVE-3766] - Enable adding hooks to hive meta store init
+ * [HIVE-3767] - BucketizedHiveInputFormat should be automatically used with Bucketized Map Joins also
+ * [HIVE-3771] - HIVE-3750 broke TestParse
+ * [HIVE-3774] - Sort merge join should work if join cols are a prefix of sort columns for each partition
+ * [HIVE-3775] - Unit test failures due to unspecified order of results in "show grant" command
+ * [HIVE-3778] - Add MapJoinDesc.isBucketMapJoin() as part of explain plan
+ * [HIVE-3782] - testCliDriver_sample_islocalmode_hook fails on hadoop-1
+ * [HIVE-3783] - stats19.q is failing on trunk
+ * [HIVE-3787] - Regression introduced from HIVE-3401
+ * [HIVE-3788] - testCliDriver_repair fails on hadoop-1
+ * [HIVE-3789] - Patch HIVE-3648 causing the majority of unit tests to fail on branch 0.9
+ * [HIVE-3795] - NPE in SELECT when WHERE-clause is an and/or/not operation involving null
+ * [HIVE-3800] - testCliDriver_combine2 fails on hadoop-1
+ * [HIVE-3801] - testCliDriver_loadpart_err fails on hadoop-1
+ * [HIVE-3802] - testCliDriver_input39 fails on hadoop-1
+ * [HIVE-3803] - explain dependency should show the dependencies hierarchically in presence of views
+ * [HIVE-3806] - Ptest failing due to "Argument list too long" errors
+ * [HIVE-3809] - Concurrency issue in RCFile: multiple threads can use the same decompressor
+ * [HIVE-3817] - Adding the name space for the maven task for the maven-publish target.
+ * [HIVE-3820] - Consider creating a literal like "D" or "BD" for representing Decimal type constants
+ * [HIVE-3824] - bug if different serdes are used for different partitions
+ * [HIVE-3826] - Rollbacks and retries of drops cause org.datanucleus.exceptions.NucleusObjectNotFoundException: No such database row)
+ * [HIVE-3828] - insert overwrite fails with stored-as-dir in cluster
+ * [HIVE-3829] - Hive CLI needs UNSET TBLPROPERTY command
+ * [HIVE-3832] - Insert overwrite doesn't create a dir if the skewed column position doesnt match
+ * [HIVE-3839] - adding .gitattributes file for normalizing line endings during cross platform development
+ * [HIVE-3840] - hive cli null representation in output is inconsistent
+ * [HIVE-3847] - ppd.remove.duplicatefilters removing filters too aggressively
+ * [HIVE-3849] - Aliased column in where clause for multi-groupby single reducer cannot be resolved
+ * [HIVE-3850] - hour() function returns 12 hour clock value when using timestamp datatype
+ * [HIVE-3852] - Multi-groupby optimization fails when same distinct column is used twice or more
+ * [HIVE-3858] - Normalize left over CRLF files
+ * [HIVE-3861] - Upgrade hbase dependency to 0.94
+ * [HIVE-3862] - testHBaseNegativeCliDriver_cascade_dbdrop fails on hadoop-1
+ * [HIVE-3872] - MAP JOIN for VIEW thorws NULL pointer exception error
+ * [HIVE-3873] - lot of tests failing for hadoop 23
+ * [HIVE-3875] - negative value for hive.stats.ndv.error should be disallowed
+ * [HIVE-3888] - wrong mapside groupby if no partition is being selected
+ * [HIVE-3893] - something wrong with the hive-default.xml
+ * [HIVE-3899] - Partition pruning fails on <constant> = <constant> expression
+ * [HIVE-3908] - create view statement's outputs contains the view and a temporary dir.
+ * [HIVE-3909] - Wrong data due to HIVE-2820
+ * [HIVE-3912] - table_access_keys_stats.q fails with hadoop 0.23
+ * [HIVE-3913] - Possible deadlock in ZK lock manager
+ * [HIVE-3915] - Union with map-only query on one side and two MR job query on the other produces wrong results
+ * [HIVE-3916] - For outer joins, when looping over the rows looking for filtered tags, it doesn't report progress
+ * [HIVE-3918] - Normalize more CRLF line endings
+ * [HIVE-3920] - Change test for HIVE-2332
+ * [HIVE-3921] - recursive_dir.q fails on 0.23
+ * [HIVE-3923] - join_filters_overlap.q fails on 0.23
+ * [HIVE-3924] - join_nullsafe.q fails on 0.23
+ * [HIVE-3927] - Potential overflow with new RCFileCat column sizes options
+ * [HIVE-3931] - Add Oracle metastore upgrade script for 0.9 to 10.0
+ * [HIVE-3932] - Hive release tarballs don't contain PostgreSQL metastore scripts
+ * [HIVE-3943] - Skewed query fails if hdfs path has special characters
+ * [HIVE-3947] - MiniMR test remains pending after test completion
+ * [HIVE-3948] - avro_nullable_fields.q is failing in trunk
+ * [HIVE-3977] - Hive 0.10 postgres schema script is broken
+ * [HIVE-3980] - Cleanup after HIVE-3403
+ * [HIVE-3984] - Maintain a clear separation between Windowing & PTF at the specification level.
+ * [HIVE-3985] - Update new UDAFs introduced for Windowing to work with new Decimal Type
+ * [HIVE-3986] - Fix select expr processing in PTF Operator
+ * [HIVE-3987] - Update PTF invocation and windowing grammar
+ * [HIVE-3992] - Hive RCFile::sync(long) does a sub-sequence linear search for sync blocks
+ * [HIVE-3995] - PostgreSQL upgrade scripts are not valid
+ * [HIVE-3998] - Oracle metastore update script will fail when upgrading from 0.9.0 to 0.10.0
+ * [HIVE-3999] - Mysql metastore upgrade script will end up with different schema than the full schema load
+ * [HIVE-4000] - Hive client goes into infinite loop at 100% cpu
+ * [HIVE-4004] - Incorrect status for AddPartition metastore event if RawStore commit fails
+ * [HIVE-4018] - MapJoin failing with Distributed Cache error
+ * [HIVE-4021] - PostgreSQL upgrade scripts are creating column with incorrect name
+ * [HIVE-4024] - Derby metastore update script will fail when upgrading from 0.9.0 to 0.10.0
+ * [HIVE-4027] - Thrift alter_table api doesnt validate column type
+ * [HIVE-4028] - Bring paranthesis handling in windowing specification in compliance with sql standard
+ * [HIVE-4029] - Hive Profiler dies with NPE
+ * [HIVE-4030] - Name windowing function in consistence with sql standard
+ * [HIVE-4033] - NPE at runtime while selecting virtual column after joining three tables on different keys
+ * [HIVE-4034] - Should be able to specify windowing spec without needing Between
+ * [HIVE-4035] - Column Pruner for PTF Op
+ * [HIVE-4036] - remove use of FunctionRegistry during PTF Op initialization
+ * [HIVE-4039] - Hive compiler sometimes fails in semantic analysis / optimisation stage when boolean variable appears in WHERE clause.
+ * [HIVE-4040] - fix ptf negative tests
+ * [HIVE-4041] - Support multiple partitionings in a single Query
+ * [HIVE-4052] - Disallow partition/sort and distribute/order combinations in windowing and partitioning spec
+ * [HIVE-4056] - Extend rcfilecat to support (un)compressed size and no. of row
+ * [HIVE-4067] - Followup to HIVE-701: reduce ambiguity in grammar
+ * [HIVE-4071] - Map-join outer join produces incorrect results.
+ * [HIVE-4072] - Hive eclipse build path update for string template jar
+ * [HIVE-4073] - Make partition by optional in over clause
+ * [HIVE-4077] - alterPartition and alterPartitions methods in ObjectStore swallow exceptions
+ * [HIVE-4078] - Delay the serialize-deserialize pair in CommonJoinTaskDispatcher
+ * [HIVE-4079] - Altering a view partition fails with NPE
+ * [HIVE-4080] - Add Lead & Lag UDAFs
+ * [HIVE-4081] - allow expressions with over clause
+ * [HIVE-4082] - Break up ptf tests in PTF, Windowing and Lead/Lag tests
+ * [HIVE-4083] - PTF ColumnPruner doesn't account for Partition & Order expressions
+ * [HIVE-4084] - Generated aliases for windowing expressions is broken
+ * [HIVE-4090] - Use of hive.exec.script.allow.partial.consumption can produce partial results
+ * [HIVE-4092] - Store complete names of tables in column access analyzer
+ * [HIVE-4093] - Remove sprintf from PTFTranslator and use String.format()
+ * [HIVE-4094] - decimal_3.q & decimal_serde.q fail on hadoop 2
+ * [HIVE-4096] - problem in hive.map.groupby.sorted with distincts
+ * [HIVE-4097] - ORC file doesn't properly interpret empty hive.io.file.readcolumn.ids
+ * [HIVE-4098] - OrcInputFormat assumes Hive always calls createValue
+ * [HIVE-4103] - Remove System.gc() call from the map-join local-task loop
+ * [HIVE-4104] - Hive localtask does not buffer disk-writes or reads
+ * [HIVE-4105] - Hive MapJoinOperator unnecessarily deserializes values for all join-keys
+ * [HIVE-4107] - Update Hive 0.10.0 RELEASE_NOTES.txt
+ * [HIVE-4108] - Allow over() clause to contain an order by with no partition by
+ * [HIVE-4109] - Partition by column does not have to be in order by
+ * [HIVE-4111] - Default value in lag is not handled correctly
+ * [HIVE-4112] - Window range specification should be more flexible
+ * [HIVE-4119] - ANALYZE TABLE ... COMPUTE STATISTICS FOR COLUMNS fails with NPE if the table is empty
+ * [HIVE-4122] - Queries fail if timestamp data not in expected format
+ * [HIVE-4126] - remove support for lead/lag UDFs outside of UDAF args
+ * [HIVE-4130] - Bring the Lead/Lag UDFs interface in line with Lead/Lag UDAFs
+ * [HIVE-4131] - Fix eclipse template classpath to include new packages added by ORC file patch
+ * [HIVE-4138] - ORC's union object inspector returns a type name that isn't parseable by TypeInfoUtils
+ * [HIVE-4139] - MiniDFS shim does not work for hadoop 2
+ * [HIVE-4140] - Specifying alias for windowing function
+ * [HIVE-4142] - Remove inferring partition specification behavior
+ * [HIVE-4143] - Incorrect column mappings with over clause
+ * [HIVE-4146] - bug with hive.auto.convert.join.noconditionaltask with outer joins
+ * [HIVE-4148] - Cleanup aisle "ivy"
+ * [HIVE-4149] - wrong results big outer joins with array of ints
+ * [HIVE-4151] - HiveProfiler NPE with ScriptOperator
+ * [HIVE-4154] - NPE reading column of empty string from ORC file
+ * [HIVE-4156] - need to add protobuf classes to hive-exec.jar
+ * [HIVE-4159] - RetryingHMSHandler doesn't retry in enough cases
+ * [HIVE-4167] - Hive converts bucket map join to SMB join even when tables are not sorted
+ * [HIVE-4169] - union_remove_*.q fail on hadoop 2
+ * [HIVE-4170] - [REGRESSION] FsShell.close closes filesystem, removing temporary directories
+ * [HIVE-4174] - Round UDF converts BigInts to double
+ * [HIVE-4178] - ORC fails with files with different numbers of columns
+ * [HIVE-4179] - NonBlockingOpDeDup does not merge SEL operators correctly
+ * [HIVE-4180] - Filter getting dropped with PTFOperator
+ * [HIVE-4182] - doAS does not work with HiveServer2 in non-kerberos mode with local job
+ * [HIVE-4184] - Document HiveServer2 setup under the admin documentation on hive wiki
+ * [HIVE-4185] - Document HiveServer2 JDBC and Beeline CLI in the user documentation
+ * [HIVE-4186] - NPE in ReduceSinkDeDuplication
+ * [HIVE-4187] - QL build-grammar target fails after HIVE-4148
+ * [HIVE-4188] - TestJdbcDriver2.testDescribeTable failing consistently
+ * [HIVE-4189] - ORC fails with String column that ends in lots of nulls
+ * [HIVE-4190] - OVER clauses with ORDER BY not getting windowing set properly
+ * [HIVE-4191] - describe table output always prints as if formatted keyword is specified
+ * [HIVE-4197] - Bring windowing support inline with SQL Standard
+ * [HIVE-4202] - reuse Partition objects in PTFOperator processing
+ * [HIVE-4208] - Clientpositive test parenthesis_star_by is non-deteministic
+ * [HIVE-4217] - Fix show_create_table_*.q test failures
+ * [HIVE-4219] - explain dependency does not capture the input table
+ * [HIVE-4235] - CREATE TABLE IF NOT EXISTS uses inefficient way to check if table exists
+ * [HIVE-4252] - hiveserver2 string representation of complex types are inconsistent with cli
+ * [HIVE-4254] - Code cleanup : debug methods, having clause associated with Windowing
+ * [HIVE-4255] - update show_functions.q.out for functions added for windowing
+ * [HIVE-4259] - SEL operator created with missing columnExprMap for unions
+ * [HIVE-4260] - union_remove_12, union_remove_13 are failing on hadoop2
+ * [HIVE-4261] - union_remove_10 is failing on hadoop2 with assertion (root task with non-empty set of parents)
+ * [HIVE-4262] - fix last_value UDAF behavior
+ * [HIVE-4269] - fix handling of binary type in hiveserver2, jdbc driver
+ * [HIVE-4270] - bug in hive.map.groupby.sorted in the presence of multiple input partitions
+ * [HIVE-4271] - Limit precision of decimal type
+ * [HIVE-4272] - partition wise metadata does not work for text files
+ * [HIVE-4275] - Hive does not differentiate scheme and authority in file uris
+ * [HIVE-4280] - TestRetryingHMSHandler is failing on trunk.
+ * [HIVE-4288] - Add IntelliJ project files files to .gitignore
+ * [HIVE-4289] - HCatalog build fails when behind a firewall
+ * [HIVE-4292] - hiveserver2 should support -hiveconf commandline parameter
+ * [HIVE-4296] - ant thriftif fails on hcatalog
+ * [HIVE-4302] - Fix how RowSchema and RowResolver are set on ReduceSinkOp that precedes PTFOp
+ * [HIVE-4303] - 2 empty java files in hcatalog
+ * [HIVE-4308] - Newly added test TestCliDriver.hiveprofiler_union0 is failing on trunk
+ * [HIVE-4311] - DOS line endings in auto_join26.q
+ * [HIVE-4315] - enable doAs in unsecure mode for hive server2, when MR job runs locally
+ * [HIVE-4318] - OperatorHooks hit performance even when not used
+ * [HIVE-4319] - Revert changes checked-in as part of HIVE-1953
+ * [HIVE-4320] - Consider extending max limit for precision to 38
+ * [HIVE-4323] - sqlline dependency is not required
+ * [HIVE-4327] - NPE in constant folding with decimal
+ * [HIVE-4332] - orc*.q tests fail on hadoop 2
+ * [HIVE-4333] - most windowing tests fail on hadoop 2
+ * [HIVE-4334] - ctas test on hadoop 2 has outdated golden file
+ * [HIVE-4335] - serde_regex test fails on hadoop 2
+ * [HIVE-4336] - Selecting from a view, and another view that also selects from that view fails
+ * [HIVE-4342] - NPE for query involving UNION ALL with nested JOIN and UNION ALL
+ * [HIVE-4352] - Guava not getting included in build package
+ * [HIVE-4356] - remove duplicate impersonation parameters for hiveserver2
+ * [HIVE-4358] - Check for Map side processing in PTFOp is no longer valid
+ * [HIVE-4365] - wrong result in left semi join
+ * [HIVE-4371] - some issue with merging join trees
+ * [HIVE-4373] - Hive Version returned by HiveDatabaseMetaData.getDatabaseProductVersion is incorrect
+ * [HIVE-4378] - Counters hit performance even when not used
+ * [HIVE-4387] - ant maven-build fails because hcatalog doesn't have a make-pom target
+ * [HIVE-4394] - test leadlag.q fails
+ * [HIVE-4398] - HS2 Resource leak: operation handles not cleaned when originating session is closed
+ * [HIVE-4407] - TestHCatStorer.testStoreFuncAllSimpleTypes fails because of null case difference
+ * [HIVE-4412] - PTFDesc tries serialize transient fields like OIs, etc.
+ * [HIVE-4419] - webhcat - support ${WEBHCAT_PREFIX}/conf/ as config directory
+ * [HIVE-4420] - HCatalog unit tests stop after a failure
+ * [HIVE-4421] - Improve memory usage by ORC dictionaries
+ * [HIVE-4447] - hcatalog version numbers need to be updated
+ * [HIVE-4455] - HCatalog build directories get included in tar file produced by "ant tar"
+ * [HIVE-4461] - hcatalog jars not getting published to maven repo
+ * [HIVE-4494] - ORC map columns get class cast exception in some context
+ * [HIVE-4498] - TestBeeLineWithArgs.testPositiveScriptFile fails
+ * [HIVE-4500] - HS2 holding too many file handles of hive_job_log_hive_*.txt files
+ * [HIVE-4505] - Hive can't load transforms added using 'ADD FILE'
+ * [HIVE-4527] - Fix eclipse project template
+
+** Improvement
+ * [HIVE-581] - improve group by syntax
+ * [HIVE-948] - more query plan optimization rules
+ * [HIVE-1953] - Hive should process comments in CliDriver
+ * [HIVE-2439] - Upgrade antlr version to 3.4
+ * [HIVE-2477] - Use name of original expression for name of CAST output
+ * [HIVE-3004] - RegexSerDe should support other column types in addition to STRING
+ * [HIVE-3231] - msck repair should find partitions already containing data files
+ * [HIVE-3252] - Add environment context to metastore Thrift calls
+ * [HIVE-3401] - Diversify grammar for split sampling
+ * [HIVE-3431] - Avoid race conditions while downloading resources from non-local filesystem
+ * [HIVE-3492] - Provide ALTER for partition changing bucket number
+ * [HIVE-3527] - Allow CREATE TABLE LIKE command to take TBLPROPERTIES
+ * [HIVE-3531] - Simple lock manager for dedicated hive server
+ * [HIVE-3564] - hivetest.py: revision number and applied patch
+ * [HIVE-3628] - Provide a way to use counters in Hive through UDF
+ * [HIVE-3633] - sort-merge join does not work with sub-queries
+ * [HIVE-3672] - Support altering partition column type in Hive
+ * [HIVE-3708] - Add mapreduce workflow information to job configuration
+ * [HIVE-3709] - Stop storing default ConfVars in temp file
+ * [HIVE-3710] - HiveConf.ConfVars.HIVE_STATS_COLLECT_RAWDATASIZE should not be checked in FileSinkOperator
+ * [HIVE-3762] - Minor fix for 'tableName' in Hive.g
+ * [HIVE-3784] - de-emphasize mapjoin hint
+ * [HIVE-3793] - Print number of fetched rows after query in CliDriver
+ * [HIVE-3796] - Multi-insert involving bucketed/sorted table turns off merging on all outputs
+ * [HIVE-3799] - Better error message if metalisteners or hookContext cannot be loaded/instantiated
+ * [HIVE-3805] - Resolve TODO in TUGIBasedProcessor
+ * [HIVE-3833] - object inspectors should be initialized based on partition metadata
+ * [HIVE-3853] - UDF unix_timestamp is deterministic if an argument is given, but it treated as non-deterministic preventing PPD
+ * [HIVE-3874] - Create a new Optimized Row Columnar file format for Hive
+ * [HIVE-3884] - Better align columns in DESCRIBE table_name output to make more human-readable
+ * [HIVE-3904] - Replace hashmaps in JoinOperators to array
+ * [HIVE-3917] - Support noscan operation for analyze command
+ * [HIVE-3950] - Remove code for merging files via MR job
+ * [HIVE-3952] - merge map-job followed by map-reduce job
+ * [HIVE-3958] - support partial scan for analyze command - RCFile
+ * [HIVE-3970] - Clean up/fix PartitionNameWhitelistPreEventListener
+ * [HIVE-3996] - Correctly enforce the memory limit on the multi-table map-join
+ * [HIVE-4001] - Add o.a.h.h.serde.Constants for backward compatibility
+ * [HIVE-4007] - Create abstract classes for serializer and deserializer
+ * [HIVE-4015] - Add ORC file to the grammar as a file format
+ * [HIVE-4016] - Remove init(fname) from TestParse.vm for each test
+ * [HIVE-4020] - Swap applying order of CP and PPD
+ * [HIVE-4023] - Improve Error Logging in MetaStore
+ * [HIVE-4025] - Add reflect UDF for member method invocation of column
+ * [HIVE-4042] - ignore mapjoin hint
+ * [HIVE-4045] - Modify PreDropPartitionEvent to pass Table parameter
+ * [HIVE-4076] - Refactor code for finding windowing expressions
+ * [HIVE-4125] - Expose metastore JMX metrics
+ * [HIVE-4128] - Support avg(decimal)
+ * [HIVE-4129] - Window handling dumps debug info on console, instead should use logger.
+ * [HIVE-4157] - ORC runs out of heap when writing
+ * [HIVE-4206] - Sort merge join does not work for outer joins for 7 inputs
+ * [HIVE-4212] - sort merge join should work for outer joins for more than 8 inputs
+ * [HIVE-4240] - optimize hive.enforce.bucketing and hive.enforce sorting insert
+ * [HIVE-4258] - Log logical plan tree for debugging
+ * [HIVE-4281] - add hive.map.groupby.sorted.testmode
+ * [HIVE-4304] - Remove unused builtins and pdk submodules
+ * [HIVE-4306] - PTFDeserializer should reconstruct OIs based on InputOI passed to PTFOperator
+ * [HIVE-4328] - Change default bigtable selection policy for sort-merge joins
+
+** New Feature
+ * [HIVE-446] - Implement TRUNCATE
+ * [HIVE-701] - lots of reserved keywords in hive
+ * [HIVE-896] - Add LEAD/LAG/FIRST/LAST analytical windowing functions to Hive.
+ * [HIVE-933] - Infer bucketing/sorting properties
+ * [HIVE-2288] - Adding the oracle nvl function to the UDF
+ * [HIVE-2691] - Specify location of log4j configuration files via configuration properties
+ * [HIVE-2693] - Add DECIMAL data type
+ * [HIVE-2935] - Implement HiveServer2
+ * [HIVE-3073] - Hive List Bucketing - DML support
+ * [HIVE-3552] - HIVE-3552 performant manner for performing cubes/rollups/grouping sets for a high number of grouping set keys
+ * [HIVE-3646] - Add 'IGNORE PROTECTION' predicate for dropping partitions
+ * [HIVE-3682] - when output hive table to file,users should could have a separator of their own choice
+ * [HIVE-3825] - Add Operator level Hooks
+ * [HIVE-3834] - Support ALTER VIEW AS SELECT in Hive
+ * [HIVE-3897] - Add a way to get the uncompressed/compressed sizes of columns from an RC File
+ * [HIVE-3898] - getReducersBucketing in SemanticAnalyzer may return more than the max number of reducers
+ * [HIVE-3903] - Allow updating bucketing/sorting metadata of a partition through the CLI
+ * [HIVE-3937] - Hive Profiler
+ * [HIVE-3951] - Allow Decimal type columns in Regex Serde
+ * [HIVE-4019] - Ability to create and drop temporary partition function
+ * [HIVE-4048] - Allow partition by/order by in partitioning spec in over clause and partition function
+ * [HIVE-4120] - Implement decimal encoding for ORC
+ * [HIVE-4127] - Testing with Hadoop 2.x causes test failure for ORC's TestFileDump
+ * [HIVE-4155] - Expose ORC's FileDump as a service
+ * [HIVE-4248] - Implement a memory manager for ORC
+
+** Task
+ * [HIVE-3679] - Unescape partition names returned by show partitions
+ * [HIVE-3718] - Add check to determine whether partition can be dropped at Semantic Analysis time
+ * [HIVE-3721] - ALTER TABLE ADD PARTS should check for valid partition spec and throw a SemanticException if part spec is not valid
+ * [HIVE-3838] - Add input table name to MetaStoreEndFunctionContext for logging purposes
+ * [HIVE-3940] - Track columns accessed in each table in a query
+ * [HIVE-3981] - Split up tests in ptf_general_queries.q
+ * [HIVE-3982] - Merge PTFDesc and PTFDef classes
+ * [HIVE-4031] - Add apache headers in new files
+ * [HIVE-4145] - Create hcatalog stub directory and add it to the build
+
+** Test
+ * [HIVE-3571] - add a way to run a small unit quickly
+ * [HIVE-3842] - Remove redundant test codes
+ * [HIVE-3944] - Make accept qfile argument for miniMR tests
+ * [HIVE-3956] - TestMetaStoreAuthorization always uses the same port
+ * [HIVE-4124] - Add more tests for windowing
+ * [HIVE-4298] - add tests for distincts for hive.map.groutp.sorted
+ * [HIVE-4337] - Update list bucketing test results
+
+** Wish
+ * [HIVE-4314] - Result of mapjoin_test_outer.q is not deterministic
+
Release Notes - Hive - Version 0.10.0
** Sub-task
Modified: hive/branches/HIVE-4115/build-common.xml
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/build-common.xml?rev=1486517&r1=1486516&r2=1486517&view=diff
==============================================================================
--- hive/branches/HIVE-4115/build-common.xml (original)
+++ hive/branches/HIVE-4115/build-common.xml Mon May 27 05:27:15 2013
@@ -59,7 +59,7 @@
<property name="test.output" value="true"/>
<property name="test.junit.output.format" value="xml"/>
<property name="test.junit.output.usefile" value="true"/>
- <property name="minimr.query.files" value="list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,schemeAuthority.q,truncate_column_buckets.q"/>
+ <property name="minimr.query.files" value="list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,schemeAuthority.q,truncate_column_buckets.q,remote_script.q"/>
<property name="minimr.query.negative.files" value="cluster_tasklog_retrieval.q,minimr_broken_pipe.q,mapreduce_stack_trace.q,mapreduce_stack_trace_turnoff.q,mapreduce_stack_trace_hadoop20.q,mapreduce_stack_trace_turnoff_hadoop20.q" />
<property name="test.silent" value="true"/>
<property name="hadoopVersion" value="${hadoop.version.ant-internal}"/>
Modified: hive/branches/HIVE-4115/build.xml
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/build.xml?rev=1486517&r1=1486516&r2=1486517&view=diff
==============================================================================
--- hive/branches/HIVE-4115/build.xml (original)
+++ hive/branches/HIVE-4115/build.xml Mon May 27 05:27:15 2013
@@ -59,6 +59,14 @@
<property name="rat.build.dir" location="${build.dir.hive}/rat"/>
<property name="md5sum.format" value="{0} {1}"/>
+ <!-- Check minimum ant version required -->
+ <fail message="Please use ant version 1.8.0 or greater for building hive.">
+ <condition>
+ <not>
+ <antversion atleast="1.8.0"/>
+ </not>
+ </condition>
+ </fail>
<condition property="is-offline" value="true" else="false">
<isset property="offline"/>
Modified: hive/branches/HIVE-4115/cli/src/java/org/apache/hadoop/hive/cli/CliSessionState.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/cli/src/java/org/apache/hadoop/hive/cli/CliSessionState.java?rev=1486517&r1=1486516&r2=1486517&view=diff
==============================================================================
--- hive/branches/HIVE-4115/cli/src/java/org/apache/hadoop/hive/cli/CliSessionState.java (original)
+++ hive/branches/HIVE-4115/cli/src/java/org/apache/hadoop/hive/cli/CliSessionState.java Mon May 27 05:27:15 2013
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.cli;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
@@ -77,11 +78,6 @@ public class CliSessionState extends Ses
private Hive hive; // currently only used (and init'ed) in getCurrentDbName
- public CliSessionState() {
- super();
- remoteMode = false;
- }
-
public CliSessionState(HiveConf conf) {
super(conf);
remoteMode = false;
@@ -112,10 +108,13 @@ public class CliSessionState extends Ses
public void close() {
try {
+ super.close();
if (remoteMode) {
client.clean();
transport.close();
}
+ } catch (IOException ioe) {
+ ioe.printStackTrace();
} catch (TException e) {
e.printStackTrace();
}
Modified: hive/branches/HIVE-4115/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1486517&r1=1486516&r2=1486517&view=diff
==============================================================================
--- hive/branches/HIVE-4115/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original)
+++ hive/branches/HIVE-4115/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Mon May 27 05:27:15 2013
@@ -201,7 +201,8 @@ public class HiveConf extends Configurat
DYNAMICPARTITIONMAXPARTS("hive.exec.max.dynamic.partitions", 1000),
DYNAMICPARTITIONMAXPARTSPERNODE("hive.exec.max.dynamic.partitions.pernode", 100),
MAXCREATEDFILES("hive.exec.max.created.files", 100000L),
- DOWNLOADED_RESOURCES_DIR("hive.downloaded.resources.dir", "/tmp/"+System.getProperty("user.name")+"/hive_resources"),
+ DOWNLOADED_RESOURCES_DIR("hive.downloaded.resources.dir",
+ "/tmp/${hive.session.id}_resources"),
DEFAULTPARTITIONNAME("hive.exec.default.partition.name", "__HIVE_DEFAULT_PARTITION__"),
DEFAULT_ZOOKEEPER_PARTITION_NAME("hive.lockmgr.zookeeper.default.partition.name", "__HIVE_DEFAULT_ZOOKEEPER_PARTITION__"),
// Whether to show a link to the most failed task + debugging tips
@@ -414,7 +415,12 @@ public class HiveConf extends Configurat
true),
HIVEJOINEMITINTERVAL("hive.join.emit.interval", 1000),
HIVEJOINCACHESIZE("hive.join.cache.size", 25000),
+
+ // hive.mapjoin.bucket.cache.size has been replaced by hive.smbjoin.cache.row,
+ // need to remove by hive .13. Also, do not change default (see SMB operator)
HIVEMAPJOINBUCKETCACHESIZE("hive.mapjoin.bucket.cache.size", 100),
+
+ HIVESMBJOINCACHEROWS("hive.smbjoin.cache.rows", 10000),
HIVEGROUPBYMAPINTERVAL("hive.groupby.mapaggr.checkinterval", 100000),
HIVEMAPAGGRHASHMEMORY("hive.map.aggr.hash.percentmemory", (float) 0.5),
HIVEMAPJOINFOLLOWEDBYMAPAGGRHASHMEMORY("hive.mapjoin.followby.map.aggr.hash.percentmemory", (float) 0.3),
@@ -435,6 +441,10 @@ public class HiveConf extends Configurat
HIVEQUERYRESULTFILEFORMAT("hive.query.result.fileformat", "TextFile"),
HIVECHECKFILEFORMAT("hive.fileformat.check", true),
+ // default serde for rcfile
+ HIVEDEFAULTRCFILESERDE("hive.default.rcfile.serde",
+ "org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe"),
+
//Location of Hive run time structured log file
HIVEHISTORYFILELOC("hive.querylog.location", "/tmp/" + System.getProperty("user.name")),
@@ -636,6 +646,8 @@ public class HiveConf extends Configurat
// Serde for FetchTask
HIVEFETCHOUTPUTSERDE("hive.fetch.output.serde", "org.apache.hadoop.hive.serde2.DelimitedJSONSerDe"),
+ HIVEEXPREVALUATIONCACHE("hive.cache.expr.evaluation", true),
+
// Hive Variables
HIVEVARIABLESUBSTITUTE("hive.variable.substitute", true),
HIVEVARIABLESUBSTITUTEDEPTH("hive.variable.substitute.depth", 40),
Modified: hive/branches/HIVE-4115/conf/hive-default.xml.template
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/conf/hive-default.xml.template?rev=1486517&r1=1486516&r2=1486517&view=diff
==============================================================================
--- hive/branches/HIVE-4115/conf/hive-default.xml.template (original)
+++ hive/branches/HIVE-4115/conf/hive-default.xml.template Mon May 27 05:27:15 2013
@@ -374,6 +374,12 @@
</property>
<property>
+ <name>hive.default.rcfile.serde</name>
+ <value>org.apache.hadoop.hive.serde2.columnar.LazyBinaryColumnarSerDe</value>
+ <description>The default SerDe hive will use for the rcfile format</description>
+</property>
+
+<property>
<name>hive.fileformat.check</name>
<value>true</value>
<description>Whether to check file format or not when loading data files</description>
@@ -589,9 +595,9 @@
</property>
<property>
- <name>hive.mapjoin.bucket.cache.size</name>
- <value>100</value>
- <description>How many values in each keys in the map-joined table should be cached in memory. </description>
+ <name>hive.smbjoin.cache.rows</name>
+ <value>10000</value>
+ <description>How many rows with the same key value should be cached in memory per smb joined table. </description>
</property>
<property>
@@ -1716,6 +1722,17 @@
</property>
<property>
+ <name>hive.cache.expr.evaluation</name>
+ <value>true</value>
+ <description>
+ If true, evaluation result of deterministic expression referenced twice or more will be cached.
+ For example, in filter condition like ".. where key + 10 > 10 or key + 10 = 0"
+ "key + 10" will be evaluated/cached once and reused for following expression ("key + 10 = 0").
+ Currently, this is applied only to expressions in select or filter operator.
+ </description>
+</property>
+
+<property>
<name>hive.hmshandler.retry.attempts</name>
<value>1</value>
<description>The number of times to retry a HMSHandler call if there were a connection error</description>
Modified: hive/branches/HIVE-4115/data/conf/hive-site.xml
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/data/conf/hive-site.xml?rev=1486517&r1=1486516&r2=1486517&view=diff
==============================================================================
--- hive/branches/HIVE-4115/data/conf/hive-site.xml (original)
+++ hive/branches/HIVE-4115/data/conf/hive-site.xml Mon May 27 05:27:15 2013
@@ -188,4 +188,10 @@
<description>The default input format, if it is not specified, the system assigns it. It is set to HiveInputFormat for hadoop versions 17, 18 and 19, whereas it is set to CombineHiveInputFormat for hadoop 20. The user can always overwrite it - if there is a bug in CombineHiveInputFormat, it can always be manually set to HiveInputFormat. </description>
</property>
+<property>
+ <name>hive.default.rcfile.serde</name>
+ <value>org.apache.hadoop.hive.serde2.columnar.ColumnarSerDe</value>
+ <description>The default SerDe hive will use for the rcfile format</description>
+</property>
+
</configuration>
Modified: hive/branches/HIVE-4115/eclipse-templates/.classpath
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/eclipse-templates/.classpath?rev=1486517&r1=1486516&r2=1486517&view=diff
==============================================================================
--- hive/branches/HIVE-4115/eclipse-templates/.classpath (original)
+++ hive/branches/HIVE-4115/eclipse-templates/.classpath Mon May 27 05:27:15 2013
@@ -45,7 +45,6 @@
<classpathentry kind="lib" path="build/ivy/lib/hadoop0.20.shim/oro-2.0.8.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/hadoop0.20.shim/servlet-api-2.5-6.1.14.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/hadoop0.20.shim/xmlenc-0.52.jar"/>
- <classpathentry kind="lib" path="build/ivy/lib/hadoop0.20.shim/guava-@guava-hadoop20.version@.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/default/avro-@avro.version@.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/default/avro-mapred-@avro.version@.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/default/jline-@jline.version@.jar"/>
@@ -59,6 +58,7 @@
<classpathentry kind="lib" path="build/ivy/lib/default/derby-@derby.version@.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/default/hbase-@hbase.version@.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/default/hbase-@hbase.version@-tests.jar"/>
+ <classpathentry kind="lib" path="build/ivy/lib/default/guava-@guava.version@.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/default/libfb303-@libfb303.version@.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/default/libthrift-@libthrift.version@.jar"/>
<classpathentry kind="lib" path="build/ivy/lib/default/protobuf-java-@protobuf.version@.jar"/>
@@ -107,8 +107,6 @@
<classpathentry kind="src" path="hwi/src/test"/>
<classpathentry kind="src" path="jdbc/src/java"/>
<classpathentry kind="src" path="jdbc/src/test"/>
- <classpathentry kind="src" path="hcatalog/src/java"/>
- <classpathentry kind="src" path="hcatalog/src/test"/>
<classpathentry kind="src" path="metastore/src/gen/thrift/gen-javabean"/>
<classpathentry kind="src" path="metastore/src/java"/>
<classpathentry kind="src" path="metastore/src/model"/>
Modified: hive/branches/HIVE-4115/hcatalog/src/test/e2e/hcatalog/build.xml
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/hcatalog/src/test/e2e/hcatalog/build.xml?rev=1486517&r1=1486516&r2=1486517&view=diff
==============================================================================
--- hive/branches/HIVE-4115/hcatalog/src/test/e2e/hcatalog/build.xml (original)
+++ hive/branches/HIVE-4115/hcatalog/src/test/e2e/hcatalog/build.xml Mon May 27 05:27:15 2013
@@ -37,6 +37,7 @@
<property name="driver.src" value="${basedir}/drivers"/>
<property name="deployer.src" value="${basedir}/deployers"/>
<property name="conf.src" value="${basedir}/conf"/>
+ <property name="resource.src" value="${basedir}/resource"/>
<property name="tool.src" value="${basedir}/tools"/>
<property name="data.dir" value="${basedir}/data"/>
@@ -105,6 +106,7 @@
<mkdir dir="${tar.dir}/drivers"/>
<mkdir dir="${tar.dir}/deployers"/>
<mkdir dir="${tar.dir}/conf"/>
+ <mkdir dir="${tar.dir}/resource"/>
<mkdir dir="${tar.dir}/libexec"/>
<mkdir dir="${tar.dir}/libexec/PigTest"/>
<mkdir dir="${tar.dir}/libexec/PigTest/test"/>
@@ -136,6 +138,10 @@
<fileset dir="${conf.src}"/>
</copy>
+ <copy todir="${tar.dir}/resource">
+ <fileset dir="${resource.src}"/>
+ </copy>
+
<copy todir="${tar.dir}/libexec/HCatTest">
<fileset dir="${tool.src}/test"/>
<fileset dir="${tool.src}/generate"/>
Modified: hive/branches/HIVE-4115/hcatalog/src/test/e2e/hcatalog/tests/hcat.conf
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/hcatalog/src/test/e2e/hcatalog/tests/hcat.conf?rev=1486517&r1=1486516&r2=1486517&view=diff
==============================================================================
--- hive/branches/HIVE-4115/hcatalog/src/test/e2e/hcatalog/tests/hcat.conf (original)
+++ hive/branches/HIVE-4115/hcatalog/src/test/e2e/hcatalog/tests/hcat.conf Mon May 27 05:27:15 2013
@@ -48,7 +48,8 @@ gpa double)
stored as textfile;
describe hcat_createtable_1;\
,'rc' => 0
- ,'expected_out_regex' => 'name string(\s)*age int(\s)*gpa double'
+ ,'expected_out_regex' =>
+ 'name(\s)+string(\s)+None(\s)+\nage(\s)+int(\s)+None(\s)+\ngpa(\s)+double(\s)+None'
,'expected_err_regex' => 'OK(.*)OK(.*)OK'
},
{
@@ -61,7 +62,8 @@ gpa double) partitioned by (b string) st
describe extended hcat_createtable_2;
\,
,'rc' => 0
- ,'expected_out_regex' => 'name string(\s)*age int(\s)*gpa double'
+ ,'expected_out_regex' =>
+ 'name(\s)+string(\s)+None(\s)+\nage(\s)+int(\s)+None(\s)+\ngpa(\s)+double(\s)+None(\s)+\nb(\s)+string(\s)+None'
},
{
'num' => 3
@@ -72,7 +74,8 @@ create table if not exists hcat_createta
describe hcat_createtable_3;
\,
,'rc' => 0
- ,'expected_out_regex' => 'name string(\s)*age int(\s)*gpa double'
+ ,'expected_out_regex' =>
+ 'name(\s)+string(\s)+None(\s)+\nage(\s)+int(\s)+None(\s)+\ngpa(\s)+double(\s)+None'
,'expected_err_regex' => 'OK(.*)OK(.*)OK(.*)OK'
},
@@ -99,7 +102,8 @@ outputformat 'org.apache.hadoop.hive.ql.
describe extended hcat_createtable_5;
\,
,'rc' => 0
- ,'expected_out_regex' => 'name string(\s)*age int(\s)*gpa double(.*(\s))*inputFormat:org.apache.hadoop.hive.ql.io.RCFileInputFormat, outputFormat:org.apache.hadoop.hive.ql.io.RCFileOutputFormat'
+ ,'expected_out_regex' =>
+ 'name(\s)+string(\s)+None(\s)+\nage(\s)+int(\s)+None(\s)+\ngpa(\s)+double(\s)+None'
,'expected_err_regex' => 'OK(.*)OK(.*)OK'
},
@@ -117,7 +121,8 @@ stored as textfile;
describe extended hcat_createtable_6;
#,
,'rc' => 0
- ,'expected_out_regex' => 'a int(\s)*b array<int>(\s)*c map<string,string>(.*(\s))*.*parameters:.*colelction.delim=,, mapkey.delim=:, serialization.format=|, line.delim=\n, field.delim=^'
+ ,'expected_out_regex' =>
+ 'a(\s)+int(\s)+None(\s)+\nb(\s)+array<int>(\s)+None(\s)+\nc(\s)+map<string,string>(\s)+None(\s)+\n(.*(\s))*.*parameters:.*colelction.delim=,, mapkey.delim=:, serialization.format=|, line.delim=\n, field.delim=^'
,'expected_err_regex' => 'OK(.*)OK(.*)OK'
},
@@ -315,7 +320,8 @@ alter table hcat_altertable_13 add parti
alter table hcat_altertable_13 add columns (x string comment 'extra');
describe hcat_altertable_13;\,
,'rc' => 0
- ,'expected_out_regex' => 'a int(\s)*b string(\s)*x string extra(\s)*c string(\s)*d string'
+ ,'expected_out_regex' =>
+ 'a(\s)+int(\s)+None(\s)+\nb(\s)+string(\s)+None(\s)+\nx(\s)+string(\s)+extra(\s)+\nc(\s)+string(\s)+None(\s)+\nd(\s)+string(\s)+None'
},
{
'num' => 14
@@ -324,7 +330,8 @@ describe hcat_altertable_13;\,
alter table hcat_altertable_13 replace columns (k int, l string, m string);
describe hcat_altertable_13;\,
,'rc' => 0
- ,'expected_out_regex' => 'k int(\s)*l string(\s)*m string(\s)*c string(\s)*d string'
+ ,'expected_out_regex' =>
+ 'k(\s)+int(\s)+None(\s)+\nl(\s)+string(\s)+None(\s)+\nm(\s)+string(\s)+None(\s)+\nc(\s)+string(\s)+None(\s)+\nd(\s)+string'
},
{
'num' => 15
@@ -410,7 +417,8 @@ drop view if exists hcat_view_2_1;
create view hcat_view_2_1(name,age) as select name, age from studenttab10k;
describe extended hcat_view_2_1;
"
- ,'expected_out_regex' => 'name string.*(\s)age int(.*(\s))*viewOriginalText:select name, age from studenttab10k.*tableType:VIRTUAL_VIEW'
+ ,'expected_out_regex' =>
+ 'name(\s)+string(\s)+None(\s)+\nage(\s)+int(\s)+None(\s)+(.*(\s))*viewOriginalText:select name, age from studenttab10k.*tableType:VIRTUAL_VIEW'
,'rc' => 0
},
],
@@ -589,7 +597,8 @@ show table extended like hcat_showdestab
,'hcat' => q\
describe extended hcat_showdestable_2 partition(c='42');\
,'rc' => 0
- ,'expected_out_regex' => 'a int first(\s)*b struct<foo:int,bar:struct<bar1:int,bar2:int>> second(\s)*c string(.*(\s))*Detailed Partition Information(\s)*Partition(.*)values:\[42\]'
+ ,'expected_out_regex' =>
+ 'a(\s)+int(\s)+first(\s)+\nb(\s)+struct<foo:int,bar:struct<bar1:int,bar2:int>>(\s)+second(\s)+\nc(\s)+string(\s)+None(\s)+(.*(\s))*Detailed Partition Information(\s)*Partition(.*)values:\[42\]'
},
{
'num' => 10
@@ -597,7 +606,7 @@ describe extended hcat_showdestable_2 pa
,'hcat' => q\
describe extended hcat_showdestable_2.b;\
,'rc' => 0
- ,'expected_out_regex' => 'foo int from deserializer(\s)*bar struct<bar1:int,bar2:int> from deserializer'
+ ,'expected_out_regex' => 'foo(\s)+int(\s)+from deserializer(\s)+\nbar(\s)+struct<bar1:int,bar2:int>(\s)+from deserializer'
},
{
'num' => 11
@@ -605,7 +614,7 @@ describe extended hcat_showdestable_2.b;
,'hcat' => q\
describe extended hcat_showdestable_2.b.foo;\
,'rc' => 0
- ,'expected_out_regex' => 'foo int from deserializer'
+ ,'expected_out_regex' => 'foo(\s)+int(\s)+from deserializer'
,'not_expected_out_regex' => 'bar'
},
{
@@ -614,7 +623,7 @@ describe extended hcat_showdestable_2.b.
,'hcat' => q\
describe extended hcat_showdestable_2.b.bar;\
,'rc' => 0
- ,'expected_out_regex' => 'bar1 int from deserializer(\s)*bar2 int from deserializer'
+ ,'expected_out_regex' => 'bar1(\s)+int(\s)+from deserializer(\s)+\nbar2(\s)+int(\s)+from deserializer'
},
{
'num' => 13
@@ -622,7 +631,7 @@ describe extended hcat_showdestable_2.b.
,'hcat' => q\
describe extended hcat_showdestable_2.b.bar.bar1;\
,'rc' => 0
- ,'expected_out_regex' => 'bar1 int from deserializer'
+ ,'expected_out_regex' => 'bar1(\s)+int(\s)+from deserializer'
,'not_expected_out_regex' => 'bar2'
},
{
Modified: hive/branches/HIVE-4115/jdbc/src/java/org/apache/hive/jdbc/HiveDriver.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/jdbc/src/java/org/apache/hive/jdbc/HiveDriver.java?rev=1486517&r1=1486516&r2=1486517&view=diff
==============================================================================
--- hive/branches/HIVE-4115/jdbc/src/java/org/apache/hive/jdbc/HiveDriver.java (original)
+++ hive/branches/HIVE-4115/jdbc/src/java/org/apache/hive/jdbc/HiveDriver.java Mon May 27 05:27:15 2013
@@ -100,8 +100,13 @@ public class HiveDriver implements Drive
return Pattern.matches(URL_PREFIX + ".*", url);
}
+ /*
+ * As per JDBC 3.0 Spec (section 9.2)
+ * "If the Driver implementation understands the URL, it will return a Connection object;
+ * otherwise it returns null"
+ */
public Connection connect(String url, Properties info) throws SQLException {
- return new HiveConnection(url, info);
+ return acceptsURL(url) ? new HiveConnection(url, info) : null;
}
/**
Modified: hive/branches/HIVE-4115/jdbc/src/test/org/apache/hive/jdbc/TestJdbcDriver2.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/jdbc/src/test/org/apache/hive/jdbc/TestJdbcDriver2.java?rev=1486517&r1=1486516&r2=1486517&view=diff
==============================================================================
--- hive/branches/HIVE-4115/jdbc/src/test/org/apache/hive/jdbc/TestJdbcDriver2.java (original)
+++ hive/branches/HIVE-4115/jdbc/src/test/org/apache/hive/jdbc/TestJdbcDriver2.java Mon May 27 05:27:15 2013
@@ -34,6 +34,7 @@ import java.sql.Types;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
+import java.util.Properties;
import java.util.Set;
import java.util.regex.Pattern;
@@ -43,6 +44,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hive.common.util.HiveVersionInfo;
+
/**
* TestJdbcDriver2
*
@@ -1263,4 +1265,14 @@ public class TestJdbcDriver2 extends Tes
assertFalse(res.next());
}
+ /**
+ * If the Driver implementation understands the URL, it will return a Connection object;
+ * otherwise it returns null
+ */
+ public void testInvalidURL() throws Exception {
+ HiveDriver driver = new HiveDriver();
+ Connection conn = driver.connect("jdbc:derby://localhost:10000/default", new Properties());
+ assertNull(conn);
+ }
+
}
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeColumnEvaluator.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeColumnEvaluator.java?rev=1486517&r1=1486516&r2=1486517&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeColumnEvaluator.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeColumnEvaluator.java Mon May 27 05:27:15 2013
@@ -29,9 +29,7 @@ import org.apache.hadoop.hive.serde2.obj
/**
* This evaluator gets the column from the row object.
*/
-public class ExprNodeColumnEvaluator extends ExprNodeEvaluator {
-
- protected ExprNodeColumnDesc expr;
+public class ExprNodeColumnEvaluator extends ExprNodeEvaluator<ExprNodeColumnDesc> {
transient boolean simpleCase;
transient StructObjectInspector inspector;
@@ -41,7 +39,7 @@ public class ExprNodeColumnEvaluator ext
transient boolean[] unionField;
public ExprNodeColumnEvaluator(ExprNodeColumnDesc expr) {
- this.expr = expr;
+ super(expr);
}
@Override
@@ -55,45 +53,44 @@ public class ExprNodeColumnEvaluator ext
simpleCase = true;
inspector = (StructObjectInspector) rowInspector;
field = inspector.getStructFieldRef(names[0]);
- return field.getFieldObjectInspector();
+ return outputOI = field.getFieldObjectInspector();
}
- else {
- simpleCase = false;
- inspectors = new StructObjectInspector[names.length];
- fields = new StructField[names.length];
- unionField = new boolean[names.length];
- int unionIndex = -1;
-
- for (int i = 0; i < names.length; i++) {
- if (i == 0) {
- inspectors[0] = (StructObjectInspector) rowInspector;
- } else {
- if (unionIndex != -1) {
- inspectors[i] = (StructObjectInspector) (
+
+ simpleCase = false;
+ inspectors = new StructObjectInspector[names.length];
+ fields = new StructField[names.length];
+ unionField = new boolean[names.length];
+ int unionIndex = -1;
+
+ for (int i = 0; i < names.length; i++) {
+ if (i == 0) {
+ inspectors[0] = (StructObjectInspector) rowInspector;
+ } else {
+ if (unionIndex != -1) {
+ inspectors[i] = (StructObjectInspector) (
(UnionObjectInspector)fields[i-1].getFieldObjectInspector()).
getObjectInspectors().get(unionIndex);
- } else {
- inspectors[i] = (StructObjectInspector) fields[i - 1]
- .getFieldObjectInspector();
- }
- }
- // to support names like _colx:1._coly
- unionfields = names[i].split("\\:");
- fields[i] = inspectors[i].getStructFieldRef(unionfields[0]);
- if (unionfields.length > 1) {
- unionIndex = Integer.parseInt(unionfields[1]);
- unionField[i] = true;
} else {
- unionIndex = -1;
- unionField[i] = false;
+ inspectors[i] = (StructObjectInspector) fields[i - 1]
+ .getFieldObjectInspector();
}
}
- return fields[names.length - 1].getFieldObjectInspector();
+ // to support names like _colx:1._coly
+ unionfields = names[i].split("\\:");
+ fields[i] = inspectors[i].getStructFieldRef(unionfields[0]);
+ if (unionfields.length > 1) {
+ unionIndex = Integer.parseInt(unionfields[1]);
+ unionField[i] = true;
+ } else {
+ unionIndex = -1;
+ unionField[i] = false;
+ }
}
+ return outputOI = fields[names.length - 1].getFieldObjectInspector();
}
@Override
- public Object evaluate(Object row) throws HiveException {
+ protected Object _evaluate(Object row, int version) throws HiveException {
if (simpleCase) {
return inspector.getStructFieldData(row, field);
}
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeConstantEvaluator.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeConstantEvaluator.java?rev=1486517&r1=1486516&r2=1486517&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeConstantEvaluator.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeConstantEvaluator.java Mon May 27 05:27:15 2013
@@ -27,13 +27,12 @@ import org.apache.hadoop.hive.serde2.obj
* ExprNodeConstantEvaluator.
*
*/
-public class ExprNodeConstantEvaluator extends ExprNodeEvaluator {
+public class ExprNodeConstantEvaluator extends ExprNodeEvaluator<ExprNodeConstantDesc> {
- protected ExprNodeConstantDesc expr;
transient ConstantObjectInspector writableObjectInspector;
public ExprNodeConstantEvaluator(ExprNodeConstantDesc expr) {
- this.expr = expr;
+ super(expr);
writableObjectInspector = expr.getWritableObjectInspector();
}
@@ -43,7 +42,7 @@ public class ExprNodeConstantEvaluator e
}
@Override
- public Object evaluate(Object row) throws HiveException {
+ protected Object _evaluate(Object row, int version) throws HiveException {
return writableObjectInspector.getWritableConstantValue();
}
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluator.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluator.java?rev=1486517&r1=1486516&r2=1486517&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluator.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluator.java Mon May 27 05:27:15 2013
@@ -19,13 +19,28 @@
package org.apache.hadoop.hive.ql.exec;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
/**
* ExprNodeEvaluator.
*
*/
-public abstract class ExprNodeEvaluator {
+public abstract class ExprNodeEvaluator<T extends ExprNodeDesc> {
+
+ protected final T expr;
+ protected ObjectInspector outputOI;
+
+ public ExprNodeEvaluator(T expr) {
+ this.expr = expr;
+ }
+
+ /**
+ * Return child evaluators if exist
+ */
+ public T getExpr() {
+ return expr;
+ }
/**
* Initialize should be called once and only once. Return the ObjectInspector
@@ -34,11 +49,40 @@ public abstract class ExprNodeEvaluator
public abstract ObjectInspector initialize(ObjectInspector rowInspector) throws HiveException;
/**
+ * Return initialized ObjectInspector. If it's not initilized, throws runtime exception
+ */
+ public ObjectInspector getOutputOI() {
+ if (outputOI == null) {
+ throw new IllegalStateException("Evaluator is not initialized");
+ }
+ return outputOI;
+ }
+
+ private transient int version = -1;
+ private transient Object evaluation;
+
+ public Object evaluate(Object row) throws HiveException {
+ return evaluate(row, -1);
+ }
+
+ /**
* Evaluate the expression given the row. This method should use the
* rowInspector passed in from initialize to inspect the row object. The
* return value will be inspected by the return value of initialize.
+ * If this evaluator is referenced by others, store it for them
*/
- public abstract Object evaluate(Object row) throws HiveException;
+ protected Object evaluate(Object row, int version) throws HiveException {
+ if (version < 0 || version != this.version) {
+ this.version = version;
+ return evaluation = _evaluate(row, version);
+ }
+ return evaluation;
+ }
+
+ /**
+ * Evaluate value
+ */
+ protected abstract Object _evaluate(Object row, int version) throws HiveException;
/**
* Return whether this node (and all children nodes) are deterministic.
@@ -46,5 +90,11 @@ public abstract class ExprNodeEvaluator
public boolean isDeterministic() {
return true;
}
-
+
+ /**
+ * Return child evaluators if exist
+ */
+ public ExprNodeEvaluator[] getChildren() {
+ return null;
+ }
}
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorFactory.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorFactory.java?rev=1486517&r1=1486516&r2=1486517&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorFactory.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorFactory.java Mon May 27 05:27:15 2013
@@ -18,6 +18,10 @@
package org.apache.hadoop.hive.ql.exec;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
@@ -34,7 +38,7 @@ public final class ExprNodeEvaluatorFact
private ExprNodeEvaluatorFactory() {
}
- public static ExprNodeEvaluator get(ExprNodeDesc desc) {
+ public static ExprNodeEvaluator get(ExprNodeDesc desc) throws HiveException {
// Constant node
if (desc instanceof ExprNodeConstantDesc) {
return new ExprNodeConstantEvaluator((ExprNodeConstantDesc) desc);
@@ -59,4 +63,56 @@ public final class ExprNodeEvaluatorFact
throw new RuntimeException(
"Cannot find ExprNodeEvaluator for the exprNodeDesc = " + desc);
}
+
+ /**
+ * Should be called before eval is initialized
+ */
+ public static ExprNodeEvaluator toCachedEval(ExprNodeEvaluator eval) {
+ if (eval instanceof ExprNodeGenericFuncEvaluator) {
+ EvaluatorContext context = new EvaluatorContext();
+ iterate(eval, context);
+ if (context.hasReference) {
+ return new ExprNodeEvaluatorHead(eval);
+ }
+ }
+ // has nothing to be cached
+ return eval;
+ }
+
+ private static ExprNodeEvaluator iterate(ExprNodeEvaluator eval, EvaluatorContext context) {
+ if (!(eval instanceof ExprNodeConstantEvaluator) && eval.isDeterministic()) {
+ ExprNodeEvaluator replace = context.getEvaluated(eval);
+ if (replace != null) {
+ return replace;
+ }
+ }
+ ExprNodeEvaluator[] children = eval.getChildren();
+ if (children != null && children.length > 0) {
+ for (int i = 0; i < children.length; i++) {
+ ExprNodeEvaluator replace = iterate(children[i], context);
+ if (replace != null) {
+ children[i] = replace;
+ }
+ }
+ }
+ return null;
+ }
+
+ private static class EvaluatorContext {
+
+ private final Map<String, ExprNodeEvaluator> cached = new HashMap<String, ExprNodeEvaluator>();
+
+ private boolean hasReference;
+
+ public ExprNodeEvaluator getEvaluated(ExprNodeEvaluator eval) {
+ String key = eval.getExpr().getExprString();
+ ExprNodeEvaluator prev = cached.get(key);
+ if (prev == null) {
+ cached.put(key, eval);
+ return null;
+ }
+ hasReference = true;
+ return new ExprNodeEvaluatorRef(prev);
+ }
+ }
}
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeFieldEvaluator.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeFieldEvaluator.java?rev=1486517&r1=1486516&r2=1486517&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeFieldEvaluator.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeFieldEvaluator.java Mon May 27 05:27:15 2013
@@ -34,9 +34,8 @@ import org.apache.hadoop.hive.serde2.obj
* is struct, then s.f is the field. If s is list of struct, then s.f is the
* list of struct field.
*/
-public class ExprNodeFieldEvaluator extends ExprNodeEvaluator {
+public class ExprNodeFieldEvaluator extends ExprNodeEvaluator<ExprNodeFieldDesc> {
- protected ExprNodeFieldDesc desc;
transient ExprNodeEvaluator leftEvaluator;
transient ObjectInspector leftInspector;
transient StructObjectInspector structObjectInspector;
@@ -44,42 +43,41 @@ public class ExprNodeFieldEvaluator exte
transient ObjectInspector structFieldObjectInspector;
transient ObjectInspector resultObjectInspector;
- public ExprNodeFieldEvaluator(ExprNodeFieldDesc desc) {
- this.desc = desc;
+ public ExprNodeFieldEvaluator(ExprNodeFieldDesc desc) throws HiveException {
+ super(desc);
leftEvaluator = ExprNodeEvaluatorFactory.get(desc.getDesc());
}
@Override
public ObjectInspector initialize(ObjectInspector rowInspector) throws HiveException {
-
leftInspector = leftEvaluator.initialize(rowInspector);
- if (desc.getIsList()) {
+ if (expr.getIsList()) {
structObjectInspector = (StructObjectInspector) ((ListObjectInspector) leftInspector)
.getListElementObjectInspector();
} else {
structObjectInspector = (StructObjectInspector) leftInspector;
}
- field = structObjectInspector.getStructFieldRef(desc.getFieldName());
+ field = structObjectInspector.getStructFieldRef(expr.getFieldName());
structFieldObjectInspector = field.getFieldObjectInspector();
- if (desc.getIsList()) {
+ if (expr.getIsList()) {
resultObjectInspector = ObjectInspectorFactory
.getStandardListObjectInspector(structFieldObjectInspector);
} else {
resultObjectInspector = structFieldObjectInspector;
}
- return resultObjectInspector;
+ return outputOI = resultObjectInspector;
}
private List<Object> cachedList = new ArrayList<Object>();
@Override
- public Object evaluate(Object row) throws HiveException {
+ protected Object _evaluate(Object row, int version) throws HiveException {
// Get the result in leftInspectableObject
- Object left = leftEvaluator.evaluate(row);
+ Object left = leftEvaluator.evaluate(row, version);
- if (desc.getIsList()) {
+ if (expr.getIsList()) {
List<?> list = ((ListObjectInspector) leftInspector).getList(left);
if (list == null) {
return null;
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java?rev=1486517&r1=1486516&r2=1486517&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeGenericFuncEvaluator.java Mon May 27 05:27:15 2013
@@ -35,16 +35,13 @@ import org.apache.hadoop.hive.serde2.obj
* ExprNodeGenericFuncEvaluator.
*
*/
-public class ExprNodeGenericFuncEvaluator extends ExprNodeEvaluator {
+public class ExprNodeGenericFuncEvaluator extends ExprNodeEvaluator<ExprNodeGenericFuncDesc> {
private static final Log LOG = LogFactory
.getLog(ExprNodeGenericFuncEvaluator.class.getName());
- protected ExprNodeGenericFuncDesc expr;
-
transient GenericUDF genericUDF;
transient Object rowObject;
- transient ObjectInspector outputOI;
transient ExprNodeEvaluator[] children;
transient GenericUDF.DeferredObject[] deferredChildren;
transient boolean isEager;
@@ -54,42 +51,38 @@ public class ExprNodeGenericFuncEvaluato
*/
class DeferredExprObject implements GenericUDF.DeferredObject {
- ExprNodeEvaluator eval;
-
- DeferredExprObject(ExprNodeEvaluator eval) {
- this.eval = eval;
- }
+ private final boolean eager;
+ private final ExprNodeEvaluator eval;
- public Object get() throws HiveException {
- return eval.evaluate(rowObject);
- }
- }
+ private transient boolean evaluated;
+ private transient int version;
+ private transient Object obj;
- /**
- * Class to force eager evaluation for GenericUDF in cases where
- * it is warranted.
- */
- class EagerExprObject implements GenericUDF.DeferredObject {
-
- ExprNodeEvaluator eval;
-
- transient Object obj;
-
- EagerExprObject(ExprNodeEvaluator eval) {
+ DeferredExprObject(ExprNodeEvaluator eval, boolean eager) {
this.eval = eval;
+ this.eager = eager;
}
- void evaluate() throws HiveException {
- obj = eval.evaluate(rowObject);
+ @Override
+ public void prepare(int version) throws HiveException {
+ this.version = version;
+ this.evaluated = false;
+ if (eager) {
+ get();
+ }
}
public Object get() throws HiveException {
+ if (!evaluated) {
+ obj = eval.evaluate(rowObject, version);
+ evaluated = true;
+ }
return obj;
}
}
- public ExprNodeGenericFuncEvaluator(ExprNodeGenericFuncDesc expr) {
- this.expr = expr;
+ public ExprNodeGenericFuncEvaluator(ExprNodeGenericFuncDesc expr) throws HiveException {
+ super(expr);
children = new ExprNodeEvaluator[expr.getChildExprs().size()];
isEager = false;
for (int i = 0; i < children.length; i++) {
@@ -109,37 +102,29 @@ public class ExprNodeGenericFuncEvaluato
}
}
}
- deferredChildren =
- new GenericUDF.DeferredObject[expr.getChildExprs().size()];
- for (int i = 0; i < deferredChildren.length; i++) {
- if (isEager) {
- deferredChildren[i] = new EagerExprObject(children[i]);
- } else {
- deferredChildren[i] = new DeferredExprObject(children[i]);
- }
+ genericUDF = expr.getGenericUDF();
+ if (isEager &&
+ (genericUDF instanceof GenericUDFCase || genericUDF instanceof GenericUDFWhen)) {
+ throw new HiveException("Stateful expressions cannot be used inside of CASE");
}
}
@Override
public ObjectInspector initialize(ObjectInspector rowInspector) throws HiveException {
+ deferredChildren = new GenericUDF.DeferredObject[children.length];
+ for (int i = 0; i < deferredChildren.length; i++) {
+ deferredChildren[i] = new DeferredExprObject(children[i], isEager);
+ }
// Initialize all children first
ObjectInspector[] childrenOIs = new ObjectInspector[children.length];
for (int i = 0; i < children.length; i++) {
childrenOIs[i] = children[i].initialize(rowInspector);
}
- genericUDF = expr.getGenericUDF();
- if (isEager &&
- ((genericUDF instanceof GenericUDFCase)
- || (genericUDF instanceof GenericUDFWhen))) {
- throw new HiveException(
- "Stateful expressions cannot be used inside of CASE");
- }
MapredContext context = MapredContext.get();
if (context != null) {
context.setup(genericUDF);
}
- this.outputOI = genericUDF.initializeAndFoldConstants(childrenOIs);
- return this.outputOI;
+ return outputOI = genericUDF.initializeAndFoldConstants(childrenOIs);
}
@Override
@@ -152,17 +137,20 @@ public class ExprNodeGenericFuncEvaluato
}
@Override
- public Object evaluate(Object row) throws HiveException {
+ public ExprNodeEvaluator[] getChildren() {
+ return children;
+ }
+
+ @Override
+ protected Object _evaluate(Object row, int version) throws HiveException {
rowObject = row;
if (ObjectInspectorUtils.isConstantObjectInspector(outputOI) &&
isDeterministic()) {
// The output of this UDF is constant, so don't even bother evaluating.
return ((ConstantObjectInspector)outputOI).getWritableConstantValue();
}
- if (isEager) {
- for (int i = 0; i < deferredChildren.length; i++) {
- ((EagerExprObject) deferredChildren[i]).evaluate();
- }
+ for (int i = 0; i < deferredChildren.length; i++) {
+ deferredChildren[i].prepare(version);
}
return genericUDF.evaluate(deferredChildren);
}
@@ -191,10 +179,8 @@ public class ExprNodeGenericFuncEvaluato
}
rowObject = row;
- if (isEager) {
- for (int i = 0; i < deferredChildren.length; i++) {
- ((EagerExprObject) deferredChildren[i]).evaluate();
- }
+ for (int i = 0; i < deferredChildren.length; i++) {
+ deferredChildren[i].prepare(-1);
}
return ((GenericUDFBaseCompare)genericUDF).compare(deferredChildren);
}
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeNullEvaluator.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeNullEvaluator.java?rev=1486517&r1=1486516&r2=1486517&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeNullEvaluator.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeNullEvaluator.java Mon May 27 05:27:15 2013
@@ -29,22 +29,19 @@ import org.apache.hadoop.hive.serde2.obj
* ExprNodeNullEvaluator.
*
*/
-public class ExprNodeNullEvaluator extends ExprNodeEvaluator {
-
- protected ExprNodeNullDesc expr;
+public class ExprNodeNullEvaluator extends ExprNodeEvaluator<ExprNodeNullDesc> {
public ExprNodeNullEvaluator(ExprNodeNullDesc expr) {
- this.expr = expr;
+ super(expr);
}
@Override
public ObjectInspector initialize(ObjectInspector rowInspector) throws HiveException {
- return PrimitiveObjectInspectorFactory.writableVoidObjectInspector;
+ return outputOI = PrimitiveObjectInspectorFactory.writableVoidObjectInspector;
}
@Override
- public Object evaluate(Object row) throws HiveException {
+ protected Object _evaluate(Object row, int version) throws HiveException {
return null;
}
-
}
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java?rev=1486517&r1=1486516&r2=1486517&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/FetchOperator.java Mon May 27 05:27:15 2013
@@ -648,8 +648,7 @@ public class FetchOperator implements Se
* @return list of file status entries
*/
private FileStatus[] listStatusUnderPath(FileSystem fs, Path p) throws IOException {
- HiveConf hiveConf = new HiveConf(job, FetchOperator.class);
- boolean recursive = hiveConf.getBoolVar(HiveConf.ConfVars.HADOOPMAPREDINPUTDIRRECURSIVE);
+ boolean recursive = HiveConf.getBoolVar(job, HiveConf.ConfVars.HADOOPMAPREDINPUTDIRRECURSIVE);
if (!recursive) {
return fs.listStatus(p);
}
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java?rev=1486517&r1=1486516&r2=1486517&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/FilterOperator.java Mon May 27 05:27:15 2013
@@ -68,6 +68,10 @@ public class FilterOperator extends Oper
heartbeatInterval = HiveConf.getIntVar(hconf,
HiveConf.ConfVars.HIVESENDHEARTBEAT);
conditionEvaluator = ExprNodeEvaluatorFactory.get(conf.getPredicate());
+ if (HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVEEXPREVALUATIONCACHE)) {
+ conditionEvaluator = ExprNodeEvaluatorFactory.toCachedEval(conditionEvaluator);
+ }
+
statsMap.put(Counter.FILTERED, filtered_count);
statsMap.put(Counter.PASSED, passed_count);
conditionInspector = null;
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java?rev=1486517&r1=1486516&r2=1486517&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/JoinUtil.java Mon May 27 05:27:15 2013
@@ -98,14 +98,14 @@ public class JoinUtil {
}
public static int populateJoinKeyValue(List<ExprNodeEvaluator>[] outMap,
- Map<Byte, List<ExprNodeDesc>> inputMap, int posBigTableAlias) {
+ Map<Byte, List<ExprNodeDesc>> inputMap, int posBigTableAlias) throws HiveException {
return populateJoinKeyValue(outMap, inputMap, null, posBigTableAlias);
}
public static int populateJoinKeyValue(List<ExprNodeEvaluator>[] outMap,
Map<Byte, List<ExprNodeDesc>> inputMap,
Byte[] order,
- int posBigTableAlias) {
+ int posBigTableAlias) throws HiveException {
int total = 0;
for (Entry<Byte, List<ExprNodeDesc>> e : inputMap.entrySet()) {
Byte key = order == null ? e.getKey() : order[e.getKey()];
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java?rev=1486517&r1=1486516&r2=1486517&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/SMBMapJoinOperator.java Mon May 27 05:27:15 2013
@@ -123,8 +123,18 @@ public class SMBMapJoinOperator extends
fetchDone = new boolean[maxAlias];
foundNextKeyGroup = new boolean[maxAlias];
- int bucketSize = HiveConf.getIntVar(hconf,
- HiveConf.ConfVars.HIVEMAPJOINBUCKETCACHESIZE);
+ int bucketSize;
+
+ // For backwards compatibility reasons we honor the older
+ // HIVEMAPJOINBUCKETCACHESIZE if set different from default.
+ // By hive 0.13 we should remove this code.
+ int oldVar = HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVEMAPJOINBUCKETCACHESIZE);
+ if (oldVar != 100) {
+ bucketSize = oldVar;
+ } else {
+ bucketSize = HiveConf.getIntVar(hconf, HiveConf.ConfVars.HIVESMBJOINCACHEROWS);
+ }
+
for (byte pos = 0; pos < order.length; pos++) {
RowContainer rc = JoinUtil.getRowContainer(hconf,
rowContainerStandardObjectInspectors[pos],
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java?rev=1486517&r1=1486516&r2=1486517&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/exec/SelectOperator.java Mon May 27 05:27:15 2013
@@ -22,6 +22,7 @@ import java.io.Serializable;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.SelectDesc;
@@ -52,6 +53,9 @@ public class SelectOperator extends Oper
for (int i = 0; i < colList.size(); i++) {
assert (colList.get(i) != null);
eval[i] = ExprNodeEvaluatorFactory.get(colList.get(i));
+ if (HiveConf.getBoolVar(hconf, HiveConf.ConfVars.HIVEEXPREVALUATIONCACHE)) {
+ eval[i] = ExprNodeEvaluatorFactory.toCachedEval(eval[i]);
+ }
}
output = new Object[eval.length];
Modified: hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java
URL: http://svn.apache.org/viewvc/hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java?rev=1486517&r1=1486516&r2=1486517&view=diff
==============================================================================
--- hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java (original)
+++ hive/branches/HIVE-4115/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java Mon May 27 05:27:15 2013
@@ -303,7 +303,7 @@ abstract public class AbstractBucketJoin
// The number of files for the table should be same as number of buckets.
int bucketCount = p.getBucketCount();
- if (fileNames.size() != bucketCount) {
+ if (fileNames.size() != 0 && fileNames.size() != bucketCount) {
String msg = "The number of buckets for table " +
tbl.getTableName() + " partition " + p.getName() + " is " +
p.getBucketCount() + ", whereas the number of files is " + fileNames.size();
@@ -333,7 +333,7 @@ abstract public class AbstractBucketJoin
Integer num = new Integer(tbl.getNumBuckets());
// The number of files for the table should be same as number of buckets.
- if (fileNames.size() != num) {
+ if (fileNames.size() != 0 && fileNames.size() != num) {
String msg = "The number of buckets for table " +
tbl.getTableName() + " is " + tbl.getNumBuckets() +
", whereas the number of files is " + fileNames.size();