You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jd...@apache.org on 2016/05/06 17:25:21 UTC
[31/39] hive git commit: Merge branch 'master' into llap
Merge branch 'master' into llap
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/e073cce2
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/e073cce2
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/e073cce2
Branch: refs/heads/master
Commit: e073cce2354801e12c4767f47a76fbf25168d026
Parents: 342668f e9a7218
Author: Jason Dere <jd...@hortonworks.com>
Authored: Mon May 2 18:29:03 2016 -0700
Committer: Jason Dere <jd...@hortonworks.com>
Committed: Mon May 2 18:29:03 2016 -0700
----------------------------------------------------------------------
.../apache/hive/beeline/DatabaseConnection.java | 12 +-
.../org/apache/hadoop/hive/cli/CliDriver.java | 5 +-
.../org/apache/hadoop/hive/conf/HiveConf.java | 19 +
data/files/keystore.jks | Bin 2248 -> 2206 bytes
data/files/keystore_exampledotcom.jks | Bin 0 -> 2222 bytes
data/files/struct1_a.txt | 4 +
data/files/struct1_b.txt | 1 +
data/files/struct1_c.txt | 1 +
data/files/struct2_a.txt | 2 +
data/files/struct2_b.txt | 2 +
data/files/struct2_c.txt | 1 +
data/files/struct2_d.txt | 1 +
data/files/struct3_a.txt | 2 +
data/files/struct3_b.txt | 1 +
data/files/struct3_c.txt | 1 +
data/files/struct4_a.txt | 2 +
data/files/struct4_b.txt | 1 +
data/files/struct4_c.txt | 1 +
data/files/truststore.jks | Bin 958 -> 1816 bytes
.../hcatalog/templeton/ExecServiceImpl.java | 7 +-
.../org/apache/hive/jdbc/miniHS2/MiniHS2.java | 35 +-
.../org/apache/hive/jdbc/TestJdbcDriver2.java | 25 +-
.../test/java/org/apache/hive/jdbc/TestSSL.java | 71 +-
itests/pom.xml | 3 -
.../test/resources/testconfiguration.properties | 34 +-
.../org/apache/hive/jdbc/HiveConnection.java | 99 +-
.../java/org/apache/hive/jdbc/JdbcColumn.java | 13 +-
.../hadoop/hive/llap/cli/LlapServiceDriver.java | 21 +-
.../llap/cli/LlapStatusOptionsProcessor.java | 44 +-
.../hive/llap/cli/LlapStatusServiceDriver.java | 74 +-
.../daemon/impl/LlapProtocolServerImpl.java | 57 +-
.../hive/llap/io/api/impl/LlapInputFormat.java | 2 +-
.../main/resources/llap-cli-log4j2.properties | 2 +-
llap-server/src/main/resources/package.py | 7 +-
packaging/src/main/assembly/bin.xml | 5 +
.../hive/ql/exec/AbstractMapOperator.java | 178 +
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 7 +
.../hadoop/hive/ql/exec/FunctionRegistry.java | 8 +
.../apache/hadoop/hive/ql/exec/MapOperator.java | 86 +-
.../apache/hadoop/hive/ql/exec/Utilities.java | 33 +-
.../hadoop/hive/ql/exec/mr/ExecMapper.java | 3 +-
.../ql/exec/spark/SparkMapRecordHandler.java | 3 +-
.../hadoop/hive/ql/exec/tez/DagUtils.java | 3 +-
.../hive/ql/exec/tez/MapRecordProcessor.java | 15 +-
.../hive/ql/exec/tez/MapRecordSource.java | 6 +-
.../hive/ql/exec/tez/ReduceRecordSource.java | 4 +-
.../vector/VectorAppMasterEventOperator.java | 16 +-
.../hive/ql/exec/vector/VectorAssignRow.java | 1111 ++--
.../ql/exec/vector/VectorAssignRowDynBatch.java | 41 -
.../exec/vector/VectorAssignRowSameBatch.java | 36 -
.../ql/exec/vector/VectorDeserializeRow.java | 1114 ++--
.../hive/ql/exec/vector/VectorExtractRow.java | 971 +---
.../exec/vector/VectorExtractRowDynBatch.java | 40 -
.../exec/vector/VectorExtractRowSameBatch.java | 36 -
.../ql/exec/vector/VectorFileSinkOperator.java | 16 +-
.../ql/exec/vector/VectorGroupByOperator.java | 13 +-
.../exec/vector/VectorMapJoinBaseOperator.java | 11 +-
.../ql/exec/vector/VectorMapJoinOperator.java | 4 +-
.../VectorMapJoinOuterFilteredOperator.java | 17 +-
.../hive/ql/exec/vector/VectorMapOperator.java | 848 ++-
.../exec/vector/VectorReduceSinkOperator.java | 16 +-
.../exec/vector/VectorSMBMapJoinOperator.java | 11 +-
.../VectorSparkHashTableSinkOperator.java | 16 +-
...VectorSparkPartitionPruningSinkOperator.java | 13 +-
.../ql/exec/vector/VectorizationContext.java | 12 +-
.../ql/exec/vector/VectorizedBatchUtil.java | 49 +
.../VectorMapJoinGenerateResultOperator.java | 8 +-
.../fast/VectorMapJoinFastLongHashUtil.java | 10 +-
.../fast/VectorMapJoinFastStringCommon.java | 10 +-
.../hadoop/hive/ql/io/HiveInputFormat.java | 6 +-
.../hadoop/hive/ql/io/NullRowsInputFormat.java | 2 +-
.../hadoop/hive/ql/io/orc/OrcInputFormat.java | 4 +-
.../ql/io/parquet/MapredParquetInputFormat.java | 2 +-
.../hive/ql/optimizer/physical/Vectorizer.java | 334 +-
.../hadoop/hive/ql/parse/FromClauseParser.g | 4 +-
.../hadoop/hive/ql/parse/IdentifiersParser.g | 6 +-
.../hadoop/hive/ql/parse/SelectClauseParser.g | 4 +-
.../apache/hadoop/hive/ql/plan/BaseWork.java | 23 +
.../org/apache/hadoop/hive/ql/plan/MapWork.java | 11 +
.../org/apache/hadoop/hive/ql/plan/TezWork.java | 9 +-
.../hive/ql/plan/VectorPartitionConversion.java | 172 +-
.../hive/ql/plan/VectorPartitionDesc.java | 164 +-
.../hadoop/hive/ql/udf/generic/BaseMaskUDF.java | 473 ++
.../hive/ql/udf/generic/GenericUDFMask.java | 334 ++
.../ql/udf/generic/GenericUDFMaskFirstN.java | 229 +
.../hive/ql/udf/generic/GenericUDFMaskHash.java | 77 +
.../ql/udf/generic/GenericUDFMaskLastN.java | 193 +
.../udf/generic/GenericUDFMaskShowFirstN.java | 248 +
.../ql/udf/generic/GenericUDFMaskShowLastN.java | 198 +
.../ql/exec/vector/TestVectorRowObject.java | 14 +-
.../hive/ql/exec/vector/TestVectorSerDeRow.java | 169 +-
.../hive/ql/io/orc/TestInputOutputFormat.java | 10 +-
.../hive/ql/plan/TestTezWorkConcurrency.java | 65 +
.../avro_schema_evolution_native.q | 18 +
.../queries/clientpositive/bucket_groupby.q | 33 +-
.../queries/clientpositive/groupby_sort_10.q | 2 +
.../schema_evol_orc_acidvec_mapwork_part.q | 3 +
.../schema_evol_orc_acidvec_mapwork_table.q | 3 +
.../schema_evol_orc_nonvec_mapwork_table.q | 2 -
.../schema_evol_orc_vec_mapwork_part.q | 3 +
.../schema_evol_orc_vec_mapwork_table.q | 7 +-
.../schema_evol_text_fetchwork_table.q | 56 -
.../schema_evol_text_mapwork_table.q | 56 -
.../schema_evol_text_nonvec_fetchwork_part.q | 98 -
.../schema_evol_text_nonvec_fetchwork_table.q | 67 -
.../schema_evol_text_nonvec_mapwork_part.q | 828 ++-
..._evol_text_nonvec_mapwork_part_all_complex.q | 159 +
...vol_text_nonvec_mapwork_part_all_primitive.q | 509 ++
.../schema_evol_text_nonvec_mapwork_table.q | 822 ++-
.../schema_evol_text_vec_mapwork_part.q | 827 +++
...ema_evol_text_vec_mapwork_part_all_complex.q | 164 +
...a_evol_text_vec_mapwork_part_all_primitive.q | 514 ++
.../schema_evol_text_vec_mapwork_table.q | 826 +++
.../schema_evol_text_vecrow_mapwork_part.q | 827 +++
..._evol_text_vecrow_mapwork_part_all_complex.q | 165 +
...vol_text_vecrow_mapwork_part_all_primitive.q | 514 ++
.../schema_evol_text_vecrow_mapwork_table.q | 826 +++
.../clientpositive/tez_schema_evolution.q | 1 +
ql/src/test/queries/clientpositive/udf_mask.q | 13 +
.../queries/clientpositive/udf_mask_first_n.q | 13 +
.../test/queries/clientpositive/udf_mask_hash.q | 13 +
.../queries/clientpositive/udf_mask_last_n.q | 13 +
.../clientpositive/udf_mask_show_first_n.q | 13 +
.../clientpositive/udf_mask_show_last_n.q | 13 +
.../beelinepositive/show_functions.q.out | 6 +
.../results/clientnegative/subq_insert.q.out | 2 +-
.../avro_schema_evolution_native.q.out | 206 +
.../results/clientpositive/bucket_groupby.q.out | 308 +-
.../results/clientpositive/groupby1_limit.q.out | 2 +-
.../clientpositive/groupby_sort_10.q.out | 8 +-
.../results/clientpositive/regexp_extract.q.out | 137 -
.../schema_evol_text_fetchwork_table.q.out | 298 --
.../schema_evol_text_mapwork_table.q.out | 298 --
...schema_evol_text_nonvec_fetchwork_part.q.out | 642 ---
...chema_evol_text_nonvec_fetchwork_table.q.out | 297 --
.../schema_evol_text_nonvec_mapwork_part.q.out | 4909 ++++++++++++++++--
...l_text_nonvec_mapwork_part_all_complex.q.out | 726 +++
...text_nonvec_mapwork_part_all_primitive.q.out | 3038 +++++++++++
.../schema_evol_text_nonvec_mapwork_table.q.out | 4376 +++++++++++++++-
.../schema_evol_text_vec_mapwork_part.q.out | 4479 ++++++++++++++++
...evol_text_vec_mapwork_part_all_complex.q.out | 730 +++
...ol_text_vec_mapwork_part_all_primitive.q.out | 3058 +++++++++++
.../schema_evol_text_vec_mapwork_table.q.out | 4221 +++++++++++++++
.../schema_evol_text_vecrow_mapwork_part.q.out | 4479 ++++++++++++++++
...l_text_vecrow_mapwork_part_all_complex.q.out | 732 +++
...text_vecrow_mapwork_part_all_primitive.q.out | 3058 +++++++++++
.../schema_evol_text_vecrow_mapwork_table.q.out | 4221 +++++++++++++++
.../results/clientpositive/show_functions.q.out | 12 +
.../tez/schema_evol_text_fetchwork_table.q.out | 298 --
.../tez/schema_evol_text_mapwork_table.q.out | 298 --
...schema_evol_text_nonvec_fetchwork_part.q.out | 642 ---
...chema_evol_text_nonvec_fetchwork_table.q.out | 297 --
.../schema_evol_text_nonvec_mapwork_part.q.out | 4453 ++++++++++++++--
...l_text_nonvec_mapwork_part_all_complex.q.out | 669 +++
...text_nonvec_mapwork_part_all_primitive.q.out | 2734 ++++++++++
.../schema_evol_text_nonvec_mapwork_table.q.out | 3920 +++++++++++++-
.../tez/schema_evol_text_vec_mapwork_part.q.out | 3999 ++++++++++++++
...evol_text_vec_mapwork_part_all_complex.q.out | 673 +++
...ol_text_vec_mapwork_part_all_primitive.q.out | 2738 ++++++++++
.../schema_evol_text_vec_mapwork_table.q.out | 3741 +++++++++++++
.../schema_evol_text_vecrow_mapwork_part.q.out | 3999 ++++++++++++++
...l_text_vecrow_mapwork_part_all_complex.q.out | 675 +++
...text_vecrow_mapwork_part_all_primitive.q.out | 2738 ++++++++++
.../schema_evol_text_vecrow_mapwork_table.q.out | 3741 +++++++++++++
.../vector_orc_string_reader_empty_dict.q.out | 62 +
.../tez/vector_partition_diff_num_cols.q.out | 1 +
.../tez/vector_tablesample_rows.q.out | 307 ++
.../test/results/clientpositive/udf_mask.q.out | 71 +
.../clientpositive/udf_mask_first_n.q.out | 68 +
.../results/clientpositive/udf_mask_hash.q.out | 59 +
.../clientpositive/udf_mask_last_n.q.out | 68 +
.../clientpositive/udf_mask_show_first_n.q.out | 68 +
.../clientpositive/udf_mask_show_last_n.q.out | 68 +
.../vector_partition_diff_num_cols.q.out | 1 +
.../vector_tablesample_rows.q.out | 2 -
.../compiler/errors/missing_overwrite.q.out | 2 +-
.../fast/BinarySortableDeserializeRead.java | 806 +--
.../hive/serde2/fast/DeserializeRead.java | 379 +-
.../lazy/fast/LazySimpleDeserializeRead.java | 704 +--
.../fast/LazyBinaryDeserializeRead.java | 944 +---
.../apache/hadoop/hive/serde2/VerifyFast.java | 75 +-
.../hive/service/auth/HiveAuthFactory.java | 21 +-
182 files changed, 84895 insertions(+), 10148 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/e073cce2/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
----------------------------------------------------------------------
diff --cc common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index 7db492f,caadf2a..5f78ae6
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@@ -2658,13 -2670,17 +2670,20 @@@ public class HiveConf extends Configura
"ZooKeeper for ZooKeeper SecretManager."),
LLAP_ZKSM_ZK_CONNECTION_STRING("hive.llap.zk.sm.connectionString", "",
"ZooKeeper connection string for ZooKeeper SecretManager."),
+ LLAP_ZK_REGISTRY_USER("hive.llap.zk.registry.user", "",
+ "In the LLAP ZooKeeper-based registry, specifies the username in the Zookeeper path.\n" +
+ "This should be the hive user or whichever user is running the LLAP daemon."),
// Note: do not rename to ..service.acl; Hadoop generates .hosts setting name from this,
// resulting in a collision with existing hive.llap.daemon.service.hosts and bizarre errors.
+ // These are read by Hadoop IPC, so you should check the usage and naming conventions (e.g.
+ // ".blocked" is a string hardcoded by Hadoop, and defaults are enforced elsewhere in Hive)
+ // before making changes or copy-pasting these.
LLAP_SECURITY_ACL("hive.llap.daemon.acl", "*", "The ACL for LLAP daemon."),
+ LLAP_SECURITY_ACL_DENY("hive.llap.daemon.acl.blocked", "", "The deny ACL for LLAP daemon."),
LLAP_MANAGEMENT_ACL("hive.llap.management.acl", "*", "The ACL for LLAP daemon management."),
+ LLAP_MANAGEMENT_ACL_DENY("hive.llap.management.acl.blocked", "",
+ "The deny ACL for LLAP daemon management."),
+
// Hadoop DelegationTokenManager default is 1 week.
LLAP_DELEGATION_TOKEN_LIFETIME("hive.llap.daemon.delegation.token.lifetime", "14d",
new TimeValidator(TimeUnit.SECONDS),
http://git-wip-us.apache.org/repos/asf/hive/blob/e073cce2/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/e073cce2/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java
index 7a3d6a6,9a9f43a..dc63d7b
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/MapRecordProcessor.java
@@@ -33,9 -33,9 +33,10 @@@ import org.slf4j.Logger
import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
+ import org.apache.hadoop.hive.ql.exec.AbstractMapOperator;
import org.apache.hadoop.hive.llap.io.api.LlapProxy;
import org.apache.hadoop.hive.ql.CompilationOpContext;
+import org.apache.hadoop.hive.llap.LlapOutputFormat;
import org.apache.hadoop.hive.ql.exec.DummyStoreOperator;
import org.apache.hadoop.hive.ql.exec.HashTableDummyOperator;
import org.apache.hadoop.hive.ql.exec.MapOperator;