You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2015/09/30 21:29:26 UTC
[14/14] hive git commit: HIVE-12000 : LLAP: Merge master into branch
(Sergey Shelukhin)
HIVE-12000 : LLAP: Merge master into branch (Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f272ccb2
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f272ccb2
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f272ccb2
Branch: refs/heads/llap
Commit: f272ccb25bc495d600a713058f6c2082ebd6d966
Parents: 7273a4c 064e37c
Author: Sergey Shelukhin <se...@apache.org>
Authored: Wed Sep 30 12:22:32 2015 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Wed Sep 30 12:22:32 2015 -0700
----------------------------------------------------------------------
.../common/metrics/common/MetricsConstant.java | 1 +
.../hive/hcatalog/templeton/AppConfig.java | 21 +++
.../apache/hive/hcatalog/templeton/Server.java | 12 +-
itests/qtest/pom.xml | 2 +-
.../test/resources/testconfiguration.properties | 27 ++-
.../hadoop/hive/ql/io/orc/MetadataReader.java | 2 +-
.../hive/ql/io/orc/MetadataReaderImpl.java | 20 ++-
.../hadoop/hive/ql/io/orc/OrcInputFormat.java | 49 ++++++
.../hadoop/hive/ql/io/orc/ReaderImpl.java | 12 +-
.../hadoop/hive/ql/io/orc/RecordReaderImpl.java | 35 ++++
.../hive/ql/io/orc/RecordReaderUtils.java | 5 +-
.../hive/ql/optimizer/StatsOptimizer.java | 46 ++++-
.../ql/optimizer/calcite/RelOptHiveTable.java | 15 ++
.../ql/optimizer/physical/Vectorizer.java.rej | 86 ----------
.../hadoop/hive/ql/session/SessionState.java | 9 +-
.../test/queries/clientpositive/add_jar_pfile.q | 8 +
.../clientpositive/metadata_only_queries.q | 15 ++
.../results/clientpositive/add_jar_pfile.q.out | 12 ++
.../clientpositive/metadata_only_queries.q.out | 158 +++++++++++++++++
.../spark/metadata_only_queries.q.out | 170 +++++++++++++++++++
.../tez/metadata_only_queries.q.out | 170 +++++++++++++++++++
.../tez/vector_groupby_reduce.q.out | 70 +++++---
.../clientpositive/vector_groupby_reduce.q.out | 69 +++++---
.../hive/service/cli/operation/Operation.java | 11 ++
.../service/cli/operation/OperationManager.java | 11 ++
.../service/cli/session/HiveSessionProxy.java | 6 +
.../service/cli/session/SessionManager.java | 6 +-
.../thrift/EmbeddedThriftBinaryCLIService.java | 2 +-
.../thrift/ThreadPoolExecutorWithOomHook.java | 55 ++++++
.../cli/thrift/ThriftBinaryCLIService.java | 12 +-
.../service/cli/thrift/ThriftCLIService.java | 3 +
.../cli/thrift/ThriftHttpCLIService.java | 10 +-
.../apache/hive/service/server/HiveServer2.java | 12 +-
.../hive/service/auth/TestPlainSaslHelper.java | 2 +-
.../session/TestPluggableHiveSessionImpl.java | 2 +-
.../cli/session/TestSessionGlobalInitFile.java | 2 +-
36 files changed, 957 insertions(+), 191 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f272ccb2/itests/qtest/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f272ccb2/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f272ccb2/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java
index 35cc05f,43d2933..cea324c
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReader.java
@@@ -18,17 -18,102 +18,17 @@@
package org.apache.hadoop.hive.ql.io.orc;
import java.io.IOException;
-import java.nio.ByteBuffer;
-import java.util.List;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hive.common.DiskRange;
-import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl.BufferChunk;
+import org.apache.hadoop.hive.ql.io.orc.OrcProto.BloomFilterIndex;
+import org.apache.hadoop.hive.ql.io.orc.OrcProto.RowIndex;
+import org.apache.hadoop.hive.ql.io.orc.OrcProto.StripeFooter;
-import com.google.common.collect.Lists;
+public interface MetadataReader {
+ RecordReaderImpl.Index readRowIndex(StripeInformation stripe, StripeFooter footer,
+ boolean[] included, RowIndex[] indexes, boolean[] sargColumns,
+ BloomFilterIndex[] bloomFilterIndices) throws IOException;
-public class MetadataReader {
- private final FSDataInputStream file;
- private final CompressionCodec codec;
- private final int bufferSize;
- private final int typeCount;
+ StripeFooter readStripeFooter(StripeInformation stripe) throws IOException;
- public MetadataReader(FileSystem fileSystem, Path path,
- CompressionCodec codec, int bufferSize, int typeCount) throws IOException {
- this(fileSystem.open(path), codec, bufferSize, typeCount);
- }
-
- public MetadataReader(FSDataInputStream file,
- CompressionCodec codec, int bufferSize, int typeCount) {
- this.file = file;
- this.codec = codec;
- this.bufferSize = bufferSize;
- this.typeCount = typeCount;
- }
-
- public RecordReaderImpl.Index readRowIndex(StripeInformation stripe, OrcProto.StripeFooter footer,
- boolean[] included, OrcProto.RowIndex[] indexes, boolean[] sargColumns,
- OrcProto.BloomFilterIndex[] bloomFilterIndices) throws IOException {
- if (footer == null) {
- footer = readStripeFooter(stripe);
- }
- if (indexes == null) {
- indexes = new OrcProto.RowIndex[typeCount];
- }
- if (bloomFilterIndices == null) {
- bloomFilterIndices = new OrcProto.BloomFilterIndex[typeCount];
- }
- long offset = stripe.getOffset();
- List<OrcProto.Stream> streams = footer.getStreamsList();
- for (int i = 0; i < streams.size(); i++) {
- OrcProto.Stream stream = streams.get(i);
- OrcProto.Stream nextStream = null;
- if (i < streams.size() - 1) {
- nextStream = streams.get(i+1);
- }
- int col = stream.getColumn();
- int len = (int) stream.getLength();
- // row index stream and bloom filter are interlaced, check if the sarg column contains bloom
- // filter and combine the io to read row index and bloom filters for that column together
- if (stream.hasKind() && (stream.getKind() == OrcProto.Stream.Kind.ROW_INDEX)) {
- boolean readBloomFilter = false;
- if (sargColumns != null && sargColumns[col] &&
- nextStream.getKind() == OrcProto.Stream.Kind.BLOOM_FILTER) {
- len += nextStream.getLength();
- i += 1;
- readBloomFilter = true;
- }
- if ((included == null || included[col]) && indexes[col] == null) {
- byte[] buffer = new byte[len];
- file.readFully(offset, buffer, 0, buffer.length);
- ByteBuffer[] bb = new ByteBuffer[] {ByteBuffer.wrap(buffer)};
- indexes[col] = OrcProto.RowIndex.parseFrom(InStream.create("index",
- bb, new long[]{0}, stream.getLength(), codec, bufferSize));
- if (readBloomFilter) {
- bb[0].position((int) stream.getLength());
- bloomFilterIndices[col] = OrcProto.BloomFilterIndex.parseFrom(
- InStream.create("bloom_filter", bb, new long[]{0}, nextStream.getLength(),
- codec, bufferSize));
- }
- }
- }
- offset += len;
- }
-
- RecordReaderImpl.Index index = new RecordReaderImpl.Index(indexes, bloomFilterIndices);
- return index;
- }
-
- public OrcProto.StripeFooter readStripeFooter(StripeInformation stripe) throws IOException {
- long offset = stripe.getOffset() + stripe.getIndexLength() + stripe.getDataLength();
- int tailLength = (int) stripe.getFooterLength();
-
- // read the footer
- ByteBuffer tailBuf = ByteBuffer.allocate(tailLength);
- file.readFully(offset, tailBuf.array(), tailBuf.arrayOffset(), tailLength);
- return OrcProto.StripeFooter.parseFrom(InStream.create("footer",
- Lists.<DiskRange>newArrayList(new BufferChunk(tailBuf, 0)),
- tailLength, codec, bufferSize));
- }
-
- public void close() throws IOException {
- file.close();
- }
+ void close() throws IOException;
- }
+ }
http://git-wip-us.apache.org/repos/asf/hive/blob/f272ccb2/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReaderImpl.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReaderImpl.java
index 83594f7,0000000..1456df3
mode 100644,000000..100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReaderImpl.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/MetadataReaderImpl.java
@@@ -1,119 -1,0 +1,123 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.io.orc;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.List;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.common.io.DiskRange;
+import org.apache.hadoop.hive.ql.io.orc.RecordReaderImpl.BufferChunk;
+
+import com.google.common.collect.Lists;
+
+public class MetadataReaderImpl implements MetadataReader {
+ private final FSDataInputStream file;
+ private final CompressionCodec codec;
+ private final int bufferSize;
+ private final int typeCount;
+
- public MetadataReaderImpl(FileSystem fileSystem, Path path, CompressionCodec codec,
- int bufferSize, int typeCount) throws IOException {
- this.file = fileSystem.open(path);
++ public MetadataReaderImpl(FileSystem fileSystem, Path path,
++ CompressionCodec codec, int bufferSize, int typeCount) throws IOException {
++ this(fileSystem.open(path), codec, bufferSize, typeCount);
++ }
++
++ public MetadataReaderImpl(FSDataInputStream file,
++ CompressionCodec codec, int bufferSize, int typeCount) {
++ this.file = file;
+ this.codec = codec;
+ this.bufferSize = bufferSize;
+ this.typeCount = typeCount;
+ }
+
+ @Override
+ public RecordReaderImpl.Index readRowIndex(StripeInformation stripe,
+ OrcProto.StripeFooter footer, boolean[] included, OrcProto.RowIndex[] indexes,
+ boolean[] sargColumns, OrcProto.BloomFilterIndex[] bloomFilterIndices) throws IOException {
+ if (footer == null) {
+ footer = readStripeFooter(stripe);
+ }
+ if (indexes == null) {
+ indexes = new OrcProto.RowIndex[typeCount];
+ }
+ if (bloomFilterIndices == null) {
+ bloomFilterIndices = new OrcProto.BloomFilterIndex[typeCount];
+ }
+ long offset = stripe.getOffset();
+ List<OrcProto.Stream> streams = footer.getStreamsList();
+ for (int i = 0; i < streams.size(); i++) {
+ OrcProto.Stream stream = streams.get(i);
+ OrcProto.Stream nextStream = null;
+ if (i < streams.size() - 1) {
+ nextStream = streams.get(i+1);
+ }
+ int col = stream.getColumn();
+ int len = (int) stream.getLength();
+ // row index stream and bloom filter are interlaced, check if the sarg column contains bloom
+ // filter and combine the io to read row index and bloom filters for that column together
+ if (stream.hasKind() && (stream.getKind() == OrcProto.Stream.Kind.ROW_INDEX)) {
+ boolean readBloomFilter = false;
+ if (sargColumns != null && sargColumns[col] &&
+ nextStream.getKind() == OrcProto.Stream.Kind.BLOOM_FILTER) {
+ len += nextStream.getLength();
+ i += 1;
+ readBloomFilter = true;
+ }
+ if ((included == null || included[col]) && indexes[col] == null) {
+ byte[] buffer = new byte[len];
++ file.readFully(offset, buffer, 0, buffer.length);
+ ByteBuffer bb = ByteBuffer.wrap(buffer);
- file.seek(offset);
- file.readFully(buffer);
+ indexes[col] = OrcProto.RowIndex.parseFrom(InStream.create(null, "index",
+ Lists.<DiskRange>newArrayList(new BufferChunk(bb, 0)), stream.getLength(),
- codec, bufferSize));
++ codec, bufferSize));
+ if (readBloomFilter) {
+ bb.position((int) stream.getLength());
+ bloomFilterIndices[col] = OrcProto.BloomFilterIndex.parseFrom(InStream.create(
+ null, "bloom_filter", Lists.<DiskRange>newArrayList(new BufferChunk(bb, 0)),
+ nextStream.getLength(), codec, bufferSize));
+ }
+ }
+ }
+ offset += len;
+ }
+
+ RecordReaderImpl.Index index = new RecordReaderImpl.Index(indexes, bloomFilterIndices);
+ return index;
+ }
+
+ @Override
+ public OrcProto.StripeFooter readStripeFooter(StripeInformation stripe) throws IOException {
+ long offset = stripe.getOffset() + stripe.getIndexLength() + stripe.getDataLength();
+ int tailLength = (int) stripe.getFooterLength();
++
+ // read the footer
+ ByteBuffer tailBuf = ByteBuffer.allocate(tailLength);
- file.seek(offset);
- file.readFully(tailBuf.array(), tailBuf.arrayOffset(), tailLength);
++ file.readFully(offset, tailBuf.array(), tailBuf.arrayOffset(), tailLength);
+ return OrcProto.StripeFooter.parseFrom(InStream.create(null, "footer",
+ Lists.<DiskRange>newArrayList(new BufferChunk(tailBuf, 0)),
+ tailLength, codec, bufferSize));
+ }
+
+ @Override
+ public void close() throws IOException {
+ file.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/f272ccb2/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f272ccb2/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ReaderImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f272ccb2/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f272ccb2/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderUtils.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f272ccb2/ql/src/test/results/clientpositive/tez/vector_groupby_reduce.q.out
----------------------------------------------------------------------