You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by om...@apache.org on 2017/04/26 22:07:34 UTC
[2/6] hive git commit: HIVE-15419. Separate storage-api to be
released independently. (omalley reviewed by Prasanth Jayachandran and Eugene
Koifman)
HIVE-15419. Separate storage-api to be released independently. (omalley
reviewed by Prasanth Jayachandran and Eugene Koifman)
Fixes #125
Signed-off-by: Owen O'Malley <om...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0d045535
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0d045535
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0d045535
Branch: refs/heads/storage-branch-2.3
Commit: 0d045535d42c62b7521a73427365d875e40fead5
Parents: 1b74102
Author: Owen O'Malley <om...@apache.org>
Authored: Fri Dec 16 16:11:02 2016 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Wed Apr 26 12:14:39 2017 -0700
----------------------------------------------------------------------
common/pom.xml | 1 -
.../hive/common/util/TestBloomFilter.java | 589 -------------------
orc/pom.xml | 1 -
packaging/pom.xml | 1 -
pom.xml | 6 +
storage-api/pom.xml | 91 ++-
.../org/apache/hadoop/hive/common/Pool.java | 8 +-
.../apache/hadoop/hive/common/io/Allocator.java | 16 +-
.../apache/hadoop/hive/common/io/DataCache.java | 7 +-
.../hadoop/hive/common/io/DiskRangeList.java | 19 +-
.../hive/common/io/encoded/MemoryBuffer.java | 6 +-
.../hive/common/type/FastHiveDecimal.java | 2 +-
.../hive/common/type/FastHiveDecimalImpl.java | 325 ++++++++--
.../hadoop/hive/common/type/HiveDecimal.java | 30 +-
.../hive/ql/exec/vector/BytesColumnVector.java | 2 +-
.../hive/ql/exec/vector/ColumnVector.java | 6 +-
.../ql/exec/vector/TimestampColumnVector.java | 2 +-
.../hadoop/hive/ql/util/TimestampUtils.java | 2 +-
.../ql/exec/vector/TestStructColumnVector.java | 36 ++
.../hive/common/util/TestBloomFilter.java | 589 +++++++++++++++++++
20 files changed, 1064 insertions(+), 675 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/common/pom.xml
----------------------------------------------------------------------
diff --git a/common/pom.xml b/common/pom.xml
index 150c492..b30ec7b 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -42,7 +42,6 @@
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-storage-api</artifactId>
- <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/common/src/test/org/apache/hive/common/util/TestBloomFilter.java
----------------------------------------------------------------------
diff --git a/common/src/test/org/apache/hive/common/util/TestBloomFilter.java b/common/src/test/org/apache/hive/common/util/TestBloomFilter.java
deleted file mode 100644
index e4ee93a..0000000
--- a/common/src/test/org/apache/hive/common/util/TestBloomFilter.java
+++ /dev/null
@@ -1,589 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.common.util;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.util.ArrayList;
-import java.util.Random;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- *
- */
-public class TestBloomFilter {
- private static final int COUNT = 100;
- Random rand = new Random(123);
-
- @Test(expected = IllegalArgumentException.class)
- public void testBloomIllegalArg1() {
- BloomFilter bf = new BloomFilter(0, 0);
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testBloomIllegalArg2() {
- BloomFilter bf = new BloomFilter(0, 0.1);
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testBloomIllegalArg3() {
- BloomFilter bf = new BloomFilter(1, 0.0);
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testBloomIllegalArg4() {
- BloomFilter bf = new BloomFilter(1, 1.0);
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testBloomIllegalArg5() {
- BloomFilter bf = new BloomFilter(-1, -1);
- }
-
-
- @Test
- public void testBloomNumBits() {
- assertEquals(0, BloomFilter.optimalNumOfBits(0, 0));
- assertEquals(0, BloomFilter.optimalNumOfBits(0, 1));
- assertEquals(0, BloomFilter.optimalNumOfBits(1, 1));
- assertEquals(7, BloomFilter.optimalNumOfBits(1, 0.03));
- assertEquals(72, BloomFilter.optimalNumOfBits(10, 0.03));
- assertEquals(729, BloomFilter.optimalNumOfBits(100, 0.03));
- assertEquals(7298, BloomFilter.optimalNumOfBits(1000, 0.03));
- assertEquals(72984, BloomFilter.optimalNumOfBits(10000, 0.03));
- assertEquals(729844, BloomFilter.optimalNumOfBits(100000, 0.03));
- assertEquals(7298440, BloomFilter.optimalNumOfBits(1000000, 0.03));
- assertEquals(6235224, BloomFilter.optimalNumOfBits(1000000, 0.05));
- assertEquals(1870567268, BloomFilter.optimalNumOfBits(300000000, 0.05));
- assertEquals(1437758756, BloomFilter.optimalNumOfBits(300000000, 0.1));
- assertEquals(432808512, BloomFilter.optimalNumOfBits(300000000, 0.5));
- assertEquals(1393332198, BloomFilter.optimalNumOfBits(3000000000L, 0.8));
- assertEquals(657882327, BloomFilter.optimalNumOfBits(3000000000L, 0.9));
- assertEquals(0, BloomFilter.optimalNumOfBits(3000000000L, 1));
- }
-
- @Test
- public void testBloomNumHashFunctions() {
- assertEquals(1, BloomFilter.optimalNumOfHashFunctions(-1, -1));
- assertEquals(1, BloomFilter.optimalNumOfHashFunctions(0, 0));
- assertEquals(1, BloomFilter.optimalNumOfHashFunctions(10, 0));
- assertEquals(1, BloomFilter.optimalNumOfHashFunctions(10, 10));
- assertEquals(7, BloomFilter.optimalNumOfHashFunctions(10, 100));
- assertEquals(1, BloomFilter.optimalNumOfHashFunctions(100, 100));
- assertEquals(1, BloomFilter.optimalNumOfHashFunctions(1000, 100));
- assertEquals(1, BloomFilter.optimalNumOfHashFunctions(10000, 100));
- assertEquals(1, BloomFilter.optimalNumOfHashFunctions(100000, 100));
- assertEquals(1, BloomFilter.optimalNumOfHashFunctions(1000000, 100));
- }
-
- @Test
- public void testBloomFilterBytes() {
- BloomFilter bf = new BloomFilter(10000);
- byte[] val = new byte[]{1, 2, 3};
- byte[] val1 = new byte[]{1, 2, 3, 4};
- byte[] val2 = new byte[]{1, 2, 3, 4, 5};
- byte[] val3 = new byte[]{1, 2, 3, 4, 5, 6};
-
- assertEquals(false, bf.test(val));
- assertEquals(false, bf.test(val1));
- assertEquals(false, bf.test(val2));
- assertEquals(false, bf.test(val3));
- bf.add(val);
- assertEquals(true, bf.test(val));
- assertEquals(false, bf.test(val1));
- assertEquals(false, bf.test(val2));
- assertEquals(false, bf.test(val3));
- bf.add(val1);
- assertEquals(true, bf.test(val));
- assertEquals(true, bf.test(val1));
- assertEquals(false, bf.test(val2));
- assertEquals(false, bf.test(val3));
- bf.add(val2);
- assertEquals(true, bf.test(val));
- assertEquals(true, bf.test(val1));
- assertEquals(true, bf.test(val2));
- assertEquals(false, bf.test(val3));
- bf.add(val3);
- assertEquals(true, bf.test(val));
- assertEquals(true, bf.test(val1));
- assertEquals(true, bf.test(val2));
- assertEquals(true, bf.test(val3));
-
- byte[] randVal = new byte[COUNT];
- for (int i = 0; i < COUNT; i++) {
- rand.nextBytes(randVal);
- bf.add(randVal);
- }
- // last value should be present
- assertEquals(true, bf.test(randVal));
- // most likely this value should not exist
- randVal[0] = 0;
- randVal[1] = 0;
- randVal[2] = 0;
- randVal[3] = 0;
- randVal[4] = 0;
- assertEquals(false, bf.test(randVal));
-
- assertEquals(7800, bf.sizeInBytes());
- }
-
- @Test
- public void testBloomFilterByte() {
- BloomFilter bf = new BloomFilter(10000);
- byte val = Byte.MIN_VALUE;
- byte val1 = 1;
- byte val2 = 2;
- byte val3 = Byte.MAX_VALUE;
-
- assertEquals(false, bf.testLong(val));
- assertEquals(false, bf.testLong(val1));
- assertEquals(false, bf.testLong(val2));
- assertEquals(false, bf.testLong(val3));
- bf.addLong(val);
- assertEquals(true, bf.testLong(val));
- assertEquals(false, bf.testLong(val1));
- assertEquals(false, bf.testLong(val2));
- assertEquals(false, bf.testLong(val3));
- bf.addLong(val1);
- assertEquals(true, bf.testLong(val));
- assertEquals(true, bf.testLong(val1));
- assertEquals(false, bf.testLong(val2));
- assertEquals(false, bf.testLong(val3));
- bf.addLong(val2);
- assertEquals(true, bf.testLong(val));
- assertEquals(true, bf.testLong(val1));
- assertEquals(true, bf.testLong(val2));
- assertEquals(false, bf.testLong(val3));
- bf.addLong(val3);
- assertEquals(true, bf.testLong(val));
- assertEquals(true, bf.testLong(val1));
- assertEquals(true, bf.testLong(val2));
- assertEquals(true, bf.testLong(val3));
-
- byte randVal = 0;
- for (int i = 0; i < COUNT; i++) {
- randVal = (byte) rand.nextInt(Byte.MAX_VALUE);
- bf.addLong(randVal);
- }
- // last value should be present
- assertEquals(true, bf.testLong(randVal));
- // most likely this value should not exist
- assertEquals(false, bf.testLong((byte) -120));
-
- assertEquals(7800, bf.sizeInBytes());
- }
-
- @Test
- public void testBloomFilterInt() {
- BloomFilter bf = new BloomFilter(10000);
- int val = Integer.MIN_VALUE;
- int val1 = 1;
- int val2 = 2;
- int val3 = Integer.MAX_VALUE;
-
- assertEquals(false, bf.testLong(val));
- assertEquals(false, bf.testLong(val1));
- assertEquals(false, bf.testLong(val2));
- assertEquals(false, bf.testLong(val3));
- bf.addLong(val);
- assertEquals(true, bf.testLong(val));
- assertEquals(false, bf.testLong(val1));
- assertEquals(false, bf.testLong(val2));
- assertEquals(false, bf.testLong(val3));
- bf.addLong(val1);
- assertEquals(true, bf.testLong(val));
- assertEquals(true, bf.testLong(val1));
- assertEquals(false, bf.testLong(val2));
- assertEquals(false, bf.testLong(val3));
- bf.addLong(val2);
- assertEquals(true, bf.testLong(val));
- assertEquals(true, bf.testLong(val1));
- assertEquals(true, bf.testLong(val2));
- assertEquals(false, bf.testLong(val3));
- bf.addLong(val3);
- assertEquals(true, bf.testLong(val));
- assertEquals(true, bf.testLong(val1));
- assertEquals(true, bf.testLong(val2));
- assertEquals(true, bf.testLong(val3));
-
- int randVal = 0;
- for (int i = 0; i < COUNT; i++) {
- randVal = rand.nextInt();
- bf.addLong(randVal);
- }
- // last value should be present
- assertEquals(true, bf.testLong(randVal));
- // most likely this value should not exist
- assertEquals(false, bf.testLong(-120));
-
- assertEquals(7800, bf.sizeInBytes());
- }
-
- @Test
- public void testBloomFilterLong() {
- BloomFilter bf = new BloomFilter(10000);
- long val = Long.MIN_VALUE;
- long val1 = 1;
- long val2 = 2;
- long val3 = Long.MAX_VALUE;
-
- assertEquals(false, bf.testLong(val));
- assertEquals(false, bf.testLong(val1));
- assertEquals(false, bf.testLong(val2));
- assertEquals(false, bf.testLong(val3));
- bf.addLong(val);
- assertEquals(true, bf.testLong(val));
- assertEquals(false, bf.testLong(val1));
- assertEquals(false, bf.testLong(val2));
- assertEquals(false, bf.testLong(val3));
- bf.addLong(val1);
- assertEquals(true, bf.testLong(val));
- assertEquals(true, bf.testLong(val1));
- assertEquals(false, bf.testLong(val2));
- assertEquals(false, bf.testLong(val3));
- bf.addLong(val2);
- assertEquals(true, bf.testLong(val));
- assertEquals(true, bf.testLong(val1));
- assertEquals(true, bf.testLong(val2));
- assertEquals(false, bf.testLong(val3));
- bf.addLong(val3);
- assertEquals(true, bf.testLong(val));
- assertEquals(true, bf.testLong(val1));
- assertEquals(true, bf.testLong(val2));
- assertEquals(true, bf.testLong(val3));
-
- long randVal = 0;
- for (int i = 0; i < COUNT; i++) {
- randVal = rand.nextLong();
- bf.addLong(randVal);
- }
- // last value should be present
- assertEquals(true, bf.testLong(randVal));
- // most likely this value should not exist
- assertEquals(false, bf.testLong(-120));
-
- assertEquals(7800, bf.sizeInBytes());
- }
-
- @Test
- public void testBloomFilterFloat() {
- BloomFilter bf = new BloomFilter(10000);
- float val = Float.MIN_VALUE;
- float val1 = 1.1f;
- float val2 = 2.2f;
- float val3 = Float.MAX_VALUE;
-
- assertEquals(false, bf.testDouble(val));
- assertEquals(false, bf.testDouble(val1));
- assertEquals(false, bf.testDouble(val2));
- assertEquals(false, bf.testDouble(val3));
- bf.addDouble(val);
- assertEquals(true, bf.testDouble(val));
- assertEquals(false, bf.testDouble(val1));
- assertEquals(false, bf.testDouble(val2));
- assertEquals(false, bf.testDouble(val3));
- bf.addDouble(val1);
- assertEquals(true, bf.testDouble(val));
- assertEquals(true, bf.testDouble(val1));
- assertEquals(false, bf.testDouble(val2));
- assertEquals(false, bf.testDouble(val3));
- bf.addDouble(val2);
- assertEquals(true, bf.testDouble(val));
- assertEquals(true, bf.testDouble(val1));
- assertEquals(true, bf.testDouble(val2));
- assertEquals(false, bf.testDouble(val3));
- bf.addDouble(val3);
- assertEquals(true, bf.testDouble(val));
- assertEquals(true, bf.testDouble(val1));
- assertEquals(true, bf.testDouble(val2));
- assertEquals(true, bf.testDouble(val3));
-
- float randVal = 0;
- for (int i = 0; i < COUNT; i++) {
- randVal = rand.nextFloat();
- bf.addDouble(randVal);
- }
- // last value should be present
- assertEquals(true, bf.testDouble(randVal));
- // most likely this value should not exist
- assertEquals(false, bf.testDouble(-120.2f));
-
- assertEquals(7800, bf.sizeInBytes());
- }
-
- @Test
- public void testBloomFilterDouble() {
- BloomFilter bf = new BloomFilter(10000);
- double val = Double.MIN_VALUE;
- double val1 = 1.1d;
- double val2 = 2.2d;
- double val3 = Double.MAX_VALUE;
-
- assertEquals(false, bf.testDouble(val));
- assertEquals(false, bf.testDouble(val1));
- assertEquals(false, bf.testDouble(val2));
- assertEquals(false, bf.testDouble(val3));
- bf.addDouble(val);
- assertEquals(true, bf.testDouble(val));
- assertEquals(false, bf.testDouble(val1));
- assertEquals(false, bf.testDouble(val2));
- assertEquals(false, bf.testDouble(val3));
- bf.addDouble(val1);
- assertEquals(true, bf.testDouble(val));
- assertEquals(true, bf.testDouble(val1));
- assertEquals(false, bf.testDouble(val2));
- assertEquals(false, bf.testDouble(val3));
- bf.addDouble(val2);
- assertEquals(true, bf.testDouble(val));
- assertEquals(true, bf.testDouble(val1));
- assertEquals(true, bf.testDouble(val2));
- assertEquals(false, bf.testDouble(val3));
- bf.addDouble(val3);
- assertEquals(true, bf.testDouble(val));
- assertEquals(true, bf.testDouble(val1));
- assertEquals(true, bf.testDouble(val2));
- assertEquals(true, bf.testDouble(val3));
-
- double randVal = 0;
- for (int i = 0; i < COUNT; i++) {
- randVal = rand.nextDouble();
- bf.addDouble(randVal);
- }
- // last value should be present
- assertEquals(true, bf.testDouble(randVal));
- // most likely this value should not exist
- assertEquals(false, bf.testDouble(-120.2d));
-
- assertEquals(7800, bf.sizeInBytes());
- }
-
- @Test
- public void testBloomFilterString() {
- BloomFilter bf = new BloomFilter(100000);
- String val = "bloo";
- String val1 = "bloom fil";
- String val2 = "bloom filter";
- String val3 = "cuckoo filter";
-
- assertEquals(false, bf.testString(val));
- assertEquals(false, bf.testString(val1));
- assertEquals(false, bf.testString(val2));
- assertEquals(false, bf.testString(val3));
- bf.addString(val);
- assertEquals(true, bf.testString(val));
- assertEquals(false, bf.testString(val1));
- assertEquals(false, bf.testString(val2));
- assertEquals(false, bf.testString(val3));
- bf.addString(val1);
- assertEquals(true, bf.testString(val));
- assertEquals(true, bf.testString(val1));
- assertEquals(false, bf.testString(val2));
- assertEquals(false, bf.testString(val3));
- bf.addString(val2);
- assertEquals(true, bf.testString(val));
- assertEquals(true, bf.testString(val1));
- assertEquals(true, bf.testString(val2));
- assertEquals(false, bf.testString(val3));
- bf.addString(val3);
- assertEquals(true, bf.testString(val));
- assertEquals(true, bf.testString(val1));
- assertEquals(true, bf.testString(val2));
- assertEquals(true, bf.testString(val3));
-
- long randVal = 0;
- for (int i = 0; i < COUNT; i++) {
- randVal = rand.nextLong();
- bf.addString(Long.toString(randVal));
- }
- // last value should be present
- assertEquals(true, bf.testString(Long.toString(randVal)));
- // most likely this value should not exist
- assertEquals(false, bf.testString(Long.toString(-120)));
-
- assertEquals(77944, bf.sizeInBytes());
- }
-
- @Test
- public void testMerge() {
- BloomFilter bf = new BloomFilter(10000);
- String val = "bloo";
- String val1 = "bloom fil";
- String val2 = "bloom filter";
- String val3 = "cuckoo filter";
- bf.addString(val);
- bf.addString(val1);
- bf.addString(val2);
- bf.addString(val3);
-
- BloomFilter bf2 = new BloomFilter(10000);
- String v = "2_bloo";
- String v1 = "2_bloom fil";
- String v2 = "2_bloom filter";
- String v3 = "2_cuckoo filter";
- bf2.addString(v);
- bf2.addString(v1);
- bf2.addString(v2);
- bf2.addString(v3);
-
- assertEquals(true, bf.testString(val));
- assertEquals(true, bf.testString(val1));
- assertEquals(true, bf.testString(val2));
- assertEquals(true, bf.testString(val3));
- assertEquals(false, bf.testString(v));
- assertEquals(false, bf.testString(v1));
- assertEquals(false, bf.testString(v2));
- assertEquals(false, bf.testString(v3));
-
- bf.merge(bf2);
-
- assertEquals(true, bf.testString(val));
- assertEquals(true, bf.testString(val1));
- assertEquals(true, bf.testString(val2));
- assertEquals(true, bf.testString(val3));
- assertEquals(true, bf.testString(v));
- assertEquals(true, bf.testString(v1));
- assertEquals(true, bf.testString(v2));
- assertEquals(true, bf.testString(v3));
- }
-
- @Test
- public void testSerialize() throws Exception {
- BloomFilter bf1 = new BloomFilter(10000);
- String[] inputs = {
- "bloo",
- "bloom fil",
- "bloom filter",
- "cuckoo filter",
- };
-
- for (String val : inputs) {
- bf1.addString(val);
- }
-
- // Serialize/deserialize
- ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
- BloomFilter.serialize(bytesOut, bf1);
- ByteArrayInputStream bytesIn = new ByteArrayInputStream(bytesOut.toByteArray());
- BloomFilter bf2 = BloomFilter.deserialize(bytesIn);
-
- for (String val : inputs) {
- assertEquals("Testing bf1 with " + val, true, bf1.testString(val));
- assertEquals("Testing bf2 with " + val, true, bf2.testString(val));
- }
- }
-
- @Test
- public void testMergeBloomFilterBytes() throws Exception {
- BloomFilter bf1 = new BloomFilter(10000);
- BloomFilter bf2 = new BloomFilter(10000);
-
- String[] inputs1 = {
- "bloo",
- "bloom fil",
- "bloom filter",
- "cuckoo filter",
- };
-
- String[] inputs2 = {
- "2_bloo",
- "2_bloom fil",
- "2_bloom filter",
- "2_cuckoo filter",
- };
-
- for (String val : inputs1) {
- bf1.addString(val);
- }
- for (String val : inputs2) {
- bf2.addString(val);
- }
-
- ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
- BloomFilter.serialize(bytesOut, bf1);
- byte[] bf1Bytes = bytesOut.toByteArray();
- bytesOut.reset();
- BloomFilter.serialize(bytesOut, bf1);
- byte[] bf2Bytes = bytesOut.toByteArray();
-
- // Merge bytes
- BloomFilter.mergeBloomFilterBytes(
- bf1Bytes, 0, bf1Bytes.length,
- bf2Bytes, 0, bf2Bytes.length);
-
- // Deserialize and test
- ByteArrayInputStream bytesIn = new ByteArrayInputStream(bf1Bytes, 0, bf1Bytes.length);
- BloomFilter bfMerged = BloomFilter.deserialize(bytesIn);
- // All values should pass test
- for (String val : inputs1) {
- bfMerged.addString(val);
- }
- for (String val : inputs2) {
- bfMerged.addString(val);
- }
- }
-
- @Test
- public void testMergeBloomFilterBytesFailureCases() throws Exception {
- BloomFilter bf1 = new BloomFilter(1000);
- BloomFilter bf2 = new BloomFilter(200);
- // Create bloom filter with same number of bits, but different # hash functions
- ArrayList<Long> bits = new ArrayList<Long>();
- for (int idx = 0; idx < bf1.getBitSet().length; ++idx) {
- bits.add(0L);
- }
- BloomFilter bf3 = new BloomFilter(bits, bf1.getBitSize(), bf1.getNumHashFunctions() + 1);
-
- // Serialize to bytes
- ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
- BloomFilter.serialize(bytesOut, bf1);
- byte[] bf1Bytes = bytesOut.toByteArray();
-
- bytesOut.reset();
- BloomFilter.serialize(bytesOut, bf2);
- byte[] bf2Bytes = bytesOut.toByteArray();
-
- bytesOut.reset();
- BloomFilter.serialize(bytesOut, bf3);
- byte[] bf3Bytes = bytesOut.toByteArray();
-
- try {
- // this should fail
- BloomFilter.mergeBloomFilterBytes(
- bf1Bytes, 0, bf1Bytes.length,
- bf2Bytes, 0, bf2Bytes.length);
- Assert.fail("Expected exception not encountered");
- } catch (IllegalArgumentException err) {
- // expected
- }
-
- try {
- // this should fail
- BloomFilter.mergeBloomFilterBytes(
- bf1Bytes, 0, bf1Bytes.length,
- bf3Bytes, 0, bf3Bytes.length);
- Assert.fail("Expected exception not encountered");
- } catch (IllegalArgumentException err) {
- // expected
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/orc/pom.xml
----------------------------------------------------------------------
diff --git a/orc/pom.xml b/orc/pom.xml
index e6ec0f4..f75b91c 100644
--- a/orc/pom.xml
+++ b/orc/pom.xml
@@ -35,7 +35,6 @@
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-storage-api</artifactId>
- <version>${project.version}</version>
</dependency>
<!-- inter-project -->
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/packaging/pom.xml
----------------------------------------------------------------------
diff --git a/packaging/pom.xml b/packaging/pom.xml
index 76e0cff..7a53856 100644
--- a/packaging/pom.xml
+++ b/packaging/pom.xml
@@ -246,7 +246,6 @@
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-storage-api</artifactId>
- <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 4158083..505e4c2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -180,6 +180,7 @@
<stax.version>1.0.1</stax.version>
<slf4j.version>1.7.10</slf4j.version>
<ST4.version>4.0.4</ST4.version>
+ <storage-api.version>2.3.0-SNAPSHOT</storage-api.version>
<tez.version>0.8.4</tez.version>
<slider.version>0.90.2-incubating</slider.version>
<super-csv.version>2.2.0</super-csv.version>
@@ -505,6 +506,11 @@
<version>${stax.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-storage-api</artifactId>
+ <version>${storage-api.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.pig</groupId>
<artifactId>pig</artifactId>
<version>${pig.version}</version>
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/storage-api/pom.xml
----------------------------------------------------------------------
diff --git a/storage-api/pom.xml b/storage-api/pom.xml
index 4095ed2..036a994 100644
--- a/storage-api/pom.xml
+++ b/storage-api/pom.xml
@@ -17,22 +17,33 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive</artifactId>
- <version>2.2.0-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
+ <groupId>org.apache</groupId>
+ <artifactId>apache</artifactId>
+ <version>18</version>
+ <relativePath></relativePath>
</parent>
+ <groupId>org.apache.hive</groupId>
<artifactId>hive-storage-api</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Hive Storage API</name>
<properties>
- <hive.path.to.root>..</hive.path.to.root>
+ <commons-lang.version>2.6</commons-lang.version>
+ <guava.version>14.0.1</guava.version>
+ <hadoop.version>2.7.2</hadoop.version>
+ <junit.version>4.11</junit.version>
+ <slf4j.version>1.7.10</slf4j.version>
</properties>
<dependencies>
- <!-- test inter-project -->
+ <!-- compile inter-project -->
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <version>${commons-lang.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
@@ -40,7 +51,15 @@
<optional>true</optional>
<exclusions>
<exclusion>
- <groupId>commmons-logging</groupId>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
<exclusion>
@@ -48,9 +67,33 @@
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
+ <groupId>javax.servlet.jsp</groupId>
+ <artifactId>jsp-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -69,5 +112,39 @@
<directory>${basedir}/src/test/resources</directory>
</testResource>
</testResources>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.1</version>
+ <configuration>
+ <source>1.7</source>
+ <target>1.7</target>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <version>2.10.4</version>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-project-info-reports-plugin</artifactId>
+ <version>2.9</version>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.19.1</version>
+ <configuration>
+ <reuseForks>false</reuseForks>
+ <argLine>-Xmx2048m</argLine>
+ <failIfNoTests>false</failIfNoTests>
+ <systemPropertyVariables>
+ <test.tmp.dir>${project.build.directory}/tmp</test.tmp.dir>
+ </systemPropertyVariables>
+ </configuration>
+ </plugin>
+ </plugins>
</build>
</project>
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/storage-api/src/java/org/apache/hadoop/hive/common/Pool.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/Pool.java b/storage-api/src/java/org/apache/hadoop/hive/common/Pool.java
index 272bbdd..5ab4ee7 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/common/Pool.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/Pool.java
@@ -21,9 +21,13 @@ package org.apache.hadoop.hive.common;
public interface Pool<T> {
/** Object helper for objects stored in the pool. */
public interface PoolObjectHelper<T> {
- /** Called to create an object when one cannot be provided. */
+ /** Called to create an object when one cannot be provided.
+ * @return a newly allocated object
+ */
T create();
- /** Called before the object is put in the pool (regardless of whether put succeeds). */
+ /** Called before the object is put in the pool (regardless of whether put succeeds).
+ * @param t the object to reset
+ */
void resetBeforeOffer(T t);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/storage-api/src/java/org/apache/hadoop/hive/common/io/Allocator.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/io/Allocator.java b/storage-api/src/java/org/apache/hadoop/hive/common/io/Allocator.java
index fd9d9c9..16b9713 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/common/io/Allocator.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/io/Allocator.java
@@ -42,12 +42,22 @@ public interface Allocator {
* Creates an unallocated memory buffer object. This object can be passed to allocateMultiple
* to allocate; this is useful if data structures are created for separate buffers that can
* later be allocated together.
+ * @return a new unallocated memory buffer
*/
MemoryBuffer createUnallocated();
- /** Deallocates a memory buffer. */
+
+ /** Deallocates a memory buffer.
+ * @param buffer the buffer to deallocate
+ */
void deallocate(MemoryBuffer buffer);
- /** Whether the allocator uses direct buffers. */
+
+ /** Whether the allocator uses direct buffers.
+ * @return are they direct buffers?
+ */
boolean isDirectAlloc();
- /** Maximum allocation size supported by this allocator. */
+
+ /** Maximum allocation size supported by this allocator.
+ * @return a number of bytes
+ */
int getMaxAllocation();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/storage-api/src/java/org/apache/hadoop/hive/common/io/DataCache.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/io/DataCache.java b/storage-api/src/java/org/apache/hadoop/hive/common/io/DataCache.java
index 1273588..e53b737 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/common/io/DataCache.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/io/DataCache.java
@@ -50,7 +50,7 @@ public interface DataCache {
* return both ranges). It should really be same as #1, however currently ORC uses estimated
* end offsets; if we don't return the end block, the caller may read it from disk needlessly.
*
- * @param fileId Unique ID of the target file on the file system.
+ * @param fileKey Unique ID of the target file on the file system.
* @param range A set of DiskRange-s (linked list) that is to be retrieved. May be modified.
* @param baseOffset base offset for the ranges (stripe/stream offset in case of ORC).
* @param factory A factory to produce DiskRangeList-s out of cached MemoryBuffer-s.
@@ -72,7 +72,7 @@ public interface DataCache {
* caller is done with it. Buffers rejected due to conflict will neither be locked, nor
* automatically deallocated. The caller must take care to discard these buffers.
*
- * @param fileId Unique ID of the target file on the file system.
+ * @param fileKey Unique ID of the target file on the file system.
* @param ranges The ranges for which the data is being cached. These objects will not be stored.
* @param data The data for the corresponding ranges.
* @param baseOffset base offset for the ranges (stripe/stream offset in case of ORC).
@@ -84,17 +84,20 @@ public interface DataCache {
/**
* Releases the buffer returned by getFileData/provided to putFileData back to cache.
* See respective javadocs for details.
+ * @param buffer the buffer to release
*/
void releaseBuffer(MemoryBuffer buffer);
/**
* Notifies the cache that the buffer returned from getFileData/provided to putFileData will
* be used by another consumer and therefore released multiple times (one more time per call).
+ * @param buffer the buffer to reuse
*/
void reuseBuffer(MemoryBuffer buffer);
/**
* Gets the allocator associated with this DataCache.
+ * @return the allocator
*/
Allocator getAllocator();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRangeList.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRangeList.java b/storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRangeList.java
index 6a89d0a..0aba08d 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRangeList.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRangeList.java
@@ -31,7 +31,10 @@ public class DiskRangeList extends DiskRange {
super(offset, end);
}
- /** Replaces this element with another in the list; returns the new element. */
+ /** Replaces this element with another in the list; returns the new element.
+ * @param other the disk range to swap into this list
+ * @return the new element
+ */
public DiskRangeList replaceSelfWith(DiskRangeList other) {
other.prev = this.prev;
other.next = this.next;
@@ -47,7 +50,8 @@ public class DiskRangeList extends DiskRange {
/**
* Inserts an intersecting range before current in the list and adjusts offset accordingly.
- * @returns the new element.
+ * @param other the element to insert
+ * @return the new element.
*/
public DiskRangeList insertPartBefore(DiskRangeList other) {
assert other.end >= this.offset;
@@ -63,7 +67,8 @@ public class DiskRangeList extends DiskRange {
/**
* Inserts an element after current in the list.
- * @returns the new element.
+ * @param other the new element to insert
+ * @return the new element.
* */
public DiskRangeList insertAfter(DiskRangeList other) {
other.next = this.next;
@@ -77,7 +82,8 @@ public class DiskRangeList extends DiskRange {
/**
* Inserts an intersecting range after current in the list and adjusts offset accordingly.
- * @returns the new element.
+ * @param other the new element to insert
+ * @return the new element.
*/
public DiskRangeList insertPartAfter(DiskRangeList other) {
assert other.offset <= this.end;
@@ -106,7 +112,10 @@ public class DiskRangeList extends DiskRange {
this.next = this.prev = null;
}
- /** Splits current element in the list, using DiskRange::slice */
+ /** Splits current element in the list, using DiskRange::slice.
+ * @param cOffset the position to split the list
+ * @return the split list
+ */
public final DiskRangeList split(long cOffset) {
insertAfter((DiskRangeList)this.sliceAndShift(cOffset, end, 0));
return replaceSelfWith((DiskRangeList)this.sliceAndShift(offset, cOffset, 0));
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/MemoryBuffer.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/MemoryBuffer.java b/storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/MemoryBuffer.java
index 4475009..201c5e2 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/MemoryBuffer.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/MemoryBuffer.java
@@ -22,7 +22,11 @@ import java.nio.ByteBuffer;
/** Abstract interface for any class wrapping a ByteBuffer. */
public interface MemoryBuffer {
- /** Note - raw buffer should not be modified. */
+ /**
+ * Get the raw byte buffer that backs this buffer.
+ * Note - raw buffer should not be modified.
+ * @return the shared byte buffer
+ */
public ByteBuffer getByteBufferRaw();
public ByteBuffer getByteBufferDup();
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimal.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimal.java b/storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimal.java
index a8c183a..7fa9fdf 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimal.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimal.java
@@ -235,7 +235,7 @@ public class FastHiveDecimal {
protected void fastFractionPortion() {
FastHiveDecimalImpl.fastFractionPortion(
- fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale,
+ fastSignum, fast0, fast1, fast2, fastScale,
this);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimalImpl.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimalImpl.java b/storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimalImpl.java
index a4fed5d..7a565dd 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimalImpl.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimalImpl.java
@@ -26,8 +26,6 @@ import java.math.BigDecimal;
import java.math.BigInteger;
import java.math.RoundingMode;
-import org.apache.commons.lang.StringUtils;
-
/**
* This class is a companion to the FastHiveDecimal class that separates the essential of code
* out of FastHiveDecimal into static methods in this class so that they can be used directly
@@ -254,8 +252,12 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* NOTE: The fastSetFromBytes method requires the caller to pass a fastResult parameter has been
* reset for better performance.
*
+ * @param bytes the bytes to copy from
+ * @param offset the starting location in bytes
+ * @param length the number of bytes to use from bytes
+ * @param trimBlanks should spaces be trimmed?
* @param fastResult True if the byte array slice was successfully converted to a decimal.
- * @return
+ * @return Was a valid number found?
*/
public static boolean fastSetFromBytes(byte[] bytes, int offset, int length, boolean trimBlanks,
FastHiveDecimal fastResult) {
@@ -681,6 +683,12 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* NOTE: The fastSetFromDigitsOnlyBytesAndScale method requires the caller to pass a fastResult
* parameter has been reset for better performance.
*
+ * @param isNegative is the number negative
+ * @param bytes the bytes to read from
+ * @param offset the position to start at
+ * @param length the number of bytes to read
+ * @param scale the scale of the number
+ * @param fastResult an object it into
* @return True if the sign, digits, and scale were successfully converted to a decimal.
*/
public static boolean fastSetFromDigitsOnlyBytesAndScale(
@@ -826,6 +834,9 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* NOTE: The fastSetFromBigDecimal method requires the caller to pass a fastResult
* parameter has been reset for better performance.
*
+ * @param bigDecimal the big decimal to copy
+ * @param allowRounding is rounding allowed?
+ * @param fastResult an object to reuse
* @return True if the BigDecimal could be converted to our decimal representation.
*/
public static boolean fastSetFromBigDecimal(
@@ -891,6 +902,9 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* NOTE: The fastSetFromString method requires the caller to pass a fastResult
* parameter has been reset for better performance.
*
+ * @param string the string to parse
+ * @param trimBlanks should the blanks be trimmed
+ * @param result an object to reuse
* @return True if the String was successfully converted to a decimal.
*/
public static boolean fastSetFromString(
@@ -904,7 +918,8 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
*
* NOTE: The fastSetFromString method requires the caller to pass a fastResult
* parameter has been reset for better performance.
- *
+ * @param intValue the value to set
+ * @param fastResult an object to reuse
*/
public static void fastSetFromInt(int intValue, FastHiveDecimal fastResult) {
if (intValue == 0) {
@@ -930,6 +945,8 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* NOTE: The fastSetFromLong method requires the caller to pass a fastResult
* parameter has been reset for better performance.
*
+ * @param longValue the value to set
+ * @param fastResult an object to reuse
*/
public static void fastSetFromLong(
long longValue, FastHiveDecimal fastResult) {
@@ -971,6 +988,10 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* NOTE: The fastSetFromLongAndScale method requires the caller to pass a fastResult
* parameter has been reset for better performance.
*
+ * @param longValue the value to set as a long
+ * @param scale the scale to use
+ * @param fastResult an object to reuse
+ * @return was the conversion successful?
*/
public static boolean fastSetFromLongAndScale(
long longValue, int scale, FastHiveDecimal fastResult) {
@@ -999,6 +1020,9 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* NOTE: The fastSetFromFloat method requires the caller to pass a fastResult
* parameter has been reset for better performance.
*
+ * @param floatValue the value to set
+ * @param fastResult an object to reuse
+ * @return was the conversion successful?
*/
public static boolean fastSetFromFloat(
float floatValue, FastHiveDecimal fastResult) {
@@ -1014,6 +1038,9 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* NOTE: The fastSetFromDouble method requires the caller to pass a fastResult
* parameter has been reset for better performance.
*
+ * @param doubleValue the value to set
+ * @param fastResult an object to reuse
+ * @return was the conversion successful?
*/
public static boolean fastSetFromDouble(
double doubleValue, FastHiveDecimal fastResult) {
@@ -1032,6 +1059,8 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* NOTE: The fastSetFromBigInteger method requires the caller to pass a fastResult
* parameter has been reset for better performance.
*
+ * @param bigInteger the value to set
+ * @param fastResult an object to reuse
* @return Return true if the BigInteger value fit within HiveDecimal.MAX_PRECISION. Otherwise,
* false for overflow.
*/
@@ -1101,6 +1130,9 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* NOTE: The fastSetFromBigInteger method requires the caller to pass a fastResult
* parameter has been reset for better performance.
*
+ * @param bigInteger the value to set as an integer
+ * @param scale the scale to use
+ * @param fastResult an object to reuse
* @return True if the BigInteger and scale were successfully converted to a decimal.
*/
public static boolean fastSetFromBigInteger(
@@ -1224,10 +1256,17 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
*
* NOTE: The fastFractionPortion method requires the caller to pass a fastResult
* parameter has been reset for better performance.
+ *
+ * @param fastSignum the sign of the number (1, 0, or -1)
+ * @param fast0 high bits
+ * @param fast1 second word bits
+ * @param fast2 third word bits
+ * @param fastScale the scale
+ * @param fastResult an object to reuse
*/
public static void fastFractionPortion(
int fastSignum, long fast0, long fast1, long fast2,
- int fastIntegerDigitCount, int fastScale,
+ int fastScale,
FastHiveDecimal fastResult) {
if (fastSignum == 0 || fastScale == 0) {
@@ -1288,6 +1327,14 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
*
* NOTE: The fastFractionPortion method requires the caller to pass a fastResult
* parameter has been reset for better performance.
+ *
+ * @param fastSignum the sign of the number (1, 0, or -1)
+ * @param fast0 high bits
+ * @param fast1 second word bits
+ * @param fast2 third word bits
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale
+ * @param fastResult an object to reuse
*/
public static void fastIntegerPortion(
int fastSignum, long fast0, long fast1, long fast2,
@@ -1326,9 +1373,12 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* We let L and M be different to support the SerializationUtil serialization where the lower
* word is 62 bits and the remaining words are 63 bits...
*
- * The fast decimal middleWordMultiplier is 2^L.
- * The fast decimal highWordMultiplier is 2^(M+L).
- *
+ * @param lowerWord the lower internal representation
+ * @param middleWord the middle internal representation
+ * @param highWord the high internal representation
+ * @param middleWordMultiplier 2^L
+ * @param highWordMultiplier 2^(M+L)
+ * @param fastResult an object to reuse
* @return True if the conversion of the 3 binary words to decimal was successful.
*/
public static boolean doBinaryToDecimalConversion(
@@ -1505,22 +1555,21 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
*
* We produce 1 binary word (remainder) and a decimal quotient for the higher portion.
*
- * So, the parameters are:
- *
- * The input decimal (dividendFast0, dividendFast1, and dividendFast2) that will produce a
- * single binary word remainder and decimal quotient.
- *
- * The fast decimal inverse of 2^N = 2^-N (fastInverseConst).
- *
- * Where in the inverse multiplication result (quotientIntegerWordNum and
- * quotientIntegerDigitNum) to find the quotient integer decimal portion.
- *
- * The fast decimal multiplier for converting the quotient integer to the larger number to
- * subtract from the input decimal to get the remainder.
- *
- * And, the scratch longs where to store the result remainder word (index 3) and result quotient
- * decimal longwords (indices 0 .. 2).
- *
+ * @param dividendFast0 The input decimal that will produce a
+ * single binary word remainder and decimal quotient.
+ * @param dividendFast1 second word
+ * @param dividendFast2 third word
+ * @param fastInverseConst the fast decimal inverse of 2^N = 2^-N
+ * @param quotientIntegerWordNum the word in the inverse multiplication result
+ * to find the quotient integer decimal portion
+ * @param quotientIntegerDigitNum the digit in the result to find the quotient
+ * integer decimal portion
+ * @param fastMultiplierConst The fast decimal multiplier for converting the
+ * quotient integer to the larger number to
+ * subtract from the input decimal to get the
+ * remainder.
+ * @param scratchLongs where to store the result remainder word (index 3) and
+ * result quotient decimal longwords (indices 0 .. 2)
* @return True if the results were produced without overflow.
*/
public static boolean doDecimalToBinaryDivisionRemainder(
@@ -1769,11 +1818,18 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* performance. Pass a FAST_SCRATCH_BUFFER_LEN_SERIALIZATION_UTILS_READ byte array for
* scratchBytes.
*
+ * @param inputStream the stream to read from
+ * @param scale the scale of the number
+ * @param scratchBytes An array for the binary to decimal conversion for better
+ * performance. Must have length of
+ * FAST_SCRATCH_BUFFER_LEN_SERIALIZATION_UTILS_READ.
+ * @param fastResult an object to reuse
* @return The deserialized decimal or null if the conversion failed.
+ * @throws IOException failures in reading the stream
*/
public static boolean fastSerializationUtilsRead(InputStream inputStream, int scale,
byte[] scratchBytes,
- FastHiveDecimal fastResult) throws IOException, EOFException {
+ FastHiveDecimal fastResult) throws IOException {
// Following a suggestion from Gopal, quickly read in the bytes from the stream.
// CONSIDER: Have ORC read the whole input stream into a big byte array with one call to
@@ -1909,7 +1965,16 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* BigInteger.bitLength -- we do not emulate that. SerializationUtils.readBigInteger will
* produce the same result for both.
*
+ * @param outputStream the stream to write to
+ * @param fastSignum the sign digit (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1 of the internal representation
+ * @param fast2 word 2 of the internal representation
+ * @param fastIntegerDigitCount unused
+ * @param fastScale unused
+ * @param scratchLongs scratch space
* @return True if the decimal was successfully serialized into the output stream.
+ * @throws IOException for problems in writing
*/
public static boolean fastSerializationUtilsWrite(OutputStream outputStream,
int fastSignum, long fast0, long fast1, long fast2,
@@ -2130,7 +2195,12 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* This method is designed for high performance and does not create an actual BigInteger during
* binary to decimal conversion.
*
- * @return
+ * @param bytes the bytes to read from
+ * @param offset the starting position in the bytes array
+ * @param length the number of bytes to read from the bytes array
+ * @param scale the scale of the number
+ * @param fastResult an object to reused
+ * @return did the conversion succeed?
*/
public static boolean fastSetFromBigIntegerBytesAndScale(
byte[] bytes, int offset, int length, int scale,
@@ -2337,8 +2407,12 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* Scratch objects necessary to do the decimal to binary conversion without actually creating a
* BigInteger object are passed for better performance.
*
- * Allocate scratchLongs with SCRATCH_LONGS_LEN longs.
- * And, allocate buffer with SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES bytes.
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param scratchLongs scratch array of SCRATCH_LONGS_LEN longs
+ * @param buffer scratch array of SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES bytes
* @return The number of bytes used for the binary result in buffer. Otherwise, 0 if the
* conversion failed.
*/
@@ -2680,10 +2754,16 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
*
* This emulates the OldHiveDecimal setScale / OldHiveDecimal getInternalStorage() behavior.
*
- * @param serializeScale
- * @param scratchLongs
- * @param buffer
- * @return
+ * @param fastSignum the sign number (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale
+ * @param serializeScale the scale to serialize
+ * @param scratchLongs a scratch array of longs
+ * @param buffer the buffer to serialize into
+ * @return the number of bytes used to serialize the number
*/
public static int fastBigIntegerBytesScaled(
final int fastSignum, long fast0, long fast1, long fast2,
@@ -2801,6 +2881,12 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* NOTE: Fractional digits are ignored in the test since fastByteValueClip() will
* remove them (round down).
*
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
* @return True when fastByteValueClip() will return a correct byte.
*/
public static boolean fastIsByte(
@@ -2953,6 +3039,12 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* NOTE: Fractional digits are ignored in the test since fastShortValueClip() will
* remove them (round down).
*
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
* @return True when fastShortValueClip() will return a correct short.
*/
public static boolean fastIsShort(
@@ -3100,6 +3192,12 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* NOTE: Fractional digits are ignored in the test since fastIntValueClip() will
* remove them (round down).
*
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
* @return True when fastIntValueClip() will return a correct int.
*/
public static boolean fastIsInt(
@@ -3247,6 +3345,12 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* NOTE: Fractional digits are ignored in the test since fastLongValueClip() will
* remove them (round down).
*
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
* @return True when fastLongValueClip() will return a correct long.
*/
public static boolean fastIsLong(
@@ -3449,6 +3553,13 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
/**
* Get a BigInteger representing the decimal's digits without a dot.
*
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
+ * @param fastSerializationScale the scale to serialize
* @return Returns a signed BigInteger.
*/
public static BigInteger fastBigIntegerValue(
@@ -3521,7 +3632,13 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* NOTE: We are not representing our decimal as BigDecimal now as OldHiveDecimal did, so this
* is now slower.
*
- * @return
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
+ * @return the BigDecimal equivalent
*/
public static BigDecimal fastBigDecimalValue(
int fastSignum, long fast0, long fast1, long fast2,
@@ -3872,6 +3989,13 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* Used by map join and other Hive internal purposes where performance is important.
*
* IMPORTANT: See comments for fastHashCode(), too.
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
+ * @return the hash code
*/
public static int fastNewFasterHashCode(
int fastSignum, long fast0, long fast1, long fast2, int fastIntegerDigitCount, int fastScale) {
@@ -3894,7 +4018,14 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
*
* NOTE: It is necessary to create a BigDecimal object and use its hash code, so this method is
* slow.
- * @return
+ *
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
+ * @return the hash code
*/
public static int fastHashCode(
int fastSignum, long fast0, long fast1, long fast2, int fastIntegerDigitCount, int fastScale) {
@@ -4773,6 +4904,15 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* ceiling(12400.8302, -2) = 12500 // E.g. Positive case FAST_ROUND_CEILING
* rr rrrr
*
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
+ * @param roundPower the power to round to
+ * @param fastResult an object to reuse
+ * @return was the operation successful
*/
public static boolean fastRoundIntegerUp(
int fastSignum, long fast0, long fast1, long fast2,
@@ -4855,6 +4995,16 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* The fraction being scaled away is thrown away.
*
* The signum will be updated if the result is 0, otherwise the original sign is unchanged.
+ *
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
+ * @param roundPower the power to round to
+ * @param fastResult an object to reuse
+ * @return was the operation successful?
*/
public static boolean fastRoundIntegerDown(
int fastSignum, long fast0, long fast1, long fast2,
@@ -4924,8 +5074,17 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
/**
* Fast decimal scale down by factor of 10 with rounding ROUND_HALF_UP.
*
- * When the fraction being scaled away is >= 0.5, the add 1.
- *
+ * When the fraction being scaled away is >= 0.5, the add 1.
+ *
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
+ * @param roundPower the power to round to
+ * @param fastResult an object to reuse
+ * @return was the operation successful?
*/
public static boolean fastRoundIntegerHalfUp(
int fastSignum, long fast0, long fast1, long fast2,
@@ -5014,10 +5173,20 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* Fast decimal scale down by factor of 10 with rounding ROUND_HALF_EVEN.
*
* When the fraction being scaled away is exactly 0.5, then round and add 1 only if aaa.
- * When fraction is not exactly 0.5, then if fraction > 0.5 then add 1.
+ * When fraction is not exactly 0.5, then if fraction > 0.5 then add 1.
* Otherwise, throw away fraction.
*
* The signum will be updated if the result is 0, otherwise the original sign is unchanged.
+ *
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
+ * @param roundPower the power to round to
+ * @param fastResult an object to reuse
+ * @return was the operation successful?
*/
public static boolean fastRoundIntegerHalfEven(
int fastSignum, long fast0, long fast1, long fast2,
@@ -5106,7 +5275,16 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
*
* When the fraction being scaled away is non-zero, return false.
*
- * The signum will be updated if the result is 0, otherwise the original sign is unchanged.
+ * The signum will be updated if the result is 0, otherwise the original sign
+ * is unchanged.
+ *
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param scaleDown the digits to scale down by
+ * @param fastResult an object to reuse
+ * @return was the operation successful?
*/
public static boolean fastScaleDownNoRound(
int fastSignum, long fast0, long fast1, long fast2,
@@ -5210,6 +5388,13 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
*
* When the fraction being scaled away is non-zero, the add 1.
*
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param scaleDown the number of integer digits to scale
+ * @param fastResult an object to reuse
+ * @return was the operation successfule?
*/
public static boolean fastRoundFractionalUp(
int fastSignum, long fast0, long fast1, long fast2,
@@ -5272,6 +5457,12 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
*
* The fraction being scaled away is thrown away.
*
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param scaleDown the number of integer digits to scale
+ * @param fastResult an object to reuse
*/
public static void fastRoundFractionalDown(
int fastSignum, long fast0, long fast1, long fast2,
@@ -5305,8 +5496,15 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
/**
* Fast decimal scale down by factor of 10 with rounding ROUND_HALF_UP.
*
- * When the fraction being scaled away is >= 0.5, the add 1.
+ * When the fraction being scaled away is >= 0.5, the add 1.
*
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param scaleDown the number of integer digits to scale
+ * @param fastResult an object to reuse
+ * @return was the operation successfule?
*/
public static boolean fastRoundFractionalHalfUp(
int fastSignum, long fast0, long fast1, long fast2,
@@ -5371,8 +5569,17 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
/**
* Fast decimal scale down by factor of 10 with rounding ROUND_HALF_UP.
*
- * When the fraction being scaled away is >= 0.5, the add 1.
- *
+ * When the fraction being scaled away is >= 0.5, the add 1.
+ *
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fast3 word 3
+ * @param fast4 word 4
+ * @param scaleDown the number of integer digits to scale
+ * @param fastResult an object to reuse
+ * @return was the operation successfule?
*/
public static boolean fastRoundFractionalHalfUp5Words(
int fastSignum, long fast0, long fast1, long fast2, long fast3, long fast4,
@@ -5562,9 +5769,16 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* Fast decimal scale down by factor of 10 with rounding ROUND_HALF_EVEN.
*
* When the fraction being scaled away is exactly 0.5, then round and add 1 only if aaa.
- * When fraction is not exactly 0.5, then if fraction > 0.5 then add 1.
+ * When fraction is not exactly 0.5, then if fraction > 0.5 then add 1.
* Otherwise, throw away fraction.
*
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param scaleDown the number of integer digits to scale
+ * @param fastResult an object to reuse
+ * @return was the operation successfule?
*/
public static boolean fastRoundFractionalHalfEven(
int fastSignum, long fast0, long fast1, long fast2,
@@ -5657,6 +5871,12 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* Fast decimal scale down by factor of 10 with NO rounding.
*
* The signum will be updated if the result is 0, otherwise the original sign is unchanged.
+ *
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param scaleDown the number of integer digits to scale
+ * @param fastResult an object to reuse
*/
public static void doFastScaleDown(
long fast0, long fast1, long fast2,
@@ -5733,6 +5953,12 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
/**
* Fast decimal scale up by factor of 10.
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param scaleUp the number of integer digits to scale up by
+ * @param fastResult an object to reuse
+ * @return was the operation successfule?
*/
public static boolean fastScaleUp(
long fast0, long fast1, long fast2,
@@ -7492,7 +7718,7 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* + 0.0000000000006711991169422033 dec2 (int digits 0, scale 28)
*
* Trying to make dec1 to have a scale of 28 (i.e. by adding trailing zeroes) would exceed
- * MAX_PRECISION (int digits 27 + 28 > 38).
+ * MAX_PRECISION (int digits 27 + 28 > 38).
*
* In this example we need to make sure we have enough integer digit room in the result to
* handle dec1's digits. In order to maintain that, we will need to get rid of lower
@@ -7507,6 +7733,19 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* So, the simplest thing is to emulate what OldHiveDecimal does and do the full digit addition
* and then fit the result afterwards.
*
+ * @param leftSignum The left sign (-1, 0, or +1)
+ * @param leftFast0 The left word 0 of reprentation
+ * @param leftFast1 word 1
+ * @param leftFast2 word 2
+ * @param leftIntegerDigitCount The left number of integer digits
+ * @param leftScale the left scale
+ * @param rightSignum The right sign (-1, 0, or +1)
+ * @param rightFast0 The right word 0 of reprentation
+ * @param rightFast1 word 1
+ * @param rightFast2 word 2
+ * @param rightIntegerDigitCount The right number of integer digits
+ * @param rightScale the right scale
+ * @param fastResult an object to reuse
* @return True if the addition was successful; Otherwise, false is returned on overflow.
*/
public static boolean fastAdd(
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java b/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java
index bf954a8..7beb7bf 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java
@@ -47,7 +47,7 @@ import java.math.BigInteger;
* DECIMAL(precision, scale)}
* }
* <p>
- * The declared scale must be <= precision.
+ * The declared scale must be <= precision.
* <p>
* Use DECIMAL instead of DOUBLE when exact numeric accuracy is required. Not all decimal numbers
* (radix 10) are exactly representable in the binary (radix 2 based) floating point type DOUBLE and
@@ -185,7 +185,7 @@ public final class HiveDecimal extends FastHiveDecimal implements Comparable<Hiv
* <p>
* The Hive function is ROUND.
* <p>
- * For result, throw away round fraction. If the round fraction is >= 0.5, then add 1 when
+ * For result, throw away round fraction. If the round fraction is >= 0.5, then add 1 when
* positive and subtract 1 when negative. So, the sign is irrelevant.
* <p>
* (Example here rounds at scale 0)
@@ -264,6 +264,8 @@ public final class HiveDecimal extends FastHiveDecimal implements Comparable<Hiv
/**
* Create a HiveDecimal from a FastHiveDecimal object. Used by HiveDecimalWritable.
+ * @param fastDec the value to set
+ * @return new hive decimal
*/
@HiveDecimalVersionV2
public static HiveDecimal createFromFast(FastHiveDecimal fastDec) {
@@ -279,14 +281,14 @@ public final class HiveDecimal extends FastHiveDecimal implements Comparable<Hiv
* 99,999,999,999,999,999,999,999,999,999,999,999,999 or 10^38 - 1.
*
* When the BigDecimal value's precision exceeds MAX_PRECISION and there are fractional digits
- * because of scale > 0, then lower digits are trimmed off with rounding to meet the
+ * because of scale > 0, then lower digits are trimmed off with rounding to meet the
* MAX_PRECISION requirement.
*
* Also, BigDecimal supports negative scale -- which means multiplying the value by 10^abs(scale).
* And, BigDecimal allows for a non-zero scale for zero. We normalize that so zero always has
* scale 0.
*
- * @param bigDecimal
+ * @param bigDecimal the value to set
* @return The HiveDecimal with the BigDecimal's value adjusted down to a maximum precision.
* Otherwise, null is returned for overflow.
*/
@@ -297,7 +299,7 @@ public final class HiveDecimal extends FastHiveDecimal implements Comparable<Hiv
/**
* Same as the above create method, except fractional digit rounding can be turned off.
- * @param bigDecimal
+ * @param bigDecimal the value to set
* @param allowRounding True requires all of the bigDecimal value be converted to the decimal
* without loss of precision.
* @return
@@ -318,7 +320,7 @@ public final class HiveDecimal extends FastHiveDecimal implements Comparable<Hiv
* We will have overflow if BigInteger exceed MAX_PRECISION digits or
* 99,999,999,999,999,999,999,999,999,999,999,999,999 or 10^38 - 1.
*
- * @param bigInteger
+ * @param bigInteger the value to set
* @return A HiveDecimal object with the exact BigInteger's value.
* Otherwise, null is returned on overflow.
*/
@@ -341,11 +343,11 @@ public final class HiveDecimal extends FastHiveDecimal implements Comparable<Hiv
* The resulting decimal will have fractional digits when the specified scale is greater than 0.
*
* When the BigInteger's value's precision exceeds MAX_PRECISION and there are fractional digits
- * because of scale > 0, then lower digits are trimmed off with rounding to meet the
+ * because of scale > 0, then lower digits are trimmed off with rounding to meet the
* MAX_PRECISION requirement.
*
- * @param bigInteger
- * @param scale
+ * @param bigInteger the value to set
+ * @param scale the scale to set
* @return A HiveDecimal object with the BigInteger's value adjusted for scale.
* Otherwise, null is returned on overflow.
*/
@@ -365,6 +367,8 @@ public final class HiveDecimal extends FastHiveDecimal implements Comparable<Hiv
* We support parsing a decimal with an exponent because the previous version
* (i.e. OldHiveDecimal) uses the BigDecimal parser and was able to.
*
+ * @param string the string to parse
+ * @return a new hive decimal
*/
@HiveDecimalVersionV1
public static HiveDecimal create(String string) {
@@ -378,9 +382,9 @@ public final class HiveDecimal extends FastHiveDecimal implements Comparable<Hiv
/**
* Same as the method above, except blanks before and after are tolerated.
- * @param string
+ * @param string the string to parse
* @param trimBlanks True specifies leading and trailing blanks are to be ignored.
- * @return
+ * @return a new hive decimal
*/
@HiveDecimalVersionV2
public static HiveDecimal create(String string, boolean trimBlanks) {
@@ -766,7 +770,7 @@ public final class HiveDecimal extends FastHiveDecimal implements Comparable<Hiv
* Is does the equivalent of a setScale(int newScale). So, more than 38 digits may be returned.
* See that method for more details on how this can happen.
* <p>
- * @param scale The number of digits after the decimal point
+ * @param formatScale The number of digits after the decimal point
* @return The scaled decimal representation string representation.
*/
@HiveDecimalVersionV1
@@ -894,7 +898,7 @@ public final class HiveDecimal extends FastHiveDecimal implements Comparable<Hiv
/**
* Are two decimal content (values) equal?
* <p>
- * @obj The 2nd decimal.
+ * @param obj The 2nd decimal.
* @return When obj is null or not class HiveDecimal, the return is false.
* Otherwise, returns true when the decimal values are exactly equal.
*/
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java
index 08e8638..f914a22 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java
@@ -442,7 +442,7 @@ public class BytesColumnVector extends ColumnVector {
}
if (noNulls || !isNull[row]) {
buffer.append('"');
- buffer.append(new String(this.buffer, start[row], length[row]));
+ buffer.append(new String(vector[row], start[row], length[row]));
buffer.append('"');
} else {
buffer.append("null");
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
index 507d46e..065c1fa 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
@@ -191,13 +191,13 @@ public abstract class ColumnVector {
* This method is deliberately *not* recursive because the complex types
* can easily have more (or less) children than the upper levels.
* @param size the new minimum size
- * @param presesrveData should the old data be preserved?
+ * @param preserveData should the old data be preserved?
*/
- public void ensureSize(int size, boolean presesrveData) {
+ public void ensureSize(int size, boolean preserveData) {
if (isNull.length < size) {
boolean[] oldArray = isNull;
isNull = new boolean[size];
- if (presesrveData && !noNulls) {
+ if (preserveData && !noNulls) {
if (isRepeating) {
isNull[0] = oldArray[0];
} else {
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java
index 3307f57..9d579ce 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java
@@ -182,7 +182,7 @@ public class TimestampColumnVector extends ColumnVector {
/**
* Return a double representation of a Timestamp.
- * @param elementNum
+ * @param timestamp
* @return
*/
public static double getDouble(Timestamp timestamp) {
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/storage-api/src/java/org/apache/hadoop/hive/ql/util/TimestampUtils.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/util/TimestampUtils.java b/storage-api/src/java/org/apache/hadoop/hive/ql/util/TimestampUtils.java
index c16d67e..dfc7272 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/ql/util/TimestampUtils.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/ql/util/TimestampUtils.java
@@ -109,7 +109,7 @@ public class TimestampUtils {
* is the nanoseconds and integer part is the number of seconds.
*
* This is a HiveDecimalWritable variation with supplied scratch objects.
- * @param decdecWritable
+ * @param decWritable
* @param scratchDecWritable1
* @param scratchDecWritable2
* @return
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java
----------------------------------------------------------------------
diff --git a/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java b/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java
index 41b4b65..9ac7ba0 100644
--- a/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java
+++ b/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java
@@ -20,6 +20,10 @@ package org.apache.hadoop.hive.ql.exec.vector;
import org.junit.Test;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.sql.Timestamp;
+
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -92,4 +96,36 @@ public class TestStructColumnVector {
assertEquals(false, input1.isRepeating);
assertEquals(true, input.noNulls);
}
+
+ @Test
+ public void testStringify() throws IOException {
+ VectorizedRowBatch batch = new VectorizedRowBatch(2);
+ LongColumnVector x1 = new LongColumnVector();
+ TimestampColumnVector x2 = new TimestampColumnVector();
+ StructColumnVector x = new StructColumnVector(1024, x1, x2);
+ BytesColumnVector y = new BytesColumnVector();
+ batch.cols[0] = x;
+ batch.cols[1] = y;
+ batch.reset();
+ Timestamp ts = Timestamp.valueOf("2000-01-01 00:00:00");
+ for(int r=0; r < 10; ++r) {
+ batch.size += 1;
+ x1.vector[r] = 3 * r;
+ ts.setTime(ts.getTime() + 1000);
+ x2.set(r, ts);
+ byte[] buffer = ("value " + r).getBytes(StandardCharsets.UTF_8);
+ y.setRef(r, buffer, 0, buffer.length);
+ }
+ final String EXPECTED = ("[[0, 2000-01-01 00:00:01.0], \"value 0\"]\n" +
+ "[[3, 2000-01-01 00:00:02.0], \"value 1\"]\n" +
+ "[[6, 2000-01-01 00:00:03.0], \"value 2\"]\n" +
+ "[[9, 2000-01-01 00:00:04.0], \"value 3\"]\n" +
+ "[[12, 2000-01-01 00:00:05.0], \"value 4\"]\n" +
+ "[[15, 2000-01-01 00:00:06.0], \"value 5\"]\n" +
+ "[[18, 2000-01-01 00:00:07.0], \"value 6\"]\n" +
+ "[[21, 2000-01-01 00:00:08.0], \"value 7\"]\n" +
+ "[[24, 2000-01-01 00:00:09.0], \"value 8\"]\n" +
+ "[[27, 2000-01-01 00:00:10.0], \"value 9\"]");
+ assertEquals(EXPECTED, batch.toString());
+ }
}