You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by om...@apache.org on 2017/04/26 22:08:04 UTC
[1/5] hive git commit: HIVE-15419. Separate storage-api to be
released independently. (omalley reviewed by Prasanth Jayachandran and Eugene
Koifman)
Repository: hive
Updated Branches:
refs/heads/branch-2.2 1b7410245 -> 9bedccc4b
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/storage-api/src/test/org/apache/hive/common/util/TestBloomFilter.java
----------------------------------------------------------------------
diff --git a/storage-api/src/test/org/apache/hive/common/util/TestBloomFilter.java b/storage-api/src/test/org/apache/hive/common/util/TestBloomFilter.java
new file mode 100644
index 0000000..e4ee93a
--- /dev/null
+++ b/storage-api/src/test/org/apache/hive/common/util/TestBloomFilter.java
@@ -0,0 +1,589 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hive.common.util;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.util.ArrayList;
+import java.util.Random;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ *
+ */
+public class TestBloomFilter {
+ private static final int COUNT = 100;
+ Random rand = new Random(123);
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testBloomIllegalArg1() {
+ BloomFilter bf = new BloomFilter(0, 0);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testBloomIllegalArg2() {
+ BloomFilter bf = new BloomFilter(0, 0.1);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testBloomIllegalArg3() {
+ BloomFilter bf = new BloomFilter(1, 0.0);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testBloomIllegalArg4() {
+ BloomFilter bf = new BloomFilter(1, 1.0);
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testBloomIllegalArg5() {
+ BloomFilter bf = new BloomFilter(-1, -1);
+ }
+
+
+ @Test
+ public void testBloomNumBits() {
+ assertEquals(0, BloomFilter.optimalNumOfBits(0, 0));
+ assertEquals(0, BloomFilter.optimalNumOfBits(0, 1));
+ assertEquals(0, BloomFilter.optimalNumOfBits(1, 1));
+ assertEquals(7, BloomFilter.optimalNumOfBits(1, 0.03));
+ assertEquals(72, BloomFilter.optimalNumOfBits(10, 0.03));
+ assertEquals(729, BloomFilter.optimalNumOfBits(100, 0.03));
+ assertEquals(7298, BloomFilter.optimalNumOfBits(1000, 0.03));
+ assertEquals(72984, BloomFilter.optimalNumOfBits(10000, 0.03));
+ assertEquals(729844, BloomFilter.optimalNumOfBits(100000, 0.03));
+ assertEquals(7298440, BloomFilter.optimalNumOfBits(1000000, 0.03));
+ assertEquals(6235224, BloomFilter.optimalNumOfBits(1000000, 0.05));
+ assertEquals(1870567268, BloomFilter.optimalNumOfBits(300000000, 0.05));
+ assertEquals(1437758756, BloomFilter.optimalNumOfBits(300000000, 0.1));
+ assertEquals(432808512, BloomFilter.optimalNumOfBits(300000000, 0.5));
+ assertEquals(1393332198, BloomFilter.optimalNumOfBits(3000000000L, 0.8));
+ assertEquals(657882327, BloomFilter.optimalNumOfBits(3000000000L, 0.9));
+ assertEquals(0, BloomFilter.optimalNumOfBits(3000000000L, 1));
+ }
+
+ @Test
+ public void testBloomNumHashFunctions() {
+ assertEquals(1, BloomFilter.optimalNumOfHashFunctions(-1, -1));
+ assertEquals(1, BloomFilter.optimalNumOfHashFunctions(0, 0));
+ assertEquals(1, BloomFilter.optimalNumOfHashFunctions(10, 0));
+ assertEquals(1, BloomFilter.optimalNumOfHashFunctions(10, 10));
+ assertEquals(7, BloomFilter.optimalNumOfHashFunctions(10, 100));
+ assertEquals(1, BloomFilter.optimalNumOfHashFunctions(100, 100));
+ assertEquals(1, BloomFilter.optimalNumOfHashFunctions(1000, 100));
+ assertEquals(1, BloomFilter.optimalNumOfHashFunctions(10000, 100));
+ assertEquals(1, BloomFilter.optimalNumOfHashFunctions(100000, 100));
+ assertEquals(1, BloomFilter.optimalNumOfHashFunctions(1000000, 100));
+ }
+
+ @Test
+ public void testBloomFilterBytes() {
+ BloomFilter bf = new BloomFilter(10000);
+ byte[] val = new byte[]{1, 2, 3};
+ byte[] val1 = new byte[]{1, 2, 3, 4};
+ byte[] val2 = new byte[]{1, 2, 3, 4, 5};
+ byte[] val3 = new byte[]{1, 2, 3, 4, 5, 6};
+
+ assertEquals(false, bf.test(val));
+ assertEquals(false, bf.test(val1));
+ assertEquals(false, bf.test(val2));
+ assertEquals(false, bf.test(val3));
+ bf.add(val);
+ assertEquals(true, bf.test(val));
+ assertEquals(false, bf.test(val1));
+ assertEquals(false, bf.test(val2));
+ assertEquals(false, bf.test(val3));
+ bf.add(val1);
+ assertEquals(true, bf.test(val));
+ assertEquals(true, bf.test(val1));
+ assertEquals(false, bf.test(val2));
+ assertEquals(false, bf.test(val3));
+ bf.add(val2);
+ assertEquals(true, bf.test(val));
+ assertEquals(true, bf.test(val1));
+ assertEquals(true, bf.test(val2));
+ assertEquals(false, bf.test(val3));
+ bf.add(val3);
+ assertEquals(true, bf.test(val));
+ assertEquals(true, bf.test(val1));
+ assertEquals(true, bf.test(val2));
+ assertEquals(true, bf.test(val3));
+
+ byte[] randVal = new byte[COUNT];
+ for (int i = 0; i < COUNT; i++) {
+ rand.nextBytes(randVal);
+ bf.add(randVal);
+ }
+ // last value should be present
+ assertEquals(true, bf.test(randVal));
+ // most likely this value should not exist
+ randVal[0] = 0;
+ randVal[1] = 0;
+ randVal[2] = 0;
+ randVal[3] = 0;
+ randVal[4] = 0;
+ assertEquals(false, bf.test(randVal));
+
+ assertEquals(7800, bf.sizeInBytes());
+ }
+
+ @Test
+ public void testBloomFilterByte() {
+ BloomFilter bf = new BloomFilter(10000);
+ byte val = Byte.MIN_VALUE;
+ byte val1 = 1;
+ byte val2 = 2;
+ byte val3 = Byte.MAX_VALUE;
+
+ assertEquals(false, bf.testLong(val));
+ assertEquals(false, bf.testLong(val1));
+ assertEquals(false, bf.testLong(val2));
+ assertEquals(false, bf.testLong(val3));
+ bf.addLong(val);
+ assertEquals(true, bf.testLong(val));
+ assertEquals(false, bf.testLong(val1));
+ assertEquals(false, bf.testLong(val2));
+ assertEquals(false, bf.testLong(val3));
+ bf.addLong(val1);
+ assertEquals(true, bf.testLong(val));
+ assertEquals(true, bf.testLong(val1));
+ assertEquals(false, bf.testLong(val2));
+ assertEquals(false, bf.testLong(val3));
+ bf.addLong(val2);
+ assertEquals(true, bf.testLong(val));
+ assertEquals(true, bf.testLong(val1));
+ assertEquals(true, bf.testLong(val2));
+ assertEquals(false, bf.testLong(val3));
+ bf.addLong(val3);
+ assertEquals(true, bf.testLong(val));
+ assertEquals(true, bf.testLong(val1));
+ assertEquals(true, bf.testLong(val2));
+ assertEquals(true, bf.testLong(val3));
+
+ byte randVal = 0;
+ for (int i = 0; i < COUNT; i++) {
+ randVal = (byte) rand.nextInt(Byte.MAX_VALUE);
+ bf.addLong(randVal);
+ }
+ // last value should be present
+ assertEquals(true, bf.testLong(randVal));
+ // most likely this value should not exist
+ assertEquals(false, bf.testLong((byte) -120));
+
+ assertEquals(7800, bf.sizeInBytes());
+ }
+
+ @Test
+ public void testBloomFilterInt() {
+ BloomFilter bf = new BloomFilter(10000);
+ int val = Integer.MIN_VALUE;
+ int val1 = 1;
+ int val2 = 2;
+ int val3 = Integer.MAX_VALUE;
+
+ assertEquals(false, bf.testLong(val));
+ assertEquals(false, bf.testLong(val1));
+ assertEquals(false, bf.testLong(val2));
+ assertEquals(false, bf.testLong(val3));
+ bf.addLong(val);
+ assertEquals(true, bf.testLong(val));
+ assertEquals(false, bf.testLong(val1));
+ assertEquals(false, bf.testLong(val2));
+ assertEquals(false, bf.testLong(val3));
+ bf.addLong(val1);
+ assertEquals(true, bf.testLong(val));
+ assertEquals(true, bf.testLong(val1));
+ assertEquals(false, bf.testLong(val2));
+ assertEquals(false, bf.testLong(val3));
+ bf.addLong(val2);
+ assertEquals(true, bf.testLong(val));
+ assertEquals(true, bf.testLong(val1));
+ assertEquals(true, bf.testLong(val2));
+ assertEquals(false, bf.testLong(val3));
+ bf.addLong(val3);
+ assertEquals(true, bf.testLong(val));
+ assertEquals(true, bf.testLong(val1));
+ assertEquals(true, bf.testLong(val2));
+ assertEquals(true, bf.testLong(val3));
+
+ int randVal = 0;
+ for (int i = 0; i < COUNT; i++) {
+ randVal = rand.nextInt();
+ bf.addLong(randVal);
+ }
+ // last value should be present
+ assertEquals(true, bf.testLong(randVal));
+ // most likely this value should not exist
+ assertEquals(false, bf.testLong(-120));
+
+ assertEquals(7800, bf.sizeInBytes());
+ }
+
+ @Test
+ public void testBloomFilterLong() {
+ BloomFilter bf = new BloomFilter(10000);
+ long val = Long.MIN_VALUE;
+ long val1 = 1;
+ long val2 = 2;
+ long val3 = Long.MAX_VALUE;
+
+ assertEquals(false, bf.testLong(val));
+ assertEquals(false, bf.testLong(val1));
+ assertEquals(false, bf.testLong(val2));
+ assertEquals(false, bf.testLong(val3));
+ bf.addLong(val);
+ assertEquals(true, bf.testLong(val));
+ assertEquals(false, bf.testLong(val1));
+ assertEquals(false, bf.testLong(val2));
+ assertEquals(false, bf.testLong(val3));
+ bf.addLong(val1);
+ assertEquals(true, bf.testLong(val));
+ assertEquals(true, bf.testLong(val1));
+ assertEquals(false, bf.testLong(val2));
+ assertEquals(false, bf.testLong(val3));
+ bf.addLong(val2);
+ assertEquals(true, bf.testLong(val));
+ assertEquals(true, bf.testLong(val1));
+ assertEquals(true, bf.testLong(val2));
+ assertEquals(false, bf.testLong(val3));
+ bf.addLong(val3);
+ assertEquals(true, bf.testLong(val));
+ assertEquals(true, bf.testLong(val1));
+ assertEquals(true, bf.testLong(val2));
+ assertEquals(true, bf.testLong(val3));
+
+ long randVal = 0;
+ for (int i = 0; i < COUNT; i++) {
+ randVal = rand.nextLong();
+ bf.addLong(randVal);
+ }
+ // last value should be present
+ assertEquals(true, bf.testLong(randVal));
+ // most likely this value should not exist
+ assertEquals(false, bf.testLong(-120));
+
+ assertEquals(7800, bf.sizeInBytes());
+ }
+
+ @Test
+ public void testBloomFilterFloat() {
+ BloomFilter bf = new BloomFilter(10000);
+ float val = Float.MIN_VALUE;
+ float val1 = 1.1f;
+ float val2 = 2.2f;
+ float val3 = Float.MAX_VALUE;
+
+ assertEquals(false, bf.testDouble(val));
+ assertEquals(false, bf.testDouble(val1));
+ assertEquals(false, bf.testDouble(val2));
+ assertEquals(false, bf.testDouble(val3));
+ bf.addDouble(val);
+ assertEquals(true, bf.testDouble(val));
+ assertEquals(false, bf.testDouble(val1));
+ assertEquals(false, bf.testDouble(val2));
+ assertEquals(false, bf.testDouble(val3));
+ bf.addDouble(val1);
+ assertEquals(true, bf.testDouble(val));
+ assertEquals(true, bf.testDouble(val1));
+ assertEquals(false, bf.testDouble(val2));
+ assertEquals(false, bf.testDouble(val3));
+ bf.addDouble(val2);
+ assertEquals(true, bf.testDouble(val));
+ assertEquals(true, bf.testDouble(val1));
+ assertEquals(true, bf.testDouble(val2));
+ assertEquals(false, bf.testDouble(val3));
+ bf.addDouble(val3);
+ assertEquals(true, bf.testDouble(val));
+ assertEquals(true, bf.testDouble(val1));
+ assertEquals(true, bf.testDouble(val2));
+ assertEquals(true, bf.testDouble(val3));
+
+ float randVal = 0;
+ for (int i = 0; i < COUNT; i++) {
+ randVal = rand.nextFloat();
+ bf.addDouble(randVal);
+ }
+ // last value should be present
+ assertEquals(true, bf.testDouble(randVal));
+ // most likely this value should not exist
+ assertEquals(false, bf.testDouble(-120.2f));
+
+ assertEquals(7800, bf.sizeInBytes());
+ }
+
+ @Test
+ public void testBloomFilterDouble() {
+ BloomFilter bf = new BloomFilter(10000);
+ double val = Double.MIN_VALUE;
+ double val1 = 1.1d;
+ double val2 = 2.2d;
+ double val3 = Double.MAX_VALUE;
+
+ assertEquals(false, bf.testDouble(val));
+ assertEquals(false, bf.testDouble(val1));
+ assertEquals(false, bf.testDouble(val2));
+ assertEquals(false, bf.testDouble(val3));
+ bf.addDouble(val);
+ assertEquals(true, bf.testDouble(val));
+ assertEquals(false, bf.testDouble(val1));
+ assertEquals(false, bf.testDouble(val2));
+ assertEquals(false, bf.testDouble(val3));
+ bf.addDouble(val1);
+ assertEquals(true, bf.testDouble(val));
+ assertEquals(true, bf.testDouble(val1));
+ assertEquals(false, bf.testDouble(val2));
+ assertEquals(false, bf.testDouble(val3));
+ bf.addDouble(val2);
+ assertEquals(true, bf.testDouble(val));
+ assertEquals(true, bf.testDouble(val1));
+ assertEquals(true, bf.testDouble(val2));
+ assertEquals(false, bf.testDouble(val3));
+ bf.addDouble(val3);
+ assertEquals(true, bf.testDouble(val));
+ assertEquals(true, bf.testDouble(val1));
+ assertEquals(true, bf.testDouble(val2));
+ assertEquals(true, bf.testDouble(val3));
+
+ double randVal = 0;
+ for (int i = 0; i < COUNT; i++) {
+ randVal = rand.nextDouble();
+ bf.addDouble(randVal);
+ }
+ // last value should be present
+ assertEquals(true, bf.testDouble(randVal));
+ // most likely this value should not exist
+ assertEquals(false, bf.testDouble(-120.2d));
+
+ assertEquals(7800, bf.sizeInBytes());
+ }
+
+ @Test
+ public void testBloomFilterString() {
+ BloomFilter bf = new BloomFilter(100000);
+ String val = "bloo";
+ String val1 = "bloom fil";
+ String val2 = "bloom filter";
+ String val3 = "cuckoo filter";
+
+ assertEquals(false, bf.testString(val));
+ assertEquals(false, bf.testString(val1));
+ assertEquals(false, bf.testString(val2));
+ assertEquals(false, bf.testString(val3));
+ bf.addString(val);
+ assertEquals(true, bf.testString(val));
+ assertEquals(false, bf.testString(val1));
+ assertEquals(false, bf.testString(val2));
+ assertEquals(false, bf.testString(val3));
+ bf.addString(val1);
+ assertEquals(true, bf.testString(val));
+ assertEquals(true, bf.testString(val1));
+ assertEquals(false, bf.testString(val2));
+ assertEquals(false, bf.testString(val3));
+ bf.addString(val2);
+ assertEquals(true, bf.testString(val));
+ assertEquals(true, bf.testString(val1));
+ assertEquals(true, bf.testString(val2));
+ assertEquals(false, bf.testString(val3));
+ bf.addString(val3);
+ assertEquals(true, bf.testString(val));
+ assertEquals(true, bf.testString(val1));
+ assertEquals(true, bf.testString(val2));
+ assertEquals(true, bf.testString(val3));
+
+ long randVal = 0;
+ for (int i = 0; i < COUNT; i++) {
+ randVal = rand.nextLong();
+ bf.addString(Long.toString(randVal));
+ }
+ // last value should be present
+ assertEquals(true, bf.testString(Long.toString(randVal)));
+ // most likely this value should not exist
+ assertEquals(false, bf.testString(Long.toString(-120)));
+
+ assertEquals(77944, bf.sizeInBytes());
+ }
+
+ @Test
+ public void testMerge() {
+ BloomFilter bf = new BloomFilter(10000);
+ String val = "bloo";
+ String val1 = "bloom fil";
+ String val2 = "bloom filter";
+ String val3 = "cuckoo filter";
+ bf.addString(val);
+ bf.addString(val1);
+ bf.addString(val2);
+ bf.addString(val3);
+
+ BloomFilter bf2 = new BloomFilter(10000);
+ String v = "2_bloo";
+ String v1 = "2_bloom fil";
+ String v2 = "2_bloom filter";
+ String v3 = "2_cuckoo filter";
+ bf2.addString(v);
+ bf2.addString(v1);
+ bf2.addString(v2);
+ bf2.addString(v3);
+
+ assertEquals(true, bf.testString(val));
+ assertEquals(true, bf.testString(val1));
+ assertEquals(true, bf.testString(val2));
+ assertEquals(true, bf.testString(val3));
+ assertEquals(false, bf.testString(v));
+ assertEquals(false, bf.testString(v1));
+ assertEquals(false, bf.testString(v2));
+ assertEquals(false, bf.testString(v3));
+
+ bf.merge(bf2);
+
+ assertEquals(true, bf.testString(val));
+ assertEquals(true, bf.testString(val1));
+ assertEquals(true, bf.testString(val2));
+ assertEquals(true, bf.testString(val3));
+ assertEquals(true, bf.testString(v));
+ assertEquals(true, bf.testString(v1));
+ assertEquals(true, bf.testString(v2));
+ assertEquals(true, bf.testString(v3));
+ }
+
+ @Test
+ public void testSerialize() throws Exception {
+ BloomFilter bf1 = new BloomFilter(10000);
+ String[] inputs = {
+ "bloo",
+ "bloom fil",
+ "bloom filter",
+ "cuckoo filter",
+ };
+
+ for (String val : inputs) {
+ bf1.addString(val);
+ }
+
+ // Serialize/deserialize
+ ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
+ BloomFilter.serialize(bytesOut, bf1);
+ ByteArrayInputStream bytesIn = new ByteArrayInputStream(bytesOut.toByteArray());
+ BloomFilter bf2 = BloomFilter.deserialize(bytesIn);
+
+ for (String val : inputs) {
+ assertEquals("Testing bf1 with " + val, true, bf1.testString(val));
+ assertEquals("Testing bf2 with " + val, true, bf2.testString(val));
+ }
+ }
+
+ @Test
+ public void testMergeBloomFilterBytes() throws Exception {
+ BloomFilter bf1 = new BloomFilter(10000);
+ BloomFilter bf2 = new BloomFilter(10000);
+
+ String[] inputs1 = {
+ "bloo",
+ "bloom fil",
+ "bloom filter",
+ "cuckoo filter",
+ };
+
+ String[] inputs2 = {
+ "2_bloo",
+ "2_bloom fil",
+ "2_bloom filter",
+ "2_cuckoo filter",
+ };
+
+ for (String val : inputs1) {
+ bf1.addString(val);
+ }
+ for (String val : inputs2) {
+ bf2.addString(val);
+ }
+
+ ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
+ BloomFilter.serialize(bytesOut, bf1);
+ byte[] bf1Bytes = bytesOut.toByteArray();
+ bytesOut.reset();
+ BloomFilter.serialize(bytesOut, bf1);
+ byte[] bf2Bytes = bytesOut.toByteArray();
+
+ // Merge bytes
+ BloomFilter.mergeBloomFilterBytes(
+ bf1Bytes, 0, bf1Bytes.length,
+ bf2Bytes, 0, bf2Bytes.length);
+
+ // Deserialize and test
+ ByteArrayInputStream bytesIn = new ByteArrayInputStream(bf1Bytes, 0, bf1Bytes.length);
+ BloomFilter bfMerged = BloomFilter.deserialize(bytesIn);
+ // All values should pass test
+ for (String val : inputs1) {
+ bfMerged.addString(val);
+ }
+ for (String val : inputs2) {
+ bfMerged.addString(val);
+ }
+ }
+
+ @Test
+ public void testMergeBloomFilterBytesFailureCases() throws Exception {
+ BloomFilter bf1 = new BloomFilter(1000);
+ BloomFilter bf2 = new BloomFilter(200);
+ // Create bloom filter with same number of bits, but different # hash functions
+ ArrayList<Long> bits = new ArrayList<Long>();
+ for (int idx = 0; idx < bf1.getBitSet().length; ++idx) {
+ bits.add(0L);
+ }
+ BloomFilter bf3 = new BloomFilter(bits, bf1.getBitSize(), bf1.getNumHashFunctions() + 1);
+
+ // Serialize to bytes
+ ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
+ BloomFilter.serialize(bytesOut, bf1);
+ byte[] bf1Bytes = bytesOut.toByteArray();
+
+ bytesOut.reset();
+ BloomFilter.serialize(bytesOut, bf2);
+ byte[] bf2Bytes = bytesOut.toByteArray();
+
+ bytesOut.reset();
+ BloomFilter.serialize(bytesOut, bf3);
+ byte[] bf3Bytes = bytesOut.toByteArray();
+
+ try {
+ // this should fail
+ BloomFilter.mergeBloomFilterBytes(
+ bf1Bytes, 0, bf1Bytes.length,
+ bf2Bytes, 0, bf2Bytes.length);
+ Assert.fail("Expected exception not encountered");
+ } catch (IllegalArgumentException err) {
+ // expected
+ }
+
+ try {
+ // this should fail
+ BloomFilter.mergeBloomFilterBytes(
+ bf1Bytes, 0, bf1Bytes.length,
+ bf3Bytes, 0, bf3Bytes.length);
+ Assert.fail("Expected exception not encountered");
+ } catch (IllegalArgumentException err) {
+ // expected
+ }
+ }
+}
[3/5] hive git commit: Add license and notice file for storage-api
Posted by om...@apache.org.
Add license and notice file for storage-api
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ae528876
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ae528876
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ae528876
Branch: refs/heads/branch-2.2
Commit: ae528876964092c823b400e4a7dbead920a152af
Parents: 0d04553
Author: Owen O'Malley <om...@apache.org>
Authored: Thu Jan 5 09:10:11 2017 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Wed Apr 26 15:03:05 2017 -0700
----------------------------------------------------------------------
storage-api/LICENSE | 203 +++++++++++++++++++++++++++++++++++++++++++++++
storage-api/NOTICE | 6 ++
2 files changed, 209 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ae528876/storage-api/LICENSE
----------------------------------------------------------------------
diff --git a/storage-api/LICENSE b/storage-api/LICENSE
new file mode 100644
index 0000000..9f29a89
--- /dev/null
+++ b/storage-api/LICENSE
@@ -0,0 +1,203 @@
+
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability contains
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "[]"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright [yyyy] [name of copyright owner]
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
http://git-wip-us.apache.org/repos/asf/hive/blob/ae528876/storage-api/NOTICE
----------------------------------------------------------------------
diff --git a/storage-api/NOTICE b/storage-api/NOTICE
new file mode 100644
index 0000000..0452bfe
--- /dev/null
+++ b/storage-api/NOTICE
@@ -0,0 +1,6 @@
+Apache Hive Storage API
+
+Copyright 2008-2017 The Apache Software Foundation
+
+This product includes software developed by The Apache Software
+Foundation (http://www.apache.org/).
[4/5] hive git commit: HIVE-15929. Fix HiveDecimalWritable
compatibility.
Posted by om...@apache.org.
HIVE-15929. Fix HiveDecimalWritable compatibility.
Fixes #149
Signed-off-by: Owen O'Malley <om...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9bedccc4
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9bedccc4
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9bedccc4
Branch: refs/heads/branch-2.2
Commit: 9bedccc4b00308b071481d7b9ab623475a9fb5fb
Parents: a685775
Author: Owen O'Malley <om...@apache.org>
Authored: Tue Feb 14 17:25:46 2017 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Wed Apr 26 15:03:05 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/serde2/io/HiveDecimalWritable.java | 5 ++++-
1 file changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/9bedccc4/storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritable.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritable.java b/storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritable.java
index 94d61b4..ffbe31a 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritable.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritable.java
@@ -947,10 +947,13 @@ public final class HiveDecimalWritable extends FastHiveDecimal
return fastHashCode();
}
+ private static final byte[] EMPTY_ARRAY = new byte[0];
+
@HiveDecimalWritableVersionV1
public byte[] getInternalStorage() {
if (!isSet()) {
- throw new RuntimeException("no value set");
+ // don't break old callers that are trying to reuse storages
+ return EMPTY_ARRAY;
}
if (internalScratchLongs == null) {
[2/5] hive git commit: HIVE-15419. Separate storage-api to be
released independently. (omalley reviewed by Prasanth Jayachandran and Eugene
Koifman)
Posted by om...@apache.org.
HIVE-15419. Separate storage-api to be released independently. (omalley
reviewed by Prasanth Jayachandran and Eugene Koifman)
Fixes #125
Signed-off-by: Owen O'Malley <om...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0d045535
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0d045535
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0d045535
Branch: refs/heads/branch-2.2
Commit: 0d045535d42c62b7521a73427365d875e40fead5
Parents: 1b74102
Author: Owen O'Malley <om...@apache.org>
Authored: Fri Dec 16 16:11:02 2016 -0800
Committer: Owen O'Malley <om...@apache.org>
Committed: Wed Apr 26 12:14:39 2017 -0700
----------------------------------------------------------------------
common/pom.xml | 1 -
.../hive/common/util/TestBloomFilter.java | 589 -------------------
orc/pom.xml | 1 -
packaging/pom.xml | 1 -
pom.xml | 6 +
storage-api/pom.xml | 91 ++-
.../org/apache/hadoop/hive/common/Pool.java | 8 +-
.../apache/hadoop/hive/common/io/Allocator.java | 16 +-
.../apache/hadoop/hive/common/io/DataCache.java | 7 +-
.../hadoop/hive/common/io/DiskRangeList.java | 19 +-
.../hive/common/io/encoded/MemoryBuffer.java | 6 +-
.../hive/common/type/FastHiveDecimal.java | 2 +-
.../hive/common/type/FastHiveDecimalImpl.java | 325 ++++++++--
.../hadoop/hive/common/type/HiveDecimal.java | 30 +-
.../hive/ql/exec/vector/BytesColumnVector.java | 2 +-
.../hive/ql/exec/vector/ColumnVector.java | 6 +-
.../ql/exec/vector/TimestampColumnVector.java | 2 +-
.../hadoop/hive/ql/util/TimestampUtils.java | 2 +-
.../ql/exec/vector/TestStructColumnVector.java | 36 ++
.../hive/common/util/TestBloomFilter.java | 589 +++++++++++++++++++
20 files changed, 1064 insertions(+), 675 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/common/pom.xml
----------------------------------------------------------------------
diff --git a/common/pom.xml b/common/pom.xml
index 150c492..b30ec7b 100644
--- a/common/pom.xml
+++ b/common/pom.xml
@@ -42,7 +42,6 @@
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-storage-api</artifactId>
- <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/common/src/test/org/apache/hive/common/util/TestBloomFilter.java
----------------------------------------------------------------------
diff --git a/common/src/test/org/apache/hive/common/util/TestBloomFilter.java b/common/src/test/org/apache/hive/common/util/TestBloomFilter.java
deleted file mode 100644
index e4ee93a..0000000
--- a/common/src/test/org/apache/hive/common/util/TestBloomFilter.java
+++ /dev/null
@@ -1,589 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hive.common.util;
-
-import static org.junit.Assert.assertEquals;
-
-import java.io.ByteArrayInputStream;
-import java.io.ByteArrayOutputStream;
-import java.util.ArrayList;
-import java.util.Random;
-
-import org.junit.Assert;
-import org.junit.Test;
-
-/**
- *
- */
-public class TestBloomFilter {
- private static final int COUNT = 100;
- Random rand = new Random(123);
-
- @Test(expected = IllegalArgumentException.class)
- public void testBloomIllegalArg1() {
- BloomFilter bf = new BloomFilter(0, 0);
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testBloomIllegalArg2() {
- BloomFilter bf = new BloomFilter(0, 0.1);
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testBloomIllegalArg3() {
- BloomFilter bf = new BloomFilter(1, 0.0);
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testBloomIllegalArg4() {
- BloomFilter bf = new BloomFilter(1, 1.0);
- }
-
- @Test(expected = IllegalArgumentException.class)
- public void testBloomIllegalArg5() {
- BloomFilter bf = new BloomFilter(-1, -1);
- }
-
-
- @Test
- public void testBloomNumBits() {
- assertEquals(0, BloomFilter.optimalNumOfBits(0, 0));
- assertEquals(0, BloomFilter.optimalNumOfBits(0, 1));
- assertEquals(0, BloomFilter.optimalNumOfBits(1, 1));
- assertEquals(7, BloomFilter.optimalNumOfBits(1, 0.03));
- assertEquals(72, BloomFilter.optimalNumOfBits(10, 0.03));
- assertEquals(729, BloomFilter.optimalNumOfBits(100, 0.03));
- assertEquals(7298, BloomFilter.optimalNumOfBits(1000, 0.03));
- assertEquals(72984, BloomFilter.optimalNumOfBits(10000, 0.03));
- assertEquals(729844, BloomFilter.optimalNumOfBits(100000, 0.03));
- assertEquals(7298440, BloomFilter.optimalNumOfBits(1000000, 0.03));
- assertEquals(6235224, BloomFilter.optimalNumOfBits(1000000, 0.05));
- assertEquals(1870567268, BloomFilter.optimalNumOfBits(300000000, 0.05));
- assertEquals(1437758756, BloomFilter.optimalNumOfBits(300000000, 0.1));
- assertEquals(432808512, BloomFilter.optimalNumOfBits(300000000, 0.5));
- assertEquals(1393332198, BloomFilter.optimalNumOfBits(3000000000L, 0.8));
- assertEquals(657882327, BloomFilter.optimalNumOfBits(3000000000L, 0.9));
- assertEquals(0, BloomFilter.optimalNumOfBits(3000000000L, 1));
- }
-
- @Test
- public void testBloomNumHashFunctions() {
- assertEquals(1, BloomFilter.optimalNumOfHashFunctions(-1, -1));
- assertEquals(1, BloomFilter.optimalNumOfHashFunctions(0, 0));
- assertEquals(1, BloomFilter.optimalNumOfHashFunctions(10, 0));
- assertEquals(1, BloomFilter.optimalNumOfHashFunctions(10, 10));
- assertEquals(7, BloomFilter.optimalNumOfHashFunctions(10, 100));
- assertEquals(1, BloomFilter.optimalNumOfHashFunctions(100, 100));
- assertEquals(1, BloomFilter.optimalNumOfHashFunctions(1000, 100));
- assertEquals(1, BloomFilter.optimalNumOfHashFunctions(10000, 100));
- assertEquals(1, BloomFilter.optimalNumOfHashFunctions(100000, 100));
- assertEquals(1, BloomFilter.optimalNumOfHashFunctions(1000000, 100));
- }
-
- @Test
- public void testBloomFilterBytes() {
- BloomFilter bf = new BloomFilter(10000);
- byte[] val = new byte[]{1, 2, 3};
- byte[] val1 = new byte[]{1, 2, 3, 4};
- byte[] val2 = new byte[]{1, 2, 3, 4, 5};
- byte[] val3 = new byte[]{1, 2, 3, 4, 5, 6};
-
- assertEquals(false, bf.test(val));
- assertEquals(false, bf.test(val1));
- assertEquals(false, bf.test(val2));
- assertEquals(false, bf.test(val3));
- bf.add(val);
- assertEquals(true, bf.test(val));
- assertEquals(false, bf.test(val1));
- assertEquals(false, bf.test(val2));
- assertEquals(false, bf.test(val3));
- bf.add(val1);
- assertEquals(true, bf.test(val));
- assertEquals(true, bf.test(val1));
- assertEquals(false, bf.test(val2));
- assertEquals(false, bf.test(val3));
- bf.add(val2);
- assertEquals(true, bf.test(val));
- assertEquals(true, bf.test(val1));
- assertEquals(true, bf.test(val2));
- assertEquals(false, bf.test(val3));
- bf.add(val3);
- assertEquals(true, bf.test(val));
- assertEquals(true, bf.test(val1));
- assertEquals(true, bf.test(val2));
- assertEquals(true, bf.test(val3));
-
- byte[] randVal = new byte[COUNT];
- for (int i = 0; i < COUNT; i++) {
- rand.nextBytes(randVal);
- bf.add(randVal);
- }
- // last value should be present
- assertEquals(true, bf.test(randVal));
- // most likely this value should not exist
- randVal[0] = 0;
- randVal[1] = 0;
- randVal[2] = 0;
- randVal[3] = 0;
- randVal[4] = 0;
- assertEquals(false, bf.test(randVal));
-
- assertEquals(7800, bf.sizeInBytes());
- }
-
- @Test
- public void testBloomFilterByte() {
- BloomFilter bf = new BloomFilter(10000);
- byte val = Byte.MIN_VALUE;
- byte val1 = 1;
- byte val2 = 2;
- byte val3 = Byte.MAX_VALUE;
-
- assertEquals(false, bf.testLong(val));
- assertEquals(false, bf.testLong(val1));
- assertEquals(false, bf.testLong(val2));
- assertEquals(false, bf.testLong(val3));
- bf.addLong(val);
- assertEquals(true, bf.testLong(val));
- assertEquals(false, bf.testLong(val1));
- assertEquals(false, bf.testLong(val2));
- assertEquals(false, bf.testLong(val3));
- bf.addLong(val1);
- assertEquals(true, bf.testLong(val));
- assertEquals(true, bf.testLong(val1));
- assertEquals(false, bf.testLong(val2));
- assertEquals(false, bf.testLong(val3));
- bf.addLong(val2);
- assertEquals(true, bf.testLong(val));
- assertEquals(true, bf.testLong(val1));
- assertEquals(true, bf.testLong(val2));
- assertEquals(false, bf.testLong(val3));
- bf.addLong(val3);
- assertEquals(true, bf.testLong(val));
- assertEquals(true, bf.testLong(val1));
- assertEquals(true, bf.testLong(val2));
- assertEquals(true, bf.testLong(val3));
-
- byte randVal = 0;
- for (int i = 0; i < COUNT; i++) {
- randVal = (byte) rand.nextInt(Byte.MAX_VALUE);
- bf.addLong(randVal);
- }
- // last value should be present
- assertEquals(true, bf.testLong(randVal));
- // most likely this value should not exist
- assertEquals(false, bf.testLong((byte) -120));
-
- assertEquals(7800, bf.sizeInBytes());
- }
-
- @Test
- public void testBloomFilterInt() {
- BloomFilter bf = new BloomFilter(10000);
- int val = Integer.MIN_VALUE;
- int val1 = 1;
- int val2 = 2;
- int val3 = Integer.MAX_VALUE;
-
- assertEquals(false, bf.testLong(val));
- assertEquals(false, bf.testLong(val1));
- assertEquals(false, bf.testLong(val2));
- assertEquals(false, bf.testLong(val3));
- bf.addLong(val);
- assertEquals(true, bf.testLong(val));
- assertEquals(false, bf.testLong(val1));
- assertEquals(false, bf.testLong(val2));
- assertEquals(false, bf.testLong(val3));
- bf.addLong(val1);
- assertEquals(true, bf.testLong(val));
- assertEquals(true, bf.testLong(val1));
- assertEquals(false, bf.testLong(val2));
- assertEquals(false, bf.testLong(val3));
- bf.addLong(val2);
- assertEquals(true, bf.testLong(val));
- assertEquals(true, bf.testLong(val1));
- assertEquals(true, bf.testLong(val2));
- assertEquals(false, bf.testLong(val3));
- bf.addLong(val3);
- assertEquals(true, bf.testLong(val));
- assertEquals(true, bf.testLong(val1));
- assertEquals(true, bf.testLong(val2));
- assertEquals(true, bf.testLong(val3));
-
- int randVal = 0;
- for (int i = 0; i < COUNT; i++) {
- randVal = rand.nextInt();
- bf.addLong(randVal);
- }
- // last value should be present
- assertEquals(true, bf.testLong(randVal));
- // most likely this value should not exist
- assertEquals(false, bf.testLong(-120));
-
- assertEquals(7800, bf.sizeInBytes());
- }
-
- @Test
- public void testBloomFilterLong() {
- BloomFilter bf = new BloomFilter(10000);
- long val = Long.MIN_VALUE;
- long val1 = 1;
- long val2 = 2;
- long val3 = Long.MAX_VALUE;
-
- assertEquals(false, bf.testLong(val));
- assertEquals(false, bf.testLong(val1));
- assertEquals(false, bf.testLong(val2));
- assertEquals(false, bf.testLong(val3));
- bf.addLong(val);
- assertEquals(true, bf.testLong(val));
- assertEquals(false, bf.testLong(val1));
- assertEquals(false, bf.testLong(val2));
- assertEquals(false, bf.testLong(val3));
- bf.addLong(val1);
- assertEquals(true, bf.testLong(val));
- assertEquals(true, bf.testLong(val1));
- assertEquals(false, bf.testLong(val2));
- assertEquals(false, bf.testLong(val3));
- bf.addLong(val2);
- assertEquals(true, bf.testLong(val));
- assertEquals(true, bf.testLong(val1));
- assertEquals(true, bf.testLong(val2));
- assertEquals(false, bf.testLong(val3));
- bf.addLong(val3);
- assertEquals(true, bf.testLong(val));
- assertEquals(true, bf.testLong(val1));
- assertEquals(true, bf.testLong(val2));
- assertEquals(true, bf.testLong(val3));
-
- long randVal = 0;
- for (int i = 0; i < COUNT; i++) {
- randVal = rand.nextLong();
- bf.addLong(randVal);
- }
- // last value should be present
- assertEquals(true, bf.testLong(randVal));
- // most likely this value should not exist
- assertEquals(false, bf.testLong(-120));
-
- assertEquals(7800, bf.sizeInBytes());
- }
-
- @Test
- public void testBloomFilterFloat() {
- BloomFilter bf = new BloomFilter(10000);
- float val = Float.MIN_VALUE;
- float val1 = 1.1f;
- float val2 = 2.2f;
- float val3 = Float.MAX_VALUE;
-
- assertEquals(false, bf.testDouble(val));
- assertEquals(false, bf.testDouble(val1));
- assertEquals(false, bf.testDouble(val2));
- assertEquals(false, bf.testDouble(val3));
- bf.addDouble(val);
- assertEquals(true, bf.testDouble(val));
- assertEquals(false, bf.testDouble(val1));
- assertEquals(false, bf.testDouble(val2));
- assertEquals(false, bf.testDouble(val3));
- bf.addDouble(val1);
- assertEquals(true, bf.testDouble(val));
- assertEquals(true, bf.testDouble(val1));
- assertEquals(false, bf.testDouble(val2));
- assertEquals(false, bf.testDouble(val3));
- bf.addDouble(val2);
- assertEquals(true, bf.testDouble(val));
- assertEquals(true, bf.testDouble(val1));
- assertEquals(true, bf.testDouble(val2));
- assertEquals(false, bf.testDouble(val3));
- bf.addDouble(val3);
- assertEquals(true, bf.testDouble(val));
- assertEquals(true, bf.testDouble(val1));
- assertEquals(true, bf.testDouble(val2));
- assertEquals(true, bf.testDouble(val3));
-
- float randVal = 0;
- for (int i = 0; i < COUNT; i++) {
- randVal = rand.nextFloat();
- bf.addDouble(randVal);
- }
- // last value should be present
- assertEquals(true, bf.testDouble(randVal));
- // most likely this value should not exist
- assertEquals(false, bf.testDouble(-120.2f));
-
- assertEquals(7800, bf.sizeInBytes());
- }
-
- @Test
- public void testBloomFilterDouble() {
- BloomFilter bf = new BloomFilter(10000);
- double val = Double.MIN_VALUE;
- double val1 = 1.1d;
- double val2 = 2.2d;
- double val3 = Double.MAX_VALUE;
-
- assertEquals(false, bf.testDouble(val));
- assertEquals(false, bf.testDouble(val1));
- assertEquals(false, bf.testDouble(val2));
- assertEquals(false, bf.testDouble(val3));
- bf.addDouble(val);
- assertEquals(true, bf.testDouble(val));
- assertEquals(false, bf.testDouble(val1));
- assertEquals(false, bf.testDouble(val2));
- assertEquals(false, bf.testDouble(val3));
- bf.addDouble(val1);
- assertEquals(true, bf.testDouble(val));
- assertEquals(true, bf.testDouble(val1));
- assertEquals(false, bf.testDouble(val2));
- assertEquals(false, bf.testDouble(val3));
- bf.addDouble(val2);
- assertEquals(true, bf.testDouble(val));
- assertEquals(true, bf.testDouble(val1));
- assertEquals(true, bf.testDouble(val2));
- assertEquals(false, bf.testDouble(val3));
- bf.addDouble(val3);
- assertEquals(true, bf.testDouble(val));
- assertEquals(true, bf.testDouble(val1));
- assertEquals(true, bf.testDouble(val2));
- assertEquals(true, bf.testDouble(val3));
-
- double randVal = 0;
- for (int i = 0; i < COUNT; i++) {
- randVal = rand.nextDouble();
- bf.addDouble(randVal);
- }
- // last value should be present
- assertEquals(true, bf.testDouble(randVal));
- // most likely this value should not exist
- assertEquals(false, bf.testDouble(-120.2d));
-
- assertEquals(7800, bf.sizeInBytes());
- }
-
- @Test
- public void testBloomFilterString() {
- BloomFilter bf = new BloomFilter(100000);
- String val = "bloo";
- String val1 = "bloom fil";
- String val2 = "bloom filter";
- String val3 = "cuckoo filter";
-
- assertEquals(false, bf.testString(val));
- assertEquals(false, bf.testString(val1));
- assertEquals(false, bf.testString(val2));
- assertEquals(false, bf.testString(val3));
- bf.addString(val);
- assertEquals(true, bf.testString(val));
- assertEquals(false, bf.testString(val1));
- assertEquals(false, bf.testString(val2));
- assertEquals(false, bf.testString(val3));
- bf.addString(val1);
- assertEquals(true, bf.testString(val));
- assertEquals(true, bf.testString(val1));
- assertEquals(false, bf.testString(val2));
- assertEquals(false, bf.testString(val3));
- bf.addString(val2);
- assertEquals(true, bf.testString(val));
- assertEquals(true, bf.testString(val1));
- assertEquals(true, bf.testString(val2));
- assertEquals(false, bf.testString(val3));
- bf.addString(val3);
- assertEquals(true, bf.testString(val));
- assertEquals(true, bf.testString(val1));
- assertEquals(true, bf.testString(val2));
- assertEquals(true, bf.testString(val3));
-
- long randVal = 0;
- for (int i = 0; i < COUNT; i++) {
- randVal = rand.nextLong();
- bf.addString(Long.toString(randVal));
- }
- // last value should be present
- assertEquals(true, bf.testString(Long.toString(randVal)));
- // most likely this value should not exist
- assertEquals(false, bf.testString(Long.toString(-120)));
-
- assertEquals(77944, bf.sizeInBytes());
- }
-
- @Test
- public void testMerge() {
- BloomFilter bf = new BloomFilter(10000);
- String val = "bloo";
- String val1 = "bloom fil";
- String val2 = "bloom filter";
- String val3 = "cuckoo filter";
- bf.addString(val);
- bf.addString(val1);
- bf.addString(val2);
- bf.addString(val3);
-
- BloomFilter bf2 = new BloomFilter(10000);
- String v = "2_bloo";
- String v1 = "2_bloom fil";
- String v2 = "2_bloom filter";
- String v3 = "2_cuckoo filter";
- bf2.addString(v);
- bf2.addString(v1);
- bf2.addString(v2);
- bf2.addString(v3);
-
- assertEquals(true, bf.testString(val));
- assertEquals(true, bf.testString(val1));
- assertEquals(true, bf.testString(val2));
- assertEquals(true, bf.testString(val3));
- assertEquals(false, bf.testString(v));
- assertEquals(false, bf.testString(v1));
- assertEquals(false, bf.testString(v2));
- assertEquals(false, bf.testString(v3));
-
- bf.merge(bf2);
-
- assertEquals(true, bf.testString(val));
- assertEquals(true, bf.testString(val1));
- assertEquals(true, bf.testString(val2));
- assertEquals(true, bf.testString(val3));
- assertEquals(true, bf.testString(v));
- assertEquals(true, bf.testString(v1));
- assertEquals(true, bf.testString(v2));
- assertEquals(true, bf.testString(v3));
- }
-
- @Test
- public void testSerialize() throws Exception {
- BloomFilter bf1 = new BloomFilter(10000);
- String[] inputs = {
- "bloo",
- "bloom fil",
- "bloom filter",
- "cuckoo filter",
- };
-
- for (String val : inputs) {
- bf1.addString(val);
- }
-
- // Serialize/deserialize
- ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
- BloomFilter.serialize(bytesOut, bf1);
- ByteArrayInputStream bytesIn = new ByteArrayInputStream(bytesOut.toByteArray());
- BloomFilter bf2 = BloomFilter.deserialize(bytesIn);
-
- for (String val : inputs) {
- assertEquals("Testing bf1 with " + val, true, bf1.testString(val));
- assertEquals("Testing bf2 with " + val, true, bf2.testString(val));
- }
- }
-
- @Test
- public void testMergeBloomFilterBytes() throws Exception {
- BloomFilter bf1 = new BloomFilter(10000);
- BloomFilter bf2 = new BloomFilter(10000);
-
- String[] inputs1 = {
- "bloo",
- "bloom fil",
- "bloom filter",
- "cuckoo filter",
- };
-
- String[] inputs2 = {
- "2_bloo",
- "2_bloom fil",
- "2_bloom filter",
- "2_cuckoo filter",
- };
-
- for (String val : inputs1) {
- bf1.addString(val);
- }
- for (String val : inputs2) {
- bf2.addString(val);
- }
-
- ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
- BloomFilter.serialize(bytesOut, bf1);
- byte[] bf1Bytes = bytesOut.toByteArray();
- bytesOut.reset();
- BloomFilter.serialize(bytesOut, bf1);
- byte[] bf2Bytes = bytesOut.toByteArray();
-
- // Merge bytes
- BloomFilter.mergeBloomFilterBytes(
- bf1Bytes, 0, bf1Bytes.length,
- bf2Bytes, 0, bf2Bytes.length);
-
- // Deserialize and test
- ByteArrayInputStream bytesIn = new ByteArrayInputStream(bf1Bytes, 0, bf1Bytes.length);
- BloomFilter bfMerged = BloomFilter.deserialize(bytesIn);
- // All values should pass test
- for (String val : inputs1) {
- bfMerged.addString(val);
- }
- for (String val : inputs2) {
- bfMerged.addString(val);
- }
- }
-
- @Test
- public void testMergeBloomFilterBytesFailureCases() throws Exception {
- BloomFilter bf1 = new BloomFilter(1000);
- BloomFilter bf2 = new BloomFilter(200);
- // Create bloom filter with same number of bits, but different # hash functions
- ArrayList<Long> bits = new ArrayList<Long>();
- for (int idx = 0; idx < bf1.getBitSet().length; ++idx) {
- bits.add(0L);
- }
- BloomFilter bf3 = new BloomFilter(bits, bf1.getBitSize(), bf1.getNumHashFunctions() + 1);
-
- // Serialize to bytes
- ByteArrayOutputStream bytesOut = new ByteArrayOutputStream();
- BloomFilter.serialize(bytesOut, bf1);
- byte[] bf1Bytes = bytesOut.toByteArray();
-
- bytesOut.reset();
- BloomFilter.serialize(bytesOut, bf2);
- byte[] bf2Bytes = bytesOut.toByteArray();
-
- bytesOut.reset();
- BloomFilter.serialize(bytesOut, bf3);
- byte[] bf3Bytes = bytesOut.toByteArray();
-
- try {
- // this should fail
- BloomFilter.mergeBloomFilterBytes(
- bf1Bytes, 0, bf1Bytes.length,
- bf2Bytes, 0, bf2Bytes.length);
- Assert.fail("Expected exception not encountered");
- } catch (IllegalArgumentException err) {
- // expected
- }
-
- try {
- // this should fail
- BloomFilter.mergeBloomFilterBytes(
- bf1Bytes, 0, bf1Bytes.length,
- bf3Bytes, 0, bf3Bytes.length);
- Assert.fail("Expected exception not encountered");
- } catch (IllegalArgumentException err) {
- // expected
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/orc/pom.xml
----------------------------------------------------------------------
diff --git a/orc/pom.xml b/orc/pom.xml
index e6ec0f4..f75b91c 100644
--- a/orc/pom.xml
+++ b/orc/pom.xml
@@ -35,7 +35,6 @@
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-storage-api</artifactId>
- <version>${project.version}</version>
</dependency>
<!-- inter-project -->
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/packaging/pom.xml
----------------------------------------------------------------------
diff --git a/packaging/pom.xml b/packaging/pom.xml
index 76e0cff..7a53856 100644
--- a/packaging/pom.xml
+++ b/packaging/pom.xml
@@ -246,7 +246,6 @@
<dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-storage-api</artifactId>
- <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.hive</groupId>
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 4158083..505e4c2 100644
--- a/pom.xml
+++ b/pom.xml
@@ -180,6 +180,7 @@
<stax.version>1.0.1</stax.version>
<slf4j.version>1.7.10</slf4j.version>
<ST4.version>4.0.4</ST4.version>
+ <storage-api.version>2.3.0-SNAPSHOT</storage-api.version>
<tez.version>0.8.4</tez.version>
<slider.version>0.90.2-incubating</slider.version>
<super-csv.version>2.2.0</super-csv.version>
@@ -505,6 +506,11 @@
<version>${stax.version}</version>
</dependency>
<dependency>
+ <groupId>org.apache.hive</groupId>
+ <artifactId>hive-storage-api</artifactId>
+ <version>${storage-api.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.pig</groupId>
<artifactId>pig</artifactId>
<version>${pig.version}</version>
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/storage-api/pom.xml
----------------------------------------------------------------------
diff --git a/storage-api/pom.xml b/storage-api/pom.xml
index 4095ed2..036a994 100644
--- a/storage-api/pom.xml
+++ b/storage-api/pom.xml
@@ -17,22 +17,33 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
- <groupId>org.apache.hive</groupId>
- <artifactId>hive</artifactId>
- <version>2.2.0-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
+ <groupId>org.apache</groupId>
+ <artifactId>apache</artifactId>
+ <version>18</version>
+ <relativePath></relativePath>
</parent>
+ <groupId>org.apache.hive</groupId>
<artifactId>hive-storage-api</artifactId>
+ <version>2.3.0-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Hive Storage API</name>
<properties>
- <hive.path.to.root>..</hive.path.to.root>
+ <commons-lang.version>2.6</commons-lang.version>
+ <guava.version>14.0.1</guava.version>
+ <hadoop.version>2.7.2</hadoop.version>
+ <junit.version>4.11</junit.version>
+ <slf4j.version>1.7.10</slf4j.version>
</properties>
<dependencies>
- <!-- test inter-project -->
+ <!-- compile inter-project -->
+ <dependency>
+ <groupId>commons-lang</groupId>
+ <artifactId>commons-lang</artifactId>
+ <version>${commons-lang.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
@@ -40,7 +51,15 @@
<optional>true</optional>
<exclusions>
<exclusion>
- <groupId>commmons-logging</groupId>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
<exclusion>
@@ -48,9 +67,33 @@
<artifactId>servlet-api</artifactId>
</exclusion>
<exclusion>
+ <groupId>javax.servlet.jsp</groupId>
+ <artifactId>jsp-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ </exclusion>
+ <exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -69,5 +112,39 @@
<directory>${basedir}/src/test/resources</directory>
</testResource>
</testResources>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.1</version>
+ <configuration>
+ <source>1.7</source>
+ <target>1.7</target>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-javadoc-plugin</artifactId>
+ <version>2.10.4</version>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-project-info-reports-plugin</artifactId>
+ <version>2.9</version>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-surefire-plugin</artifactId>
+ <version>2.19.1</version>
+ <configuration>
+ <reuseForks>false</reuseForks>
+ <argLine>-Xmx2048m</argLine>
+ <failIfNoTests>false</failIfNoTests>
+ <systemPropertyVariables>
+ <test.tmp.dir>${project.build.directory}/tmp</test.tmp.dir>
+ </systemPropertyVariables>
+ </configuration>
+ </plugin>
+ </plugins>
</build>
</project>
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/storage-api/src/java/org/apache/hadoop/hive/common/Pool.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/Pool.java b/storage-api/src/java/org/apache/hadoop/hive/common/Pool.java
index 272bbdd..5ab4ee7 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/common/Pool.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/Pool.java
@@ -21,9 +21,13 @@ package org.apache.hadoop.hive.common;
public interface Pool<T> {
/** Object helper for objects stored in the pool. */
public interface PoolObjectHelper<T> {
- /** Called to create an object when one cannot be provided. */
+ /** Called to create an object when one cannot be provided.
+ * @return a newly allocated object
+ */
T create();
- /** Called before the object is put in the pool (regardless of whether put succeeds). */
+ /** Called before the object is put in the pool (regardless of whether put succeeds).
+ * @param t the object to reset
+ */
void resetBeforeOffer(T t);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/storage-api/src/java/org/apache/hadoop/hive/common/io/Allocator.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/io/Allocator.java b/storage-api/src/java/org/apache/hadoop/hive/common/io/Allocator.java
index fd9d9c9..16b9713 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/common/io/Allocator.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/io/Allocator.java
@@ -42,12 +42,22 @@ public interface Allocator {
* Creates an unallocated memory buffer object. This object can be passed to allocateMultiple
* to allocate; this is useful if data structures are created for separate buffers that can
* later be allocated together.
+ * @return a new unallocated memory buffer
*/
MemoryBuffer createUnallocated();
- /** Deallocates a memory buffer. */
+
+ /** Deallocates a memory buffer.
+ * @param buffer the buffer to deallocate
+ */
void deallocate(MemoryBuffer buffer);
- /** Whether the allocator uses direct buffers. */
+
+ /** Whether the allocator uses direct buffers.
+ * @return are they direct buffers?
+ */
boolean isDirectAlloc();
- /** Maximum allocation size supported by this allocator. */
+
+ /** Maximum allocation size supported by this allocator.
+ * @return a number of bytes
+ */
int getMaxAllocation();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/storage-api/src/java/org/apache/hadoop/hive/common/io/DataCache.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/io/DataCache.java b/storage-api/src/java/org/apache/hadoop/hive/common/io/DataCache.java
index 1273588..e53b737 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/common/io/DataCache.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/io/DataCache.java
@@ -50,7 +50,7 @@ public interface DataCache {
* return both ranges). It should really be same as #1, however currently ORC uses estimated
* end offsets; if we don't return the end block, the caller may read it from disk needlessly.
*
- * @param fileId Unique ID of the target file on the file system.
+ * @param fileKey Unique ID of the target file on the file system.
* @param range A set of DiskRange-s (linked list) that is to be retrieved. May be modified.
* @param baseOffset base offset for the ranges (stripe/stream offset in case of ORC).
* @param factory A factory to produce DiskRangeList-s out of cached MemoryBuffer-s.
@@ -72,7 +72,7 @@ public interface DataCache {
* caller is done with it. Buffers rejected due to conflict will neither be locked, nor
* automatically deallocated. The caller must take care to discard these buffers.
*
- * @param fileId Unique ID of the target file on the file system.
+ * @param fileKey Unique ID of the target file on the file system.
* @param ranges The ranges for which the data is being cached. These objects will not be stored.
* @param data The data for the corresponding ranges.
* @param baseOffset base offset for the ranges (stripe/stream offset in case of ORC).
@@ -84,17 +84,20 @@ public interface DataCache {
/**
* Releases the buffer returned by getFileData/provided to putFileData back to cache.
* See respective javadocs for details.
+ * @param buffer the buffer to release
*/
void releaseBuffer(MemoryBuffer buffer);
/**
* Notifies the cache that the buffer returned from getFileData/provided to putFileData will
* be used by another consumer and therefore released multiple times (one more time per call).
+ * @param buffer the buffer to reuse
*/
void reuseBuffer(MemoryBuffer buffer);
/**
* Gets the allocator associated with this DataCache.
+ * @return the allocator
*/
Allocator getAllocator();
}
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRangeList.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRangeList.java b/storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRangeList.java
index 6a89d0a..0aba08d 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRangeList.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/io/DiskRangeList.java
@@ -31,7 +31,10 @@ public class DiskRangeList extends DiskRange {
super(offset, end);
}
- /** Replaces this element with another in the list; returns the new element. */
+ /** Replaces this element with another in the list; returns the new element.
+ * @param other the disk range to swap into this list
+ * @return the new element
+ */
public DiskRangeList replaceSelfWith(DiskRangeList other) {
other.prev = this.prev;
other.next = this.next;
@@ -47,7 +50,8 @@ public class DiskRangeList extends DiskRange {
/**
* Inserts an intersecting range before current in the list and adjusts offset accordingly.
- * @returns the new element.
+ * @param other the element to insert
+ * @return the new element.
*/
public DiskRangeList insertPartBefore(DiskRangeList other) {
assert other.end >= this.offset;
@@ -63,7 +67,8 @@ public class DiskRangeList extends DiskRange {
/**
* Inserts an element after current in the list.
- * @returns the new element.
+ * @param other the new element to insert
+ * @return the new element.
* */
public DiskRangeList insertAfter(DiskRangeList other) {
other.next = this.next;
@@ -77,7 +82,8 @@ public class DiskRangeList extends DiskRange {
/**
* Inserts an intersecting range after current in the list and adjusts offset accordingly.
- * @returns the new element.
+ * @param other the new element to insert
+ * @return the new element.
*/
public DiskRangeList insertPartAfter(DiskRangeList other) {
assert other.offset <= this.end;
@@ -106,7 +112,10 @@ public class DiskRangeList extends DiskRange {
this.next = this.prev = null;
}
- /** Splits current element in the list, using DiskRange::slice */
+ /** Splits current element in the list, using DiskRange::slice.
+ * @param cOffset the position to split the list
+ * @return the split list
+ */
public final DiskRangeList split(long cOffset) {
insertAfter((DiskRangeList)this.sliceAndShift(cOffset, end, 0));
return replaceSelfWith((DiskRangeList)this.sliceAndShift(offset, cOffset, 0));
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/MemoryBuffer.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/MemoryBuffer.java b/storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/MemoryBuffer.java
index 4475009..201c5e2 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/MemoryBuffer.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/io/encoded/MemoryBuffer.java
@@ -22,7 +22,11 @@ import java.nio.ByteBuffer;
/** Abstract interface for any class wrapping a ByteBuffer. */
public interface MemoryBuffer {
- /** Note - raw buffer should not be modified. */
+ /**
+ * Get the raw byte buffer that backs this buffer.
+ * Note - raw buffer should not be modified.
+ * @return the shared byte buffer
+ */
public ByteBuffer getByteBufferRaw();
public ByteBuffer getByteBufferDup();
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimal.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimal.java b/storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimal.java
index a8c183a..7fa9fdf 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimal.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimal.java
@@ -235,7 +235,7 @@ public class FastHiveDecimal {
protected void fastFractionPortion() {
FastHiveDecimalImpl.fastFractionPortion(
- fastSignum, fast0, fast1, fast2, fastIntegerDigitCount, fastScale,
+ fastSignum, fast0, fast1, fast2, fastScale,
this);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimalImpl.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimalImpl.java b/storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimalImpl.java
index a4fed5d..7a565dd 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimalImpl.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/type/FastHiveDecimalImpl.java
@@ -26,8 +26,6 @@ import java.math.BigDecimal;
import java.math.BigInteger;
import java.math.RoundingMode;
-import org.apache.commons.lang.StringUtils;
-
/**
* This class is a companion to the FastHiveDecimal class that separates the essential of code
* out of FastHiveDecimal into static methods in this class so that they can be used directly
@@ -254,8 +252,12 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* NOTE: The fastSetFromBytes method requires the caller to pass a fastResult parameter has been
* reset for better performance.
*
+ * @param bytes the bytes to copy from
+ * @param offset the starting location in bytes
+ * @param length the number of bytes to use from bytes
+ * @param trimBlanks should spaces be trimmed?
* @param fastResult True if the byte array slice was successfully converted to a decimal.
- * @return
+ * @return Was a valid number found?
*/
public static boolean fastSetFromBytes(byte[] bytes, int offset, int length, boolean trimBlanks,
FastHiveDecimal fastResult) {
@@ -681,6 +683,12 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* NOTE: The fastSetFromDigitsOnlyBytesAndScale method requires the caller to pass a fastResult
* parameter has been reset for better performance.
*
+ * @param isNegative is the number negative
+ * @param bytes the bytes to read from
+ * @param offset the position to start at
+ * @param length the number of bytes to read
+ * @param scale the scale of the number
+ * @param fastResult an object it into
* @return True if the sign, digits, and scale were successfully converted to a decimal.
*/
public static boolean fastSetFromDigitsOnlyBytesAndScale(
@@ -826,6 +834,9 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* NOTE: The fastSetFromBigDecimal method requires the caller to pass a fastResult
* parameter has been reset for better performance.
*
+ * @param bigDecimal the big decimal to copy
+ * @param allowRounding is rounding allowed?
+ * @param fastResult an object to reuse
* @return True if the BigDecimal could be converted to our decimal representation.
*/
public static boolean fastSetFromBigDecimal(
@@ -891,6 +902,9 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* NOTE: The fastSetFromString method requires the caller to pass a fastResult
* parameter has been reset for better performance.
*
+ * @param string the string to parse
+ * @param trimBlanks should the blanks be trimmed
+ * @param result an object to reuse
* @return True if the String was successfully converted to a decimal.
*/
public static boolean fastSetFromString(
@@ -904,7 +918,8 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
*
* NOTE: The fastSetFromString method requires the caller to pass a fastResult
* parameter has been reset for better performance.
- *
+ * @param intValue the value to set
+ * @param fastResult an object to reuse
*/
public static void fastSetFromInt(int intValue, FastHiveDecimal fastResult) {
if (intValue == 0) {
@@ -930,6 +945,8 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* NOTE: The fastSetFromLong method requires the caller to pass a fastResult
* parameter has been reset for better performance.
*
+ * @param longValue the value to set
+ * @param fastResult an object to reuse
*/
public static void fastSetFromLong(
long longValue, FastHiveDecimal fastResult) {
@@ -971,6 +988,10 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* NOTE: The fastSetFromLongAndScale method requires the caller to pass a fastResult
* parameter has been reset for better performance.
*
+ * @param longValue the value to set as a long
+ * @param scale the scale to use
+ * @param fastResult an object to reuse
+ * @return was the conversion successful?
*/
public static boolean fastSetFromLongAndScale(
long longValue, int scale, FastHiveDecimal fastResult) {
@@ -999,6 +1020,9 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* NOTE: The fastSetFromFloat method requires the caller to pass a fastResult
* parameter has been reset for better performance.
*
+ * @param floatValue the value to set
+ * @param fastResult an object to reuse
+ * @return was the conversion successful?
*/
public static boolean fastSetFromFloat(
float floatValue, FastHiveDecimal fastResult) {
@@ -1014,6 +1038,9 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* NOTE: The fastSetFromDouble method requires the caller to pass a fastResult
* parameter has been reset for better performance.
*
+ * @param doubleValue the value to set
+ * @param fastResult an object to reuse
+ * @return was the conversion successful?
*/
public static boolean fastSetFromDouble(
double doubleValue, FastHiveDecimal fastResult) {
@@ -1032,6 +1059,8 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* NOTE: The fastSetFromBigInteger method requires the caller to pass a fastResult
* parameter has been reset for better performance.
*
+ * @param bigInteger the value to set
+ * @param fastResult an object to reuse
* @return Return true if the BigInteger value fit within HiveDecimal.MAX_PRECISION. Otherwise,
* false for overflow.
*/
@@ -1101,6 +1130,9 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* NOTE: The fastSetFromBigInteger method requires the caller to pass a fastResult
* parameter has been reset for better performance.
*
+ * @param bigInteger the value to set as an integer
+ * @param scale the scale to use
+ * @param fastResult an object to reuse
* @return True if the BigInteger and scale were successfully converted to a decimal.
*/
public static boolean fastSetFromBigInteger(
@@ -1224,10 +1256,17 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
*
* NOTE: The fastFractionPortion method requires the caller to pass a fastResult
* parameter has been reset for better performance.
+ *
+ * @param fastSignum the sign of the number (1, 0, or -1)
+ * @param fast0 high bits
+ * @param fast1 second word bits
+ * @param fast2 third word bits
+ * @param fastScale the scale
+ * @param fastResult an object to reuse
*/
public static void fastFractionPortion(
int fastSignum, long fast0, long fast1, long fast2,
- int fastIntegerDigitCount, int fastScale,
+ int fastScale,
FastHiveDecimal fastResult) {
if (fastSignum == 0 || fastScale == 0) {
@@ -1288,6 +1327,14 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
*
* NOTE: The fastFractionPortion method requires the caller to pass a fastResult
* parameter has been reset for better performance.
+ *
+ * @param fastSignum the sign of the number (1, 0, or -1)
+ * @param fast0 high bits
+ * @param fast1 second word bits
+ * @param fast2 third word bits
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale
+ * @param fastResult an object to reuse
*/
public static void fastIntegerPortion(
int fastSignum, long fast0, long fast1, long fast2,
@@ -1326,9 +1373,12 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* We let L and M be different to support the SerializationUtil serialization where the lower
* word is 62 bits and the remaining words are 63 bits...
*
- * The fast decimal middleWordMultiplier is 2^L.
- * The fast decimal highWordMultiplier is 2^(M+L).
- *
+ * @param lowerWord the lower internal representation
+ * @param middleWord the middle internal representation
+ * @param highWord the high internal representation
+ * @param middleWordMultiplier 2^L
+ * @param highWordMultiplier 2^(M+L)
+ * @param fastResult an object to reuse
* @return True if the conversion of the 3 binary words to decimal was successful.
*/
public static boolean doBinaryToDecimalConversion(
@@ -1505,22 +1555,21 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
*
* We produce 1 binary word (remainder) and a decimal quotient for the higher portion.
*
- * So, the parameters are:
- *
- * The input decimal (dividendFast0, dividendFast1, and dividendFast2) that will produce a
- * single binary word remainder and decimal quotient.
- *
- * The fast decimal inverse of 2^N = 2^-N (fastInverseConst).
- *
- * Where in the inverse multiplication result (quotientIntegerWordNum and
- * quotientIntegerDigitNum) to find the quotient integer decimal portion.
- *
- * The fast decimal multiplier for converting the quotient integer to the larger number to
- * subtract from the input decimal to get the remainder.
- *
- * And, the scratch longs where to store the result remainder word (index 3) and result quotient
- * decimal longwords (indices 0 .. 2).
- *
+ * @param dividendFast0 The input decimal that will produce a
+ * single binary word remainder and decimal quotient.
+ * @param dividendFast1 second word
+ * @param dividendFast2 third word
+ * @param fastInverseConst the fast decimal inverse of 2^N = 2^-N
+ * @param quotientIntegerWordNum the word in the inverse multiplication result
+ * to find the quotient integer decimal portion
+ * @param quotientIntegerDigitNum the digit in the result to find the quotient
+ * integer decimal portion
+ * @param fastMultiplierConst The fast decimal multiplier for converting the
+ * quotient integer to the larger number to
+ * subtract from the input decimal to get the
+ * remainder.
+ * @param scratchLongs where to store the result remainder word (index 3) and
+ * result quotient decimal longwords (indices 0 .. 2)
* @return True if the results were produced without overflow.
*/
public static boolean doDecimalToBinaryDivisionRemainder(
@@ -1769,11 +1818,18 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* performance. Pass a FAST_SCRATCH_BUFFER_LEN_SERIALIZATION_UTILS_READ byte array for
* scratchBytes.
*
+ * @param inputStream the stream to read from
+ * @param scale the scale of the number
+ * @param scratchBytes An array for the binary to decimal conversion for better
+ * performance. Must have length of
+ * FAST_SCRATCH_BUFFER_LEN_SERIALIZATION_UTILS_READ.
+ * @param fastResult an object to reuse
* @return The deserialized decimal or null if the conversion failed.
+ * @throws IOException failures in reading the stream
*/
public static boolean fastSerializationUtilsRead(InputStream inputStream, int scale,
byte[] scratchBytes,
- FastHiveDecimal fastResult) throws IOException, EOFException {
+ FastHiveDecimal fastResult) throws IOException {
// Following a suggestion from Gopal, quickly read in the bytes from the stream.
// CONSIDER: Have ORC read the whole input stream into a big byte array with one call to
@@ -1909,7 +1965,16 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* BigInteger.bitLength -- we do not emulate that. SerializationUtils.readBigInteger will
* produce the same result for both.
*
+ * @param outputStream the stream to write to
+ * @param fastSignum the sign digit (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1 of the internal representation
+ * @param fast2 word 2 of the internal representation
+ * @param fastIntegerDigitCount unused
+ * @param fastScale unused
+ * @param scratchLongs scratch space
* @return True if the decimal was successfully serialized into the output stream.
+ * @throws IOException for problems in writing
*/
public static boolean fastSerializationUtilsWrite(OutputStream outputStream,
int fastSignum, long fast0, long fast1, long fast2,
@@ -2130,7 +2195,12 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* This method is designed for high performance and does not create an actual BigInteger during
* binary to decimal conversion.
*
- * @return
+ * @param bytes the bytes to read from
+ * @param offset the starting position in the bytes array
+ * @param length the number of bytes to read from the bytes array
+ * @param scale the scale of the number
+ * @param fastResult an object to reused
+ * @return did the conversion succeed?
*/
public static boolean fastSetFromBigIntegerBytesAndScale(
byte[] bytes, int offset, int length, int scale,
@@ -2337,8 +2407,12 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* Scratch objects necessary to do the decimal to binary conversion without actually creating a
* BigInteger object are passed for better performance.
*
- * Allocate scratchLongs with SCRATCH_LONGS_LEN longs.
- * And, allocate buffer with SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES bytes.
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param scratchLongs scratch array of SCRATCH_LONGS_LEN longs
+ * @param buffer scratch array of SCRATCH_BUFFER_LEN_BIG_INTEGER_BYTES bytes
* @return The number of bytes used for the binary result in buffer. Otherwise, 0 if the
* conversion failed.
*/
@@ -2680,10 +2754,16 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
*
* This emulates the OldHiveDecimal setScale / OldHiveDecimal getInternalStorage() behavior.
*
- * @param serializeScale
- * @param scratchLongs
- * @param buffer
- * @return
+ * @param fastSignum the sign number (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale
+ * @param serializeScale the scale to serialize
+ * @param scratchLongs a scratch array of longs
+ * @param buffer the buffer to serialize into
+ * @return the number of bytes used to serialize the number
*/
public static int fastBigIntegerBytesScaled(
final int fastSignum, long fast0, long fast1, long fast2,
@@ -2801,6 +2881,12 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* NOTE: Fractional digits are ignored in the test since fastByteValueClip() will
* remove them (round down).
*
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
* @return True when fastByteValueClip() will return a correct byte.
*/
public static boolean fastIsByte(
@@ -2953,6 +3039,12 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* NOTE: Fractional digits are ignored in the test since fastShortValueClip() will
* remove them (round down).
*
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
* @return True when fastShortValueClip() will return a correct short.
*/
public static boolean fastIsShort(
@@ -3100,6 +3192,12 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* NOTE: Fractional digits are ignored in the test since fastIntValueClip() will
* remove them (round down).
*
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
* @return True when fastIntValueClip() will return a correct int.
*/
public static boolean fastIsInt(
@@ -3247,6 +3345,12 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* NOTE: Fractional digits are ignored in the test since fastLongValueClip() will
* remove them (round down).
*
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
* @return True when fastLongValueClip() will return a correct long.
*/
public static boolean fastIsLong(
@@ -3449,6 +3553,13 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
/**
* Get a BigInteger representing the decimal's digits without a dot.
*
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
+ * @param fastSerializationScale the scale to serialize
* @return Returns a signed BigInteger.
*/
public static BigInteger fastBigIntegerValue(
@@ -3521,7 +3632,13 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* NOTE: We are not representing our decimal as BigDecimal now as OldHiveDecimal did, so this
* is now slower.
*
- * @return
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
+ * @return the BigDecimal equivalent
*/
public static BigDecimal fastBigDecimalValue(
int fastSignum, long fast0, long fast1, long fast2,
@@ -3872,6 +3989,13 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* Used by map join and other Hive internal purposes where performance is important.
*
* IMPORTANT: See comments for fastHashCode(), too.
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
+ * @return the hash code
*/
public static int fastNewFasterHashCode(
int fastSignum, long fast0, long fast1, long fast2, int fastIntegerDigitCount, int fastScale) {
@@ -3894,7 +4018,14 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
*
* NOTE: It is necessary to create a BigDecimal object and use its hash code, so this method is
* slow.
- * @return
+ *
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
+ * @return the hash code
*/
public static int fastHashCode(
int fastSignum, long fast0, long fast1, long fast2, int fastIntegerDigitCount, int fastScale) {
@@ -4773,6 +4904,15 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* ceiling(12400.8302, -2) = 12500 // E.g. Positive case FAST_ROUND_CEILING
* rr rrrr
*
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
+ * @param roundPower the power to round to
+ * @param fastResult an object to reuse
+ * @return was the operation successful
*/
public static boolean fastRoundIntegerUp(
int fastSignum, long fast0, long fast1, long fast2,
@@ -4855,6 +4995,16 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* The fraction being scaled away is thrown away.
*
* The signum will be updated if the result is 0, otherwise the original sign is unchanged.
+ *
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
+ * @param roundPower the power to round to
+ * @param fastResult an object to reuse
+ * @return was the operation successful?
*/
public static boolean fastRoundIntegerDown(
int fastSignum, long fast0, long fast1, long fast2,
@@ -4924,8 +5074,17 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
/**
* Fast decimal scale down by factor of 10 with rounding ROUND_HALF_UP.
*
- * When the fraction being scaled away is >= 0.5, the add 1.
- *
+ * When the fraction being scaled away is >= 0.5, the add 1.
+ *
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
+ * @param roundPower the power to round to
+ * @param fastResult an object to reuse
+ * @return was the operation successful?
*/
public static boolean fastRoundIntegerHalfUp(
int fastSignum, long fast0, long fast1, long fast2,
@@ -5014,10 +5173,20 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* Fast decimal scale down by factor of 10 with rounding ROUND_HALF_EVEN.
*
* When the fraction being scaled away is exactly 0.5, then round and add 1 only if aaa.
- * When fraction is not exactly 0.5, then if fraction > 0.5 then add 1.
+ * When fraction is not exactly 0.5, then if fraction > 0.5 then add 1.
* Otherwise, throw away fraction.
*
* The signum will be updated if the result is 0, otherwise the original sign is unchanged.
+ *
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fastIntegerDigitCount the number of integer digits
+ * @param fastScale the scale of the number
+ * @param roundPower the power to round to
+ * @param fastResult an object to reuse
+ * @return was the operation successful?
*/
public static boolean fastRoundIntegerHalfEven(
int fastSignum, long fast0, long fast1, long fast2,
@@ -5106,7 +5275,16 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
*
* When the fraction being scaled away is non-zero, return false.
*
- * The signum will be updated if the result is 0, otherwise the original sign is unchanged.
+ * The signum will be updated if the result is 0, otherwise the original sign
+ * is unchanged.
+ *
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param scaleDown the digits to scale down by
+ * @param fastResult an object to reuse
+ * @return was the operation successful?
*/
public static boolean fastScaleDownNoRound(
int fastSignum, long fast0, long fast1, long fast2,
@@ -5210,6 +5388,13 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
*
* When the fraction being scaled away is non-zero, the add 1.
*
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param scaleDown the number of integer digits to scale
+ * @param fastResult an object to reuse
+ * @return was the operation successfule?
*/
public static boolean fastRoundFractionalUp(
int fastSignum, long fast0, long fast1, long fast2,
@@ -5272,6 +5457,12 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
*
* The fraction being scaled away is thrown away.
*
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param scaleDown the number of integer digits to scale
+ * @param fastResult an object to reuse
*/
public static void fastRoundFractionalDown(
int fastSignum, long fast0, long fast1, long fast2,
@@ -5305,8 +5496,15 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
/**
* Fast decimal scale down by factor of 10 with rounding ROUND_HALF_UP.
*
- * When the fraction being scaled away is >= 0.5, the add 1.
+ * When the fraction being scaled away is >= 0.5, the add 1.
*
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param scaleDown the number of integer digits to scale
+ * @param fastResult an object to reuse
+ * @return was the operation successfule?
*/
public static boolean fastRoundFractionalHalfUp(
int fastSignum, long fast0, long fast1, long fast2,
@@ -5371,8 +5569,17 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
/**
* Fast decimal scale down by factor of 10 with rounding ROUND_HALF_UP.
*
- * When the fraction being scaled away is >= 0.5, the add 1.
- *
+ * When the fraction being scaled away is >= 0.5, the add 1.
+ *
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param fast3 word 3
+ * @param fast4 word 4
+ * @param scaleDown the number of integer digits to scale
+ * @param fastResult an object to reuse
+ * @return was the operation successfule?
*/
public static boolean fastRoundFractionalHalfUp5Words(
int fastSignum, long fast0, long fast1, long fast2, long fast3, long fast4,
@@ -5562,9 +5769,16 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* Fast decimal scale down by factor of 10 with rounding ROUND_HALF_EVEN.
*
* When the fraction being scaled away is exactly 0.5, then round and add 1 only if aaa.
- * When fraction is not exactly 0.5, then if fraction > 0.5 then add 1.
+ * When fraction is not exactly 0.5, then if fraction > 0.5 then add 1.
* Otherwise, throw away fraction.
*
+ * @param fastSignum the sign (-1, 0, or +1)
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param scaleDown the number of integer digits to scale
+ * @param fastResult an object to reuse
+ * @return was the operation successfule?
*/
public static boolean fastRoundFractionalHalfEven(
int fastSignum, long fast0, long fast1, long fast2,
@@ -5657,6 +5871,12 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* Fast decimal scale down by factor of 10 with NO rounding.
*
* The signum will be updated if the result is 0, otherwise the original sign is unchanged.
+ *
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param scaleDown the number of integer digits to scale
+ * @param fastResult an object to reuse
*/
public static void doFastScaleDown(
long fast0, long fast1, long fast2,
@@ -5733,6 +5953,12 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
/**
* Fast decimal scale up by factor of 10.
+ * @param fast0 word 0 of the internal representation
+ * @param fast1 word 1
+ * @param fast2 word 2
+ * @param scaleUp the number of integer digits to scale up by
+ * @param fastResult an object to reuse
+ * @return was the operation successfule?
*/
public static boolean fastScaleUp(
long fast0, long fast1, long fast2,
@@ -7492,7 +7718,7 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* + 0.0000000000006711991169422033 dec2 (int digits 0, scale 28)
*
* Trying to make dec1 to have a scale of 28 (i.e. by adding trailing zeroes) would exceed
- * MAX_PRECISION (int digits 27 + 28 > 38).
+ * MAX_PRECISION (int digits 27 + 28 > 38).
*
* In this example we need to make sure we have enough integer digit room in the result to
* handle dec1's digits. In order to maintain that, we will need to get rid of lower
@@ -7507,6 +7733,19 @@ public class FastHiveDecimalImpl extends FastHiveDecimal {
* So, the simplest thing is to emulate what OldHiveDecimal does and do the full digit addition
* and then fit the result afterwards.
*
+ * @param leftSignum The left sign (-1, 0, or +1)
+ * @param leftFast0 The left word 0 of reprentation
+ * @param leftFast1 word 1
+ * @param leftFast2 word 2
+ * @param leftIntegerDigitCount The left number of integer digits
+ * @param leftScale the left scale
+ * @param rightSignum The right sign (-1, 0, or +1)
+ * @param rightFast0 The right word 0 of reprentation
+ * @param rightFast1 word 1
+ * @param rightFast2 word 2
+ * @param rightIntegerDigitCount The right number of integer digits
+ * @param rightScale the right scale
+ * @param fastResult an object to reuse
* @return True if the addition was successful; Otherwise, false is returned on overflow.
*/
public static boolean fastAdd(
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java b/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java
index bf954a8..7beb7bf 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/type/HiveDecimal.java
@@ -47,7 +47,7 @@ import java.math.BigInteger;
* DECIMAL(precision, scale)}
* }
* <p>
- * The declared scale must be <= precision.
+ * The declared scale must be <= precision.
* <p>
* Use DECIMAL instead of DOUBLE when exact numeric accuracy is required. Not all decimal numbers
* (radix 10) are exactly representable in the binary (radix 2 based) floating point type DOUBLE and
@@ -185,7 +185,7 @@ public final class HiveDecimal extends FastHiveDecimal implements Comparable<Hiv
* <p>
* The Hive function is ROUND.
* <p>
- * For result, throw away round fraction. If the round fraction is >= 0.5, then add 1 when
+ * For result, throw away round fraction. If the round fraction is >= 0.5, then add 1 when
* positive and subtract 1 when negative. So, the sign is irrelevant.
* <p>
* (Example here rounds at scale 0)
@@ -264,6 +264,8 @@ public final class HiveDecimal extends FastHiveDecimal implements Comparable<Hiv
/**
* Create a HiveDecimal from a FastHiveDecimal object. Used by HiveDecimalWritable.
+ * @param fastDec the value to set
+ * @return new hive decimal
*/
@HiveDecimalVersionV2
public static HiveDecimal createFromFast(FastHiveDecimal fastDec) {
@@ -279,14 +281,14 @@ public final class HiveDecimal extends FastHiveDecimal implements Comparable<Hiv
* 99,999,999,999,999,999,999,999,999,999,999,999,999 or 10^38 - 1.
*
* When the BigDecimal value's precision exceeds MAX_PRECISION and there are fractional digits
- * because of scale > 0, then lower digits are trimmed off with rounding to meet the
+ * because of scale > 0, then lower digits are trimmed off with rounding to meet the
* MAX_PRECISION requirement.
*
* Also, BigDecimal supports negative scale -- which means multiplying the value by 10^abs(scale).
* And, BigDecimal allows for a non-zero scale for zero. We normalize that so zero always has
* scale 0.
*
- * @param bigDecimal
+ * @param bigDecimal the value to set
* @return The HiveDecimal with the BigDecimal's value adjusted down to a maximum precision.
* Otherwise, null is returned for overflow.
*/
@@ -297,7 +299,7 @@ public final class HiveDecimal extends FastHiveDecimal implements Comparable<Hiv
/**
* Same as the above create method, except fractional digit rounding can be turned off.
- * @param bigDecimal
+ * @param bigDecimal the value to set
* @param allowRounding True requires all of the bigDecimal value be converted to the decimal
* without loss of precision.
* @return
@@ -318,7 +320,7 @@ public final class HiveDecimal extends FastHiveDecimal implements Comparable<Hiv
* We will have overflow if BigInteger exceed MAX_PRECISION digits or
* 99,999,999,999,999,999,999,999,999,999,999,999,999 or 10^38 - 1.
*
- * @param bigInteger
+ * @param bigInteger the value to set
* @return A HiveDecimal object with the exact BigInteger's value.
* Otherwise, null is returned on overflow.
*/
@@ -341,11 +343,11 @@ public final class HiveDecimal extends FastHiveDecimal implements Comparable<Hiv
* The resulting decimal will have fractional digits when the specified scale is greater than 0.
*
* When the BigInteger's value's precision exceeds MAX_PRECISION and there are fractional digits
- * because of scale > 0, then lower digits are trimmed off with rounding to meet the
+ * because of scale > 0, then lower digits are trimmed off with rounding to meet the
* MAX_PRECISION requirement.
*
- * @param bigInteger
- * @param scale
+ * @param bigInteger the value to set
+ * @param scale the scale to set
* @return A HiveDecimal object with the BigInteger's value adjusted for scale.
* Otherwise, null is returned on overflow.
*/
@@ -365,6 +367,8 @@ public final class HiveDecimal extends FastHiveDecimal implements Comparable<Hiv
* We support parsing a decimal with an exponent because the previous version
* (i.e. OldHiveDecimal) uses the BigDecimal parser and was able to.
*
+ * @param string the string to parse
+ * @return a new hive decimal
*/
@HiveDecimalVersionV1
public static HiveDecimal create(String string) {
@@ -378,9 +382,9 @@ public final class HiveDecimal extends FastHiveDecimal implements Comparable<Hiv
/**
* Same as the method above, except blanks before and after are tolerated.
- * @param string
+ * @param string the string to parse
* @param trimBlanks True specifies leading and trailing blanks are to be ignored.
- * @return
+ * @return a new hive decimal
*/
@HiveDecimalVersionV2
public static HiveDecimal create(String string, boolean trimBlanks) {
@@ -766,7 +770,7 @@ public final class HiveDecimal extends FastHiveDecimal implements Comparable<Hiv
* Is does the equivalent of a setScale(int newScale). So, more than 38 digits may be returned.
* See that method for more details on how this can happen.
* <p>
- * @param scale The number of digits after the decimal point
+ * @param formatScale The number of digits after the decimal point
* @return The scaled decimal representation string representation.
*/
@HiveDecimalVersionV1
@@ -894,7 +898,7 @@ public final class HiveDecimal extends FastHiveDecimal implements Comparable<Hiv
/**
* Are two decimal content (values) equal?
* <p>
- * @obj The 2nd decimal.
+ * @param obj The 2nd decimal.
* @return When obj is null or not class HiveDecimal, the return is false.
* Otherwise, returns true when the decimal values are exactly equal.
*/
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java
index 08e8638..f914a22 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/BytesColumnVector.java
@@ -442,7 +442,7 @@ public class BytesColumnVector extends ColumnVector {
}
if (noNulls || !isNull[row]) {
buffer.append('"');
- buffer.append(new String(this.buffer, start[row], length[row]));
+ buffer.append(new String(vector[row], start[row], length[row]));
buffer.append('"');
} else {
buffer.append("null");
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
index 507d46e..065c1fa 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/ColumnVector.java
@@ -191,13 +191,13 @@ public abstract class ColumnVector {
* This method is deliberately *not* recursive because the complex types
* can easily have more (or less) children than the upper levels.
* @param size the new minimum size
- * @param presesrveData should the old data be preserved?
+ * @param preserveData should the old data be preserved?
*/
- public void ensureSize(int size, boolean presesrveData) {
+ public void ensureSize(int size, boolean preserveData) {
if (isNull.length < size) {
boolean[] oldArray = isNull;
isNull = new boolean[size];
- if (presesrveData && !noNulls) {
+ if (preserveData && !noNulls) {
if (isRepeating) {
isNull[0] = oldArray[0];
} else {
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java
index 3307f57..9d579ce 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/ql/exec/vector/TimestampColumnVector.java
@@ -182,7 +182,7 @@ public class TimestampColumnVector extends ColumnVector {
/**
* Return a double representation of a Timestamp.
- * @param elementNum
+ * @param timestamp
* @return
*/
public static double getDouble(Timestamp timestamp) {
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/storage-api/src/java/org/apache/hadoop/hive/ql/util/TimestampUtils.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/util/TimestampUtils.java b/storage-api/src/java/org/apache/hadoop/hive/ql/util/TimestampUtils.java
index c16d67e..dfc7272 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/ql/util/TimestampUtils.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/ql/util/TimestampUtils.java
@@ -109,7 +109,7 @@ public class TimestampUtils {
* is the nanoseconds and integer part is the number of seconds.
*
* This is a HiveDecimalWritable variation with supplied scratch objects.
- * @param decdecWritable
+ * @param decWritable
* @param scratchDecWritable1
* @param scratchDecWritable2
* @return
http://git-wip-us.apache.org/repos/asf/hive/blob/0d045535/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java
----------------------------------------------------------------------
diff --git a/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java b/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java
index 41b4b65..9ac7ba0 100644
--- a/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java
+++ b/storage-api/src/test/org/apache/hadoop/hive/ql/exec/vector/TestStructColumnVector.java
@@ -20,6 +20,10 @@ package org.apache.hadoop.hive.ql.exec.vector;
import org.junit.Test;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.sql.Timestamp;
+
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -92,4 +96,36 @@ public class TestStructColumnVector {
assertEquals(false, input1.isRepeating);
assertEquals(true, input.noNulls);
}
+
+ @Test
+ public void testStringify() throws IOException {
+ VectorizedRowBatch batch = new VectorizedRowBatch(2);
+ LongColumnVector x1 = new LongColumnVector();
+ TimestampColumnVector x2 = new TimestampColumnVector();
+ StructColumnVector x = new StructColumnVector(1024, x1, x2);
+ BytesColumnVector y = new BytesColumnVector();
+ batch.cols[0] = x;
+ batch.cols[1] = y;
+ batch.reset();
+ Timestamp ts = Timestamp.valueOf("2000-01-01 00:00:00");
+ for(int r=0; r < 10; ++r) {
+ batch.size += 1;
+ x1.vector[r] = 3 * r;
+ ts.setTime(ts.getTime() + 1000);
+ x2.set(r, ts);
+ byte[] buffer = ("value " + r).getBytes(StandardCharsets.UTF_8);
+ y.setRef(r, buffer, 0, buffer.length);
+ }
+ final String EXPECTED = ("[[0, 2000-01-01 00:00:01.0], \"value 0\"]\n" +
+ "[[3, 2000-01-01 00:00:02.0], \"value 1\"]\n" +
+ "[[6, 2000-01-01 00:00:03.0], \"value 2\"]\n" +
+ "[[9, 2000-01-01 00:00:04.0], \"value 3\"]\n" +
+ "[[12, 2000-01-01 00:00:05.0], \"value 4\"]\n" +
+ "[[15, 2000-01-01 00:00:06.0], \"value 5\"]\n" +
+ "[[18, 2000-01-01 00:00:07.0], \"value 6\"]\n" +
+ "[[21, 2000-01-01 00:00:08.0], \"value 7\"]\n" +
+ "[[24, 2000-01-01 00:00:09.0], \"value 8\"]\n" +
+ "[[27, 2000-01-01 00:00:10.0], \"value 9\"]");
+ assertEquals(EXPECTED, batch.toString());
+ }
}
[5/5] hive git commit: HIVE-13906. Remove guava from compile time
dependence in the storage-api module. (omalley reviewed by Sergey Shelukhin)
Posted by om...@apache.org.
HIVE-13906. Remove guava from compile time dependence in the storage-api
module. (omalley reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/a6857753
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/a6857753
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/a6857753
Branch: refs/heads/branch-2.2
Commit: a6857753b656f9d8291ebb8b5691ec768cdc9a49
Parents: ae52887
Author: Owen O'Malley <om...@apache.org>
Authored: Wed Jun 1 10:58:34 2016 -0700
Committer: Owen O'Malley <om...@apache.org>
Committed: Wed Apr 26 15:03:05 2017 -0700
----------------------------------------------------------------------
storage-api/pom.xml | 19 ++++++++++++++++---
.../apache/hadoop/hive/common/DiskRangeInfo.java | 5 ++---
.../hadoop/hive/ql/util/JavaDataModel.java | 4 +---
.../org/apache/hive/common/util/BloomFilter.java | 8 ++++++--
4 files changed, 25 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/a6857753/storage-api/pom.xml
----------------------------------------------------------------------
diff --git a/storage-api/pom.xml b/storage-api/pom.xml
index 036a994..097efdb 100644
--- a/storage-api/pom.xml
+++ b/storage-api/pom.xml
@@ -38,7 +38,7 @@
</properties>
<dependencies>
- <!-- compile inter-project -->
+ <!-- inter-project -->
<dependency>
<groupId>commons-lang</groupId>
<artifactId>commons-lang</artifactId>
@@ -48,7 +48,7 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<version>${hadoop.version}</version>
- <optional>true</optional>
+ <scope>provided</scope>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
@@ -95,7 +95,20 @@
<artifactId>jetty-util</artifactId>
</exclusion>
</exclusions>
- </dependency>
+ </dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>slf4j-api</artifactId>
+ <version>${slf4j.version}</version>
+ </dependency>
+
+ <!-- test inter-project -->
+ <dependency>
+ <groupId>com.google.guava</groupId>
+ <artifactId>guava</artifactId>
+ <version>${guava.version}</version>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
http://git-wip-us.apache.org/repos/asf/hive/blob/a6857753/storage-api/src/java/org/apache/hadoop/hive/common/DiskRangeInfo.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/common/DiskRangeInfo.java b/storage-api/src/java/org/apache/hadoop/hive/common/DiskRangeInfo.java
index 86b838c..e5025bf 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/common/DiskRangeInfo.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/common/DiskRangeInfo.java
@@ -17,12 +17,11 @@
*/
package org.apache.hadoop.hive.common;
+import java.util.ArrayList;
import java.util.List;
import org.apache.hadoop.hive.common.io.DiskRange;
-import com.google.common.collect.Lists;
-
/**
* Disk range information class containing disk ranges and total length.
*/
@@ -31,7 +30,7 @@ public class DiskRangeInfo {
long totalLength;
public DiskRangeInfo(int indexBaseOffset) {
- this.diskRanges = Lists.newArrayList();
+ this.diskRanges = new ArrayList<>();
// Some data is missing from the stream for PPD uncompressed read (because index offset is
// relative to the entire stream and we only read part of stream if RGs are filtered; unlike
// with compressed data where PPD only filters CBs, so we always get full CB, and index offset
http://git-wip-us.apache.org/repos/asf/hive/blob/a6857753/storage-api/src/java/org/apache/hadoop/hive/ql/util/JavaDataModel.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hadoop/hive/ql/util/JavaDataModel.java b/storage-api/src/java/org/apache/hadoop/hive/ql/util/JavaDataModel.java
index 33b70c2..4a745e4 100644
--- a/storage-api/src/java/org/apache/hadoop/hive/ql/util/JavaDataModel.java
+++ b/storage-api/src/java/org/apache/hadoop/hive/ql/util/JavaDataModel.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.hive.ql.util;
-import com.google.common.annotations.VisibleForTesting;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -257,7 +255,7 @@ public enum JavaDataModel {
private static final JavaDataModel MODEL_FOR_SYSTEM = getModelForSystem();
}
- @VisibleForTesting
+ //@VisibleForTesting
static JavaDataModel getModelForSystem() {
String props = null;
try {
http://git-wip-us.apache.org/repos/asf/hive/blob/a6857753/storage-api/src/java/org/apache/hive/common/util/BloomFilter.java
----------------------------------------------------------------------
diff --git a/storage-api/src/java/org/apache/hive/common/util/BloomFilter.java b/storage-api/src/java/org/apache/hive/common/util/BloomFilter.java
index 74bf4f5..e9f419d 100644
--- a/storage-api/src/java/org/apache/hive/common/util/BloomFilter.java
+++ b/storage-api/src/java/org/apache/hive/common/util/BloomFilter.java
@@ -23,8 +23,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
-import static com.google.common.base.Preconditions.checkArgument;
-
/**
* BloomFilter is a probabilistic data structure for set membership check. BloomFilters are
* highly space efficient when compared to using a HashSet. Because of the probabilistic nature of
@@ -56,6 +54,12 @@ public class BloomFilter {
this(expectedEntries, DEFAULT_FPP);
}
+ static void checkArgument(boolean expression, String message) {
+ if (!expression) {
+ throw new IllegalArgumentException(message);
+ }
+ }
+
public BloomFilter(long expectedEntries, double fpp) {
checkArgument(expectedEntries > 0, "expectedEntries should be > 0");
checkArgument(fpp > 0.0 && fpp < 1.0, "False positive probability should be > 0.0 & < 1.0");