You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by sa...@apache.org on 2017/02/14 23:42:22 UTC
[33/50] [abbrv] phoenix git commit: PHOENIX-1598 Column encoding to
save space and improve performance
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-core/src/test/java/org/apache/phoenix/query/EncodedColumnQualifierCellsListTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/EncodedColumnQualifierCellsListTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/EncodedColumnQualifierCellsListTest.java
new file mode 100644
index 0000000..bd70f84
--- /dev/null
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/EncodedColumnQualifierCellsListTest.java
@@ -0,0 +1,608 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.query;
+
+import static org.apache.phoenix.schema.PTable.QualifierEncodingScheme.FOUR_BYTE_QUALIFIERS;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.ConcurrentModificationException;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.NoSuchElementException;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.schema.tuple.EncodedColumnQualiferCellsList;
+import org.junit.Test;
+
+public class EncodedColumnQualifierCellsListTest {
+
+ private static final byte[] row = Bytes.toBytes("row");
+ private static final byte[] cf = Bytes.toBytes("cf");
+
+
+ @Test
+ public void testIterator() {
+ EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS);
+ Cell[] cells = new Cell[7];
+ int i = 0;
+ populateListAndArray(list, cells);
+ Iterator itr = list.iterator();
+ assertTrue(itr.hasNext());
+
+ // test itr.next()
+ i = 0;
+ while (itr.hasNext()) {
+ assertEquals(cells[i++], itr.next());
+ }
+
+ assertEquals(7, list.size());
+
+ // test itr.remove()
+ itr = list.iterator();
+ i = 0;
+ int numRemoved = 0;
+ try {
+ itr.remove();
+ fail("Remove not allowed till next() is called");
+ } catch (IllegalStateException expected) {}
+
+ while (itr.hasNext()) {
+ assertEquals(cells[i++], itr.next());
+ itr.remove();
+ numRemoved++;
+ }
+ assertEquals("Number of elements removed should have been the size of the list", 7, numRemoved);
+ }
+
+ @Test
+ public void testSize() {
+ EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS);
+ assertEquals(0, list.size());
+
+ populateList(list);
+
+ assertEquals(7, list.size());
+ int originalSize = list.size();
+
+ Iterator itr = list.iterator();
+ while (itr.hasNext()) {
+ itr.next();
+ itr.remove();
+ assertEquals(--originalSize, list.size());
+ }
+ }
+
+ @Test
+ public void testIsEmpty() throws Exception {
+ EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS);
+ assertTrue(list.isEmpty());
+ populateList(list);
+ assertFalse(list.isEmpty());
+ Iterator itr = list.iterator();
+ while (itr.hasNext()) {
+ itr.next();
+ itr.remove();
+ if (itr.hasNext()) {
+ assertFalse(list.isEmpty());
+ }
+ }
+ assertTrue(list.isEmpty());
+ }
+
+ @Test
+ public void testContains() throws Exception {
+ EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS);
+ Cell[] cells = new Cell[7];
+ populateListAndArray(list, cells);
+
+ for (Cell c : cells) {
+ assertTrue(list.contains(c));
+ }
+ assertFalse(list.contains(KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(13))));
+ }
+
+ @Test
+ public void testToArrayWithParam() throws Exception {
+ EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS);
+ Cell[] cells = new Cell[7];
+ populateListAndArray(list, cells);
+ Cell[] array = list.toArray(new Cell[0]);
+ assertTrue(Arrays.equals(cells, array));
+ }
+
+ @Test
+ public void testToArrayWithoutParam() throws Exception {
+ EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS);
+ Cell[] cells = new Cell[7];
+ populateListAndArray(list, cells);
+ Object[] array = list.toArray();
+ assertTrue(Arrays.equals(cells, array));
+ }
+
+ @Test
+ public void testRemove() throws Exception {
+ EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS);
+ Cell[] cells = new Cell[7];
+ populateListAndArray(list, cells);
+ assertTrue(list.remove(cells[0]));
+ assertEquals(6, list.size());
+ assertTrue(list.remove(cells[6]));
+ assertEquals(5, list.size());
+ assertTrue(list.remove(cells[3]));
+ assertEquals(4, list.size());
+ assertFalse(list.remove(KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(13))));
+ assertEquals(4, list.size());
+ }
+
+ @Test
+ public void testContainsAll() throws Exception {
+ EncodedColumnQualiferCellsList list1 = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS);
+ populateList(list1);
+ EncodedColumnQualiferCellsList list2 = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS);
+ populateList(list2);
+ assertTrue(list1.containsAll(list2));
+ list2.remove(KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(11)));
+ assertTrue(list1.containsAll(list2));
+ assertFalse(list2.containsAll(list1));
+ list2.add(KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(13)));
+ assertFalse(list1.containsAll(list2));
+ assertFalse(list2.containsAll(list1));
+ List<Cell> arrayList = new ArrayList<>();
+ populateList(arrayList);
+ assertTrue(list1.containsAll(arrayList));
+ }
+
+ @Test
+ public void testAddAll() throws Exception {
+ EncodedColumnQualiferCellsList list1 = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS);
+ populateList(list1);
+ EncodedColumnQualiferCellsList list2 = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS);
+ populateList(list2);
+ /*
+ * Note that we don't care about equality of the element being added with the element already
+ * present at the index.
+ */
+ assertTrue(list1.addAll(list2));
+ }
+
+ @Test
+ public void testAddAllAtIndexFails() throws Exception {
+ EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS);
+ populateList(list);
+ try {
+ list.addAll(0, new ArrayList<Cell>());
+ } catch (UnsupportedOperationException expected) {
+ }
+ }
+
+ @Test
+ public void testRemoveAll() throws Exception {
+ EncodedColumnQualiferCellsList list1 = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS);
+ populateList(list1);
+ ArrayList<Cell> list2 = new ArrayList<>();
+ populateList(list2);
+ assertTrue(list1.removeAll(list2));
+ assertTrue(list1.isEmpty());
+ assertFalse(list2.isEmpty());
+ }
+
+ @Test
+ public void testRetainAll() throws Exception {
+ EncodedColumnQualiferCellsList list1 = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS);
+ populateList(list1);
+ EncodedColumnQualiferCellsList list2 = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS);
+ populateList(list2);
+ // retainAll won't be modifying the list1 since they both have the same elements equality wise
+ assertFalse(list1.retainAll(list2));
+ list2.remove(KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(12)));
+ assertTrue(list1.retainAll(list2));
+ assertEquals(list1.size(), list2.size());
+ for (Cell c : list1) {
+ assertTrue(list2.contains(c));
+ }
+ }
+
+ @Test
+ public void testClear() throws Exception {
+ EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS);
+ populateList(list);
+ list.clear();
+ assertTrue(list.isEmpty());
+ assertEquals(0, list.size());
+ }
+
+ @Test
+ public void testGetIndex() throws Exception {
+ EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS);
+ Cell[] cells = new Cell[7];
+ populateListAndArray(list, cells);
+ for (int i = 0; i < cells.length; i++) {
+ assertEquals(cells[i], list.get(i));
+ }
+ }
+
+ @Test
+ public void testIndexOf() throws Exception {
+ EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS);
+ Cell[] cells = new Cell[7];
+ populateListAndArray(list, cells);
+ for (int i = 0; i < cells.length; i++) {
+ assertEquals(i, list.indexOf(cells[i]));
+ }
+ }
+
+ @Test
+ public void testLastIndexOf() throws Exception {
+ EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS);
+ Cell[] cells = new Cell[7];
+ populateListAndArray(list, cells);
+ for (int i = 0; i < cells.length; i++) {
+ assertEquals(i, list.lastIndexOf(cells[i]));
+ }
+ }
+
+ @Test
+ public void testListIterator() throws Exception {
+ EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS);
+ Cell[] cells = new Cell[7];
+ int i = 0;
+ populateListAndArray(list, cells);
+ ListIterator<Cell> itr = list.listIterator();
+ assertTrue(itr.hasNext());
+
+ // test itr.next()
+ i = 0;
+ while (itr.hasNext()) {
+ assertEquals(cells[i++], itr.next());
+ }
+
+ assertEquals(7, list.size());
+
+ // test itr.remove()
+ itr = list.listIterator();
+ i = 0;
+ int numRemoved = 0;
+ try {
+ itr.remove();
+ fail("Remove not allowed till next() is called");
+ } catch (IllegalStateException expected) {}
+
+ while (itr.hasNext()) {
+ assertEquals(cells[i++], itr.next());
+ itr.remove();
+ numRemoved++;
+ }
+ assertEquals("Number of elements removed should have been the size of the list", 7, numRemoved);
+ assertTrue(list.isEmpty());
+ }
+
+ @Test
+ public void testListIteratorSet() {
+ EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS);
+ Cell[] array = new Cell[7];
+ populateListAndArray(list, array);
+ ListIterator<Cell> itr = list.listIterator();
+ // This cell is KeyValue.createFirstOnRow(row, cf, getEncodedColumnQualifier(12))
+ final Cell validCell = array[4];
+ // This cell is KeyValue.createFirstOnRow(row, cf, getEncodedColumnQualifier(14))
+ final Cell invalidCell = array[5];
+ String validCellName = "Valid Cell";
+ String invalidCellName = "Invalid Cell";
+ Cell validReplacementCell = new DelegateCell(validCell, validCellName);
+ Cell invalidReplacementCell = new DelegateCell(invalidCell, invalidCellName);
+ int i = 0;
+ while (itr.hasNext()) {
+ Cell c = itr.next();
+ if (i == 4) {
+ itr.set(validReplacementCell);
+ }
+ if (i == 6) {
+ try {
+ itr.set(invalidReplacementCell);
+ fail("This should have failed since " + invalidReplacementCell + " cannot be added where " + c + " is.");
+ } catch (IllegalArgumentException expected) {
+ }
+ }
+ i++;
+ }
+ itr = list.listIterator();
+ i = 0;
+ // Assert that the valid cell was added and invalid cell wasn't.
+ while (itr.hasNext()) {
+ Cell c = itr.next();
+ if (i == 4) {
+ assertEquals(validCellName, c.toString());
+ }
+ if (i == 6) {
+ assertNotEquals(invalidCellName, c.toString());
+ }
+ i++;
+ }
+ }
+
+ @Test
+ public void testListIteratorNextAndPrevious() throws Exception {
+ EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS);
+ Cell[] array = new Cell[7];
+ populateListAndArray(list, array);
+ ListIterator<Cell> itr = list.listIterator();
+ try {
+ itr.previous();
+ fail("Call to itr.previous() should have failed since the iterator hasn't been moved forward yet");
+ } catch (NoSuchElementException expected) {
+
+ }
+ Cell c = itr.next();
+ Cell d = itr.previous();
+ Cell e = itr.next();
+ Cell f = itr.previous();
+ assertTrue(c.equals(d) && c.equals(f) && c.equals(e));
+ itr = list.listIterator();
+ int i = 0;
+ assertEquals(array[i++], itr.next());
+ assertEquals(array[i++], itr.next());
+ assertEquals(array[i++], itr.next());
+ assertEquals(array[--i], itr.previous());
+ assertEquals(array[--i], itr.previous());
+ assertEquals(array[i++], itr.next());
+
+ // move itr forward till next() is exhausted
+ while (itr.hasNext()) {
+ itr.next();
+ }
+ i = 6;
+ while (itr.hasPrevious()) {
+ assertEquals(array[i--], itr.previous());
+ }
+ assertEquals("Not all elements navigated using previous()", -1, i);
+ // now that previous is exhausted, move itr() forward till next() is exhausted
+ i = 0;
+ while (itr.hasNext()) {
+ assertEquals(array[i++], itr.next());
+ }
+ assertEquals("Not all elements navigated using next()", 7, i);
+ }
+
+ @Test
+ public void testSetNull() throws Exception {
+ EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS);
+ try {
+ list.add(null);
+ fail("Adding null elements to the list is not allowed");
+ } catch (NullPointerException expected) {
+
+ }
+ }
+
+ @Test
+ public void testFailFastIterator() throws Exception {
+ EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS);
+ populateList(list);
+ int i = 0;
+ Iterator<Cell> itr = list.iterator();
+ while (itr.hasNext()) {
+ i++;
+ try {
+ itr.next();
+ list.add(KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(0)));
+ if (i == 2) {
+ fail("ConcurrentModificationException should have been thrown as the list is being modified while being iterated through");
+ }
+ } catch (ConcurrentModificationException expected) {
+ assertEquals("Exception should have been thrown when getting the second element",
+ 2, i);
+ break;
+ }
+ }
+ }
+
+ @Test
+ public void testFailFastListIterator() throws Exception {
+ EncodedColumnQualiferCellsList list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS);
+ populateList(list);
+ ListIterator<Cell> itr = list.listIterator();
+ itr.next();
+ list.add(KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(0)));
+ try {
+ itr.next();
+ fail("ConcurrentModificationException should have been thrown as the list was modified without using iterator");
+ } catch (ConcurrentModificationException expected) {
+
+ }
+ list = new EncodedColumnQualiferCellsList(11, 16, FOUR_BYTE_QUALIFIERS);
+ populateList(list);
+ itr = list.listIterator();
+ itr.next();
+ itr.next();
+ itr.remove();
+ itr.next();
+ list.remove(KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(0)));
+ try {
+ itr.next();
+ fail("ConcurrentModificationException should have been thrown as the list was modified without using iterator");
+ } catch (ConcurrentModificationException expected) {
+
+ }
+ }
+
+ private void populateListAndArray(List<Cell> list, Cell[] cells) {
+ // add elements in reserved range
+ list.add(cells[0] = KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(0)));
+ list.add(cells[1] = KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(5)));
+ list.add(cells[2] = KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(10)));
+
+ // add elements in qualifier range
+ list.add(cells[6] = KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(16)));
+ list.add(cells[4] = KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(12)));
+ list.add(cells[5] = KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(14)));
+ list.add(cells[3] = KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(11)));
+ }
+
+ private void populateList(List<Cell> list) {
+ // add elements in reserved range
+ list.add(KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(0)));
+ list.add(KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(5)));
+ list.add(KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(10)));
+
+ // add elements in qualifier range
+ list.add(KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(16)));
+ list.add(KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(12)));
+ list.add(KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(14)));
+ list.add(KeyValue.createFirstOnRow(row, cf, FOUR_BYTE_QUALIFIERS.encode(11)));
+ }
+
+ private class DelegateCell implements Cell {
+ private final Cell delegate;
+ private final String name;
+ public DelegateCell(Cell delegate, String name) {
+ this.delegate = delegate;
+ this.name = name;
+ }
+
+ @Override
+ public int getValueOffset() {
+ return delegate.getValueOffset();
+ }
+
+ @Override
+ public int getValueLength() {
+ return delegate.getValueLength();
+ }
+
+ @Override
+ public byte[] getValueArray() {
+ return delegate.getValueArray();
+ }
+
+ @Override
+ public byte[] getValue() {
+ return delegate.getValue();
+ }
+
+ @Override
+ public byte getTypeByte() {
+ return delegate.getTypeByte();
+ }
+
+ @Override
+ public long getTimestamp() {
+ return delegate.getTimestamp();
+ }
+
+ @Override
+ public int getTagsOffset() {
+ return delegate.getTagsOffset();
+ }
+
+ @Override
+ public int getTagsLengthUnsigned() {
+ return delegate.getTagsLengthUnsigned();
+ }
+
+ @Override
+ public short getTagsLength() {
+ return delegate.getTagsLength();
+ }
+
+ @Override
+ public byte[] getTagsArray() {
+ return delegate.getTagsArray();
+ }
+
+ @Override
+ public int getRowOffset() {
+ return delegate.getRowOffset();
+ }
+
+ @Override
+ public short getRowLength() {
+ return delegate.getRowLength();
+ }
+
+ @Override
+ public byte[] getRowArray() {
+ return delegate.getRowArray();
+ }
+
+ @Override
+ public byte[] getRow() {
+ return delegate.getRow();
+ }
+
+ @Override
+ public int getQualifierOffset() {
+ return delegate.getQualifierOffset();
+ }
+
+ @Override
+ public int getQualifierLength() {
+ return delegate.getQualifierLength();
+ }
+
+ @Override
+ public byte[] getQualifierArray() {
+ return delegate.getQualifierArray();
+ }
+
+ @Override
+ public byte[] getQualifier() {
+ return delegate.getQualifier();
+ }
+
+ @Override
+ public long getMvccVersion() {
+ return delegate.getMvccVersion();
+ }
+
+ @Override
+ public int getFamilyOffset() {
+ return delegate.getFamilyOffset();
+ }
+
+ @Override
+ public byte getFamilyLength() {
+ return delegate.getFamilyLength();
+ }
+
+ @Override
+ public byte[] getFamilyArray() {
+ return delegate.getFamilyArray();
+ }
+
+ @Override
+ public byte[] getFamily() {
+ return delegate.getFamily();
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-core/src/test/java/org/apache/phoenix/schema/ImmutableStorageSchemeTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/schema/ImmutableStorageSchemeTest.java b/phoenix-core/src/test/java/org/apache/phoenix/schema/ImmutableStorageSchemeTest.java
new file mode 100644
index 0000000..d8c5cdb
--- /dev/null
+++ b/phoenix-core/src/test/java/org/apache/phoenix/schema/ImmutableStorageSchemeTest.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.schema;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
+import org.apache.phoenix.expression.DelegateExpression;
+import org.apache.phoenix.expression.Expression;
+import org.apache.phoenix.expression.LiteralExpression;
+import org.apache.phoenix.expression.SingleCellConstructorExpression;
+import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
+import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.schema.PTable.ImmutableStorageScheme;
+import org.apache.phoenix.schema.tuple.Tuple;
+import org.apache.phoenix.schema.types.PVarbinary;
+import org.apache.phoenix.util.ByteUtil;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+import com.google.common.collect.Lists;
+
+@RunWith(Parameterized.class)
+public class ImmutableStorageSchemeTest {
+
+ protected static final LiteralExpression CONSTANT_EXPRESSION = LiteralExpression.newConstant(QueryConstants.EMPTY_COLUMN_VALUE_BYTES);
+ protected static final byte[] BYTE_ARRAY1 = new byte[]{1,2,3,4,5};
+ protected static final byte[] BYTE_ARRAY2 = new byte[]{6,7,8};
+ protected Expression FALSE_EVAL_EXPRESSION = new DelegateExpression(LiteralExpression.newConstant(null)) {
+ @Override
+ public boolean evaluate(Tuple tuple, ImmutableBytesWritable ptr) {
+ return false;
+ }
+ };
+ private ImmutableStorageScheme immutableStorageScheme;
+
+ @Parameters(name="ImmutableStorageSchemeTest_immutableStorageScheme={0}}") // name is used by failsafe as file name in reports
+ public static ImmutableStorageScheme[] data() {
+ ImmutableStorageScheme[] values = ImmutableStorageScheme.values();
+ // skip ONE_CELL_PER_COLUMN
+ return Arrays.copyOfRange(values, 1, values.length);
+ }
+
+ public ImmutableStorageSchemeTest(ImmutableStorageScheme immutableStorageScheme) {
+ this.immutableStorageScheme = immutableStorageScheme;
+ }
+
+ @Test
+ public void testWithExpressionsThatEvaluatetoFalse() throws Exception {
+ List<Expression> children = Lists.newArrayListWithExpectedSize(4);
+ children.add(CONSTANT_EXPRESSION);
+ children.add(FALSE_EVAL_EXPRESSION);
+ children.add(LiteralExpression.newConstant(BYTE_ARRAY1, PVarbinary.INSTANCE));
+ children.add(FALSE_EVAL_EXPRESSION);
+ children.add(LiteralExpression.newConstant(BYTE_ARRAY2, PVarbinary.INSTANCE));
+ SingleCellConstructorExpression singleCellConstructorExpression = new SingleCellConstructorExpression(immutableStorageScheme, children);
+ ImmutableBytesPtr ptr = new ImmutableBytesPtr();
+ singleCellConstructorExpression.evaluate(null, ptr);
+
+ ImmutableBytesPtr ptrCopy = new ImmutableBytesPtr(ptr);
+ ColumnValueDecoder decoder = immutableStorageScheme.getDecoder();
+ assertTrue(decoder.decode(ptrCopy, 0));
+ assertArrayEquals(QueryConstants.EMPTY_COLUMN_VALUE_BYTES, ptrCopy.copyBytesIfNecessary());
+ ptrCopy = new ImmutableBytesPtr(ptr);
+ assertFalse(decoder.decode(ptrCopy, 1));
+ assertArrayEquals(ByteUtil.EMPTY_BYTE_ARRAY, ptrCopy.copyBytesIfNecessary());
+ ptrCopy = new ImmutableBytesPtr(ptr);
+ assertTrue(decoder.decode(ptrCopy, 2));
+ assertArrayEquals(BYTE_ARRAY1, ptrCopy.copyBytesIfNecessary());
+ ptrCopy = new ImmutableBytesPtr(ptr);
+ assertFalse(decoder.decode(ptrCopy, 3));
+ assertArrayEquals(ByteUtil.EMPTY_BYTE_ARRAY, ptrCopy.copyBytesIfNecessary());
+ ptrCopy = new ImmutableBytesPtr(ptr);
+ assertTrue(decoder.decode(ptrCopy, 4));
+ assertArrayEquals(BYTE_ARRAY2, ptrCopy.copyBytesIfNecessary());
+ }
+
+ @Test
+ public void testWithMaxOffsetLargerThanShortMax() throws Exception {
+ int numElements = Short.MAX_VALUE+2;
+ List<Expression> children = Lists.newArrayListWithExpectedSize(numElements);
+ for (int i=0; i<numElements; ++i) {
+ children.add(CONSTANT_EXPRESSION);
+ }
+ SingleCellConstructorExpression singleCellConstructorExpression = new SingleCellConstructorExpression(immutableStorageScheme, children);
+ ImmutableBytesPtr ptr = new ImmutableBytesPtr();
+ singleCellConstructorExpression.evaluate(null, ptr);
+
+ ImmutableBytesPtr ptrCopy = new ImmutableBytesPtr(ptr);
+ ColumnValueDecoder decoder = immutableStorageScheme.getDecoder();
+ assertTrue(decoder.decode(ptrCopy, 0));
+ assertArrayEquals(QueryConstants.EMPTY_COLUMN_VALUE_BYTES, ptrCopy.copyBytesIfNecessary());
+
+ ptrCopy = new ImmutableBytesPtr(ptr);
+ assertTrue(decoder.decode(ptrCopy, 14999));
+ assertArrayEquals(QueryConstants.EMPTY_COLUMN_VALUE_BYTES, ptrCopy.copyBytesIfNecessary());
+
+ ptrCopy = new ImmutableBytesPtr(ptr);
+ assertTrue(decoder.decode(ptrCopy, numElements-1));
+ assertArrayEquals(QueryConstants.EMPTY_COLUMN_VALUE_BYTES, ptrCopy.copyBytesIfNecessary());
+ }
+
+ @Test
+ public void testWithMaxOffsetSmallerThanShortMin() throws Exception {
+ int numElements = Short.MAX_VALUE+2;
+ List<Expression> children = Lists.newArrayListWithExpectedSize(numElements);
+ for (int i=0; i<=numElements; i+=2) {
+ children.add(CONSTANT_EXPRESSION);
+ children.add(FALSE_EVAL_EXPRESSION);
+ }
+ SingleCellConstructorExpression singleCellConstructorExpression = new SingleCellConstructorExpression(immutableStorageScheme, children);
+ ImmutableBytesPtr ptr = new ImmutableBytesPtr();
+ singleCellConstructorExpression.evaluate(null, ptr);
+
+ ImmutableBytesPtr ptrCopy = new ImmutableBytesPtr(ptr);
+ ColumnValueDecoder decoder = immutableStorageScheme.getDecoder();
+ assertTrue(decoder.decode(ptrCopy, 0));
+ assertArrayEquals(QueryConstants.EMPTY_COLUMN_VALUE_BYTES, ptrCopy.copyBytesIfNecessary());
+
+ ptrCopy = new ImmutableBytesPtr(ptr);
+ assertFalse(decoder.decode(ptrCopy, 1));
+ assertArrayEquals(ByteUtil.EMPTY_BYTE_ARRAY, ptrCopy.copyBytesIfNecessary());
+
+ ptrCopy = new ImmutableBytesPtr(ptr);
+ assertTrue(decoder.decode(ptrCopy, numElements-1));
+ assertArrayEquals(QueryConstants.EMPTY_COLUMN_VALUE_BYTES, ptrCopy.copyBytesIfNecessary());
+
+ ptrCopy = new ImmutableBytesPtr(ptr);
+ assertFalse(decoder.decode(ptrCopy, numElements));
+ assertArrayEquals(ByteUtil.EMPTY_BYTE_ARRAY, ptrCopy.copyBytesIfNecessary());
+ }
+
+ @Test
+ public void testLeadingNulls() throws Exception {
+ List<Expression> children = Lists.newArrayListWithExpectedSize(4);
+ LiteralExpression nullExpression = LiteralExpression.newConstant(null);
+ children.add(nullExpression);
+ children.add(nullExpression);
+ children.add(LiteralExpression.newConstant(BYTE_ARRAY1, PVarbinary.INSTANCE));
+ children.add(LiteralExpression.newConstant(BYTE_ARRAY2, PVarbinary.INSTANCE));
+ SingleCellConstructorExpression singleCellConstructorExpression = new SingleCellConstructorExpression(immutableStorageScheme, children);
+ ImmutableBytesPtr ptr = new ImmutableBytesPtr();
+ singleCellConstructorExpression.evaluate(null, ptr);
+
+ ImmutableBytesPtr ptrCopy = new ImmutableBytesPtr(ptr);
+ ColumnValueDecoder decoder = immutableStorageScheme.getDecoder();
+ assertTrue(decoder.decode(ptrCopy, 0));
+ assertArrayEquals(ByteUtil.EMPTY_BYTE_ARRAY, ptrCopy.copyBytesIfNecessary());
+ ptrCopy = new ImmutableBytesPtr(ptr);
+ assertTrue(decoder.decode(ptrCopy, 1));
+ assertArrayEquals(ByteUtil.EMPTY_BYTE_ARRAY, ptrCopy.copyBytesIfNecessary());
+ ptrCopy = new ImmutableBytesPtr(ptr);
+ assertTrue(decoder.decode(ptrCopy, 2));
+ assertArrayEquals(BYTE_ARRAY1, ptrCopy.copyBytesIfNecessary());
+ ptrCopy = new ImmutableBytesPtr(ptr);
+ assertTrue(decoder.decode(ptrCopy, 3));
+ assertArrayEquals(BYTE_ARRAY2, ptrCopy.copyBytesIfNecessary());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-core/src/test/java/org/apache/phoenix/schema/types/PDataTypeForArraysTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/schema/types/PDataTypeForArraysTest.java b/phoenix-core/src/test/java/org/apache/phoenix/schema/types/PDataTypeForArraysTest.java
index 333fbf9..2aeeeb8 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/schema/types/PDataTypeForArraysTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/schema/types/PDataTypeForArraysTest.java
@@ -324,7 +324,7 @@ public class PDataTypeForArraysTest {
PVarchar.INSTANCE, strArr);
byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr);
ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes);
- PArrayDataType.positionAtArrayElement(ptr, 4, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
+ PArrayDataTypeDecoder.positionAtArrayElement(ptr, 4, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
int offset = ptr.getOffset();
int length = ptr.getLength();
byte[] bs = ptr.get();
@@ -342,7 +342,7 @@ public class PDataTypeForArraysTest {
PVarchar.INSTANCE, strArr);
byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr);
ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes);
- PArrayDataType.positionAtArrayElement(ptr, 0, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
+ PArrayDataTypeDecoder.positionAtArrayElement(ptr, 0, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
int offset = ptr.getOffset();
int length = ptr.getLength();
byte[] bs = ptr.get();
@@ -365,7 +365,7 @@ public class PDataTypeForArraysTest {
PVarchar.INSTANCE, strArr);
byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr);
ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes);
- PArrayDataType.positionAtArrayElement(ptr, 3, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
+ PArrayDataTypeDecoder.positionAtArrayElement(ptr, 3, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
int offset = ptr.getOffset();
int length = ptr.getLength();
byte[] bs = ptr.get();
@@ -402,7 +402,7 @@ public class PDataTypeForArraysTest {
PVarchar.INSTANCE, strArr);
byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr);
ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes);
- PArrayDataType.positionAtArrayElement(ptr, 3, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
+ PArrayDataTypeDecoder.positionAtArrayElement(ptr, 3, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
int offset = ptr.getOffset();
int length = ptr.getLength();
byte[] bs = ptr.get();
@@ -423,7 +423,7 @@ public class PDataTypeForArraysTest {
PVarchar.INSTANCE, strArr);
byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr);
ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes);
- PArrayDataType.positionAtArrayElement(ptr, 2, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
+ PArrayDataTypeDecoder.positionAtArrayElement(ptr, 2, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
int offset = ptr.getOffset();
int length = ptr.getLength();
byte[] bs = ptr.get();
@@ -444,7 +444,7 @@ public class PDataTypeForArraysTest {
PVarchar.INSTANCE, strArr);
byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr);
ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes);
- PArrayDataType.positionAtArrayElement(ptr, 2, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
+ PArrayDataTypeDecoder.positionAtArrayElement(ptr, 2, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
int offset = ptr.getOffset();
int length = ptr.getLength();
byte[] bs = ptr.get();
@@ -464,7 +464,7 @@ public class PDataTypeForArraysTest {
PVarchar.INSTANCE, strArr);
byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr);
ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes);
- PArrayDataType.positionAtArrayElement(ptr, 4, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
+ PArrayDataTypeDecoder.positionAtArrayElement(ptr, 4, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
int offset = ptr.getOffset();
int length = ptr.getLength();
byte[] bs = ptr.get();
@@ -485,7 +485,7 @@ public class PDataTypeForArraysTest {
PVarchar.INSTANCE, strArr);
byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr);
ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes);
- PArrayDataType.positionAtArrayElement(ptr, 3, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
+ PArrayDataTypeDecoder.positionAtArrayElement(ptr, 3, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
int offset = ptr.getOffset();
int length = ptr.getLength();
byte[] bs = ptr.get();
@@ -506,7 +506,7 @@ public class PDataTypeForArraysTest {
PVarchar.INSTANCE, strArr);
byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr);
ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes);
- PArrayDataType.positionAtArrayElement(ptr, 4, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
+ PArrayDataTypeDecoder.positionAtArrayElement(ptr, 4, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
int offset = ptr.getOffset();
int length = ptr.getLength();
byte[] bs = ptr.get();
@@ -528,7 +528,7 @@ public class PDataTypeForArraysTest {
PVarchar.INSTANCE, strArr);
byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr);
ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes);
- PArrayDataType.positionAtArrayElement(ptr, 4, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
+ PArrayDataTypeDecoder.positionAtArrayElement(ptr, 4, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
int offset = ptr.getOffset();
int length = ptr.getLength();
byte[] bs = ptr.get();
@@ -548,7 +548,7 @@ public class PDataTypeForArraysTest {
PVarchar.INSTANCE, strArr);
byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr);
ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes);
- PArrayDataType.positionAtArrayElement(ptr, 3, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
+ PArrayDataTypeDecoder.positionAtArrayElement(ptr, 3, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
int offset = ptr.getOffset();
int length = ptr.getLength();
byte[] bs = ptr.get();
@@ -569,7 +569,7 @@ public class PDataTypeForArraysTest {
PVarchar.INSTANCE, strArr);
byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr);
ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes);
- PArrayDataType.positionAtArrayElement(ptr, 3, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
+ PArrayDataTypeDecoder.positionAtArrayElement(ptr, 3, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
int offset = ptr.getOffset();
int length = ptr.getLength();
byte[] bs = ptr.get();
@@ -590,7 +590,7 @@ public class PDataTypeForArraysTest {
PVarchar.INSTANCE, strArr);
byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr);
ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes);
- PArrayDataType.positionAtArrayElement(ptr, 0, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
+ PArrayDataTypeDecoder.positionAtArrayElement(ptr, 0, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
int offset = ptr.getOffset();
int length = ptr.getLength();
byte[] bs = ptr.get();
@@ -611,7 +611,7 @@ public class PDataTypeForArraysTest {
PVarchar.INSTANCE, strArr);
byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr);
ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes);
- PArrayDataType.positionAtArrayElement(ptr, 4, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
+ PArrayDataTypeDecoder.positionAtArrayElement(ptr, 4, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
int offset = ptr.getOffset();
int length = ptr.getLength();
byte[] bs = ptr.get();
@@ -632,7 +632,7 @@ public class PDataTypeForArraysTest {
PVarchar.INSTANCE, strArr);
byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr);
ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes);
- PArrayDataType.positionAtArrayElement(ptr, 4, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
+ PArrayDataTypeDecoder.positionAtArrayElement(ptr, 4, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
int offset = ptr.getOffset();
int length = ptr.getLength();
byte[] bs = ptr.get();
@@ -649,7 +649,7 @@ public class PDataTypeForArraysTest {
PVarchar.INSTANCE, strArr);
byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr);
ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes);
- PArrayDataType.positionAtArrayElement(ptr, 0, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
+ PArrayDataTypeDecoder.positionAtArrayElement(ptr, 0, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
int offset = ptr.getOffset();
int length = ptr.getLength();
byte[] bs = ptr.get();
@@ -667,7 +667,7 @@ public class PDataTypeForArraysTest {
PVarchar.INSTANCE, strArr);
byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr);
ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes);
- PArrayDataType.positionAtArrayElement(ptr, 1, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
+ PArrayDataTypeDecoder.positionAtArrayElement(ptr, 1, PVarchar.INSTANCE, PVarchar.INSTANCE.getByteSize());
int offset = ptr.getOffset();
int length = ptr.getLength();
byte[] bs = ptr.get();
@@ -688,7 +688,7 @@ public class PDataTypeForArraysTest {
PLongArray.INSTANCE.toObject(arr, PLongArray.INSTANCE);
byte[] bytes = PLongArray.INSTANCE.toBytes(arr);
ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes);
- PArrayDataType.positionAtArrayElement(ptr, 2, PLong.INSTANCE, PLong.INSTANCE.getByteSize());
+ PArrayDataTypeDecoder.positionAtArrayElement(ptr, 2, PLong.INSTANCE, PLong.INSTANCE.getByteSize());
int offset = ptr.getOffset();
int length = ptr.getLength();
byte[] bs = ptr.get();
@@ -1196,7 +1196,7 @@ public class PDataTypeForArraysTest {
PhoenixArray arr = new PhoenixArray(PVarchar.INSTANCE, objects);
byte[] bytes = PVarcharArray.INSTANCE.toBytes(arr, PVarchar.INSTANCE, SortOrder.DESC);
ImmutableBytesWritable ptr = new ImmutableBytesWritable(bytes);
- PArrayDataType.positionAtArrayElement(ptr, 2, PVarchar.INSTANCE, null);
+ PArrayDataTypeDecoder.positionAtArrayElement(ptr, 2, PVarchar.INSTANCE, null);
String value = (String)PVarchar.INSTANCE.toObject(ptr, SortOrder.DESC);
assertEquals(null, value);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-core/src/test/java/org/apache/phoenix/util/PhoenixRuntimeTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/PhoenixRuntimeTest.java b/phoenix-core/src/test/java/org/apache/phoenix/util/PhoenixRuntimeTest.java
index 430c20b..7b81c8d 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/util/PhoenixRuntimeTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/PhoenixRuntimeTest.java
@@ -26,6 +26,8 @@ import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
import java.sql.Connection;
import java.sql.Date;
import java.sql.DriverManager;
@@ -37,6 +39,7 @@ import java.util.List;
import java.util.Properties;
import org.apache.commons.lang.exception.ExceptionUtils;
+import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.phoenix.compile.QueryPlan;
import org.apache.phoenix.expression.Expression;
@@ -45,6 +48,7 @@ import org.apache.phoenix.jdbc.PhoenixStatement;
import org.apache.phoenix.query.BaseConnectionlessQueryTest;
import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTable.QualifierEncodingScheme;
import org.apache.phoenix.schema.TableNotFoundException;
import org.apache.phoenix.schema.types.PDataType;
import org.junit.Test;
@@ -342,5 +346,6 @@ public class PhoenixRuntimeTest extends BaseConnectionlessQueryTest {
long skewedTs = ts + QueryConstants.MILLIS_IN_DAY; // skew of a day
// Even with a day of skew, we won't consider the ts a nanos timestamp
assertEquals(skewedTs, PhoenixRuntime.getWallClockTimeFromCellTimeStamp(skewedTs));
- }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-core/src/test/java/org/apache/phoenix/util/QualifierEncodingSchemeTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/QualifierEncodingSchemeTest.java b/phoenix-core/src/test/java/org/apache/phoenix/util/QualifierEncodingSchemeTest.java
new file mode 100644
index 0000000..2b08d7d
--- /dev/null
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/QualifierEncodingSchemeTest.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.util;
+
+import static org.apache.phoenix.schema.PTable.QualifierEncodingScheme.FOUR_BYTE_QUALIFIERS;
+import static org.apache.phoenix.schema.PTable.QualifierEncodingScheme.ONE_BYTE_QUALIFIERS;
+import static org.apache.phoenix.schema.PTable.QualifierEncodingScheme.THREE_BYTE_QUALIFIERS;
+import static org.apache.phoenix.schema.PTable.QualifierEncodingScheme.TWO_BYTE_QUALIFIERS;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import org.apache.phoenix.schema.PTable.QualifierEncodingScheme.InvalidQualifierBytesException;
+import org.junit.Test;
+
+public class QualifierEncodingSchemeTest {
+
+ @Test
+ public void testOneByteQualifierEncodeDecode() {
+ assertEquals(1, ONE_BYTE_QUALIFIERS.decode(ONE_BYTE_QUALIFIERS.encode(1)));
+ assertEquals(127, ONE_BYTE_QUALIFIERS.decode(ONE_BYTE_QUALIFIERS.encode(127)));
+ assertEquals(63, ONE_BYTE_QUALIFIERS.decode(ONE_BYTE_QUALIFIERS.encode(63)));
+ assertEquals(130, ONE_BYTE_QUALIFIERS.decode(ONE_BYTE_QUALIFIERS.encode(130)));
+ assertEquals(255, ONE_BYTE_QUALIFIERS.decode(ONE_BYTE_QUALIFIERS.encode(255)));
+ byte[] arr1 = ONE_BYTE_QUALIFIERS.encode(255);
+ byte[] arr2 = new byte[] {-128, arr1[0]};
+ assertEquals(255, ONE_BYTE_QUALIFIERS.decode(arr2, 1, 1));
+ try {
+ ONE_BYTE_QUALIFIERS.decode(arr2);
+ fail();
+ } catch (InvalidQualifierBytesException expected) {}
+ try {
+ ONE_BYTE_QUALIFIERS.decode(arr2, 0, 2);
+ fail();
+ } catch (InvalidQualifierBytesException expected) {}
+
+ }
+
+ @Test
+ public void testTwoByteQualifierEncodeDecode() {
+ assertEquals(1, TWO_BYTE_QUALIFIERS.decode(TWO_BYTE_QUALIFIERS.encode(1)));
+ assertEquals(127, TWO_BYTE_QUALIFIERS.decode(TWO_BYTE_QUALIFIERS.encode(127)));
+ assertEquals(63, TWO_BYTE_QUALIFIERS.decode(TWO_BYTE_QUALIFIERS.encode(63)));
+ assertEquals(130, TWO_BYTE_QUALIFIERS.decode(TWO_BYTE_QUALIFIERS.encode(130)));
+ assertEquals(128, TWO_BYTE_QUALIFIERS.decode(TWO_BYTE_QUALIFIERS.encode(128)));
+ assertEquals(129, TWO_BYTE_QUALIFIERS.decode(TWO_BYTE_QUALIFIERS.encode(129)));
+ assertEquals(32767, TWO_BYTE_QUALIFIERS.decode(TWO_BYTE_QUALIFIERS.encode(32767)));
+ assertEquals(32768, TWO_BYTE_QUALIFIERS.decode(TWO_BYTE_QUALIFIERS.encode(32768)));
+ assertEquals(65535, TWO_BYTE_QUALIFIERS.decode(TWO_BYTE_QUALIFIERS.encode(65535)));
+ byte[] arr1 = TWO_BYTE_QUALIFIERS.encode(65535);
+ byte[] arr2 = new byte[] {-128, arr1[0], arr1[1]};
+ assertEquals(65535, TWO_BYTE_QUALIFIERS.decode(arr2, 1, 2));
+ try {
+ TWO_BYTE_QUALIFIERS.decode(arr2);
+ fail();
+ } catch (InvalidQualifierBytesException expected) {}
+ }
+
+ @Test
+ public void testThreeByteQualifierEncodeDecode() {
+ assertEquals(1, THREE_BYTE_QUALIFIERS.decode(THREE_BYTE_QUALIFIERS.encode(1)));
+ assertEquals(127, THREE_BYTE_QUALIFIERS.decode(THREE_BYTE_QUALIFIERS.encode(127)));
+ assertEquals(63, THREE_BYTE_QUALIFIERS.decode(THREE_BYTE_QUALIFIERS.encode(63)));
+ assertEquals(130, THREE_BYTE_QUALIFIERS.decode(THREE_BYTE_QUALIFIERS.encode(130)));
+ assertEquals(128, THREE_BYTE_QUALIFIERS.decode(THREE_BYTE_QUALIFIERS.encode(128)));
+ assertEquals(129, THREE_BYTE_QUALIFIERS.decode(THREE_BYTE_QUALIFIERS.encode(129)));
+ assertEquals(32767, THREE_BYTE_QUALIFIERS.decode(THREE_BYTE_QUALIFIERS.encode(32767)));
+ assertEquals(32768, THREE_BYTE_QUALIFIERS.decode(THREE_BYTE_QUALIFIERS.encode(32768)));
+ assertEquals(65535, THREE_BYTE_QUALIFIERS.decode(THREE_BYTE_QUALIFIERS.encode(65535)));
+ assertEquals(16777215, THREE_BYTE_QUALIFIERS.decode(THREE_BYTE_QUALIFIERS.encode(16777215)));
+ byte[] arr1 = THREE_BYTE_QUALIFIERS.encode(16777215);
+ byte[] arr2 = new byte[] {-128, arr1[0], arr1[1], arr1[2]};
+ assertEquals(16777215, THREE_BYTE_QUALIFIERS.decode(arr2, 1, 3));
+ try {
+ THREE_BYTE_QUALIFIERS.decode(arr2, 0, 2);
+ fail();
+ } catch (InvalidQualifierBytesException expected) {}
+ }
+
+ @Test
+ public void testFourByteQualifierEncodeDecode() {
+ assertEquals(1, FOUR_BYTE_QUALIFIERS.decode(FOUR_BYTE_QUALIFIERS.encode(1)));
+ assertEquals(127, FOUR_BYTE_QUALIFIERS.decode(FOUR_BYTE_QUALIFIERS.encode(127)));
+ assertEquals(63, FOUR_BYTE_QUALIFIERS.decode(FOUR_BYTE_QUALIFIERS.encode(63)));
+ assertEquals(130, FOUR_BYTE_QUALIFIERS.decode(FOUR_BYTE_QUALIFIERS.encode(130)));
+ assertEquals(128, FOUR_BYTE_QUALIFIERS.decode(FOUR_BYTE_QUALIFIERS.encode(128)));
+ assertEquals(129, FOUR_BYTE_QUALIFIERS.decode(FOUR_BYTE_QUALIFIERS.encode(129)));
+ assertEquals(32767, FOUR_BYTE_QUALIFIERS.decode(FOUR_BYTE_QUALIFIERS.encode(32767)));
+ assertEquals(32768, FOUR_BYTE_QUALIFIERS.decode(FOUR_BYTE_QUALIFIERS.encode(32768)));
+ assertEquals(65535, FOUR_BYTE_QUALIFIERS.decode(FOUR_BYTE_QUALIFIERS.encode(65535)));
+ assertEquals(Integer.MAX_VALUE, FOUR_BYTE_QUALIFIERS.decode(FOUR_BYTE_QUALIFIERS.encode(Integer.MAX_VALUE)));
+ byte[] arr1 = FOUR_BYTE_QUALIFIERS.encode(Integer.MAX_VALUE);
+ byte[] arr2 = new byte[] {-128, arr1[0], arr1[1], arr1[2], arr1[3]};
+ assertEquals(Integer.MAX_VALUE, FOUR_BYTE_QUALIFIERS.decode(arr2, 1, 4));
+ try {
+ FOUR_BYTE_QUALIFIERS.decode(arr2);
+ fail();
+ } catch (InvalidQualifierBytesException expected) {}
+ try {
+ FOUR_BYTE_QUALIFIERS.decode(arr2, 0, 3);
+ fail();
+ } catch (InvalidQualifierBytesException expected) {}
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
index 5feedb1..ead712b 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
@@ -74,6 +74,7 @@ import org.apache.phoenix.compile.StatementContext;
import org.apache.phoenix.coprocessor.generated.MetaDataProtos.ClearCacheRequest;
import org.apache.phoenix.coprocessor.generated.MetaDataProtos.ClearCacheResponse;
import org.apache.phoenix.coprocessor.generated.MetaDataProtos.MetaDataService;
+import org.apache.phoenix.execute.MutationState;
import org.apache.phoenix.expression.AndExpression;
import org.apache.phoenix.expression.ByteBasedLikeExpression;
import org.apache.phoenix.expression.ComparisonExpression;
@@ -108,6 +109,7 @@ import org.apache.phoenix.schema.PColumn;
import org.apache.phoenix.schema.PLongColumn;
import org.apache.phoenix.schema.PName;
import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTableKey;
import org.apache.phoenix.schema.RowKeyValueAccessor;
import org.apache.phoenix.schema.SortOrder;
import org.apache.phoenix.schema.TableRef;
@@ -610,7 +612,7 @@ public class TestUtil {
}
public static void analyzeTable(Connection conn, String tableName) throws IOException, SQLException {
- analyzeTable(conn, tableName, false);
+ analyzeTable(conn, tableName, false);
}
public static void analyzeTable(Connection conn, String tableName, boolean transactional) throws IOException, SQLException {
@@ -652,17 +654,17 @@ public class TestUtil {
Date date = new Date(DateUtil.parseDate("2015-01-01 00:00:00").getTime() + (i - 1) * MILLIS_IN_DAY);
stmt.setDate(6, date);
}
-
+
public static void validateRowKeyColumns(ResultSet rs, int i) throws SQLException {
- assertTrue(rs.next());
- assertEquals(rs.getString(1), "varchar" + String.valueOf(i));
- assertEquals(rs.getString(2), "char" + String.valueOf(i));
- assertEquals(rs.getInt(3), i);
- assertEquals(rs.getInt(4), i);
- assertEquals(rs.getBigDecimal(5), new BigDecimal(i*0.5d));
- Date date = new Date(DateUtil.parseDate("2015-01-01 00:00:00").getTime() + (i - 1) * MILLIS_IN_DAY);
- assertEquals(rs.getDate(6), date);
- }
+ assertTrue(rs.next());
+ assertEquals(rs.getString(1), "varchar" + String.valueOf(i));
+ assertEquals(rs.getString(2), "char" + String.valueOf(i));
+ assertEquals(rs.getInt(3), i);
+ assertEquals(rs.getInt(4), i);
+ assertEquals(rs.getBigDecimal(5), new BigDecimal(i*0.5d));
+ Date date = new Date(DateUtil.parseDate("2015-01-01 00:00:00").getTime() + (i - 1) * MILLIS_IN_DAY);
+ assertEquals(rs.getDate(6), date);
+ }
public static String getTableName(Boolean mutable, Boolean transactional) {
StringBuilder tableNameBuilder = new StringBuilder(DEFAULT_DATA_TABLE_NAME);
@@ -694,7 +696,7 @@ public class TestUtil {
@Override
public SortOrder getSortOrder() {
- return SortOrder.getDefault();
+ return SortOrder.getDefault();
}
@Override
@@ -720,10 +722,14 @@ public class TestUtil {
public boolean isRowTimestamp() {
return false;
}
- @Override
- public boolean isDynamic() {
- return false;
- }
+ @Override
+ public boolean isDynamic() {
+ return false;
+ }
+ @Override
+ public byte[] getColumnQualifierBytes() {
+ return SINGLE_COLUMN_NAME.getBytes();
+ }
})), null);
aggregationManager.setAggregators(new ClientAggregators(Collections.<SingleAggregateFunction>singletonList(func), 1));
ClientAggregators aggregators = aggregationManager.getAggregators();
@@ -765,15 +771,26 @@ public class TestUtil {
// We simply write a marker row, request a major compaction, and then wait until the marker
// row is gone
+ PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class);
+ PTable table = pconn.getTable(new PTableKey(pconn.getTenantId(), tableName));
ConnectionQueryServices services = conn.unwrap(PhoenixConnection.class).getQueryServices();
- try (HTableInterface htable = services.getTable(Bytes.toBytes(tableName))) {
+ MutationState mutationState = pconn.getMutationState();
+ if (table.isTransactional()) {
+ mutationState.startTransaction();
+ }
+ try (HTableInterface htable = mutationState.getHTable(table)) {
byte[] markerRowKey = Bytes.toBytes("TO_DELETE");
-
+
Put put = new Put(markerRowKey);
- put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, HConstants.EMPTY_BYTE_ARRAY,
- HConstants.EMPTY_BYTE_ARRAY);
+ put.add(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_VALUE_BYTES, QueryConstants.EMPTY_COLUMN_VALUE_BYTES);
htable.put(put);
- htable.delete(new Delete(markerRowKey));
+ Delete delete = new Delete(markerRowKey);
+ delete.deleteColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_VALUE_BYTES);
+ htable.delete(delete);
+ htable.close();
+ if (table.isTransactional()) {
+ mutationState.commit();
+ }
HBaseAdmin hbaseAdmin = services.getAdmin();
hbaseAdmin.flush(tableName);
@@ -782,19 +799,28 @@ public class TestUtil {
boolean compactionDone = false;
while (!compactionDone) {
- Thread.sleep(2000L);
+ Thread.sleep(6000L);
Scan scan = new Scan();
scan.setStartRow(markerRowKey);
scan.setStopRow(Bytes.add(markerRowKey, new byte[] { 0 }));
scan.setRaw(true);
- ResultScanner scanner = htable.getScanner(scan);
- List<Result> results = Lists.newArrayList(scanner);
- LOG.info("Results: " + results);
- compactionDone = results.isEmpty();
- scanner.close();
-
+ try (HTableInterface htableForRawScan = services.getTable(Bytes.toBytes(tableName))) {
+ ResultScanner scanner = htableForRawScan.getScanner(scan);
+ List<Result> results = Lists.newArrayList(scanner);
+ LOG.info("Results: " + results);
+ compactionDone = results.isEmpty();
+ scanner.close();
+ }
LOG.info("Compaction done: " + compactionDone);
+
+ // need to run compaction after the next txn snapshot has been written so that compaction can remove deleted rows
+ if (!compactionDone && table.isTransactional()) {
+ hbaseAdmin = services.getAdmin();
+ hbaseAdmin.flush(tableName);
+ hbaseAdmin.majorCompact(tableName);
+ hbaseAdmin.close();
+ }
}
}
}
@@ -821,4 +847,3 @@ public class TestUtil {
}
}
-
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-protocol/src/main/PTable.proto
----------------------------------------------------------------------
diff --git a/phoenix-protocol/src/main/PTable.proto b/phoenix-protocol/src/main/PTable.proto
index a16263f..f2eb46c 100644
--- a/phoenix-protocol/src/main/PTable.proto
+++ b/phoenix-protocol/src/main/PTable.proto
@@ -47,6 +47,7 @@ message PColumn {
optional string expression = 12;
optional bool isRowTimestamp = 13;
optional bool isDynamic = 14;
+ optional bytes columnQualifierBytes = 15;
}
message PTableStats {
@@ -95,4 +96,12 @@ message PTable {
optional string autoParititonSeqName = 31;
optional bool isAppendOnlySchema = 32;
optional bytes parentNameBytes = 33;
+ optional bytes storageScheme = 34;
+ optional bytes encodingScheme = 35;
+ repeated EncodedCQCounter encodedCQCounters = 36;
+}
+
+message EncodedCQCounter {
+ required string colFamily = 1;
+ required int32 counter = 2;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/phoenix-protocol/src/main/ServerCachingService.proto
----------------------------------------------------------------------
diff --git a/phoenix-protocol/src/main/ServerCachingService.proto b/phoenix-protocol/src/main/ServerCachingService.proto
index a45b18f..044c111 100644
--- a/phoenix-protocol/src/main/ServerCachingService.proto
+++ b/phoenix-protocol/src/main/ServerCachingService.proto
@@ -30,12 +30,47 @@ message ImmutableBytesWritable {
required int32 length = 3;
}
+message ColumnReference {
+ required bytes family = 1;
+ required bytes qualifier = 2;
+}
+
+message ColumnInfo {
+ optional string familyName = 1;
+ required string columnName = 2;
+}
+
+message IndexMaintainer {
+ required int32 saltBuckets = 1;
+ required bool isMultiTenant = 2;
+ optional bytes viewIndexId = 3;
+ repeated ColumnReference indexedColumns = 4;
+ repeated int32 indexedColumnTypeOrdinal = 5;
+ repeated ColumnReference dataTableColRefForCoveredColumns = 6;
+ repeated ColumnReference indexTableColRefForCoveredColumns = 7;
+ required bool isLocalIndex = 8;
+ required bytes indexTableName = 9;
+ required bool rowKeyOrderOptimizable = 10;
+ required bytes dataTableEmptyKeyValueColFamily = 11;
+ required ImmutableBytesWritable emptyKeyValueColFamily = 12;
+ optional bytes indexedExpressions = 13;
+ required bytes rowKeyMetadata = 14;
+ required int32 numDataTableColFamilies = 15;
+ required bool indexWalDisabled = 16;
+ required int32 indexRowKeyByteSize = 17;
+ required bool immutable = 18;
+ repeated ColumnInfo indexedColumnInfo = 19;
+ required int32 encodingScheme = 20;
+ required int32 immutableStorageScheme = 21;
+}
+
message AddServerCacheRequest {
optional bytes tenantId = 1;
required bytes cacheId = 2;
required ImmutableBytesWritable cachePtr = 3;
required ServerCacheFactory cacheFactory = 4;
optional bytes txState = 5;
+ optional bool hasProtoBufIndexMaintainer = 6;
}
message AddServerCacheResponse {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/b49fc0d1/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 00c4dec..3657d8a 100644
--- a/pom.xml
+++ b/pom.xml
@@ -299,7 +299,7 @@
<forkCount>${numForkedIT}</forkCount>
<runOrder>alphabetical</runOrder>
<reuseForks>true</reuseForks>
- <argLine>-enableassertions -Xmx2000m -XX:MaxPermSize=128m -Djava.security.egd=file:/dev/./urandom "-Djava.library.path=${hadoop.library.path}${path.separator}${java.library.path}" -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=./target/</argLine>
+ <argLine>-enableassertions -Xmx2500m -XX:MaxPermSize=128m -Djava.security.egd=file:/dev/./urandom "-Djava.library.path=${hadoop.library.path}${path.separator}${java.library.path}" -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=./target/</argLine>
<redirectTestOutputToFile>${test.output.tofile}</redirectTestOutputToFile>
<testSourceDirectory>${basedir}/src/it/java</testSourceDirectory>
<groups>org.apache.phoenix.end2end.ClientManagedTimeTest</groups>