You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by el...@apache.org on 2017/06/02 22:02:42 UTC
[4/4] phoenix git commit: PHOENIX-3895 Upgrade to Apache Avatica
1.10.0
PHOENIX-3895 Upgrade to Apache Avatica 1.10.0
With the Avatica Array support, Phoenix code may see an Array
implementation which _isn't_ PhoenixArray. We can make a
best-effort to convert the provided Array into a PhoenixArray.
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/ef29d3aa
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/ef29d3aa
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/ef29d3aa
Branch: refs/heads/4.x-HBase-0.98
Commit: ef29d3aa52ba6212e53ff506638c32fa0c58f3c3
Parents: 774a31e
Author: Josh Elser <el...@apache.org>
Authored: Wed May 31 16:56:35 2017 -0400
Committer: Josh Elser <el...@apache.org>
Committed: Fri Jun 2 17:56:52 2017 -0400
----------------------------------------------------------------------
.../phoenix/schema/types/PArrayDataType.java | 32 +++-
.../apache/phoenix/schema/types/PDataType.java | 18 ++-
.../schema/types/PDataTypeForArraysTest.java | 73 +++++++++
.../phoenix/end2end/QueryServerBasicsIT.java | 155 ++++++++++++++++++-
pom.xml | 2 +-
5 files changed, 272 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/ef29d3aa/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataType.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataType.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataType.java
index f31f272..2a6d1f8 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataType.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PArrayDataType.java
@@ -20,6 +20,8 @@ package org.apache.phoenix.schema.types;
import java.io.DataOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.sql.Array;
+import java.sql.SQLException;
import java.sql.Types;
import java.text.Format;
import java.util.regex.Pattern;
@@ -86,10 +88,34 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
public byte[] toBytes(Object object, PDataType baseType, SortOrder sortOrder) {
return toBytes(object, baseType, sortOrder, true);
}
+
+ /**
+ * Ensures that the provided {@code object} is a PhoenixArray, attempting a conversion in the
+ * case when it is not.
+ */
+ PhoenixArray toPhoenixArray(Object object, PDataType baseType) {
+ if (object instanceof PhoenixArray) {
+ return (PhoenixArray) object;
+ }
+ if (!(object instanceof Array)) {
+ throw new IllegalArgumentException("Expected an Array but got " + object.getClass());
+ }
+ Array arr = (Array) object;
+ try {
+ Object untypedArrayData = arr.getArray();
+ if (!(untypedArrayData instanceof Object[])) {
+ throw new IllegalArgumentException("Array data is required to be Object[] but data for "
+ + arr.getClass() + " is " + untypedArrayData.getClass());
+ }
+ return this.getArrayFactory().newArray(baseType, (Object[]) untypedArrayData);
+ } catch (SQLException e) {
+ throw new IllegalArgumentException("Could not convert Array data", e);
+ }
+ }
public byte[] toBytes(Object object, PDataType baseType, SortOrder sortOrder, boolean rowKeyOrderOptimizable) {
if (object == null) { throw new ConstraintViolationException(this + " may not be null"); }
- PhoenixArray arr = ((PhoenixArray)object);
+ PhoenixArray arr = toPhoenixArray(object, baseType);
int noOfElements = arr.numElements;
if (noOfElements == 0) { return ByteUtil.EMPTY_BYTE_ARRAY; }
TrustedByteArrayOutputStream byteStream = null;
@@ -115,7 +141,7 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
}
DataOutputStream oStream = new DataOutputStream(byteStream);
// Handles bit inversion also
- return createArrayBytes(byteStream, oStream, (PhoenixArray)object, noOfElements, baseType, sortOrder, rowKeyOrderOptimizable);
+ return createArrayBytes(byteStream, oStream, arr, noOfElements, baseType, sortOrder, rowKeyOrderOptimizable);
}
public static int serializeNulls(DataOutputStream oStream, int nulls) throws IOException {
@@ -376,7 +402,7 @@ public abstract class PArrayDataType<T> extends PDataType<T> {
@Override
public Object toObject(Object object, PDataType actualType) {
- return object;
+ return toPhoenixArray(object, arrayBaseType(actualType));
}
public Object toObject(Object object, PDataType actualType, SortOrder sortOrder) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/ef29d3aa/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PDataType.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PDataType.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PDataType.java
index de1e63f..c3366f4 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PDataType.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/types/PDataType.java
@@ -21,6 +21,8 @@ import java.math.BigDecimal;
import java.math.BigInteger;
import java.math.MathContext;
import java.math.RoundingMode;
+import java.sql.Array;
+import java.sql.SQLException;
import java.text.Format;
import java.util.Random;
@@ -1152,9 +1154,19 @@ public abstract class PDataType<T> implements DataType<T>, Comparable<PDataType<
if (value == null) { return null; }
for (PDataType type : PDataType.values()) {
if (type.isArrayType()) {
- PhoenixArray arr = (PhoenixArray)value;
- if ((type.getSqlType() == arr.baseType.sqlType + PDataType.ARRAY_TYPE_BASE)
- && type.getJavaClass().isInstance(value)) { return type; }
+ if (value instanceof PhoenixArray) {
+ PhoenixArray arr = (PhoenixArray)value;
+ if ((type.getSqlType() == arr.baseType.sqlType + PDataType.ARRAY_TYPE_BASE)
+ && type.getJavaClass().isInstance(value)) { return type; }
+ } else {
+ Array arr = (Array) value;
+ try {
+ // Does the array's component type make sense for what we were told it is
+ if (arr.getBaseType() == type.getSqlType() - PDataType.ARRAY_TYPE_BASE) {
+ return type;
+ }
+ } catch (SQLException e) { /* Passthrough to fail */ }
+ }
} else {
if (type.getJavaClass().isInstance(value)) { return type; }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/ef29d3aa/phoenix-core/src/test/java/org/apache/phoenix/schema/types/PDataTypeForArraysTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/schema/types/PDataTypeForArraysTest.java b/phoenix-core/src/test/java/org/apache/phoenix/schema/types/PDataTypeForArraysTest.java
index 2aeeeb8..290c80f 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/schema/types/PDataTypeForArraysTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/schema/types/PDataTypeForArraysTest.java
@@ -18,9 +18,14 @@
package org.apache.phoenix.schema.types;
import java.math.BigDecimal;
+import java.sql.Array;
import java.sql.Date;
+import java.sql.ResultSet;
+import java.sql.SQLException;
import java.sql.Time;
import java.sql.Timestamp;
+import java.sql.Types;
+import java.util.Map;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.util.Bytes;
@@ -1211,4 +1216,72 @@ public class PDataTypeForArraysTest {
}
}
}
+
+ @Test
+ public void testArrayConversion() {
+ final String[] data = new String[] {"asdf", "qwerty"};
+ PhoenixArray phxArray = PArrayDataType.instantiatePhoenixArray(PVarchar.INSTANCE, data);
+ assertTrue("Converting a PhoenixArray to a PhoenixArray should return the same object",
+ phxArray == PVarcharArray.INSTANCE.toPhoenixArray(phxArray, PVarchar.INSTANCE));
+ // Create a skeleton of an Array which isn't a PhoenixArray. Make sure we can convert that.
+ Array customArray = new Array() {
+
+ @Override
+ public String getBaseTypeName() throws SQLException {
+ return "VARCHAR";
+ }
+
+ @Override
+ public int getBaseType() throws SQLException {
+ return Types.VARCHAR;
+ }
+
+ @Override
+ public Object getArray() throws SQLException {
+ return data;
+ }
+
+ @Override
+ public Object getArray(Map<String,Class<?>> map) throws SQLException {
+ return null;
+ }
+
+ @Override
+ public Object getArray(long index, int count) throws SQLException {
+ return null;
+ }
+
+ @Override
+ public Object getArray(long index, int count, Map<String,Class<?>> map)
+ throws SQLException {
+ return null;
+ }
+
+ @Override
+ public ResultSet getResultSet() throws SQLException {
+ return null;
+ }
+
+ @Override
+ public ResultSet getResultSet(Map<String,Class<?>> map) throws SQLException {
+ return null;
+ }
+
+ @Override
+ public ResultSet getResultSet(long index, int count) throws SQLException {
+ return null;
+ }
+
+ @Override
+ public ResultSet getResultSet(long index, int count, Map<String,Class<?>> map)
+ throws SQLException {
+ return null;
+ }
+
+ @Override public void free() throws SQLException {}
+ };
+
+ PhoenixArray copy = PVarcharArray.INSTANCE.toPhoenixArray(customArray, PVarchar.INSTANCE);
+ assertEquals(phxArray, copy);
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/ef29d3aa/phoenix-queryserver/src/it/java/org/apache/phoenix/end2end/QueryServerBasicsIT.java
----------------------------------------------------------------------
diff --git a/phoenix-queryserver/src/it/java/org/apache/phoenix/end2end/QueryServerBasicsIT.java b/phoenix-queryserver/src/it/java/org/apache/phoenix/end2end/QueryServerBasicsIT.java
index 219a0a8..ca4cf0b 100644
--- a/phoenix-queryserver/src/it/java/org/apache/phoenix/end2end/QueryServerBasicsIT.java
+++ b/phoenix-queryserver/src/it/java/org/apache/phoenix/end2end/QueryServerBasicsIT.java
@@ -29,6 +29,7 @@ import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.sql.Array;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
@@ -45,7 +46,9 @@ import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.queryserver.client.ThinClientUtil;
import org.junit.AfterClass;
import org.junit.BeforeClass;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.TestName;
/**
* Smoke test for query server.
@@ -58,6 +61,9 @@ public class QueryServerBasicsIT extends BaseHBaseManagedTimeIT {
private static Configuration CONF;
private static String CONN_STRING;
+ @Rule
+ public TestName name = new TestName();
+
@BeforeClass
public static void beforeClass() throws Exception {
CONF = getTestClusterConfig();
@@ -123,7 +129,7 @@ public class QueryServerBasicsIT extends BaseHBaseManagedTimeIT {
@Test
public void smokeTest() throws Exception {
- final String tableName = getClass().getSimpleName().toUpperCase() + System.currentTimeMillis();
+ final String tableName = name.getMethodName();
try (final Connection connection = DriverManager.getConnection(CONN_STRING)) {
assertThat(connection.isClosed(), is(false));
connection.setAutoCommit(true);
@@ -162,4 +168,151 @@ public class QueryServerBasicsIT extends BaseHBaseManagedTimeIT {
}
}
}
+
+ @Test
+ public void arrayTest() throws Exception {
+ final String tableName = name.getMethodName();
+ try (Connection conn = DriverManager.getConnection(CONN_STRING);
+ Statement stmt = conn.createStatement()) {
+ conn.setAutoCommit(false);
+ assertFalse(stmt.execute("DROP TABLE IF EXISTS " + tableName));
+ assertFalse(stmt.execute("CREATE TABLE " + tableName + " ("
+ + "pk VARCHAR NOT NULL PRIMARY KEY, "
+ + "histogram INTEGER[])")
+ );
+ conn.commit();
+ int numRows = 10;
+ int numEvenElements = 4;
+ int numOddElements = 6;
+ for (int i = 0; i < numRows; i++) {
+ int arrayLength = i % 2 == 0 ? numEvenElements : numOddElements;
+ StringBuilder sb = new StringBuilder();
+ for (int arrayOffset = 0; arrayOffset < arrayLength; arrayOffset++) {
+ if (sb.length() > 0) {
+ sb.append(", ");
+ }
+ sb.append(getArrayValueForOffset(arrayOffset));
+ }
+ String updateSql = "UPSERT INTO " + tableName + " values('" + i + "', " + "ARRAY[" + sb.toString() + "])";
+ assertEquals(1, stmt.executeUpdate(updateSql));
+ }
+ conn.commit();
+ try (ResultSet rs = stmt.executeQuery("SELECT * FROM " + tableName)) {
+ for (int i = 0; i < numRows; i++) {
+ assertTrue(rs.next());
+ assertEquals(i, Integer.parseInt(rs.getString(1)));
+ Array array = rs.getArray(2);
+ Object untypedArrayData = array.getArray();
+ assertTrue("Expected array data to be an int array, but was " + untypedArrayData.getClass(), untypedArrayData instanceof Object[]);
+ Object[] arrayData = (Object[]) untypedArrayData;
+ int expectedArrayLength = i % 2 == 0 ? numEvenElements : numOddElements;
+ assertEquals(expectedArrayLength, arrayData.length);
+ for (int arrayOffset = 0; arrayOffset < expectedArrayLength; arrayOffset++) {
+ assertEquals(getArrayValueForOffset(arrayOffset), arrayData[arrayOffset]);
+ }
+ }
+ assertFalse(rs.next());
+ }
+ }
+ }
+
+ @Test
+ public void preparedStatementArrayTest() throws Exception {
+ final String tableName = name.getMethodName();
+ try (Connection conn = DriverManager.getConnection(CONN_STRING);
+ Statement stmt = conn.createStatement()) {
+ conn.setAutoCommit(false);
+ assertFalse(stmt.execute("DROP TABLE IF EXISTS " + tableName));
+ assertFalse(stmt.execute("CREATE TABLE " + tableName + " ("
+ + "pk VARCHAR NOT NULL PRIMARY KEY, "
+ + "histogram INTEGER[])")
+ );
+ conn.commit();
+ int numRows = 10;
+ int numEvenElements = 4;
+ int numOddElements = 6;
+ try (PreparedStatement pstmt = conn.prepareStatement("UPSERT INTO " + tableName + " values(?, ?)")) {
+ for (int i = 0; i < numRows; i++) {
+ pstmt.setString(1, Integer.toString(i));
+ int arrayLength = i % 2 == 0 ? numEvenElements : numOddElements;
+ Object[] arrayData = new Object[arrayLength];
+ for (int arrayOffset = 0; arrayOffset < arrayLength; arrayOffset++) {
+ arrayData[arrayOffset] = getArrayValueForOffset(arrayOffset);
+ }
+ pstmt.setArray(2, conn.createArrayOf("INTEGER", arrayData));
+ assertEquals(1, pstmt.executeUpdate());
+ }
+ conn.commit();
+ }
+ conn.commit();
+ try (ResultSet rs = stmt.executeQuery("SELECT * FROM " + tableName)) {
+ for (int i = 0; i < numRows; i++) {
+ assertTrue(rs.next());
+ assertEquals(i, Integer.parseInt(rs.getString(1)));
+ Array array = rs.getArray(2);
+ Object untypedArrayData = array.getArray();
+ assertTrue("Expected array data to be an int array, but was " + untypedArrayData.getClass(), untypedArrayData instanceof Object[]);
+ Object[] arrayData = (Object[]) untypedArrayData;
+ int expectedArrayLength = i % 2 == 0 ? numEvenElements : numOddElements;
+ assertEquals(expectedArrayLength, arrayData.length);
+ for (int arrayOffset = 0; arrayOffset < expectedArrayLength; arrayOffset++) {
+ assertEquals(getArrayValueForOffset(arrayOffset), arrayData[arrayOffset]);
+ }
+ }
+ assertFalse(rs.next());
+ }
+ }
+ }
+
+ @Test
+ public void preparedStatementVarcharArrayTest() throws Exception {
+ final String tableName = name.getMethodName();
+ try (Connection conn = DriverManager.getConnection(CONN_STRING);
+ Statement stmt = conn.createStatement()) {
+ conn.setAutoCommit(false);
+ assertFalse(stmt.execute("DROP TABLE IF EXISTS " + tableName));
+ assertFalse(stmt.execute("CREATE TABLE " + tableName + " ("
+ + "pk VARCHAR NOT NULL PRIMARY KEY, "
+ + "histogram VARCHAR[])")
+ );
+ conn.commit();
+ int numRows = 10;
+ int numEvenElements = 4;
+ int numOddElements = 6;
+ try (PreparedStatement pstmt = conn.prepareStatement("UPSERT INTO " + tableName + " values(?, ?)")) {
+ for (int i = 0; i < numRows; i++) {
+ pstmt.setString(1, Integer.toString(i));
+ int arrayLength = i % 2 == 0 ? numEvenElements : numOddElements;
+ Object[] arrayData = new Object[arrayLength];
+ for (int arrayOffset = 0; arrayOffset < arrayLength; arrayOffset++) {
+ arrayData[arrayOffset] = Integer.toString(getArrayValueForOffset(arrayOffset));
+ }
+ pstmt.setArray(2, conn.createArrayOf("VARCHAR", arrayData));
+ assertEquals(1, pstmt.executeUpdate());
+ }
+ conn.commit();
+ }
+ conn.commit();
+ try (ResultSet rs = stmt.executeQuery("SELECT * FROM " + tableName)) {
+ for (int i = 0; i < numRows; i++) {
+ assertTrue(rs.next());
+ assertEquals(i, Integer.parseInt(rs.getString(1)));
+ Array array = rs.getArray(2);
+ Object untypedArrayData = array.getArray();
+ assertTrue("Expected array data to be an int array, but was " + untypedArrayData.getClass(), untypedArrayData instanceof Object[]);
+ Object[] arrayData = (Object[]) untypedArrayData;
+ int expectedArrayLength = i % 2 == 0 ? numEvenElements : numOddElements;
+ assertEquals(expectedArrayLength, arrayData.length);
+ for (int arrayOffset = 0; arrayOffset < expectedArrayLength; arrayOffset++) {
+ assertEquals(Integer.toString(getArrayValueForOffset(arrayOffset)), arrayData[arrayOffset]);
+ }
+ }
+ assertFalse(rs.next());
+ }
+ }
+ }
+
+ private int getArrayValueForOffset(int arrayOffset) {
+ return arrayOffset * 2 + 1;
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/ef29d3aa/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 65e1301..5e746b3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -98,7 +98,7 @@
<!-- Do not change jodatime.version until HBASE-15199 is fixed -->
<jodatime.version>1.6</jodatime.version>
<joni.version>2.1.2</joni.version>
- <avatica.version>1.9.0</avatica.version>
+ <avatica.version>1.10.0</avatica.version>
<jettyVersion>8.1.7.v20120910</jettyVersion>
<tephra.version>0.12.0-incubating</tephra.version>
<spark.version>2.0.2</spark.version>