You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by go...@apache.org on 2020/11/16 17:19:52 UTC
[phoenix] branch 4.x-PHOENIX-5923 updated: PHOENIX-6220 CREATE
INDEX shouldn't ignore IMMUTABLE_STORAGE_SCHEME and COLUMN_ENDCODED_BYTES
This is an automated email from the ASF dual-hosted git repository.
gokcen pushed a commit to branch 4.x-PHOENIX-5923
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/4.x-PHOENIX-5923 by this push:
new a2cd0a1 PHOENIX-6220 CREATE INDEX shouldn't ignore IMMUTABLE_STORAGE_SCHEME and COLUMN_ENDCODED_BYTES
a2cd0a1 is described below
commit a2cd0a13ae5b11a13306e7f6d023559bc3f3e49c
Author: Gokcen Iskender <gi...@salesforce.com>
AuthorDate: Wed Nov 11 16:37:39 2020 -0800
PHOENIX-6220 CREATE INDEX shouldn't ignore IMMUTABLE_STORAGE_SCHEME and COLUMN_ENDCODED_BYTES
Signed-off-by: Gokcen Iskender <gi...@salesforce.com>
---
.../org/apache/phoenix/end2end/CreateTableIT.java | 91 ++++++++++++++++++
.../org/apache/phoenix/schema/MetaDataClient.java | 104 +++++++++++++++------
2 files changed, 167 insertions(+), 28 deletions(-)
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
index 4f6ccab..c92be7c 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
@@ -622,6 +622,97 @@ public class CreateTableIT extends ParallelStatsDisabledIT {
}
}
+ @Test
+ public void testCreateIndexWithDifferentStorageAndEncoding() throws Exception {
+ verifyIndexSchemeChange(false, false);
+ verifyIndexSchemeChange(false, true);
+ verifyIndexSchemeChange(true, false);
+ verifyIndexSchemeChange(true, true);
+
+ String tableName = generateUniqueName();
+ String indexName = generateUniqueName();
+ String createTableDDL = "create IMMUTABLE TABLE " + tableName + "(id char(1) NOT NULL, col1 char(1), col2 char(1) "
+ + "CONSTRAINT NAME_PK PRIMARY KEY (id)) IMMUTABLE_STORAGE_SCHEME=SINGLE_CELL_ARRAY_WITH_OFFSETS";
+ String createIndexDDL = "create INDEX " + indexName + " ON " + tableName + " (col1) INCLUDE (col2) IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN";
+
+ Properties props = PropertiesUtil.deepCopy(TestUtil.TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ conn.createStatement().execute(createTableDDL);
+ assertColumnEncodingMetadata(QualifierEncodingScheme.TWO_BYTE_QUALIFIERS,
+ ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS, tableName, conn);
+
+ boolean failed = false;
+ try {
+ conn.createStatement().execute(createIndexDDL);
+ } catch (SQLException e) {
+ assertEquals(e.getErrorCode(), SQLExceptionCode.INVALID_IMMUTABLE_STORAGE_SCHEME_CHANGE.getErrorCode());
+ failed = true;
+ }
+ assertEquals(true, failed);
+ }
+ }
+
+ private void verifyIndexSchemeChange(boolean immutable, boolean multiTenant) throws Exception{
+ Properties props = PropertiesUtil.deepCopy(TestUtil.TEST_PROPERTIES);
+ String nonEncodedOneCellPerColumnTable = generateUniqueName();
+ String createTableDDL;
+ String createIndexDDL;
+ String tableName= "[TABLE_NAME]";
+ String indexName= "[INDEX_NAME]";
+ String createTableBaseDDL = "create " + (immutable? " IMMUTABLE ":"") + " TABLE [TABLE_NAME] ("
+ + " id char(1) NOT NULL," + " col1 integer NOT NULL,"
+ + " col2 bigint NOT NULL,"
+ + " CONSTRAINT NAME_PK PRIMARY KEY (id, col1, col2)) MULTI_TENANT=" + (multiTenant? "true,":"false,");
+
+ String createIndexBaseDDL = "create index [INDEX_NAME] ON [TABLE_NAME] (col1) INCLUDE (col2) ";
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ createTableDDL = createTableBaseDDL.replace(tableName, nonEncodedOneCellPerColumnTable);
+ createTableDDL += "IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, COLUMN_ENCODED_BYTES=0";
+ conn.createStatement().execute(createTableDDL);
+ assertColumnEncodingMetadata(QualifierEncodingScheme.NON_ENCODED_QUALIFIERS,
+ ImmutableStorageScheme.ONE_CELL_PER_COLUMN,
+ nonEncodedOneCellPerColumnTable, conn);
+
+ String idxName = "IDX_" + generateUniqueName();
+ // Don't specify anything to see if it inherits from parent
+ createIndexDDL = createIndexBaseDDL.replace(indexName, idxName).replace(tableName, nonEncodedOneCellPerColumnTable);
+ conn.createStatement().execute(createIndexDDL);
+ assertColumnEncodingMetadata(QualifierEncodingScheme.NON_ENCODED_QUALIFIERS,
+ ImmutableStorageScheme.ONE_CELL_PER_COLUMN,
+ idxName, conn);
+
+ idxName = "IDX_" + generateUniqueName();
+ createIndexDDL = createIndexBaseDDL.replace(indexName, idxName).replace(tableName, nonEncodedOneCellPerColumnTable);
+ createIndexDDL += "IMMUTABLE_STORAGE_SCHEME=SINGLE_CELL_ARRAY_WITH_OFFSETS";
+ conn.createStatement().execute(createIndexDDL);
+ // Check if it sets the encoding to 2
+ assertColumnEncodingMetadata(QualifierEncodingScheme.TWO_BYTE_QUALIFIERS,
+ ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS,
+ idxName, conn);
+
+ idxName = "IDX_" + generateUniqueName();
+ createIndexDDL = createIndexBaseDDL.replace(indexName, idxName).replace(tableName, nonEncodedOneCellPerColumnTable);
+ createIndexDDL += "IMMUTABLE_STORAGE_SCHEME=SINGLE_CELL_ARRAY_WITH_OFFSETS, COLUMN_ENCODED_BYTES=3";
+ conn.createStatement().execute(createIndexDDL);
+ assertColumnEncodingMetadata(QualifierEncodingScheme.THREE_BYTE_QUALIFIERS,
+ ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS,
+ idxName, conn);
+
+ createIndexDDL = createIndexBaseDDL.replace(indexName, idxName).replace(tableName, nonEncodedOneCellPerColumnTable);
+ createIndexDDL += "IMMUTABLE_STORAGE_SCHEME=SINGLE_CELL_ARRAY_WITH_OFFSETS, COLUMN_ENCODED_BYTES=0";
+ // should fail
+ boolean failed = false;
+ try {
+ conn.createStatement().execute(createIndexDDL);
+ } catch (SQLException e) {
+ failed = true;
+ assertEquals(SQLExceptionCode.INVALID_IMMUTABLE_STORAGE_SCHEME_AND_COLUMN_QUALIFIER_BYTES.getErrorCode(),e.getErrorCode());
+ }
+ assertEquals(true, failed);
+ }
+ }
+
private void verifyUCFValueInSysCat(String tableName, String createTableString,
Properties props, long expectedUCFInSysCat) throws SQLException {
String readSysCatQuery = "SELECT TABLE_NAME, UPDATE_CACHE_FREQUENCY FROM SYSTEM.CATALOG "
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index 0a45981..86a5cc1 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -2534,28 +2534,46 @@ public class MetaDataClient {
*
*/
if (parent != null) {
- encodingScheme = parent.getEncodingScheme();
- immutableStorageScheme = parent.getImmutableStorageScheme();
- } else {
- Byte encodingSchemeSerializedByte = (Byte) TableProperty.COLUMN_ENCODED_BYTES.getValue(tableProps);
- if (encodingSchemeSerializedByte == null) {
- // Ignore default if transactional and column encoding is not supported (as with OMID)
- if (transactionProvider == null || !transactionProvider.getTransactionProvider().isUnsupported(PhoenixTransactionProvider.Feature.COLUMN_ENCODING) ) {
- encodingSchemeSerializedByte = (byte)connection.getQueryServices().getProps().getInt(QueryServices.DEFAULT_COLUMN_ENCODED_BYTES_ATRRIB,
- QueryServicesOptions.DEFAULT_COLUMN_ENCODED_BYTES);
- encodingScheme = QualifierEncodingScheme.fromSerializedValue(encodingSchemeSerializedByte);
- }
+ Byte encodingSchemeSerializedByte = (Byte) TableProperty.COLUMN_ENCODED_BYTES.getValue(tableProps);
+ // Table has encoding scheme defined
+ if (encodingSchemeSerializedByte != null) {
+ encodingScheme = getEncodingScheme(tableProps, schemaName, tableName, transactionProvider);
} else {
- encodingScheme = QualifierEncodingScheme.fromSerializedValue(encodingSchemeSerializedByte);
- if (encodingScheme != NON_ENCODED_QUALIFIERS && transactionProvider != null && transactionProvider.getTransactionProvider().isUnsupported(PhoenixTransactionProvider.Feature.COLUMN_ENCODING) ) {
- throw new SQLExceptionInfo.Builder(
- SQLExceptionCode.UNSUPPORTED_COLUMN_ENCODING_FOR_TXN_PROVIDER)
- .setSchemaName(schemaName).setTableName(tableName)
- .setMessage(transactionProvider.name())
- .build()
- .buildException();
+ encodingScheme = parent.getEncodingScheme();
+ }
+
+ ImmutableStorageScheme immutableStorageSchemeProp = (ImmutableStorageScheme) TableProperty.IMMUTABLE_STORAGE_SCHEME.getValue(tableProps);
+ if (immutableStorageSchemeProp == null) {
+ immutableStorageScheme = parent.getImmutableStorageScheme();
+ } else {
+ immutableStorageScheme = getImmutableStorageScheme(immutableStorageSchemeProp, schemaName, tableName, transactionProvider);
+ }
+
+ if (immutableStorageScheme == SINGLE_CELL_ARRAY_WITH_OFFSETS) {
+ if (encodingScheme == NON_ENCODED_QUALIFIERS) {
+ if (encodingSchemeSerializedByte != null) {
+ // encoding scheme is set as non-encoded on purpose, so we should fail
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_IMMUTABLE_STORAGE_SCHEME_AND_COLUMN_QUALIFIER_BYTES)
+ .setSchemaName(schemaName).setTableName(tableName).build().buildException();
+ } else {
+ // encoding scheme is inherited from parent but it is not compatible with Single Cell.
+ encodingScheme =
+ QualifierEncodingScheme.fromSerializedValue(
+ (byte) QueryServicesOptions.DEFAULT_COLUMN_ENCODED_BYTES);
+ }
}
}
+
+ if (parent.getImmutableStorageScheme() == SINGLE_CELL_ARRAY_WITH_OFFSETS && immutableStorageScheme == ONE_CELL_PER_COLUMN) {
+ throw new SQLExceptionInfo.Builder(
+ SQLExceptionCode.INVALID_IMMUTABLE_STORAGE_SCHEME_CHANGE)
+ .setSchemaName(schemaName).setTableName(tableName).build()
+ .buildException();
+ }
+ LOGGER.info(String.format("STORAGE--ENCODING: %s--%s", immutableStorageScheme, encodingScheme));
+ } else {
+ encodingScheme = getEncodingScheme(tableProps, schemaName, tableName, transactionProvider);
+
if (isImmutableRows) {
ImmutableStorageScheme immutableStorageSchemeProp =
(ImmutableStorageScheme) TableProperty.IMMUTABLE_STORAGE_SCHEME
@@ -2586,15 +2604,7 @@ public class MetaDataClient {
}
}
} else {
- immutableStorageScheme = immutableStorageSchemeProp;
- if (immutableStorageScheme != ONE_CELL_PER_COLUMN && transactionProvider != null && transactionProvider.getTransactionProvider().isUnsupported(PhoenixTransactionProvider.Feature.COLUMN_ENCODING) ) {
- throw new SQLExceptionInfo.Builder(
- SQLExceptionCode.UNSUPPORTED_STORAGE_FORMAT_FOR_TXN_PROVIDER)
- .setSchemaName(schemaName).setTableName(tableName)
- .setMessage(transactionProvider.name())
- .build()
- .buildException();
- }
+ immutableStorageScheme = getImmutableStorageScheme(immutableStorageSchemeProp, schemaName, tableName, transactionProvider);
}
if (immutableStorageScheme != ONE_CELL_PER_COLUMN
&& encodingScheme == NON_ENCODED_QUALIFIERS) {
@@ -3133,6 +3143,44 @@ public class MetaDataClient {
}
}
+ private QualifierEncodingScheme getEncodingScheme(Map<String, Object> tableProps, String schemaName, String tableName, TransactionFactory.Provider transactionProvider)
+ throws SQLException {
+ QualifierEncodingScheme encodingScheme = null;
+ Byte encodingSchemeSerializedByte = (Byte) TableProperty.COLUMN_ENCODED_BYTES.getValue(tableProps);
+ if (encodingSchemeSerializedByte == null) {
+ // Ignore default if transactional and column encoding is not supported (as with OMID)
+ if (transactionProvider == null || !transactionProvider.getTransactionProvider().isUnsupported(PhoenixTransactionProvider.Feature.COLUMN_ENCODING) ) {
+ encodingSchemeSerializedByte = (byte)connection.getQueryServices().getProps().getInt(QueryServices.DEFAULT_COLUMN_ENCODED_BYTES_ATRRIB,
+ QueryServicesOptions.DEFAULT_COLUMN_ENCODED_BYTES);
+ encodingScheme = QualifierEncodingScheme.fromSerializedValue(encodingSchemeSerializedByte);
+ } else {
+ encodingScheme = NON_ENCODED_QUALIFIERS;
+ }
+ } else {
+ encodingScheme = QualifierEncodingScheme.fromSerializedValue(encodingSchemeSerializedByte);
+ if (encodingScheme != NON_ENCODED_QUALIFIERS && transactionProvider != null && transactionProvider.getTransactionProvider()
+ .isUnsupported(PhoenixTransactionProvider.Feature.COLUMN_ENCODING)) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.UNSUPPORTED_COLUMN_ENCODING_FOR_TXN_PROVIDER)
+ .setSchemaName(schemaName).setTableName(tableName).setMessage(transactionProvider.name()).build().buildException();
+ }
+ }
+
+ return encodingScheme;
+ }
+
+ private ImmutableStorageScheme getImmutableStorageScheme(ImmutableStorageScheme immutableStorageSchemeProp, String schemaName, String tableName, TransactionFactory.Provider transactionProvider)
+ throws SQLException {
+ if (immutableStorageSchemeProp != ONE_CELL_PER_COLUMN && transactionProvider != null && transactionProvider.getTransactionProvider().isUnsupported(PhoenixTransactionProvider.Feature.COLUMN_ENCODING) ) {
+ throw new SQLExceptionInfo.Builder(
+ SQLExceptionCode.UNSUPPORTED_STORAGE_FORMAT_FOR_TXN_PROVIDER)
+ .setSchemaName(schemaName).setTableName(tableName)
+ .setMessage(transactionProvider.name())
+ .build()
+ .buildException();
+ }
+ return immutableStorageSchemeProp;
+ }
+
/* This method handles mutation codes sent by phoenix server, except for TABLE_NOT_FOUND which
* is considered to be a success code. If TABLE_ALREADY_EXISTS in hbase, we don't need to add
* it in ConnectionQueryServices and we return result as true. However if code is