You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by st...@apache.org on 2023/01/30 14:30:04 UTC
[phoenix] branch master updated: PHOENIX-6720 CREATE TABLE can't recreate column encoded tables that had columns dropped
This is an automated email from the ASF dual-hosted git repository.
stoty pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/master by this push:
new 624e96be4b PHOENIX-6720 CREATE TABLE can't recreate column encoded tables that had columns dropped
624e96be4b is described below
commit 624e96be4b7d5182bfbc7a2ce2fec1beb3eba102
Author: Aron Meszaros <am...@cloudera.com>
AuthorDate: Fri Sep 23 12:36:03 2022 +0200
PHOENIX-6720 CREATE TABLE can't recreate column encoded tables that had columns dropped
---
.../org/apache/phoenix/end2end/AlterTableIT.java | 41 ++++
.../org/apache/phoenix/end2end/CreateTableIT.java | 273 +++++++++++++++++++++
.../apache/phoenix/end2end/ShowCreateTableIT.java | 173 ++++++++++++-
.../transform/TransformMonitorExtendedIT.java | 155 ++++++++++++
.../schema/tool/SchemaToolExtractionIT.java | 14 +-
phoenix-core/src/main/antlr3/PhoenixSQL.g | 21 +-
.../apache/phoenix/exception/SQLExceptionCode.java | 8 +
.../org/apache/phoenix/jdbc/PhoenixStatement.java | 16 +-
.../java/org/apache/phoenix/parse/ColumnDef.java | 15 +-
.../apache/phoenix/parse/CreateTableStatement.java | 15 +-
.../org/apache/phoenix/parse/ParseNodeFactory.java | 24 +-
.../org/apache/phoenix/schema/MetaDataClient.java | 79 +++++-
.../schema/tool/SchemaExtractionProcessor.java | 54 +++-
13 files changed, 849 insertions(+), 39 deletions(-)
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java
index 79ec0dca05..ac24c5b54f 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AlterTableIT.java
@@ -32,6 +32,7 @@ import static org.apache.phoenix.util.TestUtil.closeConnection;
import static org.apache.phoenix.util.TestUtil.closeStatement;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@@ -1811,4 +1812,44 @@ public class AlterTableIT extends ParallelStatsDisabledIT {
}
}
+ @Test
+ public void testAlterTableWithColumnQualifiers() throws Exception {
+ Properties props = new Properties();
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ String ddl = "CREATE TABLE \"" + tableName + "\"(K VARCHAR NOT NULL PRIMARY KEY, " +
+ "INT INTEGER ENCODED_QUALIFIER 11, INT2 INTEGER ENCODED_QUALIFIER 12, " +
+ "INT3 INTEGER ENCODED_QUALIFIER 14) " +
+ generateDDLOptions("IMMUTABLE_ROWS = true"
+ + (!columnEncoded ? ",IMMUTABLE_STORAGE_SCHEME=" + PTable.ImmutableStorageScheme.ONE_CELL_PER_COLUMN : "")) +
+ " COLUMN_QUALIFIER_COUNTER ('" + QueryConstants.DEFAULT_COLUMN_FAMILY + "'=15)";
+ conn.createStatement().execute(ddl);
+
+ String addDdl = "ALTER TABLE \"" + tableName + "\" ADD CHAR1 char(10)";
+ conn.createStatement().execute(addDdl);
+
+ PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class);
+ PTable table = pconn.getTable(new PTableKey(null, tableName));
+
+ QualifierEncodingScheme encodingScheme = table.getEncodingScheme();
+ if (columnEncoded) {
+ assertNotEquals(PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, encodingScheme);
+ } else {
+ assertEquals(PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, encodingScheme);
+ }
+
+ PTable.EncodedCQCounter cqCounter = table.getEncodedCQCounter();
+ assertEquals(columnEncoded ? 16 : null, cqCounter.getNextQualifier(QueryConstants.DEFAULT_COLUMN_FAMILY));
+ if (columnEncoded) {
+ assertEquals(11, encodingScheme.decode(table.getColumnForColumnName("INT")
+ .getColumnQualifierBytes()));
+ assertEquals(12, encodingScheme.decode(table.getColumnForColumnName("INT2")
+ .getColumnQualifierBytes()));
+ assertEquals(14, encodingScheme.decode(table.getColumnForColumnName("INT3")
+ .getColumnQualifierBytes()));
+ assertEquals(15, encodingScheme.decode(table.getColumnForColumnName("CHAR1")
+ .getColumnQualifierBytes()));
+ }
+ }
+
}
\ No newline at end of file
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
index 8fa3a5958a..9b76ac7452 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
@@ -53,6 +53,7 @@ import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.jdbc.PhoenixStatement;
import org.apache.phoenix.query.BaseTest;
import org.apache.phoenix.query.KeyRange;
+import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.ColumnAlreadyExistsException;
@@ -1301,6 +1302,278 @@ public class CreateTableIT extends ParallelStatsDisabledIT {
}
}
+ @Test
+ public void testCreateTableWithColumnQualifiers() throws Exception {
+ Properties props = new Properties();
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ String ddl = "CREATE TABLE \"" + tableName + "\"(K VARCHAR NOT NULL PRIMARY KEY, " +
+ "INT INTEGER ENCODED_QUALIFIER 11, INT2 INTEGER ENCODED_QUALIFIER 12, " +
+ "INT3 INTEGER ENCODED_QUALIFIER 14) COLUMN_QUALIFIER_COUNTER ('"
+ + QueryConstants.DEFAULT_COLUMN_FAMILY + "'=15)";
+ conn.createStatement().execute(ddl);
+ PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class);
+ PTable table = pconn.getTable(new PTableKey(null, tableName));
+
+ QualifierEncodingScheme encodingScheme = table.getEncodingScheme();
+ assertNotEquals(PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, encodingScheme);
+
+ PTable.EncodedCQCounter cqCounter = table.getEncodedCQCounter();
+ assertEquals(15, cqCounter.values().get(QueryConstants.DEFAULT_COLUMN_FAMILY).intValue());
+ assertEquals(11, encodingScheme.decode(table.getColumnForColumnName("INT")
+ .getColumnQualifierBytes()));
+ assertEquals(12, encodingScheme.decode(table.getColumnForColumnName("INT2")
+ .getColumnQualifierBytes()));
+ assertEquals(14, encodingScheme.decode(table.getColumnForColumnName("INT3")
+ .getColumnQualifierBytes()));
+ }
+
+ @Test
+ public void testCreateTableWithNotOrderedColumnQualifiers() throws Exception {
+ Properties props = new Properties();
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ String ddl = "CREATE TABLE \"" + tableName + "\"(K VARCHAR NOT NULL PRIMARY KEY, " +
+ "INT3 INTEGER ENCODED_QUALIFIER 14, INT INTEGER ENCODED_QUALIFIER 11, " +
+ "INT2 INTEGER ENCODED_QUALIFIER 12) COLUMN_QUALIFIER_COUNTER ('"
+ + QueryConstants.DEFAULT_COLUMN_FAMILY + "'=15)";
+ conn.createStatement().execute(ddl);
+ PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class);
+ PTable table = pconn.getTable(new PTableKey(null, tableName));
+
+ QualifierEncodingScheme encodingScheme = table.getEncodingScheme();
+ assertNotEquals(PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, encodingScheme);
+
+ PTable.EncodedCQCounter cqCounter = table.getEncodedCQCounter();
+ assertEquals(15, cqCounter.values().get(QueryConstants.DEFAULT_COLUMN_FAMILY).intValue());
+ assertEquals(11, encodingScheme.decode(table.getColumnForColumnName("INT")
+ .getColumnQualifierBytes()));
+ assertEquals(12, encodingScheme.decode(table.getColumnForColumnName("INT2")
+ .getColumnQualifierBytes()));
+ assertEquals(14, encodingScheme.decode(table.getColumnForColumnName("INT3")
+ .getColumnQualifierBytes()));
+ }
+
+ @Test
+ public void testCreateTableWithColumnQualifiersWithoutCounter() throws Exception {
+ Properties props = new Properties();
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ String ddl = "CREATE TABLE \"" + tableName + "\"(K VARCHAR NOT NULL PRIMARY KEY, " +
+ "INT INTEGER ENCODED_QUALIFIER 11, INT2 INTEGER ENCODED_QUALIFIER 12, " +
+ "INT3 INTEGER ENCODED_QUALIFIER 14)";
+ conn.createStatement().execute(ddl);
+ PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class);
+ PTable table = pconn.getTable(new PTableKey(null, tableName));
+
+ QualifierEncodingScheme encodingScheme = table.getEncodingScheme();
+ assertNotEquals(PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, encodingScheme);
+
+ PTable.EncodedCQCounter cqCounter = table.getEncodedCQCounter();
+ assertEquals(15, cqCounter.values().get(QueryConstants.DEFAULT_COLUMN_FAMILY).intValue());
+ assertEquals(11, encodingScheme.decode(table.getColumnForColumnName("INT")
+ .getColumnQualifierBytes()));
+ assertEquals(12, encodingScheme.decode(table.getColumnForColumnName("INT2")
+ .getColumnQualifierBytes()));
+ assertEquals(14, encodingScheme.decode(table.getColumnForColumnName("INT3")
+ .getColumnQualifierBytes()));
+ }
+
+ @Test
+ public void testCreateTableWithColumnQualifiersMultipleFamilies() throws Exception {
+ Properties props = new Properties();
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ String ddl = "CREATE IMMUTABLE TABLE \"" + tableName + "\"(K VARCHAR NOT NULL PRIMARY KEY, " +
+ "A.INT INTEGER ENCODED_QUALIFIER 11, A.INT2 INTEGER ENCODED_QUALIFIER 13, " +
+ "B.INT3 INTEGER ENCODED_QUALIFIER 12) " +
+ "COLUMN_QUALIFIER_COUNTER ('A'=14, 'B'=13)";
+ conn.createStatement().execute(ddl);
+ PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class);
+ PTable table = pconn.getTable(new PTableKey(null, tableName));
+
+ QualifierEncodingScheme encodingScheme = table.getEncodingScheme();
+ assertNotEquals(PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, encodingScheme);
+
+ PTable.EncodedCQCounter cqCounter = table.getEncodedCQCounter();
+ assertEquals(14, cqCounter.values().get("A").intValue());
+ assertEquals(13, cqCounter.values().get("B").intValue());
+ assertEquals(11, encodingScheme.decode(table.getColumnForColumnName("INT")
+ .getColumnQualifierBytes()));
+ assertEquals(13, encodingScheme.decode(table.getColumnForColumnName("INT2")
+ .getColumnQualifierBytes()));
+ assertEquals(12, encodingScheme.decode(table.getColumnForColumnName("INT3")
+ .getColumnQualifierBytes()));
+ }
+
+ @Test
+ public void testCreateTableWithColumnQualifiersMultipleFamiliesWithoutCounter() throws Exception {
+ Properties props = new Properties();
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ String ddl = "CREATE IMMUTABLE TABLE \"" + tableName + "\"(K VARCHAR NOT NULL PRIMARY KEY, " +
+ "A.INT INTEGER ENCODED_QUALIFIER 11, A.INT2 INTEGER ENCODED_QUALIFIER 13, " +
+ "B.INT3 INTEGER ENCODED_QUALIFIER 12)";
+ conn.createStatement().execute(ddl);
+ PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class);
+ PTable table = pconn.getTable(new PTableKey(null, tableName));
+
+ QualifierEncodingScheme encodingScheme = table.getEncodingScheme();
+ assertNotEquals(PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, encodingScheme);
+
+ PTable.EncodedCQCounter cqCounter = table.getEncodedCQCounter();
+ assertEquals(14, cqCounter.values().get("A").intValue());
+ assertEquals(13, cqCounter.values().get("B").intValue());
+ assertEquals(11, encodingScheme.decode(table.getColumnForColumnName("INT")
+ .getColumnQualifierBytes()));
+ assertEquals(13, encodingScheme.decode(table.getColumnForColumnName("INT2")
+ .getColumnQualifierBytes()));
+ assertEquals(12, encodingScheme.decode(table.getColumnForColumnName("INT3")
+ .getColumnQualifierBytes()));
+ }
+
+ @Test
+ public void testCreateTableWithColumnQualifiersDuplicateCQ() throws Exception {
+ Properties props = new Properties();
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ String ddl = "CREATE TABLE \"" + tableName + "\"(K VARCHAR NOT NULL PRIMARY KEY, " +
+ "INT INTEGER ENCODED_QUALIFIER 11, INT2 INTEGER ENCODED_QUALIFIER 11, " +
+ "INT3 INTEGER ENCODED_QUALIFIER 14) COLUMN_QUALIFIER_COUNTER ('0'=15)";
+ try {
+ conn.createStatement().execute(ddl);
+ fail("Duplicate Column Qualifiers");
+ } catch (SQLException e) {
+ // expected DUPLICATE_CQ
+ if (e.getErrorCode() != SQLExceptionCode.DUPLICATE_CQ.getErrorCode()) {
+ fail("Duplicate Column Qualifiers");
+ }
+ }
+ }
+
+ @Test
+ public void testCreateTableInvalidColumnQualifier() throws Exception {
+ Properties props = new Properties();
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ String ddl = "CREATE TABLE \"" + tableName + "\"(K VARCHAR NOT NULL PRIMARY KEY, " +
+ "INT INTEGER ENCODED_QUALIFIER 11, INT2 INTEGER ENCODED_QUALIFIER 9, " +
+ "INT3 INTEGER ENCODED_QUALIFIER 14) COLUMN_QUALIFIER_COUNTER ('0'=15)";
+ try {
+ conn.createStatement().execute(ddl);
+ fail("Invalid Column Qualifier");
+ } catch (SQLException e) {
+ // expected INVALID_CQ
+ if (e.getErrorCode() != SQLExceptionCode.INVALID_CQ.getErrorCode()) {
+ fail("Invalid Column Qualifier");
+ }
+ }
+
+ ddl = "CREATE TABLE \"" + tableName + "\"(K VARCHAR NOT NULL PRIMARY KEY, " +
+ "INT INTEGER ENCODED_QUALIFIER 11) COLUMN_QUALIFIER_COUNTER ('0'=8)";
+ try {
+ conn.createStatement().execute(ddl);
+ fail("Invalid Column Qualifier");
+ } catch (SQLException e) {
+ // expected INVALID_CQ
+ if (e.getErrorCode() != SQLExceptionCode.INVALID_CQ.getErrorCode()) {
+ fail("Invalid Column Qualifier");
+ }
+ }
+
+ ddl = "CREATE TABLE \"" + tableName + "\"(K VARCHAR NOT NULL PRIMARY KEY, " +
+ "INT INTEGER ENCODED_QUALIFIER 15) COLUMN_QUALIFIER_COUNTER ('0'=13)";
+ try {
+ conn.createStatement().execute(ddl);
+ fail("Invalid Column Qualifier");
+ } catch (SQLException e) {
+ // expected INVALID_CQ
+ if (e.getErrorCode() != SQLExceptionCode.INVALID_CQ.getErrorCode()) {
+ fail("Invalid Column Qualifier");
+ }
+ }
+
+ ddl = "CREATE IMMUTABLE TABLE \"" + tableName + "\"(K VARCHAR NOT NULL PRIMARY KEY, " +
+ "INT1 INTEGER, " +
+ "INT2 INTEGER " +
+ "DEFAULT_COLUMN_FAMILY=dF" +
+ "COLUMN_QUALIFIER_COUNTER (\"0\"=13)";
+ try {
+ conn.createStatement().execute(ddl);
+ } catch (SQLException e) {
+ //expected
+ }
+ }
+
+ @Test
+ public void testCreateTableMissingColumnQualifier() throws Exception {
+ Properties props = new Properties();
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ String ddl = "CREATE TABLE \"" + tableName + "\"(K VARCHAR NOT NULL PRIMARY KEY, " +
+ "INT INTEGER, INT2 INTEGER ENCODED_QUALIFIER 12)";
+ try {
+ conn.createStatement().execute(ddl);
+ fail("Invalid Column Qualifier");
+ } catch (SQLException e) {
+ // expected MISSING_CQ
+ if (e.getErrorCode() != SQLExceptionCode.MISSING_CQ.getErrorCode()) {
+ fail("Missing Column Qualifier");
+ }
+ }
+
+ ddl = "CREATE TABLE \"" + tableName + "\"(K VARCHAR NOT NULL PRIMARY KEY, " +
+ "INT INTEGER ENCODED_QUALIFIER 11, INT2 INTEGER) COLUMN_QUALIFIER_COUNTER ('0'=13)";
+ try {
+ conn.createStatement().execute(ddl);
+ fail("Invalid Column Qualifier");
+ } catch (SQLException e) {
+ // expected MISSING_CQ
+ if (e.getErrorCode() != SQLExceptionCode.MISSING_CQ.getErrorCode()) {
+ fail("Missing Column Qualifier");
+ }
+ }
+ }
+
+ @Test
+ public void testCreateTableDefaultColumnQualifier() throws Exception {
+ Properties props = new Properties();
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ String ddl = "CREATE IMMUTABLE TABLE \"" + tableName + "\"(K VARCHAR NOT NULL PRIMARY KEY, " +
+ "INT1 INTEGER, " +
+ "INT2 INTEGER, " +
+ "a.INT3 INTEGER, " +
+ "\"A\".INT4 INTEGER, " +
+ "\"b\".INT5 INTEGER, " +
+ "\"B\".INT6 INTEGER) " +
+ "DEFAULT_COLUMN_FAMILY=dF";
+ conn.createStatement().execute(ddl);
+ PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class);
+ PTable table = pconn.getTable(new PTableKey(null, tableName));
+
+ QualifierEncodingScheme encodingScheme = table.getEncodingScheme();
+ assertNotEquals(PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, encodingScheme);
+
+ PTable.EncodedCQCounter cqCounter = table.getEncodedCQCounter();
+ assertEquals(13, cqCounter.values().get("dF").intValue());
+ assertEquals(13, cqCounter.values().get("A").intValue());
+ assertEquals(12, cqCounter.values().get("b").intValue());
+ assertEquals(12, cqCounter.values().get("B").intValue());
+ assertEquals(11, encodingScheme.decode(table.getColumnForColumnName("INT1")
+ .getColumnQualifierBytes()));
+ assertEquals(12, encodingScheme.decode(table.getColumnForColumnName("INT2")
+ .getColumnQualifierBytes()));
+ assertEquals(11, encodingScheme.decode(table.getColumnForColumnName("INT3")
+ .getColumnQualifierBytes()));
+ assertEquals(12, encodingScheme.decode(table.getColumnForColumnName("INT4")
+ .getColumnQualifierBytes()));
+ assertEquals(11, encodingScheme.decode(table.getColumnForColumnName("INT5")
+ .getColumnQualifierBytes()));
+ assertEquals(11, encodingScheme.decode(table.getColumnForColumnName("INT6")
+ .getColumnQualifierBytes()));
+ }
+
public static long verifyLastDDLTimestamp(String dataTableFullName, long startTS, Connection conn) throws SQLException {
long endTS = EnvironmentEdgeManager.currentTimeMillis();
//Now try the PTable API
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ShowCreateTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ShowCreateTableIT.java
index fe75f85e4d..9b8619bf36 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ShowCreateTableIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ShowCreateTableIT.java
@@ -27,6 +27,8 @@ import java.sql.DriverManager;
import java.sql.ResultSet;
import java.util.Properties;
+import static org.apache.phoenix.query.QueryConstants.DEFAULT_COLUMN_FAMILY;
+import static org.apache.phoenix.query.QueryConstants.ENCODED_CQ_COUNTER_INITIAL_VALUE;
import static org.junit.Assert.assertTrue;
@Category(ParallelStatsDisabledTest.class)
@@ -36,13 +38,18 @@ public class ShowCreateTableIT extends ParallelStatsDisabledIT {
Properties props = new Properties();
Connection conn = DriverManager.getConnection(getUrl(), props);
String tableName = generateUniqueName();
- String ddl = "CREATE TABLE " + tableName + "(K VARCHAR NOT NULL PRIMARY KEY, INT INTEGER)";
+ String ddl = "CREATE TABLE " + tableName + "(K VARCHAR NOT NULL PRIMARY KEY, INT INTEGER, INT2 INTEGER)";
conn.createStatement().execute(ddl);
ResultSet rs = conn.createStatement().executeQuery("SHOW CREATE TABLE " + tableName );
assertTrue(rs.next());
- assertTrue("Expected: :" + ddl + "\nResult: " + rs.getString(1),
- rs.getString(1).contains(ddl));
+
+ String expected = "CREATE TABLE " + tableName + "(K VARCHAR NOT NULL PRIMARY KEY, " +
+ "INT INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE) +
+ ", INT2 INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE + 1) +
+ ") IMMUTABLE_STORAGE_SCHEME='ONE_CELL_PER_COLUMN'";
+ assertTrue("Expected: :" + expected + "\nResult: " + rs.getString(1),
+ rs.getString(1).equals(expected));
}
@Test
@@ -55,8 +62,12 @@ public class ShowCreateTableIT extends ParallelStatsDisabledIT {
ResultSet rs = conn.createStatement().executeQuery("SHOW CREATE TABLE \"" + tableName + "\"");
assertTrue(rs.next());
- assertTrue("Expected: :" + ddl + "\nResult: " + rs.getString(1),
- rs.getString(1).contains(ddl));
+
+ String expected = "CREATE TABLE \"" + tableName + "\"(K VARCHAR NOT NULL PRIMARY KEY, " +
+ "INT INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE) +
+ ") IMMUTABLE_STORAGE_SCHEME='ONE_CELL_PER_COLUMN'";
+ assertTrue("Expected: :" + expected + "\nResult: " + rs.getString(1),
+ rs.getString(1).equals(expected));
}
@Test
@@ -71,8 +82,158 @@ public class ShowCreateTableIT extends ParallelStatsDisabledIT {
ResultSet rs = conn.createStatement().executeQuery("SHOW CREATE TABLE " + tableFullName);
assertTrue(rs.next());
+
+ String expected = "CREATE TABLE " + tableFullName + "(K VARCHAR NOT NULL PRIMARY KEY, " +
+ "INT INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE) +
+ ") IMMUTABLE_STORAGE_SCHEME='ONE_CELL_PER_COLUMN'";
+ assertTrue("Expected: :" + expected + "\nResult: " + rs.getString(1),
+ rs.getString(1).equals(expected));
+ }
+
+ @Test
+ public void testShowCreateTableDefaultFamily() throws Exception {
+ Properties props = new Properties();
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ String ddl = "CREATE IMMUTABLE TABLE \"" + tableName + "\"(K VARCHAR NOT NULL PRIMARY KEY, " +
+ "INT1 INTEGER, " +
+ "INT2 INTEGER, " +
+ "a.INT3 INTEGER, " +
+ "\"A\".INT4 INTEGER, " +
+ "\"b\".INT5 INTEGER, " +
+ "\"B\".INT6 INTEGER) " +
+ "DEFAULT_COLUMN_FAMILY='dF'";
+ conn.createStatement().execute(ddl);
+
+ ResultSet rs = conn.createStatement().executeQuery("SHOW CREATE TABLE " + tableName );
+ assertTrue(rs.next());
+
+ String expected = "CREATE TABLE " + tableName + "(K VARCHAR NOT NULL PRIMARY KEY, " +
+ "\"dF\".INT1 INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE) +
+ ", \"dF\".INT2 INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE + 1) +
+ ", A.INT3 INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE) +
+ ", A.INT4 INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE + 1) +
+ ", \"b\".INT5 INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE) +
+ ", B.INT6 INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE) +
+ ") IMMUTABLE_ROWS=true, DEFAULT_COLUMN_FAMILY='dF'";
+ assertTrue("Expected: :" + expected + "\nResult: " + rs.getString(1),
+ rs.getString(1).equals(expected));
+ }
+
+ @Test
+ public void testShowCreateTableCounter() throws Exception {
+ Properties props = new Properties();
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ String ddl = "CREATE TABLE " + tableName + "(K VARCHAR NOT NULL PRIMARY KEY, " +
+ "INT INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE) +
+ ", INT2 INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE + 1) +
+ ") IMMUTABLE_STORAGE_SCHEME='ONE_CELL_PER_COLUMN' COLUMN_QUALIFIER_COUNTER " +
+ "('" + DEFAULT_COLUMN_FAMILY +"'=" + (ENCODED_CQ_COUNTER_INITIAL_VALUE + 3) + ")";
+ conn.createStatement().execute(ddl);
+
+ ResultSet rs = conn.createStatement().executeQuery("SHOW CREATE TABLE " + tableName );
+ assertTrue(rs.next());
assertTrue("Expected: :" + ddl + "\nResult: " + rs.getString(1),
- rs.getString(1).contains(ddl));
+ rs.getString(1).equals(ddl));
+ }
+
+ @Test
+ public void testShowCreateTableColumnQualifierDrop() throws Exception {
+ Properties props = new Properties();
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();;
+ String ddl = "CREATE TABLE " + tableName + "(K VARCHAR NOT NULL PRIMARY KEY, " +
+ "INT INTEGER, INT2 INTEGER, INT3 INTEGER)";
+ conn.createStatement().execute(ddl);
+
+ String dropInt2 = "ALTER TABLE " + tableName + " DROP COLUMN INT3";
+ conn.createStatement().execute(dropInt2);
+
+ ResultSet rs = conn.createStatement().executeQuery("SHOW CREATE TABLE \"" + tableName + "\"");
+ assertTrue(rs.next());
+
+ String expected = "CREATE TABLE " + tableName + "(K VARCHAR NOT NULL PRIMARY KEY, " +
+ "INT INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE) +
+ ", INT2 INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE + 1) +
+ ") IMMUTABLE_STORAGE_SCHEME='ONE_CELL_PER_COLUMN' COLUMN_QUALIFIER_COUNTER " +
+ "('" + DEFAULT_COLUMN_FAMILY +"'=" + (ENCODED_CQ_COUNTER_INITIAL_VALUE + 3) + ")";
+ assertTrue("Expected: :" + expected + "\nResult: " + rs.getString(1),
+ rs.getString(1).equals(expected));
+ }
+
+ @Test
+ public void testShowCreateTableColumnQualifierDropAndAdd() throws Exception {
+ Properties props = new Properties();
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();;
+ String ddl = "CREATE TABLE " + tableName + "(K VARCHAR NOT NULL PRIMARY KEY, " +
+ "INT INTEGER, INT2 INTEGER, INT3 INTEGER)";
+ conn.createStatement().execute(ddl);
+
+ String dropInt3 = "ALTER TABLE " + tableName + " DROP COLUMN INT3";
+ conn.createStatement().execute(dropInt3);
+
+ String addInt4 = "ALTER TABLE " + tableName + " ADD INT4 INTEGER";
+ conn.createStatement().execute(addInt4);
+
+ ResultSet rs = conn.createStatement().executeQuery("SHOW CREATE TABLE \"" + tableName + "\"");
+ assertTrue(rs.next());
+
+ String expected = "CREATE TABLE " + tableName + "(K VARCHAR NOT NULL PRIMARY KEY, " +
+ "INT INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE) +
+ ", INT2 INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE + 1) +
+ ", INT4 INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE + 3) +
+ ") IMMUTABLE_STORAGE_SCHEME='ONE_CELL_PER_COLUMN'";
+ assertTrue("Expected: :" + expected + "\nResult: " + rs.getString(1),
+ rs.getString(1).equals(expected));
+ }
+
+ @Test
+ public void testShowCreateTableColumnQualifierMultipleFamilies() throws Exception {
+ Properties props = new Properties();
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();;
+ String ddl = "CREATE IMMUTABLE TABLE " + tableName +
+ "(K VARCHAR NOT NULL PRIMARY KEY, A.INT INTEGER, B.INT2 INTEGER) " +
+ "IMMUTABLE_STORAGE_SCHEME=SINGLE_CELL_ARRAY_WITH_OFFSETS";
+ conn.createStatement().execute(ddl);
+ ResultSet rs = conn.createStatement().executeQuery("SHOW CREATE TABLE \"" + tableName + "\"");
+ assertTrue(rs.next());
+
+ String expected = "CREATE TABLE " + tableName + "(K VARCHAR NOT NULL PRIMARY KEY, " +
+ "A.INT INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE) +
+ ", B.INT2 INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE) +
+ ") IMMUTABLE_ROWS=true";
+ assertTrue("Expected: :" + expected + "\nResult: " + rs.getString(1),
+ rs.getString(1).equals(expected));
+ }
+
+ @Test
+ public void testShowCreateTableColumnQualifierMultipleFamiliesDrop() throws Exception {
+ Properties props = new Properties();
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();;
+ String ddl = "CREATE IMMUTABLE TABLE " + tableName +
+ "(K VARCHAR NOT NULL PRIMARY KEY, A.INT INTEGER, A.INT2 INTEGER, " +
+ "B.INT3 INTEGER, B.INT4 INTEGER) " +
+ "IMMUTABLE_STORAGE_SCHEME=SINGLE_CELL_ARRAY_WITH_OFFSETS";
+ conn.createStatement().execute(ddl);
+
+ String dropInt3 = "ALTER TABLE " + tableName + " DROP COLUMN INT4";
+ conn.createStatement().execute(dropInt3);
+
+ ResultSet rs = conn.createStatement().executeQuery("SHOW CREATE TABLE \"" + tableName + "\"");
+ assertTrue(rs.next());
+
+ String expected = "CREATE TABLE " + tableName + "(K VARCHAR NOT NULL PRIMARY KEY, " +
+ "A.INT INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE) +
+ ", A.INT2 INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE + 1) +
+ ", B.INT3 INTEGER ENCODED_QUALIFIER " + (ENCODED_CQ_COUNTER_INITIAL_VALUE) +
+ ") IMMUTABLE_ROWS=true COLUMN_QUALIFIER_COUNTER ('B'=" + (ENCODED_CQ_COUNTER_INITIAL_VALUE + 2)
+ + ")";
+ assertTrue("Expected: :" + expected + "\nResult: " + rs.getString(1),
+ rs.getString(1).equals(expected));
}
@Test
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/transform/TransformMonitorExtendedIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/transform/TransformMonitorExtendedIT.java
index 6163a76d4d..4ccc3d4cae 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/transform/TransformMonitorExtendedIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/transform/TransformMonitorExtendedIT.java
@@ -152,6 +152,40 @@ public class TransformMonitorExtendedIT extends BaseTest {
TransformToolIT.createTableAndUpsertRows(conn, fullDataTableName, numOfRows, dataTableDdl);
SingleCellIndexIT.assertMetadata(conn, PTable.ImmutableStorageScheme.ONE_CELL_PER_COLUMN, PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, fullDataTableName);
+ conn.createStatement().execute("ALTER TABLE " + fullDataTableName +
+ " SET COLUMN_ENCODED_BYTES=2");
+
+ SystemTransformRecord record = Transform.getTransformRecord(schemaName, dataTableName, null, null, conn.unwrap(PhoenixConnection.class));
+ assertNotNull(record);
+ waitForTransformToGetToState(conn.unwrap(PhoenixConnection.class), record, PTable.TransformStatus.COMPLETED);
+ SingleCellIndexIT.assertMetadata(conn, PTable.ImmutableStorageScheme.ONE_CELL_PER_COLUMN, PTable.QualifierEncodingScheme.TWO_BYTE_QUALIFIERS, record.getNewPhysicalTableName());
+ TransformToolIT.upsertRows(conn, fullDataTableName, 2, 1);
+ assertEquals(numOfRows + 1, TransformMonitorIT.countRows(conn, fullDataTableName));
+
+ ResultSet rs = conn.createStatement().executeQuery("SELECT ID, ZIP FROM " + fullDataTableName);
+ assertTrue(rs.next());
+ assertEquals("1", rs.getString(1));
+ assertEquals(95051, rs.getInt(2));
+ assertTrue(rs.next());
+ assertEquals("2", rs.getString(1));
+ assertEquals(95052, rs.getInt(2));
+ assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testTransformImmutableTableWithNamespaceEnabled() throws Exception {
+ String schemaName = "S_" + generateUniqueName();
+ String dataTableName = "TBL_" + generateUniqueName();
+ String fullDataTableName = SchemaUtil.getTableName(schemaName, dataTableName);
+ dataTableDdl += ", IMMUTABLE_ROWS=true";
+ try (Connection conn = DriverManager.getConnection(getUrl(), propsNamespace)) {
+ conn.setAutoCommit(true);
+ int numOfRows = 1;
+ conn.createStatement().execute("CREATE SCHEMA IF NOT EXISTS " + schemaName);
+ TransformToolIT.createTableAndUpsertRows(conn, fullDataTableName, numOfRows, dataTableDdl);
+ SingleCellIndexIT.assertMetadata(conn, PTable.ImmutableStorageScheme.ONE_CELL_PER_COLUMN, PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, fullDataTableName);
+
conn.createStatement().execute("ALTER TABLE " + fullDataTableName +
" SET IMMUTABLE_STORAGE_SCHEME=SINGLE_CELL_ARRAY_WITH_OFFSETS, COLUMN_ENCODED_BYTES=2");
@@ -241,6 +275,127 @@ public class TransformMonitorExtendedIT extends BaseTest {
tenantConn1.createStatement().execute(createViewSql);
}
+ try (Connection conn = DriverManager.getConnection(getUrl(), propsNamespace)) {
+ conn.setAutoCommit(true);
+ conn.createStatement().execute("ALTER TABLE " + dataTableFullName1 +
+ " SET COLUMN_ENCODED_BYTES=2");
+ SystemTransformRecord record = Transform.getTransformRecord(schemaName, dataTableName1, null, null, conn.unwrap(PhoenixConnection.class));
+ assertNotNull(record);
+ waitForTransformToGetToState(conn.unwrap(PhoenixConnection.class), record, PTable.TransformStatus.COMPLETED);
+ assertMetadata(conn, PTable.ImmutableStorageScheme.ONE_CELL_PER_COLUMN, PTable.QualifierEncodingScheme.TWO_BYTE_QUALIFIERS, record.getNewPhysicalTableName());
+
+ try (Admin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin()) {
+ admin.disableTable(TableName.valueOf(namespaceMappedDataTableName1));
+ admin.truncateTable(TableName.valueOf(namespaceMappedDataTableName1), true);
+ }
+
+ SingleCellIndexIT.dumpTable(schemaName + ":" + dataTableName1 + "_1");
+
+ String sql = "SELECT VIEW_COL1, VIEW_COL2 FROM %s WHERE DATA='GLOBAL_VIEW' ";
+ ResultSet rs1 = conn.createStatement().executeQuery(String.format(sql, view1Name));
+ assertTrue(rs1.next());
+ assertEquals(111, rs1.getInt(1));
+ assertEquals("viewcol2", rs1.getString(2));
+ assertTrue(rs1.next());
+ assertEquals("viewcol22", rs1.getString(2));
+ assertFalse(rs1.next());
+
+ rs1 = conn.createStatement().executeQuery(String.format(sql, view2Name));
+ assertTrue(rs1.next());
+ assertEquals(111, rs1.getInt(1));
+ assertEquals("viewcol2", rs1.getString(2));
+ assertTrue(rs1.next());
+ assertEquals("viewcol22", rs1.getString(2));
+ assertFalse(rs1.next());
+
+ sql = "SELECT DATA FROM %s WHERE ID=1";
+ rs1 = conn.createStatement().executeQuery(String.format(sql, dataTableFullName1));
+ assertFalse(rs1.next());
+ }
+
+ try (Connection tenantConn1 = getTenantConnection("tenant1")) {
+ String sql = "SELECT VIEW_TCOL1, VIEW_TCOL2 FROM %s ";
+ ResultSet rs1 = tenantConn1.createStatement().executeQuery(String.format(sql, tenantView));
+
+ assertTrue(rs1.next());
+ assertEquals(2001, rs1.getInt(1));
+ assertEquals("tenantviewcol", rs1.getString(2));
+
+ ResultSet rs2 = tenantConn1.createStatement().executeQuery(String.format(sql, readOnlyTenantView));
+ assertTrue(rs2.next());
+ assertEquals(2001, rs2.getInt(1));
+ assertEquals("tenantviewcol", rs2.getString(2));
+ }
+ }
+
+ @Test
+ public void testTransformWithGlobalAndTenantViewsImmutable() throws Exception {
+ String schemaName = generateUniqueName();
+ String dataTableName1 = generateUniqueName();
+ String dataTableFullName1 = SchemaUtil.getTableName(schemaName, dataTableName1);
+ String namespaceMappedDataTableName1 = SchemaUtil.getPhysicalHBaseTableName(schemaName, dataTableName1, true).getString();
+ String view1Name = SchemaUtil.getTableName(schemaName, "VW1_" + generateUniqueName());
+ String view2Name = SchemaUtil.getTableName(schemaName, "VW2_" + generateUniqueName());
+ String tenantView = SchemaUtil.getTableName(schemaName, "VWT_" + generateUniqueName());
+ String readOnlyTenantView = SchemaUtil.getTableName(schemaName, "ROVWT_" + generateUniqueName());
+ dataTableDdl += ", IMMUTABLE_ROWS=true";
+ try (Connection conn = DriverManager.getConnection(getUrl(), propsNamespace)) {
+ conn.setAutoCommit(true);
+ int numOfRows = 1;
+ conn.createStatement().execute("CREATE SCHEMA IF NOT EXISTS " + schemaName);
+ TransformToolIT.createTableAndUpsertRows(conn, dataTableFullName1, numOfRows, "TABLE_ONLY", dataTableDdl);
+
+ SingleCellIndexIT.assertMetadata(conn, PTable.ImmutableStorageScheme.ONE_CELL_PER_COLUMN, PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, dataTableFullName1);
+
+ String createViewSql = "CREATE VIEW " + view1Name + " ( VIEW_COL1 INTEGER, VIEW_COL2 VARCHAR ) AS SELECT * FROM "
+ + dataTableFullName1 + " where DATA='GLOBAL_VIEW' ";
+ conn.createStatement().execute(createViewSql);
+ PreparedStatement stmt1 = conn.prepareStatement(String.format("UPSERT INTO %s VALUES(?, ? , ?, ?, ?,?)", view1Name));
+ stmt1.setInt(1, 2);
+ stmt1.setString(2, "uname2");
+ stmt1.setInt(3, 95053);
+ stmt1.setString(4, "GLOBAL_VIEW");
+ stmt1.setInt(5, 111);
+ stmt1.setString(6, "viewcol2");
+ stmt1.executeUpdate();
+
+ createViewSql = "CREATE VIEW " + view2Name + " ( VIEW_COL1 INTEGER, VIEW_COL2 VARCHAR ) AS SELECT * FROM "
+ + dataTableFullName1 + " where DATA='GLOBAL_VIEW' AND ZIP=95053";
+ conn.createStatement().execute(createViewSql);
+ stmt1 = conn.prepareStatement(String.format("UPSERT INTO %s VALUES(?, ? , ?, ?, ?,?)", view1Name));
+ stmt1.setInt(1, 20);
+ stmt1.setString(2, "uname22");
+ stmt1.setInt(3, 95053);
+ stmt1.setString(4, "GLOBAL_VIEW");
+ stmt1.setInt(5, 111);
+ stmt1.setString(6, "viewcol22");
+ stmt1.executeUpdate();
+ }
+
+ try (Connection tenantConn1 = getTenantConnection("tenant1")) {
+ tenantConn1.setAutoCommit(true);
+ String createViewSql = "CREATE VIEW " + tenantView + " ( VIEW_TCOL1 INTEGER, VIEW_TCOL2 VARCHAR ) " +
+ " AS SELECT * FROM "
+ + dataTableFullName1 + " where DATA='TENANT_VIEW'";
+ tenantConn1.createStatement().execute(createViewSql);
+
+ PreparedStatement stmt1 = tenantConn1.prepareStatement(
+ String.format("UPSERT INTO %s (ID, NAME, ZIP, DATA, VIEW_TCOL1, VIEW_TCOL2) " +
+ "VALUES(?, ? , ?, ?, ?, ?)", tenantView));
+ stmt1.setInt(1, 4);
+ stmt1.setString(2, "uname4");
+ stmt1.setInt(3, 95054);
+ stmt1.setString(4, "TENANT_VIEW");
+ stmt1.setInt(5, 2001);
+ stmt1.setString(6, "tenantviewcol");
+ stmt1.executeUpdate();
+
+ // ZIP field values are like 95050 + i
+ createViewSql = "CREATE VIEW " + readOnlyTenantView + " ( VIEW_TCOL1 INTEGER, VIEW_TCOL2 VARCHAR ) AS SELECT * FROM "
+ + dataTableFullName1 + " where DATA='TENANT_VIEW' AND ZIP > 95050";
+ tenantConn1.createStatement().execute(createViewSql);
+ }
+
try (Connection conn = DriverManager.getConnection(getUrl(), propsNamespace)) {
conn.setAutoCommit(true);
conn.createStatement().execute("ALTER TABLE " + dataTableFullName1 +
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/schema/tool/SchemaToolExtractionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/schema/tool/SchemaToolExtractionIT.java
index 0a733aea97..d4a2aaa2d3 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/schema/tool/SchemaToolExtractionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/schema/tool/SchemaToolExtractionIT.java
@@ -66,7 +66,8 @@ public class SchemaToolExtractionIT extends ParallelStatsEnabledIT {
String schemaName = generateUniqueName();
String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
String createTableStmt = "CREATE TABLE "+ pTableFullName + "(K VARCHAR NOT NULL PRIMARY KEY, "
- + "V1 VARCHAR, V2 VARCHAR) TTL=2592000, IMMUTABLE_ROWS=TRUE, DISABLE_WAL=TRUE";
+ + "V1 VARCHAR ENCODED_QUALIFIER 11, V2 VARCHAR ENCODED_QUALIFIER 12) " +
+ "TTL=2592000, IMMUTABLE_ROWS=TRUE, DISABLE_WAL=TRUE";
List<String> queries = new ArrayList<String>(){};
queries.add(createTableStmt);
String result = runSchemaExtractionTool(schemaName, tableName, null, queries);
@@ -79,7 +80,8 @@ public class SchemaToolExtractionIT extends ParallelStatsEnabledIT {
String schemaName = "lowecaseschemaname1";
String pTableFullName = SchemaUtil.getEscapedTableName(schemaName, tableName);
String createTableStmt = "CREATE TABLE "+ pTableFullName + "(\"smallK\" VARCHAR NOT NULL PRIMARY KEY, "
- + "\"asd\".V1 VARCHAR, \"foo\".\"bar\" VARCHAR) TTL=2592000, IMMUTABLE_ROWS=true, DISABLE_WAL=true";
+ + "\"asd\".V1 VARCHAR ENCODED_QUALIFIER 11, \"foo\".\"bar\" VARCHAR ENCODED_QUALIFIER 11) " +
+ "TTL=2592000, IMMUTABLE_ROWS=true, DISABLE_WAL=true";
List<String> queries = new ArrayList<String>(){};
queries.add(createTableStmt);
String result = runSchemaExtractionTool("\"" + schemaName + "\"", "\"" + tableName + "\"", null, queries);
@@ -416,8 +418,9 @@ public class SchemaToolExtractionIT extends ParallelStatsEnabledIT {
String query = "create table " + pTableFullName +
"(a_char CHAR(15) NOT NULL, " +
"b_char CHAR(10) NOT NULL, " +
- "c_var_array VARCHAR ARRAY, " +
- "d_char_array CHAR(15) ARRAY[3] CONSTRAINT PK PRIMARY KEY (a_char, b_char)) " +
+ "c_var_array VARCHAR ARRAY ENCODED_QUALIFIER 11, " +
+ "d_char_array CHAR(15) ARRAY[3] ENCODED_QUALIFIER 12 " +
+ "CONSTRAINT PK PRIMARY KEY (a_char, b_char)) " +
"TTL=2592000, IMMUTABLE_STORAGE_SCHEME='ONE_CELL_PER_COLUMN', REPLICATION_SCOPE=1";
List<String> queries = new ArrayList<String>(){};
queries.add(query);
@@ -458,7 +461,8 @@ public class SchemaToolExtractionIT extends ParallelStatsEnabledIT {
"b_char CHAR(10) NOT NULL, " +
"\"av\".\"_\" CHAR(1), " +
"\"bv\".\"_\" CHAR(1), " +
- "\"cv\".\"_\" CHAR(1) CONSTRAINT PK PRIMARY KEY (a_char, b_char)) " + properties;
+ "\"cv\".\"_\" CHAR(1) " +
+ "CONSTRAINT PK PRIMARY KEY (a_char, b_char)) " + properties;
List<String> queries = new ArrayList<String>(){};
queries.add(query);
String result = runSchemaExtractionTool(schemaName, tableName, null, queries);
diff --git a/phoenix-core/src/main/antlr3/PhoenixSQL.g b/phoenix-core/src/main/antlr3/PhoenixSQL.g
index 19a483293e..ce2c9b2575 100644
--- a/phoenix-core/src/main/antlr3/PhoenixSQL.g
+++ b/phoenix-core/src/main/antlr3/PhoenixSQL.g
@@ -130,6 +130,8 @@ tokens
LIST = 'list';
JARS='jars';
ROW_TIMESTAMP='row_timestamp';
+ ENCODED_QUALIFIER = 'encoded_qualifier';
+ COLUMN_QUALIFIER_COUNTER = 'column_qualifier_counter';
USE='use';
OFFSET ='offset';
FETCH = 'fetch';
@@ -458,7 +460,8 @@ create_table_node returns [CreateTableStatement ret]
(LPAREN c=column_defs (pk=pk_constraint)? RPAREN)
(p=fam_properties)?
(SPLIT ON s=value_expression_list)?
- {ret = factory.createTable(t, p, c, pk, s, PTableType.TABLE, ex!=null, null, null, getBindCount(), im!=null ? true : null); }
+ (COLUMN_QUALIFIER_COUNTER LPAREN cqc=initializiation_list RPAREN)?
+ {ret = factory.createTable(t, p, c, pk, s, PTableType.TABLE, ex!=null, null, null, getBindCount(), im!=null ? true : null, cqc); }
;
// Parse a create schema statement.
@@ -704,19 +707,31 @@ column_defs returns [List<ColumnDef> ret]
: v = column_def {$ret.add(v);} (COMMA v = column_def {$ret.add(v);} )*
;
+initializiation_list returns [Map<String, Integer> ret]
+@init{ret = new HashMap<String,Integer>(); }
+ : k=STRING_LITERAL EQ v=NUMBER {$ret.put(k.getText(), Integer.parseInt( v.getText() ));}
+ (COMMA k=STRING_LITERAL EQ v=NUMBER {$ret.put(k.getText(), Integer.parseInt( v.getText() ));} )*
+ ;
+
indexes returns [List<NamedNode> ret]
@init{ret = new ArrayList<NamedNode>(); }
: v = index_name {$ret.add(v);} (COMMA v = index_name {$ret.add(v);} )*
;
column_def returns [ColumnDef ret]
- : c=column_name dt=identifier (LPAREN l=NUMBER (COMMA s=NUMBER)? RPAREN)? ar=ARRAY? (lsq=LSQUARE (a=NUMBER)? RSQUARE)? (nn=NOT? n=NULL)? (DEFAULT df=expression)? (pk=PRIMARY KEY (order=ASC|order=DESC)? rr=ROW_TIMESTAMP?)?
- { $ret = factory.columnDef(c, dt, ar != null || lsq != null, a == null ? null : Integer.parseInt( a.getText() ), nn!=null ? Boolean.FALSE : n!=null ? Boolean.TRUE : null,
+ : c=column_name dt=identifier (LPAREN l=NUMBER (COMMA s=NUMBER)? RPAREN)? ar=ARRAY? (lsq=LSQUARE (a=NUMBER)? RSQUARE)? (nn=NOT? n=NULL)? (DEFAULT df=expression)? ((pk=PRIMARY KEY (order=ASC|order=DESC)? rr=ROW_TIMESTAMP?)|(ENCODED_QUALIFIER eq=NUMBER))?
+ { $ret = factory.columnDef(
+ c,
+ dt,
+ ar != null || lsq != null,
+ a == null ? null : Integer.parseInt( a.getText() ),
+ nn!=null ? Boolean.FALSE : n!=null ? Boolean.TRUE : null,
l == null ? null : Integer.parseInt( l.getText() ),
s == null ? null : Integer.parseInt( s.getText() ),
pk != null,
order == null ? SortOrder.getDefault() : SortOrder.fromDDLValue(order.getText()),
df == null ? null : df.toString(),
+ eq == null ? null : Integer.parseInt( eq.getText() ),
rr != null); }
;
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
index 771b06a42d..d39e67b8a4 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
@@ -448,6 +448,14 @@ public enum SQLExceptionCode {
+ "Table not in this region."),
UNABLE_TO_UPSERT_TASK(1146, "XCL46",
"Error upserting records in SYSTEM.TASK table"),
+ INVALID_CQ(1148, "XCL48",
+ "ENCODED_QUALIFIER is less than INITIAL_VALUE."),
+ DUPLICATE_CQ(1149, "XCL49",
+ "Duplicate ENCODED_QUALIFIER."),
+ MISSING_CQ(1150, "XCL49",
+ "Missing ENCODED_QUALIFIER."),
+
+
/**
* Implementation defined class. Phoenix internal error. (errorcode 20, sqlstate INT).
*/
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
index 02a46420eb..00ee645103 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixStatement.java
@@ -1030,9 +1030,10 @@ public class PhoenixStatement implements PhoenixMonitoredStatement, SQLCloseable
private static class ExecutableCreateTableStatement extends CreateTableStatement implements CompilableStatement {
ExecutableCreateTableStatement(TableName tableName, ListMultimap<String,Pair<String,Object>> props, List<ColumnDef> columnDefs,
- PrimaryKeyConstraint pkConstraint, List<ParseNode> splitNodes, PTableType tableType, boolean ifNotExists,
- TableName baseTableName, ParseNode tableTypeIdNode, int bindCount, Boolean immutableRows) {
- super(tableName, props, columnDefs, pkConstraint, splitNodes, tableType, ifNotExists, baseTableName, tableTypeIdNode, bindCount, immutableRows);
+ PrimaryKeyConstraint pkConstraint, List<ParseNode> splitNodes, PTableType tableType, boolean ifNotExists,
+ TableName baseTableName, ParseNode tableTypeIdNode, int bindCount, Boolean immutableRows,
+ Map<String, Integer> familyCounters) {
+ super(tableName, props, columnDefs, pkConstraint, splitNodes, tableType, ifNotExists, baseTableName, tableTypeIdNode, bindCount, immutableRows, familyCounters);
}
@SuppressWarnings("unchecked")
@@ -1802,11 +1803,16 @@ public class PhoenixStatement implements PhoenixMonitoredStatement, SQLCloseable
public ExecutableDeleteStatement delete(NamedTableNode table, HintNode hint, ParseNode whereNode, List<OrderByNode> orderBy, LimitNode limit, int bindCount, Map<String, UDFParseNode> udfParseNodes) {
return new ExecutableDeleteStatement(table, hint, whereNode, orderBy, limit, bindCount, udfParseNodes);
}
-
+
+ @Override
+ public CreateTableStatement createTable(TableName tableName, ListMultimap<String,Pair<String,Object>> props, List<ColumnDef> columns, PrimaryKeyConstraint pkConstraint, List<ParseNode> splits, PTableType tableType, boolean ifNotExists, TableName baseTableName, ParseNode tableTypeIdNode, int bindCount, Boolean immutableRows, Map<String, Integer> cqCounters) {
+ return new ExecutableCreateTableStatement(tableName, props, columns, pkConstraint, splits, tableType, ifNotExists, baseTableName, tableTypeIdNode, bindCount, immutableRows, cqCounters);
+ }
+
@Override
public CreateTableStatement createTable(TableName tableName, ListMultimap<String,Pair<String,Object>> props, List<ColumnDef> columns, PrimaryKeyConstraint pkConstraint,
List<ParseNode> splits, PTableType tableType, boolean ifNotExists, TableName baseTableName, ParseNode tableTypeIdNode, int bindCount, Boolean immutableRows) {
- return new ExecutableCreateTableStatement(tableName, props, columns, pkConstraint, splits, tableType, ifNotExists, baseTableName, tableTypeIdNode, bindCount, immutableRows);
+ return new ExecutableCreateTableStatement(tableName, props, columns, pkConstraint, splits, tableType, ifNotExists, baseTableName, tableTypeIdNode, bindCount, immutableRows, null);
}
@Override
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/parse/ColumnDef.java b/phoenix-core/src/main/java/org/apache/phoenix/parse/ColumnDef.java
index 029c30dd80..ac4835906b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/parse/ColumnDef.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/parse/ColumnDef.java
@@ -58,6 +58,7 @@ public class ColumnDef {
private final boolean isArray;
private final Integer arrSize;
private final String expressionStr;
+ private final Integer encodedQualifier;
private final boolean isRowTimestamp;
public ColumnDef(ColumnDef def, String expressionStr) {
@@ -70,12 +71,13 @@ public class ColumnDef {
this.sortOrder = def.sortOrder;
this.isArray = def.isArray;
this.arrSize = def.arrSize;
+ this.encodedQualifier = def.encodedQualifier;
this.isRowTimestamp = def.isRowTimestamp;
this.expressionStr = expressionStr;
}
ColumnDef(ColumnName columnDefName, String sqlTypeName, boolean isArray, Integer arrSize, Boolean isNull, Integer maxLength,
- Integer scale, boolean isPK, SortOrder sortOrder, String expressionStr, boolean isRowTimestamp) {
+ Integer scale, boolean isPK, SortOrder sortOrder, String expressionStr, Integer encodedQualifier, boolean isRowTimestamp) {
try {
Preconditions.checkNotNull(sortOrder);
PDataType baseType;
@@ -149,6 +151,7 @@ public class ColumnDef {
this.sortOrder = sortOrder;
this.dataType = dataType;
this.expressionStr = expressionStr;
+ this.encodedQualifier = encodedQualifier;
this.isRowTimestamp = isRowTimestamp;
} catch (SQLException e) {
throw new ParseException(e);
@@ -156,8 +159,10 @@ public class ColumnDef {
}
ColumnDef(ColumnName columnDefName, String sqlTypeName, Boolean isNull, Integer maxLength,
- Integer scale, boolean isPK, SortOrder sortOrder, String expressionStr, boolean isRowTimestamp) {
- this(columnDefName, sqlTypeName, false, 0, isNull, maxLength, scale, isPK, sortOrder, expressionStr, isRowTimestamp);
+ Integer scale, boolean isPK, SortOrder sortOrder, String expressionStr,
+ Integer columnQualifier, boolean isRowTimestamp) {
+ this(columnDefName, sqlTypeName, false, 0, isNull, maxLength,
+ scale, isPK, sortOrder, expressionStr, columnQualifier, isRowTimestamp);
}
public ColumnName getColumnDefName() {
@@ -206,6 +211,10 @@ public class ColumnDef {
return expressionStr;
}
+ public Integer getEncodedQualifier() {
+ return encodedQualifier;
+ }
+
public boolean isRowTimestamp() {
return isRowTimestamp;
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/parse/CreateTableStatement.java b/phoenix-core/src/main/java/org/apache/phoenix/parse/CreateTableStatement.java
index fcde8f138e..2d9ea13e6e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/parse/CreateTableStatement.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/parse/CreateTableStatement.java
@@ -19,6 +19,7 @@ package org.apache.phoenix.parse;
import java.util.Collections;
import java.util.List;
+import java.util.Map;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
@@ -41,6 +42,7 @@ public class CreateTableStatement extends MutableStatement {
private final ParseNode whereClause;
// TODO change this to boolean at the next major release and remove TableProperty.IMMUTABLE_ROWS and QueryServiceOptions.IMMUTABLE_ROWS_ATTRIB
private final Boolean immutableRows;
+ private final Map<String, Integer> familyCQCounters;
public CreateTableStatement(CreateTableStatement createTable, List<ColumnDef> columns) {
this.tableName = createTable.tableName;
@@ -54,6 +56,7 @@ public class CreateTableStatement extends MutableStatement {
this.baseTableName = createTable.baseTableName;
this.whereClause = createTable.whereClause;
this.immutableRows = createTable.immutableRows;
+ this.familyCQCounters = createTable.familyCQCounters;
}
public CreateTableStatement(CreateTableStatement createTable, PrimaryKeyConstraint pkConstraint,
@@ -69,6 +72,7 @@ public class CreateTableStatement extends MutableStatement {
this.baseTableName = createTable.baseTableName;
this.whereClause = createTable.whereClause;
this.immutableRows = createTable.immutableRows;
+ this.familyCQCounters = createTable.familyCQCounters;
}
public CreateTableStatement(CreateTableStatement createTable, ListMultimap<String,Pair<String,Object>> props, List<ColumnDef> columns) {
@@ -83,11 +87,13 @@ public class CreateTableStatement extends MutableStatement {
this.baseTableName = createTable.baseTableName;
this.whereClause = createTable.whereClause;
this.immutableRows = createTable.immutableRows;
+ this.familyCQCounters = createTable.familyCQCounters;
}
protected CreateTableStatement(TableName tableName, ListMultimap<String,Pair<String,Object>> props, List<ColumnDef> columns, PrimaryKeyConstraint pkConstraint,
- List<ParseNode> splitNodes, PTableType tableType, boolean ifNotExists,
- TableName baseTableName, ParseNode whereClause, int bindCount, Boolean immutableRows) {
+ List<ParseNode> splitNodes, PTableType tableType, boolean ifNotExists,
+ TableName baseTableName, ParseNode whereClause, int bindCount, Boolean immutableRows,
+ Map<String, Integer> familyCounters) {
this.tableName = tableName;
this.props = props == null ? ImmutableListMultimap.<String,Pair<String,Object>>of() : props;
this.tableType = PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA.equals(tableName.getSchemaName()) ? PTableType.SYSTEM : tableType;
@@ -99,6 +105,7 @@ public class CreateTableStatement extends MutableStatement {
this.baseTableName = baseTableName;
this.whereClause = whereClause;
this.immutableRows = immutableRows;
+ this.familyCQCounters = familyCounters;
}
public ParseNode getWhereClause() {
@@ -145,4 +152,8 @@ public class CreateTableStatement extends MutableStatement {
public Boolean immutableRows() {
return immutableRows;
}
+
+ public Map<String, Integer> getFamilyCQCounters() {
+ return familyCQCounters;
+ }
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java
index 9ed72cf1e6..77350cc067 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/parse/ParseNodeFactory.java
@@ -289,9 +289,21 @@ public class ParseNodeFactory {
}
public ColumnDef columnDef(ColumnName columnDefName, String sqlTypeName, boolean isNull, Integer maxLength, Integer scale, boolean isPK, SortOrder sortOrder, String expressionStr, boolean isRowTimestamp) {
- return new ColumnDef(columnDefName, sqlTypeName, isNull, maxLength, scale, isPK, sortOrder, expressionStr, isRowTimestamp);
+ return new ColumnDef(columnDefName, sqlTypeName, isNull, maxLength, scale, isPK, sortOrder, expressionStr, null, isRowTimestamp);
}
+ public ColumnDef columnDef(ColumnName columnDefName, String sqlTypeName,
+ boolean isArray, Integer arrSize, Boolean isNull,
+ Integer maxLength, Integer scale, boolean isPK,
+ SortOrder sortOrder, String expressionStr, Integer encodedQualifier,
+ boolean isRowTimestamp) {
+ return new ColumnDef(columnDefName, sqlTypeName,
+ isArray, arrSize, isNull,
+ maxLength, scale, isPK,
+ sortOrder, expressionStr, encodedQualifier, isRowTimestamp);
+ }
+
+
public ColumnDef columnDef(ColumnName columnDefName, String sqlTypeName,
boolean isArray, Integer arrSize, Boolean isNull,
Integer maxLength, Integer scale, boolean isPK,
@@ -299,12 +311,12 @@ public class ParseNodeFactory {
return new ColumnDef(columnDefName, sqlTypeName,
isArray, arrSize, isNull,
maxLength, scale, isPK,
- sortOrder, expressionStr, isRowTimestamp);
+ sortOrder, expressionStr, null, isRowTimestamp);
}
public ColumnDef columnDef(ColumnName columnDefName, String sqlTypeName, boolean isArray, Integer arrSize, Boolean isNull, Integer maxLength, Integer scale, boolean isPK,
SortOrder sortOrder, boolean isRowTimestamp) {
- return new ColumnDef(columnDefName, sqlTypeName, isArray, arrSize, isNull, maxLength, scale, isPK, sortOrder, null, isRowTimestamp);
+ return new ColumnDef(columnDefName, sqlTypeName, isArray, arrSize, isNull, maxLength, scale, isPK, sortOrder, null, null, isRowTimestamp);
}
public ColumnDefInPkConstraint columnDefInPkConstraint(ColumnName columnDefName, SortOrder sortOrder, boolean isRowTimestamp) {
@@ -319,8 +331,12 @@ public class ParseNodeFactory {
return new IndexKeyConstraint(parseNodeAndSortOrder);
}
+ public CreateTableStatement createTable(TableName tableName, ListMultimap<String,Pair<String,Object>> props, List<ColumnDef> columns, PrimaryKeyConstraint pkConstraint, List<ParseNode> splits, PTableType tableType, boolean ifNotExists, TableName baseTableName, ParseNode tableTypeIdNode, int bindCount, Boolean immutableRows, Map<String, Integer> cqCounters) {
+ return new CreateTableStatement(tableName, props, columns, pkConstraint, splits, tableType, ifNotExists, baseTableName, tableTypeIdNode, bindCount, immutableRows, cqCounters);
+ }
+
public CreateTableStatement createTable(TableName tableName, ListMultimap<String,Pair<String,Object>> props, List<ColumnDef> columns, PrimaryKeyConstraint pkConstraint, List<ParseNode> splits, PTableType tableType, boolean ifNotExists, TableName baseTableName, ParseNode tableTypeIdNode, int bindCount, Boolean immutableRows) {
- return new CreateTableStatement(tableName, props, columns, pkConstraint, splits, tableType, ifNotExists, baseTableName, tableTypeIdNode, bindCount, immutableRows);
+ return new CreateTableStatement(tableName, props, columns, pkConstraint, splits, tableType, ifNotExists, baseTableName, tableTypeIdNode, bindCount, immutableRows, null);
}
public CreateSchemaStatement createSchema(String schemaName, boolean ifNotExists) {
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index bd6b58e6c5..c94e5f29ff 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -2452,6 +2452,9 @@ public class MetaDataClient {
int pkPositionOffset = pkColumns.size();
int position = positionOffset;
EncodedCQCounter cqCounter = NULL_COUNTER;
+ Map<String, Integer> changedCqCounters = new HashMap<>(colDefs.size());
+ // Check for duplicate column qualifiers
+ Map<String, Set<Integer>> inputCqCounters = new HashMap<>();
PTable viewPhysicalTable = null;
if (tableType == PTableType.VIEW) {
/*
@@ -2497,7 +2500,8 @@ public class MetaDataClient {
if (immutableStorageSchemeProp == null) {
immutableStorageScheme = parent.getImmutableStorageScheme();
} else {
- immutableStorageScheme = getImmutableStorageSchemeForIndex(immutableStorageSchemeProp, schemaName, tableName, transactionProvider);
+ checkImmutableStorageSchemeForIndex(immutableStorageSchemeProp, schemaName, tableName, transactionProvider);
+ immutableStorageScheme = immutableStorageSchemeProp;
}
if (immutableStorageScheme == SINGLE_CELL_ARRAY_WITH_OFFSETS) {
@@ -2558,11 +2562,11 @@ public class MetaDataClient {
}
}
} else {
- immutableStorageScheme = getImmutableStorageSchemeForIndex(immutableStorageSchemeProp, schemaName, tableName, transactionProvider);
+ immutableStorageScheme = isImmutableRows ? immutableStorageSchemeProp : ONE_CELL_PER_COLUMN;
+ checkImmutableStorageSchemeForIndex(immutableStorageScheme, schemaName, tableName, transactionProvider);
}
if (immutableStorageScheme != ONE_CELL_PER_COLUMN
&& encodingScheme == NON_ENCODED_QUALIFIERS) {
- getEncodingScheme(tableProps, schemaName, tableName, transactionProvider);
throw new SQLExceptionInfo.Builder(
SQLExceptionCode.INVALID_IMMUTABLE_STORAGE_SCHEME_AND_COLUMN_QUALIFIER_BYTES)
.setSchemaName(schemaName).setTableName(tableName).build()
@@ -2570,9 +2574,21 @@ public class MetaDataClient {
}
}
cqCounter = encodingScheme != NON_ENCODED_QUALIFIERS ? new EncodedCQCounter() : NULL_COUNTER;
+ if (encodingScheme != NON_ENCODED_QUALIFIERS && statement.getFamilyCQCounters() != null)
+ {
+ for (Map.Entry<String, Integer> cq : statement.getFamilyCQCounters().entrySet()) {
+ if (cq.getValue() < QueryConstants.ENCODED_CQ_COUNTER_INITIAL_VALUE) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_CQ)
+ .setSchemaName(schemaName)
+ .setTableName(tableName).build().buildException();
+ }
+ cqCounter.setValue(cq.getKey(), cq.getValue());
+ changedCqCounters.put(cq.getKey(), cqCounter.getNextQualifier(cq.getKey()));
+ inputCqCounters.putIfAbsent(cq.getKey(), new HashSet<Integer>());
+ }
+ }
}
- Map<String, Integer> changedCqCounters = new HashMap<>(colDefs.size());
boolean wasPKDefined = false;
// Keep track of all columns that are newly added to a view
Set<Integer> viewNewColumnPositions =
@@ -2614,7 +2630,54 @@ public class MetaDataClient {
}
// Use position as column qualifier if APPEND_ONLY_SCHEMA to prevent gaps in
// the column encoding (PHOENIX-4737).
- Integer encodedCQ = isPkColumn ? null : isAppendOnlySchema ? Integer.valueOf(ENCODED_CQ_COUNTER_INITIAL_VALUE + position) : cqCounter.getNextQualifier(cqCounterFamily);
+ Integer encodedCQ = null;
+ if (!isPkColumn) {
+ if (colDef.getEncodedQualifier() != null && encodingScheme != NON_ENCODED_QUALIFIERS) {
+ if (cqCounter.getNextQualifier(cqCounterFamily) > ENCODED_CQ_COUNTER_INITIAL_VALUE &&
+ !inputCqCounters.containsKey(cqCounterFamily)) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.MISSING_CQ)
+ .setSchemaName(schemaName)
+ .setTableName(tableName).build().buildException();
+ }
+
+ if (statement.getFamilyCQCounters() == null ||
+ statement.getFamilyCQCounters().get(cqCounterFamily) == null) {
+ if (colDef.getEncodedQualifier() >= cqCounter.getNextQualifier(cqCounterFamily)) {
+ cqCounter.setValue(cqCounterFamily, colDef.getEncodedQualifier());
+ cqCounter.increment(cqCounterFamily);
+ }
+ changedCqCounters.put(cqCounterFamily, cqCounter.getNextQualifier(cqCounterFamily));
+ }
+
+ encodedCQ = colDef.getEncodedQualifier();
+ if (encodedCQ < QueryConstants.ENCODED_CQ_COUNTER_INITIAL_VALUE ||
+ encodedCQ >= cqCounter.getNextQualifier(cqCounterFamily)) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.INVALID_CQ)
+ .setSchemaName(schemaName)
+ .setTableName(tableName).build().buildException();
+ }
+
+ inputCqCounters.putIfAbsent(cqCounterFamily, new HashSet<Integer>());
+ Set<Integer> familyCounters = inputCqCounters.get(cqCounterFamily);
+ if (!familyCounters.add(encodedCQ)) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.DUPLICATE_CQ)
+ .setSchemaName(schemaName)
+ .setTableName(tableName).build().buildException();
+ }
+ } else {
+ if (inputCqCounters.containsKey(cqCounterFamily)) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.MISSING_CQ)
+ .setSchemaName(schemaName)
+ .setTableName(tableName).build().buildException();
+ }
+
+ if (isAppendOnlySchema) {
+ encodedCQ = Integer.valueOf(ENCODED_CQ_COUNTER_INITIAL_VALUE + position);
+ } else {
+ encodedCQ = cqCounter.getNextQualifier(cqCounterFamily);
+ }
+ }
+ }
byte[] columnQualifierBytes = null;
try {
columnQualifierBytes = EncodedColumnsUtil.getColumnQualifierBytes(columnDefName.getColumnName(), encodedCQ, encodingScheme, isPkColumn);
@@ -2625,7 +2688,8 @@ public class MetaDataClient {
.setTableName(tableName).build().buildException();
}
PColumn column = newColumn(position++, colDef, pkConstraint, defaultFamilyName, false, columnQualifierBytes, isImmutableRows);
- if (!isAppendOnlySchema && cqCounter.increment(cqCounterFamily)) {
+ if (!isAppendOnlySchema && colDef.getEncodedQualifier() == null
+ && cqCounter.increment(cqCounterFamily)) {
changedCqCounters.put(cqCounterFamily, cqCounter.getNextQualifier(cqCounterFamily));
}
if (SchemaUtil.isPKColumn(column)) {
@@ -3222,7 +3286,7 @@ public class MetaDataClient {
return encodingScheme;
}
- private ImmutableStorageScheme getImmutableStorageSchemeForIndex(ImmutableStorageScheme immutableStorageSchemeProp, String schemaName, String tableName, TransactionFactory.Provider transactionProvider)
+ private void checkImmutableStorageSchemeForIndex(ImmutableStorageScheme immutableStorageSchemeProp, String schemaName, String tableName, TransactionFactory.Provider transactionProvider)
throws SQLException {
if (immutableStorageSchemeProp != ONE_CELL_PER_COLUMN && transactionProvider != null && transactionProvider.getTransactionProvider().isUnsupported(PhoenixTransactionProvider.Feature.COLUMN_ENCODING) ) {
throw new SQLExceptionInfo.Builder(
@@ -3232,7 +3296,6 @@ public class MetaDataClient {
.build()
.buildException();
}
- return immutableStorageSchemeProp;
}
/* This method handles mutation codes sent by phoenix server, except for TABLE_NOT_FOUND which
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/tool/SchemaExtractionProcessor.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/tool/SchemaExtractionProcessor.java
index 899f42faf6..41a8d16afd 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/tool/SchemaExtractionProcessor.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/tool/SchemaExtractionProcessor.java
@@ -52,6 +52,7 @@ import java.util.Arrays;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTION_PROVIDER;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY;
+import static org.apache.phoenix.schema.PTable.ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS;
import static org.apache.phoenix.util.MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES;
public class SchemaExtractionProcessor implements SchemaProcessor {
@@ -303,16 +304,19 @@ public class SchemaExtractionProcessor implements SchemaProcessor {
String columnInfoString = getColumnInfoStringForTable(table);
String propertiesString = convertPropertiesToString(false);
+ String columnQualifierString = convertColumnQualifierCountersToString(table);
- return generateTableDDLString(columnInfoString, propertiesString, pSchemaName, pTableName);
+ return generateTableDDLString(columnInfoString, propertiesString, columnQualifierString,
+ pSchemaName, pTableName);
}
private String generateTableDDLString(String columnInfoString, String propertiesString,
- String pSchemaName, String pTableName) {
+ String columnQualifierString, String pSchemaName, String pTableName) {
String quotedTableFullName = SchemaUtil.getFullTableNameWithQuotes(pSchemaName, pTableName);
StringBuilder outputBuilder = new StringBuilder(String.format(CREATE_TABLE,
quotedTableFullName));
- outputBuilder.append(columnInfoString).append(" ").append(propertiesString);
+ outputBuilder.append(columnInfoString).append(" ").append(propertiesString)
+ .append(columnQualifierString);
return outputBuilder.toString();
}
@@ -396,6 +400,36 @@ public class SchemaExtractionProcessor implements SchemaProcessor {
}
}
+ private String convertColumnQualifierCountersToString(PTable table) {
+ StringBuilder cqBuilder = new StringBuilder();
+ if (shouldGenerateWithDefaults) {
+ return cqBuilder.toString();
+ }
+ Map<String, Integer> cqCounterValues = table.getEncodedCQCounter().values();
+ ArrayList<String> cqCounters = new ArrayList<>(cqCounterValues.size());
+
+ for (Map.Entry<String, Integer> entry : cqCounterValues.entrySet()) {
+ Boolean include = table.getColumns().stream()
+ .filter(c -> !table.getPKColumns().contains(c))
+ .filter(pColumn -> table.getImmutableStorageScheme() == SINGLE_CELL_ARRAY_WITH_OFFSETS ?
+ pColumn.getFamilyName().getString().equalsIgnoreCase(entry.getKey()) : true)
+ .map(o -> table.getEncodingScheme().decode(o.getColumnQualifierBytes()))
+ .max(Integer::compare).map(maxCounter -> maxCounter != entry.getValue() - 1)
+ .orElse(false);
+ if (include) {
+ String def = "'" + entry.getKey() + "'=" + entry.getValue().toString();
+ cqCounters.add(def);
+ }
+ }
+ if (cqCounters.size() > 0) {
+ cqBuilder.append(" COLUMN_QUALIFIER_COUNTER");
+ cqBuilder.append(" (");
+ cqBuilder.append(StringUtils.join( ", ", cqCounters));
+ cqBuilder.append(')');
+ }
+ return cqBuilder.toString();
+ }
+
private String convertPropertiesToString(boolean forIndex) {
StringBuilder optionBuilder = new StringBuilder();
for(Map.Entry<String, String> entry : definedProps.entrySet()) {
@@ -468,14 +502,28 @@ public class SchemaExtractionProcessor implements SchemaProcessor {
return getColumnInfoString(table, colInfo, columns, pkColumns);
}
+ private boolean shouldContainQualifier(PTable table, List<PColumn> columns)
+ {
+ if (columns.size() == 0 || shouldGenerateWithDefaults) {
+ return false;
+ }
+
+ return table.getType() == PTableType.TABLE && table.getEncodingScheme() != PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS;
+ }
+
private String getColumnInfoString(PTable table, StringBuilder colInfo, List<PColumn> columns,
List<PColumn> pkColumns) {
+ boolean appendQualifier = shouldContainQualifier(table, columns);
ArrayList<String> colDefs = new ArrayList<>(columns.size());
for (PColumn col : columns) {
String def = extractColumn(col);
if (pkColumns.size() == 1 && pkColumns.contains(col)) {
def += " PRIMARY KEY" + extractPKColumnAttributes(col);
}
+ if (appendQualifier && !pkColumns.contains(col)) {
+ def += " ENCODED_QUALIFIER " +
+ table.getEncodingScheme().decode(col.getColumnQualifierBytes());
+ }
colDefs.add(def);
}
if (colDefs.size() > 0) {