You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by sk...@apache.org on 2020/08/05 20:51:26 UTC
[phoenix] branch 4.x-PHOENIX-5946 updated: Phoenix-5947 Add tests
and extensions to Schema Extraction Tool utility (#848)
This is an automated email from the ASF dual-hosted git repository.
skadam pushed a commit to branch 4.x-PHOENIX-5946
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/4.x-PHOENIX-5946 by this push:
new 2b72758 Phoenix-5947 Add tests and extensions to Schema Extraction Tool utility (#848)
2b72758 is described below
commit 2b72758fa5ad1cfdbe94cfe6f941414bb5ccaaef
Author: Qinrui Li <qi...@salesforce.com>
AuthorDate: Wed Aug 5 16:51:17 2020 -0400
Phoenix-5947 Add tests and extensions to Schema Extraction Tool utility (#848)
---
.../java/org/apache/phoenix/schema/PTableImpl.java | 6 +
.../phoenix/schema/SchemaExtractionToolIT.java | 395 ++++++++++++++-------
.../phoenix/schema/SchemaExtractionProcessor.java | 116 ++++--
3 files changed, 350 insertions(+), 167 deletions(-)
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
index 9c8c5a0..f336559 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
@@ -27,6 +27,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DISABLE_WAL;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ENCODING_SCHEME;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_ROWS;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_COLUMN_FAMILY_NAME;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_STORAGE_SCHEME;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.MULTI_TENANT;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SALT_BUCKETS;
@@ -43,6 +44,7 @@ import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_TRANSACTION_
import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_UPDATE_CACHE_FREQUENCY;
import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_USE_STATS_FOR_PARALLELIZATION;
import static org.apache.phoenix.schema.SaltingUtil.SALTING_COLUMN;
+import static org.apache.phoenix.schema.TableProperty.DEFAULT_COLUMN_FAMILY;
import static org.apache.phoenix.schema.types.PDataType.TRUE_BYTES;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_TTL_NOT_DEFINED;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.MIN_VIEW_TTL_HWM;
@@ -422,6 +424,9 @@ public class PTableImpl implements PTable {
}
public Builder setDefaultFamilyName(PName defaultFamilyName) {
+ if (defaultFamilyName != null){
+ propertyValues.put(DEFAULT_COLUMN_FAMILY_NAME, defaultFamilyName.getString());
+ }
this.defaultFamilyName = defaultFamilyName;
return this;
}
@@ -2086,6 +2091,7 @@ public class PTableImpl implements PTable {
map.put(TRANSACTIONAL, String.valueOf(DEFAULT_TRANSACTIONAL));
map.put(MULTI_TENANT, String.valueOf(DEFAULT_MULTI_TENANT));
map.put(SALT_BUCKETS, String.valueOf(DEFAULT_SALT_BUCKETS));
+ map.put(DEFAULT_COLUMN_FAMILY_NAME, String.valueOf(DEFAULT_COLUMN_FAMILY));
return Collections.unmodifiableMap(map);
}
diff --git a/phoenix-tools/src/it/java/org/apache/phoenix/schema/SchemaExtractionToolIT.java b/phoenix-tools/src/it/java/org/apache/phoenix/schema/SchemaExtractionToolIT.java
index 7acb96c..ff672e4 100644
--- a/phoenix-tools/src/it/java/org/apache/phoenix/schema/SchemaExtractionToolIT.java
+++ b/phoenix-tools/src/it/java/org/apache/phoenix/schema/SchemaExtractionToolIT.java
@@ -13,9 +13,13 @@ import org.junit.Test;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
-import java.util.Collections;
-
+import java.util.List;
import java.util.Map;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Set;
+import java.util.HashSet;
+import java.util.Arrays;
import java.util.Properties;
import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
@@ -32,24 +36,92 @@ public class SchemaExtractionToolIT extends BaseTest {
public void testCreateTableStatement() throws Exception {
String tableName = generateUniqueName();
String schemaName = generateUniqueName();
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String createTableStmt = "CREATE TABLE "+ pTableFullName + "(K VARCHAR NOT NULL PRIMARY KEY, "
+ + "V1 VARCHAR, V2 VARCHAR) TTL=2592000, IMMUTABLE_ROWS=TRUE, DISABLE_WAL=TRUE";
+ List<String> queries = new ArrayList<String>(){};
+ queries.add(createTableStmt);
+ String result = runSchemaExtractionTool(schemaName, tableName, null, queries);
+ Assert.assertEquals(createTableStmt, result.toUpperCase());
+ }
+
+ @Test
+ public void testCreateIndexStatement() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ String indexName = generateUniqueName();
+ String indexName1 = generateUniqueName();
+ String indexName2 = generateUniqueName();
+ String properties = "TTL=2592000,IMMUTABLE_ROWS=true,DISABLE_WAL=true";
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String createTableStatement = "CREATE TABLE "+pTableFullName + "(k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)"
+ + properties;
+ String createIndexStatement = "CREATE INDEX "+indexName + " ON "+pTableFullName+"(v1 DESC) INCLUDE (v2)";
+ String createIndexStatement1 = "CREATE INDEX "+indexName1 + " ON "+pTableFullName+"(v2 DESC) INCLUDE (v1)";
+ String createIndexStatement2 = "CREATE INDEX "+indexName2 + " ON "+pTableFullName+"(k)";
+ List<String> queries = new ArrayList<String>(){};
+ queries.add(createTableStatement);
+ queries.add(createIndexStatement);
+ queries.add(createIndexStatement1);
+ queries.add(createIndexStatement2);
+
+ String result = runSchemaExtractionTool(schemaName, indexName2, null, queries);
+ Assert.assertEquals(createIndexStatement2.toUpperCase(), result.toUpperCase());
+ }
+
+ @Test
+ public void testCreateViewStatement() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ String viewName = generateUniqueName();
String properties = "TTL=2592000,IMMUTABLE_ROWS=true,DISABLE_WAL=true";
- try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String createTableStmt = "CREATE TABLE "+pTableFullName + "(k BIGINT NOT NULL PRIMARY KEY, "
+ + "v1 VARCHAR, v2 VARCHAR)"
+ + properties;
+ String viewFullName = SchemaUtil.getQualifiedTableName(schemaName, viewName);
+ String viewFullName1 = SchemaUtil.getQualifiedTableName(schemaName, viewName+"1");
+ String createView = "CREATE VIEW "+viewFullName + "(id1 BIGINT, id2 BIGINT NOT NULL, "
+ + "id3 VARCHAR NOT NULL CONSTRAINT PKVIEW PRIMARY KEY (id2, id3 DESC)) "
+ + "AS SELECT * FROM "+pTableFullName;
+ String createView1 = "CREATE VIEW "+viewFullName1 + "(id1 BIGINT, id2 BIGINT NOT NULL, "
+ + "id3 VARCHAR NOT NULL CONSTRAINT PKVIEW PRIMARY KEY (id2, id3 DESC)) "
+ + "AS SELECT * FROM "+pTableFullName;
- String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
- conn.createStatement().execute("CREATE TABLE "+ pTableFullName + "(k VARCHAR NOT NULL PRIMARY KEY, "
- + "v1 VARCHAR, v2 VARCHAR)"
- + properties);
- conn.commit();
- String [] args = {"-tb", tableName, "-s", schemaName};
+ List<String> queries = new ArrayList<String>(){};
+ queries.add(createTableStmt);
+ queries.add(createView);
+ queries.add(createView1);
+ String result = runSchemaExtractionTool(schemaName, viewName, null, queries);
+ Assert.assertEquals(createView.toUpperCase(), result.toUpperCase());
- SchemaExtractionTool set = new SchemaExtractionTool();
- set.setConf(conn.unwrap(PhoenixConnection.class).getQueryServices().getConfiguration());
- set.run(args);
- String actualProperties = set.getOutput().substring(set.getOutput().lastIndexOf(")")+1).replace(" ","");
- Assert.assertEquals(3, actualProperties.split(",").length);
- }
+ }
+
+ @Test
+ public void testCreateViewIndexStatement() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ String viewName = generateUniqueName();
+ String childView = generateUniqueName();
+ String indexName = generateUniqueName();
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String createTableStmt = "CREATE TABLE "+pTableFullName + "(k BIGINT NOT NULL PRIMARY KEY, "
+ + "v1 VARCHAR, v2 VARCHAR)";
+ String viewFullName = SchemaUtil.getQualifiedTableName(schemaName, viewName);
+ String childviewName = SchemaUtil.getQualifiedTableName(schemaName, childView);
+ String createView = "CREATE VIEW "+viewFullName + "(id1 BIGINT, id2 BIGINT NOT NULL, "
+ + "id3 VARCHAR NOT NULL CONSTRAINT PKVIEW PRIMARY KEY (id2, id3 DESC)) "
+ + "AS SELECT * FROM "+pTableFullName;
+ String createView1 = "CREATE VIEW "+childviewName + " AS SELECT * FROM "+viewFullName;
+ String createIndexStatement = "CREATE INDEX "+indexName + " ON "+childviewName+"(id1) INCLUDE (v1)";
+ List<String> queries = new ArrayList<String>(){};
+ queries.add(createTableStmt);
+ queries.add(createView);
+ queries.add(createView1);
+ queries.add(createIndexStatement);
+ String result = runSchemaExtractionTool(schemaName, indexName, null, queries);
+ Assert.assertEquals(createIndexStatement.toUpperCase(), result.toUpperCase());
}
@Test
@@ -58,156 +130,213 @@ public class SchemaExtractionToolIT extends BaseTest {
String viewName = generateUniqueName();
String schemaName = generateUniqueName();
String tenantId = "abc";
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
- String properties = "TTL=2592000,IMMUTABLE_ROWS=true,DISABLE_WAL=true";
- SchemaExtractionTool set;
+ String createTableStmt = "CREATE TABLE "+pTableFullName + "(k BIGINT NOT NULL PRIMARY KEY, "
+ + "v1 VARCHAR, v2 VARCHAR)";
String viewFullName = SchemaUtil.getQualifiedTableName(schemaName, viewName);
- String createView = "CREATE VIEW "+viewFullName + "(id1 BIGINT, id2 BIGINT NOT NULL, "
+ String createViewStmt = "CREATE VIEW "+viewFullName + "(id1 BIGINT, id2 BIGINT NOT NULL, "
+ "id3 VARCHAR NOT NULL CONSTRAINT PKVIEW PRIMARY KEY (id2, id3 DESC)) "
+ "AS SELECT * FROM "+pTableFullName;
- try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
-
- conn.createStatement().execute("CREATE TABLE "+pTableFullName + "(k BIGINT NOT NULL PRIMARY KEY, "
- + "v1 VARCHAR, v2 VARCHAR)"
- + properties);
- set = new SchemaExtractionTool();
- set.setConf(conn.unwrap(PhoenixConnection.class).getQueryServices().getConfiguration());
- conn.commit();
- }
- try (Connection conn = getTenantConnection(getUrl(), tenantId)) {
- conn.createStatement().execute(createView);
- conn.commit();
- }
- String [] args = {"-tb", viewName, "-s", schemaName, "-t", tenantId};
- set.run(args);
- Assert.assertEquals(createView.toUpperCase(), set.getOutput().toUpperCase());
+ List<String> queries1 = new ArrayList<String>(){};
+ queries1.add(createTableStmt);
+ runSchemaExtractionTool(schemaName, tableName, null, queries1);
+ List<String> queries2 = new ArrayList<String>();
+ queries2.add(createViewStmt);
+ String result2 = runSchemaExtractionTool(schemaName, viewName, tenantId, queries2);
+ Assert.assertEquals(createViewStmt.toUpperCase(), result2.toUpperCase());
}
- private Connection getTenantConnection(String url, String tenantId) throws SQLException {
- Properties props = new Properties();
- props.setProperty(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId);
- return DriverManager.getConnection(url, props);
+ @Test
+ public void testSaltedTableStatement() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String query = "create table " + pTableFullName +
+ "(a_integer integer not null CONSTRAINT pk PRIMARY KEY (a_integer)) SALT_BUCKETS=16";
+ List<String> queries = new ArrayList<String>(){};
+ queries.add(query);
+ String result = runSchemaExtractionTool(schemaName, tableName, null, queries);
+ Assert.assertTrue(getProperties(result).contains("SALT_BUCKETS=16"));
}
@Test
- public void testCreateIndexStatement() throws Exception {
+ public void testCreateTableWithPKConstraint() throws Exception {
String tableName = generateUniqueName();
String schemaName = generateUniqueName();
- String indexName = generateUniqueName();
- String indexName1 = generateUniqueName();
- String indexName2 = generateUniqueName();
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- String properties = "TTL=2592000,IMMUTABLE_ROWS=true,DISABLE_WAL=true";
-
- try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
-
- String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
- conn.createStatement().execute("CREATE TABLE "+pTableFullName + "(k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR)"
- + properties);
-
- String createIndexStatement = "CREATE INDEX "+indexName + " ON "+pTableFullName+"(v1 DESC) INCLUDE (v2)";
-
- String createIndexStatement1 = "CREATE INDEX "+indexName1 + " ON "+pTableFullName+"(v2 DESC) INCLUDE (v1)";
-
- String createIndexStatement2 = "CREATE INDEX "+indexName2 + " ON "+pTableFullName+"(k)";
-
- conn.createStatement().execute(createIndexStatement);
- conn.createStatement().execute(createIndexStatement1);
- conn.createStatement().execute(createIndexStatement2);
- conn.commit();
- SchemaExtractionTool set = new SchemaExtractionTool();
- set.setConf(conn.unwrap(PhoenixConnection.class).getQueryServices().getConfiguration());
-
- String [] args = {"-tb", indexName, "-s", schemaName};
- set.run(args);
- Assert.assertEquals(createIndexStatement.toUpperCase(), set.getOutput().toUpperCase());
-
- String [] args1 = {"-tb", indexName1, "-s", schemaName};
- set.run(args1);
- Assert.assertEquals(createIndexStatement1.toUpperCase(), set.getOutput().toUpperCase());
-
- String [] args2 = {"-tb", indexName2, "-s", schemaName};
- set.run(args2);
- Assert.assertEquals(createIndexStatement2.toUpperCase(), set.getOutput().toUpperCase());
- }
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String query = "create table " + pTableFullName +
+ "(a_char CHAR(15) NOT NULL, " +
+ "b_char CHAR(15) NOT NULL, " +
+ "c_bigint BIGINT NOT NULL CONSTRAINT PK PRIMARY KEY (a_char, b_char, c_bigint)) IMMUTABLE_ROWS=TRUE";
+ List<String> queries = new ArrayList<String>(){};
+ queries.add(query);
+ String result = runSchemaExtractionTool(schemaName, tableName, null, queries);
+ Assert.assertEquals(query.toUpperCase(), result.toUpperCase());
}
@Test
- public void testCreateViewStatement() throws Exception {
+ public void testCreateTableWithArrayColumn() throws Exception {
String tableName = generateUniqueName();
String schemaName = generateUniqueName();
- String viewName = generateUniqueName();
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- String properties = "TTL=2592000,IMMUTABLE_ROWS=true,DISABLE_WAL=true";
-
- try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
-
- String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
- conn.createStatement().execute("CREATE TABLE "+pTableFullName + "(k BIGINT NOT NULL PRIMARY KEY, "
- + "v1 VARCHAR, v2 VARCHAR)"
- + properties);
- String viewFullName = SchemaUtil.getQualifiedTableName(schemaName, viewName);
- String viewFullName1 = SchemaUtil.getQualifiedTableName(schemaName, viewName+"1");
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String query = "create table " + pTableFullName +
+ "(a_char CHAR(15) NOT NULL, " +
+ "b_char CHAR(10) NOT NULL, " +
+ "c_var_array VARCHAR ARRAY, " +
+ "d_char_array CHAR(15) ARRAY[3] CONSTRAINT PK PRIMARY KEY (a_char, b_char)) " +
+ "TTL=2592000, IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, REPLICATION_SCOPE=1";
+ List<String> queries = new ArrayList<String>(){};
+ queries.add(query);
+ String result = runSchemaExtractionTool(schemaName, tableName, null, queries);
+ Assert.assertEquals(query.toUpperCase(), result.toUpperCase());
+ }
+ @Test
+ public void testCreateTableWithDefaultCFProperties() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ String properties = "KEEP_DELETED_CELLS=TRUE, TTL=1209600, IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, REPLICATION_SCOPE=1, DEFAULT_COLUMN_FAMILY=cv, SALT_BUCKETS=16, MULTI_TENANT=true";
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String query = "create table " + pTableFullName +
+ "(a_char CHAR(15) NOT NULL, " +
+ "b_char CHAR(10) NOT NULL, " +
+ "\"av\".\"_\" CHAR(1), " +
+ "\"bv\".\"_\" CHAR(1), " +
+ "\"cv\".\"_\" CHAR(1), " +
+ "\"dv\".\"_\" CHAR(1) CONSTRAINT PK PRIMARY KEY (a_char, b_char)) " + properties;
+ List<String> queries = new ArrayList<String>(){};
+ queries.add(query);
+ String result = runSchemaExtractionTool(schemaName, tableName, null, queries);
+ Assert.assertTrue(compareProperties(properties, getProperties(result)));
+ }
- String createView = "CREATE VIEW "+viewFullName + "(id1 BIGINT, id2 BIGINT NOT NULL, "
- + "id3 VARCHAR NOT NULL CONSTRAINT PKVIEW PRIMARY KEY (id2, id3 DESC)) "
- + "AS SELECT * FROM "+pTableFullName;
- String createView1 = "CREATE VIEW "+viewFullName1 + "(id1 BIGINT, id2 BIGINT NOT NULL, "
- + "id3 VARCHAR NOT NULL CONSTRAINT PKVIEW PRIMARY KEY (id2, id3 DESC)) "
- + "AS SELECT * FROM "+pTableFullName;
+ @Test
+ public void testCreateTableWithCFProperties() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ String properties = "\"av\".VERSIONS=2, \"bv\".VERSIONS=2, " +
+ "DATA_BLOCK_ENCODING=DIFF, " +
+ "IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, SALT_BUCKETS=16, MULTI_TENANT=true";
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String query = "create table " + pTableFullName +
+ "(a_char CHAR(15) NOT NULL, " +
+ "b_char CHAR(10) NOT NULL, " +
+ "\"av\".\"_\" CHAR(1), " +
+ "\"bv\".\"_\" CHAR(1), " +
+ "\"cv\".\"_\" CHAR(1) CONSTRAINT PK PRIMARY KEY (a_char, b_char)) " + properties;
+ List<String> queries = new ArrayList<String>(){};
+ queries.add(query);
+ String result = runSchemaExtractionTool(schemaName, tableName, null, queries);
+ Assert.assertTrue(compareProperties(properties, getProperties(result)));
+ }
- conn.createStatement().execute(createView);
- conn.createStatement().execute(createView1);
- conn.commit();
- String [] args = {"-tb", viewName, "-s", schemaName};
+ @Test
+ public void testCreateTableWithMultipleCF() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ String properties = "\"av\".VERSIONS=2, \"bv\".VERSIONS=3, " +
+ "\"cv\".VERSIONS=4, DATA_BLOCK_ENCODING=DIFF, " +
+ "IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, SALT_BUCKETS=16, MULTI_TENANT=true";
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ final String query = "create table " + pTableFullName +
+ "(a_char CHAR(15) NOT NULL, " +
+ "b_char CHAR(10) NOT NULL, " +
+ "\"av\".\"_\" CHAR(1), " +
+ "\"bv\".\"_\" CHAR(1), " +
+ "\"cv\".\"_\" CHAR(1), " +
+ "\"dv\".\"_\" CHAR(1) CONSTRAINT PK PRIMARY KEY (a_char, b_char)) " + properties;
+ List<String> queries = new ArrayList<String>(){};
+ queries.add(query);
+ String result = runSchemaExtractionTool(schemaName, tableName, null, queries);
+ Assert.assertTrue(compareProperties(properties, getProperties(result)));
+ }
- SchemaExtractionTool set = new SchemaExtractionTool();
- set.setConf(conn.unwrap(PhoenixConnection.class).getQueryServices().getConfiguration());
- set.run(args);
- Assert.assertEquals(createView.toUpperCase(), set.getOutput().toUpperCase());
- }
+ @Test
+ public void testCreateTableWithMultipleCFProperties() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ String properties = "\"av\".DATA_BLOCK_ENCODING=DIFF, \"bv\".DATA_BLOCK_ENCODING=DIFF, \"cv\".DATA_BLOCK_ENCODING=DIFF, " +
+ "IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, SALT_BUCKETS=16, MULTI_TENANT=true";
+ String simplifiedProperties = "DATA_BLOCK_ENCODING=DIFF, IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, SALT_BUCKETS=16, MULTI_TENANT=true";
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String query = "create table " + pTableFullName +
+ "(a_char CHAR(15) NOT NULL, " +
+ "b_char CHAR(10) NOT NULL, " +
+ "\"av\".\"_\" CHAR(1), " +
+ "\"bv\".\"_\" CHAR(1), " +
+ "\"cv\".\"_\" CHAR(1) CONSTRAINT PK PRIMARY KEY (a_char, b_char)) " + properties;
+ List<String> queries = new ArrayList<String>(){};
+ queries.add(query);
+ String result = runSchemaExtractionTool(schemaName, tableName, null, queries);
+ Assert.assertTrue(compareProperties(simplifiedProperties, getProperties(result)));
}
@Test
- public void testCreateViewIndexStatement() throws Exception {
+ public void testCreateIndexStatementWithColumnFamily() throws Exception {
String tableName = generateUniqueName();
String schemaName = generateUniqueName();
- String viewName = generateUniqueName();
- String childView = generateUniqueName();
String indexName = generateUniqueName();
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- String properties = "TTL=2592000,IMMUTABLE_ROWS=true,DISABLE_WAL=true";
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String createTableStmt = "CREATE TABLE "+pTableFullName + "(k VARCHAR NOT NULL PRIMARY KEY, \"av\".\"_\" CHAR(1), v2 VARCHAR)";
+ String createIndexStmt = "CREATE INDEX "+ indexName + " ON "+pTableFullName+ "(\"av\".\"_\")";
+ List<String> queries = new ArrayList<String>() {};
+ queries.add(createTableStmt);
+ queries.add(createIndexStmt);
+ String result = runSchemaExtractionTool(schemaName, indexName, null, queries);
+ Assert.assertEquals(createIndexStmt.toUpperCase(), result.toUpperCase());
+ }
- try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ private Connection getTenantConnection(String url, String tenantId) throws SQLException {
+ Properties props = new Properties();
+ props.setProperty(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId);
+ return DriverManager.getConnection(url, props);
+ }
- String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
- conn.createStatement().execute("CREATE TABLE "+pTableFullName + "(k BIGINT NOT NULL PRIMARY KEY, "
- + "v1 VARCHAR, v2 VARCHAR)"
- + properties);
- String viewFullName = SchemaUtil.getQualifiedTableName(schemaName, viewName);
- String childviewName = SchemaUtil.getQualifiedTableName(schemaName, childView);
+ private String runSchemaExtractionTool(String schemaName, String tableName, String tenantId, List<String> queries) throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ String output;
+ if (tenantId == null){
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ executeCreateStatements(conn, queries);
+ String [] args = {"-tb", tableName, "-s", schemaName};
+ output = extractSchema(conn, args);
+ }
+ } else {
+ try (Connection conn = getTenantConnection(getUrl(), tenantId)) {
+ executeCreateStatements(conn, queries);
+ String [] args = {"-tb", tableName, "-s", schemaName, "-t", tenantId};
+ output = extractSchema(conn, args);
+ }
+ }
+ return output;
+ }
- String createView = "CREATE VIEW "+viewFullName + "(id1 BIGINT, id2 BIGINT NOT NULL, "
- + "id3 VARCHAR NOT NULL CONSTRAINT PKVIEW PRIMARY KEY (id2, id3 DESC)) "
- + "AS SELECT * FROM "+pTableFullName;
+ private void executeCreateStatements(Connection conn, List<String> queries) throws SQLException {
+ for (String query: queries){
+ conn.createStatement().execute(query);
+ }
+ conn.commit();
+ }
- String createView1 = "CREATE VIEW "+childviewName + " AS SELECT * FROM "+viewFullName;
+ private String extractSchema(Connection conn, String [] args) throws Exception {
+ SchemaExtractionTool set = new SchemaExtractionTool();
+ set.setConf(conn.unwrap(PhoenixConnection.class).getQueryServices().getConfiguration());
+ set.run(args);
+ return set.getOutput();
+ }
- String createIndexStatement = "CREATE INDEX "+indexName + " ON "+childviewName+"(id1) INCLUDE (v1)";
+ private String getProperties(String query){
+ return query.substring(query.lastIndexOf(")")+1);
+ }
- conn.createStatement().execute(createView);
- conn.createStatement().execute(createView1);
- conn.createStatement().execute(createIndexStatement);
- conn.commit();
- String [] args = {"-tb", indexName, "-s", schemaName};
+ private boolean compareProperties(String prop1, String prop2){
+ String[] propArray1 = prop1.toUpperCase().replaceAll("\\s+","").split(",");
+ String[] propArray2 = prop2.toUpperCase().replaceAll("\\s+","").split(",");
- SchemaExtractionTool set = new SchemaExtractionTool();
- set.setConf(conn.unwrap(PhoenixConnection.class).getQueryServices().getConfiguration());
- set.run(args);
- Assert.assertEquals(createIndexStatement.toUpperCase(), set.getOutput().toUpperCase());
- }
+ Set<String> set1 = new HashSet<>(Arrays.asList(propArray1));
+ Set<String> set2 = new HashSet<>(Arrays.asList(propArray2));
+
+ return set1.equals(set2);
}
}
diff --git a/phoenix-tools/src/main/java/org/apache/phoenix/schema/SchemaExtractionProcessor.java b/phoenix-tools/src/main/java/org/apache/phoenix/schema/SchemaExtractionProcessor.java
index cd756c3..e3004e1 100644
--- a/phoenix-tools/src/main/java/org/apache/phoenix/schema/SchemaExtractionProcessor.java
+++ b/phoenix-tools/src/main/java/org/apache/phoenix/schema/SchemaExtractionProcessor.java
@@ -11,6 +11,7 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.mapreduce.util.ConnectionUtil;
import org.apache.phoenix.query.ConnectionQueryServices;
+import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.util.MetaDataUtil;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.SchemaUtil;
@@ -18,18 +19,17 @@ import org.apache.phoenix.util.SchemaUtil;
import java.io.IOException;
import java.sql.Connection;
import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.HashMap;
import java.util.HashSet;
-import java.util.List;
+import java.util.HashMap;
import java.util.Map;
import java.util.Set;
+import java.util.List;
+import java.util.ArrayList;
-public class SchemaExtractionProcessor {
- public static final String
- FEATURE_NOT_SUPPORTED_ON_EXTRACTION_TOOL =
- "Multiple CF feature not supported on extraction tool";
+import static org.apache.phoenix.util.MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES;
+
+public class SchemaExtractionProcessor {
Map<String, String> defaultProps = new HashMap<>();
Map<String, String> definedProps = new HashMap<>();
@@ -54,9 +54,6 @@ public class SchemaExtractionProcessor {
if (ddl != null) {
return ddl;
}
- if(this.table.getColumnFamilies().size() > 1 ) {
- throw new RuntimeException(FEATURE_NOT_SUPPORTED_ON_EXTRACTION_TOOL);
- }
if(this.table.getType().equals(PTableType.TABLE)) {
ddl = extractCreateTableDDL(this.table);
} else if(this.table.getType().equals(PTableType.INDEX)) {
@@ -116,7 +113,7 @@ public class SchemaExtractionProcessor {
indexedColumnsBuilder.append(", ");
}
indexedColumnsBuilder.append(column);
- if(sortOrderMap.get(column)!= SortOrder.getDefault()) {
+ if(sortOrderMap.containsKey(column) && sortOrderMap.get(column) != SortOrder.getDefault()) {
indexedColumnsBuilder.append(" ");
indexedColumnsBuilder.append(sortOrderMap.get(column));
}
@@ -129,7 +126,8 @@ public class SchemaExtractionProcessor {
if(columnNameSplit[0].equals("") || columnNameSplit[0].equalsIgnoreCase(defaultCF)) {
return columnNameSplit[1];
} else {
- return columnName.replace(":", ".");
+ return columnNameSplit.length > 1 ?
+ String.format("\"%s\".\"%s\"", columnNameSplit[0], columnNameSplit[1]) : columnNameSplit[0];
}
}
@@ -199,12 +197,11 @@ public class SchemaExtractionProcessor {
ConnectionQueryServices cqsi = getCQSIObject();
HTableDescriptor htd = getTableDescriptor(cqsi, table);
- HColumnDescriptor hcd = htd.getFamily(SchemaUtil.getEmptyColumnFamily(table));
-
+ HColumnDescriptor[] hcds = htd.getColumnFamilies();
populateDefaultProperties(table);
setPTableProperties(table);
setHTableProperties(htd);
- setHColumnFamilyProperties(hcd);
+ setHColumnFamilyProperties(hcds);
String columnInfoString = getColumnInfoStringForTable(table);
String propertiesString = convertPropertiesToString();
@@ -250,12 +247,31 @@ public class SchemaExtractionProcessor {
}
}
- private void setHColumnFamilyProperties(HColumnDescriptor columnDescriptor) {
- Map<ImmutableBytesWritable, ImmutableBytesWritable> propsMap = columnDescriptor.getValues();
- for (Map.Entry<ImmutableBytesWritable, ImmutableBytesWritable> entry : propsMap.entrySet()) {
+ private void setHColumnFamilyProperties(HColumnDescriptor[] columnDescriptors) {
+ Map<ImmutableBytesWritable, ImmutableBytesWritable> propsMap = columnDescriptors[0].getValues();
+ for(Map.Entry<ImmutableBytesWritable, ImmutableBytesWritable> entry : propsMap.entrySet()) {
ImmutableBytesWritable key = entry.getKey();
- ImmutableBytesWritable value = entry.getValue();
- definedProps.put(Bytes.toString(key.get()), Bytes.toString(value.get()));
+ ImmutableBytesWritable globalValue = entry.getValue();
+ Map<String, String> cfToPropertyValueMap = new HashMap<String, String>();
+ Set<ImmutableBytesWritable> cfPropertyValueSet = new HashSet<ImmutableBytesWritable>();
+ for(HColumnDescriptor columnDescriptor: columnDescriptors) {
+ String columnFamilyName = Bytes.toString(columnDescriptor.getName());
+ ImmutableBytesWritable value = columnDescriptor.getValues().get(key);
+ // check if it is universal properties
+ if (SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES.contains(Bytes.toString(key.get()))) {
+ definedProps.put(Bytes.toString(key.get()), Bytes.toString(value.get()));
+ break;
+ }
+ cfToPropertyValueMap.put(columnFamilyName, Bytes.toString(value.get()));
+ cfPropertyValueSet.add(value);
+ }
+ if (cfPropertyValueSet.size() > 1) {
+ for(Map.Entry<String, String> mapEntry: cfToPropertyValueMap.entrySet()) {
+ definedProps.put(String.format("%s.%s", mapEntry.getKey(), Bytes.toString(key.get())), mapEntry.getValue());
+ }
+ } else {
+ definedProps.put(Bytes.toString(key.get()), Bytes.toString(globalValue.get()));
+ }
}
}
@@ -278,14 +294,22 @@ public class SchemaExtractionProcessor {
private String convertPropertiesToString() {
StringBuilder optionBuilder = new StringBuilder();
-
for(Map.Entry<String, String> entry : definedProps.entrySet()) {
String key = entry.getKey();
String value = entry.getValue();
+ String columnFamilyName = QueryConstants.DEFAULT_COLUMN_FAMILY;
+
+ String[] colPropKey = key.split("\\.");
+ if (colPropKey.length > 1) {
+ columnFamilyName = colPropKey[0];
+ key = colPropKey[1];
+ }
+
if(value!=null && defaultProps.get(key) != null && !value.equals(defaultProps.get(key))) {
if (optionBuilder.length() != 0) {
optionBuilder.append(", ");
}
+ key = columnFamilyName.equals(QueryConstants.DEFAULT_COLUMN_FAMILY)? key : String.format("\"%s\".%s", columnFamilyName, key);
optionBuilder.append(key+"="+value);
}
}
@@ -312,9 +336,8 @@ public class SchemaExtractionProcessor {
private String getColumnInfoStringForTable(PTable table) {
StringBuilder colInfo = new StringBuilder();
-
- List<PColumn> columns = table.getColumns();
- List<PColumn> pkColumns = table.getPKColumns();
+ List<PColumn> columns = table.getBucketNum() == null ? table.getColumns() : table.getColumns().subList(1, table.getColumns().size());
+ List<PColumn> pkColumns = table.getBucketNum() == null ? table.getPKColumns() : table.getColumns().subList(1, table.getPKColumns().size());
return getColumnInfoString(table, colInfo, columns, pkColumns);
}
@@ -374,20 +397,33 @@ public class SchemaExtractionProcessor {
private String extractColumn(PColumn column) {
String colName = column.getName().getString();
+ if (column.getFamilyName() != null){
+ String colFamilyName = column.getFamilyName().getString();
+ // check if it is default column family name
+ colName = colFamilyName.equals(QueryConstants.DEFAULT_COLUMN_FAMILY)? colName : String.format("\"%s\".\"%s\"", colFamilyName, colName);
+ }
+ boolean isArrayType = column.getDataType().isArrayType();
String type = column.getDataType().getSqlTypeName();
+ Integer maxLength = column.getMaxLength();
+ Integer arrSize = column.getArraySize();
+ Integer scale = column.getScale();
StringBuilder buf = new StringBuilder(colName);
buf.append(' ');
- buf.append(type);
- Integer maxLength = column.getMaxLength();
- if (maxLength != null) {
- buf.append('(');
- buf.append(maxLength);
- Integer scale = column.getScale();
- if (scale != null) {
- buf.append(',');
- buf.append(scale); // has both max length and scale. For ex- decimal(10,2)
+
+ if (isArrayType) {
+ String arrayPrefix = type.split("\\s+")[0];
+ buf.append(arrayPrefix);
+ appendMaxLengthAndScale(buf, maxLength, scale);
+ buf.append(' ');
+ buf.append("ARRAY");
+ if (arrSize != null) {
+ buf.append('[');
+ buf.append(arrSize);
+ buf.append(']');
}
- buf.append(')');
+ } else {
+ buf.append(type);
+ appendMaxLengthAndScale(buf, maxLength, scale);
}
if (!column.isNullable()) {
@@ -398,6 +434,18 @@ public class SchemaExtractionProcessor {
return buf.toString();
}
+ private void appendMaxLengthAndScale(StringBuilder buf, Integer maxLength, Integer scale){
+ if (maxLength != null) {
+ buf.append('(');
+ buf.append(maxLength);
+ if (scale != null) {
+ buf.append(',');
+ buf.append(scale); // has both max length and scale. For ex- decimal(10,2)
+ }
+ buf.append(')');
+ }
+ }
+
private String extractPKColumnAttributes(PColumn column) {
StringBuilder buf = new StringBuilder();