You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@phoenix.apache.org by GitBox <gi...@apache.org> on 2020/07/30 21:10:03 UTC
[GitHub] [phoenix] gjacoby126 commented on a change in pull request #848: Phoenix-5947 Add tests and extensions to Schema Extraction Tool utility
gjacoby126 commented on a change in pull request #848:
URL: https://github.com/apache/phoenix/pull/848#discussion_r463259391
##########
File path: phoenix-tools/src/it/java/org/apache/phoenix/schema/SchemaExtractionToolIT.java
##########
@@ -210,4 +204,242 @@ public void testCreateViewIndexStatement() throws Exception {
Assert.assertEquals(createIndexStatement.toUpperCase(), set.getOutput().toUpperCase());
}
}
+
+ @Test
+ public void testSaltedTableStatement() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String query = "create table " + pTableFullName +
+ "(a_integer integer not null CONSTRAINT pk PRIMARY KEY (a_integer)) SALT_BUCKETS=16";
+ conn.createStatement().execute(query);
+ conn.commit();
+ String [] args = {"-tb", tableName, "-s", schemaName};
+ SchemaExtractionTool set = new SchemaExtractionTool();
+ set.setConf(conn.unwrap(PhoenixConnection.class).getQueryServices().getConfiguration());
+ set.run(args);
+ String actualProperties = set.getOutput().substring(set.getOutput().lastIndexOf(")")+1);
+ Assert.assertEquals(true, actualProperties.contains("SALT_BUCKETS=16"));
+ }
+ }
+
+ @Test
+ public void testCreateTableWithPKConstraint() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
Review comment:
Several of these tests are of the form:
1. Create a table according to some SQL and table name
2. Run SchemaExtractionTool
3. Assert that output is equal to the original SQL.
Can we extract that logic into a helper function and you just pass in different tablename and SQL in each test?
##########
File path: phoenix-tools/src/it/java/org/apache/phoenix/schema/SchemaExtractionToolIT.java
##########
@@ -210,4 +204,242 @@ public void testCreateViewIndexStatement() throws Exception {
Assert.assertEquals(createIndexStatement.toUpperCase(), set.getOutput().toUpperCase());
}
}
+
+ @Test
+ public void testSaltedTableStatement() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String query = "create table " + pTableFullName +
+ "(a_integer integer not null CONSTRAINT pk PRIMARY KEY (a_integer)) SALT_BUCKETS=16";
+ conn.createStatement().execute(query);
+ conn.commit();
+ String [] args = {"-tb", tableName, "-s", schemaName};
+ SchemaExtractionTool set = new SchemaExtractionTool();
+ set.setConf(conn.unwrap(PhoenixConnection.class).getQueryServices().getConfiguration());
+ set.run(args);
+ String actualProperties = set.getOutput().substring(set.getOutput().lastIndexOf(")")+1);
+ Assert.assertEquals(true, actualProperties.contains("SALT_BUCKETS=16"));
+ }
+ }
+
+ @Test
+ public void testCreateTableWithPKConstraint() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String query = "create table " + pTableFullName +
+ "(a_char CHAR(15) NOT NULL, " +
+ "b_char CHAR(15) NOT NULL, " +
+ "c_bigint BIGINT NOT NULL CONSTRAINT PK PRIMARY KEY (a_char, b_char, c_bigint)) IMMUTABLE_ROWS=TRUE";
+ conn.createStatement().execute(query);
+ conn.commit();
+ String [] args = {"-tb", tableName, "-s", schemaName};
+ SchemaExtractionTool set = new SchemaExtractionTool();
+ set.setConf(conn.unwrap(PhoenixConnection.class).getQueryServices().getConfiguration());
+ set.run(args);
+
+ Assert.assertEquals(query.toUpperCase(), set.getOutput().toUpperCase());
+ }
+ }
+
+ @Test
+ public void testCreateTableWithArrayColumn() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String query = "create table " + pTableFullName +
+ "(a_char CHAR(15) NOT NULL, " +
+ "b_char CHAR(10) NOT NULL, " +
+ "c_var_array VARCHAR ARRAY, " +
+ "d_char_array CHAR(15) ARRAY[3] CONSTRAINT PK PRIMARY KEY (a_char, b_char)) " +
+ "TTL=2592000, IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, DISABLE_TABLE_SOR=true, REPLICATION_SCOPE=1";
+ conn.createStatement().execute(query);
+ conn.commit();
+ String[] args = {"-tb", tableName, "-s", schemaName};
+ SchemaExtractionTool set = new SchemaExtractionTool();
+ set.setConf(conn.unwrap(PhoenixConnection.class).getQueryServices().getConfiguration());
+ set.run(args);
+
+ Assert.assertEquals(query.toUpperCase(), set.getOutput().toUpperCase());
+ }
+ }
+
+ @Test
+ public void testCreateTableWithNonDefaultColumnFamily() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String query = "create table " + pTableFullName +
+ "(a_char CHAR(15) NOT NULL, " +
+ "b_char CHAR(10) NOT NULL, " +
+ "\"av\".\"_\" CHAR(1), " +
+ "\"bv\".\"_\" CHAR(1), " +
+ "\"cv\".\"_\" CHAR(1), " +
+ "\"dv\".\"_\" CHAR(1) CONSTRAINT PK PRIMARY KEY (a_char, b_char)) " +
+ "TTL=1209600, IMMUTABLE_ROWS=true, IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, SALT_BUCKETS=16, DISABLE_TABLE_SOR=true, MULTI_TENANT=true";
+ conn.createStatement().execute(query);
+ conn.commit();
+ String[] args = {"-tb", tableName, "-s", schemaName};
+ SchemaExtractionTool set = new SchemaExtractionTool();
+ set.setConf(conn.unwrap(PhoenixConnection.class).getQueryServices().getConfiguration());
+ set.run(args);
+
+ Assert.assertEquals(query.toUpperCase(), set.getOutput().toUpperCase());
+ }
+ }
+
+ @Test
+ public void testCreateTableWithUniversalCFProperties() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ String properties = "KEEP_DELETED_CELLS=TRUE, TTL=1209600, IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, REPLICATION_SCOPE=1";
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String query = "create table " + pTableFullName +
+ "(a_char CHAR(15) NOT NULL, " +
+ "b_char CHAR(10) NOT NULL, " +
+ "\"av\".\"_\" CHAR(1), " +
+ "\"bv\".\"_\" CHAR(1), " +
+ "\"cv\".\"_\" CHAR(1), " +
+ "\"dv\".\"_\" CHAR(1) CONSTRAINT PK PRIMARY KEY (a_char, b_char)) " + properties;
+ conn.createStatement().execute(query);
+ conn.commit();
+ String[] args = {"-tb", tableName, "-s", schemaName};
+ SchemaExtractionTool set = new SchemaExtractionTool();
+ set.setConf(conn.unwrap(PhoenixConnection.class).getQueryServices().getConfiguration());
+ set.run(args);
+
+ Assert.assertEquals(query.toUpperCase(), set.getOutput().toUpperCase());
+ }
+ }
+
+ @Test
+ public void testCreateTableWithDefaultCFProperties() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ String properties = "KEEP_DELETED_CELLS=TRUE, TTL=1209600, IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, REPLICATION_SCOPE=1, DEFAULT_COLUMN_FAMILY=cv";
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String query = "create table " + pTableFullName +
+ "(a_char CHAR(15) NOT NULL, " +
+ "b_char CHAR(10) NOT NULL, " +
+ "\"av\".\"_\" CHAR(1), " +
+ "\"bv\".\"_\" CHAR(1), " +
+ "\"cv\".\"_\" CHAR(1), " +
+ "\"dv\".\"_\" CHAR(1) CONSTRAINT PK PRIMARY KEY (a_char, b_char)) " + properties;
+ conn.createStatement().execute(query);
+ conn.commit();
+ String[] args = {"-tb", tableName, "-s", schemaName};
+ SchemaExtractionTool set = new SchemaExtractionTool();
+ set.setConf(conn.unwrap(PhoenixConnection.class).getQueryServices().getConfiguration());
+ set.run(args);
+
+ Assert.assertTrue(compareProperties(properties, set.getOutput().substring(set.getOutput().lastIndexOf(")")+1)));
+ }
+ }
+
+ @Test
+ public void testCreateTableWithMultipleCFProperties() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ String properties = "\"av\".VERSIONS=2, \"bv\".VERSIONS=3, " +
+ "\"cv\".VERSIONS=4, DATA_BLOCK_ENCODING=DIFF, " +
+ "IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, SALT_BUCKETS=16, DISABLE_TABLE_SOR=true, MULTI_TENANT=true";
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String query = "create table " + pTableFullName +
+ "(a_char CHAR(15) NOT NULL, " +
+ "b_char CHAR(10) NOT NULL, " +
+ "\"av\".\"_\" CHAR(1), " +
+ "\"bv\".\"_\" CHAR(1), " +
+ "\"cv\".\"_\" CHAR(1), " +
+ "\"dv\".\"_\" CHAR(1) CONSTRAINT PK PRIMARY KEY (a_char, b_char)) " + properties;
+ conn.createStatement().execute(query);
+ conn.commit();
+ String[] args = {"-tb", tableName, "-s", schemaName};
+ SchemaExtractionTool set = new SchemaExtractionTool();
+ set.setConf(conn.unwrap(PhoenixConnection.class).getQueryServices().getConfiguration());
+ set.run(args);
+
+ Assert.assertTrue(compareProperties(properties, set.getOutput().substring(set.getOutput().lastIndexOf(")")+1)));
+ }
+ }
+
+ @Test
+ public void testCreateTableWithMultipleCFProperties2() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ String properties = "\"av\".VERSIONS=2, \"bv\".VERSIONS=2, " +
+ "DATA_BLOCK_ENCODING=DIFF, " +
+ "IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, SALT_BUCKETS=16, DISABLE_TABLE_SOR=true, MULTI_TENANT=true";
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String query = "create table " + pTableFullName +
+ "(a_char CHAR(15) NOT NULL, " +
+ "b_char CHAR(10) NOT NULL, " +
+ "\"av\".\"_\" CHAR(1), " +
+ "\"bv\".\"_\" CHAR(1), " +
+ "\"cv\".\"_\" CHAR(1) CONSTRAINT PK PRIMARY KEY (a_char, b_char)) " + properties;
+ conn.createStatement().execute(query);
+ conn.commit();
+ String[] args = {"-tb", tableName, "-s", schemaName};
+ SchemaExtractionTool set = new SchemaExtractionTool();
+ set.setConf(conn.unwrap(PhoenixConnection.class).getQueryServices().getConfiguration());
+ set.run(args);
+
+ Assert.assertTrue(compareProperties(properties, set.getOutput().substring(set.getOutput().lastIndexOf(")")+1)));
Review comment:
likewise, please specify what you're trying to show here. You also might want to extract "set.getOutput().substring(set.getOutput().lastIndexOf(")")+1)" into a function with a user-friendly name since you do this operation in many tests, I think to get the properties.
##########
File path: phoenix-tools/src/it/java/org/apache/phoenix/schema/SchemaExtractionToolIT.java
##########
@@ -210,4 +204,242 @@ public void testCreateViewIndexStatement() throws Exception {
Assert.assertEquals(createIndexStatement.toUpperCase(), set.getOutput().toUpperCase());
}
}
+
+ @Test
+ public void testSaltedTableStatement() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String query = "create table " + pTableFullName +
+ "(a_integer integer not null CONSTRAINT pk PRIMARY KEY (a_integer)) SALT_BUCKETS=16";
+ conn.createStatement().execute(query);
+ conn.commit();
+ String [] args = {"-tb", tableName, "-s", schemaName};
+ SchemaExtractionTool set = new SchemaExtractionTool();
+ set.setConf(conn.unwrap(PhoenixConnection.class).getQueryServices().getConfiguration());
+ set.run(args);
+ String actualProperties = set.getOutput().substring(set.getOutput().lastIndexOf(")")+1);
+ Assert.assertEquals(true, actualProperties.contains("SALT_BUCKETS=16"));
+ }
+ }
+
+ @Test
+ public void testCreateTableWithPKConstraint() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String query = "create table " + pTableFullName +
+ "(a_char CHAR(15) NOT NULL, " +
+ "b_char CHAR(15) NOT NULL, " +
+ "c_bigint BIGINT NOT NULL CONSTRAINT PK PRIMARY KEY (a_char, b_char, c_bigint)) IMMUTABLE_ROWS=TRUE";
+ conn.createStatement().execute(query);
+ conn.commit();
+ String [] args = {"-tb", tableName, "-s", schemaName};
+ SchemaExtractionTool set = new SchemaExtractionTool();
+ set.setConf(conn.unwrap(PhoenixConnection.class).getQueryServices().getConfiguration());
+ set.run(args);
+
+ Assert.assertEquals(query.toUpperCase(), set.getOutput().toUpperCase());
+ }
+ }
+
+ @Test
+ public void testCreateTableWithArrayColumn() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String query = "create table " + pTableFullName +
+ "(a_char CHAR(15) NOT NULL, " +
+ "b_char CHAR(10) NOT NULL, " +
+ "c_var_array VARCHAR ARRAY, " +
+ "d_char_array CHAR(15) ARRAY[3] CONSTRAINT PK PRIMARY KEY (a_char, b_char)) " +
+ "TTL=2592000, IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, DISABLE_TABLE_SOR=true, REPLICATION_SCOPE=1";
+ conn.createStatement().execute(query);
+ conn.commit();
+ String[] args = {"-tb", tableName, "-s", schemaName};
+ SchemaExtractionTool set = new SchemaExtractionTool();
+ set.setConf(conn.unwrap(PhoenixConnection.class).getQueryServices().getConfiguration());
+ set.run(args);
+
+ Assert.assertEquals(query.toUpperCase(), set.getOutput().toUpperCase());
+ }
+ }
+
+ @Test
+ public void testCreateTableWithNonDefaultColumnFamily() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String query = "create table " + pTableFullName +
+ "(a_char CHAR(15) NOT NULL, " +
+ "b_char CHAR(10) NOT NULL, " +
+ "\"av\".\"_\" CHAR(1), " +
+ "\"bv\".\"_\" CHAR(1), " +
+ "\"cv\".\"_\" CHAR(1), " +
+ "\"dv\".\"_\" CHAR(1) CONSTRAINT PK PRIMARY KEY (a_char, b_char)) " +
+ "TTL=1209600, IMMUTABLE_ROWS=true, IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, SALT_BUCKETS=16, DISABLE_TABLE_SOR=true, MULTI_TENANT=true";
+ conn.createStatement().execute(query);
+ conn.commit();
+ String[] args = {"-tb", tableName, "-s", schemaName};
+ SchemaExtractionTool set = new SchemaExtractionTool();
+ set.setConf(conn.unwrap(PhoenixConnection.class).getQueryServices().getConfiguration());
+ set.run(args);
+
+ Assert.assertEquals(query.toUpperCase(), set.getOutput().toUpperCase());
+ }
+ }
+
+ @Test
+ public void testCreateTableWithUniversalCFProperties() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ String properties = "KEEP_DELETED_CELLS=TRUE, TTL=1209600, IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, REPLICATION_SCOPE=1";
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String query = "create table " + pTableFullName +
+ "(a_char CHAR(15) NOT NULL, " +
+ "b_char CHAR(10) NOT NULL, " +
+ "\"av\".\"_\" CHAR(1), " +
+ "\"bv\".\"_\" CHAR(1), " +
+ "\"cv\".\"_\" CHAR(1), " +
+ "\"dv\".\"_\" CHAR(1) CONSTRAINT PK PRIMARY KEY (a_char, b_char)) " + properties;
+ conn.createStatement().execute(query);
+ conn.commit();
+ String[] args = {"-tb", tableName, "-s", schemaName};
+ SchemaExtractionTool set = new SchemaExtractionTool();
+ set.setConf(conn.unwrap(PhoenixConnection.class).getQueryServices().getConfiguration());
+ set.run(args);
+
+ Assert.assertEquals(query.toUpperCase(), set.getOutput().toUpperCase());
+ }
+ }
+
+ @Test
+ public void testCreateTableWithDefaultCFProperties() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ String properties = "KEEP_DELETED_CELLS=TRUE, TTL=1209600, IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, REPLICATION_SCOPE=1, DEFAULT_COLUMN_FAMILY=cv";
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String query = "create table " + pTableFullName +
+ "(a_char CHAR(15) NOT NULL, " +
+ "b_char CHAR(10) NOT NULL, " +
+ "\"av\".\"_\" CHAR(1), " +
+ "\"bv\".\"_\" CHAR(1), " +
+ "\"cv\".\"_\" CHAR(1), " +
+ "\"dv\".\"_\" CHAR(1) CONSTRAINT PK PRIMARY KEY (a_char, b_char)) " + properties;
+ conn.createStatement().execute(query);
+ conn.commit();
+ String[] args = {"-tb", tableName, "-s", schemaName};
+ SchemaExtractionTool set = new SchemaExtractionTool();
+ set.setConf(conn.unwrap(PhoenixConnection.class).getQueryServices().getConfiguration());
+ set.run(args);
+
+ Assert.assertTrue(compareProperties(properties, set.getOutput().substring(set.getOutput().lastIndexOf(")")+1)));
+ }
+ }
+
+ @Test
+ public void testCreateTableWithMultipleCFProperties() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ String properties = "\"av\".VERSIONS=2, \"bv\".VERSIONS=3, " +
+ "\"cv\".VERSIONS=4, DATA_BLOCK_ENCODING=DIFF, " +
+ "IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, SALT_BUCKETS=16, DISABLE_TABLE_SOR=true, MULTI_TENANT=true";
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String query = "create table " + pTableFullName +
+ "(a_char CHAR(15) NOT NULL, " +
+ "b_char CHAR(10) NOT NULL, " +
+ "\"av\".\"_\" CHAR(1), " +
+ "\"bv\".\"_\" CHAR(1), " +
+ "\"cv\".\"_\" CHAR(1), " +
+ "\"dv\".\"_\" CHAR(1) CONSTRAINT PK PRIMARY KEY (a_char, b_char)) " + properties;
+ conn.createStatement().execute(query);
+ conn.commit();
+ String[] args = {"-tb", tableName, "-s", schemaName};
+ SchemaExtractionTool set = new SchemaExtractionTool();
+ set.setConf(conn.unwrap(PhoenixConnection.class).getQueryServices().getConfiguration());
+ set.run(args);
+
+ Assert.assertTrue(compareProperties(properties, set.getOutput().substring(set.getOutput().lastIndexOf(")")+1)));
+ }
+ }
+
+ @Test
+ public void testCreateTableWithMultipleCFProperties2() throws Exception {
Review comment:
nit: please use a more informative name.
##########
File path: phoenix-tools/src/main/java/org/apache/phoenix/schema/SchemaExtractionProcessor.java
##########
@@ -250,12 +247,31 @@ private void setHTableProperties(HTableDescriptor htd) {
}
}
- private void setHColumnFamilyProperties(HColumnDescriptor columnDescriptor) {
- Map<ImmutableBytesWritable, ImmutableBytesWritable> propsMap = columnDescriptor.getValues();
- for (Map.Entry<ImmutableBytesWritable, ImmutableBytesWritable> entry : propsMap.entrySet()) {
+ private void setHColumnFamilyProperties(HColumnDescriptor[] columnDescriptors) {
+ Map<ImmutableBytesWritable, ImmutableBytesWritable> propsMap = columnDescriptors[0].getValues();
+ for(Map.Entry<ImmutableBytesWritable, ImmutableBytesWritable> entry : propsMap.entrySet()) {
ImmutableBytesWritable key = entry.getKey();
- ImmutableBytesWritable value = entry.getValue();
- definedProps.put(Bytes.toString(key.get()), Bytes.toString(value.get()));
+ ImmutableBytesWritable globalValue = entry.getValue();
+ Map<String, String> cfToPropertyValueMap = new HashMap<String, String>();
+ Set<ImmutableBytesWritable> cfPropertyValueSet = new HashSet<ImmutableBytesWritable>();
+ for(HColumnDescriptor columnDescriptor: columnDescriptors) {
+ String columnFamilyName = Bytes.toString(columnDescriptor.getName());
+ ImmutableBytesWritable value = columnDescriptor.getValues().get(key);
+ // check if it is universal properties
+ if (SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES.contains(Bytes.toString(key.get()))) {
+ definedProps.put(Bytes.toString(key.get()), Bytes.toString(value.get()));
+ break;
+ }
+ cfToPropertyValueMap.put(columnFamilyName, Bytes.toString(value.get()));
+ cfPropertyValueSet.add(value);
+ }
+ if (cfPropertyValueSet.size() > 1) {
+ for(Map.Entry<String, String> mapEntry: cfToPropertyValueMap.entrySet()) {
+ definedProps.put(String.format("%s.%s", mapEntry.getKey(), Bytes.toString(key.get())), mapEntry.getValue());
Review comment:
I'm curious why we're qualifying the CF properties in one flat HashSet instead of a HashSet<String, HashSet<String, String>>? Can we guarantee that no CF properties will ever have a period in them?
##########
File path: phoenix-tools/src/it/java/org/apache/phoenix/schema/SchemaExtractionToolIT.java
##########
@@ -210,4 +204,242 @@ public void testCreateViewIndexStatement() throws Exception {
Assert.assertEquals(createIndexStatement.toUpperCase(), set.getOutput().toUpperCase());
}
}
+
+ @Test
+ public void testSaltedTableStatement() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String query = "create table " + pTableFullName +
+ "(a_integer integer not null CONSTRAINT pk PRIMARY KEY (a_integer)) SALT_BUCKETS=16";
+ conn.createStatement().execute(query);
+ conn.commit();
+ String [] args = {"-tb", tableName, "-s", schemaName};
+ SchemaExtractionTool set = new SchemaExtractionTool();
+ set.setConf(conn.unwrap(PhoenixConnection.class).getQueryServices().getConfiguration());
+ set.run(args);
+ String actualProperties = set.getOutput().substring(set.getOutput().lastIndexOf(")")+1);
+ Assert.assertEquals(true, actualProperties.contains("SALT_BUCKETS=16"));
+ }
+ }
+
+ @Test
+ public void testCreateTableWithPKConstraint() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String query = "create table " + pTableFullName +
+ "(a_char CHAR(15) NOT NULL, " +
+ "b_char CHAR(15) NOT NULL, " +
+ "c_bigint BIGINT NOT NULL CONSTRAINT PK PRIMARY KEY (a_char, b_char, c_bigint)) IMMUTABLE_ROWS=TRUE";
+ conn.createStatement().execute(query);
+ conn.commit();
+ String [] args = {"-tb", tableName, "-s", schemaName};
+ SchemaExtractionTool set = new SchemaExtractionTool();
+ set.setConf(conn.unwrap(PhoenixConnection.class).getQueryServices().getConfiguration());
+ set.run(args);
+
+ Assert.assertEquals(query.toUpperCase(), set.getOutput().toUpperCase());
+ }
+ }
+
+ @Test
+ public void testCreateTableWithArrayColumn() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String query = "create table " + pTableFullName +
+ "(a_char CHAR(15) NOT NULL, " +
+ "b_char CHAR(10) NOT NULL, " +
+ "c_var_array VARCHAR ARRAY, " +
+ "d_char_array CHAR(15) ARRAY[3] CONSTRAINT PK PRIMARY KEY (a_char, b_char)) " +
+ "TTL=2592000, IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, DISABLE_TABLE_SOR=true, REPLICATION_SCOPE=1";
+ conn.createStatement().execute(query);
+ conn.commit();
+ String[] args = {"-tb", tableName, "-s", schemaName};
+ SchemaExtractionTool set = new SchemaExtractionTool();
+ set.setConf(conn.unwrap(PhoenixConnection.class).getQueryServices().getConfiguration());
+ set.run(args);
+
+ Assert.assertEquals(query.toUpperCase(), set.getOutput().toUpperCase());
+ }
+ }
+
+ @Test
+ public void testCreateTableWithNonDefaultColumnFamily() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String query = "create table " + pTableFullName +
+ "(a_char CHAR(15) NOT NULL, " +
+ "b_char CHAR(10) NOT NULL, " +
+ "\"av\".\"_\" CHAR(1), " +
+ "\"bv\".\"_\" CHAR(1), " +
+ "\"cv\".\"_\" CHAR(1), " +
+ "\"dv\".\"_\" CHAR(1) CONSTRAINT PK PRIMARY KEY (a_char, b_char)) " +
+ "TTL=1209600, IMMUTABLE_ROWS=true, IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, SALT_BUCKETS=16, DISABLE_TABLE_SOR=true, MULTI_TENANT=true";
+ conn.createStatement().execute(query);
+ conn.commit();
+ String[] args = {"-tb", tableName, "-s", schemaName};
+ SchemaExtractionTool set = new SchemaExtractionTool();
+ set.setConf(conn.unwrap(PhoenixConnection.class).getQueryServices().getConfiguration());
+ set.run(args);
+
+ Assert.assertEquals(query.toUpperCase(), set.getOutput().toUpperCase());
+ }
+ }
+
+ @Test
+ public void testCreateTableWithUniversalCFProperties() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ String properties = "KEEP_DELETED_CELLS=TRUE, TTL=1209600, IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, REPLICATION_SCOPE=1";
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String query = "create table " + pTableFullName +
+ "(a_char CHAR(15) NOT NULL, " +
+ "b_char CHAR(10) NOT NULL, " +
+ "\"av\".\"_\" CHAR(1), " +
+ "\"bv\".\"_\" CHAR(1), " +
+ "\"cv\".\"_\" CHAR(1), " +
+ "\"dv\".\"_\" CHAR(1) CONSTRAINT PK PRIMARY KEY (a_char, b_char)) " + properties;
+ conn.createStatement().execute(query);
+ conn.commit();
+ String[] args = {"-tb", tableName, "-s", schemaName};
+ SchemaExtractionTool set = new SchemaExtractionTool();
+ set.setConf(conn.unwrap(PhoenixConnection.class).getQueryServices().getConfiguration());
+ set.run(args);
+
+ Assert.assertEquals(query.toUpperCase(), set.getOutput().toUpperCase());
+ }
+ }
+
+ @Test
+ public void testCreateTableWithDefaultCFProperties() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ String properties = "KEEP_DELETED_CELLS=TRUE, TTL=1209600, IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, REPLICATION_SCOPE=1, DEFAULT_COLUMN_FAMILY=cv";
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String query = "create table " + pTableFullName +
+ "(a_char CHAR(15) NOT NULL, " +
+ "b_char CHAR(10) NOT NULL, " +
+ "\"av\".\"_\" CHAR(1), " +
+ "\"bv\".\"_\" CHAR(1), " +
+ "\"cv\".\"_\" CHAR(1), " +
+ "\"dv\".\"_\" CHAR(1) CONSTRAINT PK PRIMARY KEY (a_char, b_char)) " + properties;
+ conn.createStatement().execute(query);
+ conn.commit();
+ String[] args = {"-tb", tableName, "-s", schemaName};
+ SchemaExtractionTool set = new SchemaExtractionTool();
+ set.setConf(conn.unwrap(PhoenixConnection.class).getQueryServices().getConfiguration());
+ set.run(args);
+
+ Assert.assertTrue(compareProperties(properties, set.getOutput().substring(set.getOutput().lastIndexOf(")")+1)));
Review comment:
nit: comment on what exactly you're checking here would be useful.
##########
File path: phoenix-tools/src/it/java/org/apache/phoenix/schema/SchemaExtractionToolIT.java
##########
@@ -210,4 +204,242 @@ public void testCreateViewIndexStatement() throws Exception {
Assert.assertEquals(createIndexStatement.toUpperCase(), set.getOutput().toUpperCase());
}
}
+
+ @Test
+ public void testSaltedTableStatement() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String query = "create table " + pTableFullName +
+ "(a_integer integer not null CONSTRAINT pk PRIMARY KEY (a_integer)) SALT_BUCKETS=16";
+ conn.createStatement().execute(query);
+ conn.commit();
+ String [] args = {"-tb", tableName, "-s", schemaName};
+ SchemaExtractionTool set = new SchemaExtractionTool();
+ set.setConf(conn.unwrap(PhoenixConnection.class).getQueryServices().getConfiguration());
+ set.run(args);
+ String actualProperties = set.getOutput().substring(set.getOutput().lastIndexOf(")")+1);
+ Assert.assertEquals(true, actualProperties.contains("SALT_BUCKETS=16"));
+ }
+ }
+
+ @Test
+ public void testCreateTableWithPKConstraint() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String query = "create table " + pTableFullName +
+ "(a_char CHAR(15) NOT NULL, " +
+ "b_char CHAR(15) NOT NULL, " +
+ "c_bigint BIGINT NOT NULL CONSTRAINT PK PRIMARY KEY (a_char, b_char, c_bigint)) IMMUTABLE_ROWS=TRUE";
+ conn.createStatement().execute(query);
+ conn.commit();
+ String [] args = {"-tb", tableName, "-s", schemaName};
+ SchemaExtractionTool set = new SchemaExtractionTool();
+ set.setConf(conn.unwrap(PhoenixConnection.class).getQueryServices().getConfiguration());
+ set.run(args);
+
+ Assert.assertEquals(query.toUpperCase(), set.getOutput().toUpperCase());
+ }
+ }
+
+ @Test
+ public void testCreateTableWithArrayColumn() throws Exception {
+ String tableName = generateUniqueName();
+ String schemaName = generateUniqueName();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String pTableFullName = SchemaUtil.getQualifiedTableName(schemaName, tableName);
+ String query = "create table " + pTableFullName +
+ "(a_char CHAR(15) NOT NULL, " +
+ "b_char CHAR(10) NOT NULL, " +
+ "c_var_array VARCHAR ARRAY, " +
+ "d_char_array CHAR(15) ARRAY[3] CONSTRAINT PK PRIMARY KEY (a_char, b_char)) " +
Review comment:
Good edge case to check!
##########
File path: phoenix-tools/src/main/java/org/apache/phoenix/schema/SchemaExtractionProcessor.java
##########
@@ -312,9 +336,8 @@ public Connection getConnection() throws SQLException {
private String getColumnInfoStringForTable(PTable table) {
StringBuilder colInfo = new StringBuilder();
-
- List<PColumn> columns = table.getColumns();
- List<PColumn> pkColumns = table.getPKColumns();
+ List<PColumn> columns = table.getBucketNum() == null ? table.getColumns() : table.getColumns().subList(1, table.getColumns().size());
Review comment:
Good catch for salted tables, those are easy to miss.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org