You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@phoenix.apache.org by GitBox <gi...@apache.org> on 2020/12/17 02:25:23 UTC
[GitHub] [phoenix] gjacoby126 commented on a change in pull request #1021: PHOENIX-6267 View Index PK Fixed Width Field Truncation
gjacoby126 commented on a change in pull request #1021:
URL: https://github.com/apache/phoenix/pull/1021#discussion_r544758977
##########
File path: phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java
##########
@@ -458,6 +457,113 @@ public void testMultiTenantViewGlobalIndex() throws Exception {
}
}
+ @Test
+ public void testRowKeyComposition() throws Exception {
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ // View fixed, index variable
+ createTableForRowKeyTests(conn, "DATE_TIME1, INT1", "TEXT1");
+
+ // Variable lens in the middle. Both end with fixed
+ createTableForRowKeyTests(conn, "DATE_TIME1, TEXT1, TEXT3, INT1",
+ "TEXT1, TEXT4, EMAIL1 DESC, DATE_TIME1 DESC, DOUBLE1");
+ createTableForRowKeyTests(conn, "DATE_TIME1, TEXT1, TEXT3, INT1",
+ "TEXT1, TEXT4, EMAIL1, DATE_TIME1 DESC, DOUBLE1");
+ createTableForRowKeyTests(conn, "DATE_TIME1, TEXT1, INT1",
+ "TEXT1, DATE_TIME1 DESC, DOUBLE1");
+ createTableForRowKeyTests(conn, "DATE_TIME1, TEXT1, INT1",
+ "TEXT1, DATE_TIME1 DESC, INT1, DOUBLE1");
+
+ // index ends with fix length, data var length
+ createTableForRowKeyTests(conn, "DATE_TIME1, INT1, EMAIL1",
+ "TEXT1, DATE_TIME1 DESC, DOUBLE1");
+
+ // Desc separators
+ createTableForRowKeyTests(conn, "DATE_TIME1, INT1", "TEXT1 DESC");
+ createTableForRowKeyTests(conn, "DATE_TIME1, INT1 DESC", "TEXT1 DESC");
+ createTableForRowKeyTests(conn, "DATE_TIME1, INT1 DESC", "TEXT1");
+ createTableForRowKeyTests(conn,"DATE_TIME1, TEXT3 DESC, INT1 DESC, TEXT2 DESC, TEXT4 DESC", "TEXT1 DESC, DOUBLE1 DESC");
+ createTableForRowKeyTests(conn, "DATE_TIME1 DESC, TEXT3 DESC, INT1, DOUBLE1 DESC", "TEXT1");
+
+ // Both index and data end with var length
+ createTableForRowKeyTests(conn,"DATE_TIME1 DESC, TEXT3 DESC, INT1, TEXT4, EMAIL1", "TEXT1");
+ createTableForRowKeyTests(conn,"DATE_TIME1 DESC, TEXT3, INT1, EMAIL1", "TEXT1, TEXT4");
+ }
+ }
+
+ private void createTableForRowKeyTests(Connection conn, String viewPkColumns, String indexPKColumns)
+ throws SQLException {
+ final String fullTableName = "TBL_"+generateUniqueName();
+ final String fullViewName = "VW_" + generateUniqueName();
+ final String fullIndexName = "IDX_" + generateUniqueName();
+ String keyPrefix = "9Ab";
+ String tableDdl = String.format("CREATE TABLE IF NOT EXISTS %s "
+ + " (ORGANIZATION_ID CHAR(15) NOT NULL, KEY_PREFIX CHAR(3) NOT NULL, LAST_UPDATE DATE "
+ + " CONSTRAINT PK PRIMARY KEY (ORGANIZATION_ID, KEY_PREFIX)"
+ + " )VERSIONS=1, IMMUTABLE_ROWS=%s, MULTI_TENANT=%s, REPLICATION_SCOPE=1, IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, COLUMN_ENCODED_BYTES=0", fullTableName, (isNamespaceMapped? "true" : "false"),(isNamespaceMapped? "true" : "false"));
Review comment:
nit: line too long
##########
File path: phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java
##########
@@ -458,6 +457,113 @@ public void testMultiTenantViewGlobalIndex() throws Exception {
}
}
+ @Test
+ public void testRowKeyComposition() throws Exception {
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ // View fixed, index variable
+ createTableForRowKeyTests(conn, "DATE_TIME1, INT1", "TEXT1");
+
+ // Variable lens in the middle. Both end with fixed
+ createTableForRowKeyTests(conn, "DATE_TIME1, TEXT1, TEXT3, INT1",
+ "TEXT1, TEXT4, EMAIL1 DESC, DATE_TIME1 DESC, DOUBLE1");
+ createTableForRowKeyTests(conn, "DATE_TIME1, TEXT1, TEXT3, INT1",
+ "TEXT1, TEXT4, EMAIL1, DATE_TIME1 DESC, DOUBLE1");
+ createTableForRowKeyTests(conn, "DATE_TIME1, TEXT1, INT1",
+ "TEXT1, DATE_TIME1 DESC, DOUBLE1");
+ createTableForRowKeyTests(conn, "DATE_TIME1, TEXT1, INT1",
+ "TEXT1, DATE_TIME1 DESC, INT1, DOUBLE1");
+
+ // index ends with fix length, data var length
+ createTableForRowKeyTests(conn, "DATE_TIME1, INT1, EMAIL1",
+ "TEXT1, DATE_TIME1 DESC, DOUBLE1");
+
+ // Desc separators
+ createTableForRowKeyTests(conn, "DATE_TIME1, INT1", "TEXT1 DESC");
+ createTableForRowKeyTests(conn, "DATE_TIME1, INT1 DESC", "TEXT1 DESC");
+ createTableForRowKeyTests(conn, "DATE_TIME1, INT1 DESC", "TEXT1");
+ createTableForRowKeyTests(conn,"DATE_TIME1, TEXT3 DESC, INT1 DESC, TEXT2 DESC, TEXT4 DESC", "TEXT1 DESC, DOUBLE1 DESC");
+ createTableForRowKeyTests(conn, "DATE_TIME1 DESC, TEXT3 DESC, INT1, DOUBLE1 DESC", "TEXT1");
+
+ // Both index and data end with var length
+ createTableForRowKeyTests(conn,"DATE_TIME1 DESC, TEXT3 DESC, INT1, TEXT4, EMAIL1", "TEXT1");
+ createTableForRowKeyTests(conn,"DATE_TIME1 DESC, TEXT3, INT1, EMAIL1", "TEXT1, TEXT4");
+ }
+ }
+
+ private void createTableForRowKeyTests(Connection conn, String viewPkColumns, String indexPKColumns)
+ throws SQLException {
+ final String fullTableName = "TBL_"+generateUniqueName();
+ final String fullViewName = "VW_" + generateUniqueName();
+ final String fullIndexName = "IDX_" + generateUniqueName();
+ String keyPrefix = "9Ab";
+ String tableDdl = String.format("CREATE TABLE IF NOT EXISTS %s "
+ + " (ORGANIZATION_ID CHAR(15) NOT NULL, KEY_PREFIX CHAR(3) NOT NULL, LAST_UPDATE DATE "
+ + " CONSTRAINT PK PRIMARY KEY (ORGANIZATION_ID, KEY_PREFIX)"
+ + " )VERSIONS=1, IMMUTABLE_ROWS=%s, MULTI_TENANT=%s, REPLICATION_SCOPE=1, IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, COLUMN_ENCODED_BYTES=0", fullTableName, (isNamespaceMapped? "true" : "false"),(isNamespaceMapped? "true" : "false"));
+ String viewDdl = String.format("CREATE VIEW IF NOT EXISTS %s (DATE_TIME1 DATE NOT NULL, TEXT1 VARCHAR, TEXT3 VARCHAR, INT1 BIGINT NOT NULL, DATE_TIME2 DATE, DATE_TIME3 DATE, INT2 BIGINT, INT3 INTEGER, DOUBLE1 DECIMAL(12, 3), "
+ + "DOUBLE2 DECIMAL(12, 3), DOUBLE3 DECIMAL(12, 3), TEXT2 VARCHAR, TEXT4 VARCHAR, EMAIL1 VARCHAR, PHONE1 VARCHAR, URL1 VARCHAR "
+ + "CONSTRAINT PKVIEW PRIMARY KEY (%s)) AS SELECT * FROM %s WHERE KEY_PREFIX = '%s'", fullViewName,viewPkColumns, fullTableName, keyPrefix);
+ String indexDdl = String.format("CREATE INDEX IF NOT EXISTS %s "
+ + "ON %s (%s)\n"
+ + "INCLUDE (INT2, DOUBLE3, DOUBLE2, DATE_TIME3, DATE_TIME2, TEXT2, URL1, INT3, LAST_UPDATE,"
+ + " PHONE1)", fullIndexName, fullViewName, indexPKColumns);
+
+ conn.createStatement().execute(tableDdl);
+ conn.createStatement().execute(viewDdl);
+ conn.createStatement().execute(indexDdl);
+
+ String childViewName = String.format("S_%s.\"%s\"", generateUniqueName(), keyPrefix);
+ String viewChildDDl = String.format("CREATE VIEW IF NOT EXISTS %s AS SELECT * FROM %s", childViewName, fullViewName);
+ Connection conn2 = null;
+ if (isNamespaceMapped) {
+ conn2 = getTenantConnection(TENANT1);
+ } else {
+ conn2 = conn;
+ }
+ conn2.createStatement().execute(viewChildDDl);
+
+ // We picked this number because it ends with the separator byte.
+ int int1= 1792;
+ String upsert = "UPSERT INTO " + childViewName +
+ "(DATE_TIME1, INT1, TEXT1, TEXT2, DOUBLE1, DATE_TIME2, DATE_TIME3, INT3, DOUBLE2, DOUBLE3, "
+ + "TEXT3, TEXT4, EMAIL1, PHONE1, URL1"
+ + (isNamespaceMapped? ") " : ",ORGANIZATION_ID) ")
+ + " VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?,?" + (isNamespaceMapped? ")":",?)");
+ PreparedStatement upsertStmt = conn2.prepareStatement(upsert);
+ Date date = Date.valueOf("2019-02-17");
+ upsertStmt.setDate(1, date);
+ upsertStmt.setInt(2, int1);
+ upsertStmt.setString(3, "text1");
+ upsertStmt.setString(4, "text2"); // TEXT2
+ upsertStmt.setDouble(5,254.564);
+ upsertStmt.setDate(6, Date.valueOf("2019-01-04"));
+ upsertStmt.setDate(7, Date.valueOf("2019-07-16"));
+ upsertStmt.setInt(8, int1);
+ upsertStmt.setDouble(9, 4.09);
+ upsertStmt.setDouble(10, 0.249);
+ upsertStmt.setString(11,"text3"); // TEXT3
+ upsertStmt.setString(12, null); // TEXT4
+ upsertStmt.setString(13,"VScZBIjkO3QyUCMtkUEgmvL9xH0KJjwKi1gpxRv1ghonWcUMoksTWFKR4SD2yUg9@gmail.com");
+ upsertStmt.setString(14, "5827959502");
+ upsertStmt.setString(15, "https://www.sssssss.com");
+ if (!isNamespaceMapped) {
+ upsertStmt.setString(16, TENANT1);
+ }
+ upsertStmt.executeUpdate();
+ conn2.commit();
+
+ String select = "SELECT INT1 FROM " + childViewName
+ + " WHERE TEXT1='text1' LIMIT 10";
+ ResultSet rs1 = conn2.createStatement().executeQuery("EXPLAIN " + select);
+ String actualExplainPlan = QueryUtil.getExplainPlan(rs1);
+ System.out.println(actualExplainPlan);
Review comment:
Shouldn't print to console in a unit test
##########
File path: phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java
##########
@@ -458,6 +457,113 @@ public void testMultiTenantViewGlobalIndex() throws Exception {
}
}
+ @Test
+ public void testRowKeyComposition() throws Exception {
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ // View fixed, index variable
+ createTableForRowKeyTests(conn, "DATE_TIME1, INT1", "TEXT1");
+
+ // Variable lens in the middle. Both end with fixed
+ createTableForRowKeyTests(conn, "DATE_TIME1, TEXT1, TEXT3, INT1",
+ "TEXT1, TEXT4, EMAIL1 DESC, DATE_TIME1 DESC, DOUBLE1");
+ createTableForRowKeyTests(conn, "DATE_TIME1, TEXT1, TEXT3, INT1",
+ "TEXT1, TEXT4, EMAIL1, DATE_TIME1 DESC, DOUBLE1");
+ createTableForRowKeyTests(conn, "DATE_TIME1, TEXT1, INT1",
+ "TEXT1, DATE_TIME1 DESC, DOUBLE1");
+ createTableForRowKeyTests(conn, "DATE_TIME1, TEXT1, INT1",
+ "TEXT1, DATE_TIME1 DESC, INT1, DOUBLE1");
+
+ // index ends with fix length, data var length
+ createTableForRowKeyTests(conn, "DATE_TIME1, INT1, EMAIL1",
+ "TEXT1, DATE_TIME1 DESC, DOUBLE1");
+
+ // Desc separators
+ createTableForRowKeyTests(conn, "DATE_TIME1, INT1", "TEXT1 DESC");
+ createTableForRowKeyTests(conn, "DATE_TIME1, INT1 DESC", "TEXT1 DESC");
+ createTableForRowKeyTests(conn, "DATE_TIME1, INT1 DESC", "TEXT1");
+ createTableForRowKeyTests(conn,"DATE_TIME1, TEXT3 DESC, INT1 DESC, TEXT2 DESC, TEXT4 DESC", "TEXT1 DESC, DOUBLE1 DESC");
+ createTableForRowKeyTests(conn, "DATE_TIME1 DESC, TEXT3 DESC, INT1, DOUBLE1 DESC", "TEXT1");
+
+ // Both index and data end with var length
+ createTableForRowKeyTests(conn,"DATE_TIME1 DESC, TEXT3 DESC, INT1, TEXT4, EMAIL1", "TEXT1");
+ createTableForRowKeyTests(conn,"DATE_TIME1 DESC, TEXT3, INT1, EMAIL1", "TEXT1, TEXT4");
+ }
+ }
+
+ private void createTableForRowKeyTests(Connection conn, String viewPkColumns, String indexPKColumns)
Review comment:
this function not only creates but verifies
##########
File path: phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java
##########
@@ -458,6 +457,113 @@ public void testMultiTenantViewGlobalIndex() throws Exception {
}
}
+ @Test
+ public void testRowKeyComposition() throws Exception {
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ // View fixed, index variable
+ createTableForRowKeyTests(conn, "DATE_TIME1, INT1", "TEXT1");
+
+ // Variable lens in the middle. Both end with fixed
+ createTableForRowKeyTests(conn, "DATE_TIME1, TEXT1, TEXT3, INT1",
+ "TEXT1, TEXT4, EMAIL1 DESC, DATE_TIME1 DESC, DOUBLE1");
+ createTableForRowKeyTests(conn, "DATE_TIME1, TEXT1, TEXT3, INT1",
+ "TEXT1, TEXT4, EMAIL1, DATE_TIME1 DESC, DOUBLE1");
+ createTableForRowKeyTests(conn, "DATE_TIME1, TEXT1, INT1",
+ "TEXT1, DATE_TIME1 DESC, DOUBLE1");
+ createTableForRowKeyTests(conn, "DATE_TIME1, TEXT1, INT1",
+ "TEXT1, DATE_TIME1 DESC, INT1, DOUBLE1");
+
+ // index ends with fix length, data var length
+ createTableForRowKeyTests(conn, "DATE_TIME1, INT1, EMAIL1",
+ "TEXT1, DATE_TIME1 DESC, DOUBLE1");
+
+ // Desc separators
+ createTableForRowKeyTests(conn, "DATE_TIME1, INT1", "TEXT1 DESC");
+ createTableForRowKeyTests(conn, "DATE_TIME1, INT1 DESC", "TEXT1 DESC");
+ createTableForRowKeyTests(conn, "DATE_TIME1, INT1 DESC", "TEXT1");
+ createTableForRowKeyTests(conn,"DATE_TIME1, TEXT3 DESC, INT1 DESC, TEXT2 DESC, TEXT4 DESC", "TEXT1 DESC, DOUBLE1 DESC");
+ createTableForRowKeyTests(conn, "DATE_TIME1 DESC, TEXT3 DESC, INT1, DOUBLE1 DESC", "TEXT1");
+
+ // Both index and data end with var length
+ createTableForRowKeyTests(conn,"DATE_TIME1 DESC, TEXT3 DESC, INT1, TEXT4, EMAIL1", "TEXT1");
+ createTableForRowKeyTests(conn,"DATE_TIME1 DESC, TEXT3, INT1, EMAIL1", "TEXT1, TEXT4");
+ }
+ }
+
+ private void createTableForRowKeyTests(Connection conn, String viewPkColumns, String indexPKColumns)
+ throws SQLException {
+ final String fullTableName = "TBL_"+generateUniqueName();
+ final String fullViewName = "VW_" + generateUniqueName();
+ final String fullIndexName = "IDX_" + generateUniqueName();
+ String keyPrefix = "9Ab";
+ String tableDdl = String.format("CREATE TABLE IF NOT EXISTS %s "
+ + " (ORGANIZATION_ID CHAR(15) NOT NULL, KEY_PREFIX CHAR(3) NOT NULL, LAST_UPDATE DATE "
+ + " CONSTRAINT PK PRIMARY KEY (ORGANIZATION_ID, KEY_PREFIX)"
+ + " )VERSIONS=1, IMMUTABLE_ROWS=%s, MULTI_TENANT=%s, REPLICATION_SCOPE=1, IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, COLUMN_ENCODED_BYTES=0", fullTableName, (isNamespaceMapped? "true" : "false"),(isNamespaceMapped? "true" : "false"));
+ String viewDdl = String.format("CREATE VIEW IF NOT EXISTS %s (DATE_TIME1 DATE NOT NULL, TEXT1 VARCHAR, TEXT3 VARCHAR, INT1 BIGINT NOT NULL, DATE_TIME2 DATE, DATE_TIME3 DATE, INT2 BIGINT, INT3 INTEGER, DOUBLE1 DECIMAL(12, 3), "
+ + "DOUBLE2 DECIMAL(12, 3), DOUBLE3 DECIMAL(12, 3), TEXT2 VARCHAR, TEXT4 VARCHAR, EMAIL1 VARCHAR, PHONE1 VARCHAR, URL1 VARCHAR "
Review comment:
Do we need this many fields to reproduce the issue / prove its fixed? Looks like we never use more than 5 at a time?
##########
File path: phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java
##########
@@ -458,6 +457,113 @@ public void testMultiTenantViewGlobalIndex() throws Exception {
}
}
+ @Test
+ public void testRowKeyComposition() throws Exception {
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ // View fixed, index variable
+ createTableForRowKeyTests(conn, "DATE_TIME1, INT1", "TEXT1");
+
+ // Variable lens in the middle. Both end with fixed
+ createTableForRowKeyTests(conn, "DATE_TIME1, TEXT1, TEXT3, INT1",
+ "TEXT1, TEXT4, EMAIL1 DESC, DATE_TIME1 DESC, DOUBLE1");
+ createTableForRowKeyTests(conn, "DATE_TIME1, TEXT1, TEXT3, INT1",
+ "TEXT1, TEXT4, EMAIL1, DATE_TIME1 DESC, DOUBLE1");
+ createTableForRowKeyTests(conn, "DATE_TIME1, TEXT1, INT1",
+ "TEXT1, DATE_TIME1 DESC, DOUBLE1");
+ createTableForRowKeyTests(conn, "DATE_TIME1, TEXT1, INT1",
+ "TEXT1, DATE_TIME1 DESC, INT1, DOUBLE1");
+
+ // index ends with fix length, data var length
+ createTableForRowKeyTests(conn, "DATE_TIME1, INT1, EMAIL1",
+ "TEXT1, DATE_TIME1 DESC, DOUBLE1");
+
+ // Desc separators
+ createTableForRowKeyTests(conn, "DATE_TIME1, INT1", "TEXT1 DESC");
+ createTableForRowKeyTests(conn, "DATE_TIME1, INT1 DESC", "TEXT1 DESC");
+ createTableForRowKeyTests(conn, "DATE_TIME1, INT1 DESC", "TEXT1");
+ createTableForRowKeyTests(conn,"DATE_TIME1, TEXT3 DESC, INT1 DESC, TEXT2 DESC, TEXT4 DESC", "TEXT1 DESC, DOUBLE1 DESC");
+ createTableForRowKeyTests(conn, "DATE_TIME1 DESC, TEXT3 DESC, INT1, DOUBLE1 DESC", "TEXT1");
+
+ // Both index and data end with var length
+ createTableForRowKeyTests(conn,"DATE_TIME1 DESC, TEXT3 DESC, INT1, TEXT4, EMAIL1", "TEXT1");
+ createTableForRowKeyTests(conn,"DATE_TIME1 DESC, TEXT3, INT1, EMAIL1", "TEXT1, TEXT4");
+ }
+ }
+
+ private void createTableForRowKeyTests(Connection conn, String viewPkColumns, String indexPKColumns)
+ throws SQLException {
+ final String fullTableName = "TBL_"+generateUniqueName();
+ final String fullViewName = "VW_" + generateUniqueName();
+ final String fullIndexName = "IDX_" + generateUniqueName();
+ String keyPrefix = "9Ab";
+ String tableDdl = String.format("CREATE TABLE IF NOT EXISTS %s "
+ + " (ORGANIZATION_ID CHAR(15) NOT NULL, KEY_PREFIX CHAR(3) NOT NULL, LAST_UPDATE DATE "
+ + " CONSTRAINT PK PRIMARY KEY (ORGANIZATION_ID, KEY_PREFIX)"
+ + " )VERSIONS=1, IMMUTABLE_ROWS=%s, MULTI_TENANT=%s, REPLICATION_SCOPE=1, IMMUTABLE_STORAGE_SCHEME=ONE_CELL_PER_COLUMN, COLUMN_ENCODED_BYTES=0", fullTableName, (isNamespaceMapped? "true" : "false"),(isNamespaceMapped? "true" : "false"));
+ String viewDdl = String.format("CREATE VIEW IF NOT EXISTS %s (DATE_TIME1 DATE NOT NULL, TEXT1 VARCHAR, TEXT3 VARCHAR, INT1 BIGINT NOT NULL, DATE_TIME2 DATE, DATE_TIME3 DATE, INT2 BIGINT, INT3 INTEGER, DOUBLE1 DECIMAL(12, 3), "
Review comment:
nit: line too long
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org