You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@phoenix.apache.org by GitBox <gi...@apache.org> on 2022/01/10 20:20:47 UTC
[GitHub] [phoenix] gjacoby126 commented on a change in pull request #1371: PHOENIX-6620 TransformTool to fix unverified rows and do validation
gjacoby126 commented on a change in pull request #1371:
URL: https://github.com/apache/phoenix/pull/1371#discussion_r781516024
##########
File path: phoenix-core/src/it/java/org/apache/phoenix/end2end/TransformToolIT.java
##########
@@ -413,59 +431,314 @@ public void testTransformMutationFailureRepair() throws Exception {
String schemaName = generateUniqueName();
String dataTableName = generateUniqueName();
String dataTableFullName = SchemaUtil.getTableName(schemaName, dataTableName);
+ String parentViewName = "VWP_" + generateUniqueName();
+ String viewName = "VW_" + generateUniqueName();
+ String viewIdxName = "VWIDX_" + generateUniqueName();
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.setAutoCommit(true);
- IndexRegionObserver.setFailPreIndexUpdatesForTesting(true);
int numOfRows = 0;
createTableAndUpsertRows(conn, dataTableFullName, numOfRows);
+ String createParentViewSql = "CREATE VIEW " + parentViewName + " ( PARENT_VIEW_COL1 VARCHAR ) AS SELECT * FROM " + dataTableFullName;
+ conn.createStatement().execute(createParentViewSql);
+
+ String createViewSql = "CREATE VIEW " + viewName + " ( VIEW_COL1 INTEGER, VIEW_COL2 VARCHAR ) AS SELECT * FROM " + parentViewName;
+ conn.createStatement().execute(createViewSql);
+
+ String createViewIdxSql = "CREATE INDEX " + viewIdxName + " ON " + viewName + " (VIEW_COL1) include (VIEW_COL2) ";
+ conn.createStatement().execute(createViewIdxSql);
+
SingleCellIndexIT.assertMetadata(conn, PTable.ImmutableStorageScheme.ONE_CELL_PER_COLUMN, PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, dataTableFullName);
conn.createStatement().execute("ALTER TABLE " + dataTableFullName +
" SET IMMUTABLE_STORAGE_SCHEME=SINGLE_CELL_ARRAY_WITH_OFFSETS, COLUMN_ENCODED_BYTES=2");
SystemTransformRecord record = Transform.getTransformRecord(schemaName, dataTableName, null, null, conn.unwrap(PhoenixConnection.class));
assertNotNull(record);
assertMetadata(conn, PTable.ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS, PTable.QualifierEncodingScheme.TWO_BYTE_QUALIFIERS, record.getNewPhysicalTableName());
-
- String upsertQuery = String.format("UPSERT INTO %s VALUES(?, ?, ?)", dataTableFullName);
+ IndexRegionObserver.setFailPreIndexUpdatesForTesting(true);
+ String upsertQuery = String.format("UPSERT INTO %s VALUES(?, ?, ?)", viewName);
PreparedStatement stmt1 = conn.prepareStatement(upsertQuery);
try {
IndexToolIT.upsertRow(stmt1, 1);
fail("Transform table upsert should have failed");
} catch (Exception e) {
}
assertEquals(0, getRowCount(conn, record.getNewPhysicalTableName()));
- assertEquals(0, getRowCount(conn,dataTableFullName));
+ assertEquals(0, getRowCount(conn, dataTableFullName));
IndexRegionObserver.setFailPreIndexUpdatesForTesting(false);
IndexRegionObserver.setFailPostIndexUpdatesForTesting(true);
IndexToolIT.upsertRow(stmt1, 2);
+ IndexToolIT.upsertRow(stmt1, 3);
- assertEquals(1, getRowCount(conn, record.getNewPhysicalTableName()));
- assertEquals(1, getRowCount(conn,dataTableFullName));
+ assertEquals(2, getRowCount(conn, record.getNewPhysicalTableName()));
+ assertEquals(2, getRowCount(conn,dataTableFullName));
IndexRegionObserver.setFailPostIndexUpdatesForTesting(false);
IndexRegionObserver.setFailDataTableUpdatesForTesting(true);
try {
- IndexToolIT.upsertRow(stmt1, 3);
+ IndexToolIT.upsertRow(stmt1, 4);
fail("Data table upsert should have failed");
} catch (Exception e) {
}
- assertEquals(2, getRowCount(conn, record.getNewPhysicalTableName()));
- assertEquals(1, getRowCount(conn,dataTableFullName));
+ try {
+ IndexToolIT.upsertRow(stmt1, 5);
+ fail("Data table upsert should have failed");
+ } catch (Exception e) {
+ }
+ assertEquals(4, getRowCount(conn, record.getNewPhysicalTableName()));
+ assertEquals(2, getRowCount(conn,dataTableFullName));
IndexRegionObserver.setFailDataTableUpdatesForTesting(false);
List<String> args = getArgList(schemaName, dataTableName, null,
- null, null, null, false, false, false, false);
+ null, null, null, false, false, false, false, true);
runTransformTool(args.toArray(new String[0]), 0);
- //TODO: Implement GlobalIndexChecker like repair for unverified rows
- assertEquals(1, getRowCount(conn.unwrap(PhoenixConnection.class).getQueryServices()
+ assertEquals(2, getRowCount(conn.unwrap(PhoenixConnection.class).getQueryServices()
.getTable(Bytes.toBytes(dataTableFullName)), false));
-// assertEquals(1, getRowCount(conn.unwrap(PhoenixConnection.class).getQueryServices()
-// .getTable(Bytes.toBytes(record.getNewPhysicalTableName())), false));
+ assertEquals(0, getRowCountForEmptyColValue(conn, record.getNewPhysicalTableName(), UNVERIFIED_BYTES));
+ assertEquals(2, getRowCountForEmptyColValue(conn, record.getNewPhysicalTableName(), VERIFIED_BYTES));
+ } finally {
+ IndexRegionObserver.setFailPreIndexUpdatesForTesting(false);
+ IndexRegionObserver.setFailDataTableUpdatesForTesting(false);
+ IndexRegionObserver.setFailPostIndexUpdatesForTesting(false);
+ }
+ }
+
+ @Test
+ public void testTransformVerifiedForTransactionalTable() throws Exception {
+ // TODO: Column encoding is not supported for OMID. Have omid test for other type of transforms.
+ // For now, we don't support transforms other than storage and column encoding type change.
+ //testVerifiedForTransactionalTable("OMID");
+ testVerifiedForTransactionalTable("TEPHRA");
+ }
+
+ private void testVerifiedForTransactionalTable(String provider) throws Exception{
+ String tableOptions = tableDDLOptions + " ,TRANSACTIONAL=true,TRANSACTION_PROVIDER='" + provider + "'";
+
+ String schemaName = generateUniqueName();
+ String dataTableName = generateUniqueName();
+ String dataTableFullName = SchemaUtil.getTableName(schemaName, dataTableName);
+
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ conn.setAutoCommit(true);
+ int numOfRows = 1;
+ createTableAndUpsertRows(conn, dataTableFullName, numOfRows, tableOptions);
+ SingleCellIndexIT.assertMetadata(conn, PTable.ImmutableStorageScheme.ONE_CELL_PER_COLUMN, PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, dataTableFullName);
+
+ conn.createStatement().execute("ALTER TABLE " + dataTableFullName +
+ " SET IMMUTABLE_STORAGE_SCHEME=SINGLE_CELL_ARRAY_WITH_OFFSETS, COLUMN_ENCODED_BYTES=2");
+ SystemTransformRecord record = Transform.getTransformRecord(schemaName, dataTableName, null, null, conn.unwrap(PhoenixConnection.class));
+ assertNotNull(record);
+ assertMetadata(conn, PTable.ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS, PTable.QualifierEncodingScheme.TWO_BYTE_QUALIFIERS, record.getNewPhysicalTableName());
+
+ List<String> args = getArgList(schemaName, dataTableName, null,
+ null, null, null, false, false, false, false, false);
+ runTransformTool(args.toArray(new String[0]), 0);
+ assertEquals(1, getRowCountForEmptyColValue(conn, record.getNewPhysicalTableName(), VERIFIED_BYTES));
+
+
+ String upsertQuery = String.format("UPSERT INTO %s VALUES(?, ?, ?)", dataTableFullName);
+ PreparedStatement stmt1 = conn.prepareStatement(upsertQuery);
+
+ // Run again to check that VERIFIED row still remains verified
+ runTransformTool(args.toArray(new String[0]), 0);
+ assertEquals(1, getRowCountForEmptyColValue(conn, record.getNewPhysicalTableName(), VERIFIED_BYTES));
Review comment:
Consensus on the dev list seems to be that Tephra will be deprecated and removed soon, and Omid isn't supported, so I won't ask for a bunch of work that will just get removed soon anyway. But if Tephra wasn't being deprecated I'd ask for a test that made sure that this worked right when a transaction was rolled back.
##########
File path: phoenix-core/src/it/java/org/apache/phoenix/end2end/TransformToolIT.java
##########
@@ -413,59 +432,314 @@ public void testTransformMutationFailureRepair() throws Exception {
String schemaName = generateUniqueName();
String dataTableName = generateUniqueName();
String dataTableFullName = SchemaUtil.getTableName(schemaName, dataTableName);
+ String parentViewName = "VWP_" + generateUniqueName();
+ String viewName = "VW_" + generateUniqueName();
+ String viewIdxName = "VWIDX_" + generateUniqueName();
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.setAutoCommit(true);
- IndexRegionObserver.setFailPreIndexUpdatesForTesting(true);
int numOfRows = 0;
createTableAndUpsertRows(conn, dataTableFullName, numOfRows);
+ String createParentViewSql = "CREATE VIEW " + parentViewName + " ( PARENT_VIEW_COL1 VARCHAR ) AS SELECT * FROM " + dataTableFullName;
+ conn.createStatement().execute(createParentViewSql);
+
+ String createViewSql = "CREATE VIEW " + viewName + " ( VIEW_COL1 INTEGER, VIEW_COL2 VARCHAR ) AS SELECT * FROM " + parentViewName;
+ conn.createStatement().execute(createViewSql);
+
+ String createViewIdxSql = "CREATE INDEX " + viewIdxName + " ON " + viewName + " (VIEW_COL1) include (VIEW_COL2) ";
+ conn.createStatement().execute(createViewIdxSql);
+
SingleCellIndexIT.assertMetadata(conn, PTable.ImmutableStorageScheme.ONE_CELL_PER_COLUMN, PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, dataTableFullName);
conn.createStatement().execute("ALTER TABLE " + dataTableFullName +
" SET IMMUTABLE_STORAGE_SCHEME=SINGLE_CELL_ARRAY_WITH_OFFSETS, COLUMN_ENCODED_BYTES=2");
SystemTransformRecord record = Transform.getTransformRecord(schemaName, dataTableName, null, null, conn.unwrap(PhoenixConnection.class));
assertNotNull(record);
assertMetadata(conn, PTable.ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS, PTable.QualifierEncodingScheme.TWO_BYTE_QUALIFIERS, record.getNewPhysicalTableName());
-
- String upsertQuery = String.format("UPSERT INTO %s VALUES(?, ?, ?)", dataTableFullName);
+ IndexRegionObserver.setFailPreIndexUpdatesForTesting(true);
+ String upsertQuery = String.format("UPSERT INTO %s VALUES(?, ?, ?)", viewName);
PreparedStatement stmt1 = conn.prepareStatement(upsertQuery);
try {
IndexToolIT.upsertRow(stmt1, 1);
fail("Transform table upsert should have failed");
} catch (Exception e) {
}
assertEquals(0, getRowCount(conn, record.getNewPhysicalTableName()));
- assertEquals(0, getRowCount(conn,dataTableFullName));
+ assertEquals(0, getRowCount(conn, dataTableFullName));
IndexRegionObserver.setFailPreIndexUpdatesForTesting(false);
IndexRegionObserver.setFailPostIndexUpdatesForTesting(true);
IndexToolIT.upsertRow(stmt1, 2);
+ IndexToolIT.upsertRow(stmt1, 3);
- assertEquals(1, getRowCount(conn, record.getNewPhysicalTableName()));
- assertEquals(1, getRowCount(conn,dataTableFullName));
+ assertEquals(2, getRowCount(conn, record.getNewPhysicalTableName()));
+ assertEquals(2, getRowCount(conn,dataTableFullName));
IndexRegionObserver.setFailPostIndexUpdatesForTesting(false);
IndexRegionObserver.setFailDataTableUpdatesForTesting(true);
try {
- IndexToolIT.upsertRow(stmt1, 3);
+ IndexToolIT.upsertRow(stmt1, 4);
fail("Data table upsert should have failed");
} catch (Exception e) {
}
- assertEquals(2, getRowCount(conn, record.getNewPhysicalTableName()));
- assertEquals(1, getRowCount(conn,dataTableFullName));
+ try {
+ IndexToolIT.upsertRow(stmt1, 5);
+ fail("Data table upsert should have failed");
+ } catch (Exception e) {
+ }
+ assertEquals(4, getRowCount(conn, record.getNewPhysicalTableName()));
+ assertEquals(2, getRowCount(conn,dataTableFullName));
IndexRegionObserver.setFailDataTableUpdatesForTesting(false);
List<String> args = getArgList(schemaName, dataTableName, null,
- null, null, null, false, false, false, false);
+ null, null, null, false, false, false, false, true);
runTransformTool(args.toArray(new String[0]), 0);
- //TODO: Implement GlobalIndexChecker like repair for unverified rows
- assertEquals(1, getRowCount(conn.unwrap(PhoenixConnection.class).getQueryServices()
+ assertEquals(2, getRowCount(conn.unwrap(PhoenixConnection.class).getQueryServices()
.getTable(Bytes.toBytes(dataTableFullName)), false));
-// assertEquals(1, getRowCount(conn.unwrap(PhoenixConnection.class).getQueryServices()
-// .getTable(Bytes.toBytes(record.getNewPhysicalTableName())), false));
+ assertEquals(0, getRowCountForEmptyColValue(conn, record.getNewPhysicalTableName(), UNVERIFIED_BYTES));
+ assertEquals(2, getRowCountForEmptyColValue(conn, record.getNewPhysicalTableName(), VERIFIED_BYTES));
+ } finally {
+ IndexRegionObserver.setFailPreIndexUpdatesForTesting(false);
+ IndexRegionObserver.setFailDataTableUpdatesForTesting(false);
+ IndexRegionObserver.setFailPostIndexUpdatesForTesting(false);
+ }
+ }
+
+ @Test
+ public void testTransformVerifiedForTransactionalTable() throws Exception {
+ // TODO: Column encoding is not supported for OMID. Have omid test for other type of transforms.
+ // For now, we don't support transforms other than storage and column encoding type change.
+ //testVerifiedForTransactionalTable("OMID");
+ testVerifiedForTransactionalTable("TEPHRA");
+ }
+
+ private void testVerifiedForTransactionalTable(String provider) throws Exception{
Review comment:
@gokceni - I see existing error messages UNSUPPORTED_COLUMN_ENCODING_FOR_TXN_PROVIDER and UNSUPPORTED_STORAGE_FORMAT_FOR_TXN_PROVIDER -- do those get thrown on an ALTER of an OMID table or just a CREATE? If just a CREATE, it should probably do so for ALTER too. (Can be a separate JIRA.)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@phoenix.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org