You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by gj...@apache.org on 2017/10/11 17:57:58 UTC
phoenix git commit: PHOENIX-4229 - Parent-Child linking rows in
System.Catalog break tenant view replication
Repository: phoenix
Updated Branches:
refs/heads/master 024f407f2 -> ff8055553
PHOENIX-4229 - Parent-Child linking rows in System.Catalog break tenant view replication
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/ff805555
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/ff805555
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/ff805555
Branch: refs/heads/master
Commit: ff80555537ef103f73258115fc766bbe89c430a0
Parents: 024f407
Author: gjacoby <gj...@apache.org>
Authored: Tue Oct 10 11:54:17 2017 -0700
Committer: gjacoby <gj...@apache.org>
Committed: Wed Oct 11 10:25:56 2017 -0700
----------------------------------------------------------------------
.../SystemCatalogWALEntryFilterIT.java | 81 ++++++++++++++++----
.../SystemCatalogWALEntryFilter.java | 40 +++++++++-
2 files changed, 104 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/ff805555/phoenix-core/src/it/java/org/apache/phoenix/replication/SystemCatalogWALEntryFilterIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/replication/SystemCatalogWALEntryFilterIT.java b/phoenix-core/src/it/java/org/apache/phoenix/replication/SystemCatalogWALEntryFilterIT.java
index 4657cca..776e300 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/replication/SystemCatalogWALEntryFilterIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/replication/SystemCatalogWALEntryFilterIT.java
@@ -32,6 +32,7 @@ import org.apache.phoenix.mapreduce.util.ConnectionUtil;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.TestUtil;
import org.junit.AfterClass;
import org.junit.Assert;
@@ -122,36 +123,63 @@ public class SystemCatalogWALEntryFilterIT extends ParallelStatsDisabledIT {
//now create WAL.Entry objects that refer to cells in those view rows in System.Catalog
- Get tenantGet = getGet(catalogTable, TENANT_BYTES, TENANT_VIEW_NAME);
- Get nonTenantGet = getGet(catalogTable, DEFAULT_TENANT_BYTES, NONTENANT_VIEW_NAME);
+ Get tenantViewGet = getTenantViewGet(catalogTable, TENANT_BYTES, TENANT_VIEW_NAME);
+ Get nonTenantViewGet = getTenantViewGet(catalogTable,
+ DEFAULT_TENANT_BYTES, NONTENANT_VIEW_NAME);
- WAL.Entry nonTenantEntry = getEntry(systemCatalogTableName, nonTenantGet);
- WAL.Entry tenantEntry = getEntry(systemCatalogTableName, tenantGet);
+ Get tenantLinkGet = getParentChildLinkGet(catalogTable, TENANT_BYTES, TENANT_VIEW_NAME);
+ Get nonTenantLinkGet = getParentChildLinkGet(catalogTable,
+ DEFAULT_TENANT_BYTES, NONTENANT_VIEW_NAME);
+
+ WAL.Entry nonTenantViewEntry = getEntry(systemCatalogTableName, nonTenantViewGet);
+ WAL.Entry tenantViewEntry = getEntry(systemCatalogTableName, tenantViewGet);
+
+ WAL.Entry nonTenantLinkEntry = getEntry(systemCatalogTableName, nonTenantLinkGet);
+ WAL.Entry tenantLinkEntry = getEntry(systemCatalogTableName, tenantLinkGet);
//verify that the tenant view WAL.Entry passes the filter and the non-tenant view does not
SystemCatalogWALEntryFilter filter = new SystemCatalogWALEntryFilter();
- Assert.assertNull(filter.filter(nonTenantEntry));
- WAL.Entry filteredTenantEntry = filter.filter(tenantEntry);
+ Assert.assertNull(filter.filter(nonTenantViewEntry));
+ WAL.Entry filteredTenantEntry = filter.filter(tenantViewEntry);
Assert.assertNotNull("Tenant view was filtered when it shouldn't be!", filteredTenantEntry);
- Assert.assertEquals(tenantEntry.getEdit().size(),
- filter.filter(tenantEntry).getEdit().size());
+ Assert.assertEquals(tenantViewEntry.getEdit().size(),
+ filter.filter(tenantViewEntry).getEdit().size());
//now check that a WAL.Entry with cells from both a tenant and a non-tenant
//catalog row only allow the tenant cells through
WALEdit comboEdit = new WALEdit();
- comboEdit.getCells().addAll(nonTenantEntry.getEdit().getCells());
- comboEdit.getCells().addAll(tenantEntry.getEdit().getCells());
+ comboEdit.getCells().addAll(nonTenantViewEntry.getEdit().getCells());
+ comboEdit.getCells().addAll(tenantViewEntry.getEdit().getCells());
WAL.Entry comboEntry = new WAL.Entry(walKey, comboEdit);
- Assert.assertEquals(tenantEntry.getEdit().size() + nonTenantEntry.getEdit().size()
+ Assert.assertEquals(tenantViewEntry.getEdit().size() + nonTenantViewEntry.getEdit().size()
, comboEntry.getEdit().size());
- Assert.assertEquals(tenantEntry.getEdit().size(),
+ Assert.assertEquals(tenantViewEntry.getEdit().size(),
filter.filter(comboEntry).getEdit().size());
+
+ //now check that the parent-child links (which have the tenant_id of the view's parent,
+ // but are a part of the view's metadata) are migrated in the tenant case
+ // but not the non-tenant. The view's tenant_id is in th System.Catalog.COLUMN_NAME field
+
+ Assert.assertNull("Non-tenant parent-child link was not filtered " +
+ "when it should be!", filter.filter(nonTenantLinkEntry));
+ Assert.assertNotNull("Tenant parent-child link was filtered when it should not be!",
+ filter.filter(tenantLinkEntry));
+ Assert.assertEquals(tenantLinkEntry.getEdit().size(),
+ filter.filter(tenantLinkEntry).getEdit().size());
+ //add the parent-child link to the tenant view WAL entry,
+ //since they'll usually be together and they both need to
+ //be replicated
+
+ tenantViewEntry.getEdit().getCells().addAll(tenantLinkEntry.getEdit().getCells());
+ Assert.assertEquals(tenantViewEntry.getEdit().size(), tenantViewEntry.getEdit().size());
+
+
}
- public Get getGet(PTable catalogTable, byte[] tenantId, String viewName) {
+ public Get getTenantViewGet(PTable catalogTable, byte[] tenantBytes, String viewName) {
byte[][] tenantKeyParts = new byte[5][];
- tenantKeyParts[0] = tenantId;
+ tenantKeyParts[0] = tenantBytes;
tenantKeyParts[1] = Bytes.toBytes(SCHEMA_NAME.toUpperCase());
tenantKeyParts[2] = Bytes.toBytes(viewName.toUpperCase());
tenantKeyParts[3] = Bytes.toBytes(VIEW_COLUMN_NAME);
@@ -163,6 +191,28 @@ public class SystemCatalogWALEntryFilterIT extends ParallelStatsDisabledIT {
return new Get(key.copyBytes());
}
+ public Get getParentChildLinkGet(PTable catalogTable, byte[] tenantBytes, String viewName) {
+ /* For parent-child link, the system.catalog key becomes
+ 1. Parent tenant id
+ 2. Parent Schema
+ 3. Parent Table name
+ 4. View tenant_id
+ 5. Combined view SCHEMA.TABLENAME
+ */
+ byte[][] tenantKeyParts = new byte[5][];
+ tenantKeyParts[0] = null; //null tenant_id
+ tenantKeyParts[1] = null; //null parent schema
+ tenantKeyParts[2] = Bytes.toBytes(TestUtil.ENTITY_HISTORY_TABLE_NAME);
+ tenantKeyParts[3] = tenantBytes;
+ tenantKeyParts[4] = Bytes.toBytes(SchemaUtil.getTableName(SCHEMA_NAME.toUpperCase(), viewName.toUpperCase()));
+ ImmutableBytesWritable key = new ImmutableBytesWritable();
+ catalogTable.newKey(key, tenantKeyParts);
+ //the backing byte array of key might have extra space at the end.
+ // need to just slice "the good parts" which we do by calling copyBytes
+ return new Get(key.copyBytes());
+
+ }
+
public WAL.Entry getEntry(TableName tableName, Get get) throws IOException {
WAL.Entry entry = null;
try(Connection conn = ConnectionFactory.createConnection(getUtility().getConfiguration())){
@@ -176,7 +226,8 @@ public class SystemCatalogWALEntryFilterIT extends ParallelStatsDisabledIT {
edit.add(c);
}
}
- Assert.assertTrue("Didn't retrieve any cells from SYSTEM.CATALOG", edit.getCells().size() > 0);
+ Assert.assertTrue("Didn't retrieve any cells from SYSTEM.CATALOG",
+ edit.getCells().size() > 0);
WALKey key = new WALKey(REGION, tableName, 0, 0, uuid);
entry = new WAL.Entry(key, edit);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/ff805555/phoenix-core/src/main/java/org/apache/phoenix/replication/SystemCatalogWALEntryFilter.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/replication/SystemCatalogWALEntryFilter.java b/phoenix-core/src/main/java/org/apache/phoenix/replication/SystemCatalogWALEntryFilter.java
index 6d6055a..d1b71ef 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/replication/SystemCatalogWALEntryFilter.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/replication/SystemCatalogWALEntryFilter.java
@@ -22,7 +22,9 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.replication.WALEntryFilter;
import org.apache.hadoop.hbase.wal.WAL;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.util.SchemaUtil;
import java.util.List;
@@ -32,10 +34,15 @@ import java.util.List;
* may change between the source and target clusters at different times, in particular
* during cluster upgrades. However, tenant-owned data such as tenant-owned views need to
* be copied. This WALEntryFilter will only allow tenant-owned rows in SYSTEM.CATALOG to
- * be replicated. Data from all other tables is automatically passed.
+ * be replicated. Data from all other tables is automatically passed. It will also copy
+ * child links in SYSTEM.CATALOG that are globally-owned but point to tenant-owned views.
+ *
*/
public class SystemCatalogWALEntryFilter implements WALEntryFilter {
+ private static byte[] CHILD_TABLE_BYTES =
+ new byte[]{PTable.LinkType.CHILD_TABLE.getSerializedValue()};
+
@Override
public WAL.Entry filter(WAL.Entry entry) {
@@ -64,6 +71,35 @@ public class SystemCatalogWALEntryFilter implements WALEntryFilter {
ImmutableBytesWritable key =
new ImmutableBytesWritable(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
//rows in system.catalog that aren't tenant-owned will have a leading separator byte
- return key.get()[key.getOffset()] != QueryConstants.SEPARATOR_BYTE;
+ boolean isTenantRowCell = key.get()[key.getOffset()] != QueryConstants.SEPARATOR_BYTE;
+
+ /* In addition to the tenant view rows, there are parent-child links (see PHOENIX-2051) that
+ * provide an efficient way for a parent table or view to look up its children.
+ * These rows override SYSTEM_CATALOG.COLUMN_NAME with the child tenant_id,
+ * if any, and contain only a single Cell, LINK_TYPE, which is of PTable.LinkType.Child
+ */
+ boolean isChildLinkToTenantView = false;
+ if (!isTenantRowCell) {
+ ImmutableBytesWritable columnQualifier = new ImmutableBytesWritable(cell.getQualifierArray(),
+ cell.getQualifierOffset(), cell.getQualifierLength());
+ boolean isChildLink = columnQualifier.compareTo(PhoenixDatabaseMetaData.LINK_TYPE_BYTES) == 0;
+ if (isChildLink) {
+ ImmutableBytesWritable columnValue = new ImmutableBytesWritable(cell.getValueArray(),
+ cell.getValueOffset(), cell.getValueLength());
+ if (columnValue.compareTo(CHILD_TABLE_BYTES) == 0) {
+ byte[][] rowViewKeyMetadata = new byte[5][];
+ SchemaUtil.getVarChars(key.get(), key.getOffset(),
+ key.getLength(), 0, rowViewKeyMetadata);
+ //if the child link is to a tenant-owned view,
+ // the COLUMN_NAME field will be the byte[] of the tenant
+ //otherwise, it will be an empty byte array
+ // (NOT QueryConstants.SEPARATOR_BYTE, but a byte[0])
+ isChildLinkToTenantView =
+ rowViewKeyMetadata[PhoenixDatabaseMetaData.COLUMN_NAME_INDEX].length != 0;
+ }
+ }
+
+ }
+ return isTenantRowCell || isChildLinkToTenantView;
}
}