You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by td...@apache.org on 2017/03/09 21:24:34 UTC
phoenix git commit: PHOENIX-2051 Link record is in the format
CHILD-PARENT for phoenix views and it has to scan the entire table to find
the parent suffix
Repository: phoenix
Updated Branches:
refs/heads/system-catalog [created] 91ee8cc14
PHOENIX-2051 Link record is in the format CHILD-PARENT for phoenix views and it has to scan the entire table to find the parent suffix
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/91ee8cc1
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/91ee8cc1
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/91ee8cc1
Branch: refs/heads/system-catalog
Commit: 91ee8cc1429532d30827607164b3ff77676d0ac0
Parents: c6a7a71
Author: Thomas D'Silva <td...@salesforce.com>
Authored: Fri Feb 17 15:50:24 2017 -0800
Committer: Thomas D'Silva <td...@apache.org>
Committed: Thu Mar 9 13:19:35 2017 -0800
----------------------------------------------------------------------
.../coprocessor/MetaDataEndpointImpl.java | 160 +++++++++----------
.../phoenix/coprocessor/MetaDataProtocol.java | 3 +-
.../phoenix/jdbc/PhoenixDatabaseMetaData.java | 4 +-
.../query/ConnectionQueryServicesImpl.java | 4 +
.../apache/phoenix/schema/MetaDataClient.java | 25 ++-
.../java/org/apache/phoenix/schema/PTable.java | 10 +-
.../org/apache/phoenix/util/MetaDataUtil.java | 17 ++
.../org/apache/phoenix/util/UpgradeUtil.java | 73 +++++++++
8 files changed, 203 insertions(+), 93 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/91ee8cc1/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index 1d418e9..57263cd 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@ -55,6 +55,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.MULTI_TENANT_BYTES
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NULLABLE_BYTES;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.NUM_ARGS_BYTES;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ORDINAL_POSITION_BYTES;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.PARENT_TENANT_ID_BYTES;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.PK_NAME_BYTES;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.RETURN_TYPE_BYTES;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SALT_BUCKETS_BYTES;
@@ -116,7 +117,6 @@ import org.apache.hadoop.hbase.coprocessor.CoprocessorException;
import org.apache.hadoop.hbase.coprocessor.CoprocessorService;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp;
-import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.FirstKeyOnlyFilter;
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
@@ -988,7 +988,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
Integer value = PInteger.INSTANCE.getCodec().decodeInt(colKv.getValueArray(), colKv.getValueOffset(), SortOrder.ASC);
cqCounter.setValue(famName.getString(), value);
} else {
- if (colName.getString().isEmpty() && famName != null) {
+ if (Bytes.compareTo(LINK_TYPE_BYTES, 0, LINK_TYPE_BYTES.length, colKv.getQualifierArray(), colKv.getQualifierOffset(), colKv.getQualifierLength())==0) {
LinkType linkType = LinkType.fromSerializedValue(colKv.getValueArray()[colKv.getValueOffset()]);
if (linkType == LinkType.INDEX_TABLE) {
addIndexToTable(tenantId, schemaName, famName, tableName, clientTimeStamp, indexes);
@@ -1630,7 +1630,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
return rowLock;
}
- private static final byte[] PHYSICAL_TABLE_BYTES = new byte[] {PTable.LinkType.PHYSICAL_TABLE.getSerializedValue()};
+ private static final byte[] CHILD_TABLE_BYTES = new byte[] {PTable.LinkType.CHILD_TABLE.getSerializedValue()};
private PSchema loadSchema(RegionCoprocessorEnvironment env, byte[] key, ImmutableBytesPtr cacheKey,
long clientTimeStamp, long asOfTimeStamp) throws IOException, SQLException {
@@ -1649,41 +1649,36 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
&& (schema = buildDeletedSchema(key, cacheKey, region, clientTimeStamp)) != null) { return schema; }
return null;
}
+
+ private void findAllChildViews(HRegion region, byte[] tenantId, byte[] schemaName, byte[] tableName, TableViewFinderResult result) throws IOException {
+ TableViewFinderResult currResult = findChildViews(region, tenantId, schemaName, tableName);
+ result.addResult(currResult);
+ for (Result viewResult : currResult.getResults()) {
+ byte[][] rowViewKeyMetaData = new byte[5][];
+ getVarChars(viewResult.getRow(), 5, rowViewKeyMetaData);
+ byte[] viewtenantId = rowViewKeyMetaData[PhoenixDatabaseMetaData.COLUMN_NAME_INDEX];
+ byte[] viewSchema = SchemaUtil.getSchemaNameFromFullName(rowViewKeyMetaData[PhoenixDatabaseMetaData.FAMILY_NAME_INDEX]).getBytes();
+ byte[] viewTable = SchemaUtil.getTableNameFromFullName(rowViewKeyMetaData[PhoenixDatabaseMetaData.FAMILY_NAME_INDEX]).getBytes();
+ findAllChildViews(region, viewtenantId, viewSchema, viewTable, result);
+ }
+ }
/**
* @param tableName parent table's name
* Looks for whether child views exist for the table specified by table.
* TODO: should we pass a timestamp here?
*/
-
- private TableViewFinderResult findChildViews(HRegion region, byte[] tenantId, PTable table, byte[] linkTypeBytes) throws IOException {
- byte[] schemaName = table.getSchemaName().getBytes();
- byte[] tableName = table.getTableName().getBytes();
- boolean isMultiTenant = table.isMultiTenant();
+ private TableViewFinderResult findChildViews(HRegion region, byte[] tenantId, byte[] schemaName, byte[] tableName) throws IOException {
Scan scan = new Scan();
- // If the table is multi-tenant, we need to check across all tenant_ids,
- // so we can't constrain the row key. Otherwise, any views would have
- // the same tenantId.
- if (!isMultiTenant) {
- byte[] startRow = ByteUtil.concat(tenantId, QueryConstants.SEPARATOR_BYTE_ARRAY);
- byte[] stopRow = ByteUtil.nextKey(startRow);
- scan.setStartRow(startRow);
- scan.setStopRow(stopRow);
- }
- SingleColumnValueFilter linkFilter = new SingleColumnValueFilter(TABLE_FAMILY_BYTES, LINK_TYPE_BYTES, CompareOp.EQUAL, linkTypeBytes);
- SingleColumnValueFilter tableTypeFilter = new SingleColumnValueFilter(TABLE_FAMILY_BYTES, TABLE_TYPE_BYTES,
- CompareOp.EQUAL, PTableType.VIEW.getSerializedValue().getBytes());
- tableTypeFilter.setFilterIfMissing(false);
+ byte[] startRow = SchemaUtil.getTableKey(tenantId, schemaName, tableName);
+ byte[] stopRow = ByteUtil.nextKey(startRow);
+ scan.setStartRow(startRow);
+ scan.setStopRow(stopRow);
+ SingleColumnValueFilter linkFilter = new SingleColumnValueFilter(TABLE_FAMILY_BYTES, LINK_TYPE_BYTES, CompareOp.EQUAL, CHILD_TABLE_BYTES);
linkFilter.setFilterIfMissing(true);
- byte[] suffix = ByteUtil.concat(QueryConstants.SEPARATOR_BYTE_ARRAY, SchemaUtil
- .getPhysicalTableName(SchemaUtil.getTableNameAsBytes(schemaName, tableName), table.isNamespaceMapped())
- .getName());
- SuffixFilter rowFilter = new SuffixFilter(suffix);
- FilterList filter = new FilterList(linkFilter,tableTypeFilter,rowFilter);
- scan.setFilter(filter);
+ scan.setFilter(linkFilter);
scan.addColumn(TABLE_FAMILY_BYTES, LINK_TYPE_BYTES);
- scan.addColumn(TABLE_FAMILY_BYTES, TABLE_TYPE_BYTES);
- scan.addColumn(TABLE_FAMILY_BYTES, TABLE_SEQ_NUM_BYTES);
+ scan.addColumn(TABLE_FAMILY_BYTES, PARENT_TENANT_ID_BYTES);
// Original region-only scanner modified due to PHOENIX-1208
// RegionScanner scanner = region.getScanner(scan);
@@ -1710,7 +1705,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
}
results.add(result);
}
- TableViewFinderResult tableViewFinderResult = new TableViewFinderResult(results, table);
+ TableViewFinderResult tableViewFinderResult = new TableViewFinderResult(results);
if (numOfChildViews > 0 && !allViewsInCurrentRegion) {
tableViewFinderResult.setAllViewsNotInSingleRegion();
}
@@ -1851,12 +1846,9 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
EnvironmentEdgeManager.currentTimeMillis(), null);
}
- // Only tables may have views, so prevent the running of this potentially
- // expensive full table scan over the SYSTEM.CATALOG table unless it's needed.
if (tableType == PTableType.TABLE || tableType == PTableType.SYSTEM) {
// Handle any child views that exist
- TableViewFinderResult tableViewFinderResult = findChildViews(region, tenantId, table,
- PHYSICAL_TABLE_BYTES);
+ TableViewFinderResult tableViewFinderResult = findChildViews(region, tenantId, table.getSchemaName().getBytes(), table.getTableName().getBytes());
if (tableViewFinderResult.hasViews()) {
if (isCascade) {
if (tableViewFinderResult.allViewsInMultipleRegions()) {
@@ -1867,11 +1859,11 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
} else if (tableViewFinderResult.allViewsInSingleRegion()) {
// Recursively delete views - safe as all the views as all in the same region
for (Result viewResult : tableViewFinderResult.getResults()) {
- byte[][] rowKeyMetaData = new byte[3][];
- getVarChars(viewResult.getRow(), 3, rowKeyMetaData);
- byte[] viewTenantId = rowKeyMetaData[PhoenixDatabaseMetaData.TENANT_ID_INDEX];
- byte[] viewSchemaName = rowKeyMetaData[PhoenixDatabaseMetaData.SCHEMA_NAME_INDEX];
- byte[] viewName = rowKeyMetaData[PhoenixDatabaseMetaData.TABLE_NAME_INDEX];
+ byte[][] rowViewKeyMetaData = new byte[5][];
+ getVarChars(viewResult.getRow(), 5, rowViewKeyMetaData);
+ byte[] viewTenantId = rowViewKeyMetaData[PhoenixDatabaseMetaData.COLUMN_NAME_INDEX];
+ byte[] viewSchemaName = SchemaUtil.getSchemaNameFromFullName(rowViewKeyMetaData[PhoenixDatabaseMetaData.FAMILY_NAME_INDEX]).getBytes();
+ byte[] viewName = SchemaUtil.getTableNameFromFullName(rowViewKeyMetaData[PhoenixDatabaseMetaData.FAMILY_NAME_INDEX]).getBytes();
byte[] viewKey = SchemaUtil.getTableKey(viewTenantId, viewSchemaName, viewName);
Delete delete = new Delete(viewKey, clientTimeStamp);
rowsToDelete.add(delete);
@@ -1903,12 +1895,20 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
Cell kv = results.get(LINK_TYPE_INDEX);
int nColumns = getVarChars(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength(), 0, rowKeyMetaData);
if (nColumns == 5
- && rowKeyMetaData[PhoenixDatabaseMetaData.COLUMN_NAME_INDEX].length == 0
- && rowKeyMetaData[PhoenixDatabaseMetaData.INDEX_NAME_INDEX].length > 0
+ && rowKeyMetaData[PhoenixDatabaseMetaData.FAMILY_NAME_INDEX].length > 0
&& Bytes.compareTo(kv.getQualifierArray(), kv.getQualifierOffset(), kv.getQualifierLength(),
- LINK_TYPE_BYTES, 0, LINK_TYPE_BYTES.length) == 0
- && LinkType.fromSerializedValue(kv.getValueArray()[kv.getValueOffset()]) == LinkType.INDEX_TABLE) {
- indexNames.add(rowKeyMetaData[PhoenixDatabaseMetaData.INDEX_NAME_INDEX]);
+ LINK_TYPE_BYTES, 0, LINK_TYPE_BYTES.length) == 0) {
+ LinkType linkType = LinkType.fromSerializedValue(kv.getValueArray()[kv.getValueOffset()]);
+ if (rowKeyMetaData[PhoenixDatabaseMetaData.COLUMN_NAME_INDEX].length == 0 && linkType == LinkType.INDEX_TABLE) {
+ indexNames.add(rowKeyMetaData[PhoenixDatabaseMetaData.FAMILY_NAME_INDEX]);
+ } else if (linkType == LinkType.PARENT_TABLE || linkType == LinkType.PHYSICAL_TABLE) {
+ // delete parent->child link for views
+ Cell parentTenantIdCell = MetaDataUtil.getCell(results, PhoenixDatabaseMetaData.PARENT_TENANT_ID_BYTES);
+ PName parentTenantId = parentTenantIdCell!=null ? PNameFactory.newName(parentTenantIdCell.getValueArray(), parentTenantIdCell.getValueOffset(), parentTenantIdCell.getValueLength()) : null;
+ byte[] linkKey = MetaDataUtil.getChildLinkKey(parentTenantId, table.getParentSchemaName(), table.getParentTableName(), table.getTenantId(), table.getName());
+ Delete linkDelete = new Delete(linkKey, clientTimeStamp);
+ rowsToDelete.add(linkDelete);
+ }
}
// FIXME: Remove when unintentionally deprecated method is fixed (HBASE-7870).
// FIXME: the version of the Delete constructor without the lock args was introduced
@@ -1920,7 +1920,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
scanner.next(results);
} while (!results.isEmpty());
}
-
+
// Recursively delete indexes
for (byte[] indexName : indexNames) {
byte[] indexKey = SchemaUtil.getTableKey(tenantId, schemaName, indexName);
@@ -2255,11 +2255,12 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
short deltaNumPkColsSoFar = 0;
short columnsAddedToView = 0;
short columnsAddedToBaseTable = 0;
- byte[][] rowViewKeyMetaData = new byte[3][];
- getVarChars(viewResult.getRow(), 3, rowViewKeyMetaData);
- byte[] viewKey = SchemaUtil.getTableKey(rowViewKeyMetaData[PhoenixDatabaseMetaData.TENANT_ID_INDEX],
- rowViewKeyMetaData[PhoenixDatabaseMetaData.SCHEMA_NAME_INDEX],
- rowViewKeyMetaData[PhoenixDatabaseMetaData.TABLE_NAME_INDEX]);
+ byte[][] rowViewKeyMetaData = new byte[5][];
+ getVarChars(viewResult.getRow(), 5, rowViewKeyMetaData);
+ byte[] tenantId = rowViewKeyMetaData[PhoenixDatabaseMetaData.COLUMN_NAME_INDEX];
+ byte[] schema = SchemaUtil.getSchemaNameFromFullName(rowViewKeyMetaData[PhoenixDatabaseMetaData.FAMILY_NAME_INDEX]).getBytes();
+ byte[] table = SchemaUtil.getTableNameFromFullName(rowViewKeyMetaData[PhoenixDatabaseMetaData.FAMILY_NAME_INDEX]).getBytes();
+ byte[] viewKey = SchemaUtil.getTableKey(tenantId, schema, table);
// lock the rows corresponding to views so that no other thread can modify the view meta-data
RowLock viewRowLock = acquireLock(region, viewKey, locks);
@@ -2580,13 +2581,12 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
}
for (Result viewResult : childViewsResult.getResults()) {
short numColsDeleted = 0;
- byte[][] rowViewKeyMetaData = new byte[3][];
- getVarChars(viewResult.getRow(), 3, rowViewKeyMetaData);
- byte[] viewKey =
- SchemaUtil.getTableKey(
- rowViewKeyMetaData[PhoenixDatabaseMetaData.TENANT_ID_INDEX],
- rowViewKeyMetaData[PhoenixDatabaseMetaData.SCHEMA_NAME_INDEX],
- rowViewKeyMetaData[PhoenixDatabaseMetaData.TABLE_NAME_INDEX]);
+ byte[][] rowViewKeyMetaData = new byte[5][];
+ getVarChars(viewResult.getRow(), 5, rowViewKeyMetaData);
+ byte[] viewTenantId = rowViewKeyMetaData[PhoenixDatabaseMetaData.COLUMN_NAME_INDEX];
+ byte[] viewSchemaName = SchemaUtil.getSchemaNameFromFullName(rowViewKeyMetaData[PhoenixDatabaseMetaData.FAMILY_NAME_INDEX]).getBytes();
+ byte[] viewName = SchemaUtil.getTableNameFromFullName(rowViewKeyMetaData[PhoenixDatabaseMetaData.FAMILY_NAME_INDEX]).getBytes();
+ byte[] viewKey = SchemaUtil.getTableKey(viewTenantId, viewSchemaName, viewName);
// lock the rows corresponding to views so that no other thread can modify the view
// meta-data
@@ -2882,17 +2882,10 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
schemaName, tableName);
// Size for worst case - all new columns are PK column
List<Mutation> mutationsForAddingColumnsToViews = Lists.newArrayListWithExpectedSize(tableMetaData.size() * ( 1 + table.getIndexes().size()));
- /*
- * If adding a column to a view, we don't want to propagate those meta-data changes to the child
- * view hierarchy. This is because our check of finding child views is expensive and we want making
- * meta-data changes to views to be light-weight. The side-effect of this change is that a child
- * won't have it's parent views columns i.e. it would have diverged itself from the parent view. See
- * https://issues.apache.org/jira/browse/PHOENIX-2051 for a proper way to fix the performance issue
- * and https://issues.apache.org/jira/browse/PHOENIX-2054 for enabling meta-data changes to a view
- * to be propagated to its view hierarchy.
- */
+ // TODO propagate to grandchild views as well
if (type == PTableType.TABLE || type == PTableType.SYSTEM) {
- TableViewFinderResult childViewsResult = findChildViews(region, tenantId, table, PHYSICAL_TABLE_BYTES);
+ TableViewFinderResult childViewsResult = new TableViewFinderResult();
+ findAllChildViews(region, tenantId, table.getSchemaName().getBytes(), table.getTableName().getBytes(), childViewsResult);
if (childViewsResult.hasViews()) {
/*
* Dis-allow if:
@@ -3166,13 +3159,10 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
List<Mutation> additionalTableMetaData = Lists.newArrayList();
PTableType type = table.getType();
- // Only tables may have views, so prevent the running of this potentially
- // expensive full table scan over the SYSTEM.CATALOG table unless it's needed.
- // In the case of a view, we allow a column to be dropped without checking for
- // child views, but in the future we'll allow it and propagate it as necessary.
+ // TODO propagate to grandchild views as well
if (type == PTableType.TABLE || type == PTableType.SYSTEM) {
- TableViewFinderResult childViewsResult =
- findChildViews(region, tenantId, table, PHYSICAL_TABLE_BYTES);
+ TableViewFinderResult childViewsResult = new TableViewFinderResult();
+ findAllChildViews(region, tenantId, table.getSchemaName().getBytes(), table.getTableName().getBytes(), childViewsResult);
if (childViewsResult.hasViews()) {
MetaDataMutationResult mutationResult =
dropColumnsFromChildViews(region, table,
@@ -3559,21 +3549,16 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
private List<Result> results = Lists.newArrayList();
private boolean allViewsNotInSingleRegion = false;
- private PTable table;
- private TableViewFinderResult(List<Result> results, PTable table) {
+ private TableViewFinderResult() {
+ }
+
+ private TableViewFinderResult(List<Result> results) {
this.results = results;
- this.table = table;
}
-
+
public boolean hasViews() {
- int localIndexesCount = 0;
- for(PTable index : table.getIndexes()) {
- if(index.getIndexType().equals(IndexType.LOCAL)) {
- localIndexesCount++;
- }
- }
- return results.size()-localIndexesCount > 0;
+ return !results.isEmpty();
}
private void setAllViewsNotInSingleRegion() {
@@ -3597,6 +3582,13 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
private boolean allViewsInMultipleRegions() {
return results.size() > 0 && allViewsNotInSingleRegion;
}
+
+ private void addResult(TableViewFinderResult result) {
+ this.results.addAll(result.getResults());
+ if (result.allViewsInMultipleRegions()) {
+ this.setAllViewsNotInSingleRegion();
+ }
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/91ee8cc1/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
index f2abbdb..e59a967 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
@@ -86,8 +86,9 @@ public abstract class MetaDataProtocol extends MetaDataService {
public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_8_1 = MIN_TABLE_TIMESTAMP + 18;
public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0 = MIN_TABLE_TIMESTAMP + 20;
public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0 = MIN_TABLE_TIMESTAMP + 25;
+ public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_11_0 = MIN_TABLE_TIMESTAMP + 25;
// MIN_SYSTEM_TABLE_TIMESTAMP needs to be set to the max of all the MIN_SYSTEM_TABLE_TIMESTAMP_* constants
- public static final long MIN_SYSTEM_TABLE_TIMESTAMP = MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0;
+ public static final long MIN_SYSTEM_TABLE_TIMESTAMP = MIN_SYSTEM_TABLE_TIMESTAMP_4_11_0;
// ALWAYS update this map whenever rolling out a new release (major, minor or patch release).
// Key is the SYSTEM.CATALOG timestamp for the version and value is the version string.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/91ee8cc1/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
index 0bf7a71..e09f5ba 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
@@ -78,7 +78,6 @@ import com.google.common.collect.Lists;
*
*/
public class PhoenixDatabaseMetaData implements DatabaseMetaData {
- public static final int INDEX_NAME_INDEX = 4; // Shared with FAMILY_NAME_INDEX
public static final int FAMILY_NAME_INDEX = 4;
public static final int COLUMN_NAME_INDEX = 3;
public static final int TABLE_NAME_INDEX = 2;
@@ -124,6 +123,7 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData {
public static final byte[] PHYSICAL_NAME_BYTES = Bytes.toBytes(PHYSICAL_NAME);
public static final String COLUMN_FAMILY = "COLUMN_FAMILY";
+ public static final byte[] COLUMN_FAMILY_BYTES = Bytes.toBytes(COLUMN_FAMILY);
public static final String TABLE_CAT = "TABLE_CAT";
public static final String TABLE_CATALOG = "TABLE_CATALOG";
public static final String TABLE_SCHEM = "TABLE_SCHEM";
@@ -514,6 +514,8 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData {
if (colPattern == null || colPattern.length() == 0) {
appendConjunction(where);
where.append(COLUMN_NAME + " is not null" );
+ appendConjunction(where);
+ where.append(LINK_TYPE + " is null" );
}
boolean isTenantSpecificConnection = connection.getTenantId() != null;
if (isTenantSpecificConnection) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/91ee8cc1/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index b113ad9..584d30c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -59,6 +59,7 @@ import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_RENEW_LEASE_
import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_RENEW_LEASE_THRESHOLD_MILLISECONDS;
import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_RUN_RENEW_LEASE_FREQUENCY_INTERVAL_MILLISECONDS;
import static org.apache.phoenix.util.UpgradeUtil.getSysCatalogSnapshotName;
+import static org.apache.phoenix.util.UpgradeUtil.upgradeTo4_11_0;
import static org.apache.phoenix.util.UpgradeUtil.upgradeTo4_5_0;
import java.io.IOException;
@@ -2765,6 +2766,9 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0);
clearCache();
}
+ if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_11_0) {
+ upgradeTo4_11_0(metaConnection);
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/91ee8cc1/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index d09daeb..096c7ba 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -313,7 +313,17 @@ public class MetaDataClient {
COLUMN_QUALIFIER_COUNTER +
") VALUES (?, ?, ?, ?, ?)";
- public static final String INCREMENT_SEQ_NUM =
+ private static final String CREATE_CHILD_LINK =
+ "UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"( " +
+ TENANT_ID + "," +
+ TABLE_SCHEM + "," +
+ TABLE_NAME + "," +
+ COLUMN_NAME + "," +
+ COLUMN_FAMILY + "," +
+ LINK_TYPE +
+ ") VALUES (?, ?, ?, ?, ?, ?)";
+
+ private static final String INCREMENT_SEQ_NUM =
"UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"( " +
TENANT_ID + "," +
TABLE_SCHEM + "," +
@@ -2063,9 +2073,7 @@ public class MetaDataClient {
}
pkColumns = newLinkedHashSet(parent.getPKColumns());
- // Add row linking from view to its parent table
- // FIXME: not currently used, but see PHOENIX-1367
- // as fixing that will require it's usage.
+ // Add row linking view to its parent
PreparedStatement linkStatement = connection.prepareStatement(CREATE_VIEW_LINK);
linkStatement.setString(1, tenantIdStr);
linkStatement.setString(2, schemaName);
@@ -2074,6 +2082,15 @@ public class MetaDataClient {
linkStatement.setByte(5, LinkType.PARENT_TABLE.getSerializedValue());
linkStatement.setString(6, parent.getTenantId() == null ? null : parent.getTenantId().getString());
linkStatement.execute();
+ // Add row linking parent to view
+ linkStatement = connection.prepareStatement(CREATE_CHILD_LINK);
+ linkStatement.setString(1, parent.getTenantId() == null ? null : parent.getTenantId().getString());
+ linkStatement.setString(2, parent.getSchemaName() == null ? null : parent.getSchemaName().getString());
+ linkStatement.setString(3, parent.getTableName().getString());
+ linkStatement.setString(4, tenantIdStr);
+ linkStatement.setString(5, SchemaUtil.getTableName(schemaName, tableName));
+ linkStatement.setByte(6, LinkType.CHILD_TABLE.getSerializedValue());
+ linkStatement.execute();
}
} else {
columns = new LinkedHashMap<PColumn,PColumn>(colDefs.size());
http://git-wip-us.apache.org/repos/asf/phoenix/blob/91ee8cc1/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
index 91a41a3..24a494f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
@@ -141,14 +141,18 @@ public interface PTable extends PMetaDataEntity {
*/
INDEX_TABLE((byte)1),
/**
- * Link from a view to its physical table
+ * Link from a view or index to its physical table
*/
PHYSICAL_TABLE((byte)2),
/**
* Link from a view to its parent table
*/
- PARENT_TABLE((byte)3);
-
+ PARENT_TABLE((byte)3),
+ /**
+ * Link from a parent table to its child view
+ */
+ CHILD_TABLE((byte)4);
+
private final byte[] byteValue;
private final byte serializedValue;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/91ee8cc1/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
index 0b3849e..ca13281 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
@@ -306,6 +306,23 @@ public class MetaDataUtil {
return ByteUtil.concat(tenantId == null ? ByteUtil.EMPTY_BYTE_ARRAY : tenantId, QueryConstants.SEPARATOR_BYTE_ARRAY, schemaName == null ? ByteUtil.EMPTY_BYTE_ARRAY : schemaName, QueryConstants.SEPARATOR_BYTE_ARRAY, tableName, QueryConstants.SEPARATOR_BYTE_ARRAY, QueryConstants.SEPARATOR_BYTE_ARRAY, indexName);
}
+ public static byte[] getChildLinkKey(PName parentTenantId, PName parentSchemaName, PName parentTableName, PName viewTenantId, PName viewName) {
+ return ByteUtil.concat(parentTenantId == null ? ByteUtil.EMPTY_BYTE_ARRAY : parentTenantId.getBytes(), QueryConstants.SEPARATOR_BYTE_ARRAY,
+ parentSchemaName == null ? ByteUtil.EMPTY_BYTE_ARRAY : parentSchemaName.getBytes(), QueryConstants.SEPARATOR_BYTE_ARRAY,
+ parentTableName.getBytes(), QueryConstants.SEPARATOR_BYTE_ARRAY,
+ viewTenantId == null ? ByteUtil.EMPTY_BYTE_ARRAY : viewTenantId.getBytes(), QueryConstants.SEPARATOR_BYTE_ARRAY,
+ viewName.getBytes());
+ }
+
+ public static Cell getCell(List<Cell> cells, byte[] cq) {
+ for (Cell cell : cells) {
+ if (Bytes.compareTo(cell.getQualifierArray(), cell.getQualifierOffset(), cell.getQualifierLength(), cq, 0, cq.length) == 0) {
+ return cell;
+ }
+ }
+ return null;
+ }
+
public static boolean isMultiTenant(Mutation m, KeyValueBuilder builder, ImmutableBytesWritable ptr) {
if (getMutationValue(m, PhoenixDatabaseMetaData.MULTI_TENANT_BYTES, builder, ptr)) {
return Boolean.TRUE.equals(PBoolean.INSTANCE.toObject(ptr));
http://git-wip-us.apache.org/repos/asf/phoenix/blob/91ee8cc1/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
index 2b04ac1..279e6a2 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
@@ -903,6 +903,7 @@ public class UpgradeUtil {
"TABLE_SCHEM %s " +
"AND TABLE_NAME = ? " +
"AND COLUMN_NAME IS NOT NULL " +
+ "AND LINK_TYPE IS NULL " +
"ORDER BY " +
ORDINAL_POSITION;
@@ -1073,6 +1074,78 @@ public class UpgradeUtil {
}
}
+ /**
+ * Upgrade the metadata in the catalog table to enable adding columns to tables with views
+ * @param oldMetaConnection caller should take care of closing the passed connection appropriately
+ * @throws SQLException
+ */
+ public static void upgradeTo4_11_0(PhoenixConnection oldMetaConnection) throws SQLException {
+ PhoenixConnection metaConnection = null;
+ try {
+ // Need to use own connection with max time stamp to be able to read all data from SYSTEM.CATALOG
+ metaConnection = new PhoenixConnection(oldMetaConnection, HConstants.LATEST_TIMESTAMP);
+ logger.info("Upgrading metadata to add parent to child links for views");
+ metaConnection.commit();
+ // physical table
+ // |
+ // child view
+ // |
+ // grand child view
+ // Create parent table to child view CHILD link. As the PARENT link from child view to physical table is not there (it gets overwritten with the PHYSICAL link) use the PHYSICAL link instead.
+ // We need to filter out grand child views PHYSICAL links while running this query
+ String createChildLink = "UPSERT INTO SYSTEM.CATALOG(TENANT_ID,TABLE_SCHEM,TABLE_NAME,COLUMN_NAME,COLUMN_FAMILY,LINK_TYPE)" +
+ "SELECT PARENT_TENANT_ID," +
+ " CASE INSTR(COLUMN_FAMILY,'.')" +
+ " WHEN 0 THEN NULL" +
+ " ELSE REGEXP_SUBSTR(COLUMN_FAMILY,'[^\\.]+')" +
+ " END AS PARENT_SCHEMA," +
+ " CASE INSTR(COLUMN_FAMILY,'.')" +
+ " WHEN 0 THEN COLUMN_FAMILY" +
+ " ELSE SUBSTR(COLUMN_FAMILY,INSTR(COLUMN_FAMILY,'.')+1)" +
+ " END AS PARENT_TABLE," +
+ " TENANT_ID," +
+ " CASE WHEN TABLE_SCHEM IS NULL THEN TABLE_NAME" +
+ " ELSE TABLE_SCHEM||'.'||TABLE_NAME" +
+ " END AS VIEW_NAME," +
+ " 4 AS LINK_TYPE " +
+ "FROM SYSTEM.CATALOG(PARENT_TENANT_ID VARCHAR)" +
+ "WHERE LINK_TYPE = 2 " +
+ "AND (TENANT_ID, TABLE_SCHEM, TABLE_NAME) NOT IN ( " +
+ " SELECT TENANT_ID, " +
+ " TABLE_SCHEM, " +
+ " TABLE_NAME " +
+ " FROM SYSTEM.CATALOG " +
+ " WHERE LINK_TYPE = 3 )";
+ metaConnection.createStatement().execute(createChildLink);
+ metaConnection.commit();
+ // Create child view to grand child view CHILD link using grand child view to child view PARENT link.
+ String createGrandChildLink = "UPSERT INTO SYSTEM.CATALOG(TENANT_ID,TABLE_SCHEM,TABLE_NAME,COLUMN_NAME,COLUMN_FAMILY,LINK_TYPE)" +
+ "SELECT PARENT_TENANT_ID," +
+ " CASE INSTR(COLUMN_FAMILY,'.')" +
+ " WHEN 0 THEN NULL" +
+ " ELSE REGEXP_SUBSTR(COLUMN_FAMILY,'[^\\.]+')" +
+ " END AS PARENT_SCHEMA," +
+ " CASE INSTR(COLUMN_FAMILY,'.')" +
+ " WHEN 0 THEN COLUMN_FAMILY" +
+ " ELSE SUBSTR(COLUMN_FAMILY,INSTR(COLUMN_FAMILY,'.')+1)" +
+ " END AS PARENT_TABLE," +
+ " TENANT_ID," +
+ " CASE WHEN TABLE_SCHEM IS NULL THEN TABLE_NAME" +
+ " ELSE TABLE_SCHEM||'.'||TABLE_NAME" +
+ " END AS VIEW_NAME," +
+ " 4 AS LINK_TYPE " +
+ "FROM SYSTEM.CATALOG(PARENT_TENANT_ID VARCHAR)" +
+ "WHERE LINK_TYPE = 3 ";
+ metaConnection.createStatement().execute(createGrandChildLink);
+ metaConnection.commit();
+ metaConnection.getQueryServices().clearCache();
+ } finally {
+ if (metaConnection != null) {
+ metaConnection.close();
+ }
+ }
+ }
+
private static void upsertBaseColumnCountInHeaderRow(PhoenixConnection metaConnection,
String tenantId, String schemaName, String viewOrTableName, int baseColumnCount)
throws SQLException {