You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ja...@apache.org on 2014/02/18 11:04:57 UTC
[3/4] PHOENIX-21: Support indexes on multi-tenant views. Still needs
more testing
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/e2bd0ee0/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java b/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
index 26009ca..0d72c87 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
@@ -43,6 +43,7 @@ import org.apache.phoenix.schema.PColumn;
import org.apache.phoenix.schema.PDatum;
import org.apache.phoenix.schema.PIndexState;
import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTable.ViewType;
import org.apache.phoenix.schema.PTableType;
import com.google.common.collect.Lists;
@@ -85,7 +86,7 @@ public class QueryOptimizer {
}
PTable dataTable = dataPlan.getTableRef().getTable();
List<PTable>indexes = Lists.newArrayList(dataTable.getIndexes());
- if (indexes.isEmpty() || dataPlan.getTableRef().hasDynamicCols() || select.getHint().hasHint(Hint.NO_INDEX)) {
+ if (indexes.isEmpty() || dataPlan.isDegenerate() || dataPlan.getTableRef().hasDynamicCols() || select.getHint().hasHint(Hint.NO_INDEX)) {
return dataPlan;
}
@@ -110,7 +111,14 @@ public class QueryOptimizer {
return hintedPlan;
}
for (PTable index : indexes) {
- addPlan(statement, translatedIndexSelect, index, targetColumns, parallelIteratorFactory, plans);
+ QueryPlan plan = addPlan(statement, translatedIndexSelect, index, targetColumns, parallelIteratorFactory, dataPlan);
+ if (plan != null) {
+ // Query can't possibly return anything so just return this plan.
+ if (plan.isDegenerate()) {
+ return plan;
+ }
+ plans.add(plan);
+ }
}
return chooseBestPlan(select, plans);
@@ -151,8 +159,9 @@ public class QueryOptimizer {
int indexPos = getIndexPosition(indexes, indexName);
if (indexPos >= 0) {
// Hinted index is applicable, so return it. It'll be the plan at position 1, after the data plan
- if (addPlan(statement, select, indexes.get(indexPos), targetColumns, parallelIteratorFactory, plans)) {
- return plans.get(1);
+ QueryPlan plan = addPlan(statement, select, indexes.get(indexPos), targetColumns, parallelIteratorFactory, dataPlan);
+ if (plan != null) {
+ return plan;
}
indexes.remove(indexPos);
}
@@ -171,8 +180,7 @@ public class QueryOptimizer {
return -1;
}
- private static boolean addPlan(PhoenixStatement statement, SelectStatement select, PTable index, List<? extends PDatum> targetColumns, ParallelIteratorFactory parallelIteratorFactory, List<QueryPlan> plans) throws SQLException {
- QueryPlan dataPlan = plans.get(0);
+ private static QueryPlan addPlan(PhoenixStatement statement, SelectStatement select, PTable index, List<? extends PDatum> targetColumns, ParallelIteratorFactory parallelIteratorFactory, QueryPlan dataPlan) throws SQLException {
int nColumns = dataPlan.getProjector().getColumnCount();
String alias = '"' + dataPlan.getTableRef().getTableAlias() + '"'; // double quote in case it's case sensitive
String schemaName = dataPlan.getTableRef().getTable().getSchemaName().getString();
@@ -183,13 +191,15 @@ public class QueryOptimizer {
try {
SelectStatement indexSelect = FACTORY.select(select, tables);
ColumnResolver resolver = FromCompiler.getResolver(indexSelect, statement.getConnection());
- QueryCompiler compiler = new QueryCompiler(statement, indexSelect, resolver, targetColumns, parallelIteratorFactory);
- QueryPlan plan = compiler.compile();
- // Checking the index status and number of columns handles the wildcard cases correctly
- // We can't check the status earlier, because the index table may be out-of-date.
- if (plan.getTableRef().getTable().getIndexState() == PIndexState.ACTIVE && plan.getProjector().getColumnCount() == nColumns) {
- plans.add(plan);
- return true;
+ // Check index state of now potentially updated index table to make sure it's active
+ if (PIndexState.ACTIVE.equals(resolver.getTables().get(0).getTable().getIndexState())) {
+ QueryCompiler compiler = new QueryCompiler(statement, indexSelect, resolver, targetColumns, parallelIteratorFactory);
+ QueryPlan plan = compiler.compile();
+ // Checking number of columns handles the wildcard cases correctly, as in that case the index
+ // must contain all columns from the data table to be able to be used.
+ if (plan.getTableRef().getTable().getIndexState() == PIndexState.ACTIVE && plan.getProjector().getColumnCount() == nColumns) {
+ return plan;
+ }
}
} catch (ColumnNotFoundException e) {
/* Means that a column is being used that's not in our index.
@@ -198,7 +208,7 @@ public class QueryOptimizer {
* the index table to the data table.
*/
}
- return false;
+ return null;
}
/**
@@ -215,9 +225,9 @@ public class QueryOptimizer {
* @return
*/
private QueryPlan chooseBestPlan(SelectStatement select, List<QueryPlan> plans) {
- QueryPlan firstPlan = plans.get(0);
+ final QueryPlan dataPlan = plans.get(0);
if (plans.size() == 1) {
- return firstPlan;
+ return dataPlan;
}
/**
@@ -253,12 +263,31 @@ public class QueryOptimizer {
if (bestCandidates.isEmpty()) {
bestCandidates.addAll(stillCandidates);
}
+
+ int nViewConstants = 0;
+ PTable dataTable = dataPlan.getTableRef().getTable();
+ if (dataTable.getViewType() == ViewType.UPDATABLE) {
+ for (PColumn column : dataTable.getColumns()) {
+ if (column.getViewConstant() != null) {
+ nViewConstants++;
+ }
+ }
+ }
+ final int boundRanges = nViewConstants;
final int comparisonOfDataVersusIndexTable = select.getHint().hasHint(Hint.USE_DATA_OVER_INDEX_TABLE) ? -1 : 1;
Collections.sort(bestCandidates, new Comparator<QueryPlan>() {
@Override
public int compare(QueryPlan plan1, QueryPlan plan2) {
+ PTable table1 = plan1.getTableRef().getTable();
+ PTable table2 = plan2.getTableRef().getTable();
int c = plan2.getContext().getScanRanges().getRanges().size() - plan1.getContext().getScanRanges().getRanges().size();
+ // Account for potential view constants which are always bound
+ if (plan1 == dataPlan) { // plan2 is index plan. Ignore the viewIndexId if present
+ c += boundRanges - (table2.getViewIndexId() == null ? 0 : 1);
+ } else { // plan1 is index plan. Ignore the viewIndexId if present
+ c -= boundRanges - (table1.getViewIndexId() == null ? 0 : 1);
+ }
if (c != 0) return c;
if (plan1.getGroupBy()!=null && plan2.getGroupBy()!=null) {
if (plan1.getGroupBy().isOrderPreserving() != plan2.getGroupBy().isOrderPreserving()) {
@@ -266,8 +295,6 @@ public class QueryOptimizer {
}
}
// Use smaller table (table with fewest kv columns)
- PTable table1 = plan1.getTableRef().getTable();
- PTable table2 = plan2.getTableRef().getTable();
c = (table1.getColumns().size() - table1.getPKColumns().size()) - (table2.getColumns().size() - table2.getPKColumns().size());
if (c != 0) return c;
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/e2bd0ee0/phoenix-core/src/main/java/org/apache/phoenix/parse/CreateSequenceStatement.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/parse/CreateSequenceStatement.java b/phoenix-core/src/main/java/org/apache/phoenix/parse/CreateSequenceStatement.java
index 325554b..44d5f34 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/parse/CreateSequenceStatement.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/parse/CreateSequenceStatement.java
@@ -19,6 +19,10 @@ package org.apache.phoenix.parse;
public class CreateSequenceStatement extends MutableStatement {
+ public static CreateSequenceStatement create(TableName sequenceName) {
+ return new CreateSequenceStatement(sequenceName, null, null, null, true, 0);
+ }
+
private final TableName sequenceName;
private final ParseNode startWith;
private final ParseNode incrementBy;
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/e2bd0ee0/phoenix-core/src/main/java/org/apache/phoenix/parse/LiteralParseNode.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/parse/LiteralParseNode.java b/phoenix-core/src/main/java/org/apache/phoenix/parse/LiteralParseNode.java
index 4636caf..f9bbea1 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/parse/LiteralParseNode.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/parse/LiteralParseNode.java
@@ -48,6 +48,13 @@ public class LiteralParseNode extends TerminalParseNode {
this.value = this.type == null ? null : this.type.toObject(value, this.type);
}
+ public LiteralParseNode(Object value, PDataType type) {
+ this.type = type;
+ // This will make the value null if the value passed through represents null for the given type.
+ // For example, an empty string is treated as a null.
+ this.value = this.type == null ? null : this.type.toObject(value, this.type);
+ }
+
public PDataType getType() {
return type;
}
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/e2bd0ee0/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
index a2e8eaa..192db11 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
@@ -34,6 +34,7 @@ import org.apache.phoenix.compile.MutationPlan;
import org.apache.phoenix.coprocessor.MetaDataProtocol.MetaDataMutationResult;
import org.apache.phoenix.execute.MutationState;
import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.schema.SequenceKey;
@@ -67,7 +68,7 @@ public interface ConnectionQueryServices extends QueryServices, MetaDataMutated
public MetaDataMutationResult getTable(byte[] tenantId, byte[] schemaName, byte[] tableName, long tableTimestamp, long clientTimetamp) throws SQLException;
public MetaDataMutationResult createTable(List<Mutation> tableMetaData, byte[] tableName, PTableType tableType, Map<String,Object> tableProps, List<Pair<byte[],Map<String,Object>>> families, byte[][] splits) throws SQLException;
public MetaDataMutationResult dropTable(List<Mutation> tableMetadata, PTableType tableType) throws SQLException;
- public MetaDataMutationResult addColumn(List<Mutation> tableMetaData, PTableType tableType, List<Pair<byte[],Map<String,Object>>> families) throws SQLException;
+ public MetaDataMutationResult addColumn(List<Mutation> tableMetaData, List<Pair<byte[],Map<String,Object>>> families, PTable table) throws SQLException;
public MetaDataMutationResult dropColumn(List<Mutation> tableMetadata, PTableType tableType) throws SQLException;
public MetaDataMutationResult updateIndexState(List<Mutation> tableMetadata, String parentTableName) throws SQLException;
public MutationState updateData(MutationPlan plan) throws SQLException;
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/e2bd0ee0/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 8b73e49..02e3f2b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.TableExistsException;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.client.Append;
@@ -80,13 +81,17 @@ import org.apache.phoenix.schema.EmptySequenceCacheException;
import org.apache.phoenix.schema.MetaDataSplitPolicy;
import org.apache.phoenix.schema.NewerTableAlreadyExistsException;
import org.apache.phoenix.schema.PColumn;
+import org.apache.phoenix.schema.PColumnFamily;
+import org.apache.phoenix.schema.PDataType;
import org.apache.phoenix.schema.PMetaData;
import org.apache.phoenix.schema.PMetaDataImpl;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.schema.ReadOnlyTableException;
+import org.apache.phoenix.schema.SaltingUtil;
import org.apache.phoenix.schema.Sequence;
import org.apache.phoenix.schema.SequenceKey;
+import org.apache.phoenix.schema.TableAlreadyExistsException;
import org.apache.phoenix.schema.TableNotFoundException;
import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.JDBCUtil;
@@ -275,6 +280,9 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
currentKey = regionLocation.getRegionInfo().getEndKey();
} while (!Bytes.equals(currentKey, HConstants.EMPTY_END_ROW));
return locations;
+ } catch (org.apache.hadoop.hbase.TableNotFoundException e) {
+ String fullName = Bytes.toString(tableName);
+ throw new TableNotFoundException(SchemaUtil.getSchemaNameFromFullName(fullName), SchemaUtil.getTableNameFromFullName(fullName));
} catch (IOException e) {
if (retryCount++ < maxRetryCount) { // One retry, in case split occurs while navigating
reload = true;
@@ -522,7 +530,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
if (tableType == PTableType.VIEW) {
String fullTableName = Bytes.toString(tableName);
throw new ReadOnlyTableException(
- "The HBase column families for a read-only table must already exist",
+ "The HBase column families for a VIEW must already exist",
SchemaUtil.getSchemaNameFromFullName(fullTableName),
SchemaUtil.getTableNameFromFullName(fullTableName),
Bytes.toString(family.getFirst()));
@@ -530,9 +538,11 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
columnDescriptor = generateColumnFamilyDescriptor(family, tableType );
} else {
columnDescriptor = new HColumnDescriptor(oldDescriptor);
- if (tableType != PTableType.VIEW) {
- modifyColumnFamilyDescriptor(columnDescriptor, family);
+ // Don't attempt to make any metadata changes for a VIEW
+ if (tableType == PTableType.VIEW) {
+ return;
}
+ modifyColumnFamilyDescriptor(columnDescriptor, family);
}
if (columnDescriptor.equals(oldDescriptor)) {
@@ -573,12 +583,13 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
/**
*
* @param tableName
- * @param familyNames
* @param splits
+ * @param modifyExistingMetaData TODO
+ * @param familyNames
* @return true if table was created and false if it already exists
* @throws SQLException
*/
- private boolean ensureTableCreated(byte[] tableName, PTableType tableType , Map<String,Object> props, List<Pair<byte[],Map<String,Object>>> families, byte[][] splits) throws SQLException {
+ private HTableDescriptor ensureTableCreated(byte[] tableName, PTableType tableType , Map<String,Object> props, List<Pair<byte[],Map<String,Object>>> families, byte[][] splits, boolean modifyExistingMetaData) throws SQLException {
HBaseAdmin admin = null;
SQLException sqlE = null;
HTableDescriptor existingDesc = null;
@@ -618,7 +629,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
} catch (TableExistsException e) {
// We can ignore this, as it just means that another client beat us
// to creating the HBase metadata.
- return false;
+ return null;
}
if (isMetaTable) {
checkClientServerCompatibility();
@@ -633,14 +644,14 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
admin.modifyTable(tableName, newDesc);
admin.enableTable(tableName);
}
- return true;
+ return null;
} else {
- if (existingDesc.equals(newDesc)) {
+ if (!modifyExistingMetaData || existingDesc.equals(newDesc)) {
// Table is already created. Note that the presplits are ignored in this case
if (isMetaTable) {
checkClientServerCompatibility();
}
- return false;
+ return existingDesc;
}
if (isMetaTable) {
@@ -656,7 +667,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
// TODO: What if not all existing column families are present?
admin.modifyTable(tableName, newDesc);
admin.enableTable(tableName);
- return false;
+ return newDesc;
}
} catch (IOException e) {
@@ -678,7 +689,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
}
}
- return true; // will never make it here
+ return null; // will never make it here
}
private static boolean isInvalidMutableIndexConfig(Long serverVersion) {
@@ -786,22 +797,95 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
}
+ private void ensureViewIndexTableCreated(byte[] physicalTableName, Map<String,Object> tableProps, List<Pair<byte[],Map<String,Object>>> families, byte[][] splits, long timestamp) throws SQLException {
+ Long maxFileSize = (Long)tableProps.get(HTableDescriptor.MAX_FILESIZE);
+ if (maxFileSize == null) {
+ maxFileSize = this.config.getLong(HConstants.HREGION_MAX_FILESIZE, HConstants.DEFAULT_MAX_FILE_SIZE);
+ }
+ byte[] physicalIndexName = MetaDataUtil.getViewIndexPhysicalName(physicalTableName);
+
+ int indexMaxFileSizePerc;
+ // Get percentage to use from table props first and then fallback to config
+ Integer indexMaxFileSizePercProp = (Integer)tableProps.remove(QueryServices.INDEX_MAX_FILESIZE_PERC_ATTRIB);
+ if (indexMaxFileSizePercProp == null) {
+ indexMaxFileSizePerc = config.getInt(QueryServices.INDEX_MAX_FILESIZE_PERC_ATTRIB, QueryServicesOptions.DEFAULT_INDEX_MAX_FILESIZE_PERC);
+ } else {
+ indexMaxFileSizePerc = indexMaxFileSizePercProp;
+ }
+ long indexMaxFileSize = maxFileSize * indexMaxFileSizePerc / 100;
+ tableProps.put(HTableDescriptor.MAX_FILESIZE, indexMaxFileSize);
+ tableProps.put(MetaDataUtil.IS_VIEW_INDEX_TABLE_PROP_NAME, PDataType.TRUE_BYTES);
+ // Only use splits if table is salted, otherwise it may not be applicable
+ HTableDescriptor desc = ensureTableCreated(physicalIndexName, PTableType.TABLE, tableProps, families, splits, false);
+ if (desc != null) {
+ if (!Boolean.TRUE.equals(PDataType.BOOLEAN.toObject(desc.getValue(MetaDataUtil.IS_VIEW_INDEX_TABLE_PROP_BYTES)))) {
+ String fullTableName = Bytes.toString(physicalIndexName);
+ throw new TableAlreadyExistsException(
+ "Unable to create shared physical table for indexes on views.",
+ SchemaUtil.getSchemaNameFromFullName(fullTableName),
+ SchemaUtil.getTableNameFromFullName(fullTableName));
+ }
+ }
+ }
+
+ private boolean ensureViewIndexTableDropped(byte[] physicalTableName, long timestamp) throws SQLException {
+ byte[] physicalIndexName = MetaDataUtil.getViewIndexPhysicalName(physicalTableName);
+ HTableDescriptor desc = null;
+ HBaseAdmin admin = null;
+ boolean wasDeleted = false;
+ try {
+ admin = new HBaseAdmin(config);
+ try {
+ desc = admin.getTableDescriptor(physicalIndexName);
+ if (Boolean.TRUE.equals(PDataType.BOOLEAN.toObject(desc.getValue(MetaDataUtil.IS_VIEW_INDEX_TABLE_PROP_BYTES)))) {
+ final ReadOnlyProps props = this.getProps();
+ final boolean dropMetadata = props.getBoolean(DROP_METADATA_ATTRIB, DEFAULT_DROP_METADATA);
+ if (dropMetadata) {
+ admin.disableTable(physicalIndexName);
+ admin.deleteTable(physicalIndexName);
+ wasDeleted = true;
+ }
+ }
+ } catch (org.apache.hadoop.hbase.TableNotFoundException ignore) {
+ // Ignore, as we may never have created a view index table
+ }
+ } catch (IOException e) {
+ throw ServerUtil.parseServerException(e);
+ } finally {
+ try {
+ if (admin != null) admin.close();
+ } catch (IOException e) {
+ logger.warn("",e);
+ }
+ }
+ return wasDeleted;
+ }
+
@Override
public MetaDataMutationResult createTable(final List<Mutation> tableMetaData, byte[] physicalTableName, PTableType tableType,
Map<String,Object> tableProps, final List<Pair<byte[],Map<String,Object>>> families, byte[][] splits) throws SQLException {
byte[][] rowKeyMetadata = new byte[3][];
- Mutation m = tableMetaData.get(0);
+ Mutation m = MetaDataUtil.getPutOnlyTableHeaderRow(tableMetaData);
byte[] key = m.getRow();
SchemaUtil.getVarChars(key, rowKeyMetadata);
byte[] tenantIdBytes = rowKeyMetadata[PhoenixDatabaseMetaData.TENANT_ID_INDEX];
byte[] schemaBytes = rowKeyMetadata[PhoenixDatabaseMetaData.SCHEMA_NAME_INDEX];
byte[] tableBytes = rowKeyMetadata[PhoenixDatabaseMetaData.TABLE_NAME_INDEX];
- if (tableType != PTableType.VIEW || physicalTableName != null) {
- if (physicalTableName == null) {
- physicalTableName = SchemaUtil.getTableNameAsBytes(schemaBytes, tableBytes);
+ byte[] tableName = physicalTableName != null ? physicalTableName : SchemaUtil.getTableNameAsBytes(schemaBytes, tableBytes);
+ if ((tableType == PTableType.VIEW && physicalTableName != null) || (tableType != PTableType.VIEW && physicalTableName == null)) {
+ // For views this will ensure that metadata already exists
+ ensureTableCreated(tableName, tableType, tableProps, families, splits, true);
+ }
+
+ if (tableType == PTableType.INDEX && physicalTableName != null) { // Index on view
+ // Physical index table created up front for multi tenant
+ // TODO: if viewIndexId is Short.MIN_VALUE, then we don't need to attempt to create it
+ if (!MetaDataUtil.isMultiTenant(m)) {
+ ensureViewIndexTableCreated(physicalTableName, MetaDataUtil.getClientTimeStamp(m));
}
- ensureTableCreated(physicalTableName, tableType, tableProps, families, splits);
+ } else if (tableType == PTableType.TABLE && MetaDataUtil.isMultiTenant(m)) { // Create view index table up front for multi tenant tables
+ ensureViewIndexTableCreated(tableName, tableProps, families, MetaDataUtil.isSalted(m) ? splits : null, MetaDataUtil.getClientTimeStamp(m));
}
byte[] tableKey = SchemaUtil.getTableKey(tenantIdBytes, schemaBytes, tableBytes);
@@ -848,11 +932,16 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
final MutationCode code = result.getMutationCode();
switch(code) {
case TABLE_ALREADY_EXISTS:
- final ReadOnlyProps props = this.getProps();
- final boolean dropMetadata = props.getBoolean(DROP_METADATA_ATTRIB, DEFAULT_DROP_METADATA);
+ ReadOnlyProps props = this.getProps();
+ boolean dropMetadata = props.getBoolean(DROP_METADATA_ATTRIB, DEFAULT_DROP_METADATA);
if (dropMetadata) {
dropTables(result.getTableNamesToDelete());
}
+ if (tableType == PTableType.TABLE) {
+ byte[] physicalTableName = SchemaUtil.getTableNameAsBytes(schemaBytes, tableBytes);
+ long timestamp = MetaDataUtil.getClientTimeStamp(tableMetaData);
+ ensureViewIndexTableDropped(physicalTableName, timestamp);
+ }
break;
default:
break;
@@ -895,20 +984,76 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
}
+ private static Map<String,Object> createPropertiesMap(Map<ImmutableBytesWritable,ImmutableBytesWritable> htableProps) {
+ Map<String,Object> props = Maps.newHashMapWithExpectedSize(htableProps.size());
+ for (Map.Entry<ImmutableBytesWritable,ImmutableBytesWritable> entry : htableProps.entrySet()) {
+ ImmutableBytesWritable key = entry.getKey();
+ ImmutableBytesWritable value = entry.getValue();
+ props.put(Bytes.toString(key.get(), key.getOffset(), key.getLength()), Bytes.toString(value.get(), value.getOffset(), value.getLength()));
+ }
+ return props;
+ }
+
+ private void ensureViewIndexTableCreated(byte[] physicalIndexTableName, long timestamp) throws SQLException {
+ PTable table;
+ String name = Bytes.toString(
+ physicalIndexTableName,
+ MetaDataUtil.VIEW_INDEX_TABLE_PREFIX_BYTES.length,
+ physicalIndexTableName.length-MetaDataUtil.VIEW_INDEX_TABLE_PREFIX_BYTES.length);
+ try {
+ table = latestMetaData.getTable(name);
+ if (table.getTimeStamp() >= timestamp) { // Table in cache is newer than client timestamp which shouldn't be the case
+ throw new TableNotFoundException(table.getSchemaName().getString(), table.getTableName().getString());
+ }
+ } catch (TableNotFoundException e) {
+ byte[] schemaName = Bytes.toBytes(SchemaUtil.getSchemaNameFromFullName(name));
+ byte[] tableName = Bytes.toBytes(SchemaUtil.getTableNameFromFullName(name));
+ MetaDataMutationResult result = this.getTable(null, schemaName, tableName, HConstants.LATEST_TIMESTAMP, timestamp);
+ table = result.getTable();
+ if (table == null) {
+ throw e;
+ }
+ }
+ ensureViewIndexTableCreated(table, timestamp);
+ }
+
+ private void ensureViewIndexTableCreated(PTable table, long timestamp) throws SQLException {
+ byte[] physicalTableName = table.getPhysicalName().getBytes();
+ HTableDescriptor htableDesc = this.getTableDescriptor(physicalTableName);
+ Map<String,Object> tableProps = createPropertiesMap(htableDesc.getValues());
+ List<Pair<byte[],Map<String,Object>>> families = Lists.newArrayListWithExpectedSize(Math.max(1, table.getColumnFamilies().size()));
+ if (families.isEmpty()) {
+ byte[] familyName = SchemaUtil.getEmptyColumnFamily(table);
+ Map<String,Object> familyProps = createPropertiesMap(htableDesc.getFamily(familyName).getValues());
+ families.add(new Pair<byte[],Map<String,Object>>(familyName, familyProps));
+ } else {
+ for (PColumnFamily family : table.getColumnFamilies()) {
+ byte[] familyName = SchemaUtil.getEmptyColumnFamily(table);
+ Map<String,Object> familyProps = createPropertiesMap(htableDesc.getFamily(familyName).getValues());
+ families.add(new Pair<byte[],Map<String,Object>>(family.getName().getBytes(), familyProps));
+ }
+ }
+ byte[][] splits = null;
+ if (table.getBucketNum() != null) {
+ splits = SaltingUtil.getSalteByteSplitPoints(table.getBucketNum());
+ }
+
+ ensureViewIndexTableCreated(physicalTableName, tableProps, families, splits, timestamp);
+ }
+
@Override
- public MetaDataMutationResult addColumn(final List<Mutation> tableMetaData, PTableType tableType, List<Pair<byte[],Map<String,Object>>> families) throws SQLException {
+ public MetaDataMutationResult addColumn(final List<Mutation> tableMetaData, List<Pair<byte[],Map<String,Object>>> families, PTable table) throws SQLException {
byte[][] rowKeyMetaData = new byte[3][];
+ PTableType tableType = table.getType();
- byte[] rowKey = tableMetaData.get(0).getRow();
+ Mutation m = tableMetaData.get(0);
+ byte[] rowKey = m.getRow();
SchemaUtil.getVarChars(rowKey, rowKeyMetaData);
byte[] tenantIdBytes = rowKeyMetaData[PhoenixDatabaseMetaData.TENANT_ID_INDEX];
byte[] schemaBytes = rowKeyMetaData[PhoenixDatabaseMetaData.SCHEMA_NAME_INDEX];
byte[] tableBytes = rowKeyMetaData[PhoenixDatabaseMetaData.TABLE_NAME_INDEX];
byte[] tableKey = SchemaUtil.getTableKey(tenantIdBytes, schemaBytes, tableBytes);
- byte[] tableName = SchemaUtil.getTableNameAsBytes(schemaBytes, tableBytes);
for ( Pair<byte[],Map<String,Object>> family : families) {
-
- PTable table = latestMetaData.getTable(Bytes.toString(tableName));
ensureFamilyCreated(table.getPhysicalName().getBytes(), tableType, family);
}
// Special case for call during drop table to ensure that the empty column family exists.
@@ -926,6 +1071,18 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
return instance.addColumn(tableMetaData);
}
});
+ if (tableType == PTableType.TABLE && result.getMutationCode() == MutationCode.COLUMN_NOT_FOUND) { // Success
+ // If we're changing MULTI_TENANT to true or false, create or drop the view index table
+ KeyValue kv = MetaDataUtil.getMutationKeyValue(m, PhoenixDatabaseMetaData.MULTI_TENANT_BYTES);
+ if (kv != null) {
+ long timestamp = MetaDataUtil.getClientTimeStamp(m);
+ if (Boolean.TRUE.equals(PDataType.BOOLEAN.toObject(kv.getBuffer(), kv.getValueOffset(), kv.getValueLength()))) {
+ this.ensureViewIndexTableCreated(table, timestamp);
+ } else {
+ this.ensureViewIndexTableDropped(table.getPhysicalName().getBytes(), timestamp);
+ }
+ }
+ }
return result;
}
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/e2bd0ee0/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
index 59761f1..6927d0a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
@@ -176,7 +176,7 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
}
@Override
- public MetaDataMutationResult addColumn(List<Mutation> tableMetaData, PTableType readOnly, List<Pair<byte[],Map<String,Object>>> families) throws SQLException {
+ public MetaDataMutationResult addColumn(List<Mutation> tableMetaData, List<Pair<byte[],Map<String,Object>>> families, PTable table) throws SQLException {
return new MetaDataMutationResult(MutationCode.TABLE_ALREADY_EXISTS, 0, null);
}
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/e2bd0ee0/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
index 7d92ab3..f586c9e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
@@ -118,8 +118,8 @@ public class DelegateConnectionQueryServices extends DelegateQueryServices imple
}
@Override
- public MetaDataMutationResult addColumn(List<Mutation> tabeMetaData, PTableType tableType, List<Pair<byte[],Map<String,Object>>> families ) throws SQLException {
- return getDelegate().addColumn(tabeMetaData, tableType, families);
+ public MetaDataMutationResult addColumn(List<Mutation> tabeMetaData, List<Pair<byte[],Map<String,Object>>> families, PTable table) throws SQLException {
+ return getDelegate().addColumn(tabeMetaData, families, table);
}
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/e2bd0ee0/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
index b62c3e9..4987158 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
@@ -24,7 +24,6 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CACHE_SIZE;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CHAR_OCTET_LENGTH;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_COUNT;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_DEF;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SORT_ORDER;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_NAME;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.COLUMN_SIZE;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CURRENT_VALUE;
@@ -53,6 +52,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SCOPE_TABLE;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SELF_REFERENCING_COL_NAME_NAME;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SEQUENCE_NAME;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SEQUENCE_SCHEMA;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SORT_ORDER;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SOURCE_DATA_TYPE;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SQL_DATA_TYPE;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SQL_DATETIME_SUB;
@@ -67,6 +67,8 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_NAME;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_SCHEMA;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_SEQUENCE;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_TABLE;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_CONSTANT;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_STATEMENT;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_TYPE;
@@ -167,6 +169,16 @@ public interface QueryConstants {
REF_GENERATION_NAME + " VARCHAR," +
TABLE_SEQ_NUM + " BIGINT," +
COLUMN_COUNT + " INTEGER," +
+ SALT_BUCKETS + " INTEGER," +
+ DATA_TABLE_NAME + " VARCHAR," +
+ INDEX_STATE + " CHAR(1),\n" +
+ IMMUTABLE_ROWS + " BOOLEAN,\n" +
+ VIEW_STATEMENT + " VARCHAR,\n" +
+ DEFAULT_COLUMN_FAMILY_NAME + " VARCHAR,\n" +
+ DISABLE_WAL + " BOOLEAN,\n" +
+ MULTI_TENANT + " BOOLEAN,\n" +
+ VIEW_TYPE + " UNSIGNED_TINYINT,\n" +
+ VIEW_INDEX_ID + " SMALLINT,\n" +
// Column metadata (will be null for table row)
COLUMN_SIZE + " INTEGER," +
BUFFER_LENGTH + " INTEGER," +
@@ -182,23 +194,13 @@ public interface QueryConstants {
SCOPE_CATALOG + " VARCHAR," +
SCOPE_SCHEMA + " VARCHAR," +
SCOPE_TABLE + " VARCHAR," +
- SOURCE_DATA_TYPE + " INTEGER," + // supposed to be SHORT
+ SOURCE_DATA_TYPE + " SMALLINT," +
IS_AUTOINCREMENT + " VARCHAR," +
- // Columns added in 1.2.1
SORT_ORDER + " INTEGER," +
- SALT_BUCKETS + " INTEGER," +
- // Columns added in 2.0.0
- DATA_TABLE_NAME + " VARCHAR," +
- INDEX_STATE + " CHAR(1),\n" +
- IMMUTABLE_ROWS + " BOOLEAN,\n" +
- // Columns added in 3.0.0
- VIEW_STATEMENT + " VARCHAR,\n" +
- DEFAULT_COLUMN_FAMILY_NAME + " VARCHAR,\n" +
- DISABLE_WAL + " BOOLEAN,\n" +
- MULTI_TENANT + " BOOLEAN,\n" +
- VIEW_TYPE + " UNSIGNED_TINYINT,\n" +
- LINK_TYPE + " UNSIGNED_TINYINT,\n" +
ARRAY_SIZE + " INTEGER,\n" +
+ VIEW_CONSTANT + " VARBINARY,\n" +
+ // Link metadata (only set on rows linking table to index or view)
+ LINK_TYPE + " UNSIGNED_TINYINT,\n" +
"CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TENANT_ID + ","
+ TABLE_SCHEM_NAME + "," + TABLE_NAME_NAME + "," + COLUMN_NAME + "," + TABLE_CAT_NAME + "))\n" +
HConstants.VERSIONS + "=" + MetaDataProtocol.DEFAULT_MAX_META_DATA_VERSIONS + ",\n" +
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/e2bd0ee0/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index b91068a..b6a65c0 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -20,7 +20,6 @@ package org.apache.phoenix.query;
import java.util.concurrent.ExecutorService;
import org.apache.http.annotation.Immutable;
-
import org.apache.phoenix.iterate.SpoolTooBigToDiskException;
import org.apache.phoenix.memory.MemoryManager;
import org.apache.phoenix.optimize.QueryOptimizer;
@@ -93,6 +92,7 @@ public interface QueryServices extends SQLCloseable {
public static final String ZOOKEEPER_ROOT_NODE_ATTRIB = "zookeeper.znode.parent";
public static final String DISTINCT_VALUE_COMPRESS_THRESHOLD_ATTRIB = "phoenix.distinct.value.compress.threshold";
public static final String SEQUENCE_CACHE_SIZE_ATTRIB = "phoenix.sequence.cacheSize";
+ public static final String INDEX_MAX_FILESIZE_PERC_ATTRIB = "phoenix.index.maxDataFileSizePerc";
/**
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/e2bd0ee0/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index 4a07245..f9fdbfe 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -22,8 +22,8 @@ import static org.apache.phoenix.query.QueryServices.CALL_QUEUE_ROUND_ROBIN_ATTR
import static org.apache.phoenix.query.QueryServices.DATE_FORMAT_ATTRIB;
import static org.apache.phoenix.query.QueryServices.DROP_METADATA_ATTRIB;
import static org.apache.phoenix.query.QueryServices.GROUPBY_MAX_CACHE_SIZE_ATTRIB;
-import static org.apache.phoenix.query.QueryServices.GROUPBY_SPILL_FILES_ATTRIB;
import static org.apache.phoenix.query.QueryServices.GROUPBY_SPILLABLE_ATTRIB;
+import static org.apache.phoenix.query.QueryServices.GROUPBY_SPILL_FILES_ATTRIB;
import static org.apache.phoenix.query.QueryServices.IMMUTABLE_ROWS_ATTRIB;
import static org.apache.phoenix.query.QueryServices.INDEX_MUTATE_BATCH_SIZE_THRESHOLD_ATTRIB;
import static org.apache.phoenix.query.QueryServices.KEEP_ALIVE_MS_ATTRIB;
@@ -57,7 +57,6 @@ import java.util.Map.Entry;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.regionserver.wal.WALEditCodec;
-
import org.apache.phoenix.util.DateUtil;
import org.apache.phoenix.util.ReadOnlyProps;
@@ -111,6 +110,7 @@ public class QueryServicesOptions {
public static final long DEFAULT_GROUPBY_MAX_CACHE_MAX = 1024L*1024L*100L; // 100 Mb
public static final int DEFAULT_SEQUENCE_CACHE_SIZE = 100; // reserve 100 sequences at a time
+ public static final int DEFAULT_INDEX_MAX_FILESIZE_PERC = 50; // % of data table max file size for index table
private final Configuration config;
http://git-wip-us.apache.org/repos/asf/incubator-phoenix/blob/e2bd0ee0/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateColumn.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateColumn.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateColumn.java
index c916555..aa2fb3e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateColumn.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/DelegateColumn.java
@@ -66,4 +66,9 @@ public class DelegateColumn extends DelegateDatum implements PColumn {
public Integer getArraySize() {
return getDelegate().getArraySize();
}
+
+ @Override
+ public byte[] getViewConstant() {
+ return getDelegate().getViewConstant();
+ }
}