You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ja...@apache.org on 2017/11/15 18:48:13 UTC
[18/37] phoenix git commit: PHOENIX-4290 Full table scan performed
for DELETE with table having immutable indexes
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e0df4b2e/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
index f88b34b..b5293bb 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
@@ -17,8 +17,6 @@
*/
package org.apache.phoenix.compile;
-import static org.apache.phoenix.schema.PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS;
-
import java.sql.SQLException;
import java.sql.SQLFeatureNotSupportedException;
import java.util.ArrayList;
@@ -66,6 +64,7 @@ import org.apache.phoenix.schema.ColumnNotFoundException;
import org.apache.phoenix.schema.ColumnRef;
import org.apache.phoenix.schema.FunctionNotFoundException;
import org.apache.phoenix.schema.MetaDataClient;
+import org.apache.phoenix.schema.MetaDataEntityNotFoundException;
import org.apache.phoenix.schema.PColumn;
import org.apache.phoenix.schema.PColumnFamily;
import org.apache.phoenix.schema.PColumnFamilyImpl;
@@ -73,9 +72,9 @@ import org.apache.phoenix.schema.PColumnImpl;
import org.apache.phoenix.schema.PName;
import org.apache.phoenix.schema.PNameFactory;
import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTable.ImmutableStorageScheme;
import org.apache.phoenix.schema.PTable.IndexType;
import org.apache.phoenix.schema.PTable.QualifierEncodingScheme;
-import org.apache.phoenix.schema.PTable.ImmutableStorageScheme;
import org.apache.phoenix.schema.PTableImpl;
import org.apache.phoenix.schema.PTableKey;
import org.apache.phoenix.schema.PTableType;
@@ -871,7 +870,9 @@ public class FromCompiler {
TableRef tableRef = iterator.next();
try {
PColumnFamily columnFamily = tableRef.getTable().getColumnFamily(cfName);
- if (theColumnFamilyRef != null) { throw new TableNotFoundException(cfName); }
+ if (columnFamily == null) {
+ throw new TableNotFoundException(cfName);
+ }
theColumnFamilyRef = new ColumnFamilyRef(tableRef, columnFamily);
} catch (ColumnFamilyNotFoundException e) {}
}
@@ -914,10 +915,42 @@ public class FromCompiler {
PColumn column = tableRef.getTable().getColumnForColumnName(colName);
return new ColumnRef(tableRef, column.getPosition());
} catch (TableNotFoundException e) {
- // Try using the tableName as a columnFamily reference instead
- ColumnFamilyRef cfRef = resolveColumnFamily(schemaName, tableName);
- PColumn column = cfRef.getFamily().getPColumnForColumnName(colName);
- return new ColumnRef(cfRef.getTableRef(), column.getPosition());
+ TableRef theTableRef = null;
+ PColumn theColumn = null;
+ PColumnFamily theColumnFamily = null;
+ if (schemaName != null) {
+ try {
+ // Try schemaName as the tableName and use tableName as column family name
+ theTableRef = resolveTable(null, schemaName);
+ theColumnFamily = theTableRef.getTable().getColumnFamily(tableName);
+ theColumn = theColumnFamily.getPColumnForColumnName(colName);
+ } catch (MetaDataEntityNotFoundException e2) {
+ }
+ }
+ if (theColumn == null) {
+ // Try using the tableName as a columnFamily reference instead
+ // and resolve column in each column family.
+ Iterator<TableRef> iterator = tables.iterator();
+ while (iterator.hasNext()) {
+ TableRef tableRef = iterator.next();
+ try {
+ PColumnFamily columnFamily = tableRef.getTable().getColumnFamily(tableName);
+ PColumn column = columnFamily.getPColumnForColumnName(colName);
+ if (theColumn != null) {
+ throw new AmbiguousColumnException(colName);
+ }
+ theTableRef = tableRef;
+ theColumnFamily = columnFamily;
+ theColumn = column;
+ } catch (MetaDataEntityNotFoundException e1) {
+ }
+ }
+ if (theColumn == null) {
+ throw new ColumnNotFoundException(colName);
+ }
+ }
+ ColumnFamilyRef cfRef = new ColumnFamilyRef(theTableRef, theColumnFamily);
+ return new ColumnRef(cfRef.getTableRef(), theColumn.getPosition());
}
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e0df4b2e/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
index 4ebca90..796dad0 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
@@ -153,7 +153,7 @@ public class TupleProjectionCompiler {
PTableType.PROJECTED, table.getIndexState(), table.getTimeStamp(), table.getSequenceNumber(),
table.getPKName(), table.getBucketNum(), projectedColumns, table.getParentSchemaName(),
table.getParentTableName(), table.getIndexes(), table.isImmutableRows(), Collections.<PName> emptyList(),
- null, null, table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(),
+ table.getDefaultFamilyName(), table.getViewStatement(), table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(),
table.getViewIndexId(),
table.getIndexType(), table.rowKeyOrderOptimizable(), table.isTransactional(), table.getUpdateCacheFrequency(),
table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter(), table.useStatsForParallelization());
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e0df4b2e/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
index f25f7f1..cfeb212 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
@@ -244,7 +244,6 @@ public enum SQLExceptionCode {
SET_UNSUPPORTED_PROP_ON_ALTER_TABLE(1025, "42Y83", "Unsupported property set in ALTER TABLE command."),
CANNOT_ADD_NOT_NULLABLE_COLUMN(1038, "42Y84", "Only nullable columns may be added for a pre-existing table."),
NO_MUTABLE_INDEXES(1026, "42Y85", "Mutable secondary indexes are only supported for HBase version " + MetaDataUtil.decodeHBaseVersionAsString(PhoenixDatabaseMetaData.MUTABLE_SI_VERSION_THRESHOLD) + " and above."),
- INVALID_FILTER_ON_IMMUTABLE_ROWS(1027, "42Y86", "All columns referenced in a WHERE clause must be available in every index for a table with immutable rows."),
INVALID_INDEX_STATE_TRANSITION(1028, "42Y87", "Invalid index state transition."),
INVALID_MUTABLE_INDEX_CONFIG(1029, "42Y88", "Mutable secondary indexes must have the "
+ IndexManagementUtil.WAL_EDIT_CODEC_CLASS_KEY + " property set to "
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e0df4b2e/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
index df069a6..9c26575 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/MutationState.java
@@ -189,7 +189,9 @@ public class MutationState implements SQLCloseable {
public MutationState(TableRef table, Map<ImmutableBytesPtr,RowMutationState> mutations, long sizeOffset, long maxSize, long maxSizeBytes, PhoenixConnection connection) throws SQLException {
this(maxSize, maxSizeBytes, connection, false, null, sizeOffset);
- this.mutations.put(table, mutations);
+ if (!mutations.isEmpty()) {
+ this.mutations.put(table, mutations);
+ }
this.numRows = mutations.size();
throwIfTooBig();
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e0df4b2e/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
index b4566a4..500ac4b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/index/IndexMaintainer.java
@@ -366,7 +366,6 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> {
private IndexMaintainer(final PTable dataTable, final PTable index, PhoenixConnection connection) {
this(dataTable.getRowKeySchema(), dataTable.getBucketNum() != null);
- assert(dataTable.getType() == PTableType.SYSTEM || dataTable.getType() == PTableType.TABLE || dataTable.getType() == PTableType.VIEW);
this.rowKeyOrderOptimizable = index.rowKeyOrderOptimizable();
this.isMultiTenant = dataTable.isMultiTenant();
this.viewIndexId = index.getViewIndexId() == null ? null : MetaDataUtil.getViewIndexIdDataType().toBytes(index.getViewIndexId());
@@ -411,15 +410,14 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> {
int nDataPKColumns = dataRowKeySchema.getFieldCount() - dataPosOffset;
// For indexes on views, we need to remember which data columns are "constants"
// These are the values in a VIEW where clause. For these, we don't put them in the
- // index, as they're the same for every row in the index.
- if (dataTable.getType() == PTableType.VIEW) {
- List<PColumn>dataPKColumns = dataTable.getPKColumns();
- for (int i = dataPosOffset; i < dataPKColumns.size(); i++) {
- PColumn dataPKColumn = dataPKColumns.get(i);
- if (dataPKColumn.getViewConstant() != null) {
- bitSet.set(i);
- nDataPKColumns--;
- }
+ // index, as they're the same for every row in the index. The data table can be
+ // either a VIEW or PROJECTED
+ List<PColumn>dataPKColumns = dataTable.getPKColumns();
+ for (int i = dataPosOffset; i < dataPKColumns.size(); i++) {
+ PColumn dataPKColumn = dataPKColumns.get(i);
+ if (dataPKColumn.getViewConstant() != null) {
+ bitSet.set(i);
+ nDataPKColumns--;
}
}
this.indexTableName = indexTableName;
@@ -543,11 +541,14 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> {
for (int i = 0; i < index.getColumnFamilies().size(); i++) {
PColumnFamily family = index.getColumnFamilies().get(i);
for (PColumn indexColumn : family.getColumns()) {
- PColumn dataColumn = IndexUtil.getDataColumn(dataTable, indexColumn.getName().getString());
- byte[] dataColumnCq = dataColumn.getColumnQualifierBytes();
- byte[] indexColumnCq = indexColumn.getColumnQualifierBytes();
- this.coveredColumnsMap.put(new ColumnReference(dataColumn.getFamilyName().getBytes(), dataColumnCq),
- new ColumnReference(indexColumn.getFamilyName().getBytes(), indexColumnCq));
+ PColumn dataColumn = IndexUtil.getDataColumnOrNull(dataTable, indexColumn.getName().getString());
+ // This can happen during deletion where we don't need covered columns
+ if (dataColumn != null) {
+ byte[] dataColumnCq = dataColumn.getColumnQualifierBytes();
+ byte[] indexColumnCq = indexColumn.getColumnQualifierBytes();
+ this.coveredColumnsMap.put(new ColumnReference(dataColumn.getFamilyName().getBytes(), dataColumnCq),
+ new ColumnReference(indexColumn.getFamilyName().getBytes(), indexColumnCq));
+ }
}
}
this.estimatedIndexRowKeyBytes = estimateIndexRowKeyByteSize(indexColByteSize);
@@ -758,8 +759,10 @@ public class IndexMaintainer implements Writable, Iterable<ColumnReference> {
int minLength = length - maxTrailingNulls;
byte[] dataRowKey = stream.getBuffer();
// Remove trailing nulls
- while (length > minLength && dataRowKey[length-1] == QueryConstants.SEPARATOR_BYTE) {
+ int index = dataRowKeySchema.getFieldCount() - 1;
+ while (index >= 0 && !dataRowKeySchema.getField(index).getDataType().isFixedWidth() && length > minLength && dataRowKey[length-1] == QueryConstants.SEPARATOR_BYTE) {
length--;
+ index--;
}
// TODO: need to capture nDataSaltBuckets instead of just a boolean. For now,
// we store this in nIndexSaltBuckets, as we only use this function for local indexes
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e0df4b2e/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java b/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
index ca7ff2c..b3df50b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/optimize/QueryOptimizer.java
@@ -429,7 +429,7 @@ public class QueryOptimizer {
});
- return bestCandidates;
+ return stopAtBestPlan ? bestCandidates.subList(0, 1) : bestCandidates;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e0df4b2e/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
index ae91d17..1cf61a2 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
@@ -295,6 +295,16 @@ public class PTableImpl implements PTable {
table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter(), table.useStatsForParallelization());
}
+ public static PTableImpl makePTable(PTable table, PTableType type, Collection<PColumn> columns) throws SQLException {
+ return new PTableImpl(
+ table.getTenantId(), table.getSchemaName(), table.getTableName(), type, table.getIndexState(), table.getTimeStamp(),
+ table.getSequenceNumber(), table.getPKName(), table.getBucketNum(), columns, table.getParentSchemaName(), table.getParentTableName(),
+ table.getIndexes(), table.isImmutableRows(), table.getPhysicalNames(), table.getDefaultFamilyName(), table.getViewStatement(),
+ table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexId(), table.getIndexType(),
+ table.getBaseColumnCount(), table.rowKeyOrderOptimizable(), table.isTransactional(), table.getUpdateCacheFrequency(),
+ table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter(), table.useStatsForParallelization());
+ }
+
public static PTableImpl makePTable(PTable table, Collection<PColumn> columns, PName defaultFamily) throws SQLException {
return new PTableImpl(
table.getTenantId(), table.getSchemaName(), table.getTableName(), table.getType(), table.getIndexState(), table.getTimeStamp(),
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e0df4b2e/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
index 1b6f9d5..b23ea1b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
@@ -207,27 +207,35 @@ public class IndexUtil {
}
public static PColumn getDataColumn(PTable dataTable, String indexColumnName) {
+ PColumn column = getDataColumnOrNull(dataTable, indexColumnName);
+ if (column == null) {
+ throw new IllegalArgumentException("Could not find column \"" + SchemaUtil.getColumnName(getDataColumnFamilyName(indexColumnName), getDataColumnName(indexColumnName)) + " in " + dataTable);
+ }
+ return column;
+ }
+
+ public static PColumn getDataColumnOrNull(PTable dataTable, String indexColumnName) {
int pos = indexColumnName.indexOf(INDEX_COLUMN_NAME_SEP);
if (pos < 0) {
- throw new IllegalArgumentException("Could not find expected '" + INDEX_COLUMN_NAME_SEP + "' separator in index column name of \"" + indexColumnName + "\"");
+ return null;
}
if (pos == 0) {
try {
return dataTable.getPKColumn(indexColumnName.substring(1));
} catch (ColumnNotFoundException e) {
- throw new IllegalArgumentException("Could not find PK column \"" + indexColumnName.substring(pos+1) + "\" in index column name of \"" + indexColumnName + "\"", e);
+ return null;
}
}
PColumnFamily family;
try {
family = dataTable.getColumnFamily(getDataColumnFamilyName(indexColumnName));
} catch (ColumnFamilyNotFoundException e) {
- throw new IllegalArgumentException("Could not find column family \"" + indexColumnName.substring(0, pos) + "\" in index column name of \"" + indexColumnName + "\"", e);
+ return null;
}
try {
return family.getPColumnForColumnName(indexColumnName.substring(pos+1));
} catch (ColumnNotFoundException e) {
- throw new IllegalArgumentException("Could not find column \"" + indexColumnName.substring(pos+1) + "\" in index column name of \"" + indexColumnName + "\"", e);
+ return null;
}
}
@@ -686,7 +694,7 @@ public class IndexUtil {
}
public static byte[][] getViewConstants(PTable dataTable) {
- if (dataTable.getType() != PTableType.VIEW) return null;
+ if (dataTable.getType() != PTableType.VIEW && dataTable.getType() != PTableType.PROJECTED) return null;
int dataPosOffset = (dataTable.getBucketNum() != null ? 1 : 0) + (dataTable.isMultiTenant() ? 1 : 0);
ImmutableBytesWritable ptr = new ImmutableBytesWritable();
List<byte[]> viewConstants = new ArrayList<byte[]>();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/e0df4b2e/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
index ca4be2f..b3c7dca 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
@@ -1235,33 +1235,6 @@ public class QueryCompilerTest extends BaseConnectionlessQueryTest {
}
@Test
- public void testDeleteFromImmutableWithKV() throws Exception {
- String ddl = "CREATE TABLE t (k1 VARCHAR, v1 VARCHAR, v2 VARCHAR CONSTRAINT pk PRIMARY KEY(k1)) immutable_rows=true";
- String indexDDL = "CREATE INDEX i ON t (v1)";
- Connection conn = DriverManager.getConnection(getUrl());
- try {
- conn.createStatement().execute(ddl);
- assertImmutableRows(conn, "T", true);
- conn.createStatement().execute(indexDDL);
- assertImmutableRows(conn, "I", true);
- conn.createStatement().execute("DELETE FROM t WHERE v2 = 'foo'");
- fail();
- } catch (SQLException e) {
- assertEquals(SQLExceptionCode.INVALID_FILTER_ON_IMMUTABLE_ROWS.getErrorCode(), e.getErrorCode());
- }
- // Test with one index having the referenced key value column, but one not having it.
- // Still should fail
- try {
- indexDDL = "CREATE INDEX i2 ON t (v2)";
- conn.createStatement().execute(indexDDL);
- conn.createStatement().execute("DELETE FROM t WHERE v2 = 'foo'");
- fail();
- } catch (SQLException e) {
- assertEquals(SQLExceptionCode.INVALID_FILTER_ON_IMMUTABLE_ROWS.getErrorCode(), e.getErrorCode());
- }
- }
-
- @Test
public void testInvalidNegativeArrayIndex() throws Exception {
String query = "SELECT a_double_array[-20] FROM table_with_array";
Connection conn = DriverManager.getConnection(getUrl());