You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ja...@apache.org on 2018/11/10 20:59:08 UTC
[05/22] phoenix git commit: PHOENIX-4891: An index should inherit
UPDATE_CACHE_FREQUENCY setting rom parent table
PHOENIX-4891: An index should inherit UPDATE_CACHE_FREQUENCY setting rom parent table
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/64788794
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/64788794
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/64788794
Branch: refs/heads/omid2
Commit: 647887947f631d624d6897d729bce3acb07a94bf
Parents: fe466eb
Author: Chinmay Kulkarni <ch...@gmail.com>
Authored: Tue Oct 30 14:52:26 2018 -0700
Committer: Thomas D'Silva <td...@apache.org>
Committed: Tue Oct 30 16:23:42 2018 -0700
----------------------------------------------------------------------
.../phoenix/end2end/PropertiesInSyncIT.java | 172 ++++++++++++++-----
.../phoenix/end2end/index/IndexMetadataIT.java | 145 +++++++++++++++-
.../org/apache/phoenix/rpc/UpdateCacheIT.java | 134 ++++++++++-----
.../phoenix/exception/SQLExceptionCode.java | 7 +-
.../query/ConnectionQueryServicesImpl.java | 6 +-
.../apache/phoenix/schema/MetaDataClient.java | 33 +++-
.../org/apache/phoenix/util/MetaDataUtil.java | 6 +-
.../org/apache/phoenix/util/UpgradeUtil.java | 79 ++++++++-
8 files changed, 476 insertions(+), 106 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/64788794/phoenix-core/src/it/java/org/apache/phoenix/end2end/PropertiesInSyncIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/PropertiesInSyncIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/PropertiesInSyncIT.java
index db44735..348b195 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/PropertiesInSyncIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/PropertiesInSyncIT.java
@@ -23,9 +23,12 @@ import org.apache.hadoop.hbase.KeepDeletedCells;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
import org.apache.phoenix.exception.SQLExceptionCode;
import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.schema.PName;
import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.util.MetaDataUtil;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.SchemaUtil;
@@ -33,17 +36,23 @@ import org.junit.Test;
import java.sql.Connection;
import java.sql.DriverManager;
+import java.sql.PreparedStatement;
import java.sql.SQLException;
+import java.util.HashMap;
import java.util.HashSet;
+import java.util.Map;
import java.util.Properties;
import java.util.Set;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import static org.apache.phoenix.query.QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES;
-import static org.apache.phoenix.util.MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES;
+import static org.apache.phoenix.util.MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES;
import static org.apache.phoenix.util.MetaDataUtil.VIEW_INDEX_TABLE_PREFIX;
+import static org.apache.phoenix.util.UpgradeUtil.UPSERT_UPDATE_CACHE_FREQUENCY;
import static org.apache.phoenix.util.UpgradeUtil.syncTableAndIndexProperties;
+import static org.apache.phoenix.util.UpgradeUtil.syncUpdateCacheFreqAllIndexes;
+import static org.apache.phoenix.end2end.index.IndexMetadataIT.assertUpdateCacheFreq;
/**
* Test properties that need to be kept in sync amongst all column families and indexes of a table
@@ -56,12 +65,16 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
private static final int INITIAL_TTL_VALUE = 700;
private static final KeepDeletedCells INITIAL_KEEP_DELETED_CELLS_VALUE = KeepDeletedCells.TRUE;
private static final int INITIAL_REPLICATION_SCOPE_VALUE = 1;
+ private static final int INITIAL_UPDATE_CACHE_FREQUENCY = 100;
+ private static final int INITIAL_UPDATE_CACHE_FREQUENCY_VIEWS = 900;
private static final int MODIFIED_TTL_VALUE = INITIAL_TTL_VALUE + 300;
private static final KeepDeletedCells MODIFIED_KEEP_DELETED_CELLS_VALUE =
- (INITIAL_KEEP_DELETED_CELLS_VALUE == KeepDeletedCells.TRUE)
- ? KeepDeletedCells.FALSE: KeepDeletedCells.TRUE;
+ (INITIAL_KEEP_DELETED_CELLS_VALUE == KeepDeletedCells.TRUE) ?
+ KeepDeletedCells.FALSE: KeepDeletedCells.TRUE;
private static final int MODIFIED_REPLICATION_SCOPE_VALUE =
(INITIAL_REPLICATION_SCOPE_VALUE == 1) ? 0 : 1;
+ private static final int MODIFIED_UPDATE_CACHE_FREQUENCY = INITIAL_UPDATE_CACHE_FREQUENCY + 300;
+ private static final int MODIFIED_UPDATE_CACHE_FREQUENCY_VIEWS = INITIAL_UPDATE_CACHE_FREQUENCY_VIEWS + 300;
// Test that we disallow specifying synced properties to be set per column family
@@ -70,7 +83,7 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
public void testDisallowSyncedPropsToBeSetColFamSpecificCreateTable() throws Exception {
Connection conn = DriverManager.getConnection(getUrl(), new Properties());
String tableName = generateUniqueName();
- for (String propName: SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES) {
+ for (String propName: SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES) {
try {
conn.createStatement().execute("create table " + tableName
+ " (id INTEGER not null primary key, "
@@ -107,7 +120,7 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
String viewName = "VIEW_" + tableName;
conn.createStatement().execute("create view " + viewName
+ " (new_col SMALLINT) as select * from " + tableName + " where id > 1");
- for (String propName: SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES) {
+ for (String propName: SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES) {
try {
conn.createStatement().execute("create local index " + localIndexName
+ " on " + tableName + "(name) "
@@ -148,8 +161,8 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
public void testSyncedPropsBaseTableCreateIndex() throws Exception {
Connection conn = DriverManager.getConnection(getUrl(), new Properties());
String tableName = createBaseTableWithProps(conn);
- createIndexTable(conn, tableName, PTable.IndexType.LOCAL);
- String globalIndexName = createIndexTable(conn, tableName, PTable.IndexType.GLOBAL);
+ createIndexTable(conn, tableName, PTable.IndexType.LOCAL).getSecond();
+ String globalIndexName = createIndexTable(conn, tableName, PTable.IndexType.GLOBAL).getSecond();
// We pass the base table as the physical HBase table since our check includes checking
// the local index column family too
@@ -164,7 +177,7 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
public void testSyncedPropsBaseTableCreateViewIndex() throws Exception {
Connection conn = DriverManager.getConnection(getUrl(), new Properties());
String tableName = createBaseTableWithProps(conn);
- String viewIndexName = createIndexTable(conn, tableName, null);
+ String viewIndexName = createIndexTable(conn, tableName, null).getSecond();
verifyHBaseColumnFamilyProperties(tableName, conn, false, false);
verifyHBaseColumnFamilyProperties(viewIndexName, conn, false, false);
@@ -179,7 +192,7 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
String tableName = createBaseTableWithProps(conn);
StringBuilder alterAllSyncedPropsString = new StringBuilder();
String modPropString = COL_FAM1 + ".%s=" + DUMMY_PROP_VALUE + ",";
- for (String propName: SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES) {
+ for (String propName: SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES) {
try {
conn.createStatement().execute("alter table " + tableName
+ " set " + COL_FAM1 + "." + propName + "=" + DUMMY_PROP_VALUE);
@@ -216,11 +229,11 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
Set<String> tablesToCheck = new HashSet<>();
tablesToCheck.add(tableName);
for (int i=0; i<2; i++) {
- tablesToCheck.add(createIndexTable(conn, tableName, PTable.IndexType.LOCAL));
- tablesToCheck.add(createIndexTable(conn, tableName, PTable.IndexType.GLOBAL));
+ tablesToCheck.add(createIndexTable(conn, tableName, PTable.IndexType.LOCAL).getSecond());
+ tablesToCheck.add(createIndexTable(conn, tableName, PTable.IndexType.GLOBAL).getSecond());
}
// Create a view and view index
- tablesToCheck.add(createIndexTable(conn, tableName, null));
+ tablesToCheck.add(createIndexTable(conn, tableName, null).getSecond());
// Now alter the base table's properties. This should get propagated to all index tables
conn.createStatement().execute("alter table " + tableName + " set TTL=" + MODIFIED_TTL_VALUE
@@ -232,8 +245,8 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
}
// Any indexes created henceforth should have the modified properties
- String newGlobalIndex = createIndexTable(conn, tableName, PTable.IndexType.GLOBAL);
- String newViewIndex = createIndexTable(conn, tableName, null);
+ String newGlobalIndex = createIndexTable(conn, tableName, PTable.IndexType.GLOBAL).getSecond();
+ String newViewIndex = createIndexTable(conn, tableName, null).getSecond();
verifyHBaseColumnFamilyProperties(newGlobalIndex, conn, true, false);
verifyHBaseColumnFamilyProperties(newViewIndex, conn, true, false);
conn.close();
@@ -245,9 +258,8 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
Connection conn = DriverManager.getConnection(getUrl(), new Properties());
String tableName = createBaseTableWithProps(conn);
- // Test that we are not allowed to set any property to be kept in sync, specific
- // to the new column family to be added
- for (String propName: SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES) {
+ // Test that we are not allowed to set any property to be kept in sync, specific to the new column family to be added
+ for (String propName: SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES) {
try {
conn.createStatement().execute(
"alter table " + tableName + " add " + NEW_CF + ".new_column varchar(2) "
@@ -268,11 +280,11 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
Set<String> tablesToCheck = new HashSet<>();
tablesToCheck.add(tableName);
for (int i=0; i<2; i++) {
- tablesToCheck.add(createIndexTable(conn, tableName, PTable.IndexType.LOCAL));
- tablesToCheck.add(createIndexTable(conn, tableName, PTable.IndexType.GLOBAL));
+ tablesToCheck.add(createIndexTable(conn, tableName, PTable.IndexType.LOCAL).getSecond());
+ tablesToCheck.add(createIndexTable(conn, tableName, PTable.IndexType.GLOBAL).getSecond());
}
// Create a view and view index
- tablesToCheck.add(createIndexTable(conn, tableName, null));
+ tablesToCheck.add(createIndexTable(conn, tableName, null).getSecond());
// Now add a new column family while simultaneously modifying properties to be kept in sync,
// as well as a property which does not need to be kept in sync. Properties to be kept
@@ -307,8 +319,8 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
public void testDisallowAlterGlobalIndexTable() throws Exception {
Connection conn = DriverManager.getConnection(getUrl(), new Properties());
String tableName = createBaseTableWithProps(conn);
- String globalIndexName = createIndexTable(conn, tableName, PTable.IndexType.GLOBAL);
- for (String propName: SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES) {
+ String globalIndexName = createIndexTable(conn, tableName, PTable.IndexType.GLOBAL).getSecond();
+ for (String propName: SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES) {
try {
conn.createStatement().execute("alter table " + globalIndexName + " set "
+ propName + "=" + DUMMY_PROP_VALUE);
@@ -336,12 +348,12 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
createdTables.add(baseTableName1);
// Create different indexes on the base table
for (int i=0; i<2; i++) {
- createdTables.add(createIndexTable(conn, baseTableName, PTable.IndexType.GLOBAL));
- createdTables.add(createIndexTable(conn, baseTableName, PTable.IndexType.LOCAL));
- createdTables.add(createIndexTable(conn, baseTableName, null));
- createdTables.add(createIndexTable(conn, baseTableName1, PTable.IndexType.GLOBAL));
- createdTables.add(createIndexTable(conn, baseTableName1, PTable.IndexType.LOCAL));
- createdTables.add(createIndexTable(conn, baseTableName1, null));
+ createdTables.add(createIndexTable(conn, baseTableName, PTable.IndexType.GLOBAL).getSecond());
+ createdTables.add(createIndexTable(conn, baseTableName, PTable.IndexType.LOCAL).getSecond());
+ createdTables.add(createIndexTable(conn, baseTableName, null).getSecond());
+ createdTables.add(createIndexTable(conn, baseTableName1, PTable.IndexType.GLOBAL).getSecond());
+ createdTables.add(createIndexTable(conn, baseTableName1, PTable.IndexType.LOCAL).getSecond());
+ createdTables.add(createIndexTable(conn, baseTableName1, null).getSecond());
}
for (String t: createdTables) {
verifyHBaseColumnFamilyProperties(t, conn, false, false);
@@ -394,6 +406,77 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
conn.close();
}
+ @Test
+ public void testOldClientSyncUpdateCacheFreqUpgradePath() throws Exception {
+ PTable base, index;
+ String baseTableName, viewName, viewName2;
+ Map<String, Set<String>> createdTablesAndViews = new HashMap<>();
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), new Properties())) {
+ baseTableName = createBaseTableWithProps(conn);
+ createdTablesAndViews.put(baseTableName, new HashSet<String>());
+ Set<String> indexes = createdTablesAndViews.get(baseTableName);
+ indexes.add(createIndexTable(conn, baseTableName, PTable.IndexType.GLOBAL).getSecond());
+ indexes.add(createIndexTable(conn, baseTableName, PTable.IndexType.LOCAL).getFirst());
+
+ viewName = createViewOnBaseTableOrView(conn, baseTableName);
+ createdTablesAndViews.put(viewName, new HashSet<String>());
+ indexes = createdTablesAndViews.get(viewName);
+ indexes.add(createIndexTable(conn, viewName, PTable.IndexType.GLOBAL).getSecond());
+
+ viewName2 = createViewOnBaseTableOrView(conn, viewName);
+ createdTablesAndViews.put(viewName2, new HashSet<String>());
+ indexes = createdTablesAndViews.get(viewName2);
+ indexes.add(createIndexTable(conn, viewName2, PTable.IndexType.LOCAL).getFirst());
+
+ // Intentionally make UPDATE_CACHE_FREQUENCY out of sync for indexes
+ PreparedStatement stmt = conn.prepareStatement(UPSERT_UPDATE_CACHE_FREQUENCY);
+ for (String tableOrViewName : createdTablesAndViews.keySet()) {
+ base = PhoenixRuntime.getTable(conn, tableOrViewName);
+ for (String indexTableName : createdTablesAndViews.get(tableOrViewName)) {
+ index = PhoenixRuntime.getTable(conn, indexTableName);
+ PName tenantId = index.getTenantId();
+ stmt.setString(1, tenantId == null ? null : tenantId.getString());
+ stmt.setString(2, index.getSchemaName().getString());
+ stmt.setString(3, index.getTableName().getString());
+ stmt.setLong(4, base.getType() == PTableType.TABLE ?
+ MODIFIED_UPDATE_CACHE_FREQUENCY : MODIFIED_UPDATE_CACHE_FREQUENCY_VIEWS);
+ stmt.addBatch();
+ }
+ }
+ stmt.executeBatch();
+ conn.commit();
+
+ // Clear the server-side cache so that we get the latest built PTables
+ conn.unwrap(PhoenixConnection.class).getQueryServices().clearCache();
+ // Verify that the modified values are reflected
+ for (String tableOrViewName : createdTablesAndViews.keySet()) {
+ assertUpdateCacheFreq(conn, tableOrViewName, baseTableName.equals(tableOrViewName) ?
+ INITIAL_UPDATE_CACHE_FREQUENCY : INITIAL_UPDATE_CACHE_FREQUENCY_VIEWS);
+ for (String indexName : createdTablesAndViews.get(tableOrViewName)) {
+ assertUpdateCacheFreq(conn, indexName, baseTableName.equals(tableOrViewName) ?
+ MODIFIED_UPDATE_CACHE_FREQUENCY : MODIFIED_UPDATE_CACHE_FREQUENCY_VIEWS);
+ }
+ }
+
+ PhoenixConnection upgradeConn = conn.unwrap(PhoenixConnection.class);
+ upgradeConn.setRunningUpgrade(true);
+ syncUpdateCacheFreqAllIndexes(upgradeConn,
+ PhoenixRuntime.getTableNoCache(conn, baseTableName));
+
+ conn.unwrap(PhoenixConnection.class).getQueryServices().clearCache();
+ // Verify that indexes have the synced values for UPDATE_CACHE_FREQUENCY
+ for (String tableOrViewName : createdTablesAndViews.keySet()) {
+ long expectedVal = baseTableName.equals(tableOrViewName) ?
+ INITIAL_UPDATE_CACHE_FREQUENCY : INITIAL_UPDATE_CACHE_FREQUENCY_VIEWS;
+ assertUpdateCacheFreq(conn, tableOrViewName, expectedVal);
+ for (String indexOnTableOrView : createdTablesAndViews.get(tableOrViewName)) {
+ assertUpdateCacheFreq(conn, indexOnTableOrView, expectedVal);
+ }
+ }
+ }
+ }
+
/**
* Helper method to modify the synced properties for a column family descriptor
* @param cfd The column family descriptor object
@@ -417,9 +500,9 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
conn.createStatement().execute("create table " + tableName
+ " (id INTEGER not null primary key, type varchar(5), "
+ COL_FAM1 + ".name varchar(10), " + COL_FAM2 + ".flag boolean) "
- + "TTL=" + INITIAL_TTL_VALUE + ",KEEP_DELETED_CELLS="
- + INITIAL_KEEP_DELETED_CELLS_VALUE
- + ",REPLICATION_SCOPE=" + INITIAL_REPLICATION_SCOPE_VALUE);
+ + "TTL=" + INITIAL_TTL_VALUE + ",KEEP_DELETED_CELLS=" + INITIAL_KEEP_DELETED_CELLS_VALUE
+ + ",REPLICATION_SCOPE=" + INITIAL_REPLICATION_SCOPE_VALUE
+ + ",UPDATE_CACHE_FREQUENCY=" + INITIAL_UPDATE_CACHE_FREQUENCY);
return tableName;
}
@@ -429,34 +512,43 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
* @param baseTableName Name of the HBase base table on which to create an index
* @param indexType LOCAL, GLOBAL or if we pass in null as the indexType,
* we create a view and an index on that view for the given base table
- * @return The physical HBase table corresponding to the index created
+ * @return A pair consisting of the index name and the name of the physical HBase table
+ * corresponding to the index created
* @throws SQLException
*/
- private String createIndexTable(Connection conn, String baseTableName,
+ private Pair<String,String> createIndexTable(Connection conn, String baseTableName,
PTable.IndexType indexType) throws SQLException {
// Create a view on top of the base table and then an index on that view
if (indexType == null) {
- String viewName = "VIEW_" + baseTableName + "_" + generateUniqueName();
+ String viewName = createViewOnBaseTableOrView(conn, baseTableName);
String viewIndexName = VIEW_INDEX_TABLE_PREFIX + baseTableName;
- conn.createStatement().execute("create view " + viewName
- + " (new_col SMALLINT) as select * from " + baseTableName + " where id > 1");
conn.createStatement().execute("create index view_index_" + generateUniqueName()
+ " on " + viewName + " (flag)");
- return viewIndexName;
+ return new Pair<>(viewIndexName, viewIndexName);
}
switch(indexType) {
case LOCAL:
String localIndexName = baseTableName + "_LOCAL_" + generateUniqueName();
conn.createStatement().execute(
"create local index " + localIndexName + " on " + baseTableName + "(flag)");
- return baseTableName;
+ return new Pair<>(localIndexName, baseTableName);
case GLOBAL:
String globalIndexName = baseTableName + "_GLOBAL_" + generateUniqueName();
conn.createStatement()
.execute("create index " + globalIndexName + " on " + baseTableName + "(name)");
- return globalIndexName;
+ return new Pair<>(globalIndexName, globalIndexName);
+ default:
+ return new Pair<>(baseTableName, baseTableName);
}
- return baseTableName;
+ }
+
+ private String createViewOnBaseTableOrView(Connection conn, String baseTableOrView) throws SQLException {
+ String viewName = "VIEW_" + baseTableOrView + "_" + generateUniqueName();
+ conn.createStatement().execute("create view " + viewName
+ + " (" + generateUniqueName() + " SMALLINT) as select * from "
+ + baseTableOrView + " where id > 1 UPDATE_CACHE_FREQUENCY="
+ + INITIAL_UPDATE_CACHE_FREQUENCY_VIEWS);
+ return viewName;
}
/**
http://git-wip-us.apache.org/repos/asf/phoenix/blob/64788794/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexMetadataIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexMetadataIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexMetadataIT.java
index 21fa312..f797b70 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexMetadataIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexMetadataIT.java
@@ -19,6 +19,8 @@ package org.apache.phoenix.end2end.index;
import static org.apache.phoenix.util.TestUtil.INDEX_DATA_SCHEMA;
import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.apache.phoenix.exception.SQLExceptionCode.CANNOT_SET_OR_ALTER_UPDATE_CACHE_FREQ_FOR_INDEX;
+import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_UPDATE_CACHE_FREQUENCY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -675,15 +677,12 @@ public class IndexMetadataIT extends ParallelStatsDisabledIT {
}
}
-
-
@Test
public void testIndexAlterPhoenixProperty() throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
Connection conn = DriverManager.getConnection(getUrl(), props);
String testTable = generateUniqueName();
-
String ddl = "create table " + testTable + " (k varchar primary key, v1 varchar)";
Statement stmt = conn.createStatement();
stmt.execute(ddl);
@@ -703,6 +702,123 @@ public class IndexMetadataIT extends ParallelStatsDisabledIT {
assertEquals(20,rs.getInt(1));
}
+ @Test
+ public void testCreateIndexSetUpdateCacheFreqFails() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String testTable = generateUniqueName();
+
+ String ddl = "CREATE TABLE " + testTable + " (k varchar primary key, v1 varchar)";
+ Statement stmt = conn.createStatement();
+ stmt.execute(ddl);
+ String indexName = "IDX_" + generateUniqueName();
+
+ ddl = "CREATE INDEX " + indexName + " ON " + testTable + " (v1) " +
+ "UPDATE_CACHE_FREQUENCY=10000";
+ try {
+ stmt.execute(ddl);
+ fail("Should fail trying to set UPDATE_CACHE_FREQUENCY when creating an index");
+ } catch (SQLException sqlE) {
+ assertEquals("Unexpected error occurred",
+ CANNOT_SET_OR_ALTER_UPDATE_CACHE_FREQ_FOR_INDEX.getErrorCode(), sqlE.getErrorCode());
+ }
+ }
+
+ @Test
+ public void testIndexGetsUpdateCacheFreqFromBaseTable() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String testTable = generateUniqueName();
+
+ long updateCacheFreq = 10000;
+ String ddl = "CREATE TABLE " + testTable + " (k varchar primary key, v1 varchar) " +
+ "UPDATE_CACHE_FREQUENCY=" + updateCacheFreq;
+ Statement stmt = conn.createStatement();
+ stmt.execute(ddl);
+
+ String localIndex = "LOCAL_" + generateUniqueName();
+ String globalIndex = "GLOBAL_" + generateUniqueName();
+
+ ddl = "CREATE LOCAL INDEX " + localIndex + " ON " + testTable + " (v1) ";
+ stmt.execute(ddl);
+ ddl = "CREATE INDEX " + globalIndex + " ON " + testTable + " (v1) ";
+ stmt.execute(ddl);
+
+ // Check that local and global index both have the propagated UPDATE_CACHE_FREQUENCY value
+ assertUpdateCacheFreq(conn, testTable, updateCacheFreq);
+ assertUpdateCacheFreq(conn, localIndex, updateCacheFreq);
+ assertUpdateCacheFreq(conn, globalIndex, updateCacheFreq);
+ }
+
+ @Test
+ public void testAlterTablePropagatesUpdateCacheFreqToIndexes() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String testTable = generateUniqueName();
+
+ String ddl = "CREATE TABLE " + testTable + " (k varchar primary key, v1 varchar) ";
+ Statement stmt = conn.createStatement();
+ stmt.execute(ddl);
+
+ String localIndex = "LOCAL_" + generateUniqueName();
+ String globalIndex = "GLOBAL_" + generateUniqueName();
+
+ ddl = "CREATE LOCAL INDEX " + localIndex + " ON " + testTable + " (v1) ";
+ stmt.execute(ddl);
+ ddl = "CREATE INDEX " + globalIndex + " ON " + testTable + " (v1) ";
+ stmt.execute(ddl);
+
+ assertUpdateCacheFreq(conn, testTable, DEFAULT_UPDATE_CACHE_FREQUENCY);
+ assertUpdateCacheFreq(conn, localIndex, DEFAULT_UPDATE_CACHE_FREQUENCY);
+ assertUpdateCacheFreq(conn, globalIndex, DEFAULT_UPDATE_CACHE_FREQUENCY);
+
+ // Alter UPDATE_CACHE_FREQUENCY on the base table
+ long updateCacheFreq = 10000;
+ ddl = "ALTER TABLE " + testTable + " SET UPDATE_CACHE_FREQUENCY=" + updateCacheFreq;
+ stmt.execute(ddl);
+
+ // Check that local and global index both have the propagated UPDATE_CACHE_FREQUENCY value
+ assertUpdateCacheFreq(conn, testTable, updateCacheFreq);
+ assertUpdateCacheFreq(conn, localIndex, updateCacheFreq);
+ assertUpdateCacheFreq(conn, globalIndex, updateCacheFreq);
+ }
+
+ @Test
+ public void testIndexAlterUpdateCacheFreqFails() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String testTable = generateUniqueName();
+
+ String ddl = "CREATE TABLE " + testTable + " (k varchar primary key, v1 varchar)";
+ Statement stmt = conn.createStatement();
+ stmt.execute(ddl);
+
+ String localIndex = "LOCAL_" + generateUniqueName();
+ String globalIndex = "GLOBAL_" + generateUniqueName();
+
+ ddl = "CREATE LOCAL INDEX " + localIndex + " ON " + testTable + " (v1) ";
+ stmt.execute(ddl);
+ ddl = "CREATE INDEX " + globalIndex + " ON " + testTable + " (v1) ";
+ stmt.execute(ddl);
+
+ try {
+ stmt.execute("ALTER INDEX " + localIndex + " ON " + testTable +
+ " ACTIVE SET UPDATE_CACHE_FREQUENCY=NEVER");
+ fail("Should fail trying to alter UPDATE_CACHE_FREQUENCY on index");
+ } catch (SQLException sqlE) {
+ assertEquals("Unexpected error occurred",
+ CANNOT_SET_OR_ALTER_UPDATE_CACHE_FREQ_FOR_INDEX.getErrorCode(), sqlE.getErrorCode());
+ }
+
+ try {
+ stmt.execute("ALTER INDEX " + globalIndex + " ON " + testTable +
+ " ACTIVE SET UPDATE_CACHE_FREQUENCY=NEVER");
+ fail("Should fail trying to alter UPDATE_CACHE_FREQUENCY on index");
+ } catch (SQLException sqlE) {
+ assertEquals("Unexpected error occurred",
+ CANNOT_SET_OR_ALTER_UPDATE_CACHE_FREQ_FOR_INDEX.getErrorCode(), sqlE.getErrorCode());
+ }
+ }
@Test
public void testIndexAlterHBaseProperty() throws Exception {
@@ -718,10 +834,10 @@ public class IndexMetadataIT extends ParallelStatsDisabledIT {
ddl = "CREATE INDEX " + indexName + " ON " + testTable + " (v1) ";
stmt.execute(ddl);
- conn.createStatement().execute("ALTER INDEX "+indexName+" ON " + testTable +" ACTIVE SET DISABLE_WAL=false");
- asssertIsWALDisabled(conn,indexName,false);
conn.createStatement().execute("ALTER INDEX "+indexName+" ON " + testTable +" ACTIVE SET DISABLE_WAL=true");
asssertIsWALDisabled(conn,indexName,true);
+ conn.createStatement().execute("ALTER INDEX "+indexName+" ON " + testTable +" ACTIVE SET DISABLE_WAL=false");
+ asssertIsWALDisabled(conn,indexName,false);
}
private static void asssertIsWALDisabled(Connection conn, String fullTableName, boolean expectedValue) throws SQLException {
@@ -729,4 +845,23 @@ public class IndexMetadataIT extends ParallelStatsDisabledIT {
assertEquals(expectedValue, pconn.getTable(new PTableKey(pconn.getTenantId(), fullTableName)).isWALDisabled());
}
+ /**
+ * Helper method to assert the value of UPDATE_CACHE_FREQUENCY for a table/index/view
+ * @param conn Phoenix connection
+ * @param name table/view/index name
+ * @param expectedUpdateCacheFreq expected value of UPDATE_CACHE_FREQUENCY
+ * @throws SQLException
+ */
+ public static void assertUpdateCacheFreq(Connection conn, String name,
+ long expectedUpdateCacheFreq) throws SQLException {
+ ResultSet rs = conn.createStatement().executeQuery(
+ "select UPDATE_CACHE_FREQUENCY from SYSTEM.\"CATALOG\" where TABLE_NAME='" +
+ name + "'");
+ assertTrue(rs.next());
+ assertEquals("Mismatch found for " + name, expectedUpdateCacheFreq, rs.getLong(1));
+ assertEquals("Mismatch in UPDATE_CACHE_FREQUENCY for PTable of " + name,
+ expectedUpdateCacheFreq, PhoenixRuntime.getTableNoCache(conn, name)
+ .getUpdateCacheFrequency());
+ }
+
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/64788794/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
index 0ddbed3..2959b99 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
@@ -19,6 +19,7 @@ package org.apache.phoenix.rpc;
import static org.apache.phoenix.util.TestUtil.INDEX_DATA_SCHEMA;
import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertFalse;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Matchers.eq;
@@ -73,7 +74,7 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT {
String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + tableName;
Connection conn = DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES));
conn.createStatement().execute("create table " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + "TRANSACTIONAL=true,TRANSACTION_PROVIDER='" + provider + "'");
- helpTestUpdateCache(fullTableName, new int[] {1, 1});
+ helpTestUpdateCache(fullTableName, new int[] {1, 1}, false);
}
}
}
@@ -84,14 +85,14 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT {
String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + tableName;
Connection conn = DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES));
conn.createStatement().execute("create table " + fullTableName + TestUtil.TEST_TABLE_SCHEMA);
- helpTestUpdateCache(fullTableName, new int[] {1, 3});
+ helpTestUpdateCache(fullTableName, new int[] {1, 3}, false);
}
@Test
public void testUpdateCacheForNonTxnSystemTable() throws Exception {
String fullTableName = "\""+ QueryConstants.SYSTEM_SCHEMA_NAME + "\""+ QueryConstants.NAME_SEPARATOR + generateUniqueName();
setupSystemTable(fullTableName);
- helpTestUpdateCache(fullTableName, new int[] {0, 0});
+ helpTestUpdateCache(fullTableName, new int[] {0, 0}, false);
}
@Test
@@ -104,7 +105,7 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT {
conn.createStatement().execute(
"alter table " + fullTableName + " SET UPDATE_CACHE_FREQUENCY=NEVER");
}
- helpTestUpdateCache(fullTableName, new int[] {0, 0});
+ helpTestUpdateCache(fullTableName, new int[] {0, 0}, false);
}
@Test
@@ -114,7 +115,7 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT {
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.createStatement().execute("CREATE TABLE " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + " UPDATE_CACHE_FREQUENCY=always");
}
- helpTestUpdateCache(fullTableName, new int[] {1, 3});
+ helpTestUpdateCache(fullTableName, new int[] {1, 3}, false);
}
@Test
@@ -124,9 +125,9 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT {
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.createStatement().execute("CREATE TABLE " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + " UPDATE_CACHE_FREQUENCY=" + 10000);
}
- helpTestUpdateCache(fullTableName, new int[] {0, 0});
+ helpTestUpdateCache(fullTableName, new int[] {0, 0}, false);
Thread.sleep(10000);
- helpTestUpdateCache(fullTableName, new int[] {1, 0});
+ helpTestUpdateCache(fullTableName, new int[] {1, 0}, false);
}
@Test
@@ -136,14 +137,55 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT {
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.createStatement().execute("CREATE TABLE " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + " UPDATE_CACHE_FREQUENCY=never");
}
- helpTestUpdateCache(fullTableName, new int[] {0, 0});
+ helpTestUpdateCache(fullTableName, new int[] {0, 0}, false);
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.createStatement().execute("ALTER TABLE " + fullTableName + " SET UPDATE_CACHE_FREQUENCY=ALWAYS");
}
- helpTestUpdateCache(fullTableName, new int[] {1, 3});
+ helpTestUpdateCache(fullTableName, new int[] {1, 3}, false);
+ }
+
+ @Test
+ public void testUpdateCacheFreqPropagatedToIndexes() throws Exception {
+ String baseTableName = generateUniqueName();
+ String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + baseTableName;
+ String localIndex = "LOCAL_" + baseTableName;
+ String globalIndex = "GLOBAL_" + baseTableName;
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ conn.createStatement().execute("CREATE TABLE " + fullTableName +
+ TestUtil.TEST_TABLE_SCHEMA + " UPDATE_CACHE_FREQUENCY=never");
+
+ // Create local and global indexes on the base table
+ conn.createStatement().execute("CREATE LOCAL INDEX " + localIndex
+ + " on " + fullTableName + " (a.date1, b.varchar_col2)");
+ conn.createStatement().execute("CREATE INDEX " + globalIndex + " on "
+ + fullTableName + " (a.int_col1, a.long_col1)");
+ }
+
+ // The indexes should have got the UPDATE_CACHE_FREQUENCY value of their base table
+ helpTestUpdateCache(fullTableName, new int[] {0, 0}, false);
+ helpTestUpdateCache(INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + localIndex,
+ new int[] {0}, true);
+ helpTestUpdateCache(INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + globalIndex,
+ new int[] {0}, true);
+
+ // Now alter the UPDATE_CACHE_FREQUENCY value of the base table
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ conn.createStatement()
+ .execute("ALTER TABLE " + fullTableName + " SET UPDATE_CACHE_FREQUENCY=ALWAYS");
+ }
+ // Even the indexes should now have the modified value of UPDATE_CACHE_FREQUENCY
+ // Note that when we query the base table, during query plan generation, we make 2 getTable
+ // requests (to retrieve the base table) for each index of the base table
+ helpTestUpdateCache(fullTableName, new int[] {1, 15}, false);
+ helpTestUpdateCache(INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + localIndex,
+ new int[] {3}, true);
+ helpTestUpdateCache(INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + globalIndex,
+ new int[] {3}, true);
}
- private static void helpTestUpdateCache(String fullTableName, int[] expectedRPCs) throws Exception {
+ private static void helpTestUpdateCache(String fullTableName, int[] expectedRPCs,
+ boolean skipUpsertForIndexes) throws Exception {
String tableName = SchemaUtil.getTableNameFromFullName(fullTableName);
String schemaName = SchemaUtil.getSchemaNameFromFullName(fullTableName);
String selectSql = "SELECT * FROM "+fullTableName;
@@ -154,46 +196,31 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT {
Connection conn = connectionQueryServices.connect(getUrl(), props);
try {
conn.setAutoCommit(false);
- String upsert = "UPSERT INTO " + fullTableName + "(varchar_pk, char_pk, int_pk, long_pk, decimal_pk, date_pk) VALUES(?, ?, ?, ?, ?, ?)";
- PreparedStatement stmt = conn.prepareStatement(upsert);
- // upsert three rows
- TestUtil.setRowKeyColumns(stmt, 1);
- stmt.execute();
- TestUtil.setRowKeyColumns(stmt, 2);
- stmt.execute();
- TestUtil.setRowKeyColumns(stmt, 3);
- stmt.execute();
- conn.commit();
- int numUpsertRpcs = expectedRPCs[0];
- // verify only 0 or 1 rpc to fetch table metadata,
- verify(connectionQueryServices, times(numUpsertRpcs)).getTable((PName) isNull(),
- eq(PVarchar.INSTANCE.toBytes(schemaName)), eq(PVarchar.INSTANCE.toBytes(tableName)),
- anyLong(), anyLong(), eq(false), eq(false), (PTable)isNull());
- reset(connectionQueryServices);
-
- ResultSet rs = conn.createStatement().executeQuery(selectSql);
- TestUtil.validateRowKeyColumns(rs, 1);
- TestUtil.validateRowKeyColumns(rs, 2);
- TestUtil.validateRowKeyColumns(rs, 3);
- assertFalse(rs.next());
-
- rs = conn.createStatement().executeQuery(selectSql);
- TestUtil.validateRowKeyColumns(rs, 1);
- TestUtil.validateRowKeyColumns(rs, 2);
- TestUtil.validateRowKeyColumns(rs, 3);
- assertFalse(rs.next());
-
- rs = conn.createStatement().executeQuery(selectSql);
- TestUtil.validateRowKeyColumns(rs, 1);
- TestUtil.validateRowKeyColumns(rs, 2);
- TestUtil.validateRowKeyColumns(rs, 3);
- assertFalse(rs.next());
-
+ if (!skipUpsertForIndexes) {
+ String upsert = "UPSERT INTO " + fullTableName + "(varchar_pk, char_pk, int_pk, long_pk, decimal_pk, date_pk) VALUES(?, ?, ?, ?, ?, ?)";
+ PreparedStatement stmt = conn.prepareStatement(upsert);
+ // upsert three rows
+ for (int i=0; i<3; i++) {
+ TestUtil.setRowKeyColumns(stmt, i);
+ stmt.execute();
+ }
+ conn.commit();
+ int numUpsertRpcs = expectedRPCs[0];
+ // verify only 0 or 1 rpc to fetch table metadata,
+ verify(connectionQueryServices, times(numUpsertRpcs)).getTable((PName) isNull(),
+ eq(PVarchar.INSTANCE.toBytes(schemaName)), eq(PVarchar.INSTANCE.toBytes(tableName)),
+ anyLong(), anyLong(), eq(false), eq(false), (PTable)isNull());
+ reset(connectionQueryServices);
+ }
+ validateSelectRowKeyCols(conn, selectSql, skipUpsertForIndexes);
+ validateSelectRowKeyCols(conn, selectSql, skipUpsertForIndexes);
+ validateSelectRowKeyCols(conn, selectSql, skipUpsertForIndexes);
+
// for non-transactional tables without a scn : verify one rpc to getTable occurs *per* query
// for non-transactional tables with a scn : verify *only* one rpc occurs
// for transactional tables : verify *only* one rpc occurs
// for non-transactional, system tables : verify no rpc occurs
- int numRpcs = expectedRPCs[1];
+ int numRpcs = skipUpsertForIndexes ? expectedRPCs[0] : expectedRPCs[1];
verify(connectionQueryServices, times(numRpcs)).getTable((PName) isNull(),
eq(PVarchar.INSTANCE.toBytes(schemaName)), eq(PVarchar.INSTANCE.toBytes(tableName)),
anyLong(), anyLong(), eq(false), eq(false), (PTable)isNull());
@@ -202,4 +229,19 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT {
conn.close();
}
}
+
+ private static void validateSelectRowKeyCols(Connection conn, String selectSql,
+ boolean skipUpsertForIndexes) throws SQLException {
+ ResultSet rs = conn.createStatement().executeQuery(selectSql);
+ if (skipUpsertForIndexes) {
+ for (int i=0; i<3; i++) {
+ assertTrue(rs.next());
+ }
+ } else {
+ for (int i=0; i<3; i++) {
+ TestUtil.validateRowKeyColumns(rs, i);
+ }
+ }
+ assertFalse(rs.next());
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/64788794/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
index 6696521..5bffed5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
@@ -23,6 +23,7 @@ import java.util.Map;
import org.apache.phoenix.coprocessor.MetaDataProtocol;
import org.apache.phoenix.hbase.index.util.IndexManagementUtil;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.schema.AmbiguousColumnException;
@@ -263,7 +264,7 @@ public enum SQLExceptionCode {
NO_LOCAL_INDEX_ON_TABLE_WITH_IMMUTABLE_ROWS(1048,"43A05","Local indexes aren't allowed on tables with immutable rows."),
COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY(1049, "43A06", "Column family not allowed for table properties."),
COLUMN_FAMILY_NOT_ALLOWED_FOR_PROPERTY(1050, "43A07", "Setting or altering any of the following properties: "
- + MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES.toString()
+ + MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES.toString()
+ " for a column family is not supported since they must be kept in sync. You can only set these properties for the entire table."),
CANNOT_ALTER_PROPERTY(1051, "43A08", "Property can be specified or changed only when creating a table."),
CANNOT_SET_PROPERTY_FOR_COLUMN_NOT_ADDED(1052, "43A09", "Property cannot be specified for a column family that is not being added or modified."),
@@ -308,7 +309,9 @@ public enum SQLExceptionCode {
TTL_UNSUPPORTED_FOR_TXN_TABLE(10947, "44A28", "TTL is not supported for"),
CANNOT_CREATE_LOCAL_INDEX_FOR_TXN_TABLE(10948, "44A29", "Local indexes cannot be created for"),
CANNOT_SET_OR_ALTER_PROPERTY_FOR_INDEX(10949, "44A30", "Cannot set or alter the following properties on an index: "
- + MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES.toString()),
+ + MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES.toString()),
+ CANNOT_SET_OR_ALTER_UPDATE_CACHE_FREQ_FOR_INDEX(10950, "44A31", "Cannot set or alter "
+ + PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY + " on an index"),
/** Sequence related */
SEQUENCE_ALREADY_EXIST(1200, "42Z00", "Sequence already exists.", new Factory() {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/64788794/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index ebcb7b9..1381247 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -2519,7 +2519,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
* Keep the TTL, KEEP_DELETED_CELLS and REPLICATION_SCOPE properties of new column families
* in sync with the existing column families. Note that we use the new values for these properties in case they
* are passed from our alter table command, if not, we use the default column family's value for each property
- * See {@link MetaDataUtil#SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES}
+ * See {@link MetaDataUtil#SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES}
* @param allFamiliesProps Map of all column family properties
* @param table original table
* @param tableDesc new table descriptor
@@ -2565,7 +2565,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
/**
* Set the new values for properties that are to be kept in sync amongst those column families of the table which are
* not referenced in the context of our alter table command, including the local index column family if it exists
- * See {@link MetaDataUtil#SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES}
+ * See {@link MetaDataUtil#SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES}
* @param tableDesc original table descriptor
* @param allFamiliesProps Map of all column family properties
* @param newTTL new value of TTL
@@ -2587,7 +2587,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
/**
* Set properties to be kept in sync for global indexes of a table, as well as
* the physical table corresponding to indexes created on views of a table
- * See {@link MetaDataUtil#SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES} and
+ * See {@link MetaDataUtil#SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES} and
* @param table base table
* @param tableAndIndexDescriptorMappings old to new table descriptor mappings
* @param applyPropsToAllIndexesDefaultCF new properties to apply to all index column families
http://git-wip-us.apache.org/repos/asf/phoenix/blob/64788794/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index 6563c40..df0e5d4 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -272,6 +272,7 @@ public class MetaDataClient {
TABLE_NAME + "," +
SYNC_INDEX_CREATED_DATE + " " + PDate.INSTANCE.getSqlTypeName() +
") VALUES (?, ?, ?, ?)";
+
private static final String CREATE_TABLE =
"UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"( " +
TENANT_ID + "," +
@@ -1147,7 +1148,7 @@ public class MetaDataClient {
/**
* Populate properties for the table and common properties for all column families of the table
* @param statementProps Properties specified in SQL statement
- * @param tableProps Properties for an HTableDescriptor
+ * @param tableProps Properties for an HTableDescriptor and Phoenix Table Properties
* @param commonFamilyProps Properties common to all column families
* @param tableType Used to distinguish between index creation vs. base table creation paths
* @throws SQLException
@@ -1164,9 +1165,17 @@ public class MetaDataClient {
.setMessage("Property: " + prop.getFirst()).build()
.buildException();
}
+ // HTableDescriptor property or Phoenix Table Property
if (defaultDescriptor.getValue(Bytes.toBytes(prop.getFirst())) == null) {
+ // See PHOENIX-4891
+ if (tableType == PTableType.INDEX && UPDATE_CACHE_FREQUENCY.equals(prop.getFirst())) {
+ throw new SQLExceptionInfo.Builder(
+ SQLExceptionCode.CANNOT_SET_OR_ALTER_UPDATE_CACHE_FREQ_FOR_INDEX)
+ .build()
+ .buildException();
+ }
tableProps.put(prop.getFirst(), prop.getSecond());
- } else {
+ } else { // HColumnDescriptor property
commonFamilyProps.put(prop.getFirst(), prop.getSecond());
}
}
@@ -2127,8 +2136,11 @@ public class MetaDataClient {
}
long updateCacheFrequency = connection.getQueryServices().getProps().getLong(
QueryServices.DEFAULT_UPDATE_CACHE_FREQUENCY_ATRRIB, QueryServicesOptions.DEFAULT_UPDATE_CACHE_FREQUENCY);
+ if (tableType == PTableType.INDEX && parent != null) {
+ updateCacheFrequency = parent.getUpdateCacheFrequency();
+ }
Long updateCacheFrequencyProp = (Long) TableProperty.UPDATE_CACHE_FREQUENCY.getValue(tableProps);
- if (updateCacheFrequencyProp != null) {
+ if (tableType != PTableType.INDEX && updateCacheFrequencyProp != null) {
updateCacheFrequency = updateCacheFrequencyProp;
}
String autoPartitionSeq = (String) TableProperty.AUTO_PARTITION_SEQ.getValue(tableProps);
@@ -3613,9 +3625,13 @@ public class MetaDataClient {
}
}
- if (!table.getIndexes().isEmpty() && (numPkColumnsAdded>0 || metaProperties.getNonTxToTx())) {
+ if (!table.getIndexes().isEmpty() &&
+ (numPkColumnsAdded>0 || metaProperties.getNonTxToTx() ||
+ metaPropertiesEvaluated.getUpdateCacheFrequency() != null)) {
for (PTable index : table.getIndexes()) {
- incrementTableSeqNum(index, index.getType(), numPkColumnsAdded, metaProperties.getNonTxToTx() ? Boolean.TRUE : null, metaPropertiesEvaluated.getUpdateCacheFrequency());
+ incrementTableSeqNum(index, index.getType(), numPkColumnsAdded,
+ metaProperties.getNonTxToTx() ? Boolean.TRUE : null,
+ metaPropertiesEvaluated.getUpdateCacheFrequency());
}
tableMetaData.addAll(connection.getMutationState().toMutations(timeStamp).next().getSecond());
connection.rollback();
@@ -4573,6 +4589,13 @@ public class MetaDataClient {
}
if (metaProperties.getUpdateCacheFrequencyProp() != null) {
+ // See PHOENIX-4891
+ if (table.getType() == PTableType.INDEX) {
+ throw new SQLExceptionInfo.Builder(
+ SQLExceptionCode.CANNOT_SET_OR_ALTER_UPDATE_CACHE_FREQ_FOR_INDEX)
+ .build()
+ .buildException();
+ }
if (metaProperties.getUpdateCacheFrequencyProp().longValue() != table.getUpdateCacheFrequency()) {
metaPropertiesEvaluated.setUpdateCacheFrequency(metaProperties.getUpdateCacheFrequencyProp());
changingPhoenixTableProperty = true;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/64788794/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
index df7530c..8cf514f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
@@ -109,7 +109,7 @@ public class MetaDataUtil {
public static final byte[] DATA_TABLE_NAME_PROP_BYTES = Bytes.toBytes(DATA_TABLE_NAME_PROP_NAME);
// See PHOENIX-3955
- public static final List<String> SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES = ImmutableList.of(
+ public static final List<String> SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES = ImmutableList.of(
HColumnDescriptor.TTL,
HColumnDescriptor.KEEP_DELETED_CELLS,
HColumnDescriptor.REPLICATION_SCOPE);
@@ -725,13 +725,13 @@ public class MetaDataUtil {
}
public static boolean propertyNotAllowedToBeOutOfSync(String colFamProp) {
- return SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES.contains(colFamProp);
+ return SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES.contains(colFamProp);
}
public static Map<String, Object> getSyncedProps(HColumnDescriptor defaultCFDesc) {
Map<String, Object> syncedProps = new HashMap<>();
if (defaultCFDesc != null) {
- for (String propToKeepInSync: SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES) {
+ for (String propToKeepInSync: SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES) {
syncedProps.put(propToKeepInSync, Bytes.toString(
defaultCFDesc.getValue(Bytes.toBytes(propToKeepInSync))));
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/64788794/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
index ede14b8..8d0aca1 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
@@ -46,6 +46,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SEQ_NUM;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_TYPE;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID;
import static org.apache.phoenix.query.QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT;
import static org.apache.phoenix.query.QueryConstants.DIVERGED_VIEW_BASE_COLUMN_COUNT;
@@ -140,6 +141,14 @@ public class UpgradeUtil {
+ "(TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY, BASE_COLUMN_COUNT) "
+ "VALUES (?, ?, ?, ?, ?, ?) ";
+ public static final String UPSERT_UPDATE_CACHE_FREQUENCY =
+ "UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"( " +
+ TENANT_ID + "," +
+ TABLE_SCHEM + "," +
+ TABLE_NAME + "," +
+ UPDATE_CACHE_FREQUENCY +
+ ") VALUES (?, ?, ?, ?)";
+
public static String SELECT_BASE_COLUMN_COUNT_FROM_HEADER_ROW = "SELECT "
+ "BASE_COLUMN_COUNT "
+ "FROM \"SYSTEM\".CATALOG "
@@ -1227,7 +1236,7 @@ public class UpgradeUtil {
for (HColumnDescriptor currentColFam: tableDesc.getColumnFamilies()) {
if (!currentColFam.equals(defaultColFam)) {
HColumnDescriptor newColFamDesc = new HColumnDescriptor(currentColFam);
- for (String prop: MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES) {
+ for (String prop: MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES) {
String existingPropVal = Bytes.toString(currentColFam.getValue(Bytes.toBytes(prop)));
String expectedPropVal = syncedProps.get(prop).toString();
if (existingPropVal == null || !existingPropVal.toLowerCase().equals(expectedPropVal.toLowerCase())) {
@@ -1299,6 +1308,71 @@ public class UpgradeUtil {
}
}
+ private static void syncUpdateCacheFreqForIndexesOfTable(PTable baseTable,
+ PreparedStatement stmt) throws SQLException {
+ for (PTable index : baseTable.getIndexes()) {
+ if (index.getUpdateCacheFrequency() == baseTable.getUpdateCacheFrequency()) {
+ continue;
+ }
+ stmt.setString(2, index.getSchemaName().getString());
+ stmt.setString(3, index.getTableName().getString());
+ stmt.setLong(4, baseTable.getUpdateCacheFrequency());
+ stmt.addBatch();
+ }
+ }
+
+ /**
+ * See PHOENIX-4891. We set the UPDATE_CACHE_FREQUENCY of indexes to be same as their parent.
+ * We do this for both physical base tables as well as views
+ * @param conn Phoenix Connection object
+ * @param table PTable corresponding to a physical base table
+ * @throws SQLException
+ * @throws IOException
+ */
+ public static void syncUpdateCacheFreqAllIndexes(PhoenixConnection conn, PTable table)
+ throws SQLException, IOException {
+ // Use own connection with max time stamp to be able to read all data from SYSTEM.CATALOG
+ try(PhoenixConnection newConn = new PhoenixConnection(conn, HConstants.LATEST_TIMESTAMP)) {
+ // Clear the server-side cache so that we get the latest built PTables
+ newConn.unwrap(PhoenixConnection.class).getQueryServices().clearCache();
+ byte[] tenantId = newConn.getTenantId() != null ?
+ newConn.getTenantId().getBytes() : null;
+
+ PreparedStatement stmt =
+ newConn.prepareStatement(UPSERT_UPDATE_CACHE_FREQUENCY);
+ stmt.setString(1, Bytes.toString(tenantId));
+ syncUpdateCacheFreqForIndexesOfTable(table, stmt);
+
+ TableViewFinderResult childViewsResult = new TableViewFinderResult();
+ try (Table childLinkTable = newConn.getQueryServices()
+ .getTable(SchemaUtil.getPhysicalName(
+ PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_NAME_BYTES,
+ newConn.getQueryServices().getProps())
+ .getName())) {
+ ViewFinder.findAllRelatives(childLinkTable, tenantId,
+ table.getSchemaName().getBytes(), table.getTableName().getBytes(),
+ LinkType.CHILD_TABLE, childViewsResult);
+
+ // Iterate over the chain of child views
+ for (TableInfo tableInfo: childViewsResult.getLinks()) {
+ PTable view;
+ String viewName = SchemaUtil.getTableName(tableInfo.getSchemaName(),
+ tableInfo.getTableName());
+ try {
+ view = PhoenixRuntime.getTable(newConn, viewName);
+ } catch (TableNotFoundException e) {
+ // Ignore
+ logger.warn("Error getting PTable for view: " + viewName);
+ continue;
+ }
+ syncUpdateCacheFreqForIndexesOfTable(view, stmt);
+ }
+ }
+ stmt.executeBatch();
+ newConn.commit();
+ }
+ }
+
/**
* Make sure that all tables have necessary column family properties in sync
* with each other and also in sync with all the table's indexes
@@ -1316,7 +1390,7 @@ public class UpgradeUtil {
// Ignore physical view index tables since we handle them for each base table already
continue;
}
- PTable table = null;
+ PTable table;
String tableName = origTableDesc.getTableName().getNameAsString();
try {
table = PhoenixRuntime.getTable(conn, tableName);
@@ -1329,6 +1403,7 @@ public class UpgradeUtil {
// Ignore global index tables since we handle them for each base table already
continue;
}
+ syncUpdateCacheFreqAllIndexes(conn, table);
HColumnDescriptor defaultColFam = origTableDesc.getFamily(SchemaUtil.getEmptyColumnFamily(table));
Map<String, Object> syncedProps = MetaDataUtil.getSyncedProps(defaultColFam);