You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ja...@apache.org on 2018/04/13 17:31:05 UTC
[1/2] phoenix git commit: PHOENIX-4579 Add a config to conditionally
create Phoenix meta tables on first client connection (Chinmay Kulkarni)
Repository: phoenix
Updated Branches:
refs/heads/master 7b2c7e135 -> 87fdda8b1
http://git-wip-us.apache.org/repos/asf/phoenix/blob/87fdda8b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 914f62a..8b328d3 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -170,6 +170,7 @@ import org.apache.phoenix.exception.RetriableUpgradeException;
import org.apache.phoenix.exception.SQLExceptionCode;
import org.apache.phoenix.exception.SQLExceptionInfo;
import org.apache.phoenix.exception.UpgradeInProgressException;
+import org.apache.phoenix.exception.UpgradeRequiredException;
import org.apache.phoenix.exception.UpgradeNotRequiredException;
import org.apache.phoenix.execute.MutationState;
import org.apache.phoenix.hbase.index.IndexRegionSplitPolicy;
@@ -1041,23 +1042,69 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
/**
*
- * @param tableName
+ * @param physicalTableName
+ * @param tableType
+ * @param props
+ * @param families
* @param splits
- * @param modifyExistingMetaData TODO
+ * @param modifyExistingMetaData
+ * @param isNamespaceMapped
+ * @param isDoNotUpgradePropSet
* @return true if table was created and false if it already exists
* @throws SQLException
*/
private HTableDescriptor ensureTableCreated(byte[] physicalTableName, PTableType tableType, Map<String, Object> props,
List<Pair<byte[], Map<String, Object>>> families, byte[][] splits, boolean modifyExistingMetaData,
- boolean isNamespaceMapped) throws SQLException {
+ boolean isNamespaceMapped, boolean isDoNotUpgradePropSet) throws SQLException {
SQLException sqlE = null;
HTableDescriptor existingDesc = null;
boolean isMetaTable = SchemaUtil.isMetaTable(physicalTableName);
boolean tableExist = true;
try (HBaseAdmin admin = getAdmin()) {
final String quorum = ZKConfig.getZKQuorumServersString(config);
- final String znode = this.props.get(HConstants.ZOOKEEPER_ZNODE_PARENT);
+ final String znode = this.getProps().get(HConstants.ZOOKEEPER_ZNODE_PARENT);
logger.debug("Found quorum: " + quorum + ":" + znode);
+
+ if (isMetaTable) {
+ if(SchemaUtil.isNamespaceMappingEnabled(PTableType.SYSTEM, this.getProps())) {
+ try {
+ // SYSTEM namespace needs to be created via HBase APIs because "CREATE SCHEMA" statement tries to write
+ // its metadata in SYSTEM:CATALOG table. Without SYSTEM namespace, SYSTEM:CATALOG table cannot be created
+ ensureNamespaceCreated(QueryConstants.SYSTEM_SCHEMA_NAME);
+ } catch (PhoenixIOException e) {
+ // We could either:
+ // 1) Not access the NS descriptor. The NS may or may not exist at this point
+ // 2) We could not create the NS
+ // Regardless of the case 1 or 2, if we eventually try to migrate SYSTEM tables to the SYSTEM
+ // namespace using the {@link ensureSystemTablesMigratedToSystemNamespace(ReadOnlyProps)} method,
+ // if the NS does not exist, we will error as expected, or
+ // if the NS does exist and tables are already mapped, the check will exit gracefully
+ }
+ if (admin.tableExists(SchemaUtil.getPhysicalTableName(SYSTEM_CATALOG_NAME_BYTES, false))) {
+ // SYSTEM.CATALOG exists, so at this point, we have 3 cases:
+ // 1) If server-side namespace mapping is disabled, throw Inconsistent namespace mapping exception
+ // 2) If server-side namespace mapping is enabled and SYSCAT needs to be upgraded, upgrade SYSCAT
+ // and also migrate SYSTEM tables to the SYSTEM namespace
+ // 3. If server-side namespace mapping is enabled and SYSCAT doesn't need to be upgraded, we still
+ // need to migrate SYSTEM tables to the SYSTEM namespace using the
+ // {@link ensureSystemTablesMigratedToSystemNamespace(ReadOnlyProps)} method (as part of
+ // {@link upgradeSystemTables(String, Properties)})
+ checkClientServerCompatibility(SYSTEM_CATALOG_NAME_BYTES);
+ // Thrown so we can force an upgrade which will just migrate SYSTEM tables to the SYSTEM namespace
+ throw new UpgradeRequiredException(MIN_SYSTEM_TABLE_TIMESTAMP);
+ }
+ } else if (admin.tableExists(SchemaUtil.getPhysicalTableName(SYSTEM_CATALOG_NAME_BYTES, true))) {
+ // If SYSTEM:CATALOG exists, but client-side namespace mapping for SYSTEM tables is disabled, throw an exception
+ throw new SQLExceptionInfo.Builder(
+ SQLExceptionCode.INCONSISTENT_NAMESPACE_MAPPING_PROPERTIES)
+ .setMessage("Cannot initiate connection as "
+ + SchemaUtil.getPhysicalTableName(SYSTEM_CATALOG_NAME_BYTES, true)
+ + " is found but client does not have "
+ + IS_NAMESPACE_MAPPING_ENABLED + " enabled")
+ .build().buildException();
+ }
+ }
+
try {
existingDesc = admin.getTableDescriptor(physicalTableName);
} catch (org.apache.hadoop.hbase.TableNotFoundException e) {
@@ -1075,6 +1122,10 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
splits, isNamespaceMapped);
if (!tableExist) {
+ if (isMetaTable && !isUpgradeRequired() && (!isAutoUpgradeEnabled || isDoNotUpgradePropSet)) {
+ // Disallow creating the SYSTEM.CATALOG or SYSTEM:CATALOG HBase table
+ throw new UpgradeRequiredException();
+ }
if (newDesc.getValue(MetaDataUtil.IS_LOCAL_INDEX_TABLE_PROP_BYTES) != null && Boolean.TRUE.equals(
PBoolean.INSTANCE.toObject(newDesc.getValue(MetaDataUtil.IS_LOCAL_INDEX_TABLE_PROP_BYTES)))) {
newDesc.setValue(HTableDescriptor.SPLIT_POLICY, IndexRegionSplitPolicy.class.getName());
@@ -1092,9 +1143,12 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
} catch (TableExistsException e) {
// We can ignore this, as it just means that another client beat us
// to creating the HBase metadata.
+ if (isMetaTable && !isUpgradeRequired()) {
+ checkClientServerCompatibility(SchemaUtil.getPhysicalName(SYSTEM_CATALOG_NAME_BYTES, this.getProps()).getName());
+ }
return null;
}
- if (isMetaTable) {
+ if (isMetaTable && !isUpgradeRequired()) {
checkClientServerCompatibility(SchemaUtil.getPhysicalName(SYSTEM_CATALOG_NAME_BYTES, this.getProps()).getName());
/*
* Now we modify the table to add the split policy, since we know that the client and
@@ -1106,7 +1160,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
return null;
} else {
- if (isMetaTable) {
+ if (isMetaTable && !isUpgradeRequired()) {
checkClientServerCompatibility(SchemaUtil.getPhysicalName(SYSTEM_CATALOG_NAME_BYTES, this.getProps()).getName());
} else {
for(Pair<byte[],Map<String,Object>> family: families) {
@@ -1120,7 +1174,6 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
}
-
if (!modifyExistingMetaData) {
return existingDesc; // Caller already knows that no metadata was changed
}
@@ -1143,7 +1196,10 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
return null; // Indicate that no metadata was changed
}
- modifyTable(physicalTableName, newDesc, true);
+ // Do not call modifyTable for SYSTEM tables
+ if (tableType != PTableType.SYSTEM) {
+ modifyTable(physicalTableName, newDesc, true);
+ }
return newDesc;
}
@@ -1198,6 +1254,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
boolean isIncompatible = false;
int minHBaseVersion = Integer.MAX_VALUE;
boolean isTableNamespaceMappingEnabled = false;
+ long systemCatalogTimestamp = Long.MAX_VALUE;
HTableInterface ht = null;
try {
List<HRegionLocation> locations = this
@@ -1214,36 +1271,44 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
ht = this.getTable(metaTable);
- final Map<byte[], Long> results =
- ht.coprocessorService(MetaDataService.class, null, null, new Batch.Call<MetaDataService,Long>() {
+ final Map<byte[], GetVersionResponse> results =
+ ht.coprocessorService(MetaDataService.class, null, null, new Batch.Call<MetaDataService,GetVersionResponse>() {
@Override
- public Long call(MetaDataService instance) throws IOException {
+ public GetVersionResponse call(MetaDataService instance) throws IOException {
ServerRpcController controller = new ServerRpcController();
BlockingRpcCallback<GetVersionResponse> rpcCallback =
- new BlockingRpcCallback<GetVersionResponse>();
+ new BlockingRpcCallback<>();
GetVersionRequest.Builder builder = GetVersionRequest.newBuilder();
builder.setClientVersion(VersionUtil.encodeVersion(PHOENIX_MAJOR_VERSION, PHOENIX_MINOR_VERSION, PHOENIX_PATCH_NUMBER));
instance.getVersion(controller, builder.build(), rpcCallback);
if(controller.getFailedOn() != null) {
throw controller.getFailedOn();
}
- return rpcCallback.get().getVersion();
+ return rpcCallback.get();
}
});
- for (Map.Entry<byte[],Long> result : results.entrySet()) {
+ for (Map.Entry<byte[],GetVersionResponse> result : results.entrySet()) {
// This is the "phoenix.jar" is in-place, but server is out-of-sync with client case.
- long version = result.getValue();
- isTableNamespaceMappingEnabled |= MetaDataUtil.decodeTableNamespaceMappingEnabled(version);
+ GetVersionResponse versionResponse = result.getValue();
+ long serverJarVersion = versionResponse.getVersion();
+ isTableNamespaceMappingEnabled |= MetaDataUtil.decodeTableNamespaceMappingEnabled(serverJarVersion);
- if (!isCompatible(result.getValue())) {
+ if (!isCompatible(serverJarVersion)) {
isIncompatible = true;
HRegionLocation name = regionMap.get(result.getKey());
buf.append(name);
buf.append(';');
}
- hasIndexWALCodec &= hasIndexWALCodec(result.getValue());
- if (minHBaseVersion > MetaDataUtil.decodeHBaseVersion(result.getValue())) {
- minHBaseVersion = MetaDataUtil.decodeHBaseVersion(result.getValue());
+ hasIndexWALCodec &= hasIndexWALCodec(serverJarVersion);
+ if (minHBaseVersion > MetaDataUtil.decodeHBaseVersion(serverJarVersion)) {
+ minHBaseVersion = MetaDataUtil.decodeHBaseVersion(serverJarVersion);
+ }
+ // In case this is the first time connecting to this cluster, the system catalog table does not have an
+ // entry for itself yet, so we cannot get the timestamp and this will not be returned from the
+ // GetVersionResponse message object
+ if (versionResponse.hasSystemCatalogTimestamp()) {
+ systemCatalogTimestamp = systemCatalogTimestamp < versionResponse.getSystemCatalogTimestamp() ?
+ systemCatalogTimestamp: versionResponse.getSystemCatalogTimestamp();
}
}
if (isTableNamespaceMappingEnabled != SchemaUtil.isNamespaceMappingEnabled(PTableType.TABLE,
@@ -1274,6 +1339,9 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
buf.setLength(buf.length()-1);
throw new SQLExceptionInfo.Builder(SQLExceptionCode.OUTDATED_JARS).setMessage(buf.toString()).build().buildException();
}
+ if (systemCatalogTimestamp < MIN_SYSTEM_TABLE_TIMESTAMP) {
+ throw new UpgradeRequiredException(systemCatalogTimestamp);
+ }
}
/**
@@ -1334,14 +1402,14 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
tableProps.put(MetaDataUtil.IS_VIEW_INDEX_TABLE_PROP_NAME, TRUE_BYTES_AS_STRING);
HTableDescriptor desc = ensureTableCreated(physicalIndexName, PTableType.TABLE, tableProps, families, splits,
- false, isNamespaceMapped);
+ false, isNamespaceMapped, false);
if (desc != null) {
if (!Boolean.TRUE.equals(PBoolean.INSTANCE.toObject(desc.getValue(MetaDataUtil.IS_VIEW_INDEX_TABLE_PROP_BYTES)))) {
String fullTableName = Bytes.toString(physicalIndexName);
throw new TableAlreadyExistsException(
- "Unable to create shared physical table for indexes on views.",
- SchemaUtil.getSchemaNameFromFullName(fullTableName),
- SchemaUtil.getTableNameFromFullName(fullTableName));
+ SchemaUtil.getSchemaNameFromFullName(fullTableName),
+ SchemaUtil.getTableNameFromFullName(fullTableName),
+ "Unable to create shared physical table for indexes on views.");
}
}
}
@@ -1409,8 +1477,8 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
@Override
public MetaDataMutationResult createTable(final List<Mutation> tableMetaData, final byte[] physicalTableName,
PTableType tableType, Map<String, Object> tableProps,
- final List<Pair<byte[], Map<String, Object>>> families, byte[][] splits, boolean isNamespaceMapped, final boolean allocateIndexId)
- throws SQLException {
+ final List<Pair<byte[], Map<String, Object>>> families, byte[][] splits, boolean isNamespaceMapped,
+ final boolean allocateIndexId, final boolean isDoNotUpgradePropSet) throws SQLException {
byte[][] rowKeyMetadata = new byte[3][];
Mutation m = MetaDataUtil.getPutOnlyTableHeaderRow(tableMetaData);
byte[] key = m.getRow();
@@ -1430,7 +1498,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
if ((tableType == PTableType.VIEW && physicalTableName != null) || (tableType != PTableType.VIEW && (physicalTableName == null || localIndexTable))) {
// For views this will ensure that metadata already exists
// For tables and indexes, this will create the metadata if it doesn't already exist
- ensureTableCreated(tableName, tableType, tableProps, families, splits, true, isNamespaceMapped);
+ ensureTableCreated(tableName, tableType, tableProps, families, splits, true, isNamespaceMapped, isDoNotUpgradePropSet);
}
ImmutableBytesWritable ptr = new ImmutableBytesWritable();
if (tableType == PTableType.INDEX) { // Index on view
@@ -2436,30 +2504,6 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
openConnection();
hConnectionEstablished = true;
boolean isDoNotUpgradePropSet = UpgradeUtil.isNoUpgradeSet(props);
- try (HBaseAdmin admin = getAdmin()) {
- boolean mappedSystemCatalogExists = admin
- .tableExists(SchemaUtil.getPhysicalTableName(SYSTEM_CATALOG_NAME_BYTES, true));
- if (SchemaUtil.isNamespaceMappingEnabled(PTableType.SYSTEM,
- ConnectionQueryServicesImpl.this.getProps())) {
- if (admin.tableExists(SYSTEM_CATALOG_NAME_BYTES)) {
- //check if the server is already updated and have namespace config properly set.
- checkClientServerCompatibility(SYSTEM_CATALOG_NAME_BYTES);
- }
-
- // If SYSTEM tables exist, they are migrated to HBase SYSTEM namespace
- // If they don't exist, this method will create HBase SYSTEM namespace and return
- ensureSystemTablesMigratedToSystemNamespace(ConnectionQueryServicesImpl.this.getProps());
- } else if (mappedSystemCatalogExists) {
- throw new SQLExceptionInfo.Builder(
- SQLExceptionCode.INCONSISTENT_NAMESPACE_MAPPING_PROPERTIES)
- .setMessage("Cannot initiate connection as "
- + SchemaUtil.getPhysicalTableName(
- SYSTEM_CATALOG_NAME_BYTES, true)
- + " is found but client does not have "
- + IS_NAMESPACE_MAPPING_ENABLED + " enabled")
- .build().buildException();
- }
- }
Properties scnProps = PropertiesUtil.deepCopy(props);
scnProps.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB,
Long.toString(getSystemTableVersion()));
@@ -2508,38 +2552,37 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
initializationException = e;
}
return null;
+ } catch (UpgradeRequiredException e) {
+ // This will occur in 3 cases:
+ // 1. SYSTEM.CATALOG does not exist and we don't want to allow the user to create it i.e.
+ // !isAutoUpgradeEnabled or isDoNotUpgradePropSet is set
+ // 2. SYSTEM.CATALOG exists and its timestamp < MIN_SYSTEM_TABLE_TIMESTAMP
+ // 3. SYSTEM.CATALOG exists, but client and server-side namespace mapping is enabled so
+ // we need to migrate SYSTEM tables to the SYSTEM namespace
+ setUpgradeRequired();
}
- // HBase Namespace SYSTEM is created by {@link ensureSystemTablesMigratedToSystemNamespace(ReadOnlyProps)} method
- // This statement will create its entry in SYSCAT table, so that GRANT/REVOKE commands can work
- // with SYSTEM Namespace. (See PHOENIX-4227 https://issues.apache.org/jira/browse/PHOENIX-4227)
- if (SchemaUtil.isNamespaceMappingEnabled(PTableType.SYSTEM,
- ConnectionQueryServicesImpl.this.getProps())) {
- try {
- metaConnection.createStatement().execute("CREATE SCHEMA IF NOT EXISTS "
- + PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA);
- } catch (NewerSchemaAlreadyExistsException e) {
- // Older clients with appropriate perms may try getting a new connection
- // This results in NewerSchemaAlreadyExistsException, so we can safely ignore it here
- } catch (PhoenixIOException e) {
- if (!Iterables.isEmpty(Iterables.filter(Throwables.getCausalChain(e), AccessDeniedException.class))) {
- // Ignore ADE
- } else {
- throw e;
- }
- }
- }
if (!ConnectionQueryServicesImpl.this.upgradeRequired.get()) {
createOtherSystemTables(metaConnection, hBaseAdmin);
+ // In case namespace mapping is enabled and system table to system namespace mapping is also enabled,
+ // create an entry for the SYSTEM namespace in the SYSCAT table, so that GRANT/REVOKE commands can work
+ // with SYSTEM Namespace
+ createSchemaIfNotExistsSystemNSMappingEnabled(metaConnection);
} else if (isAutoUpgradeEnabled && !isDoNotUpgradePropSet) {
+ // Upgrade is required and we are allowed to automatically upgrade
upgradeSystemTables(url, props);
+ } else {
+ // We expect the user to manually run the "EXECUTE UPGRADE" command first.
+ // This exception will get caught below as a RetriableUpgradeException
+ throw new UpgradeRequiredException();
}
}
scheduleRenewLeaseTasks();
success = true;
} catch (RetriableUpgradeException e) {
- // Don't set it as initializationException because otherwise the clien't won't be able
- // to retry establishing connection.
+ // Set success to true and don't set the exception as an initializationException,
+ // because otherwise the client won't be able to retry establishing the connection.
+ success = true;
throw e;
} catch (Exception e) {
if (e instanceof SQLException) {
@@ -2580,7 +2623,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
}
- void createSysMutexTableIfNotExists(HBaseAdmin admin, ReadOnlyProps props) throws IOException, SQLException {
+ void createSysMutexTableIfNotExists(HBaseAdmin admin) throws IOException, SQLException {
try {
if(admin.tableExists(PhoenixDatabaseMetaData.SYSTEM_MUTEX_NAME) || admin.tableExists(TableName.valueOf(
PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME,PhoenixDatabaseMetaData.SYSTEM_MUTEX_TABLE_NAME))) {
@@ -2588,7 +2631,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
return;
}
final TableName mutexTableName = SchemaUtil.getPhysicalTableName(
- PhoenixDatabaseMetaData.SYSTEM_MUTEX_NAME, props);
+ PhoenixDatabaseMetaData.SYSTEM_MUTEX_NAME, this.getProps());
HTableDescriptor tableDesc = new HTableDescriptor(mutexTableName);
HColumnDescriptor columnDesc = new HColumnDescriptor(
PhoenixDatabaseMetaData.SYSTEM_MUTEX_FAMILY_NAME_BYTES);
@@ -2640,7 +2683,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
} catch (TableAlreadyExistsException ignore) {}
// Catch the IOException to log the error message and then bubble it up for the client to retry.
try {
- createSysMutexTableIfNotExists(hbaseAdmin, ConnectionQueryServicesImpl.this.getProps());
+ createSysMutexTableIfNotExists(hbaseAdmin);
} catch (IOException exception) {
logger.error("Failed to created SYSMUTEX table. Upgrade or migration is not possible without it. Please retry.");
throw exception;
@@ -2648,6 +2691,265 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
/**
+ * Create an entry for the SYSTEM namespace in the SYSCAT table in case namespace mapping is enabled and system table
+ * to system namespace mapping is also enabled. If not enabled, this method returns immediately without doing anything
+ * @param metaConnection
+ * @throws SQLException
+ */
+ private void createSchemaIfNotExistsSystemNSMappingEnabled(PhoenixConnection metaConnection) throws SQLException {
+ // HBase Namespace SYSTEM is assumed to be already created inside {@link ensureTableCreated(byte[], PTableType,
+ // Map<String, Object>, List<Pair<byte[], Map<String, Object>>>, byte[][], boolean, boolean, boolean)}.
+ // This statement will create an entry for the SYSTEM namespace in the SYSCAT table, so that GRANT/REVOKE
+ // commands can work with SYSTEM Namespace. (See PHOENIX-4227 https://issues.apache.org/jira/browse/PHOENIX-4227)
+ if (SchemaUtil.isNamespaceMappingEnabled(PTableType.SYSTEM,
+ ConnectionQueryServicesImpl.this.getProps())) {
+ try {
+ metaConnection.createStatement().execute("CREATE SCHEMA IF NOT EXISTS "
+ + PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA);
+ } catch (NewerSchemaAlreadyExistsException e) {
+ // Older clients with appropriate perms may try getting a new connection
+ // This results in NewerSchemaAlreadyExistsException, so we can safely ignore it here
+ } catch (PhoenixIOException e) {
+ if (!Iterables.isEmpty(Iterables.filter(Throwables.getCausalChain(e), AccessDeniedException.class))) {
+ // Ignore ADE
+ } else {
+ throw e;
+ }
+ }
+ }
+ }
+
+ /**
+ * Upgrade the SYSCAT schema if required
+ * @param metaConnection
+ * @param currentServerSideTableTimeStamp
+ * @return Phoenix connection object
+ * @throws SQLException
+ * @throws IOException
+ * @throws TimeoutException
+ * @throws InterruptedException
+ */
+ // Available for testing
+ protected PhoenixConnection upgradeSystemCatalogIfRequired(PhoenixConnection metaConnection,
+ long currentServerSideTableTimeStamp) throws SQLException, IOException, TimeoutException, InterruptedException {
+ String columnsToAdd = "";
+ // This will occur if we have an older SYSTEM.CATALOG and we need to update it to
+ // include any new columns we've added.
+ if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_3_0) {
+ // We know that we always need to add the STORE_NULLS column for 4.3 release
+ columnsToAdd = addColumn(columnsToAdd, PhoenixDatabaseMetaData.STORE_NULLS
+ + " " + PBoolean.INSTANCE.getSqlTypeName());
+ try (HBaseAdmin admin = getAdmin()) {
+ HTableDescriptor[] localIndexTables = admin
+ .listTables(MetaDataUtil.LOCAL_INDEX_TABLE_PREFIX + ".*");
+ for (HTableDescriptor table : localIndexTables) {
+ if (table.getValue(MetaDataUtil.PARENT_TABLE_KEY) == null
+ && table.getValue(MetaDataUtil.IS_LOCAL_INDEX_TABLE_PROP_NAME) != null) {
+ table.setValue(MetaDataUtil.PARENT_TABLE_KEY,
+ MetaDataUtil.getLocalIndexUserTableName(table.getNameAsString()));
+ // Explicitly disable, modify and enable the table to ensure
+ // co-location of data and index regions. If we just modify the
+ // table descriptor when online schema change enabled may reopen
+ // the region in same region server instead of following data region.
+ admin.disableTable(table.getTableName());
+ admin.modifyTable(table.getTableName(), table);
+ admin.enableTable(table.getTableName());
+ }
+ }
+ }
+ }
+
+ // If the server side schema is before MIN_SYSTEM_TABLE_TIMESTAMP_4_1_0 then
+ // we need to add INDEX_TYPE and INDEX_DISABLE_TIMESTAMP columns too.
+ // TODO: Once https://issues.apache.org/jira/browse/PHOENIX-1614 is fixed,
+ // we should just have a ALTER TABLE ADD IF NOT EXISTS statement with all
+ // the column names that have been added to SYSTEM.CATALOG since 4.0.
+ if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_1_0) {
+ columnsToAdd = addColumn(columnsToAdd, PhoenixDatabaseMetaData.INDEX_TYPE + " "
+ + PUnsignedTinyint.INSTANCE.getSqlTypeName() + ", "
+ + PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP + " "
+ + PLong.INSTANCE.getSqlTypeName());
+ }
+
+ // If we have some new columns from 4.1-4.3 to add, add them now.
+ if (!columnsToAdd.isEmpty()) {
+ // Ugh..need to assign to another local variable to keep eclipse happy.
+ PhoenixConnection newMetaConnection = addColumnsIfNotExists(metaConnection,
+ PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_3_0, columnsToAdd);
+ metaConnection = newMetaConnection;
+ }
+
+ if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_5_0) {
+ columnsToAdd = PhoenixDatabaseMetaData.BASE_COLUMN_COUNT + " "
+ + PInteger.INSTANCE.getSqlTypeName();
+ try {
+ metaConnection = addColumn(metaConnection,
+ PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_5_0, columnsToAdd,
+ false);
+ upgradeTo4_5_0(metaConnection);
+ } catch (ColumnAlreadyExistsException ignored) {
+ /*
+ * Upgrade to 4.5 is a slightly special case. We use the fact that the
+ * column BASE_COLUMN_COUNT is already part of the meta-data schema as the
+ * signal that the server side upgrade has finished or is in progress.
+ */
+ logger.debug("No need to run 4.5 upgrade");
+ }
+ Properties p = PropertiesUtil.deepCopy(metaConnection.getClientInfo());
+ p.remove(PhoenixRuntime.CURRENT_SCN_ATTRIB);
+ p.remove(PhoenixRuntime.TENANT_ID_ATTRIB);
+ PhoenixConnection conn = new PhoenixConnection(
+ ConnectionQueryServicesImpl.this, metaConnection.getURL(), p,
+ metaConnection.getMetaDataCache());
+ try {
+ List<String> tablesNeedingUpgrade = UpgradeUtil
+ .getPhysicalTablesWithDescRowKey(conn);
+ if (!tablesNeedingUpgrade.isEmpty()) {
+ logger.warn("The following tables require upgrade due to a bug causing the row key to be incorrect for descending columns and ascending BINARY columns (PHOENIX-2067 and PHOENIX-2120):\n"
+ + Joiner.on(' ').join(tablesNeedingUpgrade)
+ + "\nTo upgrade issue the \"bin/psql.py -u\" command.");
+ }
+ List<String> unsupportedTables = UpgradeUtil
+ .getPhysicalTablesWithDescVarbinaryRowKey(conn);
+ if (!unsupportedTables.isEmpty()) {
+ logger.warn("The following tables use an unsupported VARBINARY DESC construct and need to be changed:\n"
+ + Joiner.on(' ').join(unsupportedTables));
+ }
+ } catch (Exception ex) {
+ logger.error(
+ "Unable to determine tables requiring upgrade due to PHOENIX-2067",
+ ex);
+ } finally {
+ conn.close();
+ }
+ }
+ // Add these columns one at a time, each with different timestamps so that if folks
+ // have
+ // run the upgrade code already for a snapshot, we'll still enter this block (and do
+ // the
+ // parts we haven't yet done).
+ if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_6_0) {
+ columnsToAdd = PhoenixDatabaseMetaData.IS_ROW_TIMESTAMP + " "
+ + PBoolean.INSTANCE.getSqlTypeName();
+ metaConnection = addColumnsIfNotExists(metaConnection,
+ PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_6_0, columnsToAdd);
+ }
+ if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0) {
+ // Drop old stats table so that new stats table is created
+ metaConnection = dropStatsTable(metaConnection,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 4);
+ metaConnection = addColumnsIfNotExists(
+ metaConnection,
+ PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 3,
+ PhoenixDatabaseMetaData.TRANSACTIONAL + " "
+ + PBoolean.INSTANCE.getSqlTypeName());
+ metaConnection = addColumnsIfNotExists(
+ metaConnection,
+ PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 2,
+ PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY + " "
+ + PLong.INSTANCE.getSqlTypeName());
+ metaConnection = setImmutableTableIndexesImmutable(metaConnection,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 1);
+ metaConnection = updateSystemCatalogTimestamp(metaConnection,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0);
+ ConnectionQueryServicesImpl.this.removeTable(null,
+ PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0);
+ clearCache();
+ }
+
+ if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0) {
+ metaConnection = addColumnsIfNotExists(
+ metaConnection,
+ PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0 - 2,
+ PhoenixDatabaseMetaData.IS_NAMESPACE_MAPPED + " "
+ + PBoolean.INSTANCE.getSqlTypeName());
+ metaConnection = addColumnsIfNotExists(
+ metaConnection,
+ PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0 - 1,
+ PhoenixDatabaseMetaData.AUTO_PARTITION_SEQ + " "
+ + PVarchar.INSTANCE.getSqlTypeName());
+ metaConnection = addColumnsIfNotExists(
+ metaConnection,
+ PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0,
+ PhoenixDatabaseMetaData.APPEND_ONLY_SCHEMA + " "
+ + PBoolean.INSTANCE.getSqlTypeName());
+ metaConnection = UpgradeUtil.disableViewIndexes(metaConnection);
+ if (getProps().getBoolean(QueryServices.LOCAL_INDEX_CLIENT_UPGRADE_ATTRIB,
+ QueryServicesOptions.DEFAULT_LOCAL_INDEX_CLIENT_UPGRADE)) {
+ metaConnection = UpgradeUtil.upgradeLocalIndexes(metaConnection);
+ }
+ ConnectionQueryServicesImpl.this.removeTable(null,
+ PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0);
+ clearCache();
+ }
+ if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0) {
+ metaConnection = addColumnsIfNotExists(
+ metaConnection,
+ PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0,
+ PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + " "
+ + PLong.INSTANCE.getSqlTypeName());
+ ConnectionQueryServicesImpl.this.removeTable(null,
+ PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0);
+ clearCache();
+ }
+ if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0) {
+ metaConnection = addColumnQualifierColumn(metaConnection, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0 - 3);
+ metaConnection = addColumnsIfNotExists(
+ metaConnection,
+ PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0 - 2,
+ PhoenixDatabaseMetaData.IMMUTABLE_STORAGE_SCHEME + " "
+ + PTinyint.INSTANCE.getSqlTypeName());
+ metaConnection = addColumnsIfNotExists(
+ metaConnection,
+ PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0 - 1,
+ PhoenixDatabaseMetaData.ENCODING_SCHEME + " "
+ + PTinyint.INSTANCE.getSqlTypeName());
+ metaConnection = addColumnsIfNotExists(
+ metaConnection,
+ PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0,
+ PhoenixDatabaseMetaData.COLUMN_QUALIFIER_COUNTER + " "
+ + PInteger.INSTANCE.getSqlTypeName());
+ ConnectionQueryServicesImpl.this.removeTable(null,
+ PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0);
+ clearCache();
+ }
+ if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_11_0) {
+ metaConnection = addColumnsIfNotExists(
+ metaConnection,
+ PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_11_0,
+ PhoenixDatabaseMetaData.USE_STATS_FOR_PARALLELIZATION + " "
+ + PBoolean.INSTANCE.getSqlTypeName());
+ addParentToChildLinks(metaConnection);
+ }
+ if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_14_0) {
+ metaConnection = addColumnsIfNotExists(
+ metaConnection,
+ PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_14_0,
+ PhoenixDatabaseMetaData.TRANSACTION_PROVIDER + " "
+ + PTinyint.INSTANCE.getSqlTypeName());
+ }
+ return metaConnection;
+ }
+
+ /**
* There is no other locking needed here since only one connection (on the same or different JVM) will be able to
* acquire the upgrade mutex via {@link #acquireUpgradeMutex(long, byte[])}.
*/
@@ -2674,244 +2976,54 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
metaConnection = new PhoenixConnection(ConnectionQueryServicesImpl.this, globalUrl,
scnProps, newEmptyMetaData());
metaConnection.setRunningUpgrade(true);
+ // Always try to create SYSTEM.MUTEX table since we need it to acquire the upgrade mutex.
+ // Upgrade or migration is not possible without the upgrade mutex
+ try (HBaseAdmin admin = getAdmin()) {
+ createSysMutexTableIfNotExists(admin);
+ }
try {
metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_TABLE_METADATA);
} catch (NewerTableAlreadyExistsException ignore) {
// Ignore, as this will happen if the SYSTEM.CATALOG already exists at this fixed
// timestamp. A TableAlreadyExistsException is not thrown, since the table only exists
// *after* this fixed timestamp.
+ } catch (UpgradeRequiredException e) {
+ // This is thrown while trying to create SYSTEM:CATALOG to indicate that we must migrate SYSTEM tables
+ // to the SYSTEM namespace and/or upgrade SYSCAT if required
+ sysCatalogTableName = SchemaUtil.getPhysicalName(SYSTEM_CATALOG_NAME_BYTES, this.getProps()).getNameAsString();
+ if (SchemaUtil.isNamespaceMappingEnabled(PTableType.SYSTEM, ConnectionQueryServicesImpl.this.getProps())) {
+ // Try acquiring a lock in SYSMUTEX table before migrating the tables since it involves disabling the table.
+ if (acquiredMutexLock = acquireUpgradeMutex(MetaDataProtocol.MIN_SYSTEM_TABLE_MIGRATION_TIMESTAMP, mutexRowKey)) {
+ logger.debug("Acquired lock in SYSMUTEX table for migrating SYSTEM tables to SYSTEM namespace "
+ + "and/or upgrading " + sysCatalogTableName);
+ }
+ // We will not reach here if we fail to acquire the lock, since it throws UpgradeInProgressException
+
+ // If SYSTEM tables exist, they are migrated to HBase SYSTEM namespace
+ // If they don't exist or they're already migrated, this method will return immediately
+ ensureSystemTablesMigratedToSystemNamespace();
+ logger.debug("Migrated SYSTEM tables to SYSTEM namespace");
+ metaConnection = upgradeSystemCatalogIfRequired(metaConnection, e.getSystemCatalogTimeStamp());
+ }
} catch (TableAlreadyExistsException e) {
long currentServerSideTableTimeStamp = e.getTable().getTimeStamp();
sysCatalogTableName = e.getTable().getPhysicalName().getString();
if (currentServerSideTableTimeStamp < MIN_SYSTEM_TABLE_TIMESTAMP) {
- if (currentServerSideTableTimeStamp <= MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0) {
- try (HBaseAdmin admin = getAdmin()) {
- createSysMutexTableIfNotExists(admin, this.getProps());
- }
- }
+ // Try acquiring a lock in SYSMUTEX table before upgrading SYSCAT. If we cannot acquire the lock,
+ // it means some old client is either migrating SYSTEM tables or trying to upgrade the schema of
+ // SYSCAT table and hence it should not be interrupted
if (acquiredMutexLock = acquireUpgradeMutex(currentServerSideTableTimeStamp, mutexRowKey)) {
+ logger.debug("Acquired lock in SYSMUTEX table for upgrading " + sysCatalogTableName);
snapshotName = getSysCatalogSnapshotName(currentServerSideTableTimeStamp);
createSnapshot(snapshotName, sysCatalogTableName);
snapshotCreated = true;
+ logger.debug("Created snapshot for SYSCAT");
}
+ // We will not reach here if we fail to acquire the lock, since it throws UpgradeInProgressException
}
- String columnsToAdd = "";
- // This will occur if we have an older SYSTEM.CATALOG and we need to update it to
- // include any new columns we've added.
- if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_3_0) {
- // We know that we always need to add the STORE_NULLS column for 4.3 release
- columnsToAdd = addColumn(columnsToAdd, PhoenixDatabaseMetaData.STORE_NULLS
- + " " + PBoolean.INSTANCE.getSqlTypeName());
- try (HBaseAdmin admin = getAdmin()) {
- HTableDescriptor[] localIndexTables = admin
- .listTables(MetaDataUtil.LOCAL_INDEX_TABLE_PREFIX + ".*");
- for (HTableDescriptor table : localIndexTables) {
- if (table.getValue(MetaDataUtil.PARENT_TABLE_KEY) == null
- && table.getValue(MetaDataUtil.IS_LOCAL_INDEX_TABLE_PROP_NAME) != null) {
- table.setValue(MetaDataUtil.PARENT_TABLE_KEY,
- MetaDataUtil.getLocalIndexUserTableName(table.getNameAsString()));
- // Explicitly disable, modify and enable the table to ensure
- // co-location of data and index regions. If we just modify the
- // table descriptor when online schema change enabled may reopen
- // the region in same region server instead of following data region.
- admin.disableTable(table.getTableName());
- admin.modifyTable(table.getTableName(), table);
- admin.enableTable(table.getTableName());
- }
- }
- }
- }
-
- // If the server side schema is before MIN_SYSTEM_TABLE_TIMESTAMP_4_1_0 then
- // we need to add INDEX_TYPE and INDEX_DISABLE_TIMESTAMP columns too.
- // TODO: Once https://issues.apache.org/jira/browse/PHOENIX-1614 is fixed,
- // we should just have a ALTER TABLE ADD IF NOT EXISTS statement with all
- // the column names that have been added to SYSTEM.CATALOG since 4.0.
- if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_1_0) {
- columnsToAdd = addColumn(columnsToAdd, PhoenixDatabaseMetaData.INDEX_TYPE + " "
- + PUnsignedTinyint.INSTANCE.getSqlTypeName() + ", "
- + PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP + " "
- + PLong.INSTANCE.getSqlTypeName());
- }
-
- // If we have some new columns from 4.1-4.3 to add, add them now.
- if (!columnsToAdd.isEmpty()) {
- // Ugh..need to assign to another local variable to keep eclipse happy.
- PhoenixConnection newMetaConnection = addColumnsIfNotExists(metaConnection,
- PhoenixDatabaseMetaData.SYSTEM_CATALOG,
- MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_3_0, columnsToAdd);
- metaConnection = newMetaConnection;
- }
-
- if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_5_0) {
- columnsToAdd = PhoenixDatabaseMetaData.BASE_COLUMN_COUNT + " "
- + PInteger.INSTANCE.getSqlTypeName();
- try {
- metaConnection = addColumn(metaConnection,
- PhoenixDatabaseMetaData.SYSTEM_CATALOG,
- MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_5_0, columnsToAdd,
- false);
- upgradeTo4_5_0(metaConnection);
- } catch (ColumnAlreadyExistsException ignored) {
- /*
- * Upgrade to 4.5 is a slightly special case. We use the fact that the
- * column BASE_COLUMN_COUNT is already part of the meta-data schema as the
- * signal that the server side upgrade has finished or is in progress.
- */
- logger.debug("No need to run 4.5 upgrade");
- }
- Properties p = PropertiesUtil.deepCopy(metaConnection.getClientInfo());
- p.remove(PhoenixRuntime.CURRENT_SCN_ATTRIB);
- p.remove(PhoenixRuntime.TENANT_ID_ATTRIB);
- PhoenixConnection conn = new PhoenixConnection(
- ConnectionQueryServicesImpl.this, metaConnection.getURL(), p,
- metaConnection.getMetaDataCache());
- try {
- List<String> tablesNeedingUpgrade = UpgradeUtil
- .getPhysicalTablesWithDescRowKey(conn);
- if (!tablesNeedingUpgrade.isEmpty()) {
- logger.warn("The following tables require upgrade due to a bug causing the row key to be incorrect for descending columns and ascending BINARY columns (PHOENIX-2067 and PHOENIX-2120):\n"
- + Joiner.on(' ').join(tablesNeedingUpgrade)
- + "\nTo upgrade issue the \"bin/psql.py -u\" command.");
- }
- List<String> unsupportedTables = UpgradeUtil
- .getPhysicalTablesWithDescVarbinaryRowKey(conn);
- if (!unsupportedTables.isEmpty()) {
- logger.warn("The following tables use an unsupported VARBINARY DESC construct and need to be changed:\n"
- + Joiner.on(' ').join(unsupportedTables));
- }
- } catch (Exception ex) {
- logger.error(
- "Unable to determine tables requiring upgrade due to PHOENIX-2067",
- ex);
- } finally {
- conn.close();
- }
- }
- // Add these columns one at a time, each with different timestamps so that if folks
- // have
- // run the upgrade code already for a snapshot, we'll still enter this block (and do
- // the
- // parts we haven't yet done).
- if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_6_0) {
- columnsToAdd = PhoenixDatabaseMetaData.IS_ROW_TIMESTAMP + " "
- + PBoolean.INSTANCE.getSqlTypeName();
- metaConnection = addColumnsIfNotExists(metaConnection,
- PhoenixDatabaseMetaData.SYSTEM_CATALOG,
- MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_6_0, columnsToAdd);
- }
- if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0) {
- // Drop old stats table so that new stats table is created
- metaConnection = dropStatsTable(metaConnection,
- MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 4);
- metaConnection = addColumnsIfNotExists(
- metaConnection,
- PhoenixDatabaseMetaData.SYSTEM_CATALOG,
- MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 3,
- PhoenixDatabaseMetaData.TRANSACTIONAL + " "
- + PBoolean.INSTANCE.getSqlTypeName());
- metaConnection = addColumnsIfNotExists(
- metaConnection,
- PhoenixDatabaseMetaData.SYSTEM_CATALOG,
- MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 2,
- PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY + " "
- + PLong.INSTANCE.getSqlTypeName());
- metaConnection = setImmutableTableIndexesImmutable(metaConnection,
- MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 - 1);
- metaConnection = updateSystemCatalogTimestamp(metaConnection,
- MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0);
- ConnectionQueryServicesImpl.this.removeTable(null,
- PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null,
- MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0);
- clearCache();
- }
-
- if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0) {
- metaConnection = addColumnsIfNotExists(
- metaConnection,
- PhoenixDatabaseMetaData.SYSTEM_CATALOG,
- MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0 - 2,
- PhoenixDatabaseMetaData.IS_NAMESPACE_MAPPED + " "
- + PBoolean.INSTANCE.getSqlTypeName());
- metaConnection = addColumnsIfNotExists(
- metaConnection,
- PhoenixDatabaseMetaData.SYSTEM_CATALOG,
- MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0 - 1,
- PhoenixDatabaseMetaData.AUTO_PARTITION_SEQ + " "
- + PVarchar.INSTANCE.getSqlTypeName());
- metaConnection = addColumnsIfNotExists(
- metaConnection,
- PhoenixDatabaseMetaData.SYSTEM_CATALOG,
- MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0,
- PhoenixDatabaseMetaData.APPEND_ONLY_SCHEMA + " "
- + PBoolean.INSTANCE.getSqlTypeName());
- metaConnection = UpgradeUtil.disableViewIndexes(metaConnection);
- if (getProps().getBoolean(QueryServices.LOCAL_INDEX_CLIENT_UPGRADE_ATTRIB,
- QueryServicesOptions.DEFAULT_LOCAL_INDEX_CLIENT_UPGRADE)) {
- metaConnection = UpgradeUtil.upgradeLocalIndexes(metaConnection);
- }
- ConnectionQueryServicesImpl.this.removeTable(null,
- PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null,
- MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0);
- clearCache();
- }
- if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0) {
- metaConnection = addColumnsIfNotExists(
- metaConnection,
- PhoenixDatabaseMetaData.SYSTEM_CATALOG,
- MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0,
- PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + " "
- + PLong.INSTANCE.getSqlTypeName());
- ConnectionQueryServicesImpl.this.removeTable(null,
- PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null,
- MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0);
- clearCache();
- }
- if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0) {
- metaConnection = addColumnQualifierColumn(metaConnection, MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0 - 3);
- metaConnection = addColumnsIfNotExists(
- metaConnection,
- PhoenixDatabaseMetaData.SYSTEM_CATALOG,
- MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0 - 2,
- PhoenixDatabaseMetaData.IMMUTABLE_STORAGE_SCHEME + " "
- + PTinyint.INSTANCE.getSqlTypeName());
- metaConnection = addColumnsIfNotExists(
- metaConnection,
- PhoenixDatabaseMetaData.SYSTEM_CATALOG,
- MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0 - 1,
- PhoenixDatabaseMetaData.ENCODING_SCHEME + " "
- + PTinyint.INSTANCE.getSqlTypeName());
- metaConnection = addColumnsIfNotExists(
- metaConnection,
- PhoenixDatabaseMetaData.SYSTEM_CATALOG,
- MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0,
- PhoenixDatabaseMetaData.COLUMN_QUALIFIER_COUNTER + " "
- + PInteger.INSTANCE.getSqlTypeName());
- ConnectionQueryServicesImpl.this.removeTable(null,
- PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null,
- MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_10_0);
- clearCache();
- }
- if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_11_0) {
- metaConnection = addColumnsIfNotExists(
- metaConnection,
- PhoenixDatabaseMetaData.SYSTEM_CATALOG,
- MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_11_0,
- PhoenixDatabaseMetaData.USE_STATS_FOR_PARALLELIZATION + " "
- + PBoolean.INSTANCE.getSqlTypeName());
- addParentToChildLinks(metaConnection);
- }
- if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_14_0) {
- metaConnection = addColumnsIfNotExists(
- metaConnection,
- PhoenixDatabaseMetaData.SYSTEM_CATALOG,
- MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_14_0,
- PhoenixDatabaseMetaData.TRANSACTION_PROVIDER + " "
- + PTinyint.INSTANCE.getSqlTypeName());
- }
+ metaConnection = upgradeSystemCatalogIfRequired(metaConnection, currentServerSideTableTimeStamp);
}
-
int nSaltBuckets = ConnectionQueryServicesImpl.this.props.getInt(
QueryServices.SEQUENCE_SALT_BUCKETS_ATTRIB,
QueryServicesOptions.DEFAULT_SEQUENCE_TABLE_SALT_BUCKETS);
@@ -2997,6 +3109,12 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
try {
metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_LOG_METADATA);
} catch (NewerTableAlreadyExistsException e) {} catch (TableAlreadyExistsException e) {}
+
+ // In case namespace mapping is enabled and system table to system namespace mapping is also enabled,
+ // create an entry for the SYSTEM namespace in the SYSCAT table, so that GRANT/REVOKE commands can work
+ // with SYSTEM Namespace
+ createSchemaIfNotExistsSystemNSMappingEnabled(metaConnection);
+
ConnectionQueryServicesImpl.this.upgradeRequired.set(false);
success = true;
} catch (UpgradeInProgressException | UpgradeNotRequiredException e) {
@@ -3218,28 +3336,12 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
}
- void ensureSystemTablesMigratedToSystemNamespace(ReadOnlyProps props)
+ void ensureSystemTablesMigratedToSystemNamespace()
throws SQLException, IOException, IllegalArgumentException, InterruptedException {
- if (!SchemaUtil.isNamespaceMappingEnabled(PTableType.SYSTEM, props)) { return; }
-
- boolean acquiredMutexLock = false;
- byte[] mutexRowKey = SchemaUtil.getTableKey(null, PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA,
- PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE);
+ if (!SchemaUtil.isNamespaceMappingEnabled(PTableType.SYSTEM, this.getProps())) { return; }
HTableInterface metatable = null;
try (HBaseAdmin admin = getAdmin()) {
- // SYSTEM namespace needs to be created via HBase API's because "CREATE SCHEMA" statement tries to write its metadata
- // in SYSTEM:CATALOG table. Without SYSTEM namespace, SYSTEM:CATALOG table cannot be created.
- try {
- ensureNamespaceCreated(QueryConstants.SYSTEM_SCHEMA_NAME);
- } catch (PhoenixIOException e) {
- // We could either:
- // 1) Not access the NS descriptor. The NS may or may not exist at this point.
- // 2) We could not create the NS
- // Regardless of the case 1 or 2, if the NS does not exist, we will error expectedly
- // below. If the NS does exist and is mapped, the below check will exit gracefully.
- }
-
List<TableName> tableNames = getSystemTableNamesInDefaultNamespace(admin);
// No tables exist matching "SYSTEM\..*", they are all already in "SYSTEM:.*"
if (tableNames.size() == 0) { return; }
@@ -3248,33 +3350,22 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
logger.warn("Expected 5 system tables but found " + tableNames.size() + ":" + tableNames);
}
- // Try acquiring a lock in SYSMUTEX table before migrating the tables since it involves disabling the table
- // If we cannot acquire lock, it means some old client is either migrating SYSCAT or trying to upgrade the
- // schema of SYSCAT table and hence it should not be interrupted
- // Create mutex if not already created
- createSysMutexTableIfNotExists(admin, props);
- acquiredMutexLock = acquireUpgradeMutex(MetaDataProtocol.MIN_SYSTEM_TABLE_MIGRATION_TIMESTAMP, mutexRowKey);
- if(acquiredMutexLock) {
- logger.debug("Acquired lock in SYSMUTEX table for migrating SYSTEM tables to SYSTEM namespace");
- }
- // We will not reach here if we fail to acquire the lock, since it throws UpgradeInProgressException
-
// Handle the upgrade of SYSMUTEX table separately since it doesn't have any entries in SYSCAT
logger.info("Migrating SYSTEM.MUTEX table to SYSTEM namespace.");
String sysMutexSrcTableName = PhoenixDatabaseMetaData.SYSTEM_MUTEX_NAME;
- String sysMutexDestTableName = SchemaUtil.getPhysicalName(sysMutexSrcTableName.getBytes(), props).getNameAsString();
+ String sysMutexDestTableName = SchemaUtil.getPhysicalName(sysMutexSrcTableName.getBytes(), this.getProps()).getNameAsString();
UpgradeUtil.mapTableToNamespace(admin, sysMutexSrcTableName, sysMutexDestTableName, PTableType.SYSTEM);
tableNames.remove(PhoenixDatabaseMetaData.SYSTEM_MUTEX_HBASE_TABLE_NAME);
byte[] mappedSystemTable = SchemaUtil
- .getPhysicalName(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES, props).getName();
+ .getPhysicalName(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES, this.getProps()).getName();
metatable = getTable(mappedSystemTable);
if (tableNames.contains(PhoenixDatabaseMetaData.SYSTEM_CATALOG_HBASE_TABLE_NAME)) {
if (!admin.tableExists(mappedSystemTable)) {
logger.info("Migrating SYSTEM.CATALOG table to SYSTEM namespace.");
// Actual migration of SYSCAT table
UpgradeUtil.mapTableToNamespace(admin, metatable,
- PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, props, null, PTableType.SYSTEM,
+ PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, this.getProps(), null, PTableType.SYSTEM,
null);
// Invalidate the client-side metadataCache
ConnectionQueryServicesImpl.this.removeTable(null,
@@ -3285,7 +3376,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
for (TableName table : tableNames) {
logger.info(String.format("Migrating %s table to SYSTEM namespace.", table.getNameAsString()));
- UpgradeUtil.mapTableToNamespace(admin, metatable, table.getNameAsString(), props, null, PTableType.SYSTEM,
+ UpgradeUtil.mapTableToNamespace(admin, metatable, table.getNameAsString(), this.getProps(), null, PTableType.SYSTEM,
null);
ConnectionQueryServicesImpl.this.removeTable(null, table.getNameAsString(), null,
MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_1_0);
@@ -3297,9 +3388,6 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
if (metatable != null) {
metatable.close();
}
- if(acquiredMutexLock) {
- releaseUpgradeMutex(mutexRowKey);
- }
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/87fdda8b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
index 41df7b0..1c56c31 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
@@ -242,7 +242,7 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
@Override
public MetaDataMutationResult createTable(List<Mutation> tableMetaData, byte[] physicalName, PTableType tableType,
Map<String, Object> tableProps, List<Pair<byte[], Map<String, Object>>> families, byte[][] splits,
- boolean isNamespaceMapped, boolean allocateIndexId) throws SQLException {
+ boolean isNamespaceMapped, boolean allocateIndexId, boolean isDoNotUpgradePropSet) throws SQLException {
if (tableType == PTableType.INDEX && IndexUtil.isLocalIndexFamily(Bytes.toString(families.iterator().next().getFirst()))) {
Object dataTableName = tableProps.get(PhoenixDatabaseMetaData.DATA_TABLE_NAME);
List<HRegionLocation> regionLocations = tableSplits.get(dataTableName);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/87fdda8b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
index e393210..cb7ce58 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/DelegateConnectionQueryServices.java
@@ -114,9 +114,9 @@ public class DelegateConnectionQueryServices extends DelegateQueryServices imple
@Override
public MetaDataMutationResult createTable(List<Mutation> tableMetaData, byte[] physicalName, PTableType tableType,
Map<String, Object> tableProps, List<Pair<byte[], Map<String, Object>>> families, byte[][] splits,
- boolean isNamespaceMapped, boolean allocateIndexId) throws SQLException {
+ boolean isNamespaceMapped, boolean allocateIndexId, boolean isDoNotUpgradePropSet) throws SQLException {
return getDelegate().createTable(tableMetaData, physicalName, tableType, tableProps, families, splits,
- isNamespaceMapped, allocateIndexId);
+ isNamespaceMapped, allocateIndexId, isDoNotUpgradePropSet);
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/87fdda8b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index 1fb668e..b15072a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -2694,7 +2694,8 @@ public class MetaDataClient {
MetaDataMutationResult result = connection.getQueryServices().createTable(
tableMetaData,
viewType == ViewType.MAPPED || allocateIndexId ? physicalNames.get(0).getBytes() : null,
- tableType, tableProps, familyPropList, splits, isNamespaceMapped, allocateIndexId);
+ tableType, tableProps, familyPropList, splits, isNamespaceMapped, allocateIndexId,
+ UpgradeUtil.isNoUpgradeSet(connection.getClientInfo()));
MutationCode code = result.getMutationCode();
switch(code) {
case TABLE_ALREADY_EXISTS:
http://git-wip-us.apache.org/repos/asf/phoenix/blob/87fdda8b/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionQueryServicesImplTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionQueryServicesImplTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionQueryServicesImplTest.java
index b5c3e4a..46907d9 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionQueryServicesImplTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/ConnectionQueryServicesImplTest.java
@@ -47,9 +47,9 @@ public class ConnectionQueryServicesImplTest {
ConnectionQueryServicesImpl cqs = mock(ConnectionQueryServicesImpl.class);
// Invoke the real methods for these two calls
when(cqs.createSchema(any(List.class), anyString())).thenCallRealMethod();
- doCallRealMethod().when(cqs).ensureSystemTablesMigratedToSystemNamespace(any(ReadOnlyProps.class));
+ doCallRealMethod().when(cqs).ensureSystemTablesMigratedToSystemNamespace();
// Do nothing for this method, just check that it was invoked later
- doNothing().when(cqs).createSysMutexTableIfNotExists(any(HBaseAdmin.class), any(ReadOnlyProps.class));
+ doNothing().when(cqs).createSysMutexTableIfNotExists(any(HBaseAdmin.class));
// Spoof out this call so that ensureSystemTablesUpgrade() will return-fast.
when(cqs.getSystemTableNamesInDefaultNamespace(any(HBaseAdmin.class))).thenReturn(Collections.<TableName> emptyList());
@@ -60,7 +60,8 @@ public class ConnectionQueryServicesImplTest {
// Make sure that ensureSystemTablesMigratedToSystemNamespace will try to migrate the system tables.
Map<String,String> props = new HashMap<>();
props.put(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, "true");
- cqs.ensureSystemTablesMigratedToSystemNamespace(new ReadOnlyProps(props));
+ when(cqs.getProps()).thenReturn(new ReadOnlyProps(props));
+ cqs.ensureSystemTablesMigratedToSystemNamespace();
// Should be called after upgradeSystemTables()
// Proves that execution proceeded
http://git-wip-us.apache.org/repos/asf/phoenix/blob/87fdda8b/phoenix-protocol/src/main/MetaDataService.proto
----------------------------------------------------------------------
diff --git a/phoenix-protocol/src/main/MetaDataService.proto b/phoenix-protocol/src/main/MetaDataService.proto
index 2ba2b4c..369522c 100644
--- a/phoenix-protocol/src/main/MetaDataService.proto
+++ b/phoenix-protocol/src/main/MetaDataService.proto
@@ -168,6 +168,7 @@ message GetVersionRequest {
message GetVersionResponse {
required int64 version = 1;
+ optional int64 systemCatalogTimestamp = 2;
}
message ClearTableFromCacheRequest {
[2/2] phoenix git commit: PHOENIX-4579 Add a config to conditionally
create Phoenix meta tables on first client connection (Chinmay Kulkarni)
Posted by ja...@apache.org.
PHOENIX-4579 Add a config to conditionally create Phoenix meta tables on first client connection (Chinmay Kulkarni)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/87fdda8b
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/87fdda8b
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/87fdda8b
Branch: refs/heads/master
Commit: 87fdda8b180438e57d0e2f6082e5a9e988220245
Parents: 7b2c7e1
Author: James Taylor <jt...@salesforce.com>
Authored: Fri Apr 13 10:30:30 2018 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Fri Apr 13 10:30:30 2018 -0700
----------------------------------------------------------------------
.../phoenix/end2end/AppendOnlySchemaIT.java | 2 +-
.../MigrateSystemTablesToSystemNamespaceIT.java | 9 +-
.../SystemCatalogCreationOnConnectionIT.java | 626 ++++++++++++++++
.../coprocessor/MetaDataEndpointImpl.java | 21 +
.../phoenix/coprocessor/MetaDataProtocol.java | 4 +
.../coprocessor/generated/MetaDataProtos.java | 183 +++--
.../exception/UpgradeRequiredException.java | 13 +-
.../phoenix/query/ConnectionQueryServices.java | 2 +-
.../query/ConnectionQueryServicesImpl.java | 750 +++++++++++--------
.../query/ConnectionlessQueryServicesImpl.java | 2 +-
.../query/DelegateConnectionQueryServices.java | 4 +-
.../apache/phoenix/schema/MetaDataClient.java | 3 +-
.../query/ConnectionQueryServicesImplTest.java | 7 +-
phoenix-protocol/src/main/MetaDataService.proto | 1 +
14 files changed, 1236 insertions(+), 391 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/87fdda8b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AppendOnlySchemaIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AppendOnlySchemaIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AppendOnlySchemaIT.java
index 7ed64ff..d601beb 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AppendOnlySchemaIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AppendOnlySchemaIT.java
@@ -114,7 +114,7 @@ public class AppendOnlySchemaIT extends ParallelStatsDisabledIT {
// verify no create table rpcs
verify(connectionQueryServices, never()).createTable(anyListOf(Mutation.class),
any(byte[].class), any(PTableType.class), anyMap(), anyList(), any(byte[][].class),
- eq(false), eq(false));
+ eq(false), eq(false), eq(false));
reset(connectionQueryServices);
// execute alter table ddl that adds the same column
http://git-wip-us.apache.org/repos/asf/phoenix/blob/87fdda8b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
index 7cb5857..627e453 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
@@ -376,8 +376,10 @@ public class MigrateSystemTablesToSystemNamespaceIT extends BaseTest {
while(rs.next()) {
if(rs.getString("IS_NAMESPACE_MAPPED") == null) {
+ // Check that entry for SYSTEM namespace exists in SYSCAT
systemSchemaExists = rs.getString("TABLE_SCHEM").equals(PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME) ? true : systemSchemaExists;
} else if (rs.getString("COLUMN_NAME") == null) {
+ // Found the intial entry for a table in SYSCAT
String schemaName = rs.getString("TABLE_SCHEM");
String tableName = rs.getString("TABLE_NAME");
@@ -395,12 +397,11 @@ public class MigrateSystemTablesToSystemNamespaceIT extends BaseTest {
}
}
- if(!systemSchemaExists) {
- fail(PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME + " entry doesn't exist in SYSTEM.CATALOG table.");
- }
-
// The set will contain SYSMUTEX table since that table is not exposed in SYSCAT
if (systemTablesMapped) {
+ if (!systemSchemaExists) {
+ fail(PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME + " entry doesn't exist in SYSTEM.CATALOG table.");
+ }
assertTrue(namespaceMappedSystemTablesSet.size() == 1);
} else {
assertTrue(systemTablesSet.size() == 1);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/87fdda8b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemCatalogCreationOnConnectionIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemCatalogCreationOnConnectionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemCatalogCreationOnConnectionIT.java
new file mode 100644
index 0000000..689eb20
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemCatalogCreationOnConnectionIT.java
@@ -0,0 +1,626 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.phoenix.coprocessor.MetaDataProtocol;
+import org.apache.phoenix.exception.SQLExceptionCode;
+import org.apache.phoenix.exception.UpgradeRequiredException;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver;
+import org.apache.phoenix.jdbc.PhoenixTestDriver;
+import org.apache.phoenix.query.*;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.phoenix.util.UpgradeUtil;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.SQLException;
+import java.util.*;
+import java.util.concurrent.TimeoutException;
+
+import static org.junit.Assert.*;
+
+@Category(NeedsOwnMiniClusterTest.class)
+public class SystemCatalogCreationOnConnectionIT {
+ private HBaseTestingUtility testUtil = null;
+ private Set<String> hbaseTables;
+ private static boolean setOldTimestampToInduceUpgrade = false;
+ private static int countUpgradeAttempts;
+ // This flag is used to figure out if the SYSCAT schema was actually upgraded or not, based on the timestamp of SYSCAT
+ // (different from an upgrade attempt)
+ private static int actualSysCatUpgrades;
+ private static final String PHOENIX_NAMESPACE_MAPPED_SYSTEM_CATALOG = "SYSTEM:CATALOG";
+ private static final String PHOENIX_SYSTEM_CATALOG = "SYSTEM.CATALOG";
+ private static final String EXECUTE_UPGRADE_COMMAND = "EXECUTE UPGRADE";
+ private static final String MODIFIED_MAX_VERSIONS ="5";
+
+ private static final Set<String> PHOENIX_SYSTEM_TABLES = new HashSet<>(Arrays.asList(
+ "SYSTEM.CATALOG", "SYSTEM.SEQUENCE", "SYSTEM.STATS", "SYSTEM.FUNCTION",
+ "SYSTEM.MUTEX", "SYSTEM.LOG"));
+
+ private static final Set<String> PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES = new HashSet<>(
+ Arrays.asList("SYSTEM:CATALOG", "SYSTEM:SEQUENCE", "SYSTEM:STATS", "SYSTEM:FUNCTION",
+ "SYSTEM:MUTEX", "SYSTEM:LOG"));
+
+ private static class PhoenixSysCatCreationServices extends ConnectionQueryServicesImpl {
+
+ public PhoenixSysCatCreationServices(QueryServices services, PhoenixEmbeddedDriver.ConnectionInfo connectionInfo, Properties info) {
+ super(services, connectionInfo, info);
+ }
+
+ @Override
+ protected void setUpgradeRequired() {
+ super.setUpgradeRequired();
+ countUpgradeAttempts++;
+ }
+
+ @Override
+ protected long getSystemTableVersion() {
+ if (setOldTimestampToInduceUpgrade) {
+ // Return the next lower version where an upgrade was performed to induce setting the upgradeRequired flag
+ return MetaDataProtocol.getPriorUpgradeVersion();
+ }
+ return MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP;
+ }
+
+ @Override
+ protected PhoenixConnection upgradeSystemCatalogIfRequired(PhoenixConnection metaConnection,
+ long currentServerSideTableTimeStamp) throws InterruptedException, SQLException, TimeoutException, IOException {
+ PhoenixConnection newMetaConnection = super.upgradeSystemCatalogIfRequired(metaConnection, currentServerSideTableTimeStamp);
+ if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP) {
+ actualSysCatUpgrades++;
+ }
+ return newMetaConnection;
+ }
+ }
+
+ public static class PhoenixSysCatCreationTestingDriver extends PhoenixTestDriver {
+ private ConnectionQueryServices cqs;
+ private final ReadOnlyProps overrideProps;
+
+ public PhoenixSysCatCreationTestingDriver(ReadOnlyProps props) {
+ overrideProps = props;
+ }
+
+ @Override // public for testing
+ public synchronized ConnectionQueryServices getConnectionQueryServices(String url, Properties info) throws SQLException {
+ if (cqs == null) {
+ cqs = new PhoenixSysCatCreationServices(new QueryServicesTestImpl(getDefaultProps(), overrideProps), ConnectionInfo.create(url), info);
+ cqs.init(url, info);
+ }
+ return cqs;
+ }
+
+ // NOTE: Do not use this if you want to try re-establishing a connection from the client using a previously
+ // used ConnectionQueryServices instance. This is used only in cases where we need to test server-side
+ // changes and don't care about client-side properties set from the init method.
+ // Reset the Connection Query Services instance so we can create a new connection to the cluster
+ public void resetCQS() {
+ cqs = null;
+ }
+ }
+
+
+ @Before
+ public void resetVariables() {
+ setOldTimestampToInduceUpgrade = false;
+ countUpgradeAttempts = 0;
+ actualSysCatUpgrades = 0;
+ }
+
+ @After
+ public void tearDownMiniCluster() {
+ try {
+ if (testUtil != null) {
+ testUtil.shutdownMiniCluster();
+ testUtil = null;
+ }
+ } catch (Exception e) {
+ // ignore
+ }
+ }
+
+
+ // Conditions: isDoNotUpgradePropSet is true
+ // Expected: We do not create SYSTEM.CATALOG even if this is the first connection to the server
+ @Test
+ public void testFirstConnectionDoNotUpgradePropSet() throws Exception {
+ startMiniClusterWithToggleNamespaceMapping(Boolean.FALSE.toString());
+ Properties propsDoNotUpgradePropSet = new Properties();
+ // Set doNotUpgradeProperty to true
+ UpgradeUtil.doNotUpgradeOnFirstConnection(propsDoNotUpgradePropSet);
+ SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+ new SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver(ReadOnlyProps.EMPTY_PROPS);
+ try {
+ driver.getConnectionQueryServices(getJdbcUrl(), propsDoNotUpgradePropSet);
+ fail("Client should not be able to create SYSTEM.CATALOG since we set the doNotUpgrade property");
+ } catch (Exception e) {
+ assertTrue(e instanceof UpgradeRequiredException);
+ }
+ hbaseTables = getHBaseTables();
+ assertFalse(hbaseTables.contains(PHOENIX_SYSTEM_CATALOG) || hbaseTables.contains(PHOENIX_NAMESPACE_MAPPED_SYSTEM_CATALOG));
+ assertTrue(hbaseTables.size() == 0);
+ assertEquals(1, countUpgradeAttempts);
+ }
+
+
+ /********************* Testing SYSTEM.CATALOG/SYSTEM:CATALOG creation/upgrade behavior for subsequent connections *********************/
+
+
+ // Conditions: server-side namespace mapping is enabled, the first connection to the server will create unmapped
+ // SYSTEM tables i.e. SYSTEM\..*, the second connection has client-side namespace mapping enabled and
+ // system table to system namespace mapping enabled
+ // Expected: We will migrate all SYSTEM\..* tables to the SYSTEM namespace
+ @Test
+ public void testMigrateToSystemNamespace() throws Exception {
+ SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+ firstConnectionNSMappingServerEnabledClientEnabledMappingDisabled();
+ driver.resetCQS();
+ // Setting this to true to effect migration of SYSTEM tables to the SYSTEM namespace
+ Properties clientProps = getClientProperties(true, true);
+ driver.getConnectionQueryServices(getJdbcUrl(), clientProps);
+ hbaseTables = getHBaseTables();
+ assertEquals(PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES, hbaseTables);
+ assertEquals(1, countUpgradeAttempts);
+ }
+
+ // Conditions: server-side namespace mapping is enabled, the first connection to the server will create all namespace
+ // mapped SYSTEM tables i.e. SYSTEM:.*, the SYSTEM:CATALOG timestamp at creation is purposefully set to be <
+ // MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP. The subsequent connection has client-side namespace mapping enabled
+ // Expected: An upgrade is attempted when the second client connects to the server
+ @Test
+ public void testUpgradeAttempted() throws Exception {
+ setOldTimestampToInduceUpgrade = true;
+ SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+ firstConnectionNSMappingServerEnabledClientEnabled();
+ driver.resetCQS();
+ Properties clientProps = getClientProperties(true, true);
+ setOldTimestampToInduceUpgrade = false;
+ driver.getConnectionQueryServices(getJdbcUrl(), clientProps);
+ // There should be no new tables
+ assertEquals(hbaseTables, getHBaseTables());
+ // Since we set an old timestamp on purpose when creating SYSTEM:CATALOG, the second connection attempts to upgrade it
+ assertEquals(1, countUpgradeAttempts);
+ assertEquals(1, actualSysCatUpgrades);
+ }
+
+ // Conditions: server-side namespace mapping is enabled, the first connection to the server will create all namespace
+ // mapped SYSTEM tables i.e. SYSTEM:.*, the SYSTEM:CATALOG timestamp at creation is purposefully set to be <
+ // MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP. The subsequent connection has client-side namespace mapping enabled
+ // Expected: An upgrade is attempted when the second client connects to the server, but this fails since the
+ // isDoNotUpgradePropSet is set to true. We later run EXECUTE UPGRADE manually
+ @Test
+ public void testUpgradeNotAllowed() throws Exception {
+ setOldTimestampToInduceUpgrade = true;
+ SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+ firstConnectionNSMappingServerEnabledClientEnabled();
+ driver.resetCQS();
+ Properties clientProps = getClientProperties(true, true);
+ UpgradeUtil.doNotUpgradeOnFirstConnection(clientProps);
+ setOldTimestampToInduceUpgrade = false;
+ try {
+ driver.getConnectionQueryServices(getJdbcUrl(), clientProps);
+ } catch (Exception e) {
+ assertTrue(e instanceof UpgradeRequiredException);
+ }
+ // There should be no new tables
+ assertEquals(hbaseTables, getHBaseTables());
+ // Since we set an old timestamp on purpose when creating SYSTEM:CATALOG, the second connection attempts to upgrade it
+ assertEquals(1, countUpgradeAttempts);
+ // This connection is unable to actually upgrade SYSTEM:CATALOG due to isDoNotUpgradePropSet
+ assertEquals(0, actualSysCatUpgrades);
+ Connection conn = driver.getConnectionQueryServices(getJdbcUrl(), new Properties()).connect(getJdbcUrl(), new Properties());
+ try {
+ conn.createStatement().execute(EXECUTE_UPGRADE_COMMAND);
+ // Actually upgraded SYSTEM:CATALOG
+ assertEquals(1, actualSysCatUpgrades);
+ } finally {
+ conn.close();
+ }
+ }
+
+ // Conditions: server-side namespace mapping is enabled, the first connection to the server will create only SYSTEM.CATALOG,
+ // the second connection has client-side namespace mapping enabled
+ // Expected: We will migrate SYSTEM.CATALOG to SYSTEM namespace and create all other SYSTEM:.* tables
+ @Test
+ public void testMigrateSysCatCreateOthers() throws Exception {
+ SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+ firstConnectionNSMappingServerEnabledClientDisabled();
+ driver.resetCQS();
+ Properties clientProps = getClientProperties(true, true);
+ driver.getConnectionQueryServices(getJdbcUrl(), clientProps);
+ hbaseTables = getHBaseTables();
+ assertEquals(PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES, hbaseTables);
+ // SYSTEM.CATALOG migration to the SYSTEM namespace is counted as an upgrade
+ assertEquals(1, countUpgradeAttempts);
+ }
+
+ // Conditions: server-side namespace mapping is enabled, the first connection to the server will create unmapped SYSTEM
+ // tables SYSTEM\..* whose timestamp at creation is purposefully set to be < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP.
+ // The second connection has client-side namespace mapping enabled and system table to system namespace mapping enabled
+ // Expected: We will migrate all SYSTEM\..* tables to the SYSTEM namespace and also upgrade SYSTEM:CATALOG
+ @Test
+ public void testMigrateToSystemNamespaceAndUpgradeSysCat() throws Exception {
+ setOldTimestampToInduceUpgrade = true;
+ SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+ firstConnectionNSMappingServerEnabledClientEnabledMappingDisabled();
+ driver.resetCQS();
+ setOldTimestampToInduceUpgrade = false;
+ Properties clientProps = getClientProperties(true, true);
+ driver.getConnectionQueryServices(getJdbcUrl(), clientProps);
+ hbaseTables = getHBaseTables();
+ assertEquals(PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES, hbaseTables);
+ assertEquals(1, countUpgradeAttempts);
+ assertEquals(1, actualSysCatUpgrades);
+ }
+
+ // Conditions: server-side namespace mapping is enabled, the first connection to the server will create all namespace
+ // mapped SYSTEM tables i.e. SYSTEM:.*, the second connection has client-side namespace mapping disabled
+ // Expected: Throw Inconsistent namespace mapping exception from ensureTableCreated
+ @Test
+ public void testTablesExistInconsistentNSMappingFails() throws Exception {
+ SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+ firstConnectionNSMappingServerEnabledClientEnabled();
+ driver.resetCQS();
+ Properties clientProps = getClientProperties(false, false);
+ try {
+ driver.getConnectionQueryServices(getJdbcUrl(), clientProps);
+ fail("Client should not be able to connect to cluster with inconsistent client-server namespace mapping properties");
+ } catch (SQLException sqlE) {
+ assertEquals(SQLExceptionCode.INCONSISTENT_NAMESPACE_MAPPING_PROPERTIES.getErrorCode(), sqlE.getErrorCode());
+ }
+ hbaseTables = getHBaseTables();
+ assertEquals(PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES, hbaseTables);
+ assertEquals(0, countUpgradeAttempts);
+ }
+
+ // Conditions: server-side namespace mapping is enabled, the first connection to the server will create only SYSTEM.CATALOG,
+ // the second connection has client-side namespace mapping disabled
+ // Expected: Throw Inconsistent namespace mapping exception when you check client-server compatibility
+ @Test
+ public void testUnmappedSysCatExistsInconsistentNSMappingFails() throws Exception {
+ SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+ firstConnectionNSMappingServerEnabledClientDisabled();
+ driver.resetCQS();
+ Properties clientProps = getClientProperties(false, false);
+ try {
+ driver.getConnectionQueryServices(getJdbcUrl(), clientProps);
+ fail("Client should not be able to connect to cluster with inconsistent client-server namespace mapping properties");
+ } catch (SQLException sqlE) {
+ assertEquals(SQLExceptionCode.INCONSISTENT_NAMESPACE_MAPPING_PROPERTIES.getErrorCode(), sqlE.getErrorCode());
+ }
+ hbaseTables = getHBaseTables();
+ assertTrue(hbaseTables.contains(PHOENIX_SYSTEM_CATALOG));
+ assertTrue(hbaseTables.size() == 1);
+ assertEquals(0, countUpgradeAttempts);
+ }
+
+ // Conditions: server-side namespace mapping is disabled, the first connection to the server will create all unmapped
+ // SYSTEM tables i.e. SYSTEM\..*, the second connection has client-side namespace mapping enabled
+ // Expected: Throw Inconsistent namespace mapping exception when you check client-server compatibility
+ @Test
+ public void testSysTablesExistInconsistentNSMappingFails() throws Exception {
+ SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+ firstConnectionNSMappingServerDisabledClientDisabled();
+ driver.resetCQS();
+ Properties clientProps = getClientProperties(true, true);
+ try {
+ driver.getConnectionQueryServices(getJdbcUrl(), clientProps);
+ fail("Client should not be able to connect to cluster with inconsistent client-server namespace mapping properties");
+ } catch (SQLException sqlE) {
+ assertEquals(SQLExceptionCode.INCONSISTENT_NAMESPACE_MAPPING_PROPERTIES.getErrorCode(), sqlE.getErrorCode());
+ }
+ hbaseTables = getHBaseTables();
+ assertEquals(PHOENIX_SYSTEM_TABLES, hbaseTables);
+ assertEquals(0, countUpgradeAttempts);
+ }
+
+ // Conditions: server-side namespace mapping is disabled, the first connection to the server will create only SYSTEM:CATALOG
+ // and the second connection has client-side namespace mapping enabled
+ // Expected: Throw Inconsistent namespace mapping exception when you check client-server compatibility
+ @Test
+ public void testMappedSysCatExistsInconsistentNSMappingFails() throws Exception {
+ SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+ firstConnectionNSMappingServerDisabledClientEnabled();
+ driver.resetCQS();
+ Properties clientProps = getClientProperties(true, true);
+ try{
+ driver.getConnectionQueryServices(getJdbcUrl(), clientProps);
+ fail("Client should not be able to connect to cluster with inconsistent client-server namespace mapping properties");
+ } catch (SQLException sqlE) {
+ assertEquals(SQLExceptionCode.INCONSISTENT_NAMESPACE_MAPPING_PROPERTIES.getErrorCode(), sqlE.getErrorCode());
+ }
+ hbaseTables = getHBaseTables();
+ assertTrue(hbaseTables.contains(PHOENIX_NAMESPACE_MAPPED_SYSTEM_CATALOG));
+ assertTrue(hbaseTables.size() == 1);
+ assertEquals(0, countUpgradeAttempts);
+ }
+
+ // Conditions: server-side namespace mapping is disabled, the first connection to the server will create all SYSTEM\..*
+ // tables and the second connection has client-side namespace mapping disabled
+ // Expected: All SYSTEM\..* tables exist and no upgrade is required
+ @Test
+ public void testNSMappingDisabledNoUpgradeRequired() throws Exception {
+ SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+ firstConnectionNSMappingServerDisabledClientDisabled();
+ driver.resetCQS();
+ Properties clientProps = getClientProperties(false, false);
+ driver.getConnectionQueryServices(getJdbcUrl(), clientProps);
+ hbaseTables = getHBaseTables();
+ assertEquals(PHOENIX_SYSTEM_TABLES, hbaseTables);
+ assertEquals(0, countUpgradeAttempts);
+ }
+
+ // Conditions: server-side namespace mapping is disabled, the first connection to the server will create only SYSTEM:CATALOG
+ // and the second connection has client-side namespace mapping disabled
+ // Expected: The second connection should fail with Inconsistent namespace mapping exception
+ @Test
+ public void testClientNSMappingDisabledConnectionFails() throws Exception {
+ SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+ firstConnectionNSMappingServerDisabledClientEnabled();
+ driver.resetCQS();
+ Properties clientProps = getClientProperties(false, false);
+ try{
+ driver.getConnectionQueryServices(getJdbcUrl(), clientProps);
+ fail("Client should not be able to connect to cluster with inconsistent client-server namespace mapping properties");
+ } catch (SQLException sqlE) {
+ assertEquals(SQLExceptionCode.INCONSISTENT_NAMESPACE_MAPPING_PROPERTIES.getErrorCode(), sqlE.getErrorCode());
+ }
+ hbaseTables = getHBaseTables();
+ assertTrue(hbaseTables.contains(PHOENIX_NAMESPACE_MAPPED_SYSTEM_CATALOG));
+ assertTrue(hbaseTables.size() == 1);
+ assertEquals(0, countUpgradeAttempts);
+ }
+
+ // Conditions: The first connection creates all SYSTEM tables via "EXECUTE UPGRADE" since auto-upgrade is disabled
+ // and the same client alters HBase metadata for SYSTEM.CATALOG
+ // Expected: Another client connection (with a new ConnectionQueryServices instance) made to the server does not
+ // revert the metadata change
+ @Test
+ public void testMetadataAlterRemainsAutoUpgradeDisabled() throws Exception {
+ SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver = firstConnectionAutoUpgradeToggle(false);
+ assertEquals(Integer.parseInt(MODIFIED_MAX_VERSIONS), verifyModificationTableMetadata(driver, PHOENIX_SYSTEM_CATALOG));
+ }
+
+ // Conditions: The first connection creates all SYSTEM tables (auto-upgrade is enabled) and the same client alters
+ // HBase metadata for SYSTEM.CATALOG
+ // Expected: Another client connection (with a new ConnectionQueryServices instance) made to the server does not
+ // revert the metadata change
+ @Test
+ public void testMetadataAlterRemainsAutoUpgradeEnabled() throws Exception {
+ SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver = firstConnectionAutoUpgradeToggle(true);
+ assertEquals(Integer.parseInt(MODIFIED_MAX_VERSIONS), verifyModificationTableMetadata(driver, PHOENIX_SYSTEM_CATALOG));
+ }
+
+ /**
+ * Return all created HBase tables
+ * @return Set of HBase table name strings
+ * @throws IOException
+ */
+ private Set<String> getHBaseTables() throws IOException {
+ Set<String> tables = new HashSet<>();
+ for (TableName tn : testUtil.getHBaseAdmin().listTableNames()) {
+ tables.add(tn.getNameAsString());
+ }
+ return tables;
+ }
+
+ /**
+ * Alter the table metadata and return modified value
+ * @param driver
+ * @param tableName
+ * @return value of VERSIONS option for the table
+ * @throws Exception
+ */
+ private int verifyModificationTableMetadata(PhoenixSysCatCreationTestingDriver driver, String tableName) throws Exception {
+ // Modify table metadata
+ Connection conn = driver.getConnectionQueryServices(getJdbcUrl(), new Properties()).connect(getJdbcUrl(), new Properties());
+ conn.createStatement().execute("ALTER TABLE " + tableName + " SET VERSIONS = " + MODIFIED_MAX_VERSIONS);
+
+ // Connect via a client that creates a new ConnectionQueryServices instance
+ driver.resetCQS();
+ driver.getConnectionQueryServices(getJdbcUrl(), new Properties()).connect(getJdbcUrl(), new Properties());
+ HTableDescriptor descriptor = testUtil.getHBaseAdmin().getTableDescriptor(TableName.valueOf(tableName));
+ return descriptor.getFamily(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES).getMaxVersions();
+ }
+
+ /**
+ * Start the mini-cluster with server-side namespace mapping property specified
+ * @param isNamespaceMappingEnabled
+ * @throws Exception
+ */
+ private void startMiniClusterWithToggleNamespaceMapping(String isNamespaceMappingEnabled) throws Exception {
+ testUtil = new HBaseTestingUtility();
+ Configuration conf = testUtil.getConfiguration();
+ conf.set(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, isNamespaceMappingEnabled);
+ // Avoid multiple clusters trying to bind to the master's info port (16010)
+ conf.setInt(HConstants.MASTER_INFO_PORT, -1);
+ testUtil.startMiniCluster(1);
+ }
+
+ /**
+ * Get the connection string for the mini-cluster
+ * @return Phoenix connection string
+ */
+ private String getJdbcUrl() {
+ return "jdbc:phoenix:localhost:" + testUtil.getZkCluster().getClientPort() + ":/hbase";
+ }
+
+ /**
+ * Set namespace mapping related properties for the client connection
+ * @param nsMappingEnabled
+ * @param systemTableMappingEnabled
+ * @return Properties object
+ */
+ private Properties getClientProperties(boolean nsMappingEnabled, boolean systemTableMappingEnabled) {
+ Properties clientProps = new Properties();
+ clientProps.setProperty(QueryServices.IS_NAMESPACE_MAPPING_ENABLED, Boolean.valueOf(nsMappingEnabled).toString());
+ clientProps.setProperty(QueryServices.IS_SYSTEM_TABLE_MAPPED_TO_NAMESPACE, Boolean.valueOf(systemTableMappingEnabled).toString());
+ return clientProps;
+ }
+
+ /**
+ * Initiate the first connection to the server with provided auto-upgrade property
+ * @param isAutoUpgradeEnabled
+ * @return Phoenix JDBC driver
+ * @throws Exception
+ */
+ private SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver firstConnectionAutoUpgradeToggle(boolean isAutoUpgradeEnabled)
+ throws Exception {
+ if (isAutoUpgradeEnabled) {
+ return firstConnectionNSMappingServerDisabledClientDisabled();
+ }
+ return firstConnectionAutoUpgradeDisabled();
+ }
+
+ // Conditions: isAutoUpgradeEnabled is false
+ // Expected: We do not create SYSTEM.CATALOG even if this is the first connection to the server. Later, when we manually
+ // run "EXECUTE UPGRADE", we create SYSTEM tables
+ private SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver firstConnectionAutoUpgradeDisabled() throws Exception {
+ startMiniClusterWithToggleNamespaceMapping(Boolean.FALSE.toString());
+ Map<String, String> props = new HashMap<>();
+ // Note that the isAutoUpgradeEnabled property is set when instantiating connection query services, not during init
+ props.put(QueryServices.AUTO_UPGRADE_ENABLED, Boolean.FALSE.toString());
+ ReadOnlyProps readOnlyProps = new ReadOnlyProps(props);
+ SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+ new SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver(readOnlyProps);
+ try {
+ driver.getConnectionQueryServices(getJdbcUrl(), new Properties());
+ fail("Client should not be able to create SYSTEM.CATALOG since we set the isAutoUpgradeEnabled property to false");
+ } catch (Exception e) {
+ assertTrue(e instanceof UpgradeRequiredException);
+ }
+ hbaseTables = getHBaseTables();
+ assertFalse(hbaseTables.contains(PHOENIX_SYSTEM_CATALOG) || hbaseTables.contains(PHOENIX_NAMESPACE_MAPPED_SYSTEM_CATALOG));
+ assertTrue(hbaseTables.size() == 0);
+ assertEquals(1, countUpgradeAttempts);
+
+ // We use the same ConnectionQueryServices instance to run "EXECUTE UPGRADE"
+ Connection conn = driver.getConnectionQueryServices(getJdbcUrl(), new Properties()).connect(getJdbcUrl(), new Properties());
+ try {
+ conn.createStatement().execute(EXECUTE_UPGRADE_COMMAND);
+ } finally {
+ conn.close();
+ }
+ hbaseTables = getHBaseTables();
+ assertEquals(PHOENIX_SYSTEM_TABLES, hbaseTables);
+ return driver;
+ }
+
+ // Conditions: server-side namespace mapping is enabled, client-side namespace mapping is enabled and system tables
+ // are to be mapped to the SYSTEM namespace.
+ // Expected: If this is the first connection to the server, we should be able to create all namespace mapped system tables i.e. SYSTEM:.*
+ private SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver firstConnectionNSMappingServerEnabledClientEnabled()
+ throws Exception {
+ startMiniClusterWithToggleNamespaceMapping(Boolean.TRUE.toString());
+ Properties clientProps = getClientProperties(true, true);
+ SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+ new SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver(ReadOnlyProps.EMPTY_PROPS);
+ driver.getConnectionQueryServices(getJdbcUrl(), clientProps);
+ hbaseTables = getHBaseTables();
+ assertEquals(PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES, hbaseTables);
+ assertEquals(0, countUpgradeAttempts);
+ return driver;
+ }
+
+ // Conditions: server-side namespace mapping is enabled, client-side namespace mapping is enabled, but mapping
+ // SYSTEM tables to the SYSTEM namespace is disabled
+ // Expected: If this is the first connection to the server, we will create unmapped SYSTEM tables i.e. SYSTEM\..*
+ private SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver firstConnectionNSMappingServerEnabledClientEnabledMappingDisabled()
+ throws Exception {
+ startMiniClusterWithToggleNamespaceMapping(Boolean.TRUE.toString());
+ // client-side namespace mapping is enabled, but mapping SYSTEM tables to SYSTEM namespace is not
+ Properties clientProps = getClientProperties(true, false);
+ SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+ new SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver(ReadOnlyProps.EMPTY_PROPS);
+ driver.getConnectionQueryServices(getJdbcUrl(), clientProps);
+ hbaseTables = getHBaseTables();
+ assertEquals(PHOENIX_SYSTEM_TABLES, hbaseTables);
+ assertEquals(0, countUpgradeAttempts);
+ return driver;
+ }
+
+ // Conditions: server-side namespace mapping is enabled, client-side namespace mapping is disabled
+ // Expected: Since this is the first connection to the server, we will create SYSTEM.CATALOG but immediately
+ // throw an exception for inconsistent namespace mapping
+ private SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver firstConnectionNSMappingServerEnabledClientDisabled()
+ throws Exception {
+ startMiniClusterWithToggleNamespaceMapping(Boolean.TRUE.toString());
+ Properties clientProps = getClientProperties(false, false);
+ SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+ new SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver(ReadOnlyProps.EMPTY_PROPS);
+ try {
+ driver.getConnectionQueryServices(getJdbcUrl(), clientProps);
+ fail("Client should not be able to connect to cluster with inconsistent client-server namespace mapping properties");
+ } catch (SQLException sqlE) {
+ assertEquals(SQLExceptionCode.INCONSISTENT_NAMESPACE_MAPPING_PROPERTIES.getErrorCode(), sqlE.getErrorCode());
+ }
+ hbaseTables = getHBaseTables();
+ assertTrue(hbaseTables.contains(PHOENIX_SYSTEM_CATALOG));
+ assertTrue(hbaseTables.size() == 1);
+ assertEquals(0, countUpgradeAttempts);
+ return driver;
+ }
+
+ // Conditions: server-side namespace mapping is disabled, client-side namespace mapping is enabled
+ // Expected: Since this is the first connection to the server, we will create the SYSTEM namespace and create
+ // SYSTEM:CATALOG and then immediately throw an exception for inconsistent namespace mapping
+ private SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver firstConnectionNSMappingServerDisabledClientEnabled()
+ throws Exception {
+ startMiniClusterWithToggleNamespaceMapping(Boolean.FALSE.toString());
+ Properties clientProps = getClientProperties(true, true);
+ SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+ new SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver(ReadOnlyProps.EMPTY_PROPS);
+ try {
+ driver.getConnectionQueryServices(getJdbcUrl(), clientProps);
+ fail("Client should not be able to connect to cluster with inconsistent client-server namespace mapping properties");
+ } catch (SQLException sqlE) {
+ assertEquals(SQLExceptionCode.INCONSISTENT_NAMESPACE_MAPPING_PROPERTIES.getErrorCode(), sqlE.getErrorCode());
+ }
+ hbaseTables = getHBaseTables();
+ assertTrue(hbaseTables.contains(PHOENIX_NAMESPACE_MAPPED_SYSTEM_CATALOG));
+ assertTrue(hbaseTables.size() == 1);
+ assertEquals(0, countUpgradeAttempts);
+ return driver;
+ }
+
+ // Conditions: server-side namespace mapping is disabled, client-side namespace mapping is disabled
+ // Expected: Since this is the first connection to the server and auto-upgrade is enabled by default,
+ // we will create all SYSTEM\..* tables
+ private SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver firstConnectionNSMappingServerDisabledClientDisabled()
+ throws Exception {
+ startMiniClusterWithToggleNamespaceMapping(Boolean.FALSE.toString());
+ Properties clientProps = getClientProperties(false, false);
+ SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver driver =
+ new SystemCatalogCreationOnConnectionIT.PhoenixSysCatCreationTestingDriver(ReadOnlyProps.EMPTY_PROPS);
+ driver.getConnectionQueryServices(getJdbcUrl(), clientProps);
+ hbaseTables = getHBaseTables();
+ assertEquals(PHOENIX_SYSTEM_TABLES, hbaseTables);
+ assertEquals(0, countUpgradeAttempts);
+ return driver;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/87fdda8b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index a2d008b..34218d5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@ -3615,6 +3615,27 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
}
long version = MetaDataUtil.encodeVersion(env.getHBaseVersion(), config);
+ PTable systemCatalog = null;
+ byte[] tableKey =
+ SchemaUtil.getTableKey(ByteUtil.EMPTY_BYTE_ARRAY,
+ PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA_BYTES,
+ PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE_BYTES);
+ ImmutableBytesPtr cacheKey = new ImmutableBytesPtr(tableKey);
+ try {
+ systemCatalog = loadTable(env, tableKey, cacheKey, MIN_SYSTEM_TABLE_TIMESTAMP,
+ HConstants.LATEST_TIMESTAMP, request.getClientVersion());
+ } catch (Throwable t) {
+ logger.error("loading system catalog table inside getVersion failed", t);
+ ProtobufUtil.setControllerException(controller,
+ ServerUtil.createIOException(
+ SchemaUtil.getPhysicalTableName(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES,
+ isTablesMappingEnabled).toString(), t));
+ }
+ // In case this is the first connection, system catalog does not exist, and so we don't
+ // set the optional system catalog timestamp.
+ if (systemCatalog != null) {
+ builder.setSystemCatalogTimestamp(systemCatalog.getTimeStamp());
+ }
builder.setVersion(version);
done.run(builder.build());
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/87fdda8b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
index a71ce0c..26f8198 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
@@ -448,6 +448,10 @@ public abstract class MetaDataProtocol extends MetaDataService {
return iterator.next();
}
+ public static long getPriorUpgradeVersion() {
+ return TIMESTAMP_VERSION_MAP.lowerKey(TIMESTAMP_VERSION_MAP.lastKey());
+ }
+
public static String getVersion(long serverTimestamp) {
/*
* It is possible that when clients are trying to run upgrades concurrently, we could be at an intermediate
http://git-wip-us.apache.org/repos/asf/phoenix/blob/87fdda8b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/MetaDataProtos.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/MetaDataProtos.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/MetaDataProtos.java
index e041abd..8119c6e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/MetaDataProtos.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/generated/MetaDataProtos.java
@@ -14117,6 +14117,16 @@ public final class MetaDataProtos {
* <code>required int64 version = 1;</code>
*/
long getVersion();
+
+ // optional int64 systemCatalogTimestamp = 2;
+ /**
+ * <code>optional int64 systemCatalogTimestamp = 2;</code>
+ */
+ boolean hasSystemCatalogTimestamp();
+ /**
+ * <code>optional int64 systemCatalogTimestamp = 2;</code>
+ */
+ long getSystemCatalogTimestamp();
}
/**
* Protobuf type {@code GetVersionResponse}
@@ -14174,6 +14184,11 @@ public final class MetaDataProtos {
version_ = input.readInt64();
break;
}
+ case 16: {
+ bitField0_ |= 0x00000002;
+ systemCatalogTimestamp_ = input.readInt64();
+ break;
+ }
}
}
} catch (com.google.protobuf.InvalidProtocolBufferException e) {
@@ -14230,8 +14245,25 @@ public final class MetaDataProtos {
return version_;
}
+ // optional int64 systemCatalogTimestamp = 2;
+ public static final int SYSTEMCATALOGTIMESTAMP_FIELD_NUMBER = 2;
+ private long systemCatalogTimestamp_;
+ /**
+ * <code>optional int64 systemCatalogTimestamp = 2;</code>
+ */
+ public boolean hasSystemCatalogTimestamp() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ /**
+ * <code>optional int64 systemCatalogTimestamp = 2;</code>
+ */
+ public long getSystemCatalogTimestamp() {
+ return systemCatalogTimestamp_;
+ }
+
private void initFields() {
version_ = 0L;
+ systemCatalogTimestamp_ = 0L;
}
private byte memoizedIsInitialized = -1;
public final boolean isInitialized() {
@@ -14252,6 +14284,9 @@ public final class MetaDataProtos {
if (((bitField0_ & 0x00000001) == 0x00000001)) {
output.writeInt64(1, version_);
}
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ output.writeInt64(2, systemCatalogTimestamp_);
+ }
getUnknownFields().writeTo(output);
}
@@ -14265,6 +14300,10 @@ public final class MetaDataProtos {
size += com.google.protobuf.CodedOutputStream
.computeInt64Size(1, version_);
}
+ if (((bitField0_ & 0x00000002) == 0x00000002)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeInt64Size(2, systemCatalogTimestamp_);
+ }
size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
return size;
@@ -14293,6 +14332,11 @@ public final class MetaDataProtos {
result = result && (getVersion()
== other.getVersion());
}
+ result = result && (hasSystemCatalogTimestamp() == other.hasSystemCatalogTimestamp());
+ if (hasSystemCatalogTimestamp()) {
+ result = result && (getSystemCatalogTimestamp()
+ == other.getSystemCatalogTimestamp());
+ }
result = result &&
getUnknownFields().equals(other.getUnknownFields());
return result;
@@ -14310,6 +14354,10 @@ public final class MetaDataProtos {
hash = (37 * hash) + VERSION_FIELD_NUMBER;
hash = (53 * hash) + hashLong(getVersion());
}
+ if (hasSystemCatalogTimestamp()) {
+ hash = (37 * hash) + SYSTEMCATALOGTIMESTAMP_FIELD_NUMBER;
+ hash = (53 * hash) + hashLong(getSystemCatalogTimestamp());
+ }
hash = (29 * hash) + getUnknownFields().hashCode();
memoizedHashCode = hash;
return hash;
@@ -14421,6 +14469,8 @@ public final class MetaDataProtos {
super.clear();
version_ = 0L;
bitField0_ = (bitField0_ & ~0x00000001);
+ systemCatalogTimestamp_ = 0L;
+ bitField0_ = (bitField0_ & ~0x00000002);
return this;
}
@@ -14453,6 +14503,10 @@ public final class MetaDataProtos {
to_bitField0_ |= 0x00000001;
}
result.version_ = version_;
+ if (((from_bitField0_ & 0x00000002) == 0x00000002)) {
+ to_bitField0_ |= 0x00000002;
+ }
+ result.systemCatalogTimestamp_ = systemCatalogTimestamp_;
result.bitField0_ = to_bitField0_;
onBuilt();
return result;
@@ -14472,6 +14526,9 @@ public final class MetaDataProtos {
if (other.hasVersion()) {
setVersion(other.getVersion());
}
+ if (other.hasSystemCatalogTimestamp()) {
+ setSystemCatalogTimestamp(other.getSystemCatalogTimestamp());
+ }
this.mergeUnknownFields(other.getUnknownFields());
return this;
}
@@ -14536,6 +14593,39 @@ public final class MetaDataProtos {
return this;
}
+ // optional int64 systemCatalogTimestamp = 2;
+ private long systemCatalogTimestamp_ ;
+ /**
+ * <code>optional int64 systemCatalogTimestamp = 2;</code>
+ */
+ public boolean hasSystemCatalogTimestamp() {
+ return ((bitField0_ & 0x00000002) == 0x00000002);
+ }
+ /**
+ * <code>optional int64 systemCatalogTimestamp = 2;</code>
+ */
+ public long getSystemCatalogTimestamp() {
+ return systemCatalogTimestamp_;
+ }
+ /**
+ * <code>optional int64 systemCatalogTimestamp = 2;</code>
+ */
+ public Builder setSystemCatalogTimestamp(long value) {
+ bitField0_ |= 0x00000002;
+ systemCatalogTimestamp_ = value;
+ onChanged();
+ return this;
+ }
+ /**
+ * <code>optional int64 systemCatalogTimestamp = 2;</code>
+ */
+ public Builder clearSystemCatalogTimestamp() {
+ bitField0_ = (bitField0_ & ~0x00000002);
+ systemCatalogTimestamp_ = 0L;
+ onChanged();
+ return this;
+ }
+
// @@protoc_insertion_point(builder_scope:GetVersionResponse)
}
@@ -17103,51 +17193,52 @@ public final class MetaDataProtos {
"cheRequest\022\025\n\rclientVersion\030\001 \001(\005\"*\n\022Cle" +
"arCacheResponse\022\024\n\014unfreedBytes\030\001 \001(\003\"*\n" +
"\021GetVersionRequest\022\025\n\rclientVersion\030\001 \001(" +
- "\005\"%\n\022GetVersionResponse\022\017\n\007version\030\001 \002(\003" +
- "\"\205\001\n\032ClearTableFromCacheRequest\022\020\n\010tenan",
- "tId\030\001 \002(\014\022\022\n\nschemaName\030\002 \002(\014\022\021\n\ttableNa" +
- "me\030\003 \002(\014\022\027\n\017clientTimestamp\030\004 \002(\003\022\025\n\rcli" +
- "entVersion\030\005 \001(\005\"\035\n\033ClearTableFromCacheR" +
- "esponse*\365\004\n\014MutationCode\022\030\n\024TABLE_ALREAD" +
- "Y_EXISTS\020\000\022\023\n\017TABLE_NOT_FOUND\020\001\022\024\n\020COLUM" +
- "N_NOT_FOUND\020\002\022\031\n\025COLUMN_ALREADY_EXISTS\020\003" +
- "\022\035\n\031CONCURRENT_TABLE_MUTATION\020\004\022\027\n\023TABLE" +
- "_NOT_IN_REGION\020\005\022\025\n\021NEWER_TABLE_FOUND\020\006\022" +
- "\034\n\030UNALLOWED_TABLE_MUTATION\020\007\022\021\n\rNO_PK_C" +
- "OLUMNS\020\010\022\032\n\026PARENT_TABLE_NOT_FOUND\020\t\022\033\n\027",
- "FUNCTION_ALREADY_EXISTS\020\n\022\026\n\022FUNCTION_NO" +
- "T_FOUND\020\013\022\030\n\024NEWER_FUNCTION_FOUND\020\014\022\032\n\026F" +
- "UNCTION_NOT_IN_REGION\020\r\022\031\n\025SCHEMA_ALREAD" +
- "Y_EXISTS\020\016\022\026\n\022NEWER_SCHEMA_FOUND\020\017\022\024\n\020SC" +
- "HEMA_NOT_FOUND\020\020\022\030\n\024SCHEMA_NOT_IN_REGION" +
- "\020\021\022\032\n\026TABLES_EXIST_ON_SCHEMA\020\022\022\035\n\031UNALLO" +
- "WED_SCHEMA_MUTATION\020\023\022%\n!AUTO_PARTITION_" +
- "SEQUENCE_NOT_FOUND\020\024\022#\n\037CANNOT_COERCE_AU" +
- "TO_PARTITION_ID\020\025\022\024\n\020TOO_MANY_INDEXES\020\0262" +
- "\345\006\n\017MetaDataService\022/\n\010getTable\022\020.GetTab",
- "leRequest\032\021.MetaDataResponse\0227\n\014getFunct" +
- "ions\022\024.GetFunctionsRequest\032\021.MetaDataRes" +
- "ponse\0221\n\tgetSchema\022\021.GetSchemaRequest\032\021." +
- "MetaDataResponse\0225\n\013createTable\022\023.Create" +
- "TableRequest\032\021.MetaDataResponse\022;\n\016creat" +
- "eFunction\022\026.CreateFunctionRequest\032\021.Meta" +
- "DataResponse\0227\n\014createSchema\022\024.CreateSch" +
- "emaRequest\032\021.MetaDataResponse\0221\n\tdropTab" +
- "le\022\021.DropTableRequest\032\021.MetaDataResponse" +
- "\0223\n\ndropSchema\022\022.DropSchemaRequest\032\021.Met",
- "aDataResponse\0227\n\014dropFunction\022\024.DropFunc" +
- "tionRequest\032\021.MetaDataResponse\0221\n\taddCol" +
- "umn\022\021.AddColumnRequest\032\021.MetaDataRespons" +
- "e\0223\n\ndropColumn\022\022.DropColumnRequest\032\021.Me" +
- "taDataResponse\022?\n\020updateIndexState\022\030.Upd" +
- "ateIndexStateRequest\032\021.MetaDataResponse\022" +
- "5\n\nclearCache\022\022.ClearCacheRequest\032\023.Clea" +
- "rCacheResponse\0225\n\ngetVersion\022\022.GetVersio" +
- "nRequest\032\023.GetVersionResponse\022P\n\023clearTa" +
- "bleFromCache\022\033.ClearTableFromCacheReques",
- "t\032\034.ClearTableFromCacheResponseBB\n(org.a" +
- "pache.phoenix.coprocessor.generatedB\016Met" +
- "aDataProtosH\001\210\001\001\240\001\001"
+ "\005\"E\n\022GetVersionResponse\022\017\n\007version\030\001 \002(\003" +
+ "\022\036\n\026systemCatalogTimestamp\030\002 \001(\003\"\205\001\n\032Cle",
+ "arTableFromCacheRequest\022\020\n\010tenantId\030\001 \002(" +
+ "\014\022\022\n\nschemaName\030\002 \002(\014\022\021\n\ttableName\030\003 \002(\014" +
+ "\022\027\n\017clientTimestamp\030\004 \002(\003\022\025\n\rclientVersi" +
+ "on\030\005 \001(\005\"\035\n\033ClearTableFromCacheResponse*" +
+ "\365\004\n\014MutationCode\022\030\n\024TABLE_ALREADY_EXISTS" +
+ "\020\000\022\023\n\017TABLE_NOT_FOUND\020\001\022\024\n\020COLUMN_NOT_FO" +
+ "UND\020\002\022\031\n\025COLUMN_ALREADY_EXISTS\020\003\022\035\n\031CONC" +
+ "URRENT_TABLE_MUTATION\020\004\022\027\n\023TABLE_NOT_IN_" +
+ "REGION\020\005\022\025\n\021NEWER_TABLE_FOUND\020\006\022\034\n\030UNALL" +
+ "OWED_TABLE_MUTATION\020\007\022\021\n\rNO_PK_COLUMNS\020\010",
+ "\022\032\n\026PARENT_TABLE_NOT_FOUND\020\t\022\033\n\027FUNCTION" +
+ "_ALREADY_EXISTS\020\n\022\026\n\022FUNCTION_NOT_FOUND\020" +
+ "\013\022\030\n\024NEWER_FUNCTION_FOUND\020\014\022\032\n\026FUNCTION_" +
+ "NOT_IN_REGION\020\r\022\031\n\025SCHEMA_ALREADY_EXISTS" +
+ "\020\016\022\026\n\022NEWER_SCHEMA_FOUND\020\017\022\024\n\020SCHEMA_NOT" +
+ "_FOUND\020\020\022\030\n\024SCHEMA_NOT_IN_REGION\020\021\022\032\n\026TA" +
+ "BLES_EXIST_ON_SCHEMA\020\022\022\035\n\031UNALLOWED_SCHE" +
+ "MA_MUTATION\020\023\022%\n!AUTO_PARTITION_SEQUENCE" +
+ "_NOT_FOUND\020\024\022#\n\037CANNOT_COERCE_AUTO_PARTI" +
+ "TION_ID\020\025\022\024\n\020TOO_MANY_INDEXES\020\0262\345\006\n\017Meta",
+ "DataService\022/\n\010getTable\022\020.GetTableReques" +
+ "t\032\021.MetaDataResponse\0227\n\014getFunctions\022\024.G" +
+ "etFunctionsRequest\032\021.MetaDataResponse\0221\n" +
+ "\tgetSchema\022\021.GetSchemaRequest\032\021.MetaData" +
+ "Response\0225\n\013createTable\022\023.CreateTableReq" +
+ "uest\032\021.MetaDataResponse\022;\n\016createFunctio" +
+ "n\022\026.CreateFunctionRequest\032\021.MetaDataResp" +
+ "onse\0227\n\014createSchema\022\024.CreateSchemaReque" +
+ "st\032\021.MetaDataResponse\0221\n\tdropTable\022\021.Dro" +
+ "pTableRequest\032\021.MetaDataResponse\0223\n\ndrop",
+ "Schema\022\022.DropSchemaRequest\032\021.MetaDataRes" +
+ "ponse\0227\n\014dropFunction\022\024.DropFunctionRequ" +
+ "est\032\021.MetaDataResponse\0221\n\taddColumn\022\021.Ad" +
+ "dColumnRequest\032\021.MetaDataResponse\0223\n\ndro" +
+ "pColumn\022\022.DropColumnRequest\032\021.MetaDataRe" +
+ "sponse\022?\n\020updateIndexState\022\030.UpdateIndex" +
+ "StateRequest\032\021.MetaDataResponse\0225\n\nclear" +
+ "Cache\022\022.ClearCacheRequest\032\023.ClearCacheRe" +
+ "sponse\0225\n\ngetVersion\022\022.GetVersionRequest" +
+ "\032\023.GetVersionResponse\022P\n\023clearTableFromC",
+ "ache\022\033.ClearTableFromCacheRequest\032\034.Clea" +
+ "rTableFromCacheResponseBB\n(org.apache.ph" +
+ "oenix.coprocessor.generatedB\016MetaDataPro" +
+ "tosH\001\210\001\001\240\001\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -17261,7 +17352,7 @@ public final class MetaDataProtos {
internal_static_GetVersionResponse_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_GetVersionResponse_descriptor,
- new java.lang.String[] { "Version", });
+ new java.lang.String[] { "Version", "SystemCatalogTimestamp", });
internal_static_ClearTableFromCacheRequest_descriptor =
getDescriptor().getMessageTypes().get(18);
internal_static_ClearTableFromCacheRequest_fieldAccessorTable = new
http://git-wip-us.apache.org/repos/asf/phoenix/blob/87fdda8b/phoenix-core/src/main/java/org/apache/phoenix/exception/UpgradeRequiredException.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/exception/UpgradeRequiredException.java b/phoenix-core/src/main/java/org/apache/phoenix/exception/UpgradeRequiredException.java
index 9352a50..6c9706b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/exception/UpgradeRequiredException.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/exception/UpgradeRequiredException.java
@@ -17,12 +17,23 @@
*/
package org.apache.phoenix.exception;
+import org.apache.hadoop.hbase.HConstants;
public class UpgradeRequiredException extends RetriableUpgradeException {
+ private final long systemCatalogTimestamp;
public UpgradeRequiredException() {
+ this(HConstants.OLDEST_TIMESTAMP);
+ }
+
+ public UpgradeRequiredException(long systemCatalogTimeStamp) {
super("Operation not allowed since cluster hasn't been upgraded. Call EXECUTE UPGRADE. ",
- SQLExceptionCode.UPGRADE_REQUIRED.getSQLState(), SQLExceptionCode.UPGRADE_REQUIRED.getErrorCode());
+ SQLExceptionCode.UPGRADE_REQUIRED.getSQLState(), SQLExceptionCode.UPGRADE_REQUIRED.getErrorCode());
+ this.systemCatalogTimestamp = systemCatalogTimeStamp;
+ }
+
+ public long getSystemCatalogTimeStamp() {
+ return systemCatalogTimestamp;
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/87fdda8b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
index 0598a0a..092bfe9 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServices.java
@@ -84,7 +84,7 @@ public interface ConnectionQueryServices extends QueryServices, MetaDataMutated
public MetaDataMutationResult createTable(List<Mutation> tableMetaData, byte[] tableName, PTableType tableType,
Map<String, Object> tableProps, List<Pair<byte[], Map<String, Object>>> families, byte[][] splits,
- boolean isNamespaceMapped, boolean allocateIndexId) throws SQLException;
+ boolean isNamespaceMapped, boolean allocateIndexId, boolean isDoNotUpgradePropSet) throws SQLException;
public MetaDataMutationResult dropTable(List<Mutation> tableMetadata, PTableType tableType, boolean cascade) throws SQLException;
public MetaDataMutationResult dropFunction(List<Mutation> tableMetadata, boolean ifExists) throws SQLException;
public MetaDataMutationResult addColumn(List<Mutation> tableMetaData, PTable table, Map<String, List<Pair<String,Object>>> properties, Set<String> colFamiliesForPColumnsToBeAdded, List<PColumn> columns) throws SQLException;