You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ji...@apache.org on 2025/05/22 17:48:56 UTC
(phoenix) branch master updated: PHOENIX-7573 Backward Compatibility Issues in Phoenix TTL (#2130)
This is an automated email from the ASF dual-hosted git repository.
jisaac pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/master by this push:
new 1ea4d7b2a0 PHOENIX-7573 Backward Compatibility Issues in Phoenix TTL (#2130)
1ea4d7b2a0 is described below
commit 1ea4d7b2a0ecb718871387f77dfab416a7626fa9
Author: Jacob Isaac <ja...@gmail.com>
AuthorDate: Thu May 22 10:48:51 2025 -0700
PHOENIX-7573 Backward Compatibility Issues in Phoenix TTL (#2130)
---
.../BaseScannerRegionObserverConstants.java | 1 -
.../coprocessorclient/MetaDataProtocol.java | 1 +
.../phoenix/jdbc/PhoenixDatabaseMetaData.java | 1 +
.../phoenix/query/ConnectionQueryServicesImpl.java | 48 ++++------
.../org/apache/phoenix/query/QueryServices.java | 5 +-
.../apache/phoenix/query/QueryServicesOptions.java | 1 +
.../phoenix/schema/LiteralTTLExpression.java | 4 +-
.../org/apache/phoenix/schema/MetaDataClient.java | 35 ++++---
.../schema/tool/SchemaExtractionProcessor.java | 14 +--
.../java/org/apache/phoenix/util/MetaDataUtil.java | 58 ++++++++++++
.../java/org/apache/phoenix/util/ScanUtil.java | 27 +++---
.../java/org/apache/phoenix/util/UpgradeUtil.java | 102 ++++++++++++---------
.../java/org/apache/phoenix/util/ViewUtil.java | 1 +
.../coprocessor/BaseScannerRegionObserver.java | 14 +--
.../phoenix/coprocessor/CompactionScanner.java | 26 ++++--
.../phoenix/coprocessor/MetaDataEndpointImpl.java | 12 +++
.../phoenix/coprocessor/TTLRegionScanner.java | 31 +++++--
.../UngroupedAggregateRegionObserver.java | 4 +-
.../mapreduce/index/IndexScrutinyMapper.java | 12 ++-
.../phoenix/end2end/BaseRowKeyMatcherTestIT.java | 1 -
.../org/apache/phoenix/end2end/CreateTableIT.java | 25 ++---
.../phoenix/end2end/FlappingAlterTableIT.java | 14 +--
.../LongViewIndexDisabledBaseRowKeyMatcherIT.java | 5 +-
.../LongViewIndexEnabledBaseRowKeyMatcherIT.java | 2 +-
.../phoenix/end2end/MaxLookbackExtendedIT.java | 19 ++--
.../org/apache/phoenix/end2end/MaxLookbackIT.java | 8 +-
.../phoenix/end2end/MoveTTLDuringUpgradeIT.java | 86 ++++++++++++-----
.../apache/phoenix/end2end/PropertiesInSyncIT.java | 8 +-
.../org/apache/phoenix/end2end/SetPropertyIT.java | 40 ++++----
.../apache/phoenix/end2end/TTLAsPhoenixTTLIT.java | 102 ++++++++++-----------
.../it/java/org/apache/phoenix/end2end/TTLIT.java | 60 ++++++------
.../java/org/apache/phoenix/end2end/ViewTTLIT.java | 3 +-
.../end2end/ViewTTLWithLongViewIndexEnabledIT.java | 2 +-
.../index/IndexVerificationOutputRepositoryIT.java | 2 +-
.../index/IndexVerificationResultRepositoryIT.java | 2 +-
.../end2end/index/PartialSystemCatalogIndexIT.java | 1 -
.../apache/phoenix/schema/TTLExpressionTest.java | 9 +-
.../java/org/apache/phoenix/util/TestUtil.java | 69 ++++++++++++--
38 files changed, 530 insertions(+), 325 deletions(-)
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/coprocessorclient/BaseScannerRegionObserverConstants.java b/phoenix-core-client/src/main/java/org/apache/phoenix/coprocessorclient/BaseScannerRegionObserverConstants.java
index d834b7247e..74f6c67026 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/coprocessorclient/BaseScannerRegionObserverConstants.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/coprocessorclient/BaseScannerRegionObserverConstants.java
@@ -142,7 +142,6 @@ public class BaseScannerRegionObserverConstants {
public static final String INDEX_ROW_KEY = "_IndexRowKey";
public static final String READ_REPAIR_TRANSFORMING_TABLE = "_ReadRepairTransformingTable";
public static final String CDC_DATA_TABLE_DEF = "_CdcDataTableDef";
- public static final String IS_PHOENIX_TTL_SCAN_TABLE_SYSTEM = "_IsPhoenixScanTableSystem";
/**
* The scan attribute to provide the scan start rowkey for analyze table queries.
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/coprocessorclient/MetaDataProtocol.java b/phoenix-core-client/src/main/java/org/apache/phoenix/coprocessorclient/MetaDataProtocol.java
index 3378e11710..ff0f4d15df 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/coprocessorclient/MetaDataProtocol.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/coprocessorclient/MetaDataProtocol.java
@@ -116,6 +116,7 @@ public abstract class MetaDataProtocol extends MetaDataService {
public static final int MIN_PENDING_DISABLE_INDEX = VersionUtil.encodeVersion("4", "14", "0");
// The minimum client version that allows VARBINARY_ENCODED columns
public static final int MIN_VERSION_ALLOW_VBE_COLUMNS = VersionUtil.encodeVersion("5", "3", "0");
+ public static final int MIN_VERSION_TABLE_TTL_IN_SYSTEM_CATALOG = VersionUtil.encodeVersion("5", "3", "0");
// Version below which we should turn off essential column family.
public static final int ESSENTIAL_FAMILY_VERSION_THRESHOLD = VersionUtil.encodeVersion("0", "94", "7");
/** Version below which we fall back on the generic KeyValueBuilder */
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
index bf9675ca65..bdb74e1e74 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
@@ -397,6 +397,7 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData {
public static final byte[] TTL_BYTES = Bytes.toBytes(TTL);
public static final int TTL_NOT_DEFINED = 0;
public static final int DEFAULT_TTL = HConstants.FOREVER;
+ public static final int TTL_DEFINED_IN_TABLE_DESCRIPTOR = -1;
public static final String FOREVER_TTL = "FOREVER";
public static final String NONE_TTL = "NONE";
public static final String PHOENIX_TTL = "PHOENIX_TTL";
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index a5998750dd..6161732357 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -257,6 +257,7 @@ import org.apache.phoenix.parse.PSchema;
import org.apache.phoenix.protobuf.ProtobufUtil;
import org.apache.phoenix.schema.ColumnAlreadyExistsException;
import org.apache.phoenix.schema.ColumnFamilyNotFoundException;
+import org.apache.phoenix.schema.ConditionalTTLExpression;
import org.apache.phoenix.schema.ConnectionProperty;
import org.apache.phoenix.schema.EmptySequenceCacheException;
import org.apache.phoenix.schema.FunctionNotFoundException;
@@ -1363,11 +1364,6 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
return false;
}
- private boolean isPhoenixTTLEnabled() {
- return config.getBoolean(QueryServices.PHOENIX_TABLE_TTL_ENABLED,
- QueryServicesOptions.DEFAULT_PHOENIX_TABLE_TTL_ENABLED);
- }
-
private void addCoprocessors(byte[] tableName, TableDescriptorBuilder builder,
PTableType tableType, Map<String, Object> tableProps, TableDescriptor existingDesc,
@@ -3169,19 +3165,20 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
.setMessage("Property: " + propName).build()
.buildException();
}
- //Handle FOREVER and NONE case
- propValue = convertForeverAndNoneTTLValue(propValue, isPhoenixTTLEnabled());
- //If Phoenix level TTL is enabled we are using TTL as phoenix
- //Table level property.
- if (!isPhoenixTTLEnabled()) {
- // only literal TTL expression
- LiteralTTLExpression ttlExpr =
- (LiteralTTLExpression) TableProperty.TTL.getValue(propValue);
- newTTL = ttlExpr != null ? ttlExpr.getTTLValue() : null;
- //Even though TTL is really a HColumnProperty we treat it
- //specially. We enforce that all CFs have the same TTL.
- commonFamilyProps.put(propName, propValue);
- } else {
+ if (propValue != null) {
+ TTLExpression ttlExpr = MetaDataUtil.convertForeverAndNoneTTLValue(propValue, false);
+ // Keeping TTL value in HTable descriptor when expression is a Literal Expression
+ if ((ttlExpr instanceof LiteralTTLExpression ) && (table.getType() != PTableType.VIEW)) {
+ // only literal TTL expression
+ newTTL = ((LiteralTTLExpression) ttlExpr).getTTLValue();
+ //Even though TTL is really a HColumnProperty we treat it
+ //specially. We enforce that all CFs have the same TTL.
+ commonFamilyProps.put(propName, newTTL);
+ } else if (ttlExpr instanceof ConditionalTTLExpression && table.getType() == PTableType.TABLE) {
+ // Resetting the TTL value in HTable descriptor to FOREVER when expression is a Conditional Expression
+ newTTL = HConstants.FOREVER;
+ commonFamilyProps.put(propName, newTTL);
+ }
//Setting this here just to check if we need to throw Exception
//for Transaction's SET_TTL Feature.
newPhoenixTTL = (TTLExpression) TableProperty.TTL.getValue(propValue);
@@ -3494,19 +3491,6 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
tableDesc.getColumnFamily(SchemaUtil.getEmptyColumnFamily(table)).getTimeToLive();
}
- public static Object convertForeverAndNoneTTLValue(Object propValue, boolean isPhoenixTTLEnabled) {
- //Handle FOREVER and NONE value for TTL at HBase level TTL.
- if (propValue instanceof String) {
- String strValue = (String) propValue;
- if ("FOREVER".equalsIgnoreCase(strValue)) {
- propValue = HConstants.FOREVER;
- } else if ("NONE".equalsIgnoreCase(strValue)) {
- propValue = isPhoenixTTLEnabled ? TTL_NOT_DEFINED : HConstants.FOREVER;
- }
- }
- return propValue;
- }
-
/**
* Keep the TTL, KEEP_DELETED_CELLS and REPLICATION_SCOPE properties of new column families
* in sync with the existing column families. Note that we use the new values for these properties in case they
@@ -4560,7 +4544,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
*/
metaConnection = addColumnsIfNotExists(metaConnection,
PhoenixDatabaseMetaData.SYSTEM_CATALOG, MIN_SYSTEM_TABLE_TIMESTAMP_5_3_0 - 1,
- PhoenixDatabaseMetaData.TTL + " " + PInteger.INSTANCE.getSqlTypeName());
+ PhoenixDatabaseMetaData.TTL + " " + PVarchar.INSTANCE.getSqlTypeName());
metaConnection = addColumnsIfNotExists(metaConnection,
PhoenixDatabaseMetaData.SYSTEM_CATALOG, MIN_SYSTEM_TABLE_TIMESTAMP_5_3_0,
PhoenixDatabaseMetaData.ROW_KEY_MATCHER + " "
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServices.java
index 772eac78c5..dc0fab5a2c 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -374,8 +374,11 @@ public interface QueryServices extends SQLCloseable {
public static final String PHOENIX_TTL_SERVER_SIDE_MASKING_ENABLED = "phoenix.ttl.server_side.masking.enabled";
// The time limit on the amount of work to be done in one RPC call
public static final String PHOENIX_SERVER_PAGE_SIZE_MS = "phoenix.server.page.size.ms";
- // Phoenix TTL implemented by CompactionScanner and TTLRegionScanner is enabled
+
+ // TODO : Deprecate instead use PHOENIX_COMPACTION_ENABLED
public static final String PHOENIX_TABLE_TTL_ENABLED = "phoenix.table.ttl.enabled";
+ // Phoenix TTL/Compaction implemented by CompactionScanner and TTLRegionScanner is enabled
+ public static final String PHOENIX_COMPACTION_ENABLED = "phoenix.compaction.enabled";
// Copied here to avoid dependency on hbase-server
public static final String WAL_EDIT_CODEC_ATTRIB = "hbase.regionserver.wal.codec";
//Property to know whether TTL at View Level is enabled
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index ca637dbf90..d7da58f3c2 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -447,6 +447,7 @@ public class QueryServicesOptions {
public static final int DEFAULT_SCAN_PAGE_SIZE = 32768;
public static final boolean DEFAULT_APPLY_TIME_ZONE_DISPLACMENT = false;
public static final boolean DEFAULT_PHOENIX_TABLE_TTL_ENABLED = true;
+ public static final boolean DEFAULT_PHOENIX_COMPACTION_ENABLED = true;
public static final boolean DEFAULT_PHOENIX_VIEW_TTL_ENABLED = true;
public static final int DEFAULT_PHOENIX_VIEW_TTL_TENANT_VIEWS_PER_SCAN_LIMIT = 100;
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/LiteralTTLExpression.java b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/LiteralTTLExpression.java
index 8866e29700..1872ef771b 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/LiteralTTLExpression.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/LiteralTTLExpression.java
@@ -38,9 +38,11 @@ public class LiteralTTLExpression implements TTLExpression, CompiledTTLExpressio
new LiteralTTLExpression(HConstants.FOREVER);
public static final LiteralTTLExpression TTL_EXPRESSION_NOT_DEFINED =
new LiteralTTLExpression(PhoenixDatabaseMetaData.TTL_NOT_DEFINED);
+ public static final LiteralTTLExpression TTL_EXPRESSION_DEFINED_IN_TABLE_DESCRIPTOR =
+ new LiteralTTLExpression(PhoenixDatabaseMetaData.TTL_DEFINED_IN_TABLE_DESCRIPTOR);
public LiteralTTLExpression(int ttl) {
- Preconditions.checkArgument(ttl >= 0);
+ Preconditions.checkArgument(ttl == -1 || ttl >= 0);
this.ttlValue = ttl;
}
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index 85fad7b96b..a1cbfa10b4 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -139,6 +139,7 @@ import static org.apache.phoenix.schema.PTableType.VIEW;
import static org.apache.phoenix.schema.types.PDataType.FALSE_BYTES;
import static org.apache.phoenix.schema.types.PDataType.TRUE_BYTES;
import static org.apache.phoenix.util.CDCUtil.CDC_STREAM_NAME_FORMAT;
+import static org.apache.phoenix.util.MetaDataUtil.getCompatibleTTLExpression;
import java.io.BufferedReader;
import java.io.File;
@@ -181,7 +182,6 @@ import org.apache.hadoop.hbase.client.Table;
import org.apache.phoenix.coprocessorclient.TableInfo;
import org.apache.phoenix.query.ConnectionlessQueryServicesImpl;
import org.apache.phoenix.query.DelegateQueryServices;
-import org.apache.phoenix.query.ConnectionQueryServicesImpl;
import org.apache.phoenix.schema.task.SystemTaskParams;
import org.apache.phoenix.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.hbase.HConstants;
@@ -1245,17 +1245,20 @@ public class MetaDataClient {
.setMessage("Property: " + prop.getFirst()).build()
.buildException();
}
- //Keeping TTL value as Phoenix Table property irrespective of PhoenixTTLEnabled or
- //not to store the value in SYSCAT. To keep PTableImpl.getTTL() consistent
- //for client side.
+ // Handle when TTL property is set
if (prop.getFirst().equalsIgnoreCase(TTL) && tableType != PTableType.SYSTEM) {
tableProps.put(prop.getFirst(), prop.getSecond());
- if (!isPhoenixTTLEnabled()) {
- //Handling FOREVER and NONE case for TTL when phoenix.table.ttl.enable is false.
- Object value = ConnectionQueryServicesImpl.convertForeverAndNoneTTLValue(prop.getSecond(), false);
- commonFamilyProps.put(prop.getFirst(), value);
+ if (prop.getSecond() != null) {
+ TTLExpression ttlExpr = MetaDataUtil.convertForeverAndNoneTTLValue(prop.getSecond(), false);
+ // Keeping TTL value in HTable descriptor when expression is a Literal Expression
+ if ((ttlExpr instanceof LiteralTTLExpression ) && (tableType != PTableType.VIEW)) {
+ // only literal TTL expression
+ Integer value = ((LiteralTTLExpression) ttlExpr).getTTLValue();
+ //Even though TTL is really a HColumnProperty we treat it
+ //specially. We enforce that all CFs have the same TTL.
+ commonFamilyProps.put(prop.getFirst(), value);
+ }
}
- //If phoenix.table.ttl.enabled is true doesn't store TTL as columnFamilyProp
continue;
}
@@ -1276,12 +1279,6 @@ public class MetaDataClient {
}
}
- private boolean isPhoenixTTLEnabled() {
- return connection.getQueryServices().getConfiguration().
- getBoolean(QueryServices.PHOENIX_TABLE_TTL_ENABLED,
- QueryServicesOptions.DEFAULT_PHOENIX_TABLE_TTL_ENABLED);
- }
-
private boolean isViewTTLEnabled() {
return connection.getQueryServices().getConfiguration().
getBoolean(QueryServices.PHOENIX_VIEW_TTL_ENABLED,
@@ -2566,7 +2563,7 @@ public class MetaDataClient {
.build()
.buildException();
}
- ttl = ttlProp;
+ ttl = getCompatibleTTLExpression(ttlProp, tableType);
} else {
ttlFromHierarchy = checkAndGetTTLFromHierarchy(parent, tableName);
if (!ttlFromHierarchy.equals(TTL_EXPRESSION_NOT_DEFINED)) {
@@ -2795,8 +2792,7 @@ public class MetaDataClient {
.setSchemaName(schemaName).setTableName(tableName)
.build().buildException();
}
- if ((isPhoenixTTLEnabled() ? ttlProp != null
- : TableProperty.TTL.getValue(commonFamilyProps) != null)
+ if ((ttlProp != null || TableProperty.TTL.getValue(commonFamilyProps) != null)
&& transactionProvider != null
&& transactionProvider.getTransactionProvider().isUnsupported(PhoenixTransactionProvider.Feature.SET_TTL)) {
throw new SQLExceptionInfo.Builder(PhoenixTransactionProvider.Feature.SET_TTL.getCode())
@@ -6188,7 +6184,8 @@ public class MetaDataClient {
if (metaProperties.getTTL() != table.getTTLExpression()) {
TTLExpression newTTL = metaProperties.getTTL();
newTTL.validateTTLOnAlter(connection, table);
- metaPropertiesEvaluated.setTTL(metaProperties.getTTL());
+ metaPropertiesEvaluated.setTTL(
+ getCompatibleTTLExpression(metaProperties.getTTL(), table.getType()));
changingPhoenixTableProperty = true;
}
//Updating Introducing TTL variable to true so that we will check if TTL is already
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/tool/SchemaExtractionProcessor.java b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/tool/SchemaExtractionProcessor.java
index 0f935867fd..ac8a40833a 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/schema/tool/SchemaExtractionProcessor.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/schema/tool/SchemaExtractionProcessor.java
@@ -33,11 +33,13 @@ import org.apache.phoenix.query.ConnectionQueryServices;
import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
+import org.apache.phoenix.schema.LiteralTTLExpression;
import org.apache.phoenix.schema.PColumn;
import org.apache.phoenix.schema.PColumnFamily;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.schema.SortOrder;
+import org.apache.phoenix.schema.TTLExpression;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.SchemaUtil;
@@ -60,6 +62,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TTL;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY;
import static org.apache.phoenix.schema.PTable.ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS;
import static org.apache.phoenix.util.MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES;
+import static org.apache.phoenix.util.MetaDataUtil.convertForeverAndNoneTTLValue;
public class SchemaExtractionProcessor implements SchemaProcessor {
Map<String, String> defaultProps = new HashMap<>();
@@ -77,7 +80,6 @@ public class SchemaExtractionProcessor implements SchemaProcessor {
private String ddl = null;
private String tenantId;
private boolean shouldGenerateWithDefaults = false;
- private boolean isPhoenixTTLEnabled = true;
public SchemaExtractionProcessor(String tenantId, Configuration conf,
String pSchemaName, String pTableName)
@@ -85,8 +87,6 @@ public class SchemaExtractionProcessor implements SchemaProcessor {
this.tenantId = tenantId;
this.conf = conf;
this.table = getPTable(pSchemaName, pTableName);
- this.isPhoenixTTLEnabled = conf.getBoolean(QueryServices.PHOENIX_TABLE_TTL_ENABLED,
- QueryServicesOptions.DEFAULT_PHOENIX_TABLE_TTL_ENABLED);
}
public SchemaExtractionProcessor(String tenantId, Configuration conf,
@@ -96,8 +96,6 @@ public class SchemaExtractionProcessor implements SchemaProcessor {
this.conf = conf;
this.table = pTable;
this.shouldGenerateWithDefaults = shouldGenerateWithDefaults;
- this.isPhoenixTTLEnabled = conf.getBoolean(QueryServices.PHOENIX_TABLE_TTL_ENABLED,
- QueryServicesOptions.DEFAULT_PHOENIX_TABLE_TTL_ENABLED);
}
@Override
@@ -369,9 +367,6 @@ public class SchemaExtractionProcessor implements SchemaProcessor {
for (Map.Entry<Bytes, Bytes> entry : propsMap.entrySet()) {
Bytes key = entry.getKey();
Bytes globalValue = entry.getValue();
- if (Bytes.toString(key.get()).equalsIgnoreCase(TTL) && isPhoenixTTLEnabled) {
- continue;
- }
Map<String, String> cfToPropertyValueMap = new HashMap<String, String>();
Set<Bytes> cfPropertyValueSet = new HashSet<>();
for (ColumnFamilyDescriptor columnDescriptor: columnDescriptors) {
@@ -404,7 +399,8 @@ public class SchemaExtractionProcessor implements SchemaProcessor {
if (!key.equalsIgnoreCase(TTL)) {
definedProps.put(key, value);
} else {
- if (isPhoenixTTLEnabled && Integer.parseInt(value) != TTL_NOT_DEFINED) {
+ TTLExpression ttlExpression = convertForeverAndNoneTTLValue(value, false);
+ if (ttlExpression instanceof LiteralTTLExpression) {
definedProps.put(key, value);
}
}
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/util/MetaDataUtil.java b/phoenix-core-client/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
index 8db1d3bb73..e3d5377e3b 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
@@ -18,6 +18,11 @@
package org.apache.phoenix.util;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.*;
+import static org.apache.phoenix.schema.LiteralTTLExpression.TTL_EXPRESSION_DEFINED_IN_TABLE_DESCRIPTOR;
+import static org.apache.phoenix.schema.LiteralTTLExpression.TTL_EXPRESSION_FOREVER;
+import static org.apache.phoenix.schema.LiteralTTLExpression.TTL_EXPRESSION_NOT_DEFINED;
+import static org.apache.phoenix.schema.PTableType.TABLE;
+import static org.apache.phoenix.schema.PTableType.VIEW;
import static org.apache.phoenix.util.SchemaUtil.getVarChars;
import java.nio.charset.StandardCharsets;
@@ -25,8 +30,11 @@ import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.*;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.phoenix.exception.SQLExceptionInfo;
import org.apache.phoenix.schema.ColumnFamilyNotFoundException;
import org.apache.phoenix.schema.ColumnNotFoundException;
+import org.apache.phoenix.schema.ConditionalTTLExpression;
import org.apache.phoenix.schema.PColumn;
import org.apache.phoenix.schema.PColumnFamily;
import org.apache.phoenix.schema.PName;
@@ -36,6 +44,7 @@ import org.apache.phoenix.schema.PTableRef;
import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.schema.SequenceKey;
import org.apache.phoenix.schema.SortOrder;
+import org.apache.phoenix.schema.TTLExpression;
import org.apache.phoenix.schema.TableNotFoundException;
import org.apache.phoenix.schema.TableProperty;
import org.apache.phoenix.schema.types.PVarchar;
@@ -1190,4 +1199,53 @@ public class MetaDataUtil {
connection.getMetaDataCache().getAge(tableRef) <
effectiveUpdateCacheFreq;
}
+
+
+ public static TTLExpression convertForeverAndNoneTTLValue(Object propValue, boolean forSystemCatalog) {
+ //Handle FOREVER and NONE value for TTL at HBase level TTL.
+ if (propValue == null) {
+ return forSystemCatalog ? TTL_EXPRESSION_NOT_DEFINED : TTL_EXPRESSION_FOREVER;
+ }
+ if (propValue instanceof String) {
+ String strValue = (String) propValue;
+ if ("FOREVER".equalsIgnoreCase(strValue) || "NONE".equalsIgnoreCase(strValue) || "0".equalsIgnoreCase(strValue)) {
+ propValue = forSystemCatalog ? TTL_NOT_DEFINED : HConstants.FOREVER;
+ }
+ } else if (propValue instanceof Integer && propValue.equals(TTL_NOT_DEFINED)) {
+ propValue = forSystemCatalog ? TTL_NOT_DEFINED : HConstants.FOREVER;
+ }
+ return (TTLExpression) TableProperty.TTL.getValue(propValue);
+ }
+
+ /**
+ * In 5.3.0 release the TTL column will be created in SYSTEM.CATALOG and used to store TTL values,
+ * but the TTL values for tables with literal expressions will continue to be stored in
+ * the HBase TableDescriptor for backward compatibility.
+ * When table type != TABLE (i.e. VIEW TTL) OR expression is ConditionalExpression then
+ * actual TTL value is stored in SYSTEM.CATALOG and TableDescriptor will be set to FOREVER
+ * When table type == TABLE (i.e. Table TTL) and expression is LiteralExpression then
+ * actual TTL value is stored in TableDescriptor
+ * and SYSTEM.CATALOG will be set to TTL_EXPRESSION_DEFINED_IN_TABLE_DESCRIPTOR.
+ * if expression == TTL_NOT_DEFINED OR FOREVER then
+ * TTL_NOT_DEFINED will be stores in SYSTEM.CATALOG
+ */
+ public static TTLExpression getCompatibleTTLExpression(TTLExpression expression, PTableType tableType)
+ throws SQLException {
+ if (tableType != TABLE && tableType != VIEW) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.
+ TTL_SUPPORTED_FOR_TABLES_AND_VIEWS_ONLY)
+ .build()
+ .buildException();
+ }
+ TTLExpression ttl;
+ if (tableType != TABLE || expression instanceof ConditionalTTLExpression) {
+ ttl = expression;
+ } else if (expression.equals(TTL_EXPRESSION_NOT_DEFINED) || expression.equals(TTL_EXPRESSION_FOREVER)) {
+ ttl = TTL_EXPRESSION_NOT_DEFINED;
+ } else {
+ ttl = TTL_EXPRESSION_DEFINED_IN_TABLE_DESCRIPTOR;
+ }
+ return ttl;
+ }
+
}
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/util/ScanUtil.java b/phoenix-core-client/src/main/java/org/apache/phoenix/util/ScanUtil.java
index 2800f2edd4..1941692f57 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/util/ScanUtil.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/util/ScanUtil.java
@@ -25,9 +25,12 @@ import static org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverCons
import static org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants.SCAN_STOP_ROW_SUFFIX;
import static org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants.CDC_DATA_TABLE_DEF;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_TTL;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TTL_DEFINED_IN_TABLE_DESCRIPTOR;
import static org.apache.phoenix.query.QueryConstants.ENCODED_EMPTY_COLUMN_NAME;
import static org.apache.phoenix.query.QueryServices.USE_STATS_FOR_PARALLELIZATION;
+import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_PHOENIX_TABLE_TTL_ENABLED;
import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_USE_STATS_FOR_PARALLELIZATION;
+import static org.apache.phoenix.schema.LiteralTTLExpression.TTL_EXPRESSION_DEFINED_IN_TABLE_DESCRIPTOR;
import static org.apache.phoenix.schema.LiteralTTLExpression.TTL_EXPRESSION_FOREVER;
import static org.apache.phoenix.schema.types.PDataType.TRUE_BYTES;
import static org.apache.phoenix.util.ByteUtil.EMPTY_BYTE_ARRAY;
@@ -1170,14 +1173,16 @@ public class ScanUtil {
public static CompiledTTLExpression getTTLExpression(Scan scan) throws IOException {
byte[] phoenixTTL = scan.getAttribute(BaseScannerRegionObserverConstants.TTL);
if (phoenixTTL == null) {
- return TTL_EXPRESSION_FOREVER;
+ return null;
}
return TTLExpressionFactory.create(phoenixTTL);
}
- public static boolean isPhoenixTableTTLEnabled(Configuration conf) {
+ public static boolean isPhoenixCompactionEnabled(Configuration conf) {
return conf.getBoolean(QueryServices.PHOENIX_TABLE_TTL_ENABLED,
- QueryServicesOptions.DEFAULT_PHOENIX_TABLE_TTL_ENABLED);
+ QueryServicesOptions.DEFAULT_PHOENIX_TABLE_TTL_ENABLED) &&
+ conf.getBoolean(QueryServices.PHOENIX_COMPACTION_ENABLED,
+ QueryServicesOptions.DEFAULT_PHOENIX_COMPACTION_ENABLED);
}
public static boolean isMaskTTLExpiredRows(Scan scan) {
@@ -1408,14 +1413,14 @@ public class ScanUtil {
return;
}
- // If Phoenix level TTL is not enabled OR is a system table then return.
- if (!isPhoenixTableTTLEnabled(phoenixConnection.getQueryServices().getConfiguration())) {
- if (SchemaUtil.isSystemTable(
- SchemaUtil.getTableNameAsBytes(table.getSchemaName().getString(),
- table.getTableName().getString()))) {
- scan.setAttribute(BaseScannerRegionObserverConstants.IS_PHOENIX_TTL_SCAN_TABLE_SYSTEM,
- Bytes.toBytes(true));
- }
+ // If is a system table or If Phoenix level TTL/compaction is not enabled
+ // then set the TTL scan attribute to TTL_DEFINED_IN_TABLE_DESCRIPTOR.
+ if ((!isPhoenixCompactionEnabled(phoenixConnection.getQueryServices().getConfiguration())) ||
+ SchemaUtil.isSystemTable(
+ SchemaUtil.getTableNameAsBytes(table.getSchemaName().getString(),
+ table.getTableName().getString()))) {
+ byte[] ttlForScan = TTL_EXPRESSION_DEFINED_IN_TABLE_DESCRIPTOR.serialize();
+ scan.setAttribute(BaseScannerRegionObserverConstants.TTL, ttlForScan);
return;
}
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/util/UpgradeUtil.java b/phoenix-core-client/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
index 899f40da16..121aad1afa 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
@@ -17,14 +17,13 @@
*/
package org.apache.phoenix.util;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_COLUMN_FAMILY_NAME;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.PHOENIX_TTL_BYTES;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_NAME_BYTES;
-import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM_BYTES;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_TYPE_BYTES;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TTL_BYTES;
-import static org.apache.phoenix.query.QueryConstants.SYSTEM_SCHEMA_NAME_BYTES;
-import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_PHOENIX_TABLE_TTL_ENABLED;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TTL_DEFINED_IN_TABLE_DESCRIPTOR;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TTL_NOT_DEFINED;
+import static org.apache.phoenix.schema.LiteralTTLExpression.TTL_EXPRESSION_DEFINED_IN_TABLE_DESCRIPTOR;
+import static org.apache.phoenix.schema.LiteralTTLExpression.TTL_EXPRESSION_NOT_DEFINED;
import static org.apache.phoenix.thirdparty.com.google.common.base.Preconditions.checkNotNull;
import static org.apache.phoenix.coprocessorclient.MetaDataProtocol.CURRENT_CLIENT_VERSION;
import static org.apache.phoenix.coprocessorclient.MetaDataProtocol.getVersion;
@@ -71,6 +70,7 @@ import static org.apache.phoenix.query.QueryConstants.EMPTY_COLUMN_BYTES;
import static org.apache.phoenix.query.QueryConstants.EMPTY_COLUMN_VALUE_BYTES;
import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_TIMEOUT_DURING_UPGRADE_MS;
import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_SCAN_PAGE_SIZE;
+import static org.apache.phoenix.util.SchemaUtil.getVarChars;
import java.io.IOException;
import java.math.BigInteger;
@@ -86,7 +86,6 @@ import java.sql.Types;
import java.text.Format;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -107,7 +106,12 @@ import javax.annotation.Nullable;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.CompareOperator;
+import org.apache.hadoop.hbase.filter.BinaryComparator;
+import org.apache.hadoop.hbase.filter.QualifierFilter;
+import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.phoenix.coprocessorclient.MetaDataEndpointImplConstants;
+import org.apache.phoenix.hbase.index.util.GenericKeyValueBuilder;
+import org.apache.phoenix.schema.TTLExpression;
import org.apache.phoenix.thirdparty.com.google.common.base.Strings;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
@@ -1481,7 +1485,7 @@ public class UpgradeUtil {
ReadOnlyProps readOnlyProps = oldMetaConnection.getQueryServices().getProps();
TableName sysCat = SchemaUtil.getPhysicalTableName(SYSTEM_CATALOG_NAME, readOnlyProps);
- LOGGER.debug(String.format("SYSTEM CATALOG table use for copying TTL values: %s", sysCat.toString()));
+ LOGGER.info(String.format("SYSTEM CATALOG table use for copying TTL values: %s", sysCat.toString()));
Configuration conf = oldMetaConnection.getQueryServices().getConfiguration();
try (org.apache.hadoop.hbase.client.Connection moveTTLConnection = getHBaseConnection(conf, options);
Table sysCatalogTable = moveTTLConnection.getTable(sysCat);
@@ -1494,16 +1498,18 @@ public class UpgradeUtil {
Scan scan = new Scan();
scan.addFamily(DEFAULT_COLUMN_FAMILY_BYTES);
// Push down the filter to hbase to avoid transfer
- SingleColumnValueFilter tableFilter = new SingleColumnValueFilter(
+ QualifierFilter tableTypeQualifierFilter = new QualifierFilter(
+ CompareOperator.EQUAL, new BinaryComparator(TABLE_TYPE_BYTES));
+
+ SingleColumnValueFilter tableTypeFilter = new SingleColumnValueFilter(
DEFAULT_COLUMN_FAMILY_BYTES,
TABLE_TYPE_BYTES, CompareOperator.EQUAL,
PTableType.TABLE.getSerializedValue().getBytes(StandardCharsets.UTF_8));
-
- tableFilter.setFilterIfMissing(true);
+ tableTypeFilter.setFilterIfMissing(true);
// Limit number of records
PageFilter pf = new PageFilter(DEFAULT_SCAN_PAGE_SIZE);
- scan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, pf, tableFilter));
+ scan.setFilter(new FilterList(FilterList.Operator.MUST_PASS_ALL, pf, tableTypeQualifierFilter, tableTypeFilter));
if (pageMore) {
scan.withStartRow(lastRowKey, false);
}
@@ -1511,59 +1517,51 @@ public class UpgradeUtil {
try (ResultScanner scanner = sysCatalogTable.getScanner(scan)) {
int count = 0;
List<byte[]> rowKeys = new ArrayList<>();
- List<Put> puts = new ArrayList<>();
+ List<Mutation> mutations = new ArrayList<>();
for (Result rr = scanner.next(); rr != null; rr = scanner.next()) {
count++;
lastRowKey = rr.getRow();
byte[] tmpKey = new byte[lastRowKey.length];
System.arraycopy(lastRowKey, 0, tmpKey, 0, tmpKey.length);
rowKeys.add(tmpKey);
- String tableName = SchemaUtil.getTableName(rr.getValue(
- DEFAULT_COLUMN_FAMILY_BYTES, TABLE_SCHEM_BYTES),
- rr.getValue(DEFAULT_COLUMN_FAMILY_BYTES, TABLE_NAME_BYTES));
- if (tableName == null || Arrays.equals(rr.getValue(DEFAULT_COLUMN_FAMILY_BYTES,
- TABLE_SCHEM_BYTES), SYSTEM_SCHEMA_NAME_BYTES)) {
+ byte[][] rowKeyMetaData = new byte[3][];
+ getVarChars(tmpKey, 3, rowKeyMetaData);
+ byte[] schemaName = rowKeyMetaData[PhoenixDatabaseMetaData.SCHEMA_NAME_INDEX];
+ byte[] tableName = rowKeyMetaData[PhoenixDatabaseMetaData.TABLE_NAME_INDEX];
+
+ String fullTableName = SchemaUtil.getTableName(schemaName, tableName);
+ if (SchemaUtil.isSystemTable(SchemaUtil.getTableNameAsBytes(schemaName, tableName))) {
//We do not support system table ttl through phoenix ttl, and it will be moved to a
//constant value in future commit.
continue;
}
TableDescriptor tableDesc = admin.getDescriptor(SchemaUtil.getPhysicalTableName(
- tableName, readOnlyProps));
+ fullTableName, readOnlyProps));
int ttl = tableDesc.getColumnFamily(DEFAULT_COLUMN_FAMILY_BYTES).
getTimeToLive();
- if (ttl != ColumnFamilyDescriptorBuilder.DEFAULT_TTL) {
- //As we have ttl defined fot this table create a Put to set TTL.
- long rowTS = rr.rawCells()[0].getTimestamp();
+ // As we have ttl defined for this table create a Put to set TTL with
+ // backward compatibility in mind.
+ long rowTS = EnvironmentEdgeManager.currentTimeMillis();
+
+ if (ttl != HConstants.FOREVER && ttl != TTL_NOT_DEFINED) {
+ // set the TTL column to TTL_DEFINED_IN_TABLE_DESCRIPTOR
Put put = new Put(tmpKey);
put.addColumn(DEFAULT_COLUMN_FAMILY_BYTES, EMPTY_COLUMN_BYTES, rowTS,
EMPTY_COLUMN_VALUE_BYTES);
put.addColumn(DEFAULT_COLUMN_FAMILY_BYTES, TTL_BYTES, rowTS,
- PVarchar.INSTANCE.toBytes(String.valueOf(ttl)));
- puts.add(put);
-
- //Set TTL to Default at CF level when Phoenix level ttl is enabled
- if (oldMetaConnection.getQueryServices().getConfiguration().getBoolean(
- QueryServices.PHOENIX_TABLE_TTL_ENABLED, DEFAULT_PHOENIX_TABLE_TTL_ENABLED)) {
- ColumnFamilyDescriptor columnFamilyDescriptor = ColumnFamilyDescriptorBuilder
- .newBuilder(DEFAULT_COLUMN_FAMILY_BYTES).setTimeToLive(
- ColumnFamilyDescriptorBuilder.DEFAULT_TTL).build();
- TableDescriptor tableDescriptor = TableDescriptorBuilder.newBuilder(
- admin.getDescriptor(SchemaUtil.getPhysicalTableName(
- tableName, readOnlyProps))).modifyColumnFamily(
- columnFamilyDescriptor).build();
- admin.modifyTable(tableDescriptor);
- }
+ PVarchar.INSTANCE.toBytes(TTL_EXPRESSION_DEFINED_IN_TABLE_DESCRIPTOR.getTTLExpression()));
+ mutations.add(put);
}
}
- if (!puts.isEmpty()) {
- Object[] putResults = new Object[puts.size()];
+ if (!mutations.isEmpty()) {
+ Object[] mutationResults = new Object[mutations.size()];
try (Table moveTTLTable = moveTTLConnection.getTable(sysCat)) {
// Process a batch of ttl values
- moveTTLTable.batch(puts, putResults);
+ moveTTLTable.batch(mutations, mutationResults);
int numMoved = 0;
- for (Object putResult : putResults) {
- if (java.util.Objects.nonNull(putResult)) {
+ for (Object mutationResult : mutationResults) {
+ if (java.util.Objects.nonNull(mutationResult)) {
numMoved++;
}
}
@@ -2494,6 +2492,28 @@ public class UpgradeUtil {
tableMetadata.add(put);
}
+ public static void addTTLForClientOlderThan530(List<Mutation> tableMetadata, byte[] tableHeaderRowKey, long clientTimeStamp, int clientVersion, ColumnFamilyDescriptor cfd)
+ throws IOException {
+ if (cfd.getTimeToLive() == HConstants.FOREVER || cfd.getTimeToLive() == 0) {
+ // Set the TTL column to null
+ Delete deleteColumn = new Delete(tableHeaderRowKey);
+ KeyValue kv = GenericKeyValueBuilder.INSTANCE.buildDeleteColumns(
+ new ImmutableBytesWritable(tableHeaderRowKey),
+ new ImmutableBytesWritable(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES),
+ new ImmutableBytesWritable(TTL_BYTES), clientTimeStamp);
+ deleteColumn.add(kv);
+ tableMetadata.add(deleteColumn);
+ } else {
+ Put putColumn = new Put(tableHeaderRowKey, clientTimeStamp).addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES,
+ TTL_BYTES,
+ PVarchar.INSTANCE.toBytes(TTL_EXPRESSION_DEFINED_IN_TABLE_DESCRIPTOR.getTTLExpression()));
+ tableMetadata.add(putColumn);
+ }
+ LOGGER.info("Adding backward compatible TTL (create {}) for table-key {} with ttl value {}",
+ clientVersion,
+ Bytes.toStringBinary(tableHeaderRowKey), cfd.getTimeToLive());
+ }
+
public static boolean truncateStats(Table metaTable, Table statsTable)
throws IOException, InterruptedException {
byte[] statsTableKey = SchemaUtil.getTableKey(null, PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME,
diff --git a/phoenix-core-client/src/main/java/org/apache/phoenix/util/ViewUtil.java b/phoenix-core-client/src/main/java/org/apache/phoenix/util/ViewUtil.java
index 8d737d219e..924ce27829 100644
--- a/phoenix-core-client/src/main/java/org/apache/phoenix/util/ViewUtil.java
+++ b/phoenix-core-client/src/main/java/org/apache/phoenix/util/ViewUtil.java
@@ -400,6 +400,7 @@ public class ViewUtil {
tableInfoList.add(new TableInfo(viewTenantId, viewSchemaName, viewName));
if (scanSysCatForTTLDefinedOnAnyChildPair.getFirst()) {
byte[] viewKey = SchemaUtil.getTableKey(viewTenantId, viewSchemaName, viewName);
+
Scan ttlScan = MetaDataUtil.newTableRowsScan(viewKey,
MetaDataProtocol.MIN_TABLE_TIMESTAMP, timestamp);
Result ttlResult = sysCat.getScanner(ttlScan).next();
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
index bf7ca32d89..aeb85f397f 100644
--- a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/BaseScannerRegionObserver.java
@@ -373,7 +373,7 @@ abstract public class BaseScannerRegionObserver implements RegionObserver {
ScanType scanType, ScanOptions options, CompactionLifeCycleTracker tracker,
CompactionRequest request) throws IOException {
Configuration conf = c.getEnvironment().getConfiguration();
- if (isPhoenixTableTTLEnabled(conf)) {
+ if (isPhoenixCompactionEnabled(conf)) {
setScanOptionsForFlushesAndCompactions(options);
return;
}
@@ -388,7 +388,7 @@ abstract public class BaseScannerRegionObserver implements RegionObserver {
ScanOptions options, FlushLifeCycleTracker tracker) throws IOException {
Configuration conf = c.getEnvironment().getConfiguration();
- if (isPhoenixTableTTLEnabled(conf)) {
+ if (isPhoenixCompactionEnabled(conf)) {
setScanOptionsForFlushesAndCompactions(options);
return;
}
@@ -403,7 +403,7 @@ abstract public class BaseScannerRegionObserver implements RegionObserver {
ObserverContext<RegionCoprocessorEnvironment> c, Store store, ScanOptions options)
throws IOException {
Configuration conf = c.getEnvironment().getConfiguration();
- if (isPhoenixTableTTLEnabled(conf)) {
+ if (isPhoenixCompactionEnabled(conf)) {
setScanOptionsForFlushesAndCompactions(options);
return;
}
@@ -429,7 +429,7 @@ abstract public class BaseScannerRegionObserver implements RegionObserver {
ScanOptions options) throws IOException {
Configuration conf = c.getEnvironment().getConfiguration();
- if (isPhoenixTableTTLEnabled(conf)) {
+ if (isPhoenixCompactionEnabled(conf)) {
setScanOptionsForFlushesAndCompactions(options);
return;
}
@@ -539,8 +539,10 @@ abstract public class BaseScannerRegionObserver implements RegionObserver {
return maxLookbackTime > 0L;
}
- public static boolean isPhoenixTableTTLEnabled(Configuration conf) {
+ public static boolean isPhoenixCompactionEnabled(Configuration conf) {
return conf.getBoolean(QueryServices.PHOENIX_TABLE_TTL_ENABLED,
- QueryServicesOptions.DEFAULT_PHOENIX_TABLE_TTL_ENABLED);
+ QueryServicesOptions.DEFAULT_PHOENIX_TABLE_TTL_ENABLED) &&
+ conf.getBoolean(QueryServices.PHOENIX_COMPACTION_ENABLED,
+ QueryServicesOptions.DEFAULT_PHOENIX_COMPACTION_ENABLED);
}
}
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/CompactionScanner.java b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/CompactionScanner.java
index 232a4a20c2..2b9215b740 100644
--- a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/CompactionScanner.java
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/CompactionScanner.java
@@ -108,6 +108,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_
import static org.apache.phoenix.query.QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX;
import static org.apache.phoenix.query.QueryServices.PHOENIX_VIEW_TTL_TENANT_VIEWS_PER_SCAN_LIMIT;
import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_PHOENIX_VIEW_TTL_TENANT_VIEWS_PER_SCAN_LIMIT;
+import static org.apache.phoenix.schema.LiteralTTLExpression.TTL_EXPRESSION_DEFINED_IN_TABLE_DESCRIPTOR;
import static org.apache.phoenix.schema.LiteralTTLExpression.TTL_EXPRESSION_FOREVER;
import static org.apache.phoenix.schema.LiteralTTLExpression.TTL_EXPRESSION_NOT_DEFINED;
import static org.apache.phoenix.util.ByteUtil.EMPTY_BYTE_ARRAY;
@@ -1273,10 +1274,12 @@ public class CompactionScanner implements InternalScanner {
Store store) throws IOException {
boolean isSystemTable = pTable.getType() == PTableType.SYSTEM;
+ boolean ttlFromDescriptor = false;
try {
- if (isSystemTable) {
+ if (isSystemTable || pTable.getTTLExpression().equals(TTL_EXPRESSION_DEFINED_IN_TABLE_DESCRIPTOR)) {
ColumnFamilyDescriptor cfd = store.getColumnFamilyDescriptor();
ttlExpr = TTLExpressionFactory.create(cfd.getTimeToLive());
+ ttlFromDescriptor = true;
} else {
ttlExpr = !pTable.getTTLExpression().equals(TTL_EXPRESSION_NOT_DEFINED)
? pTable.getCompiledTTLExpression(pConn) : TTL_EXPRESSION_FOREVER;
@@ -1287,8 +1290,8 @@ public class CompactionScanner implements InternalScanner {
}
LOGGER.info(String.format(
"NonPartitionedTableTTLTracker params:- " +
- "(physical-name=%s, ttl=%s, isSystemTable=%s)",
- pTable.getName().toString(), ttlExpr, isSystemTable));
+ "(physical-name=%s, ttl=%s, ttlFromDescriptor=%s, isSystemTable=%s)",
+ pTable.getName().toString(), ttlExpr, ttlFromDescriptor, isSystemTable));
}
@Override
@@ -1329,9 +1332,17 @@ public class CompactionScanner implements InternalScanner {
this.tableRowKeyMatcher =
new PartitionedTableRowKeyMatcher(table, isSalted, isSharedIndex,
isLongViewIndexEnabled, viewTTLTenantViewsPerScanLimit);
+ boolean ttlFromDescriptor = false;
try {
- this.ttlExpr = !table.getTTLExpression().equals(TTL_EXPRESSION_NOT_DEFINED)
- ? table.getCompiledTTLExpression(pConn) : TTL_EXPRESSION_FOREVER;
+ if (table.getTTLExpression().equals(TTL_EXPRESSION_DEFINED_IN_TABLE_DESCRIPTOR)) {
+ ColumnFamilyDescriptor cfd = store.getColumnFamilyDescriptor();
+ this.ttlExpr = TTLExpressionFactory.create(cfd.getTimeToLive());
+ ttlFromDescriptor = true;
+ } else {
+ this.ttlExpr = !table.getTTLExpression().equals(TTL_EXPRESSION_NOT_DEFINED)
+ ? table.getCompiledTTLExpression(pConn) : TTL_EXPRESSION_FOREVER;
+ }
+
} catch (SQLException e) {
throw ClientUtil.createIOException(
String.format("Error compiling ttl expression %s", this.ttlExpr), e);
@@ -1346,10 +1357,11 @@ public class CompactionScanner implements InternalScanner {
"PartitionedTableTTLTracker params:- " +
"region-name = %s, table-name = %s, " +
"multi-tenant = %s, shared-index = %s, salted = %s, " +
- "default-ttl = %s, startingPKPosition = %d",
+ "default-ttl = %s, ttlFromDescriptor = %s, startingPKPosition = %d",
region.getRegionInfo().getEncodedName(),
region.getRegionInfo().getTable().getNameAsString(), this.isMultiTenant,
- this.isSharedIndex, this.isSalted, this.ttlExpr, this.startingPKPosition));
+ this.isSharedIndex, this.isSalted, this.ttlExpr, ttlFromDescriptor,
+ this.startingPKPosition));
} catch (SQLException e) {
LOGGER.error(String.format("Failed to read from catalog: " + e.getMessage()));
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index 22a10fc404..9165bd278c 100644
--- a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@ -75,6 +75,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_TYPE_BYTES;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTIONAL_BYTES;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TRANSACTION_PROVIDER_BYTES;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TTL_BYTES;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TTL_NOT_DEFINED;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TYPE_BYTES;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY_BYTES;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.USE_STATS_FOR_PARALLELIZATION_BYTES;
@@ -87,6 +88,7 @@ import static org.apache.phoenix.query.QueryServices.INDEX_MUTATE_BATCH_SIZE_THR
import static org.apache.phoenix.query.QueryServices.SKIP_SYSTEM_TABLES_EXISTENCE_CHECK;
import static org.apache.phoenix.query.QueryServices.SYSTEM_CATALOG_INDEXES_ENABLED;
import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_SYSTEM_CATALOG_INDEXES_ENABLED;
+import static org.apache.phoenix.schema.LiteralTTLExpression.TTL_EXPRESSION_DEFINED_IN_TABLE_DESCRIPTOR;
import static org.apache.phoenix.schema.PTable.LinkType.PHYSICAL_TABLE;
import static org.apache.phoenix.schema.PTable.LinkType.VIEW_INDEX_PARENT_TABLE;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.CDC_INCLUDE_BYTES;
@@ -138,6 +140,7 @@ import org.apache.hadoop.hbase.KeyValue.Type;
import org.apache.hadoop.hbase.KeyValueUtil;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TagUtil;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.Delete;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Mutation;
@@ -2508,6 +2511,15 @@ TABLE_FAMILY_BYTES, TABLE_SEQ_NUM_BYTES);
if (tableType != PTableType.VIEW) {
UpgradeUtil.addRowKeyOrderOptimizableCell(tableMetadata, tableKey, clientTimeStamp);
}
+
+ // Handle backward compatibility from older clients
+ if ((tableType == PTableType.TABLE) && (clientVersion < MetaDataProtocol.MIN_VERSION_TABLE_TTL_IN_SYSTEM_CATALOG)) {
+ Table hTable = ServerUtil.getHTableForCoprocessorScan(env, TableName.valueOf(cPhysicalName));
+ ColumnFamilyDescriptor cfd =
+ hTable.getTableDescriptor().getColumnFamilies()[0];
+ UpgradeUtil.addTTLForClientOlderThan530(tableMetadata, tableKey, clientTimeStamp, clientVersion, cfd);
+ }
+
// If the parent table of the view has the auto partition sequence name attribute, modify the
// tableMetadata and set the view statement and partition column correctly
if (parentTable != null && parentTable.getAutoPartitionSeqName() != null) {
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/TTLRegionScanner.java b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/TTLRegionScanner.java
index f9535233c9..f516b1ad41 100644
--- a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/TTLRegionScanner.java
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/TTLRegionScanner.java
@@ -32,7 +32,6 @@ import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.filter.PageFilter;
import org.apache.hadoop.hbase.regionserver.RegionScanner;
import org.apache.hadoop.hbase.regionserver.ScannerContext;
-import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.schema.CompiledTTLExpression;
import org.apache.phoenix.schema.TTLExpressionFactory;
import org.apache.phoenix.util.EnvironmentEdgeManager;
@@ -40,10 +39,10 @@ import org.apache.phoenix.util.ScanUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.phoenix.coprocessor.BaseScannerRegionObserver.isPhoenixTableTTLEnabled;
+import static org.apache.phoenix.coprocessor.BaseScannerRegionObserver.isPhoenixCompactionEnabled;
import static org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants.EMPTY_COLUMN_FAMILY_NAME;
import static org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants.EMPTY_COLUMN_QUALIFIER_NAME;
-import static org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants.IS_PHOENIX_TTL_SCAN_TABLE_SYSTEM;
+import static org.apache.phoenix.schema.LiteralTTLExpression.TTL_EXPRESSION_DEFINED_IN_TABLE_DESCRIPTOR;
import static org.apache.phoenix.schema.LiteralTTLExpression.TTL_EXPRESSION_FOREVER;
/**
@@ -76,10 +75,25 @@ public class TTLRegionScanner extends BaseRegionScanner {
emptyCF = scan.getAttribute(EMPTY_COLUMN_FAMILY_NAME);
currentTime = scan.getTimeRange().getMax() == HConstants.LATEST_TIMESTAMP
? EnvironmentEdgeManager.currentTimeMillis() : scan.getTimeRange().getMax();
- byte[] isSystemTable = scan.getAttribute(IS_PHOENIX_TTL_SCAN_TABLE_SYSTEM);
- if (isPhoenixTableTTLEnabled(env.getConfiguration()) && (isSystemTable == null
- || !Bytes.toBoolean(isSystemTable))) {
- ttlExpression = ScanUtil.getTTLExpression(this.scan);
+ CompiledTTLExpression scanTTLExpression = ScanUtil.getTTLExpression(scan);
+ /**
+ * In 5.3.0 release the TTL column will be created in SYSTEM.CATALOG and used to store TTL
+ * values, but the TTL values for tables and SYSTEM tables with literal expressions
+ * will continue to be stored in the HBase TableDescriptor for backward compatibility
+ * and SYSTEM.CATALOG will store a special literal TTL_EXPRESSION_DEFINED_IN_TABLE_DESCRIPTOR.
+ *
+ * The newer 5.3.x clients will be sending a Compiled TTL expression in the
+ * BaseScannerRegionObserverConstants.TTL scan attribute.
+ * Older 5.2 client will not have any scan attribute set and thus will be null.
+ *
+ * To achieve backward compatibility
+ * When scanTTLExpression is null or equals TTL_EXPRESSION_DEFINED_IN_TABLE_DESCRIPTOR
+ * we will use the TableDescriptor as the source of truth.
+ */
+ if (isPhoenixCompactionEnabled(
+ env.getConfiguration()) && scanTTLExpression != null && !scanTTLExpression.equals(
+ TTL_EXPRESSION_DEFINED_IN_TABLE_DESCRIPTOR)) {
+ ttlExpression = scanTTLExpression;
} else {
ColumnFamilyDescriptor cfd =
env.getRegion().getTableDescriptor().getColumnFamilies()[0];
@@ -90,8 +104,7 @@ public class TTLRegionScanner extends BaseRegionScanner {
// be done here. We also disable masking when TTL is HConstants.FOREVER.
isMaskingEnabled = emptyCF != null && emptyCQ != null
&& !ttlExpression.equals(TTL_EXPRESSION_FOREVER)
- && (isPhoenixTableTTLEnabled(env.getConfiguration()) && (isSystemTable == null
- || !Bytes.toBoolean(isSystemTable)));
+ && (isPhoenixCompactionEnabled(env.getConfiguration()));
}
private void init() throws IOException {
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index b3314752a2..cf8797ecfe 100644
--- a/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -676,7 +676,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
}
}
if (table != null
- && isPhoenixTableTTLEnabled(c.getEnvironment().getConfiguration())) {
+ && isPhoenixCompactionEnabled(c.getEnvironment().getConfiguration())) {
internalScanner =
new CompactionScanner(c.getEnvironment(), store, scanner,
BaseScannerRegionObserverConstants.getMaxLookbackInMillis(
@@ -685,7 +685,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
keepDeleted, table
);
}
- else if (isPhoenixTableTTLEnabled(c.getEnvironment().getConfiguration())) {
+ else if (isPhoenixCompactionEnabled(c.getEnvironment().getConfiguration())) {
LOGGER.warn("Skipping compaction for table: {} " +
"as failed to retrieve PTable object", fullTableName);
}
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/IndexScrutinyMapper.java b/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/IndexScrutinyMapper.java
index 813820fe15..81df3e2493 100644
--- a/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/IndexScrutinyMapper.java
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/mapreduce/index/IndexScrutinyMapper.java
@@ -36,6 +36,7 @@ import java.util.Set;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.LiteralTTLExpression;
+import org.apache.phoenix.schema.TTLExpression;
import org.apache.phoenix.thirdparty.com.google.common.annotations.VisibleForTesting;
import org.apache.commons.codec.binary.Hex;
import org.apache.hadoop.conf.Configuration;
@@ -70,6 +71,7 @@ import org.slf4j.LoggerFactory;
import org.apache.phoenix.thirdparty.com.google.common.base.Joiner;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_TTL;
+import static org.apache.phoenix.schema.LiteralTTLExpression.TTL_EXPRESSION_DEFINED_IN_TABLE_DESCRIPTOR;
import static org.apache.phoenix.schema.LiteralTTLExpression.TTL_EXPRESSION_NOT_DEFINED;
/**
@@ -343,10 +345,12 @@ public class IndexScrutinyMapper extends Mapper<NullWritable, PhoenixIndexDBWrit
cqsi = connection.unwrap(PhoenixConnection.class).getQueryServices();
String physicalTable = getSourceTableName(pSourceTable,
SchemaUtil.isNamespaceMappingEnabled(null, cqsi.getProps()));
- if (configuration.getBoolean(QueryServices.PHOENIX_TABLE_TTL_ENABLED,
- QueryServicesOptions.DEFAULT_PHOENIX_TABLE_TTL_ENABLED)) {
- return pSourceTable.getTTLExpression().equals(TTL_EXPRESSION_NOT_DEFINED) ? DEFAULT_TTL
- : ((LiteralTTLExpression) pSourceTable.getTTLExpression()).getTTLValue(); // TODO
+ TTLExpression ttlExpression = pSourceTable.getTTLExpression();
+ if (ttlExpression == null) return DEFAULT_TTL;
+ if (!ttlExpression.equals(TTL_EXPRESSION_DEFINED_IN_TABLE_DESCRIPTOR)) {
+ return ttlExpression.equals(TTL_EXPRESSION_NOT_DEFINED) ? DEFAULT_TTL
+ // TODO What happens in the case of Conditional TTL
+ : ((LiteralTTLExpression) ttlExpression).getTTLValue(); // TODO
} else {
TableDescriptor tableDesc;
try (Admin admin = cqsi.getAdmin()) {
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseRowKeyMatcherTestIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseRowKeyMatcherTestIT.java
index dd1c65ee78..8a8c467b60 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseRowKeyMatcherTestIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseRowKeyMatcherTestIT.java
@@ -850,7 +850,6 @@ public abstract class BaseRowKeyMatcherTestIT extends ParallelStatsDisabledIT {
try {
List<PDataType[]> testCases = getTestCases();
SortOrder[][] sortOrders = getSortOrders();
-
String tableName = "";
tableName = createViewHierarchy(
testCases, sortOrders, 500, 5000, 3,
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
index bf3bb26fc0..fdf49ccca8 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CreateTableIT.java
@@ -96,18 +96,11 @@ import org.junit.experimental.categories.Category;
@Category(NeedsOwnMiniClusterTest.class)
public class CreateTableIT extends ParallelStatsDisabledIT {
-
@BeforeClass
public static synchronized void doSetup() throws Exception {
Map<String, String> props = Maps.newHashMapWithExpectedSize(1);
props.put(PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY, Integer.toString(60*60)); // An hour
props.put(QueryServices.USE_STATS_FOR_PARALLELIZATION, Boolean.toString(false));
- /**
- * This test checks Table properties at ColumnFamilyDescriptor level, turing phoenix_table_ttl
- * to false for them to test TTL and other props at HBase level. TTL being set at phoenix level
- * is being tested in {@link TTLAsPhoenixTTLIT}
- */
- props.put(QueryServices.PHOENIX_TABLE_TTL_ENABLED, Boolean.toString(false));
setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
}
@@ -475,8 +468,7 @@ public class CreateTableIT extends ParallelStatsDisabledIT {
assertEquals(1, columnFamilies.length);
assertEquals(86400, columnFamilies[0].getTimeToLive());
//Check if TTL is stored in SYSCAT as well and we are getting ttl from get api in PTable
- assertEquals(new LiteralTTLExpression(86400), conn.unwrap(PhoenixConnection.class).getTable(
- new PTableKey(null, tableName)).getTTLExpression());
+ TestUtil.assertTTLValue(conn, tableName, new LiteralTTLExpression(86400));
}
@Test
@@ -534,8 +526,8 @@ public class CreateTableIT extends ParallelStatsDisabledIT {
assertEquals(86400, columnFamilies[1].getTimeToLive());
assertEquals("C", columnFamilies[1].getNameAsString());
//Check if TTL is stored in SYSCAT as well and we are getting ttl from get api in PTable
- assertEquals(new LiteralTTLExpression(86400), conn.unwrap(PhoenixConnection.class).getTable(
- new PTableKey(null, tableName)).getTTLExpression());
+ TestUtil.assertTTLValue(conn, tableName,
+ new LiteralTTLExpression(86400));
}
/**
@@ -563,8 +555,7 @@ public class CreateTableIT extends ParallelStatsDisabledIT {
assertEquals("B", columnFamilies[1].getNameAsString());
assertEquals(86400, columnFamilies[1].getTimeToLive());
//Check if TTL is stored in SYSCAT as well and we are getting ttl from get api in PTable
- assertEquals(new LiteralTTLExpression(86400), conn.unwrap(PhoenixConnection.class).getTable(
- new PTableKey(null, tableName)).getTTLExpression());
+ TestUtil.assertTTLValue(conn, tableName, new LiteralTTLExpression(86400));
}
/**
@@ -644,8 +635,8 @@ public class CreateTableIT extends ParallelStatsDisabledIT {
assertEquals("a", columnFamilies[0].getNameAsString());
assertEquals(10000, columnFamilies[0].getTimeToLive());
//Check if TTL is stored in SYSCAT as well and we are getting ttl from get api in PTable
- assertEquals(new LiteralTTLExpression(10000), conn.unwrap(PhoenixConnection.class).getTable(
- new PTableKey(null, tableName)).getTTLExpression());
+ TestUtil.assertTTLValue(conn, tableName,
+ new LiteralTTLExpression(10000));
}
/**
@@ -669,8 +660,8 @@ public class CreateTableIT extends ParallelStatsDisabledIT {
assertEquals("a", columnFamilies[0].getNameAsString());
assertEquals(10000, columnFamilies[0].getTimeToLive());
//Check if TTL is stored in SYSCAT as well and we are getting ttl from get api in PTable
- assertEquals(new LiteralTTLExpression(10000), conn.unwrap(PhoenixConnection.class).getTable(
- new PTableKey(null, tableName)).getTTLExpression());
+ TestUtil.assertTTLValue(conn, tableName,
+ new LiteralTTLExpression(10000));
}
@Test
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/FlappingAlterTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/FlappingAlterTableIT.java
index 055bee49cb..7626cc833f 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/FlappingAlterTableIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/FlappingAlterTableIT.java
@@ -17,6 +17,7 @@
*/
package org.apache.phoenix.end2end;
+import static org.apache.phoenix.schema.LiteralTTLExpression.TTL_EXPRESSION_NOT_DEFINED;
import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
import static org.junit.Assert.assertEquals;
import static org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY;
@@ -31,13 +32,14 @@ import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.TableDescriptor;
-import org.apache.phoenix.coprocessor.BaseScannerRegionObserver;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.schema.LiteralTTLExpression;
import org.apache.phoenix.thirdparty.com.google.common.collect.Maps;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.SchemaUtil;
+import org.apache.phoenix.util.TestUtil;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -52,12 +54,6 @@ public class FlappingAlterTableIT extends ParallelStatsDisabledIT {
Map<String, String> props = Maps.newHashMapWithExpectedSize(1);
props.put(PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY, Integer.toString(60*60)); // An hour
props.put(QueryServices.USE_STATS_FOR_PARALLELIZATION, Boolean.toString(false));
- /**
- * This test checks Table properties at ColumnFamilyDescriptor level, turing phoenix_table_ttl
- * to false for them to test TTL and other props at HBase level. TTL being set at phoenix level
- * is being tested in {@link TTLAsPhoenixTTLIT}
- */
- props.put(QueryServices.PHOENIX_TABLE_TTL_ENABLED, Boolean.toString(false));
setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
}
@@ -68,6 +64,7 @@ public class FlappingAlterTableIT extends ParallelStatsDisabledIT {
dataTableFullName = SchemaUtil.getTableName(schemaName, dataTableName);
}
+
@Test
public void testAddColumnForNewColumnFamily() throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
@@ -82,6 +79,8 @@ public class FlappingAlterTableIT extends ParallelStatsDisabledIT {
conn1.createStatement().execute(ddl);
ddl = "ALTER TABLE " + dataTableFullName + " ADD CF.STRING VARCHAR";
conn1.createStatement().execute(ddl);
+ TestUtil.assertTTLValue(conn1, dataTableFullName, TTL_EXPRESSION_NOT_DEFINED);
+
try (Admin admin = conn1.unwrap(PhoenixConnection.class).getQueryServices().getAdmin()) {
ColumnFamilyDescriptor[] columnFamilies = admin.getDescriptor(TableName.valueOf(dataTableFullName)).getColumnFamilies();
assertEquals(2, columnFamilies.length);
@@ -106,6 +105,7 @@ public class FlappingAlterTableIT extends ParallelStatsDisabledIT {
conn1.createStatement().execute(ddl);
ddl = "ALTER TABLE " + dataTableFullName + " ADD CF.STRING VARCHAR";
conn1.createStatement().execute(ddl);
+ TestUtil.assertTTLValue(conn1, dataTableFullName, new LiteralTTLExpression((1000)));
try (Admin admin = conn1.unwrap(PhoenixConnection.class).getQueryServices().getAdmin()) {
TableDescriptor tableDesc = admin.getDescriptor(TableName.valueOf(dataTableFullName));
ColumnFamilyDescriptor[] columnFamilies = tableDesc.getColumnFamilies();
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/LongViewIndexDisabledBaseRowKeyMatcherIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/LongViewIndexDisabledBaseRowKeyMatcherIT.java
index 1136253a21..a01159f68c 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/LongViewIndexDisabledBaseRowKeyMatcherIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/LongViewIndexDisabledBaseRowKeyMatcherIT.java
@@ -34,13 +34,15 @@ import java.sql.Statement;
import java.util.HashMap;
import java.util.Map;
+import static org.apache.phoenix.query.QueryServices.SYSTEM_CATALOG_INDEXES_ENABLED;
+
@Category(NeedsOwnMiniClusterTest.class)
public class LongViewIndexDisabledBaseRowKeyMatcherIT extends BaseRowKeyMatcherTestIT {
@BeforeClass
public static synchronized void doSetup() throws Exception {
final Configuration conf = HBaseConfiguration.create();
- conf.set(QueryServices.PHOENIX_TABLE_TTL_ENABLED, String.valueOf(true));
+ conf.set(QueryServices.PHOENIX_COMPACTION_ENABLED, String.valueOf(true));
conf.set(QueryServices.SYSTEM_CATALOG_INDEXES_ENABLED, String.valueOf(true));
conf.set(QueryServices.LONG_VIEW_INDEX_ENABLED_ATTRIB, String.valueOf(false));
conf.set(QueryServices.INDEX_REGION_OBSERVER_ENABLED_ATTRIB, "true");
@@ -81,7 +83,6 @@ public class LongViewIndexDisabledBaseRowKeyMatcherIT extends BaseRowKeyMatcherT
stmt.execute("CREATE INDEX IF NOT EXISTS SYS_VIEW_INDEX_HDR_IDX ON SYSTEM.CATALOG(DECODE_VIEW_INDEX_ID(VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE), TENANT_ID, TABLE_SCHEM, TABLE_NAME) INCLUDE(TABLE_TYPE, LINK_TYPE, VIEW_INDEX_ID, VIEW_INDEX_ID_DATA_TYPE) WHERE TABLE_TYPE = 'i' AND LINK_TYPE IS NULL AND VIEW_INDEX_ID IS NOT NULL");
conn.commit();
}
-
}
@Override
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/LongViewIndexEnabledBaseRowKeyMatcherIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/LongViewIndexEnabledBaseRowKeyMatcherIT.java
index 3fa29c41b7..3d73c948ba 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/LongViewIndexEnabledBaseRowKeyMatcherIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/LongViewIndexEnabledBaseRowKeyMatcherIT.java
@@ -43,7 +43,7 @@ public class LongViewIndexEnabledBaseRowKeyMatcherIT extends BaseRowKeyMatcherTe
@BeforeClass
public static synchronized void doSetup() throws Exception {
final Configuration conf = HBaseConfiguration.create();
- conf.set(QueryServices.PHOENIX_TABLE_TTL_ENABLED, String.valueOf(true));
+ conf.set(QueryServices.PHOENIX_COMPACTION_ENABLED, String.valueOf(true));
conf.set(QueryServices.SYSTEM_CATALOG_INDEXES_ENABLED, String.valueOf(true));
conf.set(QueryServices.LONG_VIEW_INDEX_ENABLED_ATTRIB, String.valueOf(true));
conf.set(QueryServices.INDEX_REGION_OBSERVER_ENABLED_ATTRIB, "true");
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MaxLookbackExtendedIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MaxLookbackExtendedIT.java
index 987d631bfe..c66cb1a03a 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MaxLookbackExtendedIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MaxLookbackExtendedIT.java
@@ -61,7 +61,6 @@ import static org.apache.phoenix.util.TestUtil.assertRawCellCount;
import static org.apache.phoenix.util.TestUtil.assertRawRowCount;
import static org.apache.phoenix.util.TestUtil.assertRowExistsAtSCN;
import static org.apache.phoenix.util.TestUtil.assertRowHasExpectedValueAtSCN;
-import static org.apache.phoenix.util.TestUtil.assertTableHasTtl;
import static org.apache.phoenix.util.TestUtil.assertTableHasVersions;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
@@ -370,8 +369,8 @@ public class MaxLookbackExtendedIT extends BaseTest {
long afterFirstInsertSCN = EnvironmentEdgeManager.currentTimeMillis();
TableName dataTable = TableName.valueOf(dataTableName);
TableName indexTable = TableName.valueOf(indexName);
- assertTableHasTtl(conn, dataTable, ttl, true);
- assertTableHasTtl(conn, indexTable, ttl, true);
+ TestUtil.assertTTLValue(conn, dataTable, ttl);
+ TestUtil.assertTTLValue(conn, indexTable, ttl);
//first make sure we inserted correctly
String sql = String.format("SELECT val2 FROM %s WHERE id = 'a'", dataTableName);
String indexSql = String.format("SELECT val2 FROM %s WHERE val1 = 'ab'", dataTableName);
@@ -564,12 +563,13 @@ public class MaxLookbackExtendedIT extends BaseTest {
}
}
+
@Test(timeout=60000)
public void testRetainingLastRowVersion() throws Exception {
try(Connection conn = DriverManager.getConnection(getUrl())) {
String tableName = generateUniqueName();
createTable(tableName);
- long timeIntervalBetweenTwoUpserts = (ttl / 2) + 1;
+ long timeIntervalBetweenTwoUpserts = (ttl / 4) + 1;
injectEdge.setValue(System.currentTimeMillis());
EnvironmentEdgeManager.injectEdge(injectEdge);
TableName dataTableName = TableName.valueOf(tableName);
@@ -590,14 +590,21 @@ public class MaxLookbackExtendedIT extends BaseTest {
injectEdge.incrementValue(timeIntervalBetweenTwoUpserts * 1000);
flush(dataTableName);
injectEdge.incrementValue(1);
+ TestUtil.minorCompact(utility, dataTableName);
+ injectEdge.incrementValue(1);
stmt.execute("upsert into " + tableName + " values ('a', 'ab3')");
conn.commit();
- injectEdge.incrementValue(MAX_LOOKBACK_AGE * 1000);
+ injectEdge.incrementValue(timeIntervalBetweenTwoUpserts * 1000);
flush(dataTableName);
injectEdge.incrementValue(1);
+ stmt.execute("upsert into " + tableName + " values ('a', 'ab4')");
+ conn.commit();
+ injectEdge.incrementValue(1);
+ flush(dataTableName);
TestUtil.dumpTable(conn, dataTableName);
- TestUtil.minorCompact(utility, dataTableName);
injectEdge.incrementValue(1);
+ TestUtil.minorCompact(utility, dataTableName);
+ injectEdge.incrementValue(MAX_LOOKBACK_AGE * 1000 - 1);
TestUtil.dumpTable(conn, dataTableName);
majorCompact(dataTableName);
injectEdge.incrementValue(1);
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MaxLookbackIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MaxLookbackIT.java
index d1badf67f2..b77965d74c 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MaxLookbackIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MaxLookbackIT.java
@@ -22,7 +22,6 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants;
import org.apache.phoenix.exception.SQLExceptionCode;
import org.apache.phoenix.query.BaseTest;
import org.apache.phoenix.query.QueryServices;
@@ -54,7 +53,6 @@ import static org.apache.phoenix.util.TestUtil.assertRawCellCount;
import static org.apache.phoenix.util.TestUtil.assertRawRowCount;
import static org.apache.phoenix.util.TestUtil.assertRowExistsAtSCN;
import static org.apache.phoenix.util.TestUtil.assertRowHasExpectedValueAtSCN;
-import static org.apache.phoenix.util.TestUtil.assertTableHasTtl;
import static org.apache.phoenix.util.TestUtil.assertTableHasVersions;
import static org.junit.Assert.assertFalse;
@@ -73,7 +71,7 @@ public class MaxLookbackIT extends BaseTest {
Map<String, String> props = Maps.newHashMapWithExpectedSize(3);
props.put(QueryServices.GLOBAL_INDEX_ROW_AGE_THRESHOLD_TO_DELETE_MS_ATTRIB, Long.toString(0));
props.put(PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY, Integer.toString(MAX_LOOKBACK_AGE));
- props.put(QueryServices.PHOENIX_TABLE_TTL_ENABLED, Boolean.toString(Boolean.FALSE));
+ props.put(QueryServices.PHOENIX_COMPACTION_ENABLED, Boolean.toString(false));
props.put("hbase.procedure.remote.dispatcher.delay.msec", "0");
setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
}
@@ -210,8 +208,8 @@ public class MaxLookbackIT extends BaseTest {
long afterFirstInsertSCN = EnvironmentEdgeManager.currentTimeMillis();
TableName dataTable = TableName.valueOf(dataTableName);
TableName indexTable = TableName.valueOf(indexName);
- assertTableHasTtl(conn, dataTable, ttl, false);
- assertTableHasTtl(conn, indexTable, ttl, false);
+ TestUtil.assertTTLValue(conn, dataTable, ttl);
+ TestUtil.assertTTLValue(conn, indexTable, ttl);
//first make sure we inserted correctly
String sql = String.format("SELECT val2 FROM %s WHERE id = 'a'", dataTableName);
String indexSql = String.format("SELECT val2 FROM %s WHERE val1 = 'ab'", dataTableName);
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MoveTTLDuringUpgradeIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MoveTTLDuringUpgradeIT.java
index 95da46f9b6..45ace8d26a 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MoveTTLDuringUpgradeIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MoveTTLDuringUpgradeIT.java
@@ -24,13 +24,28 @@ import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.TableDescriptor;
import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.coprocessor.CompactionScanner;
+import org.apache.phoenix.exception.SQLExceptionCode;
+import org.apache.phoenix.exception.SQLExceptionInfo;
import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
+import org.apache.phoenix.query.BaseTest;
+import org.apache.phoenix.query.ConnectionQueryServicesImpl;
import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.schema.LiteralTTLExpression;
+import org.apache.phoenix.schema.PTableKey;
+import org.apache.phoenix.schema.PTableType;
+import org.apache.phoenix.util.ClientUtil;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.SchemaUtil;
+import org.apache.phoenix.util.TestUtil;
import org.apache.phoenix.util.UpgradeUtil;
+import org.junit.Ignore;
import org.junit.Test;
import org.junit.experimental.categories.Category;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.sql.Connection;
@@ -40,18 +55,25 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Types;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
import java.util.Properties;
+import java.util.Set;
+import java.util.concurrent.TimeoutException;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_TTL;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TTL_DEFINED_IN_TABLE_DESCRIPTOR;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TTL_NOT_DEFINED;
import static org.apache.phoenix.query.QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES;
import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_TIMEOUT_DURING_UPGRADE_MS;
import static org.apache.phoenix.thirdparty.com.google.common.base.Preconditions.checkNotNull;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
@Category(NeedsOwnMiniClusterTest.class)
public class MoveTTLDuringUpgradeIT extends ParallelStatsDisabledIT {
+ private static final Logger LOGGER = LoggerFactory.getLogger(MoveTTLDuringUpgradeIT.class);
+
@Test
public void testMoveHBaseLevelTTLToSYSCAT() throws Exception {
String schema = "S_" + generateUniqueName();
@@ -71,23 +93,45 @@ public class MoveTTLDuringUpgradeIT extends ParallelStatsDisabledIT {
options.put(QueryServices.ZOOKEEPER_QUORUM_ATTRIB, localQuorum);
options.put(QueryServices.ZOOKEEPER_PORT_ATTRIB, clientPort);
UpgradeUtil.moveHBaseLevelTTLToSYSCAT(phxMetaConn, options);
+ conn.getQueryServices().clearCache();
+ phxMetaConn.getQueryServices().clearCache();
+
+ //TestUtil.dumpTable(conn, TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES));
- String sql = "SELECT TABLE_NAME, TTL FROM SYSTEM.CATALOG WHERE TABLE_SCHEM = '" + schema + "'";
- ResultSet rs = conn.createStatement().executeQuery(sql);
+ String sql = String.format("SELECT TABLE_NAME, TTL FROM SYSTEM.CATALOG WHERE TABLE_SCHEM = '%s' AND TABLE_TYPE = '%s' AND TTL IS NOT NULL", schema,
+ PTableType.TABLE.getSerializedValue());
+ ResultSet rs = phxMetaConn.createStatement().executeQuery(sql);
+ int count = 0;
while (rs.next()) {
+ count++;
String table = rs.getString(1);
+ LOGGER.info("found table: {} ", table);
int ttl = tableTTLMap.get(table);
+ int ttlInSyscat = Integer.valueOf(rs.getString(2)).intValue();
//Check if TTL is moved to SYSCAT.
- if (ttl != HConstants.FOREVER) {
- assertEquals(ttl, Integer.valueOf(rs.getString(2)).intValue());
- } else {
- assertEquals(ttl, TTL_NOT_DEFINED);
- }
+ //assertEquals(ttl, );
//Check if TTL at HBase level is reset.
TableDescriptor tableDescriptor = admin.getDescriptor(TableName.valueOf(SchemaUtil.getTableName(schema, table)));
ColumnFamilyDescriptor[] columnFamilies = tableDescriptor.getColumnFamilies();
- assertEquals(DEFAULT_TTL, columnFamilies[0].getTimeToLive());
+ if (ttlInSyscat == TTL_DEFINED_IN_TABLE_DESCRIPTOR) {
+ assertEquals(ttl, columnFamilies[0].getTimeToLive());
+ assertEquals(new LiteralTTLExpression(TTL_DEFINED_IN_TABLE_DESCRIPTOR),
+ phxMetaConn.unwrap(PhoenixConnection.class).getTableNoCache(
+ null,
+ SchemaUtil.getTableName(Bytes.toBytes(schema),
+ Bytes.toBytes(table))).getTTLExpression());
+ } else {
+ // Should not have any value other than TTL_DEFINED_IN_TABLE_DESCRIPTOR when TTL is not null
+ fail();
+ }
+
}
+ if (count == 0) {
+ // Should have at least one table with TTL defined
+ fail();
+ }
+ } catch (Exception e) {
+ LOGGER.error(e.getMessage(), e);
}
}
@@ -97,32 +141,28 @@ public class MoveTTLDuringUpgradeIT extends ParallelStatsDisabledIT {
int numOfTable = 20;
int randomTTL = 0;
Map<String, Integer> tableTTLMap = new HashMap<>();
+ //Set<TableDescriptor> modifiedTableDescriptors = new HashSet<>();
+
for (int i = 0; i < numOfTable; i++ ) {
table = "T_" + generateUniqueName();
- randomTTL = i%3 == 0 ? HConstants.FOREVER : 100 + (int)(Math.random() * 1000);
+ randomTTL = i%3 == 0 ? HConstants.FOREVER : 100 + (int)(Math.random() * 1000);
tableTTLMap.put(table, randomTTL);
+ String ddl = "CREATE TABLE " + schema + "." + table +
+ " (a_string varchar not null, b_string varbinary not null, col1 integer" +
+ " CONSTRAINT pk PRIMARY KEY (a_string, b_string)) ";
+
+ BaseTest.createTestTable(getUrl(), ddl);
TableName tableName = TableName.valueOf(SchemaUtil.getTableName(schema, table));
TableDescriptorBuilder builder = TableDescriptorBuilder.newBuilder(tableName);
builder.setColumnFamily(ColumnFamilyDescriptorBuilder.newBuilder(
DEFAULT_COLUMN_FAMILY_BYTES).setTimeToLive(randomTTL).build());
- admin.createTable(builder.build());
- upsertIntoSYSCAT(schema, table);
+ admin.disableTable(tableName);
+ admin.modifyTable(builder.build());
+ admin.enableTable(tableName);
}
return tableTTLMap;
}
- private void upsertIntoSYSCAT(String schema, String table) throws SQLException {
- try (PhoenixConnection connection = getConnection(false, null).unwrap(PhoenixConnection.class)){
- String dml = "UPSERT INTO SYSTEM.CATALOG (TENANT_ID, TABLE_SCHEM, TABLE_NAME, TTL) VALUES (?,?,?,?)";
- PreparedStatement statement = connection.prepareStatement(dml);
- statement.setString(1, null);
- statement.setString(2, schema);
- statement.setString(3, table);
- statement.setNull(4, Types.INTEGER);
- }
-
- }
-
private Connection getConnection(boolean tenantSpecific, String tenantId,
boolean isNamespaceMappingEnabled, boolean copyChildLinksDuringUpgrade)
throws SQLException {
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/PropertiesInSyncIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/PropertiesInSyncIT.java
index 2e80994815..fde6e3515f 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/PropertiesInSyncIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/PropertiesInSyncIT.java
@@ -30,9 +30,11 @@ import org.apache.phoenix.coprocessor.BaseScannerRegionObserver;
import org.apache.phoenix.exception.SQLExceptionCode;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.schema.LiteralTTLExpression;
import org.apache.phoenix.schema.PName;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTableType;
+import org.apache.phoenix.schema.TTLExpression;
import org.apache.phoenix.thirdparty.com.google.common.collect.Maps;
import org.apache.phoenix.util.MetaDataUtil;
import org.apache.phoenix.util.PhoenixRuntime;
@@ -90,12 +92,6 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
Map<String, String> props = Maps.newHashMapWithExpectedSize(1);
props.put(PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY, Integer.toString(60*60)); // An hour
props.put(QueryServices.USE_STATS_FOR_PARALLELIZATION, Boolean.toString(false));
- /**
- * This test checks Table properties at ColumnFamilyDescriptor level, turing phoenix_table_ttl
- * to false for them to test TTL and other props at HBase level. TTL being set at phoenix level
- * is being tested in {@link TTLAsPhoenixTTLIT}
- */
- props.put(QueryServices.PHOENIX_TABLE_TTL_ENABLED, Boolean.toString(false));
setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SetPropertyIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SetPropertyIT.java
index 95c841d128..78e9bf7ac1 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SetPropertyIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SetPropertyIT.java
@@ -18,7 +18,6 @@
package org.apache.phoenix.end2end;
import static org.apache.phoenix.coprocessorclient.BaseScannerRegionObserverConstants.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY;
-import static org.apache.phoenix.schema.LiteralTTLExpression.TTL_EXPRESSION_FOREVER;
import static org.apache.phoenix.schema.LiteralTTLExpression.TTL_EXPRESSION_NOT_DEFINED;
import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
import static org.junit.Assert.assertEquals;
@@ -45,12 +44,12 @@ import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTableKey;
-import org.apache.phoenix.schema.TTLExpression;
import org.apache.phoenix.schema.LiteralTTLExpression;
import org.apache.phoenix.thirdparty.com.google.common.collect.Maps;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.SchemaUtil;
+import org.apache.phoenix.util.TestUtil;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Ignore;
@@ -60,6 +59,7 @@ import org.junit.runners.Parameterized;
@RunWith(Parameterized.class)
public abstract class SetPropertyIT extends ParallelStatsDisabledIT {
+
private String schemaName;
private String dataTableName;
private String viewName;
@@ -72,12 +72,6 @@ public abstract class SetPropertyIT extends ParallelStatsDisabledIT {
Map<String, String> props = Maps.newHashMapWithExpectedSize(1);
props.put(PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY, Integer.toString(60*60)); // An hour
props.put(QueryServices.USE_STATS_FOR_PARALLELIZATION, Boolean.toString(false));
- /**
- * This test checks Table properties at ColumnFamilyDescriptor level, turing phoenix_table_ttl
- * to false for them to test TTL and other props at HBase level. TTL being set at phoenix level
- * is being tested in {@link TTLAsPhoenixTTLIT}
- */
- props.put(QueryServices.PHOENIX_TABLE_TTL_ENABLED, Boolean.toString(false));
setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
}
@@ -431,9 +425,9 @@ public abstract class SetPropertyIT extends ParallelStatsDisabledIT {
assertEquals(1000, columnFamilies[1].getTimeToLive());
assertEquals(ColumnFamilyDescriptorBuilder.DEFAULT_BLOCKSIZE, columnFamilies[1].getBlocksize());
assertEquals(Boolean.toString(false), tableDesc.getValue(TableDescriptorBuilder.COMPACTION_ENABLED));
- //Check if Alter Table TTL also changes TTL stored in SYSCAT with phoenix.table.ttl disabled
- assertEquals(new LiteralTTLExpression(1000), conn.unwrap(PhoenixConnection.class).getTable(
- new PTableKey(null, dataTableFullName)).getTTLExpression());
+ //Check if Alter Table TTL also changes TTL stored in SYSCAT
+ TestUtil.assertTTLValue(conn, dataTableFullName,
+ new LiteralTTLExpression(1000));
}
}
@@ -795,8 +789,8 @@ public abstract class SetPropertyIT extends ParallelStatsDisabledIT {
assertEquals("XYZ", columnFamilies[0].getNameAsString());
assertEquals(86400, columnFamilies[0].getTimeToLive());
//Check if TTL is stored in SYSCAT as well and we are getting ttl from get api in PTable
- assertEquals(new LiteralTTLExpression(86400), conn.unwrap(PhoenixConnection.class).getTable(
- new PTableKey(null, dataTableFullName)).getTTLExpression());
+ TestUtil.assertTTLValue(conn, dataTableFullName,
+ new LiteralTTLExpression(86400));
}
ddl = "ALTER TABLE " + dataTableFullName + " SET TTL=30";
conn.createStatement().execute(ddl);
@@ -807,9 +801,9 @@ public abstract class SetPropertyIT extends ParallelStatsDisabledIT {
assertEquals(1, columnFamilies.length);
assertEquals(30, columnFamilies[0].getTimeToLive());
assertEquals("XYZ", columnFamilies[0].getNameAsString());
- //Check if Alter Table TTL also changes TTL stored in SYSCAT with phoenix.table.ttl disabled
- assertEquals(new LiteralTTLExpression(30), conn.unwrap(PhoenixConnection.class).getTable(
- new PTableKey(null, dataTableFullName)).getTTLExpression());
+ //Check if Alter Table TTL also changes TTL stored in SYSCAT
+ TestUtil.assertTTLValue(conn, dataTableFullName,
+ new LiteralTTLExpression(30));
}
} finally {
conn.close();
@@ -848,8 +842,8 @@ public abstract class SetPropertyIT extends ParallelStatsDisabledIT {
assertEquals(1, columnFamilies.length);
assertEquals(HConstants.FOREVER, columnFamilies[0].getTimeToLive());
assertEquals("XYZ", columnFamilies[0].getNameAsString());
- //Check if Alter Table TTL also changes TTL stored in SYSCAT with phoenix.table.ttl disabled
- assertEquals(TTL_EXPRESSION_FOREVER, conn.unwrap(PhoenixConnection.class).getTable(
+ //Check if Alter Table TTL also changes TTL stored in SYSCAT
+ assertEquals(TTL_EXPRESSION_NOT_DEFINED, conn.unwrap(PhoenixConnection.class).getTable(
new PTableKey(null, dataTableFullName)).getTTLExpression());
}
} finally {
@@ -998,8 +992,8 @@ public abstract class SetPropertyIT extends ParallelStatsDisabledIT {
assertEquals(false, columnFamilies[1].isInMemory());
assertEquals(86400, columnFamilies[1].getTimeToLive());
//Check if TTL is stored in SYSCAT as well and we are getting ttl from get api in PTable
- assertEquals(new LiteralTTLExpression(86400), conn.unwrap(PhoenixConnection.class).getTable(
- new PTableKey(null, dataTableFullName)).getTTLExpression());
+ TestUtil.assertTTLValue(conn, dataTableFullName,
+ new LiteralTTLExpression(86400));
}
ddl = "ALTER TABLE " + dataTableFullName + " SET TTL=1000";
@@ -1015,9 +1009,9 @@ public abstract class SetPropertyIT extends ParallelStatsDisabledIT {
assertEquals("XYZ", columnFamilies[1].getNameAsString());
assertEquals(false, columnFamilies[1].isInMemory());
assertEquals(1000, columnFamilies[1].getTimeToLive());
- //Check if Alter Table TTL also changes TTL stored in SYSCAT with phoenix.table.ttl disabled
- assertEquals(new LiteralTTLExpression(1000), conn.unwrap(PhoenixConnection.class).getTable(
- new PTableKey(null, dataTableFullName)).getTTLExpression());
+ //Check if Alter Table TTL also changes TTL stored in SYSCAT
+ TestUtil.assertTTLValue(conn, dataTableFullName,
+ new LiteralTTLExpression(1000));
}
// the new column will be assigned to the column family XYZ. With the a KV column getting added for XYZ,
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TTLAsPhoenixTTLIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TTLAsPhoenixTTLIT.java
index d415b0e6e3..9b270093ed 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TTLAsPhoenixTTLIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TTLAsPhoenixTTLIT.java
@@ -22,10 +22,13 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
+import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.schema.ColumnNotFoundException;
import org.apache.phoenix.schema.ColumnFamilyNotFoundException;
+import org.apache.phoenix.schema.LiteralTTLExpression;
import org.apache.phoenix.schema.PName;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTableKey;
@@ -33,11 +36,13 @@ import org.apache.phoenix.schema.TTLExpression;
import org.apache.phoenix.schema.TTLExpressionFactory;
import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.PhoenixRuntime;
+import org.apache.phoenix.util.TestUtil;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
+import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
@@ -49,6 +54,8 @@ import java.util.Properties;
import static org.apache.phoenix.exception.SQLExceptionCode.CANNOT_SET_OR_ALTER_PROPERTY_FOR_INDEX;
import static org.apache.phoenix.exception.SQLExceptionCode.TTL_ALREADY_DEFINED_IN_HIERARCHY;
import static org.apache.phoenix.exception.SQLExceptionCode.TTL_SUPPORTED_FOR_TABLES_AND_VIEWS_ONLY;
+import static org.apache.phoenix.query.QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES;
+import static org.apache.phoenix.schema.LiteralTTLExpression.TTL_EXPRESSION_DEFINED_IN_TABLE_DESCRIPTOR;
import static org.apache.phoenix.schema.LiteralTTLExpression.TTL_EXPRESSION_FOREVER;
import static org.apache.phoenix.schema.LiteralTTLExpression.TTL_EXPRESSION_NOT_DEFINED;
import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB;
@@ -118,7 +125,7 @@ public class TTLAsPhoenixTTLIT extends ParallelStatsDisabledIT{
try (Connection conn = DriverManager.getConnection(getUrl());) {
PTable table = conn.unwrap(PhoenixConnection.class).getTable(new PTableKey(null,
createTableWithOrWithOutTTLAsItsProperty(conn, true)));
- assertTTLValue(table, defaultTTL);
+ TestUtil.assertTTLValue(conn, table, defaultTTL);
assertTrue("RowKeyMatcher should be Null",
(Bytes.compareTo(HConstants.EMPTY_BYTE_ARRAY, table.getRowKeyMatcher()) == 0));
}
@@ -129,7 +136,7 @@ public class TTLAsPhoenixTTLIT extends ParallelStatsDisabledIT{
try (Connection conn = DriverManager.getConnection(getUrl());) {
PTable table = conn.unwrap(PhoenixConnection.class).getTable(new PTableKey(null,
createTableWithOrWithOutTTLAsItsProperty(conn, false)));
- assertTTLValue(table, TTL_EXPRESSION_NOT_DEFINED);
+ TestUtil.assertTTLValue(conn, table, TTL_EXPRESSION_NOT_DEFINED);
assertFalse(table.hasConditionalTTL());
}
}
@@ -140,7 +147,7 @@ public class TTLAsPhoenixTTLIT extends ParallelStatsDisabledIT{
String tableName = createTableWithOrWithOutTTLAsItsProperty(conn, true);
PTable table = conn.unwrap(PhoenixConnection.class).
getTable(new PTableKey(null, tableName));
- assertTTLValue(table, defaultTTL);
+ TestUtil.assertTTLValue(conn, table, defaultTTL);
assertTrue(table.hasConditionalTTL() == useExpression);
// Switch from cond ttl to value or vice versa
String alterTTL = useExpression ? String.valueOf(DEFAULT_TTL_FOR_ALTER) :
@@ -150,7 +157,7 @@ public class TTLAsPhoenixTTLIT extends ParallelStatsDisabledIT{
TTLExpression expected = useExpression ?
TTLExpressionFactory.create(DEFAULT_TTL_FOR_ALTER) :
TTLExpressionFactory.create(DEFAULT_TTL_EXPRESSION_FOR_ALTER);
- assertTTLValue(conn.unwrap(PhoenixConnection.class), expected, tableName);
+ TestUtil.assertTTLValue(conn.unwrap(PhoenixConnection.class), expected, tableName);
}
}
@@ -173,13 +180,12 @@ public class TTLAsPhoenixTTLIT extends ParallelStatsDisabledIT{
Connection conn = DriverManager.getConnection(getUrl());
conn.createStatement().execute(ddl);
TTLExpression expected = TTLExpressionFactory.create(DEFAULT_TTL_FOR_TEST);
- assertTTLValue(conn.unwrap(PhoenixConnection.class), expected, tableName);
- //Setting TTL should not be stored as CF Descriptor properties when
- //phoenix.table.ttl.enabled is true
+ TestUtil.assertTTLValue(conn.unwrap(PhoenixConnection.class), expected, tableName);
+ //Setting TTL should also stored in CF Descriptor properties for Backward compatibility
Admin admin = driver.getConnectionQueryServices(getUrl(), new Properties()).getAdmin();
ColumnFamilyDescriptor[] columnFamilies =
admin.getDescriptor(TableName.valueOf(tableName)).getColumnFamilies();
- assertEquals(ColumnFamilyDescriptorBuilder.DEFAULT_TTL, columnFamilies[0].getTimeToLive());
+ assertEquals(DEFAULT_TTL_FOR_TEST, columnFamilies[0].getTimeToLive());
}
@Test
@@ -195,19 +201,19 @@ public class TTLAsPhoenixTTLIT extends ParallelStatsDisabledIT{
+ " ) TTL=FOREVER";
try (Connection conn = DriverManager.getConnection(getUrl())) {
conn.createStatement().execute(ddl);
- assertTTLValue(conn.unwrap(PhoenixConnection.class),
- TTL_EXPRESSION_FOREVER, tableName);
+ TestUtil.assertTTLValue(conn.unwrap(PhoenixConnection.class),
+ TTL_EXPRESSION_NOT_DEFINED, tableName);
ddl = "ALTER TABLE " + tableName + " SET TTL=NONE";
conn.createStatement().execute(ddl);
- assertTTLValue(conn.unwrap(PhoenixConnection.class),
+ TestUtil.assertTTLValue(conn.unwrap(PhoenixConnection.class),
TTL_EXPRESSION_NOT_DEFINED, tableName);
//Setting TTL should not be stored as CF Descriptor properties when
//phoenix.table.ttl.enabled is true
Admin admin = driver.getConnectionQueryServices(getUrl(), new Properties()).getAdmin();
ColumnFamilyDescriptor[] columnFamilies =
admin.getDescriptor(TableName.valueOf(tableName)).getColumnFamilies();
- assertEquals(ColumnFamilyDescriptorBuilder.DEFAULT_TTL, columnFamilies[0].getTimeToLive());
+ assertEquals(HConstants.FOREVER, columnFamilies[0].getTimeToLive());
tableName = generateUniqueName();
ddl =
@@ -216,13 +222,13 @@ public class TTLAsPhoenixTTLIT extends ParallelStatsDisabledIT{
+ " CONSTRAINT NAME_PK PRIMARY KEY (id, col1)"
+ " ) TTL=NONE";
conn.createStatement().execute(ddl);
- assertTTLValue(conn.unwrap(PhoenixConnection.class),
+ TestUtil.assertTTLValue(conn.unwrap(PhoenixConnection.class),
TTL_EXPRESSION_NOT_DEFINED, tableName);
ddl = "ALTER TABLE " + tableName + " SET TTL=FOREVER";
conn.createStatement().execute(ddl);
- assertTTLValue(conn.unwrap(PhoenixConnection.class),
- TTL_EXPRESSION_FOREVER, tableName);
+ TestUtil.assertTTLValue(conn.unwrap(PhoenixConnection.class),
+ TTL_EXPRESSION_NOT_DEFINED, tableName);
//Setting TTL should not be stored as CF Descriptor properties when
//phoenix.table.ttl.enabled is true
columnFamilies =
@@ -237,17 +243,18 @@ public class TTLAsPhoenixTTLIT extends ParallelStatsDisabledIT{
PhoenixConnection pConn = conn.unwrap(PhoenixConnection.class);){
String tableName = createTableWithOrWithOutTTLAsItsProperty(conn, false);
//Checking Default TTL in case of PhoenixTTLEnabled
- assertTTLValue(conn.unwrap(PhoenixConnection.class), TTL_EXPRESSION_NOT_DEFINED, tableName);
+ TestUtil.assertTTLValue(conn.unwrap(PhoenixConnection.class), TTL_EXPRESSION_NOT_DEFINED, tableName);
String ddl = "ALTER TABLE " + tableName + " SET TTL = " + this.alterTTLDDLOption;
conn.createStatement().execute(ddl);
- assertTTLValue(conn.unwrap(PhoenixConnection.class), this.alterTTL, tableName);
+ TestUtil.assertTTLValue(conn.unwrap(PhoenixConnection.class), this.alterTTL, tableName);
//Asserting TTL should not be stored as CF Descriptor properties when
//phoenix.table.ttl.enabled is true
Admin admin = driver.getConnectionQueryServices(getUrl(), new Properties()).getAdmin();
ColumnFamilyDescriptor[] columnFamilies =
admin.getDescriptor(TableName.valueOf(tableName)).getColumnFamilies();
- assertEquals(ColumnFamilyDescriptorBuilder.DEFAULT_TTL, columnFamilies[0].getTimeToLive());
+ PTable pTable = pConn.getTableNoCache(tableName);
+ TestUtil.assertTTLValue(conn.unwrap(PhoenixConnection.class), this.alterTTL, tableName);
}
}
@@ -261,7 +268,7 @@ public class TTLAsPhoenixTTLIT extends ParallelStatsDisabledIT{
createIndexOnTableOrViewProvidedWithTTL(conn, tableName, PTable.IndexType.GLOBAL, false);
List<PTable> indexes = PhoenixRuntime.getTable(conn, tableName).getIndexes();
for (PTable index : indexes) {
- assertTTLValue(index, defaultTTL);
+ TestUtil.assertTTLValue(conn, index, defaultTTL);
}
tableName = createTableWithOrWithOutTTLAsItsProperty(conn, false);
@@ -270,7 +277,7 @@ public class TTLAsPhoenixTTLIT extends ParallelStatsDisabledIT{
String globalIndexName = createIndexOnTableOrViewProvidedWithTTL(conn, tableName, PTable.IndexType.GLOBAL, false);
indexes = conn.unwrap(PhoenixConnection.class).getTable(new PTableKey(null, tableName)).getIndexes();
for (PTable index : indexes) {
- assertTTLValue(index, TTL_EXPRESSION_NOT_DEFINED);
+ TestUtil.assertTTLValue(conn, index, TTL_EXPRESSION_NOT_DEFINED);
assertTrue(Bytes.compareTo(
index.getRowKeyMatcher(), HConstants.EMPTY_BYTE_ARRAY) == 0
);
@@ -332,10 +339,10 @@ public class TTLAsPhoenixTTLIT extends ParallelStatsDisabledIT{
String ttl = "ID2 = 34 AND COL2 > CURRENT_DATE() + 1000";
conn.createStatement().execute(String.format(ddl, tableName, ttl));
TTLExpression expected = TTLExpressionFactory.create(ttl);
- assertTTLValue(conn.unwrap(PhoenixConnection.class), expected, tableName);
+ TestUtil.assertTTLValue(conn.unwrap(PhoenixConnection.class), expected, tableName);
conn.createStatement().execute(String.format("ALTER TABLE %s SET TTL=NONE", tableName));
- assertTTLValue(conn.unwrap(PhoenixConnection.class), TTL_EXPRESSION_NOT_DEFINED, tableName);
+ TestUtil.assertTTLValue(conn.unwrap(PhoenixConnection.class), TTL_EXPRESSION_NOT_DEFINED, tableName);
try {
conn.createStatement().execute(String.format("ALTER TABLE %s SET TTL='%s'",
@@ -358,13 +365,13 @@ public class TTLAsPhoenixTTLIT extends ParallelStatsDisabledIT{
ttl = "COL2 > CURRENT_DATE() + 200 AND VINT > 123";
conn.createStatement().execute(String.format(ddl, viewName, tableName, ttl));
expected = TTLExpressionFactory.create(ttl);
- assertTTLValue(conn.unwrap(PhoenixConnection.class), expected, viewName);
+ TestUtil.assertTTLValue(conn.unwrap(PhoenixConnection.class), expected, viewName);
ttl = "COL2 > CURRENT_DATE() + 500 AND VINT > 123";
conn.createStatement().execute(String.format("ALTER VIEW %s SET TTL='%s'",
viewName, ttl));
expected = TTLExpressionFactory.create(ttl);
- assertTTLValue(conn.unwrap(PhoenixConnection.class), expected, viewName);
+ TestUtil.assertTTLValue(conn.unwrap(PhoenixConnection.class), expected, viewName);
}
}
@@ -383,7 +390,7 @@ public class TTLAsPhoenixTTLIT extends ParallelStatsDisabledIT{
Connection tenantConn1 = DriverManager.getConnection(getUrl(), props1);
String tableName = createTableWithOrWithOutTTLAsItsProperty(conn, true);
- assertTTLValue(conn.unwrap(PhoenixConnection.class), defaultTTL, tableName);
+ TestUtil.assertTTLValue(conn.unwrap(PhoenixConnection.class), defaultTTL, tableName);
//Setting TTL on views is not allowed if Table already has TTL
try {
@@ -405,14 +412,14 @@ public class TTLAsPhoenixTTLIT extends ParallelStatsDisabledIT{
//View should have gotten TTL from parent table.
String viewName = createUpdatableViewOnTableWithTTL(conn, tableName, false);
- assertTTLValue(conn.unwrap(PhoenixConnection.class), defaultTTL, viewName);
+ TestUtil.assertTTLValue(conn.unwrap(PhoenixConnection.class), defaultTTL, viewName);
//Child View's PTable gets TTL from parent View's PTable which gets from Table.
String childView = createViewOnViewWithTTL(tenantConn, viewName, false);
- assertTTLValue(tenantConn.unwrap(PhoenixConnection.class), defaultTTL, childView);
+ TestUtil.assertTTLValue(tenantConn.unwrap(PhoenixConnection.class), defaultTTL, childView);
String childView1 = createViewOnViewWithTTL(tenantConn1, viewName, false);
- assertTTLValue(tenantConn1.unwrap(PhoenixConnection.class), defaultTTL, childView1);
+ TestUtil.assertTTLValue(tenantConn1.unwrap(PhoenixConnection.class), defaultTTL, childView1);
createIndexOnTableOrViewProvidedWithTTL(conn, viewName, PTable.IndexType.GLOBAL,
false);
@@ -423,7 +430,7 @@ public class TTLAsPhoenixTTLIT extends ParallelStatsDisabledIT{
conn.unwrap(PhoenixConnection.class), viewName).getIndexes();
for (PTable index : indexes) {
- assertTTLValue(index, defaultTTL);
+ TestUtil.assertTTLValue(conn, index, defaultTTL);
}
createIndexOnTableOrViewProvidedWithTTL(conn, tableName, PTable.IndexType.GLOBAL, false);
@@ -431,7 +438,7 @@ public class TTLAsPhoenixTTLIT extends ParallelStatsDisabledIT{
conn.unwrap(PhoenixConnection.class), tableName).getIndexes();
for (PTable index : tIndexes) {
- assertTTLValue(index, defaultTTL);
+ TestUtil.assertTTLValue(conn, index, defaultTTL);
}
}
}
@@ -446,7 +453,7 @@ public class TTLAsPhoenixTTLIT extends ParallelStatsDisabledIT{
Connection tenantConn = DriverManager.getConnection(getUrl(), props);
String tableName = createTableWithOrWithOutTTLAsItsProperty(conn, true);
- assertTTLValue(conn.unwrap(PhoenixConnection.class), defaultTTL, tableName);
+ TestUtil.assertTTLValue(conn.unwrap(PhoenixConnection.class), defaultTTL, tableName);
//Setting TTL on views is not allowed if Table already has TTL
try {
@@ -460,14 +467,14 @@ public class TTLAsPhoenixTTLIT extends ParallelStatsDisabledIT{
String ddl = "ALTER TABLE " + tableName + " SET TTL=NONE";
conn.createStatement().execute(ddl);
- assertTTLValue(conn.unwrap(PhoenixConnection.class),
+ TestUtil.assertTTLValue(conn.unwrap(PhoenixConnection.class),
TTL_EXPRESSION_NOT_DEFINED, tableName);
String viewName = createUpdatableViewOnTableWithTTL(conn, tableName, true);
TTLExpression expectedChildTTl = useExpression ?
TTLExpressionFactory.create(DEFAULT_TTL_EXPRESSION) :
TTLExpressionFactory.create(DEFAULT_TTL_FOR_CHILD);
- assertTTLValue(conn.unwrap(PhoenixConnection.class), expectedChildTTl, viewName);
+ TestUtil.assertTTLValue(conn.unwrap(PhoenixConnection.class), expectedChildTTl, viewName);
try {
createViewOnViewWithTTL(tenantConn, viewName, true);
@@ -492,12 +499,12 @@ public class TTLAsPhoenixTTLIT extends ParallelStatsDisabledIT{
conn.createStatement().execute(ddl);
String childView = createViewOnViewWithTTL(tenantConn, viewName, true);
- assertTTLValue(tenantConn.unwrap(PhoenixConnection.class), expectedChildTTl, childView);
+ TestUtil.assertTTLValue(tenantConn.unwrap(PhoenixConnection.class), expectedChildTTl, childView);
ddl = "ALTER VIEW " + childView + " SET TTL=NONE";
tenantConn.createStatement().execute(ddl);
- assertTTLValue(tenantConn.unwrap(PhoenixConnection.class),
+ TestUtil.assertTTLValue(tenantConn.unwrap(PhoenixConnection.class),
TTL_EXPRESSION_NOT_DEFINED, childView);
ddl = "ALTER VIEW " + viewName + " SET TTL=" + ttlAlter;
@@ -505,7 +512,7 @@ public class TTLAsPhoenixTTLIT extends ParallelStatsDisabledIT{
TTLExpression expectedAlterTTl = useExpression ?
TTLExpressionFactory.create(DEFAULT_TTL_EXPRESSION_FOR_ALTER) :
TTLExpressionFactory.create(DEFAULT_TTL_FOR_ALTER);
- assertTTLValue(conn.unwrap(PhoenixConnection.class), expectedAlterTTl, viewName);
+ TestUtil.assertTTLValue(conn.unwrap(PhoenixConnection.class), expectedAlterTTl, viewName);
}
}
@@ -524,18 +531,18 @@ public class TTLAsPhoenixTTLIT extends ParallelStatsDisabledIT{
Connection tenantConn1 = DriverManager.getConnection(getUrl(), props1);
String tableName = createTableWithOrWithOutTTLAsItsProperty(conn, true);
- assertTTLValue(conn.unwrap(PhoenixConnection.class), defaultTTL, tableName);
+ TestUtil.assertTTLValue(conn.unwrap(PhoenixConnection.class), defaultTTL, tableName);
//View should have gotten TTL from parent table.
String viewName = createUpdatableViewOnTableWithTTL(conn, tableName, false);
- assertTTLValue(conn.unwrap(PhoenixConnection.class), defaultTTL, viewName);
+ TestUtil.assertTTLValue(conn.unwrap(PhoenixConnection.class), defaultTTL, viewName);
//Child View's PTable gets TTL from parent View's PTable which gets from Table.
String childView = createViewOnViewWithTTL(tenantConn, viewName, false);
- assertTTLValue(tenantConn.unwrap(PhoenixConnection.class), defaultTTL, childView);
+ TestUtil.assertTTLValue(tenantConn.unwrap(PhoenixConnection.class), defaultTTL, childView);
String childView1 = createViewOnViewWithTTL(tenantConn1, viewName, false);
- assertTTLValue(tenantConn1.unwrap(PhoenixConnection.class), defaultTTL, childView1);
+ TestUtil.assertTTLValue(tenantConn1.unwrap(PhoenixConnection.class), defaultTTL, childView1);
String ttlAlter = (useExpression ?
String.format("'%s'", DEFAULT_TTL_EXPRESSION_FOR_ALTER) :
@@ -553,21 +560,12 @@ public class TTLAsPhoenixTTLIT extends ParallelStatsDisabledIT{
TTLExpressionFactory.create(DEFAULT_TTL_EXPRESSION_FOR_ALTER) :
TTLExpressionFactory.create(DEFAULT_TTL_FOR_ALTER);
//Assert TTL for each entity again with altered value
- assertTTLValue(conn.unwrap(PhoenixConnection.class), expectedAlterTTl, viewName);
- assertTTLValue(tenantConn.unwrap(PhoenixConnection.class), expectedAlterTTl, childView);
- assertTTLValue(tenantConn1.unwrap(PhoenixConnection.class), expectedAlterTTl, childView1);
+ TestUtil.assertTTLValue(conn.unwrap(PhoenixConnection.class), expectedAlterTTl, viewName);
+ TestUtil.assertTTLValue(tenantConn.unwrap(PhoenixConnection.class), expectedAlterTTl, childView);
+ TestUtil.assertTTLValue(tenantConn1.unwrap(PhoenixConnection.class), expectedAlterTTl, childView1);
}
}
- private void assertTTLValue(PhoenixConnection conn, TTLExpression expected, String name) throws SQLException {
- assertEquals("TTL value did not match :-", expected,
- PhoenixRuntime.getTableNoCache(conn, name).getTTLExpression());
- }
-
- private void assertTTLValue(PTable table, TTLExpression expected) {
- assertEquals("TTL value did not match :-", expected, table.getTTLExpression());
- }
-
private String createTableWithOrWithOutTTLAsItsProperty(Connection conn, boolean withTTL) throws SQLException {
String tableName = generateUniqueName();
StringBuilder ddl = new StringBuilder();
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TTLIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TTLIT.java
index e0ecb205e4..2fe58d1393 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TTLIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TTLIT.java
@@ -17,6 +17,7 @@
*/
package org.apache.phoenix.end2end;
+import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.exception.SQLExceptionCode;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.query.PhoenixTestBuilder;
@@ -24,14 +25,17 @@ import org.apache.phoenix.query.PhoenixTestBuilder.SchemaBuilder;
import org.apache.phoenix.query.PhoenixTestBuilder.SchemaBuilder.TableOptions;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.LiteralTTLExpression;
+import org.apache.phoenix.schema.TTLExpression;
import org.apache.phoenix.thirdparty.com.google.common.collect.Lists;
import org.apache.phoenix.util.PhoenixRuntime;
+import org.apache.phoenix.util.TestUtil;
import org.junit.Assert;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
+import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.SQLException;
@@ -136,31 +140,35 @@ public class TTLIT extends ParallelStatsDisabledIT {
* Assert Methods
*/
- private void assertTTLForGivenPTable(PTable table, int ttl) {
+ private void assertTTLForGivenPTable(Connection connection, PTable table, int ttl)
+ throws SQLException, IOException {
LiteralTTLExpression expected = new LiteralTTLExpression(ttl);
- Assert.assertEquals(expected, table.getTTLExpression());
+ TestUtil.assertTTLValue(connection, table, expected);
}
-
- private void assertTTLForGivenEntity(Connection connection, String entityName, int ttl) throws SQLException {
+ private void assertTTLForGivenEntity(Connection connection, String entityName, int ttl)
+ throws SQLException, IOException {
PTable pTable = PhoenixRuntime.getTable(connection, entityName);
LiteralTTLExpression expected = new LiteralTTLExpression(ttl);
- Assert.assertEquals(expected, pTable.getTTLExpression());
+ TestUtil.assertTTLValue(connection, pTable, expected);
}
- private void assertTTLForIndexName(Connection connection, String indexName, int ttl) throws SQLException {
+ private void assertTTLForIndexName(Connection connection, String indexName, int ttl)
+ throws SQLException, IOException {
if (!indexName.equals(SKIP_ASSERT)) {
PTable index = PhoenixRuntime.getTable(connection, indexName);
LiteralTTLExpression expected = new LiteralTTLExpression(ttl);
- Assert.assertEquals(expected, index.getTTLExpression());
+ TestUtil.assertTTLValue(connection, index, expected);
}
}
- private void assertTTLForIndexFromParentPTable(Connection connection, String baseEntity, int ttl) throws SQLException {
+ private void assertTTLForIndexFromParentPTable(Connection connection, String baseEntity, int ttl)
+ throws SQLException, IOException {
PTable entity = PhoenixRuntime.getTable(connection, baseEntity);
List<PTable> indexes = entity.getIndexes();
+ LiteralTTLExpression expected = new LiteralTTLExpression(ttl);
for (PTable index : indexes) {
- assertTTLForGivenPTable(index, ttl);
+ TestUtil.assertTTLValue(connection, index, expected);
}
}
@@ -177,7 +185,7 @@ public class TTLIT extends ParallelStatsDisabledIT {
String indexName = schemaBuilder.getEntityTableIndexName();
try (Connection globalConnection = DriverManager.getConnection(getUrl())) {
- assertTTLForGivenPTable(table, DEFAULT_TEST_TTL_VALUE);
+ assertTTLForGivenPTable(globalConnection, table, DEFAULT_TEST_TTL_VALUE);
assertTTLForIndexName(globalConnection, indexName, DEFAULT_TEST_TTL_VALUE);
//Assert TTL for index by getting PTable from table's PTable
assertTTLForIndexFromParentPTable(globalConnection, schemaBuilder.getEntityTableName(), DEFAULT_TEST_TTL_VALUE);
@@ -225,7 +233,7 @@ public class TTLIT extends ParallelStatsDisabledIT {
Connection tenantConnection = DriverManager.getConnection(
getUrl() + ';' + TENANT_ID_ATTRIB + '=' + schemaBuilder.getDataOptions().getTenantId())) {
//All level entities should have TTL value equal to TTL defined at table level
- assertTTLForGivenPTable(table, DEFAULT_TEST_TTL_VALUE);
+ assertTTLForGivenPTable(globalConnection, table, DEFAULT_TEST_TTL_VALUE);
assertTTLForGivenEntity(globalConnection, globalView, DEFAULT_TEST_TTL_VALUE);
assertTTLForGivenEntity(tenantConnection, childView, DEFAULT_TEST_TTL_VALUE);
}
@@ -270,7 +278,7 @@ public class TTLIT extends ParallelStatsDisabledIT {
Connection tenantConnection = DriverManager.getConnection(
getUrl() + ';' + TENANT_ID_ATTRIB + '=' + schemaBuilder.getDataOptions().getTenantId())) {
//All level entities should have TTL value equal to TTL defined at table level
- assertTTLForGivenPTable(table, DEFAULT_TEST_TTL_VALUE);
+ assertTTLForGivenPTable(globalConnection, table, DEFAULT_TEST_TTL_VALUE);
assertTTLForGivenEntity(globalConnection, globalViewName, DEFAULT_TEST_TTL_VALUE);
assertTTLForGivenEntity(tenantConnection, childViewName, DEFAULT_TEST_TTL_VALUE);
@@ -318,7 +326,7 @@ public class TTLIT extends ParallelStatsDisabledIT {
Connection tenantConnection = DriverManager.getConnection(
getUrl() + ';' + TENANT_ID_ATTRIB + '=' + schemaBuilder.getDataOptions().getTenantId())) {
//All level entities should have TTL value equal to TTL defined at table level
- assertTTLForGivenPTable(table, DEFAULT_TEST_TTL_VALUE);
+ assertTTLForGivenPTable(globalConnection, table, DEFAULT_TEST_TTL_VALUE);
assertTTLForIndexName(globalConnection, indexName, DEFAULT_TEST_TTL_VALUE);
//Assert TTL for index by getting PTable from table's PTable
assertTTLForIndexFromParentPTable(globalConnection, schemaBuilder.getEntityTableName(), DEFAULT_TEST_TTL_VALUE);
@@ -348,7 +356,7 @@ public class TTLIT extends ParallelStatsDisabledIT {
Connection tenantConnection = DriverManager.getConnection(
getUrl() + ';' + TENANT_ID_ATTRIB + '=' + schemaBuilder.getDataOptions().getTenantId())) {
//All level entities should have TTL value equal to TTL defined at table level
- assertTTLForGivenPTable(table, DEFAULT_TEST_TTL_VALUE);
+ assertTTLForGivenPTable(globalConnection, table, DEFAULT_TEST_TTL_VALUE);
assertTTLForGivenEntity(globalConnection, globalView, DEFAULT_TEST_TTL_VALUE);
assertTTLForIndexName(globalConnection, globalViewIndexName, DEFAULT_TEST_TTL_VALUE);
@@ -377,7 +385,7 @@ public class TTLIT extends ParallelStatsDisabledIT {
Connection tenantConnection = DriverManager.getConnection(
getUrl() + ';' + TENANT_ID_ATTRIB + '=' + schemaBuilder.getDataOptions().getTenantId())) {
//All level entities should have TTL value equal to TTL defined at table level
- assertTTLForGivenPTable(table, DEFAULT_TEST_TTL_VALUE);
+ assertTTLForGivenPTable(globalConnection, table, DEFAULT_TEST_TTL_VALUE);
assertTTLForGivenEntity(globalConnection, globalView, DEFAULT_TEST_TTL_VALUE);
@@ -407,7 +415,7 @@ public class TTLIT extends ParallelStatsDisabledIT {
Connection tenantConnection = DriverManager.getConnection(
getUrl() + ';' + TENANT_ID_ATTRIB + '=' + schemaBuilder.getDataOptions().getTenantId())) {
//All level entities should have TTL value equal to TTL defined at table level
- assertTTLForGivenPTable(table, TTL_NOT_DEFINED);
+ assertTTLForGivenPTable(globalConnection, table, TTL_NOT_DEFINED);
assertTTLForGivenEntity(globalConnection, globalViewName, DEFAULT_TEST_TTL_VALUE_AT_GLOBAL);
assertTTLForIndexName(globalConnection, globalViewIndexName, DEFAULT_TEST_TTL_VALUE_AT_GLOBAL);
assertTTLForIndexFromParentPTable(globalConnection, globalViewName, DEFAULT_TEST_TTL_VALUE_AT_GLOBAL);
@@ -430,7 +438,7 @@ public class TTLIT extends ParallelStatsDisabledIT {
Connection tenantConnection = DriverManager.getConnection(
getUrl() + ';' + TENANT_ID_ATTRIB + '=' + schemaBuilder.getDataOptions().getTenantId())) {
//All level entities should have TTL value equal to TTL defined at table level
- assertTTLForGivenPTable(table, TTL_NOT_DEFINED);
+ assertTTLForGivenPTable(globalConnection, table, TTL_NOT_DEFINED);
assertTTLForGivenEntity(globalConnection, globalViewName, DEFAULT_TEST_TTL_VALUE_AT_GLOBAL);
assertTTLForGivenEntity(tenantConnection, childViewName, DEFAULT_TEST_TTL_VALUE_AT_GLOBAL);
assertTTLForIndexName(tenantConnection, childViewIndexName, DEFAULT_TEST_TTL_VALUE_AT_GLOBAL);
@@ -462,7 +470,7 @@ public class TTLIT extends ParallelStatsDisabledIT {
getUrl() + ';' + TENANT_ID_ATTRIB + '=' + schemaBuilder.getDataOptions().getTenantId())) {
//All level entities should have TTL value equal to TTL defined at table level
- assertTTLForGivenPTable(table, TTL_NOT_DEFINED);
+ assertTTLForGivenPTable(globalConnection, table, TTL_NOT_DEFINED);
assertTTLForGivenEntity(globalConnection, globalViewName, DEFAULT_TEST_TTL_VALUE_AT_GLOBAL);
assertTTLForGivenEntity(tenantConnection, childViewName, DEFAULT_TEST_TTL_VALUE_AT_GLOBAL);
@@ -484,7 +492,7 @@ public class TTLIT extends ParallelStatsDisabledIT {
Connection tenantConnection = DriverManager.getConnection(
getUrl() + ';' + TENANT_ID_ATTRIB + '=' + schemaBuilder.getDataOptions().getTenantId())) {
//All level entities should have TTL value equal to TTL defined at table level
- assertTTLForGivenPTable(table, TTL_NOT_DEFINED);
+ assertTTLForGivenPTable(globalConnection, table, TTL_NOT_DEFINED);
assertTTLForGivenEntity(globalConnection, globalViewName, DEFAULT_TEST_TTL_VALUE_AT_GLOBAL);
assertTTLForGivenEntity(tenantConnection, childViewName, DEFAULT_TEST_TTL_VALUE_AT_GLOBAL);
@@ -525,7 +533,7 @@ public class TTLIT extends ParallelStatsDisabledIT {
Connection tenantConnection = DriverManager.getConnection(
getUrl() + ';' + TENANT_ID_ATTRIB + '=' + schemaBuilder.getDataOptions().getTenantId())) {
//All level entities should have TTL value equal to TTL defined at table level
- assertTTLForGivenPTable(table, TTL_NOT_DEFINED);
+ assertTTLForGivenPTable(globalConnection, table, TTL_NOT_DEFINED);
assertTTLForGivenEntity(globalConnection, globalView, TTL_NOT_DEFINED);
assertTTLForGivenEntity(tenantConnection, childView, DEFAULT_TEST_TTL_VALUE_AT_TENANT);
assertTTLForIndexName(tenantConnection, childViewIndex, DEFAULT_TEST_TTL_VALUE_AT_TENANT);
@@ -548,7 +556,7 @@ public class TTLIT extends ParallelStatsDisabledIT {
Connection tenantConnection = DriverManager.getConnection(
getUrl() + ';' + TENANT_ID_ATTRIB + '=' + schemaBuilder.getDataOptions().getTenantId())) {
//All level entities should have TTL value equal to TTL defined at table level
- assertTTLForGivenPTable(table, TTL_NOT_DEFINED);
+ assertTTLForGivenPTable(globalConnection, table, TTL_NOT_DEFINED);
assertTTLForGivenEntity(globalConnection, globalView, TTL_NOT_DEFINED);
assertTTLForGivenEntity(tenantConnection, childView, DEFAULT_TEST_TTL_VALUE_AT_TENANT);
assertTTLForIndexName(tenantConnection, childViewIndex, DEFAULT_TEST_TTL_VALUE_AT_TENANT);
@@ -596,7 +604,7 @@ public class TTLIT extends ParallelStatsDisabledIT {
Connection tenantConnection = DriverManager.getConnection(
getUrl() + ';' + TENANT_ID_ATTRIB + '=' + schemaBuilder.getDataOptions().getTenantId())) {
//All level entities should have TTL value equal to TTL defined at table level
- assertTTLForGivenPTable(table, DEFAULT_TEST_TTL_VALUE);
+ assertTTLForGivenPTable(globalConnection, table, DEFAULT_TEST_TTL_VALUE);
assertTTLForGivenEntity(globalConnection, globalViewName, DEFAULT_TEST_TTL_VALUE);
assertTTLForGivenEntity(tenantConnection, tenantViewName, DEFAULT_TEST_TTL_VALUE);
assertTTLForIndexName(tenantConnection, tenantViewIndexName, DEFAULT_TEST_TTL_VALUE);
@@ -650,7 +658,7 @@ public class TTLIT extends ParallelStatsDisabledIT {
Connection tenantConnection = DriverManager.getConnection(
getUrl() + ';' + TENANT_ID_ATTRIB + '=' + schemaBuilder.getDataOptions().getTenantId())) {
//All level entities should have TTL value equal to TTL defined at table level
- assertTTLForGivenPTable(table, DEFAULT_TEST_TTL_VALUE);
+ assertTTLForGivenPTable(globalConnection, table, DEFAULT_TEST_TTL_VALUE);
assertTTLForGivenEntity(globalConnection, globalViewName, DEFAULT_TEST_TTL_VALUE);
assertTTLForGivenEntity(tenantConnection, tenantViewName, DEFAULT_TEST_TTL_VALUE);
assertTTLForIndexName(tenantConnection, tenantViewIndexName, DEFAULT_TEST_TTL_VALUE);
@@ -704,7 +712,7 @@ public class TTLIT extends ParallelStatsDisabledIT {
Connection tenantConnection = DriverManager.getConnection(
getUrl() + ';' + TENANT_ID_ATTRIB + '=' + schemaBuilder.getDataOptions().getTenantId())) {
//All level entities should have TTL value equal to TTL defined at table level
- assertTTLForGivenPTable(table, TTL_NOT_DEFINED);
+ assertTTLForGivenPTable(globalConnection, table, TTL_NOT_DEFINED);
assertTTLForGivenEntity(globalConnection, globalViewName, DEFAULT_TEST_TTL_VALUE_AT_GLOBAL);
assertTTLForGivenEntity(tenantConnection, tenantViewName, DEFAULT_TEST_TTL_VALUE_AT_GLOBAL);
assertTTLForIndexName(tenantConnection, tenantViewIndexName, DEFAULT_TEST_TTL_VALUE_AT_GLOBAL);
@@ -758,7 +766,7 @@ public class TTLIT extends ParallelStatsDisabledIT {
Connection tenantConnection = DriverManager.getConnection(
getUrl() + ';' + TENANT_ID_ATTRIB + '=' + schemaBuilder.getDataOptions().getTenantId())) {
//All level entities should have TTL value equal to TTL defined at table level
- assertTTLForGivenPTable(table, TTL_NOT_DEFINED);
+ assertTTLForGivenPTable(globalConnection, table, TTL_NOT_DEFINED);
assertTTLForGivenEntity(globalConnection, globalViewName, TTL_NOT_DEFINED);
assertTTLForGivenEntity(tenantConnection, tenantViewName, DEFAULT_TEST_TTL_VALUE_AT_TENANT);
assertTTLForIndexName(tenantConnection, tenantViewIndexName, DEFAULT_TEST_TTL_VALUE_AT_TENANT);
@@ -813,7 +821,7 @@ public class TTLIT extends ParallelStatsDisabledIT {
Connection tenantConnection = DriverManager.getConnection(
getUrl() + ';' + TENANT_ID_ATTRIB + '=' + schemaBuilder.getDataOptions().getTenantId())) {
//All level entities should have TTL value equal to TTL defined at table level
- assertTTLForGivenPTable(table, TTL_NOT_DEFINED);
+ assertTTLForGivenPTable(globalConnection, table, TTL_NOT_DEFINED);
assertTTLForGivenEntity(globalConnection, globalViewName, TTL_NOT_DEFINED);
assertTTLForGivenEntity(tenantConnection, tenantViewName, DEFAULT_TEST_TTL_VALUE_AT_TENANT);
assertTTLForIndexName(tenantConnection, tenantViewIndexName, DEFAULT_TEST_TTL_VALUE_AT_TENANT);
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewTTLIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewTTLIT.java
index 34cb0a2952..96eeb97159 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewTTLIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewTTLIT.java
@@ -98,7 +98,8 @@ public class ViewTTLIT extends BaseViewTTLIT {
public static void doSetup() throws Exception {
// Turn on the View TTL feature
Map<String, String> DEFAULT_PROPERTIES = new HashMap<String, String>() {{
- put(QueryServices.PHOENIX_TABLE_TTL_ENABLED, String.valueOf(true));
+ put(QueryServices.PHOENIX_COMPACTION_ENABLED, String.valueOf(true));
+ put(QueryServices.PHOENIX_VIEW_TTL_ENABLED, Boolean.toString(true));
put(QueryServices.LONG_VIEW_INDEX_ENABLED_ATTRIB, String.valueOf(false));
put("hbase.procedure.remote.dispatcher.delay.msec", "0");
// no max lookback
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewTTLWithLongViewIndexEnabledIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewTTLWithLongViewIndexEnabledIT.java
index fbb36dc181..935d472303 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewTTLWithLongViewIndexEnabledIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ViewTTLWithLongViewIndexEnabledIT.java
@@ -42,7 +42,7 @@ public class ViewTTLWithLongViewIndexEnabledIT extends BaseViewTTLIT {
public static final void doSetup() throws Exception {
// Turn on the TTL feature
Map<String, String> DEFAULT_PROPERTIES = new HashMap<String, String>() {{
- put(QueryServices.PHOENIX_TABLE_TTL_ENABLED, String.valueOf(true));
+ put(QueryServices.PHOENIX_COMPACTION_ENABLED, String.valueOf(true));
put(QueryServices.LONG_VIEW_INDEX_ENABLED_ATTRIB, String.valueOf(true));
put("hbase.procedure.remote.dispatcher.delay.msec", "0");
// no max lookback
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexVerificationOutputRepositoryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexVerificationOutputRepositoryIT.java
index 0bd152638f..125258d294 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexVerificationOutputRepositoryIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexVerificationOutputRepositoryIT.java
@@ -132,7 +132,7 @@ public class IndexVerificationOutputRepositoryIT extends ParallelStatsDisabledIT
new IndexVerificationOutputRepository(mockStringBytes, conn);
outputRepository.createOutputTable(conn);
- TestUtil.assertTableHasTtl(conn, TableName.valueOf(OUTPUT_TABLE_NAME_BYTES), DEFAULT_LOG_TTL, false);
+ TestUtil.assertTTLValue(conn, TableName.valueOf(OUTPUT_TABLE_NAME_BYTES), DEFAULT_LOG_TTL, false);
ManualEnvironmentEdge customClock = new ManualEnvironmentEdge();
customClock.setValue(EnvironmentEdgeManager.currentTimeMillis());
EnvironmentEdgeManager.injectEdge(customClock);
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexVerificationResultRepositoryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexVerificationResultRepositoryIT.java
index 4995292453..8101f83422 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexVerificationResultRepositoryIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexVerificationResultRepositoryIT.java
@@ -107,7 +107,7 @@ public class IndexVerificationResultRepositoryIT extends ParallelStatsDisabledIT
IndexVerificationResultRepository resultRepository =
new IndexVerificationResultRepository(conn, indexNameBytes);
resultRepository.createResultTable(conn);
- TestUtil.assertTableHasTtl(conn, TableName.valueOf(RESULT_TABLE_NAME_BYTES), DEFAULT_LOG_TTL, false);
+ TestUtil.assertTTLValue(conn, TableName.valueOf(RESULT_TABLE_NAME_BYTES), DEFAULT_LOG_TTL, false);
byte[] regionOne = Bytes.toBytes("a.1.00000000000000000000");
byte[] regionTwo = Bytes.toBytes("a.2.00000000000000000000");
resultRepository.logToIndexToolResultTable(expectedResult, IndexTool.IndexVerifyType.BOTH,
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java
index 781cc9263a..d6e91cb485 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/PartialSystemCatalogIndexIT.java
@@ -207,7 +207,6 @@ public class PartialSystemCatalogIndexIT extends ParallelStatsDisabledIT {
// Turn on the View TTL feature
Map<String, String> DEFAULT_PROPERTIES = new HashMap<String, String>() {{
put(QueryServices.SYSTEM_CATALOG_INDEXES_ENABLED, String.valueOf(true));
- put(QueryServices.PHOENIX_TABLE_TTL_ENABLED, String.valueOf(true));
// no max lookback
put(BaseScannerRegionObserverConstants.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY, Integer.toString(0));
put(QueryServices.PHOENIX_VIEW_TTL_ENABLED, Boolean.toString(true));
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/schema/TTLExpressionTest.java b/phoenix-core/src/test/java/org/apache/phoenix/schema/TTLExpressionTest.java
index 066221724d..2f054d738d 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/schema/TTLExpressionTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/schema/TTLExpressionTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.phoenix.schema;
+import static org.apache.phoenix.schema.LiteralTTLExpression.TTL_EXPRESSION_DEFINED_IN_TABLE_DESCRIPTOR;
import static org.apache.phoenix.schema.LiteralTTLExpression.TTL_EXPRESSION_FOREVER;
import static org.apache.phoenix.schema.LiteralTTLExpression.TTL_EXPRESSION_NOT_DEFINED;
import static org.junit.Assert.assertEquals;
@@ -60,6 +61,12 @@ public class TTLExpressionTest {
TTLExpressionFactory.create(HConstants.FOREVER));
}
+ @Test
+ public void testTableTTLInTableDescriptor() {
+ assertEquals(TTL_EXPRESSION_DEFINED_IN_TABLE_DESCRIPTOR,
+ TTLExpressionFactory.create(PhoenixDatabaseMetaData.TTL_DEFINED_IN_TABLE_DESCRIPTOR));
+ }
+
@Test
public void testNone() throws SQLException {
assertEquals(TTL_EXPRESSION_NOT_DEFINED,
@@ -71,7 +78,7 @@ public class TTLExpressionTest {
@Test(expected = IllegalArgumentException.class)
public void testInvalidLiteral() {
- TTLExpressionFactory.create(-1);
+ TTLExpressionFactory.create(-10);
}
@Test
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
index 784e8439f3..233723f001 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
@@ -22,6 +22,8 @@ import static org.apache.phoenix.query.BaseTest.generateUniqueName;
import static org.apache.phoenix.query.QueryConstants.MILLIS_IN_DAY;
import static org.apache.phoenix.query.QueryConstants.SINGLE_COLUMN_FAMILY_NAME;
import static org.apache.phoenix.query.QueryConstants.SINGLE_COLUMN_NAME;
+import static org.apache.phoenix.schema.LiteralTTLExpression.TTL_EXPRESSION_DEFINED_IN_TABLE_DESCRIPTOR;
+import static org.apache.phoenix.schema.LiteralTTLExpression.TTL_EXPRESSION_FOREVER;
import static org.apache.phoenix.util.PhoenixRuntime.CONNECTIONLESS;
import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL;
import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR;
@@ -1404,20 +1406,73 @@ public class TestUtil {
assertEquals(code.getSQLState(), se.getSQLState());
}
- public static void assertTableHasTtl(Connection conn, TableName tableName, int ttl, boolean phoenixTTLEnabled)
- throws SQLException, IOException {
+ /**
+ * In 5.3.0 release the TTL column will be created in SYSTEM.CATALOG and used to store TTL
+ * values, but the TTL values for tables with literal expressions will continue to be stored
+ * in the HBase TableDescriptor for backward compatibility. Table level TTL values will be
+ * moved from the HBase TableDescriptor and stored in SYSTEM.CATALOG in a future release.
+ */
+ public static void assertTTLValue(Connection conn, TableName tableName, int ttl,
+ boolean checkTTLInSystemCatalog) throws SQLException, IOException {
TTLExpression tableTTL;
- if (phoenixTTLEnabled) {
- tableTTL = conn.unwrap(PhoenixConnection.class).getTable(new PTableKey(null,
- tableName.getNameAsString())).getTTLExpression();
+ if (checkTTLInSystemCatalog) {
+ tableTTL =
+ conn.unwrap(PhoenixConnection.class)
+ .getTable(new PTableKey(null, tableName.getNameAsString()))
+ .getTTLExpression();
} else {
- tableTTL = TTLExpressionFactory.create(
- getColumnDescriptor(conn, tableName).getTimeToLive());
+ tableTTL =
+ TTLExpressionFactory.create(
+ getColumnDescriptor(conn, tableName).getTimeToLive());
}
TTLExpression expectedTTL = TTLExpressionFactory.create(ttl);
Assert.assertEquals(expectedTTL, tableTTL);
}
+ /**
+ * In 5.3.0 release the TTL column will be created in SYSTEM.CATALOG and used to store TTL
+ * values, but the TTL values for tables with literal expressions will continue to be stored
+ * in the HBase TableDescriptor for backward compatibility.
+ * When table type != TABLE (i.e. VIEW TTL) OR expression is ConditionalExpression then
+ * actual TTL value is stored in SYSTEM.CATALOG and TableDescriptor will be set to FOREVER
+ * When table type == TABLE (i.e. Table TTL) and expression is LiteralExpression then actual
+ * TTL value is stored in TableDescriptor and SYSTEM.CATALOG will be set to
+ * TTL_EXPRESSION_DEFINED_IN_TABLE_DESCRIPTOR. if expression == TTL_NOT_DEFINED OR FOREVER
+ * then TTL_NOT_DEFINED will be stores in SYSTEM.CATALOG
+ */
+
+ public static void assertTTLValue(Connection conn, TTLExpression expected, String name)
+ throws SQLException, IOException {
+ PTable pTable = PhoenixRuntime.getTableNoCache(conn, name);
+ assertTTLValue(conn, pTable, expected);
+ }
+
+ public static void assertTTLValue(Connection conn, TableName tableName, int ttl)
+ throws SQLException, IOException {
+ assertTTLValue(conn, tableName.getNameAsString(), TTLExpressionFactory.create(ttl));
+ }
+
+ public static void assertTTLValue(Connection conn, String name, TTLExpression expected)
+ throws SQLException, IOException {
+ PTable pTable = PhoenixRuntime.getTableNoCache(conn, name);
+ assertTTLValue(conn, pTable, expected);
+ }
+
+ public static void assertTTLValue(Connection conn, PTable pTable, TTLExpression expected)
+ throws SQLException, IOException {
+ TTLExpression actualInSyscat = pTable.getTTLExpression();
+ byte[] hbaseTableName = pTable.getPhysicalName().getBytes();
+ ColumnFamilyDescriptor cfd = getColumnDescriptor(conn, TableName.valueOf(hbaseTableName));
+ TTLExpression actual = actualInSyscat;
+ if (actualInSyscat.equals(TTL_EXPRESSION_DEFINED_IN_TABLE_DESCRIPTOR)) {
+ actual = TTLExpressionFactory.create(cfd.getTimeToLive());
+ } else {
+ assertEquals("Table descriptor TTL value did not match :-", TTL_EXPRESSION_FOREVER,
+ TTLExpressionFactory.create(cfd.getTimeToLive()));
+ }
+ assertEquals("TTL value did not match :-", expected, actual);
+ }
+
public static void assertTableHasVersions(Connection conn, TableName tableName, int versions)
throws SQLException, IOException {
ColumnFamilyDescriptor cd = getColumnDescriptor(conn, tableName);