You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by je...@apache.org on 2014/12/10 23:25:04 UTC
phoenix git commit: PHOENIX-1498: Turn KEEP_DELETED_CELLS off by
default
Repository: phoenix
Updated Branches:
refs/heads/master f84e5da33 -> 5722a4d31
PHOENIX-1498: Turn KEEP_DELETED_CELLS off by default
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/5722a4d3
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/5722a4d3
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/5722a4d3
Branch: refs/heads/master
Commit: 5722a4d31318cb04473fed8bf72c5202b33d1e0d
Parents: f84e5da
Author: Jeffrey Zhong <je...@apache.org>
Authored: Mon Dec 8 18:30:08 2014 -0800
Committer: Jeffrey Zhong <je...@apache.org>
Committed: Wed Dec 10 14:22:02 2014 -0800
----------------------------------------------------------------------
.../java/org/apache/phoenix/end2end/BaseQueryIT.java | 7 +++++++
.../java/org/apache/phoenix/end2end/GroupByIT.java | 15 +++++++++++++++
.../phoenix/end2end/QueryDatabaseMetaDataIT.java | 12 ++++++++++++
.../it/java/org/apache/phoenix/end2end/QueryIT.java | 13 +++++++++++++
.../apache/phoenix/coprocessor/MetaDataProtocol.java | 2 ++
.../phoenix/query/ConnectionQueryServicesImpl.java | 5 ++++-
.../org/apache/phoenix/query/QueryConstants.java | 9 ++++++---
.../java/org/apache/phoenix/query/QueryServices.java | 1 +
.../apache/phoenix/query/QueryServicesOptions.java | 1 +
9 files changed, 61 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5722a4d3/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseQueryIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseQueryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseQueryIT.java
index e404c17..fc088ac 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseQueryIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseQueryIT.java
@@ -67,7 +67,14 @@ public abstract class BaseQueryIT extends BaseClientManagedTimeIT {
@BeforeClass
@Shadower(classBeingShadowed = BaseClientManagedTimeIT.class)
public static void doSetup() throws Exception {
+ doSetup(null);
+ }
+
+ protected static void doSetup(Map<String,String> customProps) throws Exception {
Map<String,String> props = getDefaultProps();
+ if(customProps != null) {
+ props.putAll(customProps);
+ }
props.put(QueryServices.QUEUE_SIZE_ATTRIB, Integer.toString(5000));
props.put(IndexWriterUtils.HTABLE_THREAD_KEY, Integer.toString(100));
// Make a small batch size to test multiple calls to reserve sequences
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5722a4d3/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByIT.java
index 55bce01..31a55a0 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByIT.java
@@ -33,6 +33,7 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.sql.Connection;
+import java.sql.Date;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
@@ -40,16 +41,22 @@ import java.sql.Statement;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
+import java.util.Map;
import java.util.Properties;
+import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
import com.google.common.collect.Lists;
+import com.google.common.collect.Maps;
@RunWith(Parameterized.class)
@@ -64,6 +71,14 @@ public class GroupByIT extends BaseQueryIT {
return QueryIT.data();
}
+ @BeforeClass
+ @Shadower(classBeingShadowed = BaseQueryIT.class)
+ public static void doSetup() throws Exception {
+ Map<String,String> props = Maps.newHashMapWithExpectedSize(3);
+ props.put(QueryServices.DEFAULT_KEEP_DELETED_CELLS_ATTRIB, "true");
+ BaseQueryIT.doSetup(props);
+ }
+
@SuppressWarnings("unchecked")
@Test
public void testGroupByCondition() throws Exception {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5722a4d3/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java
index e78b812..88a0447 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java
@@ -47,6 +47,7 @@ import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Arrays;
+import java.util.Map;
import java.util.Properties;
import org.apache.hadoop.hbase.HColumnDescriptor;
@@ -65,6 +66,7 @@ import org.apache.phoenix.coprocessor.UngroupedAggregateRegionObserver;
import org.apache.phoenix.exception.SQLExceptionCode;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
+import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.schema.ColumnNotFoundException;
import org.apache.phoenix.schema.PDataType;
import org.apache.phoenix.schema.PTable.ViewType;
@@ -73,13 +75,23 @@ import org.apache.phoenix.schema.ReadOnlyTableException;
import org.apache.phoenix.schema.TableNotFoundException;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.StringUtil;
import org.apache.phoenix.util.TestUtil;
+import org.junit.BeforeClass;
import org.junit.Test;
public class QueryDatabaseMetaDataIT extends BaseClientManagedTimeIT {
+
+ @BeforeClass
+ @Shadower(classBeingShadowed = BaseClientManagedTimeIT.class)
+ public static void doSetup() throws Exception {
+ Map<String,String> props = getDefaultProps();
+ props.put(QueryServices.DEFAULT_KEEP_DELETED_CELLS_ATTRIB, "true");
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ }
@Test
public void testTableMetadataScan() throws SQLException {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5722a4d3/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java
index 4628b43..93dbcf8 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java
@@ -48,6 +48,7 @@ import java.sql.SQLException;
import java.sql.Time;
import java.sql.Timestamp;
import java.util.Arrays;
+import java.util.Map;
import java.util.Properties;
import org.apache.hadoop.hbase.TableName;
@@ -58,14 +59,18 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.exception.SQLExceptionCode;
import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.schema.PDataType;
import org.apache.phoenix.schema.SequenceNotFoundException;
import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.MetaDataUtil;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.PropertiesUtil;
+import org.junit.BeforeClass;
import org.junit.Test;
+import com.google.common.collect.Maps;
+
/**
@@ -83,6 +88,14 @@ public class QueryIT extends BaseQueryIT {
super(indexDDL);
}
+ @BeforeClass
+ @Shadower(classBeingShadowed = BaseQueryIT.class)
+ public static void doSetup() throws Exception {
+ Map<String,String> props = Maps.newHashMapWithExpectedSize(3);
+ props.put(QueryServices.DEFAULT_KEEP_DELETED_CELLS_ATTRIB, "true");
+ BaseQueryIT.doSetup(props);
+ }
+
@Test
public void testIntFilter() throws Exception {
String updateStmt =
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5722a4d3/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
index 2c9be14..0429635 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
@@ -63,6 +63,8 @@ public abstract class MetaDataProtocol extends MetaDataService {
// Incremented from 3 to 4 to salt the sequence table in 3.2/4.2
public static final long MIN_SYSTEM_TABLE_TIMESTAMP = MIN_TABLE_TIMESTAMP + 5;
public static final int DEFAULT_MAX_META_DATA_VERSIONS = 1000;
+ public static final int DEFAULT_MAX_STAT_DATA_VERSIONS = 3;
+ public static final boolean DEFAULT_META_DATA_KEEP_DELETED_CELLS = true;
// TODO: pare this down to minimum, as we don't need duplicates for both table and column errors, nor should we need
// a different code for every type of error.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5722a4d3/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index c0a9eb5..0732cff 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -546,7 +546,10 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
private HColumnDescriptor generateColumnFamilyDescriptor(Pair<byte[],Map<String,Object>> family, PTableType tableType) throws SQLException {
HColumnDescriptor columnDesc = new HColumnDescriptor(family.getFirst());
if (tableType != PTableType.VIEW) {
- columnDesc.setKeepDeletedCells(true);
+ if(props.get(QueryServices.DEFAULT_KEEP_DELETED_CELLS_ATTRIB) != null){
+ columnDesc.setKeepDeletedCells(props.getBoolean(
+ QueryServices.DEFAULT_KEEP_DELETED_CELLS_ATTRIB, QueryServicesOptions.DEFAULT_KEEP_DELETED_CELLS));
+ }
columnDesc.setDataBlockEncoding(SchemaUtil.DEFAULT_DATA_BLOCK_ENCODING);
for (Entry<String,Object> entry : family.getSecond().entrySet()) {
String key = entry.getKey();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5722a4d3/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
index f6b1cfa..555628d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
@@ -91,6 +91,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_TYPE;
import java.math.BigDecimal;
+import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.util.Bytes;
@@ -228,6 +229,7 @@ public interface QueryConstants {
"CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TENANT_ID + ","
+ TABLE_SCHEM + "," + TABLE_NAME + "," + COLUMN_NAME + "," + COLUMN_FAMILY + "))\n" +
HConstants.VERSIONS + "=" + MetaDataProtocol.DEFAULT_MAX_META_DATA_VERSIONS + ",\n" +
+ HColumnDescriptor.KEEP_DELETED_CELLS + "=" + MetaDataProtocol.DEFAULT_META_DATA_KEEP_DELETED_CELLS + ",\n" +
// Install split policy to prevent a tenant's metadata from being split across regions.
HTableDescriptor.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName() + "'\n";
@@ -246,8 +248,8 @@ public interface QueryConstants {
"CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY ("
+ PHYSICAL_NAME + ","
+ COLUMN_FAMILY + ","+ REGION_NAME+"))\n" +
- // TODO: should we support versioned stats?
- // HConstants.VERSIONS + "=" + MetaDataProtocol.DEFAULT_MAX_META_DATA_VERSIONS + ",\n" +
+ HConstants.VERSIONS + "=" + MetaDataProtocol.DEFAULT_MAX_STAT_DATA_VERSIONS + ",\n" +
+ HColumnDescriptor.KEEP_DELETED_CELLS + "=" + MetaDataProtocol.DEFAULT_META_DATA_KEEP_DELETED_CELLS + ",\n" +
// Install split policy to prevent a physical table's stats from being split across regions.
HTableDescriptor.SPLIT_POLICY + "='" + MetaDataSplitPolicy.class.getName() + "'\n";
@@ -266,5 +268,6 @@ public interface QueryConstants {
CYCLE_FLAG + " BOOLEAN, \n" +
LIMIT_REACHED_FLAG + " BOOLEAN \n" +
" CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TENANT_ID + "," + SEQUENCE_SCHEMA + "," + SEQUENCE_NAME + "))\n" +
- HConstants.VERSIONS + "=" + MetaDataProtocol.DEFAULT_MAX_META_DATA_VERSIONS + "\n";
+ HConstants.VERSIONS + "=" + MetaDataProtocol.DEFAULT_MAX_META_DATA_VERSIONS + ",\n" +
+ HColumnDescriptor.KEEP_DELETED_CELLS + "=" + MetaDataProtocol.DEFAULT_META_DATA_KEEP_DELETED_CELLS + "\n";
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5722a4d3/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index 225e5bb..cab9262 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -145,6 +145,7 @@ public interface QueryServices extends SQLCloseable {
public static final String ALLOW_ONLINE_TABLE_SCHEMA_UPDATE = "hbase.online.schema.update.enable";
public static final String NUM_RETRIES_FOR_SCHEMA_UPDATE_CHECK = "phoenix.schema.change.retries";
public static final String DELAY_FOR_SCHEMA_UPDATE_CHECK = "phoenix.schema.change.delay";
+ public static final String DEFAULT_KEEP_DELETED_CELLS_ATTRIB = "phoenix.table.default.keep.deleted.cells";
/**
* Get executor service used for parallel scans
http://git-wip-us.apache.org/repos/asf/phoenix/blob/5722a4d3/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index 70824a5..91cd4e9 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -175,6 +175,7 @@ public class QueryServicesOptions {
public static final boolean DEFAULT_ALLOW_ONLINE_TABLE_SCHEMA_UPDATE = true;
public static final int DEFAULT_RETRIES_FOR_SCHEMA_UPDATE_CHECK = 10;
public static final long DEFAULT_DELAY_FOR_SCHEMA_UPDATE_CHECK = 5 * 1000; // 5 seconds.
+ public static final boolean DEFAULT_KEEP_DELETED_CELLS = false;
private final Configuration config;