You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ma...@apache.org on 2016/10/28 21:33:56 UTC
[03/35] phoenix git commit: PHOENIX-2675 Allow stats to be configured
on a table-by-table basis
PHOENIX-2675 Allow stats to be configured on a table-by-table basis
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/711d7f06
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/711d7f06
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/711d7f06
Branch: refs/heads/calcite
Commit: 711d7f0639d294974c2f56c7d604bc37c22e0a95
Parents: ad99952
Author: James Taylor <ja...@apache.org>
Authored: Tue Oct 4 10:34:21 2016 -0700
Committer: James Taylor <ja...@apache.org>
Committed: Fri Oct 7 17:19:15 2016 -0700
----------------------------------------------------------------------
.../apache/phoenix/end2end/GroupByCaseIT.java | 33 ++++-
.../phoenix/end2end/index/ViewIndexIT.java | 129 +++++++++++++++----
.../apache/phoenix/compile/PostDDLCompiler.java | 8 +-
.../phoenix/coprocessor/MetaDataProtocol.java | 8 +-
.../UngroupedAggregateRegionObserver.java | 13 +-
.../query/ConnectionQueryServicesImpl.java | 67 +++++++++-
.../apache/phoenix/query/QueryConstants.java | 1 +
.../apache/phoenix/schema/MetaDataClient.java | 71 ++++++----
.../apache/phoenix/schema/TableProperty.java | 7 +
.../stats/DefaultStatisticsCollector.java | 109 ++++++++++++----
.../schema/stats/NoOpStatisticsCollector.java | 6 +-
.../schema/stats/StatisticsCollector.java | 7 +-
.../stats/StatisticsCollectorFactory.java | 19 +--
.../phoenix/schema/stats/StatisticsScanner.java | 3 +-
.../phoenix/schema/stats/StatisticsUtil.java | 10 +-
.../java/org/apache/phoenix/util/IndexUtil.java | 2 +-
.../org/apache/phoenix/util/MetaDataUtil.java | 40 +++++-
.../org/apache/phoenix/util/UpgradeUtil.java | 2 +-
.../phoenix/compile/QueryCompilerTest.java | 16 +++
19 files changed, 429 insertions(+), 122 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
index be59fd7..56f4503 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByCaseIT.java
@@ -23,20 +23,24 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
+import java.util.List;
import java.util.Properties;
+import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.schema.AmbiguousColumnException;
import org.apache.phoenix.schema.types.PChar;
import org.apache.phoenix.schema.types.PInteger;
import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.QueryUtil;
+import org.apache.phoenix.util.TestUtil;
import org.junit.Test;
@@ -401,12 +405,29 @@ public class GroupByCaseIT extends ParallelStatsDisabledIT {
}
@Test
- public void testAvgGroupByOrderPreserving() throws Exception {
+ public void testAvgGroupByOrderPreservingWithStats() throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
Connection conn = DriverManager.getConnection(getUrl(), props);
String tableName = generateUniqueName();
-
- PreparedStatement stmt = conn.prepareStatement("CREATE TABLE " + tableName + " (k1 char(1) not null, k2 integer not null, constraint pk primary key (k1,k2)) split on (?,?,?)");
+ initAvgGroupTable(conn, tableName, " GUIDE_POST_WIDTH=20 ");
+ testAvgGroupByOrderPreserving(conn, tableName, 13);
+ conn.createStatement().execute("ALTER TABLE " + tableName + " SET GUIDE_POST_WIDTH=" + 100);
+ testAvgGroupByOrderPreserving(conn, tableName, 6);
+ conn.createStatement().execute("ALTER TABLE " + tableName + " SET GUIDE_POST_WIDTH=null");
+ testAvgGroupByOrderPreserving(conn, tableName, 4);
+ }
+
+ @Test
+ public void testAvgGroupByOrderPreservingWithNoStats() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ initAvgGroupTable(conn, tableName, "");
+ testAvgGroupByOrderPreserving(conn, tableName, 4);
+ }
+
+ private void initAvgGroupTable(Connection conn, String tableName, String tableProps) throws SQLException {
+ PreparedStatement stmt = conn.prepareStatement("CREATE TABLE " + tableName + " (k1 char(1) not null, k2 integer not null, constraint pk primary key (k1,k2)) " + tableProps + " split on (?,?,?)");
stmt.setBytes(1, ByteUtil.concat(PChar.INSTANCE.toBytes("a"), PInteger.INSTANCE.toBytes(3)));
stmt.setBytes(2, ByteUtil.concat(PChar.INSTANCE.toBytes("j"), PInteger.INSTANCE.toBytes(3)));
stmt.setBytes(3, ByteUtil.concat(PChar.INSTANCE.toBytes("n"), PInteger.INSTANCE.toBytes(3)));
@@ -425,6 +446,9 @@ public class GroupByCaseIT extends ParallelStatsDisabledIT {
conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 3)");
conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 2)");
conn.commit();
+ }
+
+ private void testAvgGroupByOrderPreserving(Connection conn, String tableName, int nGuidePosts) throws SQLException, IOException {
String query = "SELECT k1,avg(k2) FROM " + tableName + " GROUP BY k1";
ResultSet rs = conn.createStatement().executeQuery(query);
assertTrue(rs.next());
@@ -445,6 +469,9 @@ public class GroupByCaseIT extends ParallelStatsDisabledIT {
"CLIENT PARALLEL 1-WAY FULL SCAN OVER " + tableName + "\n" +
" SERVER FILTER BY FIRST KEY ONLY\n" +
" SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]", QueryUtil.getExplainPlan(rs));
+ TestUtil.analyzeTable(conn, tableName);
+ List<KeyRange> splits = TestUtil.getAllSplits(conn, tableName);
+ assertEquals(nGuidePosts, splits.size());
}
@Test
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java
index ca50aab..b714a11 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/ViewIndexIT.java
@@ -38,7 +38,11 @@ import org.apache.phoenix.end2end.ParallelStatsDisabledIT;
import org.apache.phoenix.jdbc.PhoenixStatement;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.schema.PNameFactory;
+import org.apache.phoenix.schema.PTableType;
+import org.apache.phoenix.util.PropertiesUtil;
import org.apache.phoenix.util.QueryUtil;
+import org.apache.phoenix.util.SchemaUtil;
+import org.apache.phoenix.util.TestUtil;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
@@ -59,7 +63,7 @@ public class ViewIndexIT extends ParallelStatsDisabledIT {
if (isNamespaceMapped) {
conn.createStatement().execute("CREATE SCHEMA IF NOT EXISTS " + schemaName);
}
- String ddl = "CREATE TABLE " + tableName + " (t_id VARCHAR NOT NULL,\n" +
+ String ddl = "CREATE TABLE " + SchemaUtil.getTableName(schemaName, tableName) + " (t_id VARCHAR NOT NULL,\n" +
"k1 VARCHAR NOT NULL,\n" +
"k2 INTEGER NOT NULL,\n" +
"v1 VARCHAR,\n" +
@@ -93,30 +97,31 @@ public class ViewIndexIT extends ParallelStatsDisabledIT {
@Test
public void testDeleteViewIndexSequences() throws Exception {
String schemaName = generateUniqueName();
- String tableName = schemaName + "." + generateUniqueName();
+ String tableName = generateUniqueName();
+ String fullTableName = SchemaUtil.getTableName(schemaName, tableName);
String indexName = "IND_" + generateUniqueName();
- String VIEW_NAME = "VIEW_" + generateUniqueName();
- String viewName = schemaName + "." + VIEW_NAME;
+ String viewName = "VIEW_" + generateUniqueName();
+ String fullViewName = SchemaUtil.getTableName(schemaName, viewName);
createBaseTable(schemaName, tableName, false, null, null);
Connection conn1 = getConnection();
Connection conn2 = getConnection();
- conn1.createStatement().execute("CREATE VIEW " + viewName + " AS SELECT * FROM " + tableName);
- conn1.createStatement().execute("CREATE INDEX " + indexName + " ON " + viewName + " (v1)");
- conn2.createStatement().executeQuery("SELECT * FROM " + tableName).next();
- String sequenceName = getViewIndexSequenceName(PNameFactory.newName(tableName), null, isNamespaceMapped);
- String sequenceSchemaName = getViewIndexSequenceSchemaName(PNameFactory.newName(tableName), isNamespaceMapped);
- String seqName = getViewIndexSequenceName(PNameFactory.newName(tableName), null, !isNamespaceMapped);
- String seqSchemaName = getViewIndexSequenceSchemaName(PNameFactory.newName(tableName), !isNamespaceMapped);
+ conn1.createStatement().execute("CREATE VIEW " + fullViewName + " AS SELECT * FROM " + fullTableName);
+ conn1.createStatement().execute("CREATE INDEX " + indexName + " ON " + fullViewName + " (v1)");
+ conn2.createStatement().executeQuery("SELECT * FROM " + fullTableName).next();
+ String sequenceName = getViewIndexSequenceName(PNameFactory.newName(fullTableName), null, isNamespaceMapped);
+ String sequenceSchemaName = getViewIndexSequenceSchemaName(PNameFactory.newName(fullTableName), isNamespaceMapped);
+ String seqName = getViewIndexSequenceName(PNameFactory.newName(fullTableName), null, !isNamespaceMapped);
+ String seqSchemaName = getViewIndexSequenceSchemaName(PNameFactory.newName(fullTableName), !isNamespaceMapped);
verifySequenceValue(null, sequenceName, sequenceSchemaName, -32767);
verifySequenceValue(null, sequenceName, sequenceSchemaName, -32767);
- conn1.createStatement().execute("CREATE INDEX " + indexName + "_2 ON " + viewName + " (v1)");
+ conn1.createStatement().execute("CREATE INDEX " + indexName + "_2 ON " + fullViewName + " (v1)");
verifySequenceValue(null, sequenceName, sequenceSchemaName, -32766);
// Check other format of sequence is not there as Sequences format is different for views/indexes created on
// table which are namespace mapped and which are not.
verifySequenceNotExists(null, seqName, seqSchemaName);
- conn1.createStatement().execute("DROP VIEW " + viewName);
- conn1.createStatement().execute("DROP TABLE "+ tableName);
+ conn1.createStatement().execute("DROP VIEW " + fullViewName);
+ conn1.createStatement().execute("DROP TABLE "+ fullTableName);
verifySequenceNotExists(null, sequenceName, sequenceSchemaName);
}
@@ -126,11 +131,13 @@ public class ViewIndexIT extends ParallelStatsDisabledIT {
String schemaName = generateUniqueName();
String tableName = generateUniqueName();
String indexName = "IND_" + generateUniqueName();
- String VIEW_NAME = "VIEW_" + generateUniqueName();
+ String viewName = "VIEW_" + generateUniqueName();
+ String fullTableName = SchemaUtil.getTableName(schemaName, tableName);
+
createBaseTable(schemaName, tableName, true, null, null);
Connection conn = DriverManager.getConnection(getUrl());
PreparedStatement stmt = conn.prepareStatement(
- "UPSERT INTO " + tableName
+ "UPSERT INTO " + fullTableName
+ " VALUES(?,?,?,?,?)");
stmt.setString(1, "10");
stmt.setString(2, "a");
@@ -144,30 +151,61 @@ public class ViewIndexIT extends ParallelStatsDisabledIT {
stmt.setString(4, "x2");
stmt.setInt(5, 200);
stmt.execute();
+ stmt.setString(1, "30");
+ stmt.setString(2, "c");
+ stmt.setInt(3, 3);
+ stmt.setString(4, "x3");
+ stmt.setInt(5, 300);
+ stmt.execute();
+ stmt.setString(1, "40");
+ stmt.setString(2, "d");
+ stmt.setInt(3, 4);
+ stmt.setString(4, "x4");
+ stmt.setInt(5, 400);
+ stmt.execute();
conn.commit();
- conn.close();
- Properties props = new Properties();
+ Properties props = PropertiesUtil.deepCopy(TestUtil.TEST_PROPERTIES);
props.setProperty("TenantId", "10");
Connection conn1 = DriverManager.getConnection(getUrl(), props);
- conn1.createStatement().execute("CREATE VIEW " + VIEW_NAME
- + " AS select * from " + tableName);
+ conn1.createStatement().execute("CREATE VIEW " + viewName
+ + " AS select * from " + fullTableName);
conn1.createStatement().execute("CREATE LOCAL INDEX "
+ indexName + " ON "
- + VIEW_NAME + "(v2)");
+ + viewName + "(v2)");
conn1.commit();
- String sql = "SELECT * FROM " + VIEW_NAME + " WHERE v2 = 100";
+ String sql = "SELECT * FROM " + viewName + " WHERE v2 = 100";
ResultSet rs = conn1.prepareStatement("EXPLAIN " + sql).executeQuery();
assertEquals(
- "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + tableName + " [1,'10',100]\n" +
+ "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + SchemaUtil.getPhysicalHBaseTableName(fullTableName, isNamespaceMapped, PTableType.TABLE) + " [1,'10',100]\n" +
" SERVER FILTER BY FIRST KEY ONLY\n" +
"CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs));
rs = conn1.prepareStatement(sql).executeQuery();
assertTrue(rs.next());
assertFalse(rs.next());
+
+// TestUtil.analyzeTable(conn, fullTableName);
+// List<KeyRange> guideposts = TestUtil.getAllSplits(conn, fullTableName);
+// assertEquals(1, guideposts.size());
+// assertEquals(KeyRange.EVERYTHING_RANGE, guideposts.get(0));
+//
+// conn.createStatement().execute("ALTER TABLE " + fullTableName + " SET GUIDE_POST_WIDTH=20");
+//
+// TestUtil.analyzeTable(conn, fullTableName);
+// guideposts = TestUtil.getAllSplits(conn, fullTableName);
+// assertEquals(5, guideposts.size());
+//
+// // Confirm that when view index used, the GUIDE_POST_WIDTH from the data physical table
+// // was used
+// sql = "SELECT * FROM " + viewName + " WHERE v2 > 100";
+// stmt = conn1.prepareStatement(sql);
+// stmt.executeQuery();
+// QueryPlan plan = stmt.unwrap(PhoenixStatement.class).getQueryPlan();
+// assertEquals(5, plan.getSplits().size());
}
+
@Test
public void testCreatingIndexOnGlobalView() throws Exception {
String baseTable = generateUniqueName();
@@ -184,6 +222,30 @@ public class ViewIndexIT extends ParallelStatsDisabledIT {
stmt.setString(4, "KV1");
stmt.setString(5, "KV3");
stmt.executeUpdate();
+ stmt.setString(1, "tenantId");
+ stmt.setDate(2, new Date(100));
+ stmt.setInt(3, 2);
+ stmt.setString(4, "KV4");
+ stmt.setString(5, "KV5");
+ stmt.executeUpdate();
+ stmt.setString(1, "tenantId");
+ stmt.setDate(2, new Date(100));
+ stmt.setInt(3, 3);
+ stmt.setString(4, "KV6");
+ stmt.setString(5, "KV7");
+ stmt.executeUpdate();
+ stmt.setString(1, "tenantId");
+ stmt.setDate(2, new Date(100));
+ stmt.setInt(3, 4);
+ stmt.setString(4, "KV8");
+ stmt.setString(5, "KV9");
+ stmt.executeUpdate();
+ stmt.setString(1, "tenantId");
+ stmt.setDate(2, new Date(100));
+ stmt.setInt(3, 5);
+ stmt.setString(4, "KV10");
+ stmt.setString(5, "KV11");
+ stmt.executeUpdate();
conn.commit();
// Verify that query against the global view index works
@@ -196,6 +258,27 @@ public class ViewIndexIT extends ParallelStatsDisabledIT {
assertTrue(rs.next());
assertEquals("KV1", rs.getString(1));
assertFalse(rs.next());
+
+// TestUtil.analyzeTable(conn, baseTable);
+// List<KeyRange> guideposts = TestUtil.getAllSplits(conn, baseTable);
+// assertEquals(1, guideposts.size());
+// assertEquals(KeyRange.EVERYTHING_RANGE, guideposts.get(0));
+//
+// conn.createStatement().execute("ALTER TABLE " + baseTable + " SET GUIDE_POST_WIDTH=20");
+//
+// TestUtil.analyzeTable(conn, baseTable);
+// guideposts = TestUtil.getAllSplits(conn, baseTable);
+// assertEquals(6, guideposts.size());
+//
+// // Confirm that when view index used, the GUIDE_POST_WIDTH from the data physical table
+// // was used
+// stmt = conn.prepareStatement("SELECT KV1 FROM " + globalView + " WHERE PK3 = ? AND KV3 >= ?");
+// stmt.setInt(1, 1);
+// stmt.setString(2, "KV3");
+// rs = stmt.executeQuery();
+// plan = stmt.unwrap(PhoenixStatement.class).getQueryPlan();
+// assertTrue(plan.getTableRef().getTable().getName().getString().equals(globalViewIdx));
+// assertEquals(6, plan.getSplits().size());
}
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java
index e43b596..004e254 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/PostDDLCompiler.java
@@ -82,7 +82,7 @@ public class PostDDLCompiler {
scan.setAttribute(BaseScannerRegionObserver.UNGROUPED_AGG, QueryConstants.TRUE);
}
- public MutationPlan compile(final List<TableRef> tableRefs, final byte[] emptyCF, final byte[] projectCF, final List<PColumn> deleteList,
+ public MutationPlan compile(final List<TableRef> tableRefs, final byte[] emptyCF, final List<byte[]> projectCFs, final List<PColumn> deleteList,
final long timestamp) throws SQLException {
PhoenixStatement statement = new PhoenixStatement(connection);
final StatementContext context = new StatementContext(
@@ -244,12 +244,14 @@ public class PostDDLCompiler {
}
}
List<byte[]> columnFamilies = Lists.newArrayListWithExpectedSize(tableRef.getTable().getColumnFamilies().size());
- if (projectCF == null) {
+ if (projectCFs == null) {
for (PColumnFamily family : tableRef.getTable().getColumnFamilies()) {
columnFamilies.add(family.getName().getBytes());
}
} else {
- columnFamilies.add(projectCF);
+ for (byte[] projectCF : projectCFs) {
+ columnFamilies.add(projectCF);
+ }
}
// Need to project all column families into the scan, since we haven't yet created our empty key value
RowProjector projector = ProjectionCompiler.compile(context, SelectStatement.COUNT_ONE, GroupBy.EMPTY_GROUP_BY);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
index bb720ee..4f0a34c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataProtocol.java
@@ -62,8 +62,8 @@ import com.google.protobuf.ByteString;
*/
public abstract class MetaDataProtocol extends MetaDataService {
public static final int PHOENIX_MAJOR_VERSION = 4;
- public static final int PHOENIX_MINOR_VERSION = 8;
- public static final int PHOENIX_PATCH_NUMBER = 1;
+ public static final int PHOENIX_MINOR_VERSION = 9;
+ public static final int PHOENIX_PATCH_NUMBER = 0;
public static final int PHOENIX_VERSION =
VersionUtil.encodeVersion(PHOENIX_MAJOR_VERSION, PHOENIX_MINOR_VERSION, PHOENIX_PATCH_NUMBER);
@@ -84,8 +84,9 @@ public abstract class MetaDataProtocol extends MetaDataService {
public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0 = MIN_TABLE_TIMESTAMP + 15;
public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0 = MIN_TABLE_TIMESTAMP + 18;
public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_8_1 = MIN_TABLE_TIMESTAMP + 18;
+ public static final long MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0 = MIN_TABLE_TIMESTAMP + 19;
// MIN_SYSTEM_TABLE_TIMESTAMP needs to be set to the max of all the MIN_SYSTEM_TABLE_TIMESTAMP_* constants
- public static final long MIN_SYSTEM_TABLE_TIMESTAMP = MIN_SYSTEM_TABLE_TIMESTAMP_4_8_1;
+ public static final long MIN_SYSTEM_TABLE_TIMESTAMP = MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0;
// ALWAYS update this map whenever rolling out a new release (major, minor or patch release).
// Key is the SYSTEM.CATALOG timestamp for the version and value is the version string.
@@ -99,6 +100,7 @@ public abstract class MetaDataProtocol extends MetaDataService {
TIMESTAMP_VERSION_MAP.put(MIN_SYSTEM_TABLE_TIMESTAMP_4_6_0, "4.6.x");
TIMESTAMP_VERSION_MAP.put(MIN_SYSTEM_TABLE_TIMESTAMP_4_7_0, "4.7.x");
TIMESTAMP_VERSION_MAP.put(MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0, "4.8.x");
+ TIMESTAMP_VERSION_MAP.put(MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0, "4.9.x");
}
public static final String CURRENT_CLIENT_VERSION = PHOENIX_MAJOR_VERSION + "." + PHOENIX_MINOR_VERSION + "." + PHOENIX_PATCH_NUMBER;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index a7c6fde..2a4bfca 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -49,10 +49,8 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.KeyValue;
-import org.apache.hadoop.hbase.RegionTooBusyException;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Delete;
-import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Mutation;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Scan;
@@ -113,6 +111,7 @@ import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.ServerUtil;
import org.apache.phoenix.util.StringUtil;
import org.apache.phoenix.util.TimeKeeper;
+import org.apache.tephra.TxConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -120,8 +119,6 @@ import com.google.common.base.Throwables;
import com.google.common.collect.Lists;
import com.google.common.collect.Sets;
-import org.apache.tephra.TxConstants;
-
/**
* Region observer that aggregates ungrouped rows(i.e. SQL query with aggregation function and no GROUP BY).
@@ -229,9 +226,10 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
throws IOException {
s = super.preScannerOpen(e, scan, s);
if (ScanUtil.isAnalyzeTable(scan)) {
- if (!ScanUtil.isLocalIndex(scan)) {
- scan.getFamilyMap().clear();
- }
+// if (!ScanUtil.isLocalIndex(scan)) {
+// scan.getFamilyMap().clear();
+// }
+// scan.getFamilyMap().clear();
// We are setting the start row and stop row such that it covers the entire region. As part
// of Phonenix-1263 we are storing the guideposts against the physical table rather than
// individual tenant specific tables.
@@ -815,6 +813,7 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
long rowCount = 0;
try {
if (!compactionRunning) {
+ stats.init();
synchronized (innerScanner) {
do {
List<Cell> results = new ArrayList<Cell>();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 354a932..4982578 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -36,6 +36,7 @@ import static org.apache.phoenix.util.UpgradeUtil.upgradeTo4_5_0;
import java.io.IOException;
import java.lang.ref.WeakReference;
+import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -2257,6 +2258,41 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
}
+ private PhoenixConnection removeNotNullConstraint(PhoenixConnection oldMetaConnection, String schemaName, String tableName, long timestamp, String columnName) throws SQLException {
+ Properties props = PropertiesUtil.deepCopy(oldMetaConnection.getClientInfo());
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(timestamp));
+ // Cannot go through DriverManager or you end up in an infinite loop because it'll call init again
+ PhoenixConnection metaConnection = new PhoenixConnection(oldMetaConnection, this, props);
+ SQLException sqlE = null;
+ try {
+ metaConnection.createStatement().executeUpdate("UPSERT INTO " + SYSTEM_STATS_NAME + " (" +
+ PhoenixDatabaseMetaData.TENANT_ID + "," + PhoenixDatabaseMetaData.TABLE_SCHEM + "," +
+ PhoenixDatabaseMetaData.TABLE_NAME + "," + PhoenixDatabaseMetaData.COLUMN_NAME + "," +
+ PhoenixDatabaseMetaData.COLUMN_FAMILY + "," + PhoenixDatabaseMetaData.NULLABLE + ") VALUES (" +
+ "null," + schemaName + "," + tableName + "," + columnName + "," + QueryConstants.DEFAULT_COLUMN_FAMILY + "," +
+ ResultSetMetaData.columnNullable + ")");
+ metaConnection.commit();
+ } catch (NewerTableAlreadyExistsException e) {
+ logger.warn("Table already modified at this timestamp, so assuming column already nullable: " + columnName);
+ } catch (SQLException e) {
+ logger.warn("Add column failed due to:" + e);
+ sqlE = e;
+ } finally {
+ try {
+ oldMetaConnection.close();
+ } catch (SQLException e) {
+ if (sqlE != null) {
+ sqlE.setNextException(e);
+ } else {
+ sqlE = e;
+ }
+ }
+ if (sqlE != null) {
+ throw sqlE;
+ }
+ }
+ return metaConnection;
+ }
/**
* This closes the passed connection.
*/
@@ -2474,7 +2510,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
if (table.getValue(MetaDataUtil.PARENT_TABLE_KEY) == null
&& table.getValue(MetaDataUtil.IS_LOCAL_INDEX_TABLE_PROP_NAME) != null) {
table.setValue(MetaDataUtil.PARENT_TABLE_KEY,
- MetaDataUtil.getUserTableName(table.getNameAsString()));
+ MetaDataUtil.getLocalIndexUserTableName(table.getNameAsString()));
// Explicitly disable, modify and enable the table to ensure
// co-location of data and index regions. If we just modify the
// table descriptor when online schema change enabled may reopen
@@ -2620,6 +2656,18 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_8_0);
clearCache();
}
+ if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0) {
+ metaConnection = addColumnsIfNotExists(
+ metaConnection,
+ PhoenixDatabaseMetaData.SYSTEM_CATALOG,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0,
+ PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + " "
+ + PLong.INSTANCE.getSqlTypeName());
+ ConnectionQueryServicesImpl.this.removeTable(null,
+ PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME, null,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0);
+ clearCache();
+ }
}
@@ -2676,7 +2724,9 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
try {
metaConnection.createStatement().executeUpdate(
QueryConstants.CREATE_STATS_TABLE_METADATA);
- } catch (NewerTableAlreadyExistsException ignore) {} catch (TableAlreadyExistsException e) {
+ } catch (NewerTableAlreadyExistsException ignore) {
+
+ } catch (TableAlreadyExistsException e) {
long currentServerSideTableTimeStamp = e.getTable().getTimeStamp();
if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_3_0) {
metaConnection = addColumnsIfNotExists(
@@ -2686,6 +2736,19 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
PhoenixDatabaseMetaData.GUIDE_POSTS_ROW_COUNT + " "
+ PLong.INSTANCE.getSqlTypeName());
}
+ if (currentServerSideTableTimeStamp < MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0) {
+ // The COLUMN_FAMILY column should be nullable as we create a row in it without
+ // any column family to mark when guideposts were last collected.
+ metaConnection = removeNotNullConstraint(metaConnection,
+ PhoenixDatabaseMetaData.SYSTEM_SCHEMA_NAME,
+ PhoenixDatabaseMetaData.SYSTEM_STATS_TABLE,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0,
+ PhoenixDatabaseMetaData.COLUMN_FAMILY);
+ ConnectionQueryServicesImpl.this.removeTable(null,
+ PhoenixDatabaseMetaData.SYSTEM_STATS_NAME, null,
+ MetaDataProtocol.MIN_SYSTEM_TABLE_TIMESTAMP_4_9_0);
+ clearCache();
+ }
}
try {
metaConnection.createStatement().executeUpdate(QueryConstants.CREATE_FUNCTION_METADATA);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
index ace228b..fd37328 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
@@ -288,6 +288,7 @@ public interface QueryConstants {
IS_NAMESPACE_MAPPED + " BOOLEAN," +
AUTO_PARTITION_SEQ + " VARCHAR," +
APPEND_ONLY_SCHEMA + " BOOLEAN," +
+ GUIDE_POSTS_WIDTH + " BIGINT," +
"CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TENANT_ID + ","
+ TABLE_SCHEM + "," + TABLE_NAME + "," + COLUMN_NAME + "," + COLUMN_FAMILY + "))\n" +
HConstants.VERSIONS + "=" + MetaDataProtocol.DEFAULT_MAX_META_DATA_VERSIONS + ",\n" +
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index 1e635d8..0c53d3e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -42,6 +42,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_COLUMN_FAM
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DEFAULT_VALUE;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.DISABLE_WAL;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.FUNCTION_NAME;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.IMMUTABLE_ROWS;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_DISABLE_TIMESTAMP;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_STATE;
@@ -256,8 +257,9 @@ public class MetaDataClient {
UPDATE_CACHE_FREQUENCY + "," +
IS_NAMESPACE_MAPPED + "," +
AUTO_PARTITION_SEQ + "," +
- APPEND_ONLY_SCHEMA +
- ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
+ APPEND_ONLY_SCHEMA + "," +
+ GUIDE_POSTS_WIDTH +
+ ") VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
private static final String CREATE_SCHEMA = "UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE
+ "\"( " + TABLE_SCHEM + "," + TABLE_NAME + ") VALUES (?,?)";
@@ -993,30 +995,20 @@ public class MetaDataClient {
// If analyzing the indexes of a multi-tenant table or a table with view indexes
// then analyze all of those indexes too.
if (table.getType() != PTableType.VIEW) {
- List<PName> names = Lists.newArrayListWithExpectedSize(2);
- final List<PName> physicalNames = Lists.newArrayListWithExpectedSize(2);
if (table.isMultiTenant() || MetaDataUtil.hasViewIndexTable(connection, table.getPhysicalName())) {
- names.add(PNameFactory.newName(SchemaUtil.getTableName(
- MetaDataUtil.getViewIndexSchemaName(table.getSchemaName().getString()),
- MetaDataUtil.getViewIndexTableName(table.getTableName().getString()))));
- physicalNames.add(PNameFactory.newName(MetaDataUtil.getViewIndexPhysicalName(table.getPhysicalName().getBytes())));
- }
- if (MetaDataUtil.hasLocalIndexTable(connection, table.getPhysicalName())) {
- names.add(PNameFactory.newName(SchemaUtil.getTableName(
- MetaDataUtil.getLocalIndexSchemaName(table.getSchemaName().getString()),
- MetaDataUtil.getLocalIndexTableName(table.getTableName().getString()))));
- physicalNames.add(PNameFactory.newName(MetaDataUtil.getLocalIndexPhysicalName(table.getPhysicalName().getBytes())));
- }
- int i = 0;
- for (final PName name : names) {
- final int index = i++;
+ final PName physicalName = PNameFactory.newName(MetaDataUtil.getViewIndexPhysicalName(table.getPhysicalName().getBytes()));
PTable indexLogicalTable = new DelegateTable(table) {
@Override
public PName getPhysicalName() {
- return physicalNames.get(index);
+ return physicalName;
}
};
- rowCount += updateStatisticsInternal(name, indexLogicalTable, updateStatisticsStmt.getProps());
+ rowCount += updateStatisticsInternal(physicalName, indexLogicalTable, updateStatisticsStmt.getProps());
+ }
+ PName physicalName = table.getPhysicalName();
+ List<byte[]> localCFs = MetaDataUtil.getLocalIndexColumnFamilies(connection, physicalName.getBytes());
+ if (!localCFs.isEmpty()) {
+ rowCount += updateStatisticsInternal(physicalName, table, updateStatisticsStmt.getProps(), localCFs);
}
}
}
@@ -1030,6 +1022,10 @@ public class MetaDataClient {
}
private long updateStatisticsInternal(PName physicalName, PTable logicalTable, Map<String, Object> statsProps) throws SQLException {
+ return updateStatisticsInternal(physicalName, logicalTable, statsProps, null);
+ }
+
+ private long updateStatisticsInternal(PName physicalName, PTable logicalTable, Map<String, Object> statsProps, List<byte[]> cfs) throws SQLException {
ReadOnlyProps props = connection.getQueryServices().getProps();
final long msMinBetweenUpdates = props
.getLong(QueryServices.MIN_STATS_UPDATE_FREQ_MS_ATTRIB,
@@ -1063,7 +1059,7 @@ public class MetaDataClient {
}
};
TableRef tableRef = new TableRef(null, nonTxnLogicalTable, clientTimeStamp, false);
- MutationPlan plan = compiler.compile(Collections.singletonList(tableRef), null, null, null, clientTimeStamp);
+ MutationPlan plan = compiler.compile(Collections.singletonList(tableRef), null, cfs, null, clientTimeStamp);
Scan scan = plan.getContext().getScan();
scan.setCacheBlocks(false);
scan.setAttribute(ANALYZE_TABLE, TRUE_BYTES);
@@ -1764,6 +1760,7 @@ public class MetaDataClient {
updateCacheFrequency = updateCacheFrequencyProp;
}
String autoPartitionSeq = (String) TableProperty.AUTO_PARTITION_SEQ.getValue(tableProps);
+ Long guidePostWidth = (Long) TableProperty.GUIDE_POSTS_WIDTH.getValue(tableProps);
Boolean storeNullsProp = (Boolean) TableProperty.STORE_NULLS.getValue(tableProps);
if (storeNullsProp == null) {
@@ -2230,6 +2227,11 @@ public class MetaDataClient {
tableUpsert.setString(23, autoPartitionSeq);
}
tableUpsert.setBoolean(24, isAppendOnlySchema);
+ if (guidePostWidth == null) {
+ tableUpsert.setNull(25, Types.BIGINT);
+ } else {
+ tableUpsert.setLong(25, guidePostWidth);
+ }
tableUpsert.execute();
if (asyncCreatedDate != null) {
@@ -2631,11 +2633,12 @@ public class MetaDataClient {
}
private long incrementTableSeqNum(PTable table, PTableType expectedType, int columnCountDelta, Boolean isTransactional, Long updateCacheFrequency) throws SQLException {
- return incrementTableSeqNum(table, expectedType, columnCountDelta, isTransactional, updateCacheFrequency, null, null, null, null);
+ return incrementTableSeqNum(table, expectedType, columnCountDelta, isTransactional, updateCacheFrequency, null, null, null, null, -1L);
}
private long incrementTableSeqNum(PTable table, PTableType expectedType, int columnCountDelta,
- Boolean isTransactional, Long updateCacheFrequency, Boolean isImmutableRows, Boolean disableWAL, Boolean isMultiTenant, Boolean storeNulls)
+ Boolean isTransactional, Long updateCacheFrequency, Boolean isImmutableRows, Boolean disableWAL,
+ Boolean isMultiTenant, Boolean storeNulls, Long guidePostWidth)
throws SQLException {
String schemaName = table.getSchemaName().getString();
String tableName = table.getTableName().getString();
@@ -2673,6 +2676,9 @@ public class MetaDataClient {
if (updateCacheFrequency != null) {
mutateLongProperty(tenantId, schemaName, tableName, UPDATE_CACHE_FREQUENCY, updateCacheFrequency);
}
+ if (guidePostWidth == null || guidePostWidth >= 0) {
+ mutateLongProperty(tenantId, schemaName, tableName, GUIDE_POSTS_WIDTH, guidePostWidth);
+ }
return seqNum;
}
@@ -2694,7 +2700,7 @@ public class MetaDataClient {
}
private void mutateLongProperty(String tenantId, String schemaName, String tableName,
- String propertyName, long propertyValue) throws SQLException {
+ String propertyName, Long propertyValue) throws SQLException {
String updatePropertySql = "UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"( " +
TENANT_ID + "," +
TABLE_SCHEM + "," +
@@ -2705,7 +2711,11 @@ public class MetaDataClient {
tableBoolUpsert.setString(1, tenantId);
tableBoolUpsert.setString(2, schemaName);
tableBoolUpsert.setString(3, tableName);
- tableBoolUpsert.setLong(4, propertyValue);
+ if (propertyValue == null) {
+ tableBoolUpsert.setNull(4, Types.BIGINT);
+ } else {
+ tableBoolUpsert.setLong(4, propertyValue);
+ }
tableBoolUpsert.execute();
}
}
@@ -2733,6 +2743,7 @@ public class MetaDataClient {
Boolean storeNullsProp = null;
Boolean isTransactionalProp = null;
Long updateCacheFrequencyProp = null;
+ Long guidePostWidth = -1L;
Map<String, List<Pair<String, Object>>> properties = new HashMap<>(stmtProperties.size());
List<ColumnDef> columnDefs = null;
@@ -2791,6 +2802,8 @@ public class MetaDataClient {
isTransactionalProp = (Boolean)value;
} else if (propName.equals(UPDATE_CACHE_FREQUENCY)) {
updateCacheFrequencyProp = (Long)value;
+ } else if (propName.equals(GUIDE_POSTS_WIDTH)) {
+ guidePostWidth = (Long)value;
}
}
// if removeTableProps is true only add the property if it is not a HTable or Phoenix Table property
@@ -2857,6 +2870,9 @@ public class MetaDataClient {
changingPhoenixTableProperty = true;
}
}
+ if (guidePostWidth == null || guidePostWidth >= 0) {
+ changingPhoenixTableProperty = true;
+ }
Boolean storeNulls = null;
if (storeNullsProp != null) {
if (storeNullsProp.booleanValue() != table.getStoreNulls()) {
@@ -2996,7 +3012,8 @@ public class MetaDataClient {
}
long seqNum = table.getSequenceNumber();
if (changingPhoenixTableProperty || columnDefs.size() > 0) {
- seqNum = incrementTableSeqNum(table, tableType, columnDefs.size(), isTransactional, updateCacheFrequency, isImmutableRows, disableWAL, multiTenant, storeNulls);
+ seqNum = incrementTableSeqNum(table, tableType, columnDefs.size(), isTransactional, updateCacheFrequency, isImmutableRows,
+ disableWAL, multiTenant, storeNulls, guidePostWidth);
tableMetaData.addAll(connection.getMutationState().toMutations(timeStamp).next().getSecond());
connection.rollback();
}
@@ -3091,7 +3108,7 @@ public class MetaDataClient {
connection.setAutoCommit(true);
// Delete everything in the column. You'll still be able to do queries at earlier timestamps
long ts = (scn == null ? result.getMutationTime() : scn);
- MutationPlan plan = new PostDDLCompiler(connection).compile(Collections.singletonList(new TableRef(null, table, ts, false)), emptyCF, projectCF, null, ts);
+ MutationPlan plan = new PostDDLCompiler(connection).compile(Collections.singletonList(new TableRef(null, table, ts, false)), emptyCF, Collections.singletonList(projectCF), null, ts);
return connection.getQueryServices().updateData(plan);
}
return new MutationState(0,connection);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java
index d5d0b84..26a7718 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableProperty.java
@@ -77,6 +77,13 @@ public enum TableProperty {
},
APPEND_ONLY_SCHEMA(PhoenixDatabaseMetaData.APPEND_ONLY_SCHEMA, COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY, true, true),
+ GUIDE_POSTS_WIDTH(PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH, true, false) {
+ @Override
+ public Object getValue(Object value) {
+ return value == null ? null : ((Number) value).longValue();
+ }
+
+ },
;
private final String propertyName;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java
index 3bb9898..0cf5ed8 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/DefaultStatisticsCollector.java
@@ -21,12 +21,17 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.KeyValueUtil;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Get;
+import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
@@ -35,11 +40,14 @@ import org.apache.hadoop.hbase.regionserver.Store;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.phoenix.coprocessor.MetaDataProtocol;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.SortOrder;
import org.apache.phoenix.schema.types.PInteger;
import org.apache.phoenix.schema.types.PLong;
+import org.apache.phoenix.util.MetaDataUtil;
+import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.TimeKeeper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -53,24 +61,21 @@ class DefaultStatisticsCollector implements StatisticsCollector {
private static final Logger logger = LoggerFactory.getLogger(DefaultStatisticsCollector.class);
private final Map<ImmutableBytesPtr, Pair<Long, GuidePostsInfoBuilder>> guidePostsInfoWriterMap = Maps.newHashMap();
private final StatisticsWriter statsWriter;
- private final Pair<Long, GuidePostsInfoBuilder> cachedGps;
+ private final Pair<Long, GuidePostsInfoBuilder> cachedGuidePosts;
+ private final byte[] guidePostWidthBytes;
+ private final byte[] guidePostPerRegionBytes;
+ // Where to look for GUIDE_POST_WIDTH in SYSTEM.CATALOG
+ private final byte[] ptableKey;
+ private final RegionCoprocessorEnvironment env;
- private long guidepostDepth;
+ private long guidePostDepth;
private long maxTimeStamp = MetaDataProtocol.MIN_TABLE_TIMESTAMP;
DefaultStatisticsCollector(RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp, byte[] family,
byte[] gp_width_bytes, byte[] gp_per_region_bytes) throws IOException {
- Configuration config = env.getConfiguration();
- int guidepostPerRegion = gp_per_region_bytes == null
- ? config.getInt(QueryServices.STATS_GUIDEPOST_PER_REGION_ATTRIB,
- QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_PER_REGION)
- : PInteger.INSTANCE.getCodec().decodeInt(gp_per_region_bytes, 0, SortOrder.getDefault());
- long guidepostWidth = gp_width_bytes == null
- ? config.getLong(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB,
- QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_WIDTH_BYTES)
- : PLong.INSTANCE.getCodec().decodeInt(gp_width_bytes, 0, SortOrder.getDefault());
- this.guidepostDepth = StatisticsUtil.getGuidePostDepth(guidepostPerRegion, guidepostWidth,
- env.getRegion().getTableDesc());
+ this.env = env;
+ this.guidePostWidthBytes = gp_width_bytes;
+ this.guidePostPerRegionBytes = gp_per_region_bytes;
// Provides a means of clients controlling their timestamps to not use current time
// when background tasks are updating stats. Instead we track the max timestamp of
// the cells and use that.
@@ -80,16 +85,64 @@ class DefaultStatisticsCollector implements StatisticsCollector {
if (!useCurrentTime) {
clientTimeStamp = DefaultStatisticsCollector.NO_TIMESTAMP;
}
+ String pName = tableName;
+ // For view index, get GUIDE_POST_WIDTH from data physical table
+ // since there's no row representing those in SYSTEM.CATALOG.
+ if (MetaDataUtil.isViewIndex(tableName)) {
+ pName = MetaDataUtil.getViewIndexUserTableName(tableName);
+ }
+ ptableKey = SchemaUtil.getTableKeyFromFullName(pName);
// Get the stats table associated with the current table on which the CP is
// triggered
this.statsWriter = StatisticsWriter.newWriter(env, tableName, clientTimeStamp);
// in a compaction we know the one family ahead of time
if (family != null) {
ImmutableBytesPtr cfKey = new ImmutableBytesPtr(family);
- cachedGps = new Pair<Long, GuidePostsInfoBuilder>(0l, new GuidePostsInfoBuilder());
- guidePostsInfoWriterMap.put(cfKey, cachedGps);
+ cachedGuidePosts = new Pair<Long, GuidePostsInfoBuilder>(0l, new GuidePostsInfoBuilder());
+ guidePostsInfoWriterMap.put(cfKey, cachedGuidePosts);
+ } else {
+ cachedGuidePosts = null;
+ }
+ }
+
+ private void initGuidepostDepth() throws IOException {
+ // First check is if guidepost info set on statement itself
+ if (guidePostPerRegionBytes != null || guidePostWidthBytes != null) {
+ int guidepostPerRegion = 0;
+ long guidepostWidth = QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_WIDTH_BYTES;
+ if (guidePostPerRegionBytes != null) {
+ guidepostPerRegion = PInteger.INSTANCE.getCodec().decodeInt(guidePostPerRegionBytes, 0, SortOrder.getDefault());
+ }
+ if (guidePostWidthBytes != null) {
+ guidepostWidth = PLong.INSTANCE.getCodec().decodeInt(guidePostWidthBytes, 0, SortOrder.getDefault());
+ }
+ this.guidePostDepth = StatisticsUtil.getGuidePostDepth(guidepostPerRegion, guidepostWidth,
+ env.getRegion().getTableDesc());
} else {
- cachedGps = null;
+ // Next check for GUIDE_POST_WIDTH on table
+ HTableInterface htable = env.getTable(TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES));
+ Get get = new Get(ptableKey);
+ get.addColumn(PhoenixDatabaseMetaData.TABLE_FAMILY_BYTES, PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH_BYTES);
+ Result result = htable.get(get);
+ long guidepostWidth = 0;
+ if (!result.isEmpty()) {
+ Cell cell = result.listCells().get(0);
+ guidepostWidth = PLong.INSTANCE.getCodec().decodeLong(cell.getValueArray(), cell.getValueOffset(), SortOrder.getDefault());
+ }
+ if (guidepostWidth > 0) {
+ this.guidePostDepth = guidepostWidth;
+ } else {
+ // Last use global config value
+ Configuration config = env.getConfiguration();
+ this.guidePostDepth = StatisticsUtil.getGuidePostDepth(
+ config.getInt(
+ QueryServices.STATS_GUIDEPOST_PER_REGION_ATTRIB,
+ QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_PER_REGION),
+ config.getLong(
+ QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB,
+ QueryServicesOptions.DEFAULT_STATS_GUIDEPOST_WIDTH_BYTES),
+ env.getRegion().getTableDesc());
+ }
}
}
@@ -114,28 +167,32 @@ class DefaultStatisticsCollector implements StatisticsCollector {
commitStats(mutations);
} catch (IOException e) {
logger.error("Unable to commit new stats", e);
- } finally {
- clear();
}
}
private void writeStatsToStatsTable(final Region region, boolean delete, List<Mutation> mutations, long currentTime)
throws IOException {
try {
+ Set<ImmutableBytesPtr> fams = guidePostsInfoWriterMap.keySet();
// update the statistics table
// Delete statistics for a region if no guidepost is collected for that region during UPDATE STATISTICS
// This will not impact a stats collection of single column family during compaction as
// guidePostsInfoWriterMap cannot be empty in this case.
- if (cachedGps == null) {
+ if (cachedGuidePosts == null) {
+ boolean collectingForLocalIndex = !fams.isEmpty() && MetaDataUtil.isLocalIndexFamily(fams.iterator().next());
for (Store store : region.getStores()) {
ImmutableBytesPtr cfKey = new ImmutableBytesPtr(store.getFamily().getName());
+ boolean isLocalIndexStore = MetaDataUtil.isLocalIndexFamily(cfKey);
+ if (isLocalIndexStore != collectingForLocalIndex) {
+ continue;
+ }
if (!guidePostsInfoWriterMap.containsKey(cfKey)) {
Pair<Long, GuidePostsInfoBuilder> emptyGps = new Pair<Long, GuidePostsInfoBuilder>(0l, new GuidePostsInfoBuilder());
guidePostsInfoWriterMap.put(cfKey, emptyGps);
}
}
}
- for (ImmutableBytesPtr fam : guidePostsInfoWriterMap.keySet()) {
+ for (ImmutableBytesPtr fam : fams) {
if (delete) {
if (logger.isDebugEnabled()) {
logger.debug("Deleting the stats for the region " + region.getRegionInfo());
@@ -162,6 +219,7 @@ class DefaultStatisticsCollector implements StatisticsCollector {
*
* @param results
* next batch of {@link KeyValue}s
+ * @throws IOException
*/
@Override
public void collectStatistics(final List<Cell> results) {
@@ -171,7 +229,7 @@ class DefaultStatisticsCollector implements StatisticsCollector {
KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
maxTimeStamp = Math.max(maxTimeStamp, kv.getTimestamp());
Pair<Long, GuidePostsInfoBuilder> gps;
- if (cachedGps == null) {
+ if (cachedGuidePosts == null) {
ImmutableBytesPtr cfKey = new ImmutableBytesPtr(kv.getFamilyArray(), kv.getFamilyOffset(),
kv.getFamilyLength());
gps = guidePostsInfoWriterMap.get(cfKey);
@@ -185,16 +243,16 @@ class DefaultStatisticsCollector implements StatisticsCollector {
gps.getSecond().incrementRowCount();
}
} else {
- gps = cachedGps;
+ gps = cachedGuidePosts;
if (incrementRow) {
- cachedGps.getSecond().incrementRowCount();
+ cachedGuidePosts.getSecond().incrementRowCount();
incrementRow = false;
}
}
int kvLength = kv.getLength();
long byteCount = gps.getFirst() + kvLength;
gps.setFirst(byteCount);
- if (byteCount >= guidepostDepth) {
+ if (byteCount >= guidePostDepth) {
ImmutableBytesWritable row = new ImmutableBytesWritable(kv.getRowArray(), kv.getRowOffset(), kv.getRowLength());
if (gps.getSecond().addGuidePosts(row, byteCount, gps.getSecond().getRowCount())) {
gps.setFirst(0l);
@@ -221,9 +279,10 @@ class DefaultStatisticsCollector implements StatisticsCollector {
}
@Override
- public void clear() {
+ public void init() throws IOException {
this.guidePostsInfoWriterMap.clear();
maxTimeStamp = MetaDataProtocol.MIN_TABLE_TIMESTAMP;
+ initGuidepostDepth();
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java
index 1063229..a16a48a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/NoOpStatisticsCollector.java
@@ -21,13 +21,10 @@ import java.io.IOException;
import java.util.List;
import org.apache.hadoop.hbase.Cell;
-import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.Store;
-import org.apache.hadoop.hbase.util.Pair;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
/**
@@ -62,7 +59,8 @@ public class NoOpStatisticsCollector implements StatisticsCollector {
return delegate;
}
- @Override public void clear() {
+ @Override
+ public void init() {
// No-op
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
index 1dcab08..43ec6c7 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollector.java
@@ -23,7 +23,6 @@ import java.util.List;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
-import org.apache.hadoop.hbase.regionserver.HRegion;
import org.apache.hadoop.hbase.regionserver.InternalScanner;
import org.apache.hadoop.hbase.regionserver.Region;
import org.apache.hadoop.hbase.regionserver.Store;
@@ -50,6 +49,7 @@ public interface StatisticsCollector extends Closeable {
/**
* Collect statistics for the given list of cells. This method can be called multiple times
* during collection of statistics.
+ * @throws IOException
*/
void collectStatistics(List<Cell> results);
@@ -60,9 +60,10 @@ public interface StatisticsCollector extends Closeable {
InternalScanner delegate) throws IOException;
/**
- * Clear all statistics information that has been collected.
+ * Called before beginning the collection of statistics through {@link #collectStatistics(List)}
+ * @throws IOException
*/
- void clear();
+ void init() throws IOException;
/**
* Retrieve the calculated guide post info for the given column family.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectorFactory.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectorFactory.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectorFactory.java
index bb21f1b..bbc7cd4 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectorFactory.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsCollectorFactory.java
@@ -29,27 +29,28 @@ import org.apache.phoenix.query.QueryServices;
public class StatisticsCollectorFactory {
public static StatisticsCollector createStatisticsCollector(RegionCoprocessorEnvironment env,
- String tableName, long clientTimestamp, byte[] guidepostWidthBytes,
+ String tableName, long clientTimeStamp, byte[] guidepostWidthBytes,
byte[] guidepostsPerRegionBytes) throws IOException {
- if (statisticsEnabled(env)) {
- return new DefaultStatisticsCollector(env, tableName, clientTimestamp, null,
- guidepostWidthBytes, guidepostsPerRegionBytes);
- } else {
- return new NoOpStatisticsCollector();
- }
+ return createStatisticsCollector(env, tableName, clientTimeStamp, null, guidepostWidthBytes, guidepostsPerRegionBytes);
}
public static StatisticsCollector createStatisticsCollector(
RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp,
byte[] storeName) throws IOException {
+ return createStatisticsCollector(env, tableName, clientTimeStamp, storeName, null, null);
+ }
+
+ public static StatisticsCollector createStatisticsCollector(
+ RegionCoprocessorEnvironment env, String tableName, long clientTimeStamp,
+ byte[] storeName, byte[] guidepostWidthBytes,
+ byte[] guidepostsPerRegionBytes) throws IOException {
if (statisticsEnabled(env)) {
return new DefaultStatisticsCollector(env, tableName, clientTimeStamp, storeName,
- null, null);
+ guidepostWidthBytes, guidepostsPerRegionBytes);
} else {
return new NoOpStatisticsCollector();
}
}
-
/**
* Determines if statistics are enabled (which is the default). This is done on the
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java
index 736efc6..8306940 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsScanner.java
@@ -83,8 +83,9 @@ public class StatisticsScanner implements InternalScanner {
*
* @param results
* next batch of {@link KeyValue}s
+ * @throws IOException
*/
- private void updateStats(final List<Cell> results) {
+ private void updateStats(final List<Cell> results) throws IOException {
if (!results.isEmpty()) {
tracker.collectStatistics(results);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
index db31b69..685c24e 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
@@ -20,7 +20,6 @@ import static org.apache.phoenix.util.SchemaUtil.getVarCharLength;
import java.io.IOException;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.List;
import java.util.Set;
import java.util.SortedMap;
@@ -30,7 +29,6 @@ import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellScanner;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.HTableInterface;
import org.apache.hadoop.hbase.client.Result;
@@ -99,7 +97,7 @@ public class StatisticsUtil {
return rowKey;
}
- public static byte[] getKey(byte[] table, ImmutableBytesPtr fam) {
+ private static byte[] getKey(byte[] table, ImmutableBytesPtr fam) {
// always starts with the source table and column family
byte[] rowKey = new byte[table.length + fam.getLength() + 1];
int offset = 0;
@@ -111,11 +109,7 @@ public class StatisticsUtil {
return rowKey;
}
- public static byte[] copyRow(KeyValue kv) {
- return Arrays.copyOfRange(kv.getRowArray(), kv.getRowOffset(), kv.getRowOffset() + kv.getRowLength());
- }
-
- public static byte[] getAdjustedKey(byte[] key, byte[] tableNameBytes, ImmutableBytesPtr cf, boolean nextKey) {
+ private static byte[] getAdjustedKey(byte[] key, byte[] tableNameBytes, ImmutableBytesPtr cf, boolean nextKey) {
if (Bytes.compareTo(key, ByteUtil.EMPTY_BYTE_ARRAY) != 0) { return getRowKey(tableNameBytes, cf, key); }
key = ByteUtil.concat(getKey(tableNameBytes, cf), QueryConstants.SEPARATOR_BYTE_ARRAY);
if (nextKey) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
index 9089b68..9622880 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/IndexUtil.java
@@ -491,7 +491,7 @@ public class IndexUtil {
joinResult = dataRegion.get(get);
} else {
TableName dataTable =
- TableName.valueOf(MetaDataUtil.getUserTableName(c.getEnvironment()
+ TableName.valueOf(MetaDataUtil.getLocalIndexUserTableName(c.getEnvironment()
.getRegion().getTableDesc().getNameAsString()));
HTableInterface table = null;
try {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
index c72e404..52346a5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collection;
+import java.util.Collections;
import java.util.List;
import org.apache.hadoop.conf.Configuration;
@@ -214,7 +215,7 @@ public class MetaDataUtil {
}
public static long getSequenceNumber(List<Mutation> tableMetaData) {
- return getSequenceNumber(getTableHeaderRow(tableMetaData));
+ return getSequenceNumber(getPutOnlyTableHeaderRow(tableMetaData));
}
public static PTableType getTableType(List<Mutation> tableMetaData, KeyValueBuilder builder,
@@ -366,7 +367,7 @@ public class MetaDataUtil {
return schemaName;
}
- public static String getUserTableName(String localIndexTableName) {
+ public static String getLocalIndexUserTableName(String localIndexTableName) {
if (localIndexTableName.contains(QueryConstants.NAMESPACE_SEPARATOR)) {
String schemaName = SchemaUtil.getSchemaNameFromFullName(localIndexTableName,
QueryConstants.NAMESPACE_SEPARATOR);
@@ -384,6 +385,24 @@ public class MetaDataUtil {
}
}
+ public static String getViewIndexUserTableName(String viewIndexTableName) {
+ if (viewIndexTableName.contains(QueryConstants.NAMESPACE_SEPARATOR)) {
+ String schemaName = SchemaUtil.getSchemaNameFromFullName(viewIndexTableName,
+ QueryConstants.NAMESPACE_SEPARATOR);
+ String tableName = SchemaUtil.getTableNameFromFullName(viewIndexTableName,
+ QueryConstants.NAMESPACE_SEPARATOR);
+ String userTableName = tableName.substring(VIEW_INDEX_TABLE_PREFIX.length());
+ return (schemaName + QueryConstants.NAMESPACE_SEPARATOR + userTableName);
+ } else {
+ String schemaName = SchemaUtil.getSchemaNameFromFullName(viewIndexTableName);
+ if (!schemaName.isEmpty()) schemaName = schemaName.substring(VIEW_INDEX_TABLE_PREFIX.length());
+ String tableName = viewIndexTableName.substring(
+ (schemaName.isEmpty() ? 0 : (schemaName.length() + QueryConstants.NAME_SEPARATOR.length()))
+ + VIEW_INDEX_TABLE_PREFIX.length());
+ return SchemaUtil.getTableName(schemaName, tableName);
+ }
+ }
+
public static String getViewIndexSequenceSchemaName(PName physicalName, boolean isNamespaceMapped) {
if (!isNamespaceMapped) { return VIEW_INDEX_SEQUENCE_PREFIX + physicalName.getString(); }
return SchemaUtil.getSchemaNameFromFullName(physicalName.toString());
@@ -461,6 +480,18 @@ public class MetaDataUtil {
return families;
}
+ public static List<byte[]> getLocalIndexColumnFamilies(PhoenixConnection conn, byte[] physicalTableName) throws SQLException {
+ HTableDescriptor desc = conn.getQueryServices().getTableDescriptor(physicalTableName);
+ if(desc == null ) return Collections.emptyList();
+ List<byte[]> families = new ArrayList<byte[]>(desc.getColumnFamilies().length / 2);
+ for (HColumnDescriptor cf : desc.getColumnFamilies()) {
+ if (cf.getNameAsString().startsWith(QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX)) {
+ families.add(cf.getName());
+ }
+ }
+ return families;
+ }
+
public static void deleteViewIndexSequences(PhoenixConnection connection, PName name, boolean isNamespaceMapped)
throws SQLException {
String schemaName = getViewIndexSequenceSchemaName(name, isNamespaceMapped);
@@ -603,4 +634,9 @@ public class MetaDataUtil {
public static boolean isHTableProperty(String propName) {
return !isHColumnProperty(propName) && !TableProperty.isPhoenixTableProperty(propName);
}
+
+ public static boolean isLocalIndexFamily(ImmutableBytesPtr cfPtr) {
+ return cfPtr.getLength() >= QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX_BYTES.length &&
+ Bytes.compareTo(cfPtr.get(), cfPtr.getOffset(), QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX_BYTES.length, QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX_BYTES, 0, QueryConstants.LOCAL_INDEX_COLUMN_FAMILY_PREFIX_BYTES.length) == 0;
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
index cddebb7..93b3f73 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
@@ -348,7 +348,7 @@ public class UpgradeUtil {
HTableDescriptor[] localIndexTables = admin.listTables(MetaDataUtil.LOCAL_INDEX_TABLE_PREFIX+".*");
String localIndexSplitter = LocalIndexSplitter.class.getName();
for (HTableDescriptor table : localIndexTables) {
- HTableDescriptor dataTableDesc = admin.getTableDescriptor(TableName.valueOf(MetaDataUtil.getUserTableName(table.getNameAsString())));
+ HTableDescriptor dataTableDesc = admin.getTableDescriptor(TableName.valueOf(MetaDataUtil.getLocalIndexUserTableName(table.getNameAsString())));
HColumnDescriptor[] columnFamilies = dataTableDesc.getColumnFamilies();
boolean modifyTable = false;
for(HColumnDescriptor cf : columnFamilies) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/711d7f06/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
index c91b855..7697d8c 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/compile/QueryCompilerTest.java
@@ -2257,6 +2257,22 @@ public class QueryCompilerTest extends BaseConnectionlessQueryTest {
}
}
+ @Test
+ public void testNegativeGuidePostWidth() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);) {
+ try {
+ conn.createStatement().execute(
+ "CREATE TABLE t (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR) GUIDE_POST_WIDTH = -1");
+ fail();
+ } catch (SQLException e) {
+ assertEquals("Unexpected Exception",
+ SQLExceptionCode.PARSER_ERROR
+ .getErrorCode(), e.getErrorCode());
+ }
+ }
+ }
+
private static void assertFamilies(Scan s, String... families) {
assertEquals(families.length, s.getFamilyMap().size());
for (String fam : families) {