You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ja...@apache.org on 2014/09/22 23:36:58 UTC
[5/5] git commit: PHOENIX-180 Use stats to guide query
parallelization (Ramkrishna S. Vasudevan)
PHOENIX-180 Use stats to guide query parallelization (Ramkrishna S. Vasudevan)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/15a54d55
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/15a54d55
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/15a54d55
Branch: refs/heads/3.0
Commit: 15a54d557b3ad839fe903f255fa7578f2d79c5ff
Parents: d8766cf
Author: James Taylor <jt...@salesforce.com>
Authored: Mon Sep 22 14:41:46 2014 -0700
Committer: James Taylor <jt...@salesforce.com>
Committed: Mon Sep 22 14:41:46 2014 -0700
----------------------------------------------------------------------
.../org/apache/phoenix/end2end/ArrayIT.java | 12 +-
.../end2end/BaseClientManagedTimeIT.java | 2 +-
.../BaseParallelIteratorsRegionSplitterIT.java | 9 +-
.../org/apache/phoenix/end2end/BaseQueryIT.java | 7 +-
.../end2end/BaseTenantSpecificViewIndexIT.java | 2 +-
.../org/apache/phoenix/end2end/BaseViewIT.java | 2 -
...efaultParallelIteratorsRegionSplitterIT.java | 55 +-
.../phoenix/end2end/GuidePostsLifeCycleIT.java | 148 ++
.../org/apache/phoenix/end2end/HashJoinIT.java | 2 -
.../org/apache/phoenix/end2end/KeyOnlyIT.java | 61 +-
.../phoenix/end2end/MultiCfQueryExecIT.java | 78 +-
.../end2end/QueryDatabaseMetaDataIT.java | 4 +
.../org/apache/phoenix/end2end/QueryIT.java | 37 +-
.../phoenix/end2end/RegexpSplitFunctionIT.java | 15 +-
.../phoenix/end2end/RegexpSubstrFunctionIT.java | 21 +-
...ipRangeParallelIteratorRegionSplitterIT.java | 32 +-
.../phoenix/end2end/StatsCollectorIT.java | 230 ++
.../apache/phoenix/end2end/StatsManagerIT.java | 2 +-
.../end2end/TenantSpecificTablesDDLIT.java | 4 +
.../end2end/TenantSpecificTablesDMLIT.java | 23 +-
.../phoenix/end2end/index/MutableIndexIT.java | 1 -
.../phoenix/end2end/index/SaltedIndexIT.java | 30 +-
.../salted/SaltedTableUpsertSelectIT.java | 21 +-
phoenix-core/src/main/antlr3/PhoenixSQL.g | 10 +-
.../org/apache/phoenix/cache/GlobalCache.java | 7 +-
.../apache/phoenix/compile/WhereCompiler.java | 1 -
.../coprocessor/MetaDataEndpointImpl.java | 121 +-
.../phoenix/coprocessor/MetaDataProtocol.java | 3 +
.../apache/phoenix/execute/MutationState.java | 2 -
.../function/RoundDecimalExpression.java | 14 +-
.../DefaultParallelIteratorRegionSplitter.java | 188 +-
.../phoenix/jdbc/PhoenixDatabaseMetaData.java | 16 +
.../apache/phoenix/jdbc/PhoenixStatement.java | 49 +
.../apache/phoenix/parse/ParseNodeFactory.java | 4 +
.../parse/UpdateStatisticsStatement.java | 26 +
.../phoenix/query/ConnectionQueryServices.java | 6 +-
.../query/ConnectionQueryServicesImpl.java | 134 +-
.../query/ConnectionlessQueryServicesImpl.java | 43 +-
.../query/DelegateConnectionQueryServices.java | 16 +-
.../apache/phoenix/query/QueryConstants.java | 29 +-
.../org/apache/phoenix/query/QueryServices.java | 5 +-
.../phoenix/query/QueryServicesOptions.java | 36 +-
.../apache/phoenix/schema/MetaDataClient.java | 97 +-
.../phoenix/schema/MetaDataClient.java.orig | 2076 ++++++++++++++++++
.../apache/phoenix/schema/PColumnFamily.java | 3 +
.../phoenix/schema/PColumnFamilyImpl.java | 25 +-
.../java/org/apache/phoenix/schema/PTable.java | 6 +-
.../org/apache/phoenix/schema/PTableImpl.java | 86 +-
.../apache/phoenix/schema/stat/PTableStats.java | 6 +-
.../phoenix/schema/stat/PTableStatsImpl.java | 40 +-
.../schema/stat/PTableStatsImpl.java.orig | 68 +
.../schema/stat/PTableStatsImpl.java.rej | 74 +
.../schema/stat/StatisticsCollector.java | 425 ++++
.../stat/StatisticsCollectorProtocol.java | 29 +
.../stat/StatisticsCollectorResponse.java | 47 +
.../phoenix/schema/stat/StatisticsScanner.java | 125 ++
.../phoenix/schema/stat/StatisticsTable.java | 172 ++
.../phoenix/schema/stat/StatisticsTracker.java | 62 +
.../phoenix/schema/stat/StatisticsUtils.java | 106 +
.../java/org/apache/phoenix/query/BaseTest.java | 7 +-
.../phoenix/query/QueryServicesTestImpl.java | 8 +-
pom.xml | 2 +-
62 files changed, 4559 insertions(+), 413 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/15a54d55/phoenix-core/src/it/java/org/apache/phoenix/end2end/ArrayIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ArrayIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ArrayIT.java
index a43cfb1..a7fe827 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ArrayIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ArrayIT.java
@@ -64,10 +64,11 @@ public class ArrayIT extends BaseClientManagedTimeIT {
String query = "SELECT a_double_array, /* comment ok? */ b_string, a_float FROM table_with_array WHERE ?=organization_id and ?=a_float";
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB,
- Long.toString(ts + 2)); // Execute at timestamp 2
+ Long.toString(ts + 2)); // Execute at timestamp 2
Connection conn = DriverManager.getConnection(getUrl(), props);
+ analyzeTable(conn, TABLE_WITH_ARRAY);
try {
- PreparedStatement statement = conn.prepareStatement(query);
+ PreparedStatement statement = conn.prepareStatement(query);
statement.setString(1, tenantId);
statement.setFloat(2, 0.01f);
ResultSet rs = statement.executeQuery();
@@ -90,6 +91,12 @@ public class ArrayIT extends BaseClientManagedTimeIT {
}
}
+ private void analyzeTable(Connection conn, String tableWithArray) throws SQLException {
+ String analyse = "ANALYZE "+tableWithArray;
+ PreparedStatement statement = conn.prepareStatement(analyse);
+ statement.execute();
+ }
+
@Test
public void testScanWithArrayInWhereClause() throws Exception {
long ts = nextTimestamp();
@@ -142,6 +149,7 @@ public class ArrayIT extends BaseClientManagedTimeIT {
props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB,
Long.toString(ts + 2)); // Execute at timestamp 2
Connection conn = DriverManager.getConnection(getUrl(), props);
+ analyzeTable(conn, TABLE_WITH_ARRAY);
try {
PreparedStatement statement = conn.prepareStatement(query);
statement.setString(1, tenantId);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/15a54d55/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseClientManagedTimeIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseClientManagedTimeIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseClientManagedTimeIT.java
index 79cb64b..8b47e11 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseClientManagedTimeIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseClientManagedTimeIT.java
@@ -52,7 +52,7 @@ import org.junit.experimental.categories.Category;
public abstract class BaseClientManagedTimeIT extends BaseTest {
private static String url;
protected static PhoenixTestDriver driver;
- private static final Configuration config = HBaseConfiguration.create();
+ protected static final Configuration config = HBaseConfiguration.create();
private static boolean clusterInitialized = false;
protected final static String getUrl() {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/15a54d55/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseParallelIteratorsRegionSplitterIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseParallelIteratorsRegionSplitterIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseParallelIteratorsRegionSplitterIT.java
index d2d3c7e..cfaa8ee 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseParallelIteratorsRegionSplitterIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseParallelIteratorsRegionSplitterIT.java
@@ -28,7 +28,6 @@ import java.util.Map;
import java.util.Properties;
import org.apache.phoenix.jdbc.PhoenixConnection;
-import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.schema.PTableKey;
import org.apache.phoenix.schema.TableRef;
import org.apache.phoenix.util.PhoenixRuntime;
@@ -58,12 +57,7 @@ public class BaseParallelIteratorsRegionSplitterIT extends BaseClientManagedTime
@BeforeClass
@Shadower(classBeingShadowed = BaseClientManagedTimeIT.class)
public static void doSetup() throws Exception {
- int targetQueryConcurrency = 3;
- int maxQueryConcurrency = 5;
Map<String,String> props = Maps.newHashMapWithExpectedSize(3);
- props.put(QueryServices.MAX_QUERY_CONCURRENCY_ATTRIB, Integer.toString(maxQueryConcurrency));
- props.put(QueryServices.TARGET_QUERY_CONCURRENCY_ATTRIB, Integer.toString(targetQueryConcurrency));
- props.put(QueryServices.MAX_INTRA_REGION_PARALLELIZATION_ATTRIB, Integer.toString(Integer.MAX_VALUE));
// Must update config before starting server
setUpTestDriver(getUrl(), new ReadOnlyProps(props.entrySet().iterator()));
}
@@ -88,7 +82,8 @@ public class BaseParallelIteratorsRegionSplitterIT extends BaseClientManagedTime
protected static TableRef getTableRef(Connection conn, long ts) throws SQLException {
PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class);
- TableRef table = new TableRef(null,pconn.getMetaDataCache().getTable(new PTableKey(pconn.getTenantId(), STABLE_NAME)),ts, false);
+ TableRef table = new TableRef(null, pconn.getMetaDataCache().getTable(
+ new PTableKey(pconn.getTenantId(), STABLE_NAME)), ts, false);
return table;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/15a54d55/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseQueryIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseQueryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseQueryIT.java
index f87e86c..ba4c741 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseQueryIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseQueryIT.java
@@ -75,16 +75,11 @@ public abstract class BaseQueryIT extends BaseClientManagedTimeIT {
@BeforeClass
@Shadower(classBeingShadowed = BaseClientManagedTimeIT.class)
public static void doSetup() throws Exception {
- int targetQueryConcurrency = 2;
- int maxQueryConcurrency = 3;
Map<String,String> props = Maps.newHashMapWithExpectedSize(5);
- props.put(QueryServices.QUEUE_SIZE_ATTRIB, Integer.toString(100));
- props.put(QueryServices.MAX_QUERY_CONCURRENCY_ATTRIB, Integer.toString(maxQueryConcurrency));
- props.put(QueryServices.TARGET_QUERY_CONCURRENCY_ATTRIB, Integer.toString(targetQueryConcurrency));
+ props.put(QueryServices.QUEUE_SIZE_ATTRIB, Integer.toString(5000));
props.put(IndexWriterUtils.HTABLE_THREAD_KEY, Integer.toString(100));
// Make a small batch size to test multiple calls to reserve sequences
props.put(QueryServices.SEQUENCE_CACHE_SIZE_ATTRIB, Long.toString(BATCH_SIZE));
-
// Must update config before starting server
setUpTestDriver(getUrl(), new ReadOnlyProps(props.entrySet().iterator()));
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/15a54d55/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseTenantSpecificViewIndexIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseTenantSpecificViewIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseTenantSpecificViewIndexIT.java
index 3b5eb1f..cda44c5 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseTenantSpecificViewIndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseTenantSpecificViewIndexIT.java
@@ -104,7 +104,7 @@ public class BaseTenantSpecificViewIndexIT extends BaseHBaseManagedTimeIT {
ResultSet rs = conn.createStatement().executeQuery("EXPLAIN SELECT k1, k2, v2 FROM v WHERE v2='" + valuePrefix + "v2-1'");
assertEquals(saltBuckets == null ?
"CLIENT PARALLEL 1-WAY RANGE SCAN OVER _IDX_T ['" + tenantId + "',-32768,'" + valuePrefix + "v2-1']" :
- "CLIENT PARALLEL 4-WAY SKIP SCAN ON 3 KEYS OVER _IDX_T [0,'" + tenantId + "',-32768,'" + valuePrefix + "v2-1'] - [2,'" + tenantId + "',-32768,'" + valuePrefix + "v2-1']\n" +
+ "CLIENT PARALLEL 3-WAY SKIP SCAN ON 3 KEYS OVER _IDX_T [0,'" + tenantId + "',-32768,'" + valuePrefix + "v2-1'] - [2,'" + tenantId + "',-32768,'" + valuePrefix + "v2-1']\n" +
"CLIENT MERGE SORT", QueryUtil.getExplainPlan(rs));
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/15a54d55/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java
index b125ab2..600e0bb 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseViewIT.java
@@ -27,7 +27,6 @@ import java.sql.DriverManager;
import java.sql.ResultSet;
import java.util.Map;
-import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.ReadOnlyProps;
import org.junit.BeforeClass;
@@ -43,7 +42,6 @@ public class BaseViewIT extends BaseHBaseManagedTimeIT {
public static void doSetup() throws Exception {
Map<String,String> props = Maps.newHashMapWithExpectedSize(1);
// Don't split intra region so we can more easily know that the n-way parallelization is for the explain plan
- props.put(QueryServices.MAX_INTRA_REGION_PARALLELIZATION_ATTRIB, Integer.toString(1));
// Must update config before starting server
setUpTestDriver(getUrl(), new ReadOnlyProps(props.entrySet().iterator()));
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/15a54d55/phoenix-core/src/it/java/org/apache/phoenix/end2end/DefaultParallelIteratorsRegionSplitterIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DefaultParallelIteratorsRegionSplitterIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DefaultParallelIteratorsRegionSplitterIT.java
index cd8f1fb..dd1dc8b 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DefaultParallelIteratorsRegionSplitterIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DefaultParallelIteratorsRegionSplitterIT.java
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertEquals;
import java.sql.Connection;
import java.sql.DriverManager;
+import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.Collections;
import java.util.Comparator;
@@ -38,7 +39,6 @@ import org.apache.phoenix.iterate.DefaultParallelIteratorRegionSplitter;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixStatement;
import org.apache.phoenix.parse.HintNode;
-import org.apache.phoenix.query.ConnectionQueryServices;
import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.schema.PDataType;
import org.apache.phoenix.schema.TableRef;
@@ -85,64 +85,61 @@ public class DefaultParallelIteratorsRegionSplitterIT extends BaseParallelIterat
public void testGetSplits() throws Exception {
long ts = nextTimestamp();
initTableValues(ts);
- String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + ts;
+ String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + ts + 2;
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
Connection conn = DriverManager.getConnection(url, props);
-
+ PreparedStatement stmt = conn.prepareStatement("ANALYZE STABLE");
+ stmt.execute();
Scan scan = new Scan();
// number of regions > target query concurrency
scan.setStartRow(K1);
scan.setStopRow(K12);
List<KeyRange> keyRanges = getSplits(conn, ts, scan);
- assertEquals("Unexpected number of splits: " + keyRanges, 5, keyRanges.size());
- assertEquals(newKeyRange(KeyRange.UNBOUND, K3), keyRanges.get(0));
- assertEquals(newKeyRange(K3, K4), keyRanges.get(1));
- assertEquals(newKeyRange(K4, K9), keyRanges.get(2));
- assertEquals(newKeyRange(K9, K11), keyRanges.get(3));
- assertEquals(newKeyRange(K11, KeyRange.UNBOUND), keyRanges.get(4));
+ assertEquals("Unexpected number of splits: " + keyRanges, 7, keyRanges.size());
+ assertEquals(newKeyRange(KeyRange.UNBOUND, KMIN), keyRanges.get(0));
+ assertEquals(newKeyRange(KMIN, K3), keyRanges.get(1));
+ assertEquals(newKeyRange(K3, K4), keyRanges.get(2));
+ assertEquals(newKeyRange(K4, K9), keyRanges.get(3));
+ assertEquals(newKeyRange(K9, K11), keyRanges.get(4));
+ assertEquals(newKeyRange(K11, KMAX), keyRanges.get(5));
+ assertEquals(newKeyRange(KMAX, KeyRange.UNBOUND), keyRanges.get(6));
- // (number of regions / 2) > target query concurrency
scan.setStartRow(K3);
scan.setStopRow(K6);
keyRanges = getSplits(conn, ts, scan);
- assertEquals("Unexpected number of splits: " + keyRanges, 3, keyRanges.size());
+ assertEquals("Unexpected number of splits: " + keyRanges, 2, keyRanges.size());
// note that we get a single split from R2 due to small key space
assertEquals(newKeyRange(K3, K4), keyRanges.get(0));
- assertEquals(newKeyRange(K4, K6), keyRanges.get(1));
- assertEquals(newKeyRange(K6, K9), keyRanges.get(2));
+ assertEquals(newKeyRange(K4, K9), keyRanges.get(1));
- // (number of regions / 2) <= target query concurrency
scan.setStartRow(K5);
scan.setStopRow(K6);
keyRanges = getSplits(conn, ts, scan);
- assertEquals("Unexpected number of splits: " + keyRanges, 3, keyRanges.size());
- assertEquals(newKeyRange(K4, K5), keyRanges.get(0));
- assertEquals(newKeyRange(K5, K6), keyRanges.get(1));
- assertEquals(newKeyRange(K6, K9), keyRanges.get(2));
+ assertEquals("Unexpected number of splits: " + keyRanges, 1, keyRanges.size());
+ assertEquals(newKeyRange(K4, K9), keyRanges.get(0));
conn.close();
}
@Test
- public void testGetLowerUnboundSplits() throws Exception {
+ public void testGetLowerUnboundSplits() throws Throwable {
long ts = nextTimestamp();
initTableValues(ts);
- String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + ts;
+ String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB;
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
Connection conn = DriverManager.getConnection(url, props);
-
+ PreparedStatement stmt = conn.prepareStatement("ANALYZE STABLE");
+ stmt.execute();
+ // The query would use all the split points here
+ conn.createStatement().executeQuery("SELECT * FROM STABLE");
+ conn.close();
Scan scan = new Scan();
-
- ConnectionQueryServices services = driver.getConnectionQueryServices(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES));
- TableRef table = getTableRef(conn,ts);
- services.getStatsManager().updateStats(table);
scan.setStartRow(HConstants.EMPTY_START_ROW);
scan.setStopRow(K1);
List<KeyRange> keyRanges = getSplits(conn, ts, scan);
- assertEquals("Unexpected number of splits: " + keyRanges, 3, keyRanges.size());
- assertEquals(newKeyRange(KeyRange.UNBOUND, new byte[] {'7'}), keyRanges.get(0));
- assertEquals(newKeyRange(new byte[] {'7'}, new byte[] {'M'}), keyRanges.get(1));
- assertEquals(newKeyRange(new byte[] {'M'}, K3), keyRanges.get(2));
+ assertEquals("Unexpected number of splits: " + keyRanges, 2, keyRanges.size());
+ assertEquals(newKeyRange(KeyRange.UNBOUND, KMIN), keyRanges.get(0));
+ assertEquals(newKeyRange(KMIN, K3), keyRanges.get(1));
}
private static KeyRange newKeyRange(byte[] lowerRange, byte[] upperRange) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/15a54d55/phoenix-core/src/it/java/org/apache/phoenix/end2end/GuidePostsLifeCycleIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/GuidePostsLifeCycleIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/GuidePostsLifeCycleIT.java
new file mode 100644
index 0000000..7645040
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/GuidePostsLifeCycleIT.java
@@ -0,0 +1,148 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import static org.apache.phoenix.util.TestUtil.STABLE_NAME;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.SQLException;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.compile.SequenceManager;
+import org.apache.phoenix.compile.StatementContext;
+import org.apache.phoenix.iterate.DefaultParallelIteratorRegionSplitter;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixStatement;
+import org.apache.phoenix.parse.HintNode;
+import org.apache.phoenix.query.KeyRange;
+import org.apache.phoenix.schema.PTableKey;
+import org.apache.phoenix.schema.TableRef;
+import org.apache.phoenix.util.PhoenixRuntime;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(HBaseManagedTimeTest.class)
+public class GuidePostsLifeCycleIT extends BaseHBaseManagedTimeIT {
+
+ protected static final byte[] KMIN = new byte[] {'!'};
+ protected static final byte[] KMIN2 = new byte[] {'.'};
+ protected static final byte[] K1 = new byte[] {'a'};
+ protected static final byte[] K3 = new byte[] {'c'};
+ protected static final byte[] K4 = new byte[] {'d'};
+ protected static final byte[] K5 = new byte[] {'e'};
+ protected static final byte[] K6 = new byte[] {'f'};
+ protected static final byte[] K9 = new byte[] {'i'};
+ protected static final byte[] K11 = new byte[] {'k'};
+ protected static final byte[] K12 = new byte[] {'l'};
+ protected static final byte[] KMAX = new byte[] {'~'};
+ protected static final byte[] KMAX2 = new byte[] {'z'};
+ protected static final byte[] KR = new byte[] { 'r' };
+ protected static final byte[] KP = new byte[] { 'p' };
+
+ private static List<KeyRange> getSplits(Connection conn, final Scan scan) throws SQLException {
+ TableRef tableRef = getTableRef(conn);
+ PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class);
+ final List<HRegionLocation> regions = pconn.getQueryServices().getAllTableRegions(
+ tableRef.getTable().getPhysicalName().getBytes());
+ PhoenixStatement statement = new PhoenixStatement(pconn);
+ StatementContext context = new StatementContext(statement, null, scan, new SequenceManager(statement));
+ DefaultParallelIteratorRegionSplitter splitter = new DefaultParallelIteratorRegionSplitter(context, tableRef,
+ HintNode.EMPTY_HINT_NODE) {
+ @Override
+ protected List<HRegionLocation> getAllRegions() throws SQLException {
+ return DefaultParallelIteratorRegionSplitter.filterRegions(regions, scan.getStartRow(),
+ scan.getStopRow());
+ }
+ };
+ List<KeyRange> keyRanges = splitter.getSplits();
+ Collections.sort(keyRanges, new Comparator<KeyRange>() {
+ @Override
+ public int compare(KeyRange o1, KeyRange o2) {
+ return Bytes.compareTo(o1.getLowerRange(), o2.getLowerRange());
+ }
+ });
+ return keyRanges;
+ }
+
+ // This test ensures that as we keep adding new records the splits gets updated
+ @Test
+ public void testGuidePostsLifeCycle() throws Exception {
+ byte[][] splits = new byte[][] { K3, K9, KR };
+ ensureTableCreated(getUrl(), STABLE_NAME, splits);
+ String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB;
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(url, props);
+ PreparedStatement stmt = conn.prepareStatement("ANALYZE STABLE");
+ stmt.execute();
+ Scan scan = new Scan();
+ List<KeyRange> keyRanges = getSplits(conn, scan);
+ assertEquals(4, keyRanges.size());
+ upsert(new byte[][] { KMIN, K4, K11 });
+ stmt = conn.prepareStatement("ANALYZE STABLE");
+ stmt.execute();
+ keyRanges = getSplits(conn, scan);
+ assertEquals(7, keyRanges.size());
+ upsert(new byte[][] { KMIN2, K5, K12 });
+ stmt = conn.prepareStatement("ANALYZE STABLE");
+ stmt.execute();
+ keyRanges = getSplits(conn, scan);
+ assertEquals(10, keyRanges.size());
+ upsert(new byte[][] { K1, K6, KP });
+ stmt = conn.prepareStatement("ANALYZE STABLE");
+ stmt.execute();
+ keyRanges = getSplits(conn, scan);
+ assertEquals(13, keyRanges.size());
+ conn.close();
+ }
+
+ protected void upsert( byte[][] val) throws Exception {
+ String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB;
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(url, props);
+ PreparedStatement stmt = conn.prepareStatement("upsert into " + STABLE_NAME + " VALUES (?, ?)");
+ stmt.setString(1, new String(val[0]));
+ stmt.setInt(2, 1);
+ stmt.execute();
+ stmt.setString(1, new String(val[1]));
+ stmt.setInt(2, 2);
+ stmt.execute();
+ stmt.setString(1, new String(val[2]));
+ stmt.setInt(2, 3);
+ stmt.execute();
+ conn.commit();
+ conn.close();
+ }
+
+ protected static TableRef getTableRef(Connection conn) throws SQLException {
+ PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class);
+ TableRef table = new TableRef(null, pconn.getMetaDataCache().getTable(
+ new PTableKey(pconn.getTenantId(), STABLE_NAME)), System.currentTimeMillis(), false);
+ return table;
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/15a54d55/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java
index b6372fd..5037f0b 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/HashJoinIT.java
@@ -81,8 +81,6 @@ public class HashJoinIT extends BaseHBaseManagedTimeIT {
@Shadower(classBeingShadowed = BaseHBaseManagedTimeIT.class)
public static void doSetup() throws Exception {
Map<String,String> props = Maps.newHashMapWithExpectedSize(3);
- // Don't split intra region so we can more easily know that the n-way parallelization is for the explain plan
- props.put(QueryServices.MAX_INTRA_REGION_PARALLELIZATION_ATTRIB, Integer.toString(1));
// Forces server cache to be used
props.put(QueryServices.INDEX_MUTATE_BATCH_SIZE_THRESHOLD_ATTRIB, Integer.toString(2));
// Must update config before starting server
http://git-wip-us.apache.org/repos/asf/phoenix/blob/15a54d55/phoenix-core/src/it/java/org/apache/phoenix/end2end/KeyOnlyIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/KeyOnlyIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/KeyOnlyIT.java
index 813f4b8..4b0d07f 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/KeyOnlyIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/KeyOnlyIT.java
@@ -23,12 +23,29 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
import java.util.Properties;
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.compile.SequenceManager;
+import org.apache.phoenix.compile.StatementContext;
+import org.apache.phoenix.iterate.DefaultParallelIteratorRegionSplitter;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixStatement;
+import org.apache.phoenix.parse.HintNode;
+import org.apache.phoenix.query.KeyRange;
+import org.apache.phoenix.schema.PTableKey;
+import org.apache.phoenix.schema.TableRef;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.PropertiesUtil;
import org.junit.Test;
@@ -45,6 +62,7 @@ public class KeyOnlyIT extends BaseClientManagedTimeIT {
props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts+5));
Connection conn5 = DriverManager.getConnection(getUrl(), props);
+ analyzeTable(conn5, KEYONLY_NAME);
String query = "SELECT i1, i2 FROM KEYONLY";
PreparedStatement statement = conn5.prepareStatement(query);
ResultSet rs = statement.executeQuery();
@@ -55,6 +73,9 @@ public class KeyOnlyIT extends BaseClientManagedTimeIT {
assertEquals(3, rs.getInt(1));
assertEquals(4, rs.getInt(2));
assertFalse(rs.next());
+ Scan scan = new Scan();
+ List<KeyRange> splits = getSplits(conn5, ts, scan);
+ assertEquals(3, splits.size());
conn5.close();
props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts+6));
@@ -76,6 +97,7 @@ public class KeyOnlyIT extends BaseClientManagedTimeIT {
props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts+8));
Connection conn8 = DriverManager.getConnection(getUrl(), props);
+ analyzeTable(conn8, KEYONLY_NAME);
query = "SELECT i1 FROM KEYONLY";
statement = conn8.prepareStatement(query);
rs = statement.executeQuery();
@@ -113,6 +135,7 @@ public class KeyOnlyIT extends BaseClientManagedTimeIT {
props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts+5));
Connection conn5 = DriverManager.getConnection(getUrl(), props);
+ analyzeTable(conn5, KEYONLY_NAME);
String query = "SELECT i1 FROM KEYONLY WHERE i1 < 2 or i1 = 3";
PreparedStatement statement = conn5.prepareStatement(query);
ResultSet rs = statement.executeQuery();
@@ -142,5 +165,41 @@ public class KeyOnlyIT extends BaseClientManagedTimeIT {
conn.commit();
conn.close();
}
-
+
+ private void analyzeTable(Connection conn, String tableName) throws IOException, SQLException {
+ String query = "ANALYZE " + tableName;
+ conn.createStatement().execute(query);
+ }
+
+ private static TableRef getTableRef(Connection conn, long ts) throws SQLException {
+ PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class);
+ TableRef table = new TableRef(null, pconn.getMetaDataCache().getTable(
+ new PTableKey(pconn.getTenantId(), KEYONLY_NAME)), ts, false);
+ return table;
+ }
+
+ private static List<KeyRange> getSplits(Connection conn, long ts, final Scan scan) throws SQLException {
+ TableRef tableRef = getTableRef(conn, ts);
+ PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class);
+ final List<HRegionLocation> regions = pconn.getQueryServices().getAllTableRegions(
+ tableRef.getTable().getPhysicalName().getBytes());
+ PhoenixStatement statement = new PhoenixStatement(pconn);
+ StatementContext context = new StatementContext(statement, null, scan, new SequenceManager(statement));
+ DefaultParallelIteratorRegionSplitter splitter = new DefaultParallelIteratorRegionSplitter(context, tableRef,
+ HintNode.EMPTY_HINT_NODE) {
+ @Override
+ protected List<HRegionLocation> getAllRegions() throws SQLException {
+ return DefaultParallelIteratorRegionSplitter.filterRegions(regions, scan.getStartRow(),
+ scan.getStopRow());
+ }
+ };
+ List<KeyRange> keyRanges = splitter.getSplits();
+ Collections.sort(keyRanges, new Comparator<KeyRange>() {
+ @Override
+ public int compare(KeyRange o1, KeyRange o2) {
+ return Bytes.compareTo(o1.getLowerRange(), o2.getLowerRange());
+ }
+ });
+ return keyRanges;
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/15a54d55/phoenix-core/src/it/java/org/apache/phoenix/end2end/MultiCfQueryExecIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MultiCfQueryExecIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MultiCfQueryExecIT.java
index b5e0ef4..ebf03d0 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MultiCfQueryExecIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MultiCfQueryExecIT.java
@@ -22,13 +22,30 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import java.io.IOException;
import java.math.BigDecimal;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
import java.util.Properties;
+import org.apache.hadoop.hbase.HRegionLocation;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.compile.SequenceManager;
+import org.apache.phoenix.compile.StatementContext;
+import org.apache.phoenix.iterate.DefaultParallelIteratorRegionSplitter;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixStatement;
+import org.apache.phoenix.parse.HintNode;
+import org.apache.phoenix.query.KeyRange;
+import org.apache.phoenix.schema.PTableKey;
+import org.apache.phoenix.schema.TableRef;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.PropertiesUtil;
import org.junit.Test;
@@ -74,7 +91,12 @@ public class MultiCfQueryExecIT extends BaseClientManagedTimeIT {
stmt.setLong(7, 22222);
stmt.execute();
}
-
+
+ private void analyzeTable(Connection conn, String tableName) throws IOException, SQLException {
+ String query = "ANALYZE " + tableName;
+ conn.createStatement().execute(query);
+ }
+
@Test
public void testConstantCount() throws Exception {
long ts = nextTimestamp();
@@ -84,6 +106,7 @@ public class MultiCfQueryExecIT extends BaseClientManagedTimeIT {
Connection conn = DriverManager.getConnection(url, props);
try {
initTableValues(ts);
+ analyzeTable(conn, "MULTI_CF");
PreparedStatement statement = conn.prepareStatement(query);
ResultSet rs = statement.executeQuery();
assertTrue(rs.next());
@@ -103,6 +126,7 @@ public class MultiCfQueryExecIT extends BaseClientManagedTimeIT {
Connection conn = DriverManager.getConnection(url, props);
try {
initTableValues(ts);
+ analyzeTable(conn, "MULTI_CF");
PreparedStatement statement = conn.prepareStatement(query);
ResultSet rs = statement.executeQuery();
assertTrue(rs.next());
@@ -123,6 +147,7 @@ public class MultiCfQueryExecIT extends BaseClientManagedTimeIT {
Connection conn = DriverManager.getConnection(url, props);
try {
initTableValues(ts);
+ analyzeTable(conn, "MULTI_CF");
PreparedStatement statement = conn.prepareStatement(query);
ResultSet rs = statement.executeQuery();
assertTrue(rs.next());
@@ -143,12 +168,24 @@ public class MultiCfQueryExecIT extends BaseClientManagedTimeIT {
Connection conn = DriverManager.getConnection(url, props);
try {
initTableValues(ts);
+ analyzeTable(conn, "MULTI_CF");
PreparedStatement statement = conn.prepareStatement(query);
ResultSet rs = statement.executeQuery();
assertTrue(rs.next());
assertEquals(2222, rs.getLong(1));
assertEquals(22222, rs.getLong(2));
assertFalse(rs.next());
+ Scan scan = new Scan();
+ // See if F has splits in it
+ scan.addFamily(Bytes.toBytes("E"));
+ List<KeyRange> splits = getSplits(conn, ts, scan);
+ assertEquals(3, splits.size());
+ scan = new Scan();
+ // See if G has splits in it
+ scan.addFamily(Bytes.toBytes("G"));
+ splits = getSplits(conn, ts, scan);
+ // We get splits from different CF
+ assertEquals(3, splits.size());
} finally {
conn.close();
}
@@ -163,6 +200,7 @@ public class MultiCfQueryExecIT extends BaseClientManagedTimeIT {
Connection conn = DriverManager.getConnection(url, props);
try {
initTableValues(ts);
+ analyzeTable(conn, "MULTI_CF");
PreparedStatement statement = conn.prepareStatement(query);
ResultSet rs = statement.executeQuery();
assertTrue(rs.next());
@@ -178,11 +216,11 @@ public class MultiCfQueryExecIT extends BaseClientManagedTimeIT {
public void testDefaultCFToDisambiguate() throws Exception {
long ts = nextTimestamp();
initTableValues(ts);
-
String ddl = "ALTER TABLE multi_cf ADD response_time BIGINT";
String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + (ts + 3);
Connection conn = DriverManager.getConnection(url);
conn.createStatement().execute(ddl);
+ analyzeTable(conn, "MULTI_CF");
conn.close();
String dml = "upsert into " +
@@ -195,7 +233,7 @@ public class MultiCfQueryExecIT extends BaseClientManagedTimeIT {
conn.createStatement().execute(dml);
conn.commit();
conn.close();
-
+ analyzeTable(conn, "MULTI_CF");
String query = "SELECT ID,RESPONSE_TIME from multi_cf where RESPONSE_TIME = 333";
url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + (ts + 5); // Run query at timestamp 5
conn = DriverManager.getConnection(url);
@@ -220,6 +258,7 @@ public class MultiCfQueryExecIT extends BaseClientManagedTimeIT {
Connection conn = DriverManager.getConnection(url, props);
try {
initTableValues(ts);
+ analyzeTable(conn, "MULTI_CF");
PreparedStatement statement = conn.prepareStatement(query);
ResultSet rs = statement.executeQuery();
assertTrue(rs.next());
@@ -230,5 +269,36 @@ public class MultiCfQueryExecIT extends BaseClientManagedTimeIT {
conn.close();
}
}
-
+
+ private static TableRef getTableRef(Connection conn, long ts) throws SQLException {
+ PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class);
+ TableRef table = new TableRef(null, pconn.getMetaDataCache().getTable(
+ new PTableKey(pconn.getTenantId(), "MULTI_CF")), ts, false);
+ return table;
+ }
+
+ private static List<KeyRange> getSplits(Connection conn, long ts, final Scan scan) throws SQLException {
+ TableRef tableRef = getTableRef(conn, ts);
+ PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class);
+ final List<HRegionLocation> regions = pconn.getQueryServices().getAllTableRegions(
+ tableRef.getTable().getPhysicalName().getBytes());
+ PhoenixStatement statement = new PhoenixStatement(pconn);
+ StatementContext context = new StatementContext(statement, null, scan, new SequenceManager(statement));
+ DefaultParallelIteratorRegionSplitter splitter = new DefaultParallelIteratorRegionSplitter(context, tableRef,
+ HintNode.EMPTY_HINT_NODE) {
+ @Override
+ protected List<HRegionLocation> getAllRegions() throws SQLException {
+ return DefaultParallelIteratorRegionSplitter.filterRegions(regions, scan.getStartRow(),
+ scan.getStopRow());
+ }
+ };
+ List<KeyRange> keyRanges = splitter.getSplits();
+ Collections.sort(keyRanges, new Comparator<KeyRange>() {
+ @Override
+ public int compare(KeyRange o1, KeyRange o2) {
+ return Bytes.compareTo(o1.getLowerRange(), o2.getLowerRange());
+ }
+ });
+ return keyRanges;
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/15a54d55/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java
index d449a26..9bf2b0c 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java
@@ -113,6 +113,10 @@ public class QueryDatabaseMetaDataIT extends BaseClientManagedTimeIT {
assertEquals(rs.getString("TABLE_NAME"),TYPE_SEQUENCE);
assertEquals(PTableType.SYSTEM.toString(), rs.getString("TABLE_TYPE"));
assertTrue(rs.next());
+ assertEquals(rs.getString("TABLE_SCHEM"),SYSTEM_CATALOG_SCHEMA);
+ assertEquals(rs.getString("TABLE_NAME"),PhoenixDatabaseMetaData.SYSTEM_STATS_TABLE);
+ assertEquals(PTableType.SYSTEM.toString(), rs.getString("TABLE_TYPE"));
+ assertTrue(rs.next());
assertEquals(rs.getString("TABLE_SCHEM"),null);
assertEquals(rs.getString("TABLE_NAME"),ATABLE_NAME);
assertEquals(PTableType.TABLE.toString(), rs.getString("TABLE_TYPE"));
http://git-wip-us.apache.org/repos/asf/phoenix/blob/15a54d55/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java
index 553f615..e6c2fc4 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryIT.java
@@ -38,11 +38,13 @@ import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.io.IOException;
import java.sql.Connection;
import java.sql.Date;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
+import java.sql.SQLException;
import java.sql.Time;
import java.sql.Timestamp;
import java.util.Arrays;
@@ -100,6 +102,11 @@ public class QueryIT extends BaseQueryIT {
stmt.setInt(3, -10);
stmt.execute();
upsertConn.close();
+ url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + (ts + 6);
+ props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ upsertConn = DriverManager.getConnection(url, props);
+ analyzeTable(upsertConn, "ATABLE");
+ upsertConn.close();
String query = "SELECT entity_id FROM aTable WHERE organization_id=? and a_integer >= ?";
props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 2)); // Execute at timestamp 2
@@ -208,6 +215,9 @@ public class QueryIT extends BaseQueryIT {
stmt.setString(2, ROW5);
stmt.setString(3, value);
stmt.execute(); // should commit too
+ Connection conn1 = DriverManager.getConnection(getUrl(), props);
+ analyzeTable(conn1, "ATABLE");
+ conn1.close();
upsertConn.close();
String query = "SELECT a_string, b_string FROM aTable WHERE organization_id=? and a_integer = 5";
@@ -252,6 +262,9 @@ public class QueryIT extends BaseQueryIT {
stmt.setString(2, ROW4);
stmt.setInt(3, 5);
stmt.execute(); // should commit too
+ Connection conn1 = DriverManager.getConnection(getUrl(), props);
+ analyzeTable(conn1, "ATABLE");
+ conn1.close();
upsertConn.close();
// Override value again, but should be ignored since it's past the SCN
@@ -387,7 +400,9 @@ public class QueryIT extends BaseQueryIT {
byte[] ts1 = PDataType.TIMESTAMP.toBytes(tsValue1);
stmt.setTimestamp(3, tsValue1);
stmt.execute();
-
+ Connection conn1 = DriverManager.getConnection(getUrl(), props);
+ analyzeTable(conn1, "ATABLE");
+ conn1.close();
updateStmt =
"upsert into " +
"ATABLE(" +
@@ -406,7 +421,10 @@ public class QueryIT extends BaseQueryIT {
stmt.setTime(4, new Time(tsValue2.getTime()));
stmt.execute();
upsertConn.close();
-
+ conn1 = DriverManager.getConnection(getUrl(), props);
+ analyzeTable(conn1, "ATABLE");
+ conn1.close();
+ analyzeTable(upsertConn, "ATABLE");
assertTrue(compare(CompareOp.GREATER, new ImmutableBytesWritable(ts2), new ImmutableBytesWritable(ts1)));
assertFalse(compare(CompareOp.GREATER, new ImmutableBytesWritable(ts1), new ImmutableBytesWritable(ts1)));
@@ -732,7 +750,7 @@ public class QueryIT extends BaseQueryIT {
assertTrue(rs.next());
assertEquals(A_VALUE, rs.getString(1));
assertEquals(E_VALUE, rs.getString(2));
- assertEquals(1, rs.getLong(3));
+ assertEquals(1, rs.getLong(3));
assertFalse(rs.next());
byte[] tableName = Bytes.toBytes(ATABLE_NAME);
@@ -799,6 +817,10 @@ public class QueryIT extends BaseQueryIT {
Connection conn = DriverManager.getConnection(getUrl(), props);
conn.setAutoCommit(true);
conn.createStatement().execute("UPSERT INTO atable(organization_id,entity_id,a_integer) VALUES('" + getOrganizationId() + "','" + ROW3 + "',NULL)");
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 6));
+ Connection conn1 = DriverManager.getConnection(getUrl(), props);
+ analyzeTable(conn1, "ATABLE");
+ conn1.close();
props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 5));
conn = DriverManager.getConnection(getUrl(), props);
try {
@@ -814,6 +836,10 @@ public class QueryIT extends BaseQueryIT {
conn = DriverManager.getConnection(getUrl(), props);
conn.setAutoCommit(true);
conn.createStatement().execute("UPSERT INTO atable(organization_id,entity_id,a_integer) SELECT organization_id, entity_id, null FROM atable");
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 6));
+ conn1 = DriverManager.getConnection(getUrl(), props);
+ analyzeTable(conn1, "ATABLE");
+ conn1.close();
props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 9));
conn = DriverManager.getConnection(getUrl(), props);
try {
@@ -827,4 +853,9 @@ public class QueryIT extends BaseQueryIT {
conn.close();
}
}
+
+ private void analyzeTable(Connection conn, String tableName) throws IOException, SQLException {
+ String query = "ANALYZE " + tableName;
+ conn.createStatement().execute(query);
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/15a54d55/phoenix-core/src/it/java/org/apache/phoenix/end2end/RegexpSplitFunctionIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/RegexpSplitFunctionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/RegexpSplitFunctionIT.java
index 73f1828..0bda438 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/RegexpSplitFunctionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/RegexpSplitFunctionIT.java
@@ -17,8 +17,11 @@
*/
package org.apache.phoenix.end2end;
-import org.junit.Test;
-import org.junit.experimental.categories.Category;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
import java.sql.Array;
import java.sql.Connection;
@@ -28,12 +31,8 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Types;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
@Category(HBaseManagedTimeTest.class)
public class RegexpSplitFunctionIT extends BaseHBaseManagedTimeIT {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/15a54d55/phoenix-core/src/it/java/org/apache/phoenix/end2end/RegexpSubstrFunctionIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/RegexpSubstrFunctionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/RegexpSubstrFunctionIT.java
index 1121207..be4646b 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/RegexpSubstrFunctionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/RegexpSubstrFunctionIT.java
@@ -17,19 +17,22 @@
*/
package org.apache.phoenix.end2end;
-import org.apache.phoenix.util.PhoenixRuntime;
-import org.apache.phoenix.util.PropertiesUtil;
+import static org.apache.phoenix.util.TestUtil.GROUPBYTEST_NAME;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+
import org.junit.Before;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-import java.sql.*;
-import java.util.Properties;
-
-import static org.apache.phoenix.util.TestUtil.GROUPBYTEST_NAME;
-import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
-import static org.junit.Assert.*;
-
@Category(HBaseManagedTimeTest.class)
public class RegexpSubstrFunctionIT extends BaseHBaseManagedTimeIT {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/15a54d55/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipRangeParallelIteratorRegionSplitterIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipRangeParallelIteratorRegionSplitterIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipRangeParallelIteratorRegionSplitterIT.java
index 93abfb9..d32441b 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipRangeParallelIteratorRegionSplitterIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SkipRangeParallelIteratorRegionSplitterIT.java
@@ -45,7 +45,6 @@ import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixStatement;
import org.apache.phoenix.parse.HintNode;
import org.apache.phoenix.query.KeyRange;
-import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.schema.ColumnRef;
import org.apache.phoenix.schema.PDataType;
import org.apache.phoenix.schema.PDatum;
@@ -81,12 +80,8 @@ public class SkipRangeParallelIteratorRegionSplitterIT extends BaseClientManaged
private static final String DDL = "CREATE TABLE " + TABLE_NAME + " (id char(3) NOT NULL PRIMARY KEY, \"value\" integer)";
private static final byte[] Ka1A = Bytes.toBytes("a1A");
private static final byte[] Ka1B = Bytes.toBytes("a1B");
- private static final byte[] Ka1C = Bytes.toBytes("a1C");
- private static final byte[] Ka1D = Bytes.toBytes("a1D");
private static final byte[] Ka1E = Bytes.toBytes("a1E");
- private static final byte[] Ka1F = Bytes.toBytes("a1F");
private static final byte[] Ka1G = Bytes.toBytes("a1G");
- private static final byte[] Ka1H = Bytes.toBytes("a1H");
private static final byte[] Ka1I = Bytes.toBytes("a1I");
private static final byte[] Ka2A = Bytes.toBytes("a2A");
@@ -109,11 +104,13 @@ public class SkipRangeParallelIteratorRegionSplitterIT extends BaseClientManaged
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
Connection conn = DriverManager.getConnection(url, props);
PhoenixConnection pconn = conn.unwrap(PhoenixConnection.class);
- TableRef tableRef = new TableRef(null,pconn.getMetaDataCache().getTable(new PTableKey(pconn.getTenantId(), TABLE_NAME)),ts, false);
- List<HRegionLocation> regions = pconn.getQueryServices().getAllTableRegions(tableRef.getTable().getPhysicalName().getBytes());
- conn.close();
initTableValues();
+ PreparedStatement stmt = conn.prepareStatement("ANALYZE "+TABLE_NAME);
+ stmt.execute();
+ conn.close();
+ TableRef tableRef = new TableRef(null,pconn.getMetaDataCache().getTable(new PTableKey(pconn.getTenantId(), TABLE_NAME)),ts, false);
+ List<HRegionLocation> regions = pconn.getQueryServices().getAllTableRegions(tableRef.getTable().getPhysicalName().getBytes());
List<KeyRange> ranges = getSplits(tableRef, scan, regions, scanRanges);
assertEquals("Unexpected number of splits: " + ranges.size(), expectedSplits.size(), ranges.size());
for (int i=0; i<expectedSplits.size(); i++) {
@@ -188,9 +185,7 @@ public class SkipRangeParallelIteratorRegionSplitterIT extends BaseClientManaged
}},
new int[] {1,1,1},
new KeyRange[] {
- getKeyRange(Ka1B, true, Ka1C, false),
- getKeyRange(Ka1C, true, Ka1D, false),
- getKeyRange(Ka1D, true, Ka1E, false),
+ getKeyRange(Ka1B, true, Ka1E, false)
}));
// Scan range spans third, split into 3 due to concurrency config.
testCases.addAll(
@@ -204,9 +199,7 @@ public class SkipRangeParallelIteratorRegionSplitterIT extends BaseClientManaged
}},
new int[] {1,1,1},
new KeyRange[] {
- getKeyRange(Ka1B, true, Ka1C, false),
- getKeyRange(Ka1C, true, Ka1D, false),
- getKeyRange(Ka1D, true, Ka1E, false),
+ getKeyRange(Ka1B, true, Ka1E, false),
}));
// Scan range spans 2 ranges, split into 4 due to concurrency config.
testCases.addAll(
@@ -220,10 +213,8 @@ public class SkipRangeParallelIteratorRegionSplitterIT extends BaseClientManaged
}},
new int[] {1,1,1},
new KeyRange[] {
- getKeyRange(Ka1E, true, Ka1F, false),
- getKeyRange(Ka1F, true, Ka1G, false),
- getKeyRange(Ka1G, true, Ka1H, false),
- getKeyRange(Ka1H, true, Ka1I, false),
+ getKeyRange(Ka1E, true, Ka1G, false),
+ getKeyRange(Ka1G, true, Ka1I, false),
}));
// Scan range spans more than 3 range, no split.
testCases.addAll(
@@ -326,12 +317,7 @@ public class SkipRangeParallelIteratorRegionSplitterIT extends BaseClientManaged
@BeforeClass
@Shadower(classBeingShadowed = BaseClientManagedTimeIT.class)
public static void doSetup() throws Exception {
- int targetQueryConcurrency = 3;
- int maxQueryConcurrency = 5;
Map<String,String> props = Maps.newHashMapWithExpectedSize(3);
- props.put(QueryServices.MAX_QUERY_CONCURRENCY_ATTRIB, Integer.toString(maxQueryConcurrency));
- props.put(QueryServices.TARGET_QUERY_CONCURRENCY_ATTRIB, Integer.toString(targetQueryConcurrency));
- props.put(QueryServices.MAX_INTRA_REGION_PARALLELIZATION_ATTRIB, Integer.toString(Integer.MAX_VALUE));
// Must update config before starting server
setUpTestDriver(getUrl(), new ReadOnlyProps(props.entrySet().iterator()));
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/15a54d55/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
new file mode 100644
index 0000000..3833e56
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsCollectorIT.java
@@ -0,0 +1,230 @@
+package org.apache.phoenix.end2end;
+
+import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL;
+import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_SEPARATOR;
+import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL_TERMINATOR;
+import static org.apache.phoenix.util.PhoenixRuntime.PHOENIX_TEST_DRIVER_URL_PARAM;
+import static org.apache.phoenix.util.TestUtil.LOCALHOST;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.sql.Array;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import com.google.common.collect.Maps;
+
+@Category(HBaseManagedTimeTest.class)
+public class StatsCollectorIT extends BaseHBaseManagedTimeIT {
+ private static String url;
+ private static HBaseTestingUtility util;
+ private static int frequency = 4000;
+
+ @BeforeClass
+ public static void doSetup() throws Exception {
+ Configuration conf = HBaseConfiguration.create();
+ setUpConfigForMiniCluster(conf);
+ conf.setInt("hbase.client.retries.number", 2);
+ conf.setInt("hbase.client.pause", 5000);
+ conf.setLong(QueryServices.INDEX_FAILURE_HANDLING_REBUILD_OVERLAP_TIME_ATTRIB, 0);
+ util = new HBaseTestingUtility(conf);
+ util.startMiniCluster();
+ String clientPort = util.getConfiguration().get(QueryServices.ZOOKEEPER_PORT_ATTRIB);
+ url = JDBC_PROTOCOL + JDBC_PROTOCOL_SEPARATOR + LOCALHOST + JDBC_PROTOCOL_SEPARATOR + clientPort
+ + JDBC_PROTOCOL_TERMINATOR + PHOENIX_TEST_DRIVER_URL_PARAM;
+ int histogramDepth = 60;
+ Map<String, String> props = Maps.newHashMapWithExpectedSize(3);
+ props.put(QueryServices.HISTOGRAM_BYTE_DEPTH_ATTRIB, Integer.toString(histogramDepth));
+ props.put(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB, Integer.toString(frequency));
+ driver = initAndRegisterDriver(url, new ReadOnlyProps(props.entrySet().iterator()));
+ }
+
+ @Test
+ public void testUpdateStatsForTheTable() throws Throwable {
+ Connection conn;
+ PreparedStatement stmt;
+ ResultSet rs;
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ // props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 10));
+ conn = DriverManager.getConnection(url, props);
+ conn.createStatement().execute(
+ "CREATE TABLE t ( k VARCHAR, a_string_array VARCHAR(100) ARRAY[4], b_string_array VARCHAR(100) ARRAY[4] \n"
+ + " CONSTRAINT pk PRIMARY KEY (k, b_string_array DESC)) \n");
+ String[] s;
+ Array array;
+ conn = upsertValues(props, "t");
+ // CAll the update statistics query here. If already major compaction has run this will not get executed.
+ stmt = conn.prepareStatement("ANALYZE T");
+ stmt.execute();
+ stmt = upsertStmt(conn, "t");
+ stmt.setString(1, "z");
+ s = new String[] { "xyz", "def", "ghi", "jkll", null, null, "xxx" };
+ array = conn.createArrayOf("VARCHAR", s);
+ stmt.setArray(2, array);
+ s = new String[] { "zya", "def", "ghi", "jkll", null, null, null, "xxx" };
+ array = conn.createArrayOf("VARCHAR", s);
+ stmt.setArray(3, array);
+ stmt.execute();
+ conn.close();
+ conn = DriverManager.getConnection(getUrl(), props);
+ // This analyze would not work
+ stmt = conn.prepareStatement("ANALYZE T");
+ stmt.execute();
+ rs = conn.createStatement().executeQuery("SELECT k FROM T");
+ assertTrue(rs.next());
+ conn.close();
+ }
+
+ @Test
+ public void testUpdateStatsWithMultipleTables() throws Throwable {
+ Connection conn;
+ PreparedStatement stmt;
+ ResultSet rs;
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ // props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 10));
+ conn = DriverManager.getConnection(url, props);
+ conn.createStatement().execute(
+ "CREATE TABLE x ( k VARCHAR, a_string_array VARCHAR(100) ARRAY[4], b_string_array VARCHAR(100) ARRAY[4] \n"
+ + " CONSTRAINT pk PRIMARY KEY (k, b_string_array DESC)) \n");
+ conn.createStatement().execute(
+ "CREATE TABLE z ( k VARCHAR, a_string_array VARCHAR(100) ARRAY[4], b_string_array VARCHAR(100) ARRAY[4] \n"
+ + " CONSTRAINT pk PRIMARY KEY (k, b_string_array DESC)) \n");
+ String[] s;
+ Array array;
+ conn = upsertValues(props, "x");
+ conn = upsertValues(props, "z");
+ // CAll the update statistics query here
+ stmt = conn.prepareStatement("ANALYZE X");
+ stmt.execute();
+ stmt = conn.prepareStatement("ANALYZE Z");
+ stmt.execute();
+ stmt = upsertStmt(conn, "x");
+ stmt.setString(1, "z");
+ s = new String[] { "xyz", "def", "ghi", "jkll", null, null, "xxx" };
+ array = conn.createArrayOf("VARCHAR", s);
+ stmt.setArray(2, array);
+ s = new String[] { "zya", "def", "ghi", "jkll", null, null, null, "xxx" };
+ array = conn.createArrayOf("VARCHAR", s);
+ stmt.setArray(3, array);
+ stmt.execute();
+ stmt = upsertStmt(conn, "z");
+ stmt.setString(1, "z");
+ s = new String[] { "xyz", "def", "ghi", "jkll", null, null, "xxx" };
+ array = conn.createArrayOf("VARCHAR", s);
+ stmt.setArray(2, array);
+ s = new String[] { "zya", "def", "ghi", "jkll", null, null, null, "xxx" };
+ array = conn.createArrayOf("VARCHAR", s);
+ stmt.setArray(3, array);
+ stmt.execute();
+ conn.close();
+ conn = DriverManager.getConnection(getUrl(), props);
+ // This analyze would not work
+ stmt = conn.prepareStatement("ANALYZE Z");
+ stmt.execute();
+ rs = conn.createStatement().executeQuery("SELECT k FROM Z");
+ assertTrue(rs.next());
+ conn.close();
+ }
+
+ private Connection upsertValues(Properties props, String tableName) throws SQLException, IOException,
+ InterruptedException {
+ Connection conn;
+ PreparedStatement stmt;
+ // props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(ts + 30));
+ conn = DriverManager.getConnection(url, props);
+ stmt = upsertStmt(conn, tableName);
+ stmt.setString(1, "a");
+ String[] s = new String[] { "abc", "def", "ghi", "jkll", null, null, "xxx" };
+ Array array = conn.createArrayOf("VARCHAR", s);
+ stmt.setArray(2, array);
+ s = new String[] { "abc", "def", "ghi", "jkll", null, null, null, "xxx" };
+ array = conn.createArrayOf("VARCHAR", s);
+ stmt.setArray(3, array);
+ stmt.execute();
+ conn.commit();
+ flush(tableName);
+ stmt = upsertStmt(conn, tableName);
+ stmt.setString(1, "b");
+ s = new String[] { "xyz", "def", "ghi", "jkll", null, null, "xxx" };
+ array = conn.createArrayOf("VARCHAR", s);
+ stmt.setArray(2, array);
+ s = new String[] { "zya", "def", "ghi", "jkll", null, null, null, "xxx" };
+ array = conn.createArrayOf("VARCHAR", s);
+ stmt.setArray(3, array);
+ stmt.execute();
+ conn.commit();
+ flush(tableName);
+ stmt = upsertStmt(conn, tableName);
+ stmt.setString(1, "c");
+ s = new String[] { "xyz", "def", "ghi", "jkll", null, null, "xxx" };
+ array = conn.createArrayOf("VARCHAR", s);
+ stmt.setArray(2, array);
+ s = new String[] { "zya", "def", "ghi", "jkll", null, null, null, "xxx" };
+ array = conn.createArrayOf("VARCHAR", s);
+ stmt.setArray(3, array);
+ stmt.execute();
+ conn.commit();
+ flush(tableName);
+ stmt = upsertStmt(conn, tableName);
+ stmt.setString(1, "d");
+ s = new String[] { "xyz", "def", "ghi", "jkll", null, null, "xxx" };
+ array = conn.createArrayOf("VARCHAR", s);
+ stmt.setArray(2, array);
+ s = new String[] { "zya", "def", "ghi", "jkll", null, null, null, "xxx" };
+ array = conn.createArrayOf("VARCHAR", s);
+ stmt.setArray(3, array);
+ stmt.execute();
+ conn.commit();
+ flush(tableName);
+ stmt = upsertStmt(conn, tableName);
+ stmt.setString(1, "b");
+ s = new String[] { "xyz", "def", "ghi", "jkll", null, null, "xxx" };
+ array = conn.createArrayOf("VARCHAR", s);
+ stmt.setArray(2, array);
+ s = new String[] { "zya", "def", "ghi", "jkll", null, null, null, "xxx" };
+ array = conn.createArrayOf("VARCHAR", s);
+ stmt.setArray(3, array);
+ stmt.execute();
+ conn.commit();
+ flush(tableName);
+ stmt = upsertStmt(conn, tableName);
+ stmt.setString(1, "e");
+ s = new String[] { "xyz", "def", "ghi", "jkll", null, null, "xxx" };
+ array = conn.createArrayOf("VARCHAR", s);
+ stmt.setArray(2, array);
+ s = new String[] { "zya", "def", "ghi", "jkll", null, null, null, "xxx" };
+ array = conn.createArrayOf("VARCHAR", s);
+ stmt.setArray(3, array);
+ stmt.execute();
+ conn.commit();
+ flush(tableName);
+ return conn;
+ }
+
+ private void flush(String tableName) throws IOException, InterruptedException {
+ util.getHBaseAdmin().flush(tableName.toUpperCase());
+ }
+
+ private PreparedStatement upsertStmt(Connection conn, String tableName) throws SQLException {
+ PreparedStatement stmt;
+ stmt = conn.prepareStatement("UPSERT INTO " + tableName + " VALUES(?,?,?)");
+ return stmt;
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/15a54d55/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsManagerIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsManagerIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsManagerIT.java
index f19f776..b13379b 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsManagerIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/StatsManagerIT.java
@@ -121,7 +121,7 @@ public class StatsManagerIT extends BaseParallelIteratorsRegionSplitterIT {
String url = getUrl() + ";" + PhoenixRuntime.CURRENT_SCN_ATTRIB + "=" + ts;
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
Connection conn = DriverManager.getConnection(url, props);
- TableRef table = getTableRef(conn,ts);
+ TableRef table = getTableRef(conn, ts);
int updateFreq = 5;
int maxAge = 10;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/15a54d55/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java
index 89a67b5..2d6c30b 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java
@@ -474,6 +474,8 @@ public class TenantSpecificTablesDDLIT extends BaseTenantSpecificTablesIT {
assertTrue(rs.next());
assertTableMetaData(rs, SYSTEM_CATALOG_SCHEMA, TYPE_SEQUENCE, SYSTEM);
assertTrue(rs.next());
+ assertTableMetaData(rs, SYSTEM_CATALOG_SCHEMA, PhoenixDatabaseMetaData.SYSTEM_STATS_TABLE, SYSTEM);
+ assertTrue(rs.next());
assertTableMetaData(rs, null, PARENT_TABLE_NAME, TABLE);
assertTrue(rs.next());
assertTableMetaData(rs, null, PARENT_TABLE_NAME_NO_TENANT_TYPE_ID, TABLE);
@@ -538,6 +540,8 @@ public class TenantSpecificTablesDDLIT extends BaseTenantSpecificTablesIT {
assertTrue(rs.next());
assertTableMetaData(rs, PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA, PhoenixDatabaseMetaData.TYPE_SEQUENCE, PTableType.SYSTEM);
assertTrue(rs.next());
+ assertTableMetaData(rs, SYSTEM_CATALOG_SCHEMA, PhoenixDatabaseMetaData.SYSTEM_STATS_TABLE, PTableType.SYSTEM);
+ assertTrue(rs.next());
assertTableMetaData(rs, null, PARENT_TABLE_NAME, PTableType.TABLE);
assertTrue(rs.next());
assertTableMetaData(rs, null, PARENT_TABLE_NAME_NO_TENANT_TYPE_ID, PTableType.TABLE);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/15a54d55/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDMLIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDMLIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDMLIT.java
index 107ca34..dba4264 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDMLIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDMLIT.java
@@ -23,6 +23,7 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
@@ -47,6 +48,7 @@ public class TenantSpecificTablesDMLIT extends BaseTenantSpecificTablesIT {
conn.createStatement().executeUpdate("upsert into " + TENANT_TABLE_NAME + " (id, tenant_col) values (2, 'Viva Las Vegas')");
conn.commit();
conn.close();
+ analyzeTable(conn, TENANT_TABLE_NAME);
conn = nextConnection(PHOENIX_JDBC_TENANT_SPECIFIC_URL);
ResultSet rs = conn.createStatement().executeQuery("select tenant_col from " + TENANT_TABLE_NAME + " where id = 1");
@@ -70,11 +72,11 @@ public class TenantSpecificTablesDMLIT extends BaseTenantSpecificTablesIT {
conn1.createStatement().executeUpdate("upsert into " + TENANT_TABLE_NAME + " values ('you','" + TENANT_TYPE_ID +"',2,'Viva Las Vegas')");
conn1.commit();
-
+ analyzeTable(conn1, TENANT_TABLE_NAME);
conn2.setAutoCommit(true);
conn2.createStatement().executeUpdate("upsert into " + TENANT_TABLE_NAME + " values ('them','" + TENANT_TYPE_ID + "',1,'Long Hair')");
conn2.createStatement().executeUpdate("upsert into " + TENANT_TABLE_NAME + " values ('us','" + TENANT_TYPE_ID + "',2,'Black Hat')");
-
+ analyzeTable(conn2, TENANT_TABLE_NAME);
conn2.close();
conn1.close();
@@ -96,6 +98,7 @@ public class TenantSpecificTablesDMLIT extends BaseTenantSpecificTablesIT {
conn2 = nextConnection(PHOENIX_JDBC_TENANT_SPECIFIC_URL2);
conn2.createStatement().executeUpdate("upsert into " + TENANT_TABLE_NAME + " select * from " + TENANT_TABLE_NAME );
conn2.commit();
+ analyzeTable(conn2, TENANT_TABLE_NAME);
conn2.close();
conn2 = nextConnection(PHOENIX_JDBC_TENANT_SPECIFIC_URL2);
@@ -112,6 +115,7 @@ public class TenantSpecificTablesDMLIT extends BaseTenantSpecificTablesIT {
conn2 = nextConnection(PHOENIX_JDBC_TENANT_SPECIFIC_URL2);
conn2.setAutoCommit(true);;
conn2.createStatement().executeUpdate("upsert into " + TENANT_TABLE_NAME + " select 'all', tenant_type_id, id, 'Big ' || tenant_col from " + TENANT_TABLE_NAME );
+ analyzeTable(conn2, TENANT_TABLE_NAME);
conn2.close();
conn2 = nextConnection(PHOENIX_JDBC_TENANT_SPECIFIC_URL2);
@@ -159,6 +163,7 @@ public class TenantSpecificTablesDMLIT extends BaseTenantSpecificTablesIT {
conn.createStatement().executeUpdate("upsert into " + TENANT_TABLE_NAME + " (id, tenant_col) values (1, 'Cheap Sunglasses')");
conn.createStatement().executeUpdate("upsert into " + TENANT_TABLE_NAME + " (id, tenant_col) values (2, 'Viva Las Vegas')");
conn.commit();
+ analyzeTable(conn, TENANT_TABLE_NAME);
conn.close();
conn = nextConnection(PHOENIX_JDBC_TENANT_SPECIFIC_URL);
@@ -185,6 +190,7 @@ public class TenantSpecificTablesDMLIT extends BaseTenantSpecificTablesIT {
conn.createStatement().executeUpdate("upsert into " + PARENT_TABLE_NAME + " (tenant_id, tenant_type_id, id, user) values ('AC/DC', 'abc', 1, 'Bon Scott')");
conn.createStatement().executeUpdate("upsert into " + PARENT_TABLE_NAME + " (tenant_id, tenant_type_id, id, user) values ('" + TENANT_ID + "', '" + TENANT_TYPE_ID + "', 1, 'Billy Gibbons')");
conn.createStatement().executeUpdate("upsert into " + PARENT_TABLE_NAME + " (tenant_id, tenant_type_id, id, user) values ('" + TENANT_ID + "', 'def', 1, 'Billy Gibbons')");
+ analyzeTable(conn, PARENT_TABLE_NAME);
conn.close();
conn = nextConnection(PHOENIX_JDBC_TENANT_SPECIFIC_URL);
@@ -216,6 +222,7 @@ public class TenantSpecificTablesDMLIT extends BaseTenantSpecificTablesIT {
conn.createStatement().executeUpdate("upsert into " + PARENT_TABLE_NAME + " (tenant_id, tenant_type_id, id, user) values ('AC/DC', 'abc', 1, 'Bon Scott')");
conn.createStatement().executeUpdate("upsert into " + PARENT_TABLE_NAME + " (tenant_id, tenant_type_id, id, user) values ('" + TENANT_ID + "', '" + TENANT_TYPE_ID + "', 1, 'Billy Gibbons')");
conn.createStatement().executeUpdate("upsert into " + PARENT_TABLE_NAME + " (tenant_id, tenant_type_id, id, user) values ('" + TENANT_ID + "', 'def', 1, 'Billy Gibbons')");
+ analyzeTable(conn, PARENT_TABLE_NAME);
conn.close();
conn = nextConnection(PHOENIX_JDBC_TENANT_SPECIFIC_URL);
@@ -253,6 +260,7 @@ public class TenantSpecificTablesDMLIT extends BaseTenantSpecificTablesIT {
conn.createStatement().executeUpdate("upsert into " + PARENT_TABLE_NAME_NO_TENANT_TYPE_ID + " (tenant_id, id, user) values ('AC/DC', 1, 'Bon Scott')");
conn.createStatement().executeUpdate("upsert into " + PARENT_TABLE_NAME_NO_TENANT_TYPE_ID + " (tenant_id, id, user) values ('" + TENANT_ID + "', 1, 'Billy Gibbons')");
conn.createStatement().executeUpdate("upsert into " + PARENT_TABLE_NAME_NO_TENANT_TYPE_ID + " (tenant_id, id, user) values ('" + TENANT_ID + "', 2, 'Billy Gibbons')");
+ analyzeTable(conn, PARENT_TABLE_NAME_NO_TENANT_TYPE_ID);
conn.close();
conn = nextConnection(PHOENIX_JDBC_TENANT_SPECIFIC_URL);
@@ -289,7 +297,7 @@ public class TenantSpecificTablesDMLIT extends BaseTenantSpecificTablesIT {
conn.createStatement().executeUpdate("upsert into " + PARENT_TABLE_NAME + " (tenant_id, tenant_type_id, id, user) values ('AC/DC', 'abc', 1, 'Bon Scott')");
conn.createStatement().executeUpdate("upsert into " + PARENT_TABLE_NAME + " (tenant_id, tenant_type_id, id, user) values ('" + TENANT_ID + "', '" + TENANT_TYPE_ID + "', 1, 'Billy Gibbons')");
conn.createStatement().executeUpdate("upsert into " + PARENT_TABLE_NAME + " (tenant_id, tenant_type_id, id, user) values ('" + TENANT_ID + "', 'def', 1, 'Billy Gibbons')");
-
+ analyzeTable(conn, PARENT_TABLE_NAME);
conn.close();
conn = nextConnection(PHOENIX_JDBC_TENANT_SPECIFIC_URL);
@@ -320,7 +328,7 @@ public class TenantSpecificTablesDMLIT extends BaseTenantSpecificTablesIT {
conn.createStatement().executeUpdate("upsert into " + PARENT_TABLE_NAME_NO_TENANT_TYPE_ID + " (tenant_id, id, user) values ('AC/DC', 1, 'Bon Scott')");
conn.createStatement().executeUpdate("upsert into " + PARENT_TABLE_NAME_NO_TENANT_TYPE_ID + " (tenant_id, id, user) values ('" + TENANT_ID + "', 1, 'Billy Gibbons')");
conn.createStatement().executeUpdate("upsert into " + PARENT_TABLE_NAME_NO_TENANT_TYPE_ID + " (tenant_id, id, user) values ('" + TENANT_ID + "', 2, 'Billy Gibbons')");
-
+ analyzeTable(conn, PARENT_TABLE_NAME_NO_TENANT_TYPE_ID);
conn.close();
conn = nextConnection(PHOENIX_JDBC_TENANT_SPECIFIC_URL);
@@ -350,6 +358,7 @@ public class TenantSpecificTablesDMLIT extends BaseTenantSpecificTablesIT {
conn.createStatement().executeUpdate("upsert into " + PARENT_TABLE_NAME + " (tenant_id, tenant_type_id, id, user) values ('AC/DC', 'aaa', 1, 'Bon Scott')");
conn.createStatement().executeUpdate("upsert into " + PARENT_TABLE_NAME + " (tenant_id, tenant_type_id, id, user) values ('" + TENANT_ID + "', '" + TENANT_TYPE_ID + "', 1, 'Billy Gibbons')");
conn.createStatement().executeUpdate("upsert into " + PARENT_TABLE_NAME + " (tenant_id, tenant_type_id, id, user) values ('" + TENANT_ID + "', 'def', 2, 'Billy Gibbons')");
+ analyzeTable(conn, PARENT_TABLE_NAME);
conn.close();
conn = nextConnection(PHOENIX_JDBC_TENANT_SPECIFIC_URL);
@@ -384,6 +393,7 @@ public class TenantSpecificTablesDMLIT extends BaseTenantSpecificTablesIT {
conn.createStatement().executeUpdate("upsert into " + PARENT_TABLE_NAME + " (tenant_id, tenant_type_id, id, user) values ('AC/DC', 'aaa', 1, 'Bon Scott')");
conn.createStatement().executeUpdate("upsert into " + PARENT_TABLE_NAME + " (tenant_id, tenant_type_id, id, user) values ('" + TENANT_ID + "', '" + TENANT_TYPE_ID + "', 1, 'Billy Gibbons')");
conn.createStatement().executeUpdate("upsert into " + PARENT_TABLE_NAME + " (tenant_id, tenant_type_id, id, user) values ('" + TENANT_ID + "', 'def', 2, 'Billy Gibbons')");
+ analyzeTable(conn, PARENT_TABLE_NAME);
conn.close();
conn = nextConnection(PHOENIX_JDBC_TENANT_SPECIFIC_URL);
@@ -432,6 +442,7 @@ public class TenantSpecificTablesDMLIT extends BaseTenantSpecificTablesIT {
conn = nextConnection(PHOENIX_JDBC_TENANT_SPECIFIC_URL);
conn.setAutoCommit(true);
conn.createStatement().executeUpdate("upsert into " + PARENT_TABLE_NAME + " (tenant_type_id, id, user) values ('" + TENANT_TYPE_ID + "', 1, 'Billy Gibbons')");
+ analyzeTable(conn, PARENT_TABLE_NAME);
conn.close();
conn = nextConnection(PHOENIX_JDBC_TENANT_SPECIFIC_URL);
@@ -460,6 +471,10 @@ public class TenantSpecificTablesDMLIT extends BaseTenantSpecificTablesIT {
}
}
+ private void analyzeTable(Connection conn, String tableName) throws IOException, SQLException {
+ String query = "ANALYZE " + tableName;
+ conn.createStatement().execute(query);
+ }
@Test
public void testUpsertValuesUsingViewWithNoWhereClause() throws Exception {
Connection conn = nextConnection(PHOENIX_JDBC_TENANT_SPECIFIC_URL);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/15a54d55/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java
index 5b3fb74..75271d4 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/MutableIndexIT.java
@@ -57,7 +57,6 @@ public class MutableIndexIT extends BaseMutableIndexIT {
public static void doSetup() throws Exception {
Map<String,String> props = Maps.newHashMapWithExpectedSize(1);
// Don't split intra region so we can more easily know that the n-way parallelization is for the explain plan
- props.put(QueryServices.MAX_INTRA_REGION_PARALLELIZATION_ATTRIB, Integer.toString(1));
// Forces server cache to be used
props.put(QueryServices.INDEX_MUTATE_BATCH_SIZE_THRESHOLD_ATTRIB, Integer.toString(2));
// Must update config before starting server