You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by vj...@apache.org on 2024/02/10 17:56:19 UTC
(phoenix) branch 5.1 updated: PHOENIX-7106 Data Integrity issues due to invalid rowkeys returned by various coprocessors (#1736) (#1820)
This is an automated email from the ASF dual-hosted git repository.
vjasani pushed a commit to branch 5.1
in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/5.1 by this push:
new b311841231 PHOENIX-7106 Data Integrity issues due to invalid rowkeys returned by various coprocessors (#1736) (#1820)
b311841231 is described below
commit b311841231125e6acbef2275a701518bfc36e653
Author: Viraj Jasani <vj...@apache.org>
AuthorDate: Sat Feb 10 08:56:14 2024 -0900
PHOENIX-7106 Data Integrity issues due to invalid rowkeys returned by various coprocessors (#1736) (#1820)
---
.../end2end/AggregateQueryWithRegionMoves2IT.java | 135 ++
.../end2end/AggregateQueryWithRegionMovesIT.java | 279 +++
.../end2end/AggregateWithRegionMoves2IT.java | 123 ++
.../end2end/AggregateWithRegionMovesIT.java | 499 +++++
.../phoenix/end2end/BackwardCompatibilityIT.java | 90 +
.../end2end/BackwardCompatibilityTestUtil.java | 12 +
.../end2end/BaseAggregateWithRegionMoves2IT.java | 633 +++++++
.../end2end/BaseAggregateWithRegionMoves3IT.java | 123 ++
.../end2end/BaseAggregateWithRegionMoves4IT.java | 136 ++
.../end2end/BaseAggregateWithRegionMovesIT.java | 998 ++++++++++
.../end2end/BaseOrderByWithRegionMovesIT.java | 1406 ++++++++++++++
.../end2end/BaseQueryWithRegionMovesIT.java | 165 ++
.../java/org/apache/phoenix/end2end/GroupByIT.java | 51 +
.../phoenix/end2end/OrderByWithRegionMovesIT.java} | 21 +-
.../end2end/OrderByWithServerMemoryLimitIT.java | 10 +-
.../ParallelStatsDisabledWithRegionMovesIT.java | 310 ++++
.../end2end/ScanUncommittedWithRegionMovesIT.java | 465 +++++
.../org/apache/phoenix/end2end/ServerPagingIT.java | 9 +-
.../end2end/ServerPagingWithRegionMovesIT.java | 840 +++++++++
.../org/apache/phoenix/end2end/SortOrderIT.java | 60 +-
.../end2end/SortOrderWithRegionMoves2IT.java | 200 ++
.../end2end/SortOrderWithRegionMovesIT.java | 209 +++
.../end2end/UpsertSelectWithRegionMovesIT.java | 1927 ++++++++++++++++++++
.../end2end/index/BaseIndexWithRegionMovesIT.java | 1847 +++++++++++++++++++
.../index/GlobalIndexCheckerWithRegionMovesIT.java | 1627 +++++++++++++++++
.../phoenix/iterate/PhoenixQueryTimeoutIT.java | 115 +-
.../resources/gold_files/gold_query_offset.txt} | 36 +-
.../gold_files/gold_query_orderby_nonpk.txt} | 43 +-
.../gold_files/gold_query_ordered_groupby.txt} | 39 +-
.../gold_files/gold_query_unorder_groupby.txt} | 28 +-
.../src/it/resources/sql_files/create_offset.sql | 48 +
.../resources/sql_files/create_orderby_nonpk.sql | 48 +
.../resources/sql_files/create_ordered_groupby.sql | 48 +
.../resources/sql_files/create_unorder_groupby.sql | 44 +
.../resources/sql_files/query_offset.sql} | 21 +-
.../resources/sql_files/query_orderby_nonpk.sql} | 21 +-
.../resources/sql_files/query_ordered_groupby.sql} | 21 +-
.../resources/sql_files/query_unorder_groupby.sql} | 21 +-
.../cache/aggcache/SpillableGroupByCache.java | 66 +-
.../coprocessor/BaseScannerRegionObserver.java | 74 +-
.../apache/phoenix/coprocessor/GroupByCache.java | 1 +
.../GroupedAggregateRegionObserver.java | 501 ++++-
.../coprocessor/IndexRebuildRegionScanner.java | 56 +-
.../coprocessor/IndexRepairRegionScanner.java | 27 +-
.../phoenix/coprocessor/IndexerRegionScanner.java | 32 +-
.../phoenix/coprocessor/PagedRegionScanner.java | 94 +-
.../coprocessor/UncoveredIndexRegionScanner.java | 76 +-
.../UngroupedAggregateRegionObserver.java | 50 +-
.../UngroupedAggregateRegionScanner.java | 13 +-
.../ResultSetOutOfScanRangeException.java} | 33 +-
.../apache/phoenix/exception/SQLExceptionCode.java | 1 +
.../org/apache/phoenix/filter/PagedFilter.java | 24 +-
.../apache/phoenix/index/GlobalIndexChecker.java | 46 +-
.../phoenix/iterate/BaseResultIterators.java | 5 +-
.../iterate/NonAggregateRegionScannerFactory.java | 176 +-
.../phoenix/iterate/OffsetResultIterator.java | 47 +-
.../phoenix/iterate/OrderedResultIterator.java | 215 ++-
.../phoenix/iterate/ScanningResultIterator.java | 31 +
.../ScanningResultPostDummyResultCaller.java | 30 +
.../ScanningResultPostValidResultCaller.java | 30 +
.../phoenix/iterate/TableResultIterator.java | 36 +-
.../org/apache/phoenix/query/QueryConstants.java | 3 +-
.../org/apache/phoenix/query/QueryServices.java | 29 +
.../java/org/apache/phoenix/util/ByteUtil.java | 126 +-
.../java/org/apache/phoenix/util/QueryUtil.java | 17 +-
.../java/org/apache/phoenix/util/ScanUtil.java | 126 +-
.../java/org/apache/phoenix/util/ServerUtil.java | 16 +
.../java/org/apache/phoenix/util/TupleUtil.java | 44 +-
.../phoenix/iterate/OrderedResultIteratorTest.java | 3 +
.../java/org/apache/phoenix/query/BaseTest.java | 3 +-
70 files changed, 14285 insertions(+), 454 deletions(-)
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AggregateQueryWithRegionMoves2IT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AggregateQueryWithRegionMoves2IT.java
new file mode 100644
index 0000000000..4cab6d27c4
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AggregateQueryWithRegionMoves2IT.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.phoenix.coprocessor.BaseScannerRegionObserver;
+import org.apache.phoenix.iterate.ScanningResultPostDummyResultCaller;
+import org.apache.phoenix.iterate.ScanningResultPostValidResultCaller;
+import org.apache.phoenix.query.BaseTest;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.experimental.categories.Category;
+import org.junit.runners.Parameterized;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+@Category(NeedsOwnMiniClusterTest.class)
+public class AggregateQueryWithRegionMoves2IT extends AggregateQueryWithRegionMovesIT {
+
+ private static final Logger LOGGER =
+ LoggerFactory.getLogger(AggregateQueryWithRegionMoves2IT.class);
+
+ private static int countOfValidResults = 0;
+
+ @Parameterized.Parameters(name = "AggregateQueryWithRegionMoves2IT_{index}")
+ public static synchronized Collection<Object> data() {
+ return BaseQueryWithRegionMovesIT.allIndexes();
+ }
+
+ public AggregateQueryWithRegionMoves2IT(String indexDDL, boolean columnEncoded,
+ boolean keepDeletedCells) {
+ super(indexDDL, columnEncoded, keepDeletedCells);
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ hasTestStarted = true;
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ countOfDummyResults = 0;
+ countOfValidResults = 0;
+ TABLE_NAMES.clear();
+ hasTestStarted = false;
+ }
+
+ private static class TestScanningResultPostDummyResultCaller extends
+ ScanningResultPostDummyResultCaller {
+
+ @Override
+ public void postDummyProcess() {
+ if (hasTestStarted && (countOfDummyResults++ % 3) == 0 &&
+ (countOfDummyResults < 17 ||
+ countOfDummyResults > 28 && countOfDummyResults < 40)) {
+ LOGGER.info("Moving regions of tables {}. current count of dummy results: {}",
+ TABLE_NAMES, countOfDummyResults);
+ TABLE_NAMES.forEach(table -> {
+ try {
+ moveRegionsOfTable(table);
+ } catch (Exception e) {
+ LOGGER.error("Unable to move regions of table: {}", table);
+ }
+ });
+ }
+ }
+ }
+
+ private static class TestScanningResultPostValidResultCaller extends
+ ScanningResultPostValidResultCaller {
+
+ @Override
+ public void postValidRowProcess() {
+ if (hasTestStarted &&
+ (countOfValidResults < 17 ||
+ countOfValidResults > 28 && countOfValidResults < 40)) {
+ LOGGER.info("Moving regions of tables {}. current count of valid results: {}",
+ TABLE_NAMES, countOfValidResults);
+ countOfValidResults++;
+ TABLE_NAMES.forEach(table -> {
+ try {
+ moveRegionsOfTable(table);
+ } catch (Exception e) {
+ LOGGER.error("Unable to move regions of table: {}", table);
+ }
+ });
+ }
+ }
+ }
+
+ @BeforeClass
+ public static synchronized void doSetup() throws Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put(BaseScannerRegionObserver.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY,
+ Integer.toString(60 * 60)); // An hour
+ props.put(QueryServices.USE_STATS_FOR_PARALLELIZATION, Boolean.toString(false));
+ props.put(QueryServices.PHOENIX_SERVER_PAGE_SIZE_MS, Long.toString(0));
+ props.put(QueryServices.TESTS_MINI_CLUSTER_NUM_REGION_SERVERS, String.valueOf(2));
+ props.put(HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY, String.valueOf(1));
+ props.put(QueryServices.PHOENIX_POST_DUMMY_PROCESS,
+ TestScanningResultPostDummyResultCaller.class.getName());
+ props.put(QueryServices.PHOENIX_POST_VALID_PROCESS,
+ TestScanningResultPostValidResultCaller.class.getName());
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ }
+
+ @AfterClass
+ public static synchronized void freeResources() throws Exception {
+ BaseTest.freeResourcesIfBeyondThreshold();
+ }
+
+}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AggregateQueryWithRegionMovesIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AggregateQueryWithRegionMovesIT.java
new file mode 100644
index 0000000000..2382d82849
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AggregateQueryWithRegionMovesIT.java
@@ -0,0 +1,279 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.ClusterConnection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.RegionLocator;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.util.ByteUtil;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runners.Parameterized.Parameters;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.util.Collection;
+import java.util.Properties;
+
+import static org.apache.phoenix.util.TestUtil.A_VALUE;
+import static org.apache.phoenix.util.TestUtil.B_VALUE;
+import static org.apache.phoenix.util.TestUtil.C_VALUE;
+import static org.apache.phoenix.util.TestUtil.E_VALUE;
+import static org.apache.phoenix.util.TestUtil.ROW4;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+@Category(NeedsOwnMiniClusterTest.class)
+public class AggregateQueryWithRegionMovesIT extends BaseQueryWithRegionMovesIT {
+
+ @Parameters(name = "AggregateQueryWithRegionMovesIT_{index}")
+ public static synchronized Collection<Object> data() {
+ return BaseQueryWithRegionMovesIT.allIndexes();
+ }
+
+ public AggregateQueryWithRegionMovesIT(String indexDDL, boolean columnEncoded,
+ boolean keepDeletedCells) {
+ super(indexDDL, columnEncoded, keepDeletedCells);
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ hasTestStarted = true;
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ countOfDummyResults = 0;
+ TABLE_NAMES.clear();
+ hasTestStarted = false;
+ }
+
+ @Test
+ public void testGroupByPlusOne() throws Exception {
+ String query = "SELECT a_integer+1 FROM " + tableName +
+ " WHERE organization_id=? and a_integer = 5 GROUP BY a_integer+1";
+ TABLE_NAMES.add(tableName);
+ TABLE_NAMES.add(indexName);
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ PreparedStatement statement = conn.prepareStatement(query);
+ statement.setString(1, tenantId);
+ ResultSet rs = statement.executeQuery();
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ moveRegionsOfTable(indexName);
+ assertEquals(6, rs.getInt(1));
+ assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testSplitWithCachedMeta() throws Exception {
+ TABLE_NAMES.add(tableName);
+ TABLE_NAMES.add(indexName);
+ // Tests that you don't get an ambiguous column exception when using the same alias as the column name
+ String query = "SELECT a_string, b_string, count(1) FROM " + tableName +
+ " WHERE organization_id=? and entity_id<=? GROUP BY a_string,b_string";
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ Admin admin = null;
+ try {
+ PreparedStatement statement = conn.prepareStatement(query);
+ statement.setString(1, tenantId);
+ statement.setString(2, ROW4);
+ ResultSet rs = statement.executeQuery();
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ moveRegionsOfTable(indexName);
+ assertEquals(A_VALUE, rs.getString(1));
+ assertEquals(B_VALUE, rs.getString(2));
+ assertEquals(2, rs.getLong(3));
+ assertTrue(rs.next());
+ assertEquals(A_VALUE, rs.getString(1));
+ assertEquals(C_VALUE, rs.getString(2));
+ assertEquals(1, rs.getLong(3));
+ moveRegionsOfTable(tableName);
+ moveRegionsOfTable(indexName);
+ assertTrue(rs.next());
+ assertEquals(A_VALUE, rs.getString(1));
+ assertEquals(E_VALUE, rs.getString(2));
+ assertEquals(1, rs.getLong(3));
+ assertFalse(rs.next());
+
+ TableName tn = TableName.valueOf(tableName);
+ admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin();
+ Configuration configuration =
+ conn.unwrap(PhoenixConnection.class).getQueryServices().getConfiguration();
+ org.apache.hadoop.hbase.client.Connection hbaseConn =
+ ConnectionFactory.createConnection(configuration);
+ ((ClusterConnection) hbaseConn).clearRegionCache(TableName.valueOf(tableName));
+ RegionLocator regionLocator = hbaseConn.getRegionLocator(TableName.valueOf(tableName));
+ int nRegions = regionLocator.getAllRegionLocations().size();
+ admin.split(tn, ByteUtil.concat(Bytes.toBytes(tenantId), Bytes.toBytes("00A3")));
+ int retryCount = 0;
+ do {
+ Thread.sleep(2000);
+ retryCount++;
+ //htable.clearRegionCache();
+ } while (retryCount < 10 && regionLocator.getAllRegionLocations().size() == nRegions);
+ assertNotEquals(nRegions, regionLocator.getAllRegionLocations().size());
+
+ statement.setString(1, tenantId);
+ rs = statement.executeQuery();
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ moveRegionsOfTable(indexName);
+ assertEquals(A_VALUE, rs.getString(1));
+ assertEquals(B_VALUE, rs.getString(2));
+ assertEquals(2, rs.getLong(3));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ moveRegionsOfTable(indexName);
+ assertEquals(A_VALUE, rs.getString(1));
+ assertEquals(C_VALUE, rs.getString(2));
+ assertEquals(1, rs.getLong(3));
+ assertTrue(rs.next());
+ assertEquals(A_VALUE, rs.getString(1));
+ assertEquals(E_VALUE, rs.getString(2));
+ assertEquals(1, rs.getLong(3));
+ assertFalse(rs.next());
+ } finally {
+ if (admin != null) {
+ admin.close();
+ }
+ conn.close();
+ }
+ }
+
+ @Test
+ public void testCountIsNull() throws Exception {
+ TABLE_NAMES.add(tableName);
+ TABLE_NAMES.add(indexName);
+ String query = "SELECT count(1) FROM " + tableName + " WHERE X_DECIMAL is null";
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ PreparedStatement statement = conn.prepareStatement(query);
+ ResultSet rs = statement.executeQuery();
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ moveRegionsOfTable(indexName);
+ assertEquals(6, rs.getLong(1));
+ assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testCountWithNoScanRanges() throws Exception {
+ TABLE_NAMES.add(tableName);
+ TABLE_NAMES.add(indexName);
+ String query = "SELECT count(1) FROM " + tableName +
+ " WHERE organization_id = 'not_existing_organization_id'";
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ PreparedStatement statement = conn.prepareStatement(query);
+ ResultSet rs = statement.executeQuery();
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ moveRegionsOfTable(indexName);
+ assertEquals(0, rs.getLong(1));
+ assertFalse(rs.next());
+ query = "SELECT count(1) FROM " + tableName +
+ " WHERE organization_id = 'not_existing_organization_id' having count(*)>0";
+ rs = conn.prepareStatement(query).executeQuery();
+ assertFalse(rs.next());
+ query = "SELECT count(1) FROM " + tableName +
+ " WHERE organization_id = 'not_existing_organization_id' limit 1 offset 1";
+ rs = conn.prepareStatement(query).executeQuery();
+ assertFalse(rs.next());
+ query = "SELECT count(1),123 FROM " + tableName +
+ " WHERE organization_id = 'not_existing_organization_id'";
+ rs = conn.prepareStatement(query).executeQuery();
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ moveRegionsOfTable(indexName);
+ assertEquals(0, rs.getLong(1));
+ assertEquals("123", rs.getString(2));
+ assertFalse(rs.next());
+ query = "SELECT count(1),sum(x_decimal) FROM " + tableName +
+ " WHERE organization_id = 'not_existing_organization_id'";
+ rs = conn.prepareStatement(query).executeQuery();
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ moveRegionsOfTable(indexName);
+ assertEquals(0, rs.getLong(1));
+ assertNull(rs.getBigDecimal(2));
+ assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testCountIsNotNull() throws Exception {
+ TABLE_NAMES.add(tableName);
+ TABLE_NAMES.add(indexName);
+ String query = "SELECT count(1) FROM " + tableName + " WHERE X_DECIMAL is not null";
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ PreparedStatement statement = conn.prepareStatement(query);
+ ResultSet rs = statement.executeQuery();
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ moveRegionsOfTable(indexName);
+ assertEquals(3, rs.getLong(1));
+ assertFalse(rs.next());
+ }
+ }
+
+ /**
+ * Test to repro Null Pointer Exception
+ */
+ @Test
+ public void testInFilterOnKey() throws Exception {
+ TABLE_NAMES.add(tableName);
+ TABLE_NAMES.add(indexName);
+ String query =
+ "SELECT count(entity_id) FROM " + tableName + " WHERE organization_id IN (?,?)";
+ String url = getUrl();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(url, props)) {
+ PreparedStatement statement = conn.prepareStatement(query);
+ statement.setString(1, tenantId);
+ statement.setString(2, tenantId);
+ ResultSet rs = statement.executeQuery();
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ moveRegionsOfTable(indexName);
+ assertEquals(9, rs.getInt(1));
+ assertFalse(rs.next());
+ }
+ }
+
+}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AggregateWithRegionMoves2IT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AggregateWithRegionMoves2IT.java
new file mode 100644
index 0000000000..5f0966b9d3
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AggregateWithRegionMoves2IT.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.phoenix.coprocessor.BaseScannerRegionObserver;
+import org.apache.phoenix.iterate.ScanningResultPostDummyResultCaller;
+import org.apache.phoenix.iterate.ScanningResultPostValidResultCaller;
+import org.apache.phoenix.query.BaseTest;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.experimental.categories.Category;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+
+@Category(NeedsOwnMiniClusterTest.class)
+public class AggregateWithRegionMoves2IT extends AggregateWithRegionMovesIT {
+
+ private static final Logger LOGGER =
+ LoggerFactory.getLogger(AggregateWithRegionMoves2IT.class);
+
+ private static int countOfValidResults = 0;
+
+ @Before
+ public void setUp() throws Exception {
+ hasTestStarted = true;
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ countOfDummyResults = 0;
+ countOfValidResults = 0;
+ TABLE_NAMES.clear();
+ hasTestStarted = false;
+ }
+
+ private static class TestScanningResultPostDummyResultCaller extends
+ ScanningResultPostDummyResultCaller {
+
+ @Override
+ public void postDummyProcess() {
+ if (hasTestStarted && (countOfDummyResults++ % 3) == 0 &&
+ (countOfDummyResults < 17 ||
+ countOfDummyResults > 28 && countOfDummyResults < 40)) {
+ LOGGER.info("Moving regions of tables {}. current count of dummy results: {}",
+ TABLE_NAMES, countOfDummyResults);
+ TABLE_NAMES.forEach(table -> {
+ try {
+ moveRegionsOfTable(table);
+ } catch (Exception e) {
+ LOGGER.error("Unable to move regions of table: {}", table);
+ }
+ });
+ }
+ }
+ }
+
+ private static class TestScanningResultPostValidResultCaller extends
+ ScanningResultPostValidResultCaller {
+
+ @Override
+ public void postValidRowProcess() {
+ if (hasTestStarted &&
+ (countOfValidResults < 17 ||
+ countOfValidResults > 28 && countOfValidResults < 40)) {
+ LOGGER.info("Moving regions of tables {}. current count of valid results: {}",
+ TABLE_NAMES, countOfValidResults);
+ countOfValidResults++;
+ TABLE_NAMES.forEach(table -> {
+ try {
+ moveRegionsOfTable(table);
+ } catch (Exception e) {
+ LOGGER.error("Unable to move regions of table: {}", table);
+ }
+ });
+ }
+ }
+ }
+
+ @BeforeClass
+ public static synchronized void doSetup() throws Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put(BaseScannerRegionObserver.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY,
+ Integer.toString(60 * 60)); // An hour
+ props.put(QueryServices.USE_STATS_FOR_PARALLELIZATION, Boolean.toString(false));
+ props.put(QueryServices.PHOENIX_SERVER_PAGE_SIZE_MS, Long.toString(0));
+ props.put(QueryServices.TESTS_MINI_CLUSTER_NUM_REGION_SERVERS, String.valueOf(2));
+ props.put(HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY, String.valueOf(1));
+ props.put(QueryServices.PHOENIX_POST_DUMMY_PROCESS,
+ TestScanningResultPostDummyResultCaller.class.getName());
+ props.put(QueryServices.PHOENIX_POST_VALID_PROCESS,
+ TestScanningResultPostValidResultCaller.class.getName());
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ }
+
+ @AfterClass
+ public static synchronized void freeResources() throws Exception {
+ BaseTest.freeResourcesIfBeyondThreshold();
+ }
+
+}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AggregateWithRegionMovesIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AggregateWithRegionMovesIT.java
new file mode 100644
index 0000000000..d72e4c7027
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AggregateWithRegionMovesIT.java
@@ -0,0 +1,499 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import org.apache.hadoop.hbase.TableName;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
+import org.apache.phoenix.schema.AmbiguousColumnException;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.QueryBuilder;
+import org.apache.phoenix.util.TestUtil;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.util.Properties;
+
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+@Category(NeedsOwnMiniClusterTest.class)
+public class AggregateWithRegionMovesIT extends BaseAggregateWithRegionMovesIT {
+
+ @Before
+ public void setUp() throws Exception {
+ hasTestStarted = true;
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ countOfDummyResults = 0;
+ TABLE_NAMES.clear();
+ hasTestStarted = false;
+ }
+
+ @Test
+ public void testGroupByWithAliasWithSameColumnName() throws Exception {
+ hasTestStarted = true;
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName1 = generateUniqueName();
+ String tableName2 = generateUniqueName();
+ String tableName3 = generateUniqueName();
+ String ddl = "create table " + tableName1 + " (pk integer primary key, col integer)";
+ conn.createStatement().execute(ddl);
+ ddl = "create table " + tableName2 + " (pk integer primary key, col integer)";
+ conn.createStatement().execute(ddl);
+ ddl = "create table " + tableName3 + " (notPk integer primary key, col integer)";
+ conn.createStatement().execute(ddl);
+ TABLE_NAMES.add(tableName1);
+ TABLE_NAMES.add(tableName2);
+ TABLE_NAMES.add(tableName3);
+ conn.createStatement().execute("UPSERT INTO " + tableName1 + " VALUES (1,2)");
+ conn.createStatement().execute("UPSERT INTO " + tableName2 + " VALUES (1,2)");
+ conn.createStatement().execute("UPSERT INTO " + tableName3 + " VALUES (1,2)");
+ conn.createStatement().executeQuery(
+ "select " + tableName1 + ".pk as pk from " + tableName1 + " group by pk");
+ conn.createStatement().executeQuery(
+ "select " + tableName1 + ".pk as pk from " + tableName1 + " group by " +
+ tableName1 + ".pk");
+ conn.createStatement().executeQuery(
+ "select " + tableName1 + ".pk as pk from " + tableName1 + " as t group by t.pk");
+ conn.createStatement()
+ .executeQuery("select " + tableName1 + ".col as pk from " + tableName1);
+ conn.createStatement()
+ .executeQuery("select " + tableName1 + ".pk as pk from " + tableName1 + " join " +
+ tableName3 + " on (" + tableName1 + ".pk=" + tableName3 +
+ ".notPk) group by pk");
+ try {
+ conn.createStatement().executeQuery(
+ "select " + tableName1 + ".col as pk from " + tableName1 + " group by pk");
+ fail();
+ } catch (AmbiguousColumnException ignored) {
+ }
+ moveRegionsOfTable(tableName1);
+ moveRegionsOfTable(tableName2);
+ moveRegionsOfTable(tableName3);
+ try {
+ conn.createStatement()
+ .executeQuery("select col as pk from " + tableName1 + " group by pk");
+ fail();
+ } catch (AmbiguousColumnException ignored) {
+ }
+ try {
+ conn.createStatement()
+ .executeQuery(
+ "select " + tableName1 + ".pk as pk from " + tableName1 + " join " +
+ tableName2 + " on (" + tableName1 + ".pk=" + tableName2 +
+ ".pk) group by pk");
+ fail();
+ } catch (AmbiguousColumnException ignored) {
+ }
+ conn.close();
+ }
+
+ @Test
+ public void testGroupByCoerceExpressionBug3453() throws Exception {
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ //Type is INT
+ String intTableName = generateUniqueName();
+ String sql = "CREATE TABLE " + intTableName + "(" +
+ "ENTITY_ID INTEGER NOT NULL," +
+ "CONTAINER_ID INTEGER NOT NULL," +
+ "SCORE INTEGER NOT NULL," +
+ "CONSTRAINT TEST_PK PRIMARY KEY (ENTITY_ID DESC,CONTAINER_ID DESC,SCORE DESC))";
+ TABLE_NAMES.add(intTableName);
+
+ conn.createStatement().execute(sql);
+ conn.createStatement().execute("UPSERT INTO " + intTableName + " VALUES (1,1,1)");
+ conn.commit();
+
+ sql = "select DISTINCT entity_id, score from ( select entity_id, score from " +
+ intTableName + " limit 1)";
+ ResultSet rs = conn.prepareStatement(sql).executeQuery();
+ assertResultSet(rs, new Object[][]{{1, 1}}, intTableName);
+
+ conn.createStatement().execute("UPSERT INTO " + intTableName + " VALUES (2,2,2)");
+ conn.createStatement().execute("UPSERT INTO " + intTableName + " VALUES (3,3,3)");
+ conn.commit();
+
+ sql = "select DISTINCT entity_id, score from ( select entity_id, score from " +
+ intTableName + " limit 3) order by entity_id";
+ rs = conn.prepareStatement(sql).executeQuery();
+ assertResultSet(rs, new Object[][]{{1, 1}, {2, 2}, {3, 3}}, intTableName);
+
+ sql = "select DISTINCT entity_id, score from ( select entity_id, score from " +
+ intTableName + " limit 3) order by entity_id desc";
+ rs = conn.prepareStatement(sql).executeQuery();
+ assertResultSet(rs, new Object[][]{{3, 3}, {2, 2}, {1, 1}}, intTableName);
+
+ //Type is CHAR
+ String charTableName = generateUniqueName();
+ sql = "CREATE TABLE " + charTableName + "(" +
+ "ENTITY_ID CHAR(15) NOT NULL," +
+ "CONTAINER_ID INTEGER NOT NULL," +
+ "SCORE INTEGER NOT NULL," +
+ "CONSTRAINT TEST_PK PRIMARY KEY (ENTITY_ID DESC,CONTAINER_ID DESC,SCORE DESC))";
+ TABLE_NAMES.add(charTableName);
+
+ conn.createStatement().execute(sql);
+ conn.createStatement()
+ .execute("UPSERT INTO " + charTableName + " VALUES ('entity1',1,1)");
+ conn.createStatement()
+ .execute("UPSERT INTO " + charTableName + " VALUES ('entity2',2,2)");
+ conn.createStatement()
+ .execute("UPSERT INTO " + charTableName + " VALUES ('entity3',3,3)");
+ conn.commit();
+
+ sql = "select DISTINCT entity_id, score from ( select entity_id, score from " +
+ charTableName + " limit 3) order by entity_id";
+ rs = conn.prepareStatement(sql).executeQuery();
+ assertResultSet(rs, new Object[][]{{"entity1", 1}, {"entity2", 2}, {"entity3", 3}},
+ charTableName);
+
+ sql = "select DISTINCT entity_id, score from ( select entity_id, score from " +
+ charTableName + " limit 3) order by entity_id desc";
+ rs = conn.prepareStatement(sql).executeQuery();
+ assertResultSet(rs, new Object[][]{{"entity3", 3}, {"entity2", 2}, {"entity1", 1}},
+ charTableName);
+ }
+ }
+
+ @Test
+ public void testNestedGroupedAggregationWithBigInt() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ String tableName = generateUniqueName();
+ TABLE_NAMES.add(tableName);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);) {
+ String createQuery = "CREATE TABLE " + tableName +
+ " (a BIGINT NOT NULL,c BIGINT NOT NULL CONSTRAINT PK PRIMARY KEY (a, c))";
+ String updateQuery = "UPSERT INTO " + tableName +
+ "(a,c) VALUES(4444444444444444444, 5555555555555555555)";
+ String query = "SELECT a FROM (SELECT a, c FROM " + tableName +
+ " GROUP BY a, c) GROUP BY a, c";
+ conn.prepareStatement(createQuery).execute();
+ conn.prepareStatement(updateQuery).execute();
+ conn.commit();
+ PreparedStatement statement = conn.prepareStatement(query);
+ ResultSet rs = statement.executeQuery();
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals(4444444444444444444L, rs.getLong(1));
+ assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testAvgGroupByOrderPreservingWithStats() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectExpression("COUNT(*)")
+ .setFullTableName(PhoenixDatabaseMetaData.SYSTEM_STATS_NAME)
+ .setWhereClause(PhoenixDatabaseMetaData.PHYSICAL_NAME + " ='" + tableName + "'");
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals(0, rs.getInt(1));
+ initAvgGroupTable(conn, tableName, PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + "=20 ");
+ TestUtil.dumpTable(conn, TableName.valueOf(tableName));
+ // When ParallelStats are disabled it is 4. When enabled it is 13
+ testAvgGroupByOrderPreserving(conn, tableName, 4);
+ rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals(13, rs.getInt(1));
+ conn.setAutoCommit(true);
+ conn.createStatement().execute("DELETE FROM " + "\"SYSTEM\".\"STATS\"");
+ rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals(0, rs.getInt(1));
+ TestUtil.doMajorCompaction(conn, tableName);
+ rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals(13, rs.getInt(1));
+ testAvgGroupByOrderPreserving(conn, tableName, 4);
+ conn.createStatement().execute(
+ "ALTER TABLE " + tableName + " SET " + PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH +
+ "=100");
+ testAvgGroupByOrderPreserving(conn, tableName, 6);
+ conn.createStatement().execute(
+ "ALTER TABLE " + tableName + " SET " + PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH +
+ "=null");
+ testAvgGroupByOrderPreserving(conn, tableName, 4);
+ }
+
+ @Override
+ protected void testCountNullInNonEmptyKeyValueCF(int columnEncodedBytes) throws Exception {
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ //Type is INT
+ String intTableName = generateUniqueName();
+ TABLE_NAMES.add(intTableName);
+ String sql = "create table " + intTableName +
+ " (mykey integer not null primary key, A.COLA integer, B.COLB integer) "
+ +
+ "IMMUTABLE_ROWS=true, IMMUTABLE_STORAGE_SCHEME = ONE_CELL_PER_COLUMN, COLUMN_ENCODED_BYTES = " +
+ columnEncodedBytes + ", DISABLE_WAL=true";
+
+ conn.createStatement().execute(sql);
+ conn.createStatement().execute("UPSERT INTO " + intTableName + " VALUES (1,1)");
+ conn.createStatement().execute("UPSERT INTO " + intTableName + " VALUES (2,1)");
+ conn.createStatement().execute("UPSERT INTO " + intTableName + " VALUES (3,1,2)");
+ conn.createStatement().execute("UPSERT INTO " + intTableName + " VALUES (4,1)");
+ conn.createStatement().execute("UPSERT INTO " + intTableName + " VALUES (5,1)");
+ conn.commit();
+
+ sql = "select count(*) from " + intTableName;
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectExpression("COUNT(*)")
+ .setFullTableName(intTableName);
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals(5, rs.getLong(1));
+
+ sql = "select count(*) from " + intTableName + " where b.colb is not null";
+ queryBuilder.setWhereClause("B.COLB IS NOT NULL");
+ rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals(1, rs.getLong(1));
+
+ sql = "select count(*) from " + intTableName + " where b.colb is null";
+ queryBuilder.setWhereClause("B.COLB IS NULL");
+ rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals(4, rs.getLong(1));
+ }
+ }
+
+ @Test
+ public void testOrderByOptimizeForClientAggregatePlanBug4820() throws Exception {
+ doTestOrderByOptimizeForClientAggregatePlanBug4820(false, false);
+ doTestOrderByOptimizeForClientAggregatePlanBug4820(false, true);
+ doTestOrderByOptimizeForClientAggregatePlanBug4820(true, false);
+ doTestOrderByOptimizeForClientAggregatePlanBug4820(true, true);
+ }
+
+ private void doTestOrderByOptimizeForClientAggregatePlanBug4820(boolean desc, boolean salted)
+ throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String tableName = generateUniqueName();
+ TABLE_NAMES.add(tableName);
+ String sql = "create table " + tableName + "( " +
+ " pk1 varchar not null , " +
+ " pk2 varchar not null, " +
+ " pk3 varchar not null," +
+ " v1 varchar, " +
+ " v2 varchar, " +
+ " CONSTRAINT TEST_PK PRIMARY KEY ( " +
+ "pk1 " + (desc ? "desc" : "") + ", " +
+ "pk2 " + (desc ? "desc" : "") + ", " +
+ "pk3 " + (desc ? "desc" : "") +
+ " )) " + (salted ? "SALT_BUCKETS =4" : "split on('b')");
+ conn.createStatement().execute(sql);
+
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES ('a11','a12','a13','a14','a15')");
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES ('a21','a22','a23','a24','a25')");
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES ('a31','a32','a33','a34','a35')");
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES ('b11','b12','b13','b14','b15')");
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES ('b21','b22','b23','b24','b25')");
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES ('b31','b32','b33','b34','b35')");
+ conn.commit();
+
+ sql = "select a.ak3 " +
+ "from (select pk1 ak1,pk2 ak2,pk3 ak3, substr(v1,1,1) av1,substr(v2,1,1) av2 from " +
+ tableName + " order by pk2,pk3 limit 10) a " +
+ "group by a.ak3,a.av1 order by a.ak3 desc,a.av1";
+ ResultSet rs = conn.prepareStatement(sql).executeQuery();
+ assertResultSet(rs,
+ new Object[][]{{"b33"}, {"b23"}, {"b13"}, {"a33"}, {"a23"}, {"a13"}},
+ tableName);
+
+ sql = "select a.ak3 " +
+ "from (select pk1 ak1,pk2 ak2,pk3 ak3, substr(v1,1,1) av1,substr(v2,1,1) av2 from " +
+ tableName + " order by pk2,pk3 limit 10) a " +
+ "group by a.ak3,a.av1 order by a.ak3,a.av1";
+ rs = conn.prepareStatement(sql).executeQuery();
+ assertResultSet(rs,
+ new Object[][]{{"a13"}, {"a23"}, {"a33"}, {"b13"}, {"b23"}, {"b33"}},
+ tableName);
+
+ sql = "select a.ak3 " +
+ "from (select pk1 ak1,pk2 ak2,pk3 ak3,substr(v1,1,1) av1,substr(v2,1,1) av2 from " +
+ tableName + " order by pk2,pk3 limit 10) a " +
+ "where a.av1 = 'a' group by a.av1,a.ak3 order by a.ak3 desc";
+ rs = conn.prepareStatement(sql).executeQuery();
+ assertResultSet(rs, new Object[][]{{"a33"}, {"a23"}, {"a13"}}, tableName);
+
+ sql = "select a.ak3 " +
+ "from (select pk1 ak1,pk2 ak2,pk3 ak3,substr(v1,1,1) av1,substr(v2,1,1) av2 from " +
+ tableName + " order by pk2,pk3 limit 10) a " +
+ "where a.av1 = 'a' group by a.av1,a.ak3 order by a.ak3";
+ rs = conn.prepareStatement(sql).executeQuery();
+ assertResultSet(rs, new Object[][]{{"a13"}, {"a23"}, {"a33"}}, tableName);
+
+ sql = "select a.ak3 " +
+ "from (select pk1 ak1,pk2 ak2,pk3 ak3,substr(v1,1,1) av1,substr(v2,1,1) av2 from " +
+ tableName + " order by pk2,pk3 limit 10) a " +
+ "where a.av1 = 'b' and a.av2= 'b' group by CASE WHEN a.av1 > a.av2 THEN a.av1 ELSE a.av2 END,a.ak3,a.ak2 order by a.ak3 desc,a.ak2 desc";
+ rs = conn.prepareStatement(sql).executeQuery();
+ assertResultSet(rs, new Object[][]{{"b33"}, {"b23"}, {"b13"}}, tableName);
+
+ sql = "select a.ak3 " +
+ "from (select pk1 ak1,pk2 ak2,pk3 ak3,substr(v1,1,1) av1,substr(v2,1,1) av2 from " +
+ tableName + " order by pk2,pk3 limit 10) a " +
+ "where a.av1 = 'b' and a.av2= 'b' group by CASE WHEN a.av1 > a.av2 THEN a.av1 ELSE a.av2 END,a.ak3,a.ak2 order by a.ak3,a.ak2 desc";
+ rs = conn.prepareStatement(sql).executeQuery();
+ assertResultSet(rs, new Object[][]{{"b13"}, {"b23"}, {"b33"}}, tableName);
+
+ tableName = generateUniqueName();
+ sql = "create table " + tableName + "( " +
+ " pk1 double not null , " +
+ " pk2 double not null, " +
+ " pk3 double not null," +
+ " v1 varchar, " +
+ " CONSTRAINT TEST_PK PRIMARY KEY ( " +
+ "pk1 " + (desc ? "desc" : "") + ", " +
+ "pk2 " + (desc ? "desc" : "") + ", " +
+ "pk3 " + (desc ? "desc" : "") +
+ " )) " + (salted ? "SALT_BUCKETS =4" : "split on(2.3)");
+ conn.createStatement().execute(sql);
+ conn.createStatement()
+ .execute("UPSERT INTO " + tableName + " VALUES (2.1,2.11,2.12,'e')");
+ conn.createStatement()
+ .execute("UPSERT INTO " + tableName + " VALUES (2.2,2.21,2.23,'d')");
+ conn.createStatement()
+ .execute("UPSERT INTO " + tableName + " VALUES (2.3,2.31,2.32,'c')");
+ conn.createStatement()
+ .execute("UPSERT INTO " + tableName + " VALUES (2.4,2.41,2.42,'b')");
+ conn.createStatement()
+ .execute("UPSERT INTO " + tableName + " VALUES (2.5,2.51,2.52,'a')");
+ conn.commit();
+
+ sql = "select a.av1 " +
+ "from (select pk1 ak1,pk2 ak2,pk3 ak3, substr(v1,1,1) av1 from " + tableName +
+ " order by pk1,pk2 limit 10) a " +
+ "where cast(a.ak1 as integer)=2 group by a.ak1,a.av1 order by a.av1";
+ rs = conn.prepareStatement(sql).executeQuery();
+ assertResultSet(rs, new Object[][]{{"a"}, {"b"}, {"c"}, {"d"}, {"e"}}, tableName);
+
+ }
+ }
+
+ @Test
+ public void testDistinctAggregatingResultIteratorBug6507() throws Exception {
+ doTestDistinctAggregatingResultIteratorBug6507(false, false);
+ doTestDistinctAggregatingResultIteratorBug6507(false, true);
+ doTestDistinctAggregatingResultIteratorBug6507(true, false);
+ doTestDistinctAggregatingResultIteratorBug6507(true, true);
+ }
+
+ private void doTestDistinctAggregatingResultIteratorBug6507(boolean desc, boolean salted)
+ throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = null;
+ try {
+ conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ TABLE_NAMES.add(tableName);
+ String sql = "create table " + tableName + "( " +
+ " pk1 varchar not null , " +
+ " pk2 varchar not null, " +
+ " pk3 varchar not null," +
+ " v1 varchar, " +
+ " v2 varchar, " +
+ " CONSTRAINT TEST_PK PRIMARY KEY ( " +
+ "pk1 " + (desc ? "desc" : "") + ", " +
+ "pk2 " + (desc ? "desc" : "") + ", " +
+ "pk3 " + (desc ? "desc" : "") +
+ " )) " + (salted ? "SALT_BUCKETS =4" : "split on('b')");
+ conn.createStatement().execute(sql);
+
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES ('a11','a12','a13','a14','a15')");
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES ('a21','a22','a23','a24','a25')");
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES ('a31','a32','a33','a38','a35')");
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES ('b11','b12','b13','b14','b15')");
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES ('b21','b22','b23','b24','b25')");
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES ('b31','b32','b33','b34','b35')");
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES ('a31','c12','c13','a34','a35')");
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES ('a31','a32','c13','a34','a35')");
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES ('a31','a32','d13','a35','a35')");
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES ('d31','a32','c13','a35','a35')");
+ conn.commit();
+
+ sql = "select distinct pk1,max(v1) from " + tableName +
+ " group by pk1,pk2,pk3 order by pk1,pk2,pk3";
+
+ ResultSet rs = conn.prepareStatement(sql).executeQuery();
+ assertResultSet(rs, new Object[][]{
+ {"a11", "a14"},
+ {"a21", "a24"},
+ {"a31", "a38"},
+ {"a31", "a34"},
+ {"a31", "a35"},
+ {"b11", "b14"},
+ {"b21", "b24"},
+ {"b31", "b34"},
+ {"d31", "a35"}},
+ tableName);
+
+ sql = "select distinct pk2,max(v1) from " + tableName +
+ " group by pk2,pk3 order by pk2,pk3";
+
+ rs = conn.prepareStatement(sql).executeQuery();
+ assertResultSet(rs, new Object[][]{
+ {"a12", "a14"},
+ {"a22", "a24"},
+ {"a32", "a38"},
+ {"a32", "a35"},
+ {"b12", "b14"},
+ {"b22", "b24"},
+ {"b32", "b34"},
+ {"c12", "a34"}},
+ tableName);
+ } finally {
+ if (conn != null) {
+ conn.close();
+ }
+ }
+ }
+}
+
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityIT.java
index 741fad5aac..f1bb8677b3 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityIT.java
@@ -22,12 +22,20 @@ import static org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.ADD_DELET
import static org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.ADD_VIEW_INDEX;
import static org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.CREATE_ADD;
import static org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.CREATE_DIVERGED_VIEW;
+import static org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.CREATE_OFFSET;
+import static org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.CREATE_ORDERED_GROUP_BY;
+import static org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.CREATE_ORDER_BY_NON_PK;
+import static org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.CREATE_UNORDERED_GROUP_BY;
import static org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.INDEX_REBUILD_ASYNC;
import static org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.QUERY_ADD_DATA;
import static org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.QUERY_ADD_DELETE;
import static org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.QUERY_CREATE_ADD;
import static org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.QUERY_CREATE_DIVERGED_VIEW;
import static org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.QUERY_INDEX_REBUILD_ASYNC;
+import static org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.QUERY_OFFSET;
+import static org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.QUERY_ORDERED_GROUP_BY;
+import static org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.QUERY_ORDER_BY_NON_PK;
+import static org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.QUERY_UNORDERED_GROUP_BY;
import static org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.QUERY_VIEW_INDEX;
import static org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.assertExpectedOutput;
import static org.apache.phoenix.end2end.BackwardCompatibilityTestUtil.checkForPreConditions;
@@ -64,6 +72,7 @@ import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.schema.SystemTaskSplitPolicy;
import org.apache.phoenix.util.PhoenixRuntime;
import org.junit.After;
+import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
import org.junit.experimental.categories.Category;
@@ -439,4 +448,85 @@ public class BackwardCompatibilityIT {
executeQueryWithClientVersion(compatibleClientVersion, QUERY_VIEW_INDEX, zkQuorum);
assertExpectedOutput(QUERY_VIEW_INDEX);
}
+
+ @Test
+ public void testUnorderedGroupByAddDataByOldClientReadByNewClient() throws Exception {
+ executeQueryWithClientVersion(compatibleClientVersion, CREATE_UNORDERED_GROUP_BY, zkQuorum);
+ executeQueriesWithCurrentVersion(QUERY_UNORDERED_GROUP_BY, url, NONE);
+ assertExpectedOutput(QUERY_UNORDERED_GROUP_BY);
+ }
+
+ @Test
+ public void testUnorderedGroupByAddDataByNewClientReadByOldClient() throws Exception {
+ executeQueriesWithCurrentVersion(CREATE_UNORDERED_GROUP_BY, url, NONE);
+ executeQueryWithClientVersion(compatibleClientVersion, QUERY_UNORDERED_GROUP_BY, zkQuorum);
+ assertExpectedOutput(QUERY_UNORDERED_GROUP_BY);
+ }
+
+ @Test
+ public void testOrderedGroupByAddDataByOldClientReadByNewClient() throws Exception {
+ executeQueryWithClientVersion(compatibleClientVersion, CREATE_ORDERED_GROUP_BY, zkQuorum);
+ executeQueriesWithCurrentVersion(QUERY_ORDERED_GROUP_BY, url, NONE);
+ assertExpectedOutput(QUERY_ORDERED_GROUP_BY);
+ }
+
+ @Test
+ public void testOrderedGroupByAddDataByNewClientReadByOldClient() throws Exception {
+ Assume.assumeTrue("compatible client version should be >= 5.1.3",
+ isClientCompatibleForOrderedGroupByQuery());
+ executeQueriesWithCurrentVersion(CREATE_ORDERED_GROUP_BY, url, NONE);
+ executeQueryWithClientVersion(compatibleClientVersion, QUERY_ORDERED_GROUP_BY, zkQuorum);
+ assertExpectedOutput(QUERY_ORDERED_GROUP_BY);
+ }
+
+ @Test
+ public void testOffsetAddDataByOldClientReadByNewClient() throws Exception {
+ executeQueryWithClientVersion(compatibleClientVersion, CREATE_OFFSET, zkQuorum);
+ executeQueriesWithCurrentVersion(QUERY_OFFSET, url, NONE);
+ assertExpectedOutput(QUERY_OFFSET);
+ }
+
+ @Test
+ public void testOffsetAddDataByNewClientReadByOldClient() throws Exception {
+ executeQueriesWithCurrentVersion(CREATE_OFFSET, url, NONE);
+ executeQueryWithClientVersion(compatibleClientVersion, QUERY_OFFSET, zkQuorum);
+ assertExpectedOutput(QUERY_OFFSET);
+ }
+
+ @Test
+ public void testOrderByNonPkAddDataByOldClientReadByNewClient() throws Exception {
+ executeQueryWithClientVersion(compatibleClientVersion, CREATE_ORDER_BY_NON_PK,
+ zkQuorum);
+ executeQueriesWithCurrentVersion(QUERY_ORDER_BY_NON_PK, url, NONE);
+ assertExpectedOutput(QUERY_ORDER_BY_NON_PK);
+ }
+
+ @Test
+ public void testOrderByNonPkAddDataByNewClientReadByOldClient() throws Exception {
+ executeQueriesWithCurrentVersion(CREATE_ORDER_BY_NON_PK, url, NONE);
+ executeQueryWithClientVersion(compatibleClientVersion, QUERY_ORDER_BY_NON_PK,
+ zkQuorum);
+ assertExpectedOutput(QUERY_ORDER_BY_NON_PK);
+ }
+
+ private boolean isClientCompatibleForOrderedGroupByQuery() {
+ String[] clientVersion = compatibleClientVersion.split("\\.");
+ int majorVersion = Integer.parseInt(clientVersion[0]);
+ int minorVersion = Integer.parseInt(clientVersion[1]);
+ int patchVersion = Integer.parseInt(clientVersion[2].split("-")[0]);
+ if (majorVersion > 5) {
+ return true;
+ }
+ if (majorVersion < 5) {
+ return false;
+ }
+ if (minorVersion > 1) {
+ return true;
+ }
+ if (minorVersion < 1) {
+ return false;
+ }
+ return patchVersion >= 3;
+ }
+
}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityTestUtil.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityTestUtil.java
index 1ad17f73bc..d4b5495049 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityTestUtil.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityTestUtil.java
@@ -73,6 +73,14 @@ public final class BackwardCompatibilityTestUtil {
public static final String CREATE_ADD = "create_add";
public static final String CREATE_TMP_TABLE = "create_tmp_table";
public static final String CREATE_DIVERGED_VIEW = "create_diverged_view";
+ public static final String UNORDERED_GROUP_BY = "unorder_groupby";
+ public static final String ORDERED_GROUP_BY = "ordered_groupby";
+ public static final String ORDER_BY_NON_PK = "orderby_nonpk";
+ public static final String OFFSET = "offset";
+ public static final String CREATE_UNORDERED_GROUP_BY = "create_" + UNORDERED_GROUP_BY;
+ public static final String CREATE_ORDERED_GROUP_BY = "create_" + ORDERED_GROUP_BY;
+ public static final String CREATE_ORDER_BY_NON_PK = "create_" + ORDER_BY_NON_PK;
+ public static final String CREATE_OFFSET = "create_" + OFFSET;
public static final String ADD_DATA = "add_data";
public static final String ADD_DELETE = "add_delete";
public static final String ADD_VIEW_INDEX = "add_view_index";
@@ -80,6 +88,10 @@ public final class BackwardCompatibilityTestUtil {
public static final String DELETE_FOR_SPLITABLE_SYSCAT = "delete_for_splitable_syscat";
public static final String VIEW_INDEX = "view_index";
public static final String SELECT_AND_DROP_TABLE = "select_and_drop_table";
+ public static final String QUERY_UNORDERED_GROUP_BY = QUERY_PREFIX + UNORDERED_GROUP_BY;
+ public static final String QUERY_ORDERED_GROUP_BY = QUERY_PREFIX + ORDERED_GROUP_BY;
+ public static final String QUERY_OFFSET = QUERY_PREFIX + OFFSET;
+ public static final String QUERY_ORDER_BY_NON_PK = QUERY_PREFIX + ORDER_BY_NON_PK;
public static final String QUERY_CREATE_ADD = QUERY_PREFIX + CREATE_ADD;
public static final String QUERY_ADD_DATA = QUERY_PREFIX + ADD_DATA;
public static final String QUERY_ADD_DELETE = QUERY_PREFIX + ADD_DELETE;
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseAggregateWithRegionMoves2IT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseAggregateWithRegionMoves2IT.java
new file mode 100644
index 0000000000..2e6c15a651
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseAggregateWithRegionMoves2IT.java
@@ -0,0 +1,633 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import org.apache.phoenix.compile.ExplainPlan;
+import org.apache.phoenix.compile.ExplainPlanAttributes;
+import org.apache.phoenix.jdbc.PhoenixPreparedStatement;
+import org.apache.phoenix.query.KeyRange;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.schema.types.PChar;
+import org.apache.phoenix.schema.types.PInteger;
+import org.apache.phoenix.thirdparty.com.google.common.collect.Lists;
+import org.apache.phoenix.util.ByteUtil;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.QueryBuilder;
+import org.apache.phoenix.util.TestUtil;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.Properties;
+
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+@Category(NeedsOwnMiniClusterTest.class)
+public class BaseAggregateWithRegionMoves2IT extends ParallelStatsDisabledWithRegionMovesIT {
+
+ @Before
+ public void setUp() throws Exception {
+ hasTestStarted = true;
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ countOfDummyResults = 0;
+ TABLE_NAMES.clear();
+ hasTestStarted = false;
+ }
+
+ @Test
+ public void testGroupByArray() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+
+ String tableName = generateUniqueName();
+ TABLE_NAMES.add(tableName);
+ conn.createStatement().execute("CREATE TABLE " + tableName + "(\n" +
+ " a VARCHAR NOT NULL,\n" +
+ " b VARCHAR,\n" +
+ " c INTEGER,\n" +
+ " d VARCHAR,\n" +
+ " e VARCHAR ARRAY,\n" +
+ " f BIGINT,\n" +
+ " g BIGINT,\n" +
+ " CONSTRAINT pk PRIMARY KEY(a)\n" +
+ ")");
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES('1', 'val', 100, 'a', ARRAY ['b'], 1, 2)");
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES('2', 'val', 100, 'a', ARRAY ['b'], 3, 4)");
+ conn.createStatement().execute("UPSERT INTO " + tableName +
+ " VALUES('3', 'val', 100, 'a', ARRAY ['b','c'], 5, 6)");
+ conn.commit();
+
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectExpression("C, SUM(F + G) AS SUMONE, D, E")
+ .setSelectExpressionColumns(Lists.newArrayList("A", "B", "C", "F", "G", "D", "E"))
+ .setWhereClause("B = 'val' AND A IN ('1','2','3')")
+ .setFullTableName(tableName)
+ .setGroupByClause("C, D, E")
+ .setOrderByClause("SUMONE desc");
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals(100, rs.getInt(1));
+ assertEquals(11, rs.getLong(2));
+ assertTrue(rs.next());
+ assertEquals(100, rs.getInt(1));
+ assertEquals(10, rs.getLong(2));
+ assertFalse(rs.next());
+ conn.close();
+ }
+
+ @Test
+ public void testGroupByOrderPreserving() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ props.put(QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB, Boolean.FALSE.toString());
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ TABLE_NAMES.add(tableName);
+
+ conn.createStatement()
+ .execute("CREATE TABLE " + tableName + "(ORGANIZATION_ID char(15) not null, \n" +
+ "JOURNEY_ID char(15) not null, \n" +
+ "DATASOURCE SMALLINT not null, \n" +
+ "MATCH_STATUS TINYINT not null, \n" +
+ "EXTERNAL_DATASOURCE_KEY varchar(30), \n" +
+ "ENTITY_ID char(15) not null, \n" +
+ "CONSTRAINT PK PRIMARY KEY (\n" +
+ " ORGANIZATION_ID, \n" +
+ " JOURNEY_ID, \n" +
+ " DATASOURCE, \n" +
+ " MATCH_STATUS,\n" +
+ " EXTERNAL_DATASOURCE_KEY,\n" +
+ " ENTITY_ID))");
+ conn.createStatement().execute("UPSERT INTO " + tableName
+ + " VALUES('000001111122222', '333334444455555', 0, 0, 'abc', '666667777788888')");
+ conn.createStatement().execute("UPSERT INTO " + tableName
+ + " VALUES('000001111122222', '333334444455555', 0, 0, 'abcd', '666667777788889')");
+ conn.createStatement().execute("UPSERT INTO " + tableName
+ + " VALUES('000001111122222', '333334444455555', 0, 0, 'abc', '666667777788899')");
+ conn.commit();
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectExpression("COUNT(1), EXTERNAL_DATASOURCE_KEY As DUP_COUNT")
+ .setSelectExpressionColumns(
+ Lists.newArrayList("EXTERNAL_DATASOURCE_KEY", "MATCH_STATUS",
+ "JOURNEY_ID", "DATASOURCE", "ORGANIZATION_ID"))
+ .setWhereClause(
+ "JOURNEY_ID='333334444455555' AND DATASOURCE=0 AND MATCH_STATUS <= 1 and ORGANIZATION_ID='000001111122222'")
+ .setFullTableName(tableName)
+ .setGroupByClause("MATCH_STATUS, EXTERNAL_DATASOURCE_KEY")
+ .setHavingClause("COUNT(1) > 1");
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals(2, rs.getLong(1));
+ assertEquals("abc", rs.getString(2));
+ assertFalse(rs.next());
+
+ ExplainPlan plan = conn.prepareStatement(queryBuilder.build())
+ .unwrap(PhoenixPreparedStatement.class).optimizeQuery()
+ .getExplainPlan();
+ ExplainPlanAttributes explainPlanAttributes =
+ plan.getPlanStepsAsAttributes();
+ assertEquals("PARALLEL 1-WAY",
+ explainPlanAttributes.getIteratorTypeAndScanSize());
+ assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType());
+ assertEquals(tableName, explainPlanAttributes.getTableName());
+ assertEquals(
+ " ['000001111122222','333334444455555',0,*] - ['000001111122222','333334444455555',0,1]",
+ explainPlanAttributes.getKeyRanges());
+ assertEquals("SERVER FILTER BY FIRST KEY ONLY",
+ explainPlanAttributes.getServerWhereFilter());
+ assertEquals(
+ "SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [MATCH_STATUS, EXTERNAL_DATASOURCE_KEY]",
+ explainPlanAttributes.getServerAggregate());
+ assertEquals("COUNT(1) > 1", explainPlanAttributes.getClientFilterBy());
+ }
+
+ @Test
+ public void testGroupByOrderPreservingDescSort() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ TABLE_NAMES.add(tableName);
+ conn.createStatement().execute("DROP TABLE IF EXISTS " + tableName + " CASCADE");
+ conn.createStatement().execute(
+ "CREATE TABLE " + tableName + " (k1 char(1) not null, k2 char(1) not null," +
+ " constraint pk primary key (k1,k2)) split on ('ac','jc','nc')");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 'a')");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 'b')");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 'c')");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 'd')");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 'a')");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 'b')");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 'c')");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 'd')");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 'a')");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 'b')");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 'c')");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 'd')");
+ conn.commit();
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectExpression("K1,COUNT(*)")
+ .setSelectColumns(Lists.newArrayList("K1"))
+ .setFullTableName(tableName)
+ .setGroupByClause("K1")
+ .setOrderByClause("K1 DESC");
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals("n", rs.getString(1));
+ assertEquals(4, rs.getLong(2));
+ assertTrue(rs.next());
+ assertEquals("j", rs.getString(1));
+ assertEquals(4, rs.getLong(2));
+ assertTrue(rs.next());
+ assertEquals("a", rs.getString(1));
+ assertEquals(4, rs.getLong(2));
+ assertFalse(rs.next());
+ ExplainPlan plan = conn.prepareStatement(queryBuilder.build())
+ .unwrap(PhoenixPreparedStatement.class).optimizeQuery()
+ .getExplainPlan();
+ ExplainPlanAttributes explainPlanAttributes =
+ plan.getPlanStepsAsAttributes();
+ assertEquals("PARALLEL 1-WAY",
+ explainPlanAttributes.getIteratorTypeAndScanSize());
+ assertEquals("REVERSE", explainPlanAttributes.getClientSortedBy());
+ assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType());
+ assertEquals(tableName, explainPlanAttributes.getTableName());
+ assertEquals("SERVER FILTER BY FIRST KEY ONLY",
+ explainPlanAttributes.getServerWhereFilter());
+ assertEquals("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]",
+ explainPlanAttributes.getServerAggregate());
+ }
+
+ @Test
+ public void testSumGroupByOrderPreservingDesc() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ TABLE_NAMES.add(tableName);
+
+ PreparedStatement stmt = conn.prepareStatement("CREATE TABLE " + tableName +
+ " (k1 char(1) not null, k2 integer not null, constraint pk primary key (k1,k2)) split on (?,?,?)");
+ stmt.setBytes(1,
+ ByteUtil.concat(PChar.INSTANCE.toBytes("a"), PInteger.INSTANCE.toBytes(3)));
+ stmt.setBytes(2,
+ ByteUtil.concat(PChar.INSTANCE.toBytes("j"), PInteger.INSTANCE.toBytes(3)));
+ stmt.setBytes(3,
+ ByteUtil.concat(PChar.INSTANCE.toBytes("n"), PInteger.INSTANCE.toBytes(3)));
+ stmt.execute();
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 1)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 2)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 3)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 4)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('b', 5)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 1)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 2)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 3)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 4)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 1)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 2)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 3)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 4)");
+ conn.commit();
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectExpression("K1,SUM(K2)")
+ .setSelectExpressionColumns(Lists.newArrayList("K1", "K2"))
+ .setFullTableName(tableName)
+ .setGroupByClause("K1")
+ .setOrderByClause("K1 DESC");
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("n", rs.getString(1));
+ assertEquals(10, rs.getLong(2));
+ assertTrue(rs.next());
+ assertEquals("j", rs.getString(1));
+ assertEquals(10, rs.getLong(2));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("b", rs.getString(1));
+ assertEquals(5, rs.getLong(2));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("a", rs.getString(1));
+ assertEquals(10, rs.getLong(2));
+ assertFalse(rs.next());
+ ExplainPlan plan = conn.prepareStatement(queryBuilder.build())
+ .unwrap(PhoenixPreparedStatement.class).optimizeQuery()
+ .getExplainPlan();
+ ExplainPlanAttributes explainPlanAttributes =
+ plan.getPlanStepsAsAttributes();
+ assertEquals("PARALLEL 1-WAY",
+ explainPlanAttributes.getIteratorTypeAndScanSize());
+ assertEquals("REVERSE", explainPlanAttributes.getClientSortedBy());
+ assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType());
+ assertEquals(tableName, explainPlanAttributes.getTableName());
+ assertEquals("SERVER FILTER BY FIRST KEY ONLY",
+ explainPlanAttributes.getServerWhereFilter());
+ assertEquals("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]",
+ explainPlanAttributes.getServerAggregate());
+ }
+
+ @Test
+ public void testSumGroupByOrderPreservingDescWithoutSplit() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ TABLE_NAMES.add(tableName);
+
+ PreparedStatement stmt = conn.prepareStatement("CREATE TABLE " + tableName +
+ " (k1 char(1) not null, k2 integer not null, constraint pk primary key (k1,k2))");
+ stmt.execute();
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 1)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 2)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 3)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 4)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('b', 5)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 1)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 2)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 3)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 4)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 1)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 2)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 3)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 4)");
+ conn.commit();
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectExpression("K1,SUM(K2)")
+ .setSelectExpressionColumns(Lists.newArrayList("K1", "K2"))
+ .setFullTableName(tableName)
+ .setWhereClause("K1 >= 'a' AND K1 < 'o'")
+ .setGroupByClause("K1")
+ .setOrderByClause("K1 DESC");
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("n", rs.getString(1));
+ assertEquals(10, rs.getLong(2));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("j", rs.getString(1));
+ assertEquals(10, rs.getLong(2));
+ assertTrue(rs.next());
+ assertEquals("b", rs.getString(1));
+ assertEquals(5, rs.getLong(2));
+ assertTrue(rs.next());
+ assertEquals("a", rs.getString(1));
+ assertEquals(10, rs.getLong(2));
+ assertFalse(rs.next());
+ ExplainPlan plan = conn.prepareStatement(queryBuilder.build())
+ .unwrap(PhoenixPreparedStatement.class).optimizeQuery()
+ .getExplainPlan();
+ ExplainPlanAttributes explainPlanAttributes =
+ plan.getPlanStepsAsAttributes();
+ assertEquals("PARALLEL 1-WAY",
+ explainPlanAttributes.getIteratorTypeAndScanSize());
+ assertEquals("REVERSE", explainPlanAttributes.getClientSortedBy());
+ assertEquals("RANGE SCAN ", explainPlanAttributes.getExplainScanType());
+ assertEquals(tableName, explainPlanAttributes.getTableName());
+ }
+
+ @Test
+ public void testSumGroupByOrderPreservingAsc() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ TABLE_NAMES.add(tableName);
+
+ PreparedStatement stmt = conn.prepareStatement("CREATE TABLE " + tableName +
+ " (k1 char(1) not null, k2 integer not null, constraint pk primary key (k1,k2)) split on (?,?,?)");
+ stmt.setBytes(1,
+ ByteUtil.concat(PChar.INSTANCE.toBytes("a"), PInteger.INSTANCE.toBytes(3)));
+ stmt.setBytes(2,
+ ByteUtil.concat(PChar.INSTANCE.toBytes("j"), PInteger.INSTANCE.toBytes(3)));
+ stmt.setBytes(3,
+ ByteUtil.concat(PChar.INSTANCE.toBytes("n"), PInteger.INSTANCE.toBytes(3)));
+ stmt.execute();
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 1)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 2)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 3)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 4)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('b', 5)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 1)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 2)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 3)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 4)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 1)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 2)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 3)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 4)");
+ conn.commit();
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectExpression("K1,SUM(K2)")
+ .setSelectExpressionColumns(Lists.newArrayList("K1", "K2"))
+ .setFullTableName(tableName)
+ .setGroupByClause("K1")
+ .setOrderByClause("K1 ASC");
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("a", rs.getString(1));
+ assertEquals(10, rs.getLong(2));
+ assertTrue(rs.next());
+ assertEquals("b", rs.getString(1));
+ assertEquals(5, rs.getLong(2));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("j", rs.getString(1));
+ assertEquals(10, rs.getLong(2));
+ assertTrue(rs.next());
+ assertEquals("n", rs.getString(1));
+ assertEquals(10, rs.getLong(2));
+ assertFalse(rs.next());
+ }
+
+ @Test
+ public void testSumGroupByOrderPreservingAscWithoutSplits() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ TABLE_NAMES.add(tableName);
+
+ PreparedStatement stmt = conn.prepareStatement("CREATE TABLE " + tableName +
+ " (k1 char(1) not null, k2 integer not null, constraint pk primary key (k1,k2))");
+ stmt.execute();
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 1)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 2)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 3)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 4)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('b', 5)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 1)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 2)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 3)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 4)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 1)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 2)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 3)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 4)");
+ conn.commit();
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectExpression("K1,SUM(K2)")
+ .setSelectExpressionColumns(Lists.newArrayList("K1", "K2"))
+ .setFullTableName(tableName)
+ .setGroupByClause("K1")
+ .setOrderByClause("K1 ASC");
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("a", rs.getString(1));
+ assertEquals(10, rs.getLong(2));
+ assertTrue(rs.next());
+ assertEquals("b", rs.getString(1));
+ assertEquals(5, rs.getLong(2));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("j", rs.getString(1));
+ assertEquals(10, rs.getLong(2));
+ assertTrue(rs.next());
+ assertEquals("n", rs.getString(1));
+ assertEquals(10, rs.getLong(2));
+ assertFalse(rs.next());
+ }
+
+ @Test
+ public void testAvgGroupByOrderPreservingWithNoStats() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ initAvgGroupTable(conn, tableName, "");
+ testAvgGroupByOrderPreserving(conn, tableName, 4);
+ }
+
+ protected void initAvgGroupTable(Connection conn, String tableName, String tableProps)
+ throws SQLException {
+ PreparedStatement stmt = conn.prepareStatement("CREATE TABLE " + tableName +
+ " (k1 char(1) not null, k2 integer not null, constraint pk primary key (k1,k2)) " +
+ tableProps + " split on (?,?,?)");
+ TABLE_NAMES.add(tableName);
+ stmt.setBytes(1,
+ ByteUtil.concat(PChar.INSTANCE.toBytes("a"), PInteger.INSTANCE.toBytes(3)));
+ stmt.setBytes(2,
+ ByteUtil.concat(PChar.INSTANCE.toBytes("j"), PInteger.INSTANCE.toBytes(3)));
+ stmt.setBytes(3,
+ ByteUtil.concat(PChar.INSTANCE.toBytes("n"), PInteger.INSTANCE.toBytes(3)));
+ stmt.execute();
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 1)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 2)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 3)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 6)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('b', 5)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 1)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 2)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 3)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 10)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 1)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 2)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 3)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 2)");
+ conn.commit();
+ }
+
+ protected void testAvgGroupByOrderPreserving(Connection conn, String tableName, int splitSize)
+ throws SQLException, IOException {
+ TABLE_NAMES.add(tableName);
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectExpression("K1, AVG(K2)")
+ .setSelectExpressionColumns(Lists.newArrayList("K1", "K2"))
+ .setFullTableName(tableName)
+ .setGroupByClause("K1")
+ .setOrderByClause("K1");
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals("a", rs.getString(1));
+ assertEquals(3, rs.getDouble(2), 1e-6);
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("b", rs.getString(1));
+ assertEquals(5, rs.getDouble(2), 1e-6);
+ assertTrue(rs.next());
+ assertEquals("j", rs.getString(1));
+ assertEquals(4, rs.getDouble(2), 1e-6);
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("n", rs.getString(1));
+ assertEquals(2, rs.getDouble(2), 1e-6);
+ assertFalse(rs.next());
+ ExplainPlan plan = conn.prepareStatement(queryBuilder.build())
+ .unwrap(PhoenixPreparedStatement.class).optimizeQuery()
+ .getExplainPlan();
+ ExplainPlanAttributes explainPlanAttributes =
+ plan.getPlanStepsAsAttributes();
+ assertEquals("PARALLEL 1-WAY",
+ explainPlanAttributes.getIteratorTypeAndScanSize());
+ assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType());
+ assertEquals(tableName, explainPlanAttributes.getTableName());
+ assertEquals("SERVER FILTER BY FIRST KEY ONLY",
+ explainPlanAttributes.getServerWhereFilter());
+ assertEquals("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]",
+ explainPlanAttributes.getServerAggregate());
+ TestUtil.analyzeTable(conn, tableName);
+ List<KeyRange> splits = TestUtil.getAllSplits(conn, tableName);
+ // nGuideposts when stats are enabled, 4 when disabled
+ assertEquals(splitSize, splits.size());
+ }
+
+ @Test
+ public void testGroupByOrderByDescBug3451() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String tableName = generateUniqueName();
+ String sql = "CREATE TABLE " + tableName + " (\n" +
+ " ORGANIZATION_ID CHAR(15) NOT NULL,\n" +
+ " CONTAINER_ID CHAR(15) NOT NULL,\n" +
+ " ENTITY_ID CHAR(15) NOT NULL,\n" +
+ " SCORE DOUBLE,\n" +
+ " CONSTRAINT TEST_PK PRIMARY KEY (\n" +
+ " ORGANIZATION_ID,\n" +
+ " CONTAINER_ID,\n" +
+ " ENTITY_ID\n" +
+ " )\n" +
+ " )";
+ conn.createStatement().execute(sql);
+ String indexName = generateUniqueName();
+ TABLE_NAMES.add(tableName);
+ TABLE_NAMES.add(indexName);
+ conn.createStatement().execute("CREATE INDEX " + indexName + " ON " + tableName +
+ "(ORGANIZATION_ID,CONTAINER_ID, SCORE DESC, ENTITY_ID DESC)");
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES ('org2','container2','entityId6',1.1)");
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES ('org2','container1','entityId5',1.2)");
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES ('org2','container2','entityId4',1.3)");
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES ('org2','container1','entityId5',1.2)");
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES ('org2','container1','entityId3',1.4)");
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES ('org2','container3','entityId7',1.35)");
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES ('org2','container3','entityId8',1.45)");
+ conn.commit();
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setDistinct(true)
+ .setSelectColumns(Lists.newArrayList("ENTITY_ID", "SCORE", "ORGANIZATION_ID",
+ "CONTAINER_ID"))
+ .setFullTableName(tableName)
+ .setWhereClause(
+ "ORGANIZATION_ID = 'org2' AND CONTAINER_ID IN ('container1','container2','container3')")
+ .setOrderByClause("SCORE DESC")
+ .setLimit(2);
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("entityId8", rs.getString(1));
+ assertEquals(1.45, rs.getDouble(2), 0.001);
+ assertTrue(rs.next());
+ assertEquals("entityId3", rs.getString(1));
+ assertEquals(1.4, rs.getDouble(2), 0.001);
+ assertFalse(rs.next());
+
+ String expectedPhoenixPlan = "";
+ validateQueryPlan(conn, queryBuilder, expectedPhoenixPlan, null);
+ }
+ }
+
+ protected static void assertResultSet(ResultSet rs, Object[][] rows, String tableName)
+ throws Exception {
+ boolean switchToRegionMove = true;
+ for (int rowIndex = 0; rowIndex < rows.length; rowIndex++) {
+ assertTrue("rowIndex:[" + rowIndex + "] rs.next error!", rs.next());
+ if (switchToRegionMove) {
+ moveRegionsOfTable(tableName);
+ switchToRegionMove = false;
+ } else {
+ switchToRegionMove = true;
+ }
+ for (int columnIndex = 1; columnIndex <= rows[rowIndex].length; columnIndex++) {
+ Object realValue = rs.getObject(columnIndex);
+ Object expectedValue = rows[rowIndex][columnIndex - 1];
+ if (realValue == null) {
+ assertNull("rowIndex:[" + rowIndex + "],columnIndex:[" + columnIndex + "]",
+ expectedValue);
+ } else {
+ assertEquals("rowIndex:[" + rowIndex + "],columnIndex:[" + columnIndex + "]",
+ expectedValue,
+ realValue
+ );
+ }
+ }
+ }
+ assertFalse(rs.next());
+ }
+}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseAggregateWithRegionMoves3IT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseAggregateWithRegionMoves3IT.java
new file mode 100644
index 0000000000..d776dca2b2
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseAggregateWithRegionMoves3IT.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.phoenix.coprocessor.BaseScannerRegionObserver;
+import org.apache.phoenix.iterate.ScanningResultPostDummyResultCaller;
+import org.apache.phoenix.iterate.ScanningResultPostValidResultCaller;
+import org.apache.phoenix.query.BaseTest;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.experimental.categories.Category;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+
+@Category(NeedsOwnMiniClusterTest.class)
+public class BaseAggregateWithRegionMoves3IT extends BaseAggregateWithRegionMoves2IT {
+
+ private static final Logger LOGGER =
+ LoggerFactory.getLogger(BaseAggregateWithRegionMoves3IT.class);
+
+ private static int countOfValidResults = 0;
+
+ @Before
+ public void setUp() throws Exception {
+ hasTestStarted = true;
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ countOfDummyResults = 0;
+ countOfValidResults = 0;
+ TABLE_NAMES.clear();
+ hasTestStarted = false;
+ }
+
+ private static class TestScanningResultPostDummyResultCaller extends
+ ScanningResultPostDummyResultCaller {
+
+ @Override
+ public void postDummyProcess() {
+ if (hasTestStarted && (countOfDummyResults++ % 3) == 0 &&
+ (countOfDummyResults < 17 ||
+ countOfDummyResults > 28 && countOfDummyResults < 40)) {
+ LOGGER.info("Moving regions of tables {}. current count of dummy results: {}",
+ TABLE_NAMES, countOfDummyResults);
+ TABLE_NAMES.forEach(table -> {
+ try {
+ moveRegionsOfTable(table);
+ } catch (Exception e) {
+ LOGGER.error("Unable to move regions of table: {}", table);
+ }
+ });
+ }
+ }
+ }
+
+ private static class TestScanningResultPostValidResultCaller extends
+ ScanningResultPostValidResultCaller {
+
+ @Override
+ public void postValidRowProcess() {
+ if (hasTestStarted &&
+ (countOfValidResults < 17 ||
+ countOfValidResults > 28 && countOfValidResults < 40)) {
+ LOGGER.info("Moving regions of tables {}. current count of valid results: {}",
+ TABLE_NAMES, countOfValidResults);
+ countOfValidResults++;
+ TABLE_NAMES.forEach(table -> {
+ try {
+ moveRegionsOfTable(table);
+ } catch (Exception e) {
+ LOGGER.error("Unable to move regions of table: {}", table);
+ }
+ });
+ }
+ }
+ }
+
+ @BeforeClass
+ public static synchronized void doSetup() throws Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put(BaseScannerRegionObserver.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY,
+ Integer.toString(60 * 60)); // An hour
+ props.put(QueryServices.USE_STATS_FOR_PARALLELIZATION, Boolean.toString(false));
+ props.put(QueryServices.PHOENIX_SERVER_PAGE_SIZE_MS, Long.toString(0));
+ props.put(QueryServices.TESTS_MINI_CLUSTER_NUM_REGION_SERVERS, String.valueOf(2));
+ props.put(HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY, String.valueOf(1));
+ props.put(QueryServices.PHOENIX_POST_DUMMY_PROCESS,
+ TestScanningResultPostDummyResultCaller.class.getName());
+ props.put(QueryServices.PHOENIX_POST_VALID_PROCESS,
+ TestScanningResultPostValidResultCaller.class.getName());
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ }
+
+ @AfterClass
+ public static synchronized void freeResources() throws Exception {
+ BaseTest.freeResourcesIfBeyondThreshold();
+ }
+
+}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseAggregateWithRegionMoves4IT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseAggregateWithRegionMoves4IT.java
new file mode 100644
index 0000000000..d5c664f4b8
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseAggregateWithRegionMoves4IT.java
@@ -0,0 +1,136 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.phoenix.coprocessor.BaseScannerRegionObserver;
+import org.apache.phoenix.iterate.ScanningResultPostValidResultCaller;
+import org.apache.phoenix.query.BaseTest;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.schema.StaleRegionBoundaryCacheException;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+
+@Category(NeedsOwnMiniClusterTest.class)
+public class BaseAggregateWithRegionMoves4IT extends BaseAggregateWithRegionMoves2IT {
+
+ private static final Logger LOGGER =
+ LoggerFactory.getLogger(BaseAggregateWithRegionMoves4IT.class);
+
+ private static int countOfValidResults = 0;
+
+ @Before
+ public void setUp() throws Exception {
+ hasTestStarted = true;
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ countOfDummyResults = 0;
+ countOfValidResults = 0;
+ TABLE_NAMES.clear();
+ hasTestStarted = false;
+ }
+
+ private static class TestScanningResultPostValidResultCaller extends
+ ScanningResultPostValidResultCaller {
+
+ @Override
+ public void postValidRowProcess() {
+ if (hasTestStarted && countOfValidResults <= 1) {
+ LOGGER.info("Splitting regions of tables {}. current count of valid results: {}",
+ TABLE_NAMES, countOfValidResults);
+ countOfValidResults++;
+ TABLE_NAMES.forEach(table -> {
+ try {
+ splitAllRegionsOfTable(table, 2);
+ } catch (Exception e) {
+ LOGGER.error("Unable to move regions of table: {}", table);
+ }
+ });
+ }
+ }
+ }
+
+ @BeforeClass
+ public static synchronized void doSetup() throws Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put(BaseScannerRegionObserver.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY,
+ Integer.toString(60 * 60)); // An hour
+ props.put(QueryServices.USE_STATS_FOR_PARALLELIZATION, Boolean.toString(false));
+ props.put(QueryServices.PHOENIX_SERVER_PAGE_SIZE_MS, Long.toString(0));
+ props.put(QueryServices.TESTS_MINI_CLUSTER_NUM_REGION_SERVERS, String.valueOf(2));
+ props.put(HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY, String.valueOf(1));
+ props.put(QueryServices.PHOENIX_POST_VALID_PROCESS,
+ TestScanningResultPostValidResultCaller.class.getName());
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ }
+
+ @AfterClass
+ public static synchronized void freeResources() throws Exception {
+ BaseTest.freeResourcesIfBeyondThreshold();
+ }
+
+ @Test
+ public void testAvgGroupByOrderPreservingWithNoStats() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ initAvgGroupTable(conn, tableName, "");
+ testAvgGroupByOrderPreserving(conn, tableName, 5);
+ }
+
+ @Test
+ public void testSumGroupByOrderPreservingDescWithoutSplit() throws Exception {
+ try {
+ super.testSumGroupByOrderPreservingDescWithoutSplit();
+ } catch (StaleRegionBoundaryCacheException e) {
+ hasTestStarted = false;
+ LOGGER.error("Rows could not be scanned because of stale region boundary. Try again.");
+ super.testSumGroupByOrderPreservingDescWithoutSplit();
+ }
+ }
+
+ @Test
+ public void testGroupByOrderByDescBug3451() throws Exception {
+ try {
+ super.testGroupByOrderByDescBug3451();
+ } catch (StaleRegionBoundaryCacheException e) {
+ hasTestStarted = false;
+ LOGGER.error("Rows could not be scanned because of stale region boundary. Try again.");
+ super.testGroupByOrderByDescBug3451();
+ }
+ }
+
+}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseAggregateWithRegionMovesIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseAggregateWithRegionMovesIT.java
new file mode 100644
index 0000000000..a0dca2651e
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseAggregateWithRegionMovesIT.java
@@ -0,0 +1,998 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import org.apache.phoenix.compile.ExplainPlan;
+import org.apache.phoenix.compile.ExplainPlanAttributes;
+import org.apache.phoenix.jdbc.PhoenixPreparedStatement;
+import org.apache.phoenix.query.KeyRange;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.schema.types.PChar;
+import org.apache.phoenix.schema.types.PInteger;
+import org.apache.phoenix.thirdparty.com.google.common.collect.Lists;
+import org.apache.phoenix.util.ByteUtil;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.QueryBuilder;
+import org.apache.phoenix.util.TestUtil;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.List;
+import java.util.Properties;
+
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+public abstract class BaseAggregateWithRegionMovesIT
+ extends ParallelStatsDisabledWithRegionMovesIT {
+
+ private static void initData(Connection conn, String tableName) throws SQLException {
+ conn.createStatement().execute("create table " + tableName +
+ " (id varchar not null primary key,\n" +
+ " uri varchar, appcpu integer)");
+ TABLE_NAMES.add(tableName);
+ insertRow(conn, tableName, "Report1", 10, 1);
+ insertRow(conn, tableName, "Report2", 10, 2);
+ insertRow(conn, tableName, "Report3", 30, 3);
+ insertRow(conn, tableName, "Report4", 30, 4);
+ insertRow(conn, tableName, "SOQL1", 10, 5);
+ insertRow(conn, tableName, "SOQL2", 10, 6);
+ insertRow(conn, tableName, "SOQL3", 30, 7);
+ insertRow(conn, tableName, "SOQL4", 30, 8);
+ conn.commit();
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ hasTestStarted = true;
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ countOfDummyResults = 0;
+ TABLE_NAMES.clear();
+ hasTestStarted = false;
+ }
+
+ private static void insertRow(Connection conn, String tableName, String uri, int appcpu, int id)
+ throws SQLException {
+ PreparedStatement statement = conn.prepareStatement(
+ "UPSERT INTO " + tableName + "(id, uri, appcpu) values (?,?,?)");
+ statement.setString(1, "id" + id);
+ statement.setString(2, uri);
+ statement.setInt(3, appcpu);
+ statement.executeUpdate();
+ }
+
+ @Test
+ public void testDuplicateTrailingAggExpr() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ props.put(QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB, Boolean.FALSE.toString());
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ TABLE_NAMES.add(tableName);
+
+ conn.createStatement().execute("create table " + tableName +
+ " (nam VARCHAR(20), address VARCHAR(20), id BIGINT "
+ + "constraint my_pk primary key (id))");
+ PreparedStatement statement = conn.prepareStatement(
+ "UPSERT INTO " + tableName + "(nam, address, id) values (?,?,?)");
+ statement.setString(1, "pulkit");
+ statement.setString(2, "badaun");
+ statement.setInt(3, 1);
+ statement.executeUpdate();
+ conn.commit();
+
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setDistinct(true)
+ .setSelectExpression("'harshit' as TEST_COLUMN, trim(NAM), trim(NAM)")
+ .setSelectExpressionColumns(Lists.newArrayList("NAM"))
+ .setFullTableName(tableName);
+
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals("harshit", rs.getString(1));
+ assertEquals("pulkit", rs.getString(2));
+ assertEquals("pulkit", rs.getString(3));
+ conn.close();
+ }
+
+ @Test
+ public void testExpressionInGroupBy() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ TABLE_NAMES.add(tableName);
+ String ddl = "create table " + tableName +
+ "(tgb_id integer NOT NULL,utc_date_epoch integer NOT NULL,tgb_name varchar(40),ack_success_count integer" +
+ ",ack_success_one_ack_count integer, CONSTRAINT pk_tgb_counter PRIMARY KEY(tgb_id, utc_date_epoch))";
+
+ createTestTable(getUrl(), ddl);
+ String dml = "UPSERT INTO " + tableName + " VALUES(?,?,?,?,?)";
+ PreparedStatement stmt = conn.prepareStatement(dml);
+ stmt.setInt(1, 1);
+ stmt.setInt(2, 1000);
+ stmt.setString(3, "aaa");
+ stmt.setInt(4, 1);
+ stmt.setInt(5, 1);
+ stmt.execute();
+ stmt.setInt(1, 2);
+ stmt.setInt(2, 2000);
+ stmt.setString(3, "bbb");
+ stmt.setInt(4, 2);
+ stmt.setInt(5, 2);
+ stmt.execute();
+ conn.commit();
+
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectExpression(
+ "TGB_ID, TGB_NAME, (UTC_DATE_EPOCH/10)*10 AS UTC_EPOCH_HOUR,SUM(ACK_SUCCESS_COUNT + " +
+ "ACK_SUCCESS_ONE_ACK_COUNT) AS ACK_TX_SUM")
+ .setSelectExpressionColumns(Lists.newArrayList("TGB_ID", "TGB_NAME",
+ "UTC_DATE_EPOCH", "ACK_SUCCESS_COUNT", "ACK_SUCCESS_ONE_ACK_COUNT"))
+ .setGroupByClause("TGB_ID, TGB_NAME, UTC_EPOCH_HOUR")
+ .setFullTableName(tableName)
+ .setOrderByClause("TGB_ID, UTC_EPOCH_HOUR");
+
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals(1, rs.getInt(1));
+ assertEquals("aaa", rs.getString(2));
+ assertEquals(1000, rs.getDouble(3), 1e-6);
+ assertEquals(2, rs.getLong(4));
+ assertTrue(rs.next());
+ assertEquals(2, rs.getInt(1));
+ assertEquals("bbb", rs.getString(2));
+ assertEquals(2000, rs.getDouble(3), 1e-6);
+ assertEquals(4, rs.getLong(4));
+ assertFalse(rs.next());
+ rs.close();
+ conn.close();
+ }
+
+ @Test
+ public void testBooleanInGroupBy() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ TABLE_NAMES.add(tableName);
+ String ddl = " create table " + tableName +
+ "(id varchar primary key,v1 boolean, v2 integer, v3 integer)";
+
+ createTestTable(getUrl(), ddl);
+ PreparedStatement stmt =
+ conn.prepareStatement("UPSERT INTO " + tableName + "(id,v2,v3) VALUES(?,?,?)");
+ stmt.setString(1, "a");
+ stmt.setInt(2, 1);
+ stmt.setInt(3, 1);
+ stmt.execute();
+ stmt.close();
+ stmt = conn.prepareStatement("UPSERT INTO " + tableName + " VALUES(?,?,?,?)");
+ stmt.setString(1, "b");
+ stmt.setBoolean(2, false);
+ stmt.setInt(3, 2);
+ stmt.setInt(4, 2);
+ stmt.execute();
+ stmt.setString(1, "c");
+ stmt.setBoolean(2, true);
+ stmt.setInt(3, 3);
+ stmt.setInt(4, 3);
+ stmt.execute();
+ conn.commit();
+
+ String[] gbs = {"V1,V2,V3", "V1,V3,V2", "V2,V1,V3"};
+ for (String gb : gbs) {
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectColumns(Lists.newArrayList("V1", "V2", "V3"))
+ .setFullTableName(tableName)
+ .setGroupByClause(gb);
+
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals(false, rs.getBoolean("v1"));
+ assertTrue(rs.wasNull());
+ assertEquals(1, rs.getInt("v2"));
+ assertEquals(1, rs.getInt("v3"));
+ assertTrue(rs.next());
+ assertEquals(false, rs.getBoolean("v1"));
+ assertFalse(rs.wasNull());
+ assertEquals(2, rs.getInt("v2"));
+ assertEquals(2, rs.getInt("v3"));
+ assertTrue(rs.next());
+ assertEquals(true, rs.getBoolean("v1"));
+ assertEquals(3, rs.getInt("v2"));
+ assertEquals(3, rs.getInt("v3"));
+ assertFalse(rs.next());
+ rs.close();
+ }
+ conn.close();
+ }
+
+ @Test
+ public void testScanUri() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ TABLE_NAMES.add(tableName);
+ initData(conn, tableName);
+ Statement stmt = conn.createStatement();
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectColumns(Lists.newArrayList("URI"))
+ .setFullTableName(tableName);
+
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("Report1", rs.getString(1));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("Report2", rs.getString(1));
+ assertTrue(rs.next());
+ assertEquals("Report3", rs.getString(1));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("Report4", rs.getString(1));
+ assertTrue(rs.next());
+ assertEquals("SOQL1", rs.getString(1));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("SOQL2", rs.getString(1));
+ assertTrue(rs.next());
+ assertEquals("SOQL3", rs.getString(1));
+ assertTrue(rs.next());
+ assertEquals("SOQL4", rs.getString(1));
+ assertFalse(rs.next());
+ conn.close();
+ }
+
+ @Test
+ public void testCount() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ TABLE_NAMES.add(tableName);
+ initData(conn, tableName);
+ Statement stmt = conn.createStatement();
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectExpression("count(1)")
+ .setFullTableName(tableName);
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals(8, rs.getLong(1));
+ assertFalse(rs.next());
+ conn.close();
+ }
+
+ @Test
+ public void testGroupByCase() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ props.put(QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB, Boolean.FALSE.toString());
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ TABLE_NAMES.add(tableName);
+ initData(conn, tableName);
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectExpression(
+ "CASE WHEN URI LIKE 'REPORT%' THEN 'REPORTS' ELSE 'OTHER' END CATEGORY, AVG(APPCPU)")
+ .setSelectExpressionColumns(Lists.newArrayList("URI", "APPCPU"))
+ .setFullTableName(tableName)
+ .setGroupByClause("CATEGORY");
+ executeQuery(conn, queryBuilder);
+
+ queryBuilder.setSelectExpression(
+ "CASE URI WHEN 'REPORT%' THEN 'REPORTS' ELSE 'OTHER' END CATEGORY, AVG(APPCPU)")
+ .setSelectExpressionColumns(Lists.newArrayList("URI", "APPCPU"))
+ .setFullTableName(tableName)
+ .setGroupByClause("APPCPU, CATEGORY");
+ executeQuery(conn, queryBuilder);
+
+ queryBuilder.setSelectExpression(
+ "CASE URI WHEN 'Report%' THEN 'Reports' ELSE 'Other' END CATEGORY, AVG(APPCPU)")
+ .setSelectColumns(Lists.newArrayList("URI", "APPCPU"))
+ .setFullTableName(tableName)
+ .setGroupByClause("AVG(APPCPU), CATEGORY");
+ executeQueryThrowsException(conn, queryBuilder,
+ "Aggregate expressions may not be used in GROUP BY", "");
+ conn.close();
+ }
+
+ protected void initAvgGroupTable(Connection conn, String tableName, String tableProps)
+ throws SQLException {
+ PreparedStatement stmt = conn.prepareStatement("CREATE TABLE " + tableName +
+ " (k1 char(1) not null, k2 integer not null, constraint pk primary key (k1,k2)) " +
+ tableProps + " split on (?,?,?)");
+ TABLE_NAMES.add(tableName);
+ stmt.setBytes(1,
+ ByteUtil.concat(PChar.INSTANCE.toBytes("a"), PInteger.INSTANCE.toBytes(3)));
+ stmt.setBytes(2,
+ ByteUtil.concat(PChar.INSTANCE.toBytes("j"), PInteger.INSTANCE.toBytes(3)));
+ stmt.setBytes(3,
+ ByteUtil.concat(PChar.INSTANCE.toBytes("n"), PInteger.INSTANCE.toBytes(3)));
+ stmt.execute();
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 1)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 2)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 3)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 6)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('b', 5)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 1)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 2)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 3)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 10)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 1)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 2)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 3)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 2)");
+ conn.commit();
+ }
+
+ protected void testAvgGroupByOrderPreserving(Connection conn, String tableName, int nGuidePosts)
+ throws SQLException, IOException {
+ TABLE_NAMES.add(tableName);
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectExpression("K1, AVG(K2)")
+ .setSelectExpressionColumns(Lists.newArrayList("K1", "K2"))
+ .setFullTableName(tableName)
+ .setGroupByClause("K1")
+ .setOrderByClause("K1");
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals("a", rs.getString(1));
+ assertEquals(3, rs.getDouble(2), 1e-6);
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("b", rs.getString(1));
+ assertEquals(5, rs.getDouble(2), 1e-6);
+ assertTrue(rs.next());
+ assertEquals("j", rs.getString(1));
+ assertEquals(4, rs.getDouble(2), 1e-6);
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("n", rs.getString(1));
+ assertEquals(2, rs.getDouble(2), 1e-6);
+ assertFalse(rs.next());
+ ExplainPlan plan = conn.prepareStatement(queryBuilder.build())
+ .unwrap(PhoenixPreparedStatement.class).optimizeQuery()
+ .getExplainPlan();
+ ExplainPlanAttributes explainPlanAttributes =
+ plan.getPlanStepsAsAttributes();
+ assertEquals("PARALLEL 1-WAY",
+ explainPlanAttributes.getIteratorTypeAndScanSize());
+ assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType());
+ assertEquals(tableName, explainPlanAttributes.getTableName());
+ assertEquals("SERVER FILTER BY FIRST KEY ONLY",
+ explainPlanAttributes.getServerWhereFilter());
+ assertEquals("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]",
+ explainPlanAttributes.getServerAggregate());
+ TestUtil.analyzeTable(conn, tableName);
+ List<KeyRange> splits = TestUtil.getAllSplits(conn, tableName);
+ // nGuideposts when stats are enabled, 4 when disabled
+ assertEquals(4, splits.size());
+ }
+
+ protected void testAvgGroupByDescOrderPreserving(Connection conn, String tableName)
+ throws SQLException, IOException {
+ TABLE_NAMES.add(tableName);
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectExpression("K1, AVG(K2)")
+ .setSelectExpressionColumns(Lists.newArrayList("K1", "K2"))
+ .setFullTableName(tableName)
+ .setGroupByClause("K1")
+ .setOrderByClause("K1 DESC");
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals("n", rs.getString(1));
+ assertEquals(2, rs.getDouble(2), 1e-6);
+ moveRegionsOfTable(tableName);
+ assertTrue(rs.next());
+ assertEquals("j", rs.getString(1));
+ assertEquals(4, rs.getDouble(2), 1e-6);
+ assertTrue(rs.next());
+ assertEquals("b", rs.getString(1));
+ assertEquals(5, rs.getDouble(2), 1e-6);
+ moveRegionsOfTable(tableName);
+ assertTrue(rs.next());
+ assertEquals("a", rs.getString(1));
+ assertEquals(3, rs.getDouble(2), 1e-6);
+ assertFalse(rs.next());
+ ExplainPlan plan = conn.prepareStatement(queryBuilder.build())
+ .unwrap(PhoenixPreparedStatement.class).optimizeQuery()
+ .getExplainPlan();
+ ExplainPlanAttributes explainPlanAttributes =
+ plan.getPlanStepsAsAttributes();
+ assertEquals("PARALLEL 1-WAY",
+ explainPlanAttributes.getIteratorTypeAndScanSize());
+ assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType());
+ assertEquals(tableName, explainPlanAttributes.getTableName());
+ assertEquals("SERVER FILTER BY FIRST KEY ONLY",
+ explainPlanAttributes.getServerWhereFilter());
+ assertEquals("SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]",
+ explainPlanAttributes.getServerAggregate());
+ TestUtil.analyzeTable(conn, tableName);
+ List<KeyRange> splits = TestUtil.getAllSplits(conn, tableName);
+ // nGuideposts when stats are enabled, 4 when disabled
+ assertEquals(4, splits.size());
+ }
+
+ @Test
+ public void testDistinctGroupByBug3452WithoutMultiTenant() throws Exception {
+ doTestDistinctGroupByBug3452("");
+ }
+
+ @Test
+ public void testDistinctGroupByBug3452WithMultiTenant() throws Exception {
+ doTestDistinctGroupByBug3452(
+ "VERSIONS=1, MULTI_TENANT=TRUE, REPLICATION_SCOPE=1, TTL=31536000");
+ }
+
+ private void doTestDistinctGroupByBug3452(String options) throws Exception {
+ Connection conn = null;
+ try {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ conn = DriverManager.getConnection(getUrl(), props);
+
+ String tableName = generateUniqueName();
+ TABLE_NAMES.add(tableName);
+ conn.createStatement().execute("DROP TABLE if exists " + tableName);
+ String sql = "CREATE TABLE " + tableName + " ( " +
+ "ORGANIZATION_ID CHAR(15) NOT NULL," +
+ "CONTAINER_ID CHAR(15) NOT NULL," +
+ "ENTITY_ID CHAR(15) NOT NULL," +
+ "SCORE DOUBLE," +
+ "CONSTRAINT TEST_PK PRIMARY KEY ( " +
+ "ORGANIZATION_ID," +
+ "CONTAINER_ID," +
+ "ENTITY_ID" +
+ ")) " + options;
+ conn.createStatement().execute(sql);
+
+ String indexTableName = generateUniqueName();
+ conn.createStatement()
+ .execute("DROP INDEX IF EXISTS " + indexTableName + " ON " + tableName);
+ conn.createStatement().execute("CREATE INDEX " + indexTableName + " ON " + tableName +
+ " (CONTAINER_ID, SCORE DESC, ENTITY_ID DESC)");
+
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES ('org1','container1','entityId6',1.1)");
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES ('org1','container1','entityId5',1.2)");
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES ('org1','container1','entityId4',1.3)");
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES ('org1','container1','entityId3',1.4)");
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES ('org1','container1','entityId2',1.5)");
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES ('org1','container1','entityId1',1.6)");
+ conn.commit();
+
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setDistinct(true)
+ .setSelectColumns(Lists.newArrayList("ENTITY_ID", "SCORE", "ORGANIZATION_ID",
+ "CONTAINER_ID"))
+ .setFullTableName(tableName)
+ .setWhereClause("ORGANIZATION_ID = 'org1' AND CONTAINER_ID = 'container1'")
+ .setOrderByClause("SCORE DESC");
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ moveRegionsOfTable(indexTableName);
+ assertEquals("entityId1", rs.getString(1));
+ assertEquals(rs.getDouble(2), 1.6, 0.0001);
+
+ assertTrue(rs.next());
+ assertEquals("entityId2", rs.getString(1));
+ assertEquals(rs.getDouble(2), 1.5, 0.0001);
+
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ moveRegionsOfTable(indexTableName);
+ assertEquals("entityId3", rs.getString(1));
+ assertEquals(rs.getDouble(2), 1.4, 0.0001);
+
+ assertTrue(rs.next());
+ assertEquals("entityId4", rs.getString(1));
+ assertEquals(rs.getDouble(2), 1.3, 0.0001);
+
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ moveRegionsOfTable(indexTableName);
+ assertEquals("entityId5", rs.getString(1));
+ assertEquals(rs.getDouble(2), 1.2, 0.0001);
+
+ assertTrue(rs.next());
+ assertEquals("entityId6", rs.getString(1));
+ assertEquals(rs.getDouble(2), 1.1, 0.0001);
+ assertFalse(rs.next());
+ } finally {
+ if (conn != null) {
+ conn.close();
+ }
+ }
+ }
+
+ protected static void assertResultSet(ResultSet rs, Object[][] rows, String tableName)
+ throws Exception {
+ boolean switchToRegionMove = true;
+ for (int rowIndex = 0; rowIndex < rows.length; rowIndex++) {
+ assertTrue("rowIndex:[" + rowIndex + "] rs.next error!", rs.next());
+ if (switchToRegionMove) {
+ moveRegionsOfTable(tableName);
+ switchToRegionMove = false;
+ } else {
+ switchToRegionMove = true;
+ }
+ for (int columnIndex = 1; columnIndex <= rows[rowIndex].length; columnIndex++) {
+ Object realValue = rs.getObject(columnIndex);
+ Object expectedValue = rows[rowIndex][columnIndex - 1];
+ if (realValue == null) {
+ assertNull("rowIndex:[" + rowIndex + "],columnIndex:[" + columnIndex + "]",
+ expectedValue);
+ } else {
+ assertEquals("rowIndex:[" + rowIndex + "],columnIndex:[" + columnIndex + "]",
+ expectedValue,
+ realValue
+ );
+ }
+ }
+ }
+ assertFalse(rs.next());
+ }
+
+ @Test
+ public void testGroupByDescColumnWithNullsLastBug3452() throws Exception {
+
+ Connection conn = null;
+ try {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ conn = DriverManager.getConnection(getUrl(), props);
+
+ String tableName = generateUniqueName();
+ TABLE_NAMES.add(tableName);
+
+ String sql = "CREATE TABLE " + tableName + " ( " +
+ "ORGANIZATION_ID VARCHAR," +
+ "CONTAINER_ID VARCHAR," +
+ "ENTITY_ID VARCHAR NOT NULL," +
+ "CONSTRAINT TEST_PK PRIMARY KEY ( " +
+ "ORGANIZATION_ID DESC," +
+ "CONTAINER_ID DESC," +
+ "ENTITY_ID" +
+ "))";
+ conn.createStatement().execute(sql);
+
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES ('a',null,'11')");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES (null,'2','22')");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES ('c','3','33')");
+ conn.commit();
+
+ //-----ORGANIZATION_ID
+
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectColumns(
+ Lists.newArrayList("CONTAINER_ID", "ORGANIZATION_ID"))
+ .setFullTableName(tableName)
+ .setGroupByClause("ORGANIZATION_ID, CONTAINER_ID")
+ .setOrderByClause("ORGANIZATION_ID ASC NULLS FIRST");
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2", null}, {null, "a"}, {"3", "c"},}, tableName);
+
+ queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null, "a"}, {"3", "c"}, {"2", null}}, tableName);
+
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2", null}, {"3", "c"}, {null, "a"}}, tableName);
+
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3", "c"}, {null, "a"}, {"2", null}}, tableName);
+
+ //----CONTAINER_ID
+
+ queryBuilder.setOrderByClause("CONTAINER_ID ASC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null, "a"}, {"2", null}, {"3", "c"}}, tableName);
+
+ queryBuilder.setOrderByClause("CONTAINER_ID ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2", null}, {"3", "c"}, {null, "a"}}, tableName);
+
+ queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null, "a"}, {"3", "c"}, {"2", null}}, tableName);
+
+ queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3", "c"}, {"2", null}, {null, "a"}}, tableName);
+
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES (null,null,'44')");
+ conn.commit();
+
+ //-----ORGANIZATION_ID ASC CONTAINER_ID ASC
+
+ queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS FIRST,CONTAINER_ID NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null, null}, {"2", null}, {null, "a"},
+ {"3", "c"}}, tableName);
+
+ queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS FIRST,CONTAINER_ID NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2", null}, {null, null}, {null, "a"},
+ {"3", "c"}}, tableName);
+
+ queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS LAST,CONTAINER_ID NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null, "a"}, {"3", "c"}, {null, null},
+ {"2", null}}, tableName);
+
+ queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS LAST,CONTAINER_ID NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null, "a"}, {"3", "c"}, {"2", null},
+ {null, null}}, tableName);
+
+ //-----ORGANIZATION_ID ASC CONTAINER_ID DESC
+
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID NULLS FIRST,CONTAINER_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null, null}, {"2", null}, {null, "a"},
+ {"3", "c"}}, tableName);
+
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID NULLS FIRST,CONTAINER_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2", null}, {null, null}, {null, "a"},
+ {"3", "c"}}, tableName);
+
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID NULLS LAST,CONTAINER_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null, "a"}, {"3", "c"}, {null, null},
+ {"2", null}}, tableName);
+
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID NULLS LAST,CONTAINER_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null, "a"}, {"3", "c"}, {"2", null},
+ {null, null}}, tableName);
+
+ //-----ORGANIZATION_ID DESC CONTAINER_ID ASC
+
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null, null}, {"2", null}, {"3", "c"},
+ {null, "a"}}, tableName);
+
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2", null}, {null, null}, {"3", "c"},
+ {null, "a"}}, tableName);
+
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3", "c"}, {null, "a"}, {null, null},
+ {"2", null}}, tableName);
+
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3", "c"}, {null, "a"}, {"2", null},
+ {null, null}}, tableName);
+
+ //-----ORGANIZATION_ID DESC CONTAINER_ID DESC
+
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null, null}, {"2", null}, {"3", "c"},
+ {null, "a"}}, tableName);
+
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2", null}, {null, null}, {"3", "c"},
+ {null, "a"}}, tableName);
+
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3", "c"}, {null, "a"}, {null, null},
+ {"2", null}}, tableName);
+
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3", "c"}, {null, "a"}, {"2", null},
+ {null, null}}, tableName);
+
+ //-----CONTAINER_ID ASC ORGANIZATION_ID ASC
+
+ queryBuilder.setOrderByClause("CONTAINER_ID NULLS FIRST,ORGANIZATION_ID NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null, null}, {null, "a"}, {"2", null},
+ {"3", "c"}}, tableName);
+
+ queryBuilder.setOrderByClause("CONTAINER_ID NULLS FIRST,ORGANIZATION_ID NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null, "a"}, {null, null}, {"2", null},
+ {"3", "c"}}, tableName);
+
+ queryBuilder.setOrderByClause("CONTAINER_ID NULLS LAST,ORGANIZATION_ID NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2", null}, {"3", "c"}, {null, null},
+ {null, "a"}}, tableName);
+
+ queryBuilder.setOrderByClause("CONTAINER_ID NULLS LAST,ORGANIZATION_ID NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2", null}, {"3", "c"}, {null, "a"},
+ {null, null}}, tableName);
+
+ //-----CONTAINER_ID ASC ORGANIZATION_ID DESC
+
+ queryBuilder.setOrderByClause(
+ "CONTAINER_ID ASC NULLS FIRST,ORGANIZATION_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null, null}, {null, "a"}, {"2", null},
+ {"3", "c"}}, tableName);
+
+ queryBuilder.setOrderByClause(
+ "CONTAINER_ID ASC NULLS FIRST,ORGANIZATION_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null, "a"}, {null, null}, {"2", null},
+ {"3", "c"}}, tableName);
+
+ queryBuilder.setOrderByClause(
+ "CONTAINER_ID ASC NULLS LAST,ORGANIZATION_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2", null}, {"3", "c"}, {null, null},
+ {null, "a"}}, tableName);
+
+ queryBuilder.setOrderByClause(
+ "CONTAINER_ID ASC NULLS LAST,ORGANIZATION_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2", null}, {"3", "c"}, {null, "a"},
+ {null, null}}, tableName);
+
+ //-----CONTAINER_ID DESC ORGANIZATION_ID ASC
+
+ queryBuilder.setOrderByClause(
+ "CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID ASC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null, null}, {null, "a"}, {"3", "c"},
+ {"2", null}}, tableName);
+
+ queryBuilder.setOrderByClause(
+ "CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null, "a"}, {null, null}, {"3", "c"},
+ {"2", null}}, tableName);
+
+ queryBuilder.setOrderByClause(
+ "CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID ASC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3", "c"}, {"2", null}, {null, null},
+ {null, "a"}}, tableName);
+
+ queryBuilder.setOrderByClause(
+ "CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3", "c"}, {"2", null}, {null, "a"},
+ {null, null}}, tableName);
+
+ //-----CONTAINER_ID DESC ORGANIZATION_ID DESC
+
+ queryBuilder.setOrderByClause(
+ "CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null, null}, {null, "a"}, {"3", "c"},
+ {"2", null}}, tableName);
+
+ queryBuilder.setOrderByClause(
+ "CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null, "a"}, {null, null}, {"3", "c"},
+ {"2", null}}, tableName);
+
+ queryBuilder.setOrderByClause(
+ "CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3", "c"}, {"2", null}, {null, null},
+ {null, "a"}}, tableName);
+
+ queryBuilder.setOrderByClause(
+ "CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3", "c"}, {"2", null}, {null, "a"},
+ {null, null}}, tableName);
+ } finally {
+ if (conn != null) {
+ conn.close();
+ }
+ }
+ }
+
+ @Test
+ public void testCountNullInEncodedNonEmptyKeyValueCF() throws Exception {
+ testCountNullInNonEmptyKeyValueCF(1);
+ }
+
+ @Test
+ public void testCountNullInNonEncodedNonEmptyKeyValueCF() throws Exception {
+ testCountNullInNonEmptyKeyValueCF(0);
+ }
+
+ protected abstract void testCountNullInNonEmptyKeyValueCF(int columnEncodedBytes)
+ throws Exception;
+
+ @Test
+ public void testGroupByOrderMatchPkColumnOrderBug46901() throws Exception {
+ this.doTestGroupByOrderMatchPkColumnOrderBug4690(false, false);
+ this.doTestGroupByOrderMatchPkColumnOrderBug4690(false, true);
+ this.doTestGroupByOrderMatchPkColumnOrderBug4690(true, false);
+ this.doTestGroupByOrderMatchPkColumnOrderBug4690(true, true);
+ }
+
+ private void doTestGroupByOrderMatchPkColumnOrderBug4690(boolean desc, boolean salted)
+ throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String tableName = generateUniqueName();
+ String sql = "create table " + tableName + "( " +
+ " pk1 integer not null , " +
+ " pk2 integer not null, " +
+ " pk3 integer not null," +
+ " pk4 integer not null," +
+ " v integer, " +
+ " CONSTRAINT TEST_PK PRIMARY KEY ( " +
+ "pk1 " + (desc ? "desc" : "") + ", " +
+ "pk2 " + (desc ? "desc" : "") + ", " +
+ "pk3 " + (desc ? "desc" : "") + ", " +
+ "pk4 " + (desc ? "desc" : "") +
+ " )) " + (salted ? "SALT_BUCKETS =4" : "split on(2)");
+ conn.createStatement().execute(sql);
+
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES (1,8,10,20,30)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES (1,8,11,21,31)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES (1,9,5 ,22,32)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES (1,9,6 ,12,33)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES (1,9,6 ,13,34)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES (1,9,7 ,8,35)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES (2,3,15,25,35)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES (2,7,16,26,36)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES (2,7,17,27,37)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES (3,2,18,28,38)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES (3,2,19,29,39)");
+ conn.commit();
+
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectExpression("PK2,PK1,COUNT(V)")
+ .setSelectExpressionColumns(Lists.newArrayList("PK1", "PK2", "V"))
+ .setFullTableName(tableName)
+ .setGroupByClause("PK2, PK1")
+ .setOrderByClause("PK2, PK1");
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{2, 3, 2L}, {3, 2, 1L}, {7, 2, 2L}, {8, 1, 2L}, {9, 1, 4L}},
+ tableName);
+
+ queryBuilder.setSelectExpression("PK1, PK2, COUNT(V)");
+ queryBuilder.setOrderByClause("PK1, PK2");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{1, 8, 2L}, {1, 9, 4L}, {2, 3, 1L}, {2, 7, 2L}, {3, 2, 2L}},
+ tableName);
+
+ queryBuilder.setSelectExpression("PK2,PK1,COUNT(V)");
+ queryBuilder.setOrderByClause("PK2 DESC,PK1 DESC");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{9, 1, 4L}, {8, 1, 2L}, {7, 2, 2L}, {3, 2, 1L}, {2, 3, 2L}},
+ tableName);
+
+ queryBuilder.setSelectExpression("PK1,PK2,COUNT(V)");
+ queryBuilder.setOrderByClause("PK1 DESC,PK2 DESC");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{3, 2, 2L}, {2, 7, 2L}, {2, 3, 1L}, {1, 9, 4L}, {1, 8, 2L}},
+ tableName);
+
+ queryBuilder.setSelectExpression("PK3,PK2,COUNT(V)");
+ queryBuilder.setSelectExpressionColumns(Lists.newArrayList("PK1", "PK2", "PK3", "V"));
+ queryBuilder.setFullTableName(tableName);
+ queryBuilder.setGroupByClause("PK3,PK2");
+ queryBuilder.setOrderByClause("PK3,PK2");
+ queryBuilder.setWhereClause("PK1=1");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{5, 9, 1L}, {6, 9, 2L}, {7, 9, 1L}, {10, 8, 1L}, {11, 8, 1L}},
+ tableName);
+
+ queryBuilder.setSelectExpression("PK2,PK3,COUNT(V)");
+ queryBuilder.setOrderByClause("PK2,PK3");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{8, 10, 1L}, {8, 11, 1L}, {9, 5, 1L}, {9, 6, 2L}, {9, 7, 1L}},
+ tableName);
+
+ queryBuilder.setSelectExpression("PK3,PK2,COUNT(V)");
+ queryBuilder.setOrderByClause("PK3 DESC,PK2 DESC");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{11, 8, 1L}, {10, 8, 1L}, {7, 9, 1L}, {6, 9, 2L}, {5, 9, 1L}},
+ tableName);
+
+ queryBuilder.setSelectExpression("PK2,PK3,COUNT(V)");
+ queryBuilder.setOrderByClause("PK2 DESC,PK3 DESC");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{9, 7, 1L}, {9, 6, 2L}, {9, 5, 1L}, {8, 11, 1L}, {8, 10, 1L}},
+ tableName);
+
+
+ queryBuilder.setSelectExpression("PK4,PK3,PK1,COUNT(V)");
+ queryBuilder.setSelectExpressionColumns(
+ Lists.newArrayList("PK1", "PK2", "PK3", "PK4", "V"));
+ queryBuilder.setFullTableName(tableName);
+ queryBuilder.setWhereClause("PK2=9 ");
+ queryBuilder.setGroupByClause("PK4,PK3,PK1");
+ queryBuilder.setOrderByClause("PK4,PK3,PK1");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{8, 7, 1, 1L}, {12, 6, 1, 1L}, {13, 6, 1, 1L}, {22, 5, 1, 1L}}
+ , tableName);
+
+ queryBuilder.setSelectExpression("PK1,PK3,PK4,COUNT(V)");
+ queryBuilder.setOrderByClause("PK1,PK3,PK4");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{1, 5, 22, 1L}, {1, 6, 12, 1L}, {1, 6, 13, 1L}, {1, 7, 8, 1L}},
+ tableName);
+
+ queryBuilder.setSelectExpression("PK4,PK3,PK1,COUNT(V)");
+ queryBuilder.setOrderByClause("PK4 DESC,PK3 DESC,PK1 DESC");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{22, 5, 1, 1L}, {13, 6, 1, 1L}, {12, 6, 1, 1L}, {8, 7, 1, 1L}},
+ tableName);
+
+ queryBuilder.setSelectExpression("PK1,PK3,PK4,COUNT(V)");
+ queryBuilder.setOrderByClause("PK1 DESC,PK3 DESC,PK4 DESC");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{1, 7, 8, 1L}, {1, 6, 13, 1L}, {1, 6, 12, 1L}, {1,
+ 5, 22, 1L}}, tableName);
+ }
+ }
+}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseOrderByWithRegionMovesIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseOrderByWithRegionMovesIT.java
new file mode 100644
index 0000000000..526d4f608d
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseOrderByWithRegionMovesIT.java
@@ -0,0 +1,1406 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import org.apache.phoenix.compile.ExplainPlan;
+import org.apache.phoenix.compile.ExplainPlanAttributes;
+import org.apache.phoenix.jdbc.PhoenixPreparedStatement;
+import org.apache.phoenix.thirdparty.com.google.common.collect.Lists;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.QueryBuilder;
+import org.apache.phoenix.util.SchemaUtil;
+import org.apache.phoenix.util.TestUtil;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Properties;
+
+import static org.apache.phoenix.util.TestUtil.ROW1;
+import static org.apache.phoenix.util.TestUtil.ROW2;
+import static org.apache.phoenix.util.TestUtil.ROW3;
+import static org.apache.phoenix.util.TestUtil.ROW4;
+import static org.apache.phoenix.util.TestUtil.ROW5;
+import static org.apache.phoenix.util.TestUtil.ROW6;
+import static org.apache.phoenix.util.TestUtil.ROW7;
+import static org.apache.phoenix.util.TestUtil.ROW8;
+import static org.apache.phoenix.util.TestUtil.ROW9;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.apache.phoenix.util.TestUtil.assertResultSet;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+
+public abstract class BaseOrderByWithRegionMovesIT extends ParallelStatsDisabledWithRegionMovesIT {
+
+ @Before
+ public void setUp() throws Exception {
+ hasTestStarted = true;
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ countOfDummyResults = 0;
+ TABLE_NAMES.clear();
+ hasTestStarted = false;
+ }
+
+ @Test
+ public void testMultiOrderByExpr() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ String tenantId = getOrganizationId();
+ String tableName = initATableValues(tenantId, getDefaultSplits(tenantId), getUrl());
+ TABLE_NAMES.add(tableName);
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectColumns(
+ Lists.newArrayList("ENTITY_ID", "B_STRING"))
+ .setFullTableName(tableName)
+ .setOrderByClause("B_STRING, ENTITY_ID");
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals(ROW1, rs.getString(1));
+ assertTrue(rs.next());
+ assertEquals(ROW4, rs.getString(1));
+ assertTrue(rs.next());
+ assertEquals(ROW7, rs.getString(1));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals(ROW2, rs.getString(1));
+ assertTrue(rs.next());
+ assertEquals(ROW5, rs.getString(1));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals(ROW8, rs.getString(1));
+ assertTrue(rs.next());
+ assertEquals(ROW3, rs.getString(1));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals(ROW6, rs.getString(1));
+ assertTrue(rs.next());
+ assertEquals(ROW9, rs.getString(1));
+
+ assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testDescMultiOrderByExpr() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ String tenantId = getOrganizationId();
+ String tableName = initATableValues(tenantId, null, getUrl());
+ TABLE_NAMES.add(tableName);
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectColumns(
+ Lists.newArrayList("ENTITY_ID", "B_STRING"))
+ .setFullTableName(tableName)
+ .setOrderByClause("B_STRING || ENTITY_ID DESC");
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals(ROW9, rs.getString(1));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals(ROW6, rs.getString(1));
+ assertTrue(rs.next());
+ assertEquals(ROW3, rs.getString(1));
+ assertTrue(rs.next());
+ assertEquals(ROW8, rs.getString(1));
+ assertTrue(rs.next());
+ assertEquals(ROW5, rs.getString(1));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals(ROW2, rs.getString(1));
+ assertTrue(rs.next());
+ assertEquals(ROW7, rs.getString(1));
+ assertTrue(rs.next());
+ assertEquals(ROW4, rs.getString(1));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals(ROW1, rs.getString(1));
+ assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testDescMultiOrderByExprWithSplits() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ String tenantId = getOrganizationId();
+ String tableName = initATableValues(tenantId, getDefaultSplits(tenantId), getUrl());
+ TABLE_NAMES.add(tableName);
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectColumns(
+ Lists.newArrayList("ENTITY_ID", "B_STRING"))
+ .setFullTableName(tableName)
+ .setOrderByClause("B_STRING || ENTITY_ID DESC");
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals(ROW9, rs.getString(1));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals(ROW6, rs.getString(1));
+ assertTrue(rs.next());
+ assertEquals(ROW3, rs.getString(1));
+ assertTrue(rs.next());
+ assertEquals(ROW8, rs.getString(1));
+ assertTrue(rs.next());
+ assertEquals(ROW5, rs.getString(1));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals(ROW2, rs.getString(1));
+ assertTrue(rs.next());
+ assertEquals(ROW7, rs.getString(1));
+ assertTrue(rs.next());
+ assertEquals(ROW4, rs.getString(1));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals(ROW1, rs.getString(1));
+ assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testOrderByDifferentColumns() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ conn.setAutoCommit(false);
+ String tableName = generateUniqueName();
+ TABLE_NAMES.add(tableName);
+ String ddl = "CREATE TABLE " + tableName +
+ " (a_string varchar not null, col1 integer" +
+ " CONSTRAINT pk PRIMARY KEY (a_string))\n";
+ createTestTable(getUrl(), ddl);
+
+ String dml = "UPSERT INTO " + tableName + " VALUES(?, ?)";
+ PreparedStatement stmt = conn.prepareStatement(dml);
+ stmt.setString(1, "a");
+ stmt.setInt(2, 40);
+ stmt.execute();
+ stmt.setString(1, "b");
+ stmt.setInt(2, 20);
+ stmt.execute();
+ stmt.setString(1, "c");
+ stmt.setInt(2, 30);
+ stmt.execute();
+ conn.commit();
+
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectColumns(
+ Lists.newArrayList("COL1"))
+ .setFullTableName(tableName)
+ .setSelectExpression("count(*)")
+ .setGroupByClause("COL1")
+ .setOrderByClause("COL1");
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals(1, rs.getLong(1));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals(1, rs.getLong(1));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals(1, rs.getLong(1));
+ assertFalse(rs.next());
+
+ queryBuilder = new QueryBuilder();
+ queryBuilder.setSelectColumns(
+ Lists.newArrayList("A_STRING", "COL1"));
+ queryBuilder.setFullTableName(tableName);
+ queryBuilder.setOrderByClause("A_STRING");
+ rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("a", rs.getString(1));
+ assertEquals(40, rs.getInt(2));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("b", rs.getString(1));
+ assertEquals(20, rs.getInt(2));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("c", rs.getString(1));
+ assertEquals(30, rs.getInt(2));
+ assertFalse(rs.next());
+
+ queryBuilder.setSelectColumns(
+ Lists.newArrayList("A_STRING", "COL1"));
+ queryBuilder.setFullTableName(tableName);
+ queryBuilder.setOrderByClause("COL1");
+ rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("b", rs.getString(1));
+ assertEquals(20, rs.getInt(2));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("c", rs.getString(1));
+ assertEquals(30, rs.getInt(2));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("a", rs.getString(1));
+ assertEquals(40, rs.getInt(2));
+ assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testAggregateOrderBy() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ TABLE_NAMES.add(tableName);
+ String ddl = "create table " + tableName +
+ " (ID VARCHAR NOT NULL PRIMARY KEY, VAL1 VARCHAR, VAL2 INTEGER)";
+ conn.createStatement().execute(ddl);
+
+ conn.createStatement().execute("upsert into " + tableName + " values ('ABC','aa123', 11)");
+ conn.createStatement().execute("upsert into " + tableName + " values ('ABD','ba124', 1)");
+ conn.createStatement().execute("upsert into " + tableName + " values ('ABE','cf125', 13)");
+ conn.createStatement().execute("upsert into " + tableName + " values ('ABF','dan126', 4)");
+ conn.createStatement().execute("upsert into " + tableName + " values ('ABG','elf127', 15)");
+ conn.createStatement().execute("upsert into " + tableName + " values ('ABH','fan128', 6)");
+ conn.createStatement()
+ .execute("upsert into " + tableName + " values ('AAA','get211', 100)");
+ conn.createStatement().execute("upsert into " + tableName + " values ('AAB','hat212', 7)");
+ conn.createStatement().execute("upsert into " + tableName + " values ('AAC','aap12', 2)");
+ conn.createStatement().execute("upsert into " + tableName + " values ('AAD','ball12', 3)");
+ conn.createStatement()
+ .execute("upsert into " + tableName + " values ('AAE','inn2110', 13)");
+ conn.createStatement()
+ .execute("upsert into " + tableName + " values ('AAF','key2112', 40)");
+ conn.commit();
+
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectColumns(
+ Lists.newArrayList("VAL1", "VAL2"))
+ .setFullTableName(tableName)
+ .setOrderByClause("VAL1")
+ .setSelectExpression("DISTINCT(ID)")
+ .setSelectExpressionColumns(Lists.newArrayList("ID"))
+ .setWhereClause(
+ "ID in ('ABC','ABD','ABE','ABF','ABG','ABH','AAA', 'AAB', 'AAC','AAD','AAE','AAF')");
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("ABC", rs.getString(1));
+ assertEquals("aa123", rs.getString(2));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("aap12", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("ba124", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("ball12", rs.getString(2));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("cf125", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("dan126", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("elf127", rs.getString(2));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("fan128", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("get211", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("hat212", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("inn2110", rs.getString(2));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("AAF", rs.getString(1));
+ assertEquals("key2112", rs.getString(2));
+ assertFalse(rs.next());
+ }
+
+ @Test
+ public void testAggregateOptimizedOutOrderBy() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ TABLE_NAMES.add(tableName);
+ String ddl = "create table " + tableName +
+ " (K1 VARCHAR NOT NULL, K2 VARCHAR NOT NULL, VAL1 VARCHAR, VAL2 INTEGER, CONSTRAINT pk PRIMARY KEY(K1,K2))";
+ conn.createStatement().execute(ddl);
+
+ conn.createStatement()
+ .execute("upsert into " + tableName + " values ('ABC','ABC','aa123', 11)");
+ conn.createStatement()
+ .execute("upsert into " + tableName + " values ('ABD','ABC','ba124', 1)");
+ conn.createStatement()
+ .execute("upsert into " + tableName + " values ('ABE','ABC','cf125', 13)");
+ conn.createStatement()
+ .execute("upsert into " + tableName + " values ('ABF','ABC','dan126', 4)");
+ conn.createStatement()
+ .execute("upsert into " + tableName + " values ('ABG','ABC','elf127', 15)");
+ conn.createStatement()
+ .execute("upsert into " + tableName + " values ('ABH','ABC','fan128', 6)");
+ conn.createStatement()
+ .execute("upsert into " + tableName + " values ('AAA','ABC','get211', 100)");
+ conn.createStatement()
+ .execute("upsert into " + tableName + " values ('AAB','ABC','hat212', 7)");
+ conn.createStatement()
+ .execute("upsert into " + tableName + " values ('AAC','ABC','aap12', 2)");
+ conn.createStatement()
+ .execute("upsert into " + tableName + " values ('AAD','ABC','ball12', 3)");
+ conn.createStatement()
+ .execute("upsert into " + tableName + " values ('AAE','ABC','inn2110', 13)");
+ conn.createStatement()
+ .execute("upsert into " + tableName + " values ('AAF','ABC','key2112', 40)");
+ conn.commit();
+
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectColumns(
+ Lists.newArrayList("VAL1", "VAL2"))
+ .setFullTableName(tableName)
+ .setOrderByClause("VAL1")
+ .setSelectExpressionColumns(Lists.newArrayList("K2"))
+ .setSelectExpression("DISTINCT(K2)")
+ .setWhereClause("K2 = 'ABC'");
+
+ // verify that the phoenix query plan doesn't contain an order by
+ ExplainPlan plan = conn.prepareStatement(queryBuilder.build())
+ .unwrap(PhoenixPreparedStatement.class).optimizeQuery()
+ .getExplainPlan();
+ ExplainPlanAttributes explainPlanAttributes =
+ plan.getPlanStepsAsAttributes();
+ assertEquals("PARALLEL 1-WAY",
+ explainPlanAttributes.getIteratorTypeAndScanSize());
+ assertEquals("FULL SCAN ", explainPlanAttributes.getExplainScanType());
+ assertEquals(tableName, explainPlanAttributes.getTableName());
+ assertEquals("SERVER FILTER BY K2 = 'ABC'",
+ explainPlanAttributes.getServerWhereFilter());
+ assertEquals("SERVER AGGREGATE INTO DISTINCT ROWS BY [K2, VAL1, VAL2]",
+ explainPlanAttributes.getServerAggregate());
+ assertEquals("CLIENT MERGE SORT",
+ explainPlanAttributes.getClientSortAlgo());
+ assertNull(explainPlanAttributes.getClientSortedBy());
+ assertNull(explainPlanAttributes.getServerSortedBy());
+
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("ABC", rs.getString(1));
+ assertEquals("aa123", rs.getString(2));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("aap12", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("ba124", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("ball12", rs.getString(2));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("cf125", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("dan126", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("elf127", rs.getString(2));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("fan128", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("get211", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("hat212", rs.getString(2));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("inn2110", rs.getString(2));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("ABC", rs.getString(1));
+ assertEquals("key2112", rs.getString(2));
+ assertFalse(rs.next());
+ }
+
+ @Test
+ public void testNullsLastWithDesc() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String tableName = generateUniqueName();
+ TABLE_NAMES.add(tableName);
+ String sql = "CREATE TABLE " + tableName + " ( " +
+ "ORGANIZATION_ID VARCHAR," +
+ "CONTAINER_ID VARCHAR," +
+ "ENTITY_ID VARCHAR NOT NULL," +
+ "CONSTRAINT TEST_PK PRIMARY KEY ( " +
+ "ORGANIZATION_ID DESC," +
+ "CONTAINER_ID DESC," +
+ "ENTITY_ID" +
+ "))";
+ conn.createStatement().execute(sql);
+
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES ('a',null,'11')");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES (null,'2','22')");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES ('c','3','33')");
+ conn.commit();
+
+ //-----ORGANIZATION_ID
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID ASC NULLS FIRST";
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectColumns(
+ Lists.newArrayList("CONTAINER_ID", "ORGANIZATION_ID"))
+ .setFullTableName(tableName)
+ .setOrderByClause("ORGANIZATION_ID ASC NULLS FIRST");
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertResultSetWithRegionMoves(rs,
+ new String[][]{{"2", null}, {null, "a"}, {"3", "c"},}, tableName);
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID ASC NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSetWithRegionMoves(rs, new String[][]{{null, "a"}, {"3", "c"}, {"2", null}},
+ tableName);
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS FIRST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSetWithRegionMoves(rs, new String[][]{{"2", null}, {"3", "c"}, {null, "a"}},
+ tableName);
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSetWithRegionMoves(rs, new String[][]{{"3", "c"}, {null, "a"}, {"2", null}},
+ tableName);
+
+ //----CONTAINER_ID
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID ASC NULLS FIRST";
+ queryBuilder.setOrderByClause("CONTAINER_ID ASC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null, "a"}, {"2", null}, {"3", "c"}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID ASC NULLS LAST";
+ queryBuilder.setOrderByClause("CONTAINER_ID ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2", null}, {"3", "c"}, {null, "a"}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS FIRST";
+ queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSetWithRegionMoves(rs, new String[][]{{null, "a"}, {"3", "c"}, {"2", null}},
+ tableName);
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS LAST";
+ queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3", "c"}, {"2", null}, {null, "a"}});
+
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES (null,null,'44')");
+ conn.commit();
+
+ //-----ORGANIZATION_ID ASC CONTAINER_ID ASC
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS FIRST,CONTAINER_ID NULLS FIRST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS FIRST,CONTAINER_ID NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSetWithRegionMoves(rs,
+ new String[][]{{null, null}, {"2", null}, {null, "a"},
+ {"3", "c"}}, tableName);
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS FIRST,CONTAINER_ID NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS FIRST,CONTAINER_ID NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2", null}, {null, null}, {null, "a"},
+ {"3", "c"}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS LAST,CONTAINER_ID NULLS FIRST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS LAST,CONTAINER_ID NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null, "a"}, {"3", "c"}, {null, null},
+ {"2", null}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS LAST,CONTAINER_ID NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS LAST,CONTAINER_ID NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null, "a"}, {"3", "c"}, {"2", null},
+ {null, null}});
+
+
+ //-----ORGANIZATION_ID ASC CONTAINER_ID DESC
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS FIRST,CONTAINER_ID DESC NULLS FIRST";
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID NULLS FIRST,CONTAINER_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null, null}, {"2", null}, {null, "a"},
+ {"3", "c"}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS FIRST,CONTAINER_ID DESC NULLS LAST";
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID NULLS FIRST,CONTAINER_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSetWithRegionMoves(rs,
+ new String[][]{{"2", null}, {null, null}, {null, "a"},
+ {"3", "c"}}, tableName);
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS LAST,CONTAINER_ID DESC NULLS FIRST";
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID NULLS LAST,CONTAINER_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSetWithRegionMoves(rs, new String[][]{{null, "a"}, {"3", "c"}, {null, null},
+ {"2", null}}, tableName);
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS LAST,CONTAINER_ID DESC NULLS LAST";
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID NULLS LAST,CONTAINER_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null, "a"}, {"3", "c"}, {"2", null},
+ {null, null}});
+
+ //-----ORGANIZATION_ID DESC CONTAINER_ID ASC
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID NULLS FIRST";
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null, null}, {"2", null}, {"3", "c"},
+ {null, "a"}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID NULLS LAST";
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2", null}, {null, null}, {"3", "c"},
+ {null, "a"}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID NULLS FIRST";
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSetWithRegionMoves(rs, new String[][]{{"3", "c"}, {null, "a"}, {null, null},
+ {"2", null}}, tableName);
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID NULLS LAST";
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSetWithRegionMoves(rs, new String[][]{{"3", "c"}, {null, "a"}, {"2", null},
+ {null, null}}, tableName);
+
+ //-----ORGANIZATION_ID DESC CONTAINER_ID DESC
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID DESC NULLS FIRST";
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null, null}, {"2", null}, {"3", "c"},
+ {null, "a"}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID DESC NULLS LAST";
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2", null}, {null, null}, {"3", "c"},
+ {null, "a"}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID DESC NULLS FIRST";
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3", "c"}, {null, "a"}, {null, null},
+ {"2", null}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID DESC NULLS LAST";
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3", "c"}, {null, "a"}, {"2", null}, {null, null}});
+
+ //-----CONTAINER_ID ASC ORGANIZATION_ID ASC
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID NULLS FIRST,ORGANIZATION_ID NULLS FIRST";
+ queryBuilder.setOrderByClause("CONTAINER_ID NULLS FIRST,ORGANIZATION_ID NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null, null}, {null, "a"}, {"2", null}, {"3", "c"}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID NULLS FIRST,ORGANIZATION_ID NULLS LAST";
+ queryBuilder.setOrderByClause("CONTAINER_ID NULLS FIRST,ORGANIZATION_ID NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null, "a"}, {null, null}, {"2", null}, {"3", "c"}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID NULLS LAST,ORGANIZATION_ID NULLS FIRST";
+ queryBuilder.setOrderByClause("CONTAINER_ID NULLS LAST,ORGANIZATION_ID NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSetWithRegionMoves(rs,
+ new String[][]{{"2", null}, {"3", "c"}, {null, null}, {null,
+ "a"}}, tableName);
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID NULLS LAST,ORGANIZATION_ID NULLS LAST";
+ queryBuilder.setOrderByClause("CONTAINER_ID NULLS LAST,ORGANIZATION_ID NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2", null}, {"3", "c"}, {null, "a"}, {null, null}});
+
+ //-----CONTAINER_ID ASC ORGANIZATION_ID DESC
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID ASC NULLS FIRST,ORGANIZATION_ID DESC NULLS FIRST";
+ queryBuilder.setOrderByClause(
+ "CONTAINER_ID ASC NULLS FIRST,ORGANIZATION_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null, null}, {null, "a"}, {"2", null}, {"3", "c"}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID ASC NULLS FIRST,ORGANIZATION_ID DESC NULLS LAST";
+ queryBuilder.setOrderByClause(
+ "CONTAINER_ID ASC NULLS FIRST,ORGANIZATION_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSetWithRegionMoves(rs,
+ new String[][]{{null, "a"}, {null, null}, {"2", null}, {"3",
+ "c"}}, tableName);
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID ASC NULLS LAST,ORGANIZATION_ID DESC NULLS FIRST";
+ queryBuilder.setOrderByClause(
+ "CONTAINER_ID ASC NULLS LAST,ORGANIZATION_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2", null}, {"3", "c"}, {null, null}, {null, "a"}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID ASC NULLS LAST,ORGANIZATION_ID DESC NULLS LAST";
+ queryBuilder.setOrderByClause(
+ "CONTAINER_ID ASC NULLS LAST,ORGANIZATION_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2", null}, {"3", "c"}, {null, "a"}, {null, null}});
+
+ //-----CONTAINER_ID DESC ORGANIZATION_ID ASC
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID ASC NULLS FIRST";
+ queryBuilder.setOrderByClause(
+ "CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID ASC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null, null}, {null, "a"}, {"3", "c"}, {"2", null}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID ASC NULLS LAST";
+ queryBuilder.setOrderByClause(
+ "CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null, "a"}, {null, null}, {"3", "c"}, {"2", null}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID ASC NULLS FIRST";
+ queryBuilder.setOrderByClause(
+ "CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID ASC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3", "c"}, {"2", null}, {null, null}, {null, "a"}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID ASC NULLS LAST";
+ queryBuilder.setOrderByClause(
+ "CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3", "c"}, {"2", null}, {null, "a"}, {null, null}});
+
+ //-----CONTAINER_ID DESC ORGANIZATION_ID DESC
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID DESC NULLS FIRST";
+ queryBuilder.setOrderByClause(
+ "CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null, null}, {null, "a"}, {"3", "c"}, {"2", null}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID DESC NULLS LAST";
+ queryBuilder.setOrderByClause(
+ "CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null, "a"}, {null, null}, {"3", "c"}, {"2", null}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID DESC NULLS FIRST";
+ queryBuilder.setOrderByClause(
+ "CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3", "c"}, {"2", null}, {null, null}, {null, "a"}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID DESC NULLS LAST";
+ queryBuilder.setOrderByClause(
+ "CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSetWithRegionMoves(rs,
+ new String[][]{{"3", "c"}, {"2", null}, {null, "a"}, {null,
+ null}}, tableName);
+ }
+ }
+
+ @Test
+ public void testOrderByReverseOptimization1() throws Exception {
+ for (boolean salted : new boolean[]{true, false}) {
+ doTestOrderByReverseOptimization(salted, true, true, true);
+ doTestOrderByReverseOptimization(salted, true, true, false);
+ doTestOrderByReverseOptimization(salted, true, false, true);
+ doTestOrderByReverseOptimization(salted, true, false, false);
+ }
+ }
+
+ @Test
+ public void testOrderByReverseOptimization2() throws Exception {
+ for (boolean salted : new boolean[]{true, false}) {
+ doTestOrderByReverseOptimization(salted, false, true, true);
+ doTestOrderByReverseOptimization(salted, false, true, false);
+ doTestOrderByReverseOptimization(salted, false, false, true);
+ doTestOrderByReverseOptimization(salted, false, false, false);
+ }
+ }
+
+ private void doTestOrderByReverseOptimization(boolean salted, boolean desc1, boolean desc2,
+ boolean desc3) throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String tableName = generateUniqueName();
+ TABLE_NAMES.add(tableName);
+ String sql = "CREATE TABLE " + tableName + " ( " +
+ "ORGANIZATION_ID INTEGER NOT NULL," +
+ "CONTAINER_ID INTEGER NOT NULL," +
+ "SCORE INTEGER NOT NULL," +
+ "ENTITY_ID INTEGER NOT NULL," +
+ "CONSTRAINT TEST_PK PRIMARY KEY ( " +
+ "ORGANIZATION_ID" + (desc1 ? " DESC" : "") + "," +
+ "CONTAINER_ID" + (desc2 ? " DESC" : "") + "," +
+ "SCORE" + (desc3 ? " DESC" : "") + "," +
+ "ENTITY_ID" +
+ ")) " + (salted ? "SALT_BUCKETS =4" : "split on(4)");
+ conn.createStatement().execute(sql);
+
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES (1,1,1,1)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES (2,2,2,2)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES (3,3,3,3)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES (4,4,4,4)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES (5,5,5,5)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES (6,6,6,6)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES (1,1,1,11)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES (2,2,2,22)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES (3,3,3,33)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES (4,4,4,44)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES (5,5,5,55)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES (6,6,6,66)");
+ conn.commit();
+
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectColumns(
+ Lists.newArrayList("CONTAINER_ID", "ORGANIZATION_ID"))
+ .setFullTableName(tableName)
+ .setGroupByClause("ORGANIZATION_ID, CONTAINER_ID")
+ .setOrderByClause("ORGANIZATION_ID ASC, CONTAINER_ID ASC");
+ //groupBy orderPreserving orderBy asc asc
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC, CONTAINER_ID ASC";
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertResultSetWithRegionMoves(rs, new Object[][]{{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5,
+ 5}, {6, 6}}, tableName);
+
+ //groupBy orderPreserving orderBy asc desc
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC, CONTAINER_ID desc";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID ASC, CONTAINER_ID DESC");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}});
+
+ //groupBy orderPreserving orderBy desc asc
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC, CONTAINER_ID ASC";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC, CONTAINER_ID ASC");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{6, 6}, {5, 5}, {4, 4}, {3, 3}, {2, 2}, {1, 1}});
+
+ //groupBy orderPreserving orderBy desc desc
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC, CONTAINER_ID DESC";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC, CONTAINER_ID DESC");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{6, 6}, {5, 5}, {4, 4}, {3, 3}, {2,
+ 2}, {1, 1}});
+
+ //groupBy not orderPreserving orderBy asc asc
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC, SCORE ASC";
+ queryBuilder.setSelectColumns(
+ Lists.newArrayList("ORGANIZATION_ID", "SCORE"));
+ queryBuilder.setGroupByClause("ORGANIZATION_ID, SCORE");
+ queryBuilder.setOrderByClause("ORGANIZATION_ID ASC, SCORE ASC");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}});
+
+ //groupBy not orderPreserving orderBy asc desc
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC, SCORE DESC";
+ queryBuilder.setGroupByClause("ORGANIZATION_ID, SCORE");
+ queryBuilder.setOrderByClause("ORGANIZATION_ID ASC, SCORE DESC");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}, {6, 6}});
+
+ //groupBy not orderPreserving orderBy desc asc
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC, SCORE ASC";
+ queryBuilder.setGroupByClause("ORGANIZATION_ID, SCORE");
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC, SCORE ASC");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSetWithRegionMoves(rs, new Object[][]{{6, 6}, {5, 5}, {4, 4}, {3, 3}, {2,
+ 2}, {1, 1}}, tableName);
+
+ //groupBy not orderPreserving orderBy desc desc
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC, SCORE DESC";
+ queryBuilder.setGroupByClause("ORGANIZATION_ID, SCORE");
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC, SCORE DESC");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{6, 6}, {5, 5}, {4, 4}, {3, 3}, {2, 2}, {1, 1}});
+ }
+ }
+
+ @Test
+ public void testOrderByReverseOptimizationWithNullsLast1() throws Exception {
+ for (boolean salted : new boolean[]{true, false}) {
+ doTestOrderByReverseOptimizationWithNullsLast(salted, true, true, true);
+ doTestOrderByReverseOptimizationWithNullsLast(salted, true, true, false);
+ }
+ }
+
+ @Test
+ public void testOrderByReverseOptimizationWithNullsLast2() throws Exception {
+ for (boolean salted : new boolean[]{true, false}) {
+ doTestOrderByReverseOptimizationWithNullsLast(salted, true, false, true);
+ doTestOrderByReverseOptimizationWithNullsLast(salted, true, false, false);
+ }
+ }
+
+ @Test
+ public void testOrderByReverseOptimizationWithNullsLast3() throws Exception {
+ for (boolean salted : new boolean[]{true, false}) {
+ doTestOrderByReverseOptimizationWithNullsLast(salted, false, true, true);
+ doTestOrderByReverseOptimizationWithNullsLast(salted, false, true, false);
+ }
+ }
+
+ @Test
+ public void testOrderByReverseOptimizationWithNullsLast4() throws Exception {
+ for (boolean salted : new boolean[]{true, false}) {
+ doTestOrderByReverseOptimizationWithNullsLast(salted, false, false, true);
+ doTestOrderByReverseOptimizationWithNullsLast(salted, false, false, false);
+ }
+ }
+
+ private void doTestOrderByReverseOptimizationWithNullsLast(boolean salted, boolean desc1,
+ boolean desc2, boolean desc3)
+ throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String tableName = generateUniqueName();
+ TABLE_NAMES.add(tableName);
+ String sql = "CREATE TABLE " + tableName + " ( " +
+ "ORGANIZATION_ID VARCHAR," +
+ "CONTAINER_ID VARCHAR," +
+ "SCORE VARCHAR," +
+ "ENTITY_ID VARCHAR NOT NULL," +
+ "CONSTRAINT TEST_PK PRIMARY KEY ( " +
+ "ORGANIZATION_ID" + (desc1 ? " DESC" : "") + "," +
+ "CONTAINER_ID" + (desc2 ? " DESC" : "") + "," +
+ "SCORE" + (desc3 ? " DESC" : "") + "," +
+ "ENTITY_ID" +
+ ")) " + (salted ? "SALT_BUCKETS =4" : "split on('4')");
+ conn.createStatement().execute(sql);
+
+ for (int i = 1; i <= 6; i++) {
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES (null,'" + i + "','" + i + "','" + i +
+ "')");
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES (null,'" + i + "',null,'" + i + "')");
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES (null,null,'" + i + "','" + i + "')");
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES (null,null,null,'" + i + "')");
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES ('" + i + "','" + i + "','" + i +
+ "','" + i + "')");
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES ('" + i + "','" + i + "',null,'" + i +
+ "')");
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES ('" + i + "',null,'" + i + "','" + i +
+ "')");
+ conn.createStatement().execute(
+ "UPSERT INTO " + tableName + " VALUES ('" + i + "',null,null,'" + i + "')");
+ }
+ conn.createStatement()
+ .execute("UPSERT INTO " + tableName + " VALUES (null,null,null,'66')");
+ conn.commit();
+
+ //groupBy orderPreserving orderBy asc asc
+
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID ASC NULLS FIRST";
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectColumns(
+ Lists.newArrayList("ORGANIZATION_ID", "CONTAINER_ID"))
+ .setFullTableName(tableName)
+ .setGroupByClause("ORGANIZATION_ID, CONTAINER_ID")
+ .setOrderByClause(
+ "ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID ASC NULLS FIRST");
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertResultSetWithRegionMoves(rs,
+ new Object[][]{{null, null}, {null, "1"}, {null, "2"}, {null, "3"}, {null, "4"},
+ {null, "5"}, {null, "6"}, {"1", null}, {"1", "1"}, {"2", null},
+ {"2", "2"}, {"3", null}, {"3", "3"}, {"4", null}, {"4", "4"},
+ {"5", null}, {"5", "5"}, {"6", null}, {"6", "6"}}, tableName);
+
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID ASC NULLS LAST";
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{null, "1"}, {null, "2"}, {null, "3"}, {null, "4"}, {null, "5"},
+ {null, "6"}, {null, null}, {"1", "1"}, {"1", null}, {"2", "2"},
+ {"2", null}, {"3", "3"}, {"3", null}, {"4", "4"}, {"4", null},
+ {"5", "5"}, {"5", null}, {"6", "6"}, {"6", null}});
+
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORDER BY ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID ASC NULLS LAST";
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{null, "1"}, {null, "2"}, {null, "3"}, {null, "4"}, {null, "5"},
+ {null, "6"}, {null, null}, {"1", "1"}, {"1", null}, {"2", "2"},
+ {"2", null}, {"3", "3"}, {"3", null}, {"4", "4"}, {"4", null},
+ {"5", "5"}, {"5", null}, {"6", "6"}, {"6", null}});
+
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS LAST, CONTAINER_ID ASC NULLS LAST";
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID ASC NULLS LAST, CONTAINER_ID ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{"1", "1"}, {"1", null}, {"2", "2"}, {"2", null}, {"3", "3"},
+ {"3", null}, {"4", "4"}, {"4", null}, {"5", "5"}, {"5", null},
+ {"6", "6"}, {"6", null}, {null, "1"}, {null, "2"}, {null, "3"},
+ {null, "4"}, {null, "5"}, {null, "6"}, {null, null}});
+
+ //groupBy orderPreserving orderBy asc desc
+
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID DESC NULLS FIRST";
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{null, null}, {null, "6"}, {null, "5"}, {null, "4"}, {null, "3"},
+ {null, "2"}, {null, "1"}, {"1", null}, {"1", "1"}, {"2", null},
+ {"2", "2"}, {"3", null}, {"3", "3"}, {"4", null}, {"4", "4"},
+ {"5", null}, {"5", "5"}, {"6", null}, {"6", "6"}});
+
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID DESC NULLS LAST";
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSetWithRegionMoves(rs,
+ new Object[][]{{null, "6"}, {null, "5"}, {null, "4"}, {null, "3"}, {null, "2"},
+ {null, "1"}, {null, null}, {"1", "1"}, {"1", null}, {"2", "2"},
+ {"2", null}, {"3", "3"}, {"3", null}, {"4", "4"}, {"4", null},
+ {"5", "5"}, {"5", null}, {"6", "6"}, {"6", null}}, tableName);
+
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS LAST, CONTAINER_ID DESC NULLS FIRST";
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID ASC NULLS LAST, CONTAINER_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{"1", null}, {"1", "1"}, {"2", null}, {"2", "2"}, {"3", null},
+ {"3", "3"}, {"4", null}, {"4", "4"}, {"5", null}, {"5", "5"},
+ {"6", null}, {"6", "6"}, {null, null}, {null, "6"}, {null, "5"},
+ {null, "4"}, {null, "3"}, {null, "2"}, {null, "1"}});
+
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS LAST, CONTAINER_ID DESC NULLS LAST";
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID ASC NULLS LAST, CONTAINER_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{"1", "1"}, {"1", null}, {"2", "2"}, {"2", null}, {"3", "3"},
+ {"3", null}, {"4", "4"}, {"4", null}, {"5", "5"}, {"5", null},
+ {"6", "6"}, {"6", null}, {null, "6"}, {null, "5"}, {null, "4"},
+ {null, "3"}, {null, "2"}, {null, "1"}, {null, null}});
+
+ //groupBy orderPreserving orderBy desc asc
+
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS FIRST, CONTAINER_ID ASC NULLS FIRST";
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID DESC NULLS FIRST, CONTAINER_ID ASC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{null, null}, {null, "1"}, {null, "2"}, {null, "3"}, {null, "4"},
+ {null, "5"}, {null, "6"}, {"6", null}, {"6", "6"}, {"5", null},
+ {"5", "5"}, {"4", null}, {"4", "4"}, {"3", null}, {"3", "3"},
+ {"2", null}, {"2", "2"}, {"1", null}, {"1", "1"}});
+
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS FIRST, CONTAINER_ID ASC NULLS LAST";
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID DESC NULLS FIRST, CONTAINER_ID ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{null, "1"}, {null, "2"}, {null, "3"}, {null, "4"}, {null, "5"},
+ {null, "6"}, {null, null}, {"6", "6"}, {"6", null}, {"5", "5"},
+ {"5", null}, {"4", "4"}, {"4", null}, {"3", "3"}, {"3", null},
+ {"2", "2"}, {"2", null}, {"1", "1"}, {"1", null}});
+
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS LAST, CONTAINER_ID ASC NULLS FIRST";
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID DESC NULLS LAST, CONTAINER_ID ASC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{"6", null}, {"6", "6"}, {"5", null}, {"5", "5"}, {"4", null},
+ {"4", "4"}, {"3", null}, {"3", "3"}, {"2", null}, {"2", "2"},
+ {"1", null}, {"1", "1"}, {null, null}, {null, "1"}, {null, "2"},
+ {null, "3"}, {null, "4"}, {null, "5"}, {null, "6"}});
+
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS LAST, CONTAINER_ID ASC NULLS LAST";
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID DESC NULLS LAST, CONTAINER_ID ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{"6", "6"}, {"6", null}, {"5", "5"}, {"5", null}, {"4", "4"},
+ {"4", null}, {"3", "3"}, {"3", null}, {"2", "2"}, {"2", null},
+ {"1", "1"}, {"1", null}, {null, "1"}, {null, "2"}, {null, "3"},
+ {null, "4"}, {null, "5"}, {null, "6"}, {null, null}});
+
+ //groupBy orderPreserving orderBy desc desc
+
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS FIRST, CONTAINER_ID DESC NULLS FIRST";
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID DESC NULLS FIRST, CONTAINER_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{null, null}, {null, "6"}, {null, "5"}, {null, "4"}, {null, "3"},
+ {null, "2"}, {null, "1"}, {"6", null}, {"6", "6"}, {"5", null},
+ {"5", "5"}, {"4", null}, {"4", "4"}, {"3", null}, {"3", "3"},
+ {"2", null}, {"2", "2"}, {"1", null}, {"1", "1"}});
+
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS FIRST, CONTAINER_ID DESC NULLS LAST";
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID DESC NULLS FIRST, CONTAINER_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{null, "6"}, {null, "5"}, {null, "4"}, {null, "3"}, {null, "2"},
+ {null, "1"}, {null, null}, {"6", "6"}, {"6", null}, {"5", "5"},
+ {"5", null}, {"4", "4"}, {"4", null}, {"3", "3"}, {"3", null},
+ {"2", "2"}, {"2", null}, {"1", "1"}, {"1", null}});
+
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS LAST, CONTAINER_ID DESC NULLS FIRST";
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID DESC NULLS LAST, CONTAINER_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSetWithRegionMoves(rs,
+ new Object[][]{{"6", null}, {"6", "6"}, {"5", null}, {"5", "5"}, {"4", null},
+ {"4", "4"}, {"3", null}, {"3", "3"}, {"2", null}, {"2", "2"},
+ {"1", null}, {"1", "1"}, {null, null}, {null, "6"}, {null, "5"},
+ {null, "4"}, {null, "3"}, {null, "2"}, {null, "1"}}, tableName);
+
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS LAST, CONTAINER_ID DESC NULLS LAST";
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID DESC NULLS LAST, CONTAINER_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{"6", "6"}, {"6", null}, {"5", "5"}, {"5", null}, {"4", "4"},
+ {"4", null}, {"3", "3"}, {"3", null}, {"2", "2"}, {"2", null},
+ {"1", "1"}, {"1", null}, {null, "6"}, {null, "5"}, {null, "4"},
+ {null, "3"}, {null, "2"}, {null, "1"}, {null, null}});
+
+ //-----groupBy not orderPreserving--
+
+ //groupBy not orderPreserving orderBy asc asc
+
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS FIRST, SCORE ASC NULLS FIRST";
+ queryBuilder.setSelectColumns(
+ Lists.newArrayList("ORGANIZATION_ID", "SCORE"))
+ .setFullTableName(tableName)
+ .setGroupByClause("ORGANIZATION_ID, SCORE")
+ .setOrderByClause("ORGANIZATION_ID ASC NULLS FIRST, SCORE ASC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{null, null}, {null, "1"}, {null, "2"}, {null, "3"}, {null, "4"},
+ {null, "5"}, {null, "6"}, {"1", null}, {"1", "1"}, {"2", null},
+ {"2", "2"}, {"3", null}, {"3", "3"}, {"4", null}, {"4", "4"},
+ {"5", null}, {"5", "5"}, {"6", null}, {"6", "6"}});
+
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS FIRST, SCORE ASC NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS FIRST, SCORE ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{null, "1"}, {null, "2"}, {null, "3"}, {null, "4"}, {null, "5"},
+ {null, "6"}, {null, null}, {"1", "1"}, {"1", null}, {"2", "2"},
+ {"2", null}, {"3", "3"}, {"3", null}, {"4", "4"}, {"4", null},
+ {"5", "5"}, {"5", null}, {"6", "6"}, {"6", null}});
+
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS LAST, SCORE ASC NULLS FIRST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS LAST, SCORE ASC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{"1", null}, {"1", "1"}, {"2", null}, {"2", "2"}, {"3", null},
+ {"3", "3"}, {"4", null}, {"4", "4"}, {"5", null}, {"5", "5"},
+ {"6", null}, {"6", "6"}, {null, null}, {null, "1"}, {null, "2"},
+ {null, "3"}, {null, "4"}, {null, "5"}, {null, "6"}});
+
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS LAST, SCORE ASC NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS LAST, SCORE ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{"1", "1"}, {"1", null}, {"2", "2"}, {"2", null}, {"3", "3"},
+ {"3", null}, {"4", "4"}, {"4", null}, {"5", "5"}, {"5", null},
+ {"6", "6"}, {"6", null}, {null, "1"}, {null, "2"}, {null, "3"},
+ {null, "4"}, {null, "5"}, {null, "6"}, {null, null}});
+
+ //groupBy not orderPreserving orderBy asc desc
+
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS FIRST, SCORE DESC NULLS FIRST";
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID ASC NULLS FIRST, SCORE DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{null, null}, {null, "6"}, {null, "5"}, {null, "4"}, {null, "3"},
+ {null, "2"}, {null, "1"}, {"1", null}, {"1", "1"}, {"2", null},
+ {"2", "2"}, {"3", null}, {"3", "3"}, {"4", null}, {"4", "4"},
+ {"5", null}, {"5", "5"}, {"6", null}, {"6", "6"}});
+
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS FIRST, SCORE DESC NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS FIRST, SCORE DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{null, "6"}, {null, "5"}, {null, "4"}, {null, "3"}, {null, "2"},
+ {null, "1"}, {null, null}, {"1", "1"}, {"1", null}, {"2", "2"},
+ {"2", null}, {"3", "3"}, {"3", null}, {"4", "4"}, {"4", null},
+ {"5", "5"}, {"5", null}, {"6", "6"}, {"6", null}});
+
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS LAST, SCORE DESC NULLS FIRST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS LAST, SCORE DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{"1", null}, {"1", "1"}, {"2", null}, {"2", "2"}, {"3", null},
+ {"3", "3"}, {"4", null}, {"4", "4"}, {"5", null}, {"5", "5"},
+ {"6", null}, {"6", "6"}, {null, null}, {null, "6"}, {null, "5"},
+ {null, "4"}, {null, "3"}, {null, "2"}, {null, "1"}});
+
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS LAST, SCORE DESC NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS LAST, SCORE DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{"1", "1"}, {"1", null}, {"2", "2"}, {"2", null}, {"3", "3"},
+ {"3", null}, {"4", "4"}, {"4", null}, {"5", "5"}, {"5", null},
+ {"6", "6"}, {"6", null}, {null, "6"}, {null, "5"}, {null, "4"},
+ {null, "3"}, {null, "2"}, {null, "1"}, {null, null}});
+
+ //groupBy not orderPreserving orderBy desc asc
+
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS FIRST, SCORE ASC NULLS FIRST";
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID DESC NULLS FIRST, SCORE ASC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{null, null}, {null, "1"}, {null, "2"}, {null, "3"}, {null, "4"},
+ {null, "5"}, {null, "6"}, {"6", null}, {"6", "6"}, {"5", null},
+ {"5", "5"}, {"4", null}, {"4", "4"}, {"3", null}, {"3", "3"},
+ {"2", null}, {"2", "2"}, {"1", null}, {"1", "1"}});
+
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS FIRST, SCORE ASC NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST, SCORE ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSetWithRegionMoves(rs,
+ new Object[][]{{null, "1"}, {null, "2"}, {null, "3"}, {null, "4"}, {null, "5"},
+ {null, "6"}, {null, null}, {"6", "6"}, {"6", null}, {"5", "5"},
+ {"5", null}, {"4", "4"}, {"4", null}, {"3", "3"}, {"3", null},
+ {"2", "2"}, {"2", null}, {"1", "1"}, {"1", null}}, tableName);
+
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS LAST, SCORE ASC NULLS FIRST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST, SCORE ASC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{"6", null}, {"6", "6"}, {"5", null}, {"5", "5"}, {"4", null},
+ {"4", "4"}, {"3", null}, {"3", "3"}, {"2", null}, {"2", "2"},
+ {"1", null}, {"1", "1"}, {null, null}, {null, "1"}, {null, "2"},
+ {null, "3"}, {null, "4"}, {null, "5"}, {null, "6"}});
+
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS LAST, SCORE ASC NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST, SCORE ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{"6", "6"}, {"6", null}, {"5", "5"}, {"5", null}, {"4", "4"},
+ {"4", null}, {"3", "3"}, {"3", null}, {"2", "2"}, {"2", null},
+ {"1", "1"}, {"1", null}, {null, "1"}, {null, "2"}, {null, "3"},
+ {null, "4"}, {null, "5"}, {null, "6"}, {null, null}});
+
+ //groupBy not orderPreserving orderBy desc desc
+
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS FIRST, SCORE DESC NULLS FIRST";
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID DESC NULLS FIRST, SCORE DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{null, null}, {null, "6"}, {null, "5"}, {null, "4"}, {null, "3"},
+ {null, "2"}, {null, "1"}, {"6", null}, {"6", "6"}, {"5", null},
+ {"5", "5"}, {"4", null}, {"4", "4"}, {"3", null}, {"3", "3"},
+ {"2", null}, {"2", "2"}, {"1", null}, {"1", "1"}});
+
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS FIRST, SCORE DESC NULLS LAST";
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID DESC NULLS FIRST, SCORE DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{null, "6"}, {null, "5"}, {null, "4"}, {null, "3"}, {null, "2"},
+ {null, "1"}, {null, null}, {"6", "6"}, {"6", null}, {"5", "5"},
+ {"5", null}, {"4", "4"}, {"4", null}, {"3", "3"}, {"3", null},
+ {"2", "2"}, {"2", null}, {"1", "1"}, {"1", null}});
+
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS LAST, SCORE DESC NULLS FIRST";
+ queryBuilder.setOrderByClause(
+ "ORGANIZATION_ID DESC NULLS LAST, SCORE DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{"6", null}, {"6", "6"}, {"5", null}, {"5", "5"}, {"4", null},
+ {"4", "4"}, {"3", null}, {"3", "3"}, {"2", null}, {"2", "2"},
+ {"1", null}, {"1", "1"}, {null, null}, {null, "6"}, {null, "5"},
+ {null, "4"}, {null, "3"}, {null, "2"}, {null, "1"}});
+
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS LAST, SCORE DESC NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST, SCORE DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs,
+ new Object[][]{{"6", "6"}, {"6", null}, {"5", "5"}, {"5", null}, {"4", "4"},
+ {"4", null}, {"3", "3"}, {"3", null}, {"2", "2"}, {"2", null},
+ {"1", "1"}, {"1", null}, {null, "6"}, {null, "5"}, {null, "4"},
+ {null, "3"}, {null, "2"}, {null, "1"}, {null, null}});
+
+ //-------test only one return column----------------------------------
+
+// sql="SELECT SCORE FROM "+tableName+" group by SCORE ORDER BY SCORE ASC NULLS FIRST";
+ queryBuilder.setSelectColumns(
+ Lists.newArrayList("SCORE"))
+ .setFullTableName(tableName)
+ .setGroupByClause("SCORE")
+ .setOrderByClause("SCORE ASC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{null}, {"1"}, {"2"}, {"3"}, {"4"}, {"5"}, {"6"}});
+
+// sql="SELECT SCORE FROM "+tableName+" group by SCORE ORDER BY SCORE ASC NULLS LAST";
+ queryBuilder.setOrderByClause("SCORE ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSetWithRegionMoves(rs, new Object[][]{{"1"}, {"2"}, {"3"}, {"4"}, {"5"},
+ {"6"}, {null}}, tableName);
+
+// sql="SELECT SCORE FROM "+tableName+" group by SCORE ORDER BY SCORE DESC NULLS FIRST";
+ queryBuilder.setOrderByClause("SCORE DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{null}, {"6"}, {"5"}, {"4"}, {"3"}, {"2"}, {"1"}});
+
+// sql="SELECT SCORE FROM "+tableName+" group by SCORE ORDER BY SCORE DESC NULLS LAST";
+ queryBuilder.setOrderByClause("SCORE DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{"6"}, {"5"}, {"4"}, {"3"}, {"2"}, {"1"}, {null}});
+ }
+ }
+
+ @Test
+ public void testOrderByNullable() throws SQLException {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String sql = "CREATE TABLE IF NOT EXISTS us_population (state CHAR(2) NOT NULL," +
+ "city VARCHAR NOT NULL, population BIGINT CONSTRAINT my_pk PRIMARY KEY" +
+ " (state, city))";
+ conn.createStatement().execute(sql);
+ TABLE_NAMES.add("US_POPULATION");
+
+ sql = "select ORDINAL_POSITION from SYSTEM.CATALOG where TABLE_NAME = 'US_POPULATION'";
+ ResultSet rs = conn.createStatement().executeQuery(sql);
+ int expected = 0;
+ while (rs.next()) {
+ expected += 1;
+ }
+
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectColumns(Lists.newArrayList("*"))
+ .setFullTableName("SYSTEM.CATALOG")
+ .setWhereClause("TABLE_NAME = 'US_POPULATION'")
+ .setOrderByClause("ORDINAL_POSITION");
+ rs = executeQuery(conn, queryBuilder);
+ int linesCount = 0;
+ while (rs.next()) {
+ linesCount += 1;
+ }
+ assertEquals(expected, linesCount);
+
+ queryBuilder = new QueryBuilder()
+ .setSelectColumns(Lists.newArrayList("COLUMN_NAME"))
+ .setFullTableName("SYSTEM.CATALOG")
+ .setWhereClause("TABLE_NAME = 'US_POPULATION'")
+ .setOrderByClause("ORDINAL_POSITION");
+ rs = executeQuery(conn, queryBuilder);
+ linesCount = 0;
+ while (rs.next()) {
+ linesCount += 1;
+ }
+ assertEquals(expected, linesCount);
+ }
+ }
+
+ @Test
+ public void testPhoenix6999() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ String tableName = "TBL_" + generateUniqueName();
+ String descTableName = "TBL_" + generateUniqueName();
+
+ String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
+ String fullDescTableName =
+ SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, descTableName);
+ TABLE_NAMES.add(fullTableName);
+ TABLE_NAMES.add(fullDescTableName);
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ conn.setAutoCommit(false);
+ String ddl =
+ "CREATE TABLE " + fullTableName
+ + "(k1 varchar primary key, v1 varchar, v2 varchar)";
+ stmt.execute(ddl);
+ ddl =
+ "CREATE TABLE " + fullDescTableName
+ + "(k1 varchar primary key desc, v1 varchar, v2 varchar)";
+ stmt.execute(ddl);
+ stmt.execute("upsert into " + fullTableName + " values ('a','a','a')");
+ stmt.execute("upsert into " + fullTableName + " values ('b','b','b')");
+ stmt.execute("upsert into " + fullTableName + " values ('c','c','c')");
+ stmt.execute("upsert into " + fullDescTableName + " values ('a','a','a')");
+ stmt.execute("upsert into " + fullDescTableName + " values ('b','b','b')");
+ stmt.execute("upsert into " + fullDescTableName + " values ('c','c','c')");
+ conn.commit();
+
+ String query = "SELECT * from " + fullTableName + " where k1='b' order by k1 asc";
+ ResultSet rs = stmt.executeQuery(query);
+ assertTrue(rs.next());
+ assertEquals("b", rs.getString(1));
+
+ query = "SELECT * from " + fullTableName + " where k1='b' order by k1 desc";
+ rs = stmt.executeQuery(query);
+ assertTrue(rs.next());
+ assertEquals("b", rs.getString(1));
+
+ query = "SELECT * from " + fullDescTableName + " where k1='b' order by k1 asc";
+ rs = stmt.executeQuery(query);
+ assertTrue(rs.next());
+ assertEquals("b", rs.getString(1));
+
+ query = "SELECT * from " + fullDescTableName + " where k1='b' order by k1 desc";
+ rs = stmt.executeQuery(query);
+ assertTrue(rs.next());
+ assertEquals("b", rs.getString(1));
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseQueryWithRegionMovesIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseQueryWithRegionMovesIT.java
new file mode 100644
index 0000000000..446ae58390
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseQueryWithRegionMovesIT.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import org.apache.phoenix.thirdparty.com.google.common.collect.Lists;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.BeforeParam;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.sql.Connection;
+import java.sql.Date;
+import java.sql.DriverManager;
+import java.sql.SQLTimeoutException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Properties;
+
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+
+/**
+ * Basic tests that include region move hooks while scanning the rows.
+ * <p>
+ */
+@RunWith(Parameterized.class)
+public abstract class BaseQueryWithRegionMovesIT extends ParallelStatsDisabledWithRegionMovesIT {
+ protected static final String tenantId = getOrganizationId();
+ protected static final String NO_INDEX = "";
+ protected static final String[] GLOBAL_INDEX_DDLS =
+ new String[]{
+ "CREATE INDEX IF NOT EXISTS %s ON %s (a_integer DESC) INCLUDE (" +
+ " A_STRING, "
+ + " B_STRING, " + " A_DATE)",
+ "CREATE INDEX IF NOT EXISTS %s ON %s (a_integer, a_string) INCLUDE (" +
+ " B_STRING, "
+ + " A_DATE)",
+ "CREATE INDEX IF NOT EXISTS %s ON %s (a_integer) INCLUDE (" + " A_STRING, "
+ + " B_STRING, " + " A_DATE)",
+ NO_INDEX};
+ private static final String[] INDEX_DDLS;
+
+ static {
+ INDEX_DDLS = new String[GLOBAL_INDEX_DDLS.length];
+ int i = 0;
+ for (String s : GLOBAL_INDEX_DDLS) {
+ INDEX_DDLS[i++] = s;
+ }
+ }
+
+ protected static Date date;
+ protected static String tableName;
+ protected static String indexName;
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(BaseQueryWithRegionMovesIT.class);
+
+ @BeforeParam
+ public static synchronized void initTables(String idxDdl, boolean columnEncoded,
+ boolean keepDeletedCells) throws Exception {
+ StringBuilder optionBuilder = new StringBuilder();
+ if (!columnEncoded) {
+ optionBuilder.append("COLUMN_ENCODED_BYTES=0");
+ }
+ if (keepDeletedCells) {
+ if (optionBuilder.length() > 0) optionBuilder.append(",");
+ optionBuilder.append("KEEP_DELETED_CELLS=true");
+ }
+ String tableDDLOptions = optionBuilder.toString();
+ try {
+ tableName = initATableValues(generateUniqueName(), tenantId,
+ getDefaultSplits(tenantId),
+ date = new Date(System.currentTimeMillis()), null, getUrl(),
+ tableDDLOptions);
+ } catch (Exception e) {
+ LOGGER.error("Exception when creating aTable ", e);
+ throw e;
+ }
+ indexName = generateUniqueName();
+ if (idxDdl.length() > 0) {
+ String indexDDL =
+ String.format(idxDdl, indexName, tableName);
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ props.setProperty("phoenix.query.timeoutMs", "30000");
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ conn.createStatement().execute(indexDDL);
+ } catch (SQLTimeoutException e) {
+ LOGGER.info("Query timed out. Retrying one more time.", e);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ conn.createStatement().execute(indexDDL);
+ } catch (Exception ex) {
+ LOGGER.error("Exception while creating index during second retry: "
+ + indexDDL, ex);
+ throw ex;
+ }
+ } catch (Exception e) {
+ LOGGER.error("Exception while creating index: " + indexDDL, e);
+ throw e;
+ }
+ }
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ hasTestStarted = true;
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ countOfDummyResults = 0;
+ TABLE_NAMES.clear();
+ hasTestStarted = false;
+ }
+
+ public BaseQueryWithRegionMovesIT(String idxDdl, boolean columnEncoded,
+ boolean keepDeletedCells) {
+ }
+
+ public static Collection<Object> allIndexes() {
+ List<Object> testCases = Lists.newArrayList();
+ for (String indexDDL : INDEX_DDLS) {
+ for (boolean columnEncoded : new boolean[]{false}) {
+ testCases.add(new Object[]{indexDDL, columnEncoded, false});
+ }
+ }
+ return testCases;
+ }
+
+ public static Collection<Object> allIndexesWithEncoded() {
+ List<Object> testCases = Lists.newArrayList();
+ for (String indexDDL : INDEX_DDLS) {
+ for (boolean columnEncoded : new boolean[]{false, true}) {
+ testCases.add(new Object[]{indexDDL, columnEncoded, false});
+ }
+ }
+ return testCases;
+ }
+
+ public static Collection<Object> allIndexesWithEncodedAndKeepDeleted() {
+ List<Object> testCases = Lists.newArrayList();
+ for (String indexDDL : INDEX_DDLS) {
+ for (boolean columnEncoded : new boolean[]{false, true}) {
+ testCases.add(new Object[]{indexDDL, columnEncoded, true});
+ }
+ }
+ return testCases;
+ }
+}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByIT.java
index cde1e0fe50..7fa9329fad 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/GroupByIT.java
@@ -378,4 +378,55 @@ public class GroupByIT extends BaseQueryIT {
conn.close();
}
}
+
+ @Test
+ public void testGroupByHavingWithAlias2() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ conn.setAutoCommit(false);
+ try {
+ conn = DriverManager.getConnection(getUrl(), props);
+ conn.setAutoCommit(false);
+ String tableName = generateUniqueName();
+ String ddl = "CREATE TABLE " + tableName + " (a_string varchar not null, col1 " +
+ "integer not null, col2 varchar, col3 integer"
+ + " CONSTRAINT pk PRIMARY KEY (a_string, col1))";
+ createTestTable(getUrl(), ddl);
+
+ String dml = "UPSERT INTO " + tableName + " VALUES(?, ?, ?, ?)";
+ PreparedStatement stmt = conn.prepareStatement(dml);
+ stmt.setString(1, "a1xyz");
+ stmt.setInt(2, 40);
+ stmt.setString(3, "col2xyz1");
+ stmt.setInt(4, 50);
+ stmt.execute();
+ stmt.setString(1, "b1xyz");
+ stmt.setInt(2, 20);
+ stmt.setString(3, "col2xyz2");
+ stmt.setInt(4, 60);
+ stmt.execute();
+ stmt.setString(1, "c1xyz");
+ stmt.setInt(2, 30);
+ stmt.setString(3, "col2xyz3");
+ stmt.setInt(4, 70);
+ stmt.execute();
+ conn.commit();
+
+ String query = "SELECT a_string, col1, sum(col1) as sumCol1 FROM " + tableName
+ + " GROUP BY a_string, col1 HAVING sumCol1 > 20 ORDER BY sumCol1";
+ ResultSet rs = conn.createStatement().executeQuery(query);
+ assertTrue(rs.next());
+ assertEquals("c1xyz", rs.getString(1));
+ assertEquals(30, rs.getInt(2));
+ assertEquals(30, rs.getInt(3));
+ assertTrue(rs.next());
+ assertEquals("a1xyz", rs.getString(1));
+ assertEquals(40, rs.getInt(2));
+ assertEquals(40, rs.getInt(3));
+ assertFalse(rs.next());
+ } finally {
+ conn.close();
+ }
+ }
+
}
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupByCache.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrderByWithRegionMovesIT.java
similarity index 60%
copy from phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupByCache.java
copy to phoenix-core/src/it/java/org/apache/phoenix/end2end/OrderByWithRegionMovesIT.java
index 68d07a948a..f7a91f31c4 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/GroupByCache.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrderByWithRegionMovesIT.java
@@ -15,24 +15,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.phoenix.coprocessor;
+package org.apache.phoenix.end2end;
-import java.io.Closeable;
+import org.junit.experimental.categories.Category;
-import org.apache.hadoop.hbase.regionserver.RegionScanner;
-import org.apache.phoenix.expression.aggregator.Aggregator;
-import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
+@Category(NeedsOwnMiniClusterTest.class)
+public class OrderByWithRegionMovesIT extends BaseOrderByWithRegionMovesIT {
-/**
- *
- * Interface to abstract the way in which distinct group by
- * elements are cached
- *
- *
- * @since 3.0.0
- */
-public interface GroupByCache extends Closeable {
- long size();
- Aggregator[] cache(ImmutableBytesPtr key);
- RegionScanner getScanner(RegionScanner s);
}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrderByWithServerMemoryLimitIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrderByWithServerMemoryLimitIT.java
index 0a71e81bf0..edcf8848f1 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrderByWithServerMemoryLimitIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrderByWithServerMemoryLimitIT.java
@@ -19,10 +19,10 @@ import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
+import java.sql.SQLException;
import java.util.Map;
import java.util.Properties;
-import org.apache.phoenix.exception.PhoenixIOException;
import org.apache.phoenix.query.BaseTest;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.util.PropertiesUtil;
@@ -74,9 +74,11 @@ public class OrderByWithServerMemoryLimitIT extends BaseTest {
try {
rs.next();
fail("Expected PhoenixIOException due to IllegalStateException");
- } catch (PhoenixIOException e) {
- assertThat(e.getMessage(), containsString("java.lang.IllegalStateException: "
- + "Queue full. Consider increasing memory threshold or spooling to disk"));
+ } catch (SQLException e) {
+ assertThat(e.getMessage(), containsString(
+ "ERROR 101 (08000): Unexpected IO exception. ERROR 101 (08000): "
+ + "Queue full. Consider increasing memory threshold or "
+ + "spooling to disk"));
}
}
}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParallelStatsDisabledWithRegionMovesIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParallelStatsDisabledWithRegionMovesIT.java
new file mode 100644
index 0000000000..2594624567
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParallelStatsDisabledWithRegionMovesIT.java
@@ -0,0 +1,310 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.end2end;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.AsyncAdmin;
+import org.apache.hadoop.hbase.client.AsyncConnection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.RegionStatesCount;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.phoenix.coprocessor.BaseScannerRegionObserver;
+import org.apache.phoenix.iterate.ScanningResultPostDummyResultCaller;
+import org.apache.phoenix.query.BaseTest;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.util.ByteUtil;
+import org.apache.phoenix.util.QueryBuilder;
+import org.apache.phoenix.util.QueryUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+
+/**
+ * Base class for tests whose methods run in parallel with
+ * 1. Statistics enabled on server side (QueryServices#STATS_COLLECTION_ENABLED is true)
+ * 2. Guide Post Width for all relevant tables is 0. Stats are disabled at table level.
+ * <p>
+ * See {@link org.apache.phoenix.schema.stats.NoOpStatsCollectorIT} for tests that disable
+ * stats collection from server side.
+ * <p>
+ * You must create unique names using {@link #generateUniqueName()} for each
+ * table and sequence used to prevent collisions.
+ * <p>
+ * Any class extending this would also observe multiple region moves while tests are running.
+ */
+public abstract class ParallelStatsDisabledWithRegionMovesIT extends BaseTest {
+
+ private static final Logger LOGGER =
+ LoggerFactory.getLogger(ParallelStatsDisabledWithRegionMovesIT.class);
+
+ protected static boolean hasTestStarted = false;
+ protected static int countOfDummyResults = 0;
+ protected static final Set<String> TABLE_NAMES = new HashSet<>();
+
+ protected static class TestScanningResultPostDummyResultCaller extends
+ ScanningResultPostDummyResultCaller {
+
+ @Override
+ public void postDummyProcess() {
+ if (hasTestStarted && (countOfDummyResults++ % 3) == 0 &&
+ (countOfDummyResults < 17 ||
+ countOfDummyResults > 28 && countOfDummyResults < 40)) {
+ LOGGER.info("Moving regions of tables {}. current count of dummy results: {}",
+ TABLE_NAMES, countOfDummyResults);
+ TABLE_NAMES.forEach(table -> {
+ try {
+ moveRegionsOfTable(table);
+ } catch (Exception e) {
+ LOGGER.error("Unable to move regions of table: {}", table);
+ }
+ });
+ }
+ }
+ }
+
+ @BeforeClass
+ public static synchronized void doSetup() throws Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put(BaseScannerRegionObserver.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY,
+ Integer.toString(60 * 60)); // An hour
+ props.put(QueryServices.USE_STATS_FOR_PARALLELIZATION, Boolean.toString(false));
+ props.put(QueryServices.PHOENIX_SERVER_PAGE_SIZE_MS, Long.toString(0));
+ props.put(QueryServices.TESTS_MINI_CLUSTER_NUM_REGION_SERVERS, String.valueOf(2));
+ props.put(HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY, String.valueOf(1));
+ props.put(QueryServices.PHOENIX_POST_DUMMY_PROCESS,
+ TestScanningResultPostDummyResultCaller.class.getName());
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ }
+
+ @AfterClass
+ public static synchronized void freeResources() throws Exception {
+ BaseTest.freeResourcesIfBeyondThreshold();
+ }
+
+ protected static void moveRegionsOfTable(String tableName)
+ throws IOException {
+ try (AsyncConnection asyncConnection =
+ ConnectionFactory.createAsyncConnection(getUtility().getConfiguration())
+ .get()) {
+ AsyncAdmin admin = asyncConnection.getAdmin();
+ List<ServerName> servers =
+ new ArrayList<>(admin.getRegionServers().get());
+ ServerName server1 = servers.get(0);
+ ServerName server2 = servers.get(1);
+ List<RegionInfo> regionsOnServer1;
+ regionsOnServer1 = admin.getRegions(server1).get();
+ List<RegionInfo> regionsOnServer2;
+ regionsOnServer2 = admin.getRegions(server2).get();
+ regionsOnServer1.forEach(regionInfo -> {
+ if (regionInfo.getTable().equals(TableName.valueOf(tableName))) {
+ try {
+ for (int i = 0; i < 2; i++) {
+ RegionStatesCount regionStatesCount =
+ admin.getClusterMetrics().get().getTableRegionStatesCount()
+ .get(TableName.valueOf(tableName));
+ if (regionStatesCount.getRegionsInTransition() == 0 &&
+ regionStatesCount.getOpenRegions() ==
+ regionStatesCount.getTotalRegions()) {
+ LOGGER.info("Moving region {} to {}",
+ regionInfo.getRegionNameAsString(), server2);
+ admin.move(regionInfo.getEncodedNameAsBytes(), server2).get(3,
+ TimeUnit.SECONDS);
+ break;
+ } else {
+ LOGGER.info("Table {} has some region(s) in RIT or not online",
+ tableName);
+ }
+ }
+ } catch (InterruptedException | ExecutionException | TimeoutException e) {
+ LOGGER.error("Something went wrong", e);
+ throw new RuntimeException(e);
+ }
+ }
+ });
+ regionsOnServer2.forEach(regionInfo -> {
+ if (regionInfo.getTable().equals(TableName.valueOf(tableName))) {
+ try {
+ for (int i = 0; i < 2; i++) {
+ RegionStatesCount regionStatesCount =
+ admin.getClusterMetrics().get().getTableRegionStatesCount()
+ .get(TableName.valueOf(tableName));
+ if (regionStatesCount.getRegionsInTransition() == 0 &&
+ regionStatesCount.getOpenRegions() ==
+ regionStatesCount.getTotalRegions()) {
+ admin.move(regionInfo.getEncodedNameAsBytes(), server1)
+ .get(3, TimeUnit.SECONDS);
+ LOGGER.info("Moving region {} to {}",
+ regionInfo.getRegionNameAsString(), server1);
+ break;
+ } else {
+ LOGGER.info("Table {} has some region(s) in RIT or not online",
+ tableName);
+ }
+ }
+ } catch (InterruptedException | ExecutionException | TimeoutException e) {
+ LOGGER.error("Something went wrong", e);
+ throw new RuntimeException(e);
+ }
+ }
+ });
+ } catch (Exception e) {
+ LOGGER.error("Something went wrong..", e);
+ }
+ }
+
+ protected static void splitAllRegionsOfTable(String tableName, int splitAtRow) {
+ try (AsyncConnection asyncConnection =
+ ConnectionFactory.createAsyncConnection(getUtility().getConfiguration())
+ .get()) {
+ AsyncAdmin admin = asyncConnection.getAdmin();
+ List<RegionInfo> regionsOfTable =
+ admin.getRegions(TableName.valueOf(tableName)).get();
+ regionsOfTable.forEach(regionInfo -> {
+ if (regionInfo.getTable().equals(TableName.valueOf(tableName))) {
+ try {
+ for (int i = 0; i < 5; i++) {
+ RegionStatesCount regionStatesCount =
+ admin.getClusterMetrics().get().getTableRegionStatesCount()
+ .get(TableName.valueOf(tableName));
+ byte[] splitPoint;
+ if (regionStatesCount.getRegionsInTransition() == 0 &&
+ regionStatesCount.getOpenRegions() ==
+ regionStatesCount.getTotalRegions()) {
+ try (Table table =
+ utility.getConnection()
+ .getTable(TableName.valueOf(tableName))) {
+ try (ResultScanner resultScanner = table.getScanner(
+ new Scan())) {
+ Result result = null;
+ for (int rowCount = 0; rowCount < splitAtRow; rowCount++) {
+ result = resultScanner.next();
+ if (result == null) {
+ LOGGER.info("Table {} has only {} rows, splitting" +
+ " at row number {} not possible",
+ tableName, rowCount, splitAtRow);
+ return;
+ }
+ }
+ splitPoint = result == null ? null :
+ ByteUtil.closestPossibleRowAfter(result.getRow());
+ }
+ }
+ LOGGER.info("Splitting region {}",
+ regionInfo.getRegionNameAsString());
+ admin.flushRegion(regionInfo.getRegionName()).get(3,
+ TimeUnit.SECONDS);
+ admin.splitRegion(regionInfo.getRegionName(), splitPoint).get(4,
+ TimeUnit.SECONDS);
+ break;
+ } else {
+ LOGGER.info("Table {} has some region(s) in RIT or not online",
+ tableName);
+ Thread.sleep(1000);
+ }
+ }
+ } catch (InterruptedException | ExecutionException | TimeoutException |
+ IOException e) {
+ LOGGER.error("Something went wrong", e);
+ throw new RuntimeException(e);
+ }
+ }
+ });
+ } catch (Exception e) {
+ LOGGER.error("Something went wrong..", e);
+ }
+ }
+
+ protected ResultSet executeQueryThrowsException(Connection conn, QueryBuilder queryBuilder,
+ String expectedPhoenixExceptionMsg,
+ String expectedSparkExceptionMsg) {
+ ResultSet rs = null;
+ try {
+ rs = executeQuery(conn, queryBuilder);
+ fail();
+ } catch (Exception e) {
+ assertTrue(e.getMessage().contains(expectedPhoenixExceptionMsg));
+ }
+ return rs;
+ }
+
+ public static void validateQueryPlan(Connection conn, QueryBuilder queryBuilder,
+ String expectedPhoenixPlan, String expectedSparkPlan)
+ throws SQLException {
+ if (StringUtils.isNotBlank(expectedPhoenixPlan)) {
+ ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + queryBuilder.build());
+ assertEquals(expectedPhoenixPlan, QueryUtil.getExplainPlan(rs));
+ }
+ }
+
+ protected static void assertResultSetWithRegionMoves(ResultSet rs, Object[][] rows,
+ String tableName)
+ throws Exception {
+ for (int rowIndex = 0; rowIndex < rows.length; rowIndex++) {
+ assertTrue("rowIndex:[" + rowIndex + "] rs.next error!", rs.next());
+ if (rowIndex == 0) {
+ moveRegionsOfTable(tableName);
+ }
+ for (int columnIndex = 1; columnIndex <= rows[rowIndex].length; columnIndex++) {
+ Object realValue = rs.getObject(columnIndex);
+ Object expectedValue = rows[rowIndex][columnIndex - 1];
+ if (realValue == null) {
+ assertNull("rowIndex:[" + rowIndex + "],columnIndex:[" + columnIndex + "]",
+ expectedValue);
+ } else {
+ assertEquals("rowIndex:[" + rowIndex + "],columnIndex:[" + columnIndex + "]",
+ expectedValue,
+ realValue
+ );
+ }
+ }
+ }
+ assertFalse(rs.next());
+ }
+
+}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ScanUncommittedWithRegionMovesIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ScanUncommittedWithRegionMovesIT.java
new file mode 100644
index 0000000000..45fa4d77bd
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ScanUncommittedWithRegionMovesIT.java
@@ -0,0 +1,465 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.phoenix.iterate.ScanningResultPostDummyResultCaller;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.thirdparty.com.google.common.collect.Maps;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.junit.After;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Tests to ensure we perform READ_COMMITTED scan i.e. after the scanner is opened, any updated
+ * data should not be visible to the scanner until it is closed. The ongoing scanner should
+ * only be able to cover the region state that was present before the scanner was opened. HBase
+ * provides this guarantee by using region level MVCC.
+ */
+@Category(NeedsOwnMiniClusterTest.class)
+public class ScanUncommittedWithRegionMovesIT extends ParallelStatsDisabledIT {
+
+ private static final Logger LOGGER =
+ LoggerFactory.getLogger(ScanUncommittedWithRegionMovesIT.class);
+
+ private static boolean hasTestStarted = false;
+ private static int countOfDummyResults = 0;
+ private static final Set<String> TABLE_NAMES = new HashSet<>();
+
+ private static class TestScanningResultPostDummyResultCaller extends
+ ScanningResultPostDummyResultCaller {
+
+ @Override
+ public void postDummyProcess() {
+ if (hasTestStarted && (countOfDummyResults++ % 5) == 0 &&
+ (countOfDummyResults < 17 ||
+ countOfDummyResults > 28 && countOfDummyResults < 40)) {
+ LOGGER.info("Moving regions of tables {}. current count of dummy results: {}",
+ TABLE_NAMES, countOfDummyResults);
+ TABLE_NAMES.forEach(table -> {
+ try {
+ moveRegionsOfTable(table);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+ }
+ }
+
+ @BeforeClass
+ public static synchronized void doSetup() throws Exception {
+ Map<String, String> props = Maps.newHashMapWithExpectedSize(2);
+ props.put(QueryServices.PHOENIX_SERVER_PAGE_SIZE_MS, Long.toString(0));
+ props.put(QueryServices.COLLECT_REQUEST_LEVEL_METRICS, String.valueOf(true));
+ props.put(QueryServices.TESTS_MINI_CLUSTER_NUM_REGION_SERVERS, String.valueOf(2));
+ props.put(HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY, String.valueOf(1));
+ props.put(QueryServices.PHOENIX_POST_DUMMY_PROCESS,
+ TestScanningResultPostDummyResultCaller.class.getName());
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ TABLE_NAMES.clear();
+ hasTestStarted = false;
+ countOfDummyResults = 0;
+ }
+
+ private static void moveRegionsOfTable(String tableName)
+ throws IOException {
+ if (StringUtils.isEmpty(tableName)) {
+ return;
+ }
+ Admin admin = getUtility().getAdmin();
+ List<ServerName> servers =
+ new ArrayList<>(admin.getRegionServers());
+ ServerName server1 = servers.get(0);
+ ServerName server2 = servers.get(1);
+ List<RegionInfo> regionsOnServer1;
+ try {
+ regionsOnServer1 = admin.getRegions(server1);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ List<RegionInfo> regionsOnServer2;
+ try {
+ regionsOnServer2 = admin.getRegions(server2);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ regionsOnServer1.forEach(regionInfo -> {
+ if (regionInfo.getTable().equals(TableName.valueOf(tableName))) {
+ try {
+ admin.move(regionInfo.getEncodedNameAsBytes(), server2);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ });
+ regionsOnServer2.forEach(regionInfo -> {
+ if (regionInfo.getTable().equals(TableName.valueOf(tableName))) {
+ try {
+ admin.move(regionInfo.getEncodedNameAsBytes(), server1);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testCoveredIndex() throws Exception {
+ hasTestStarted = true;
+ String dataTableName = generateUniqueName();
+ populateTable(dataTableName);
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ String indexTableName = generateUniqueName();
+ conn.createStatement().execute("CREATE INDEX "
+ + indexTableName + " on " + dataTableName + " (val1) INCLUDE (val2, val3)");
+ populateTableWithAdditionalRows(dataTableName, conn);
+ conn.commit();
+ TABLE_NAMES.add(dataTableName);
+ TABLE_NAMES.add(indexTableName);
+ String selectSql;
+
+ // Verify that an index hint is not necessary for an uncovered index
+ selectSql = "SELECT val2, val3 from " + dataTableName +
+ " WHERE val1 IS NOT NULL";
+
+ try (ResultSet rs = conn.createStatement().executeQuery(selectSql)) {
+ assertTrue(rs.next());
+ assertEquals("abc", rs.getString(1));
+ assertEquals("abcd", rs.getString(2));
+ moveRegionsOfTable(dataTableName);
+ moveRegionsOfTable(indexTableName);
+ try (Connection newConn = DriverManager.getConnection(getUrl())) {
+ newConn.createStatement().execute("upsert into " + dataTableName
+ + " (id, val1, val2, val3) values ('c1', 'cd1','cde1', 'cdef1')");
+ newConn.createStatement().execute("upsert into " + dataTableName
+ + " (id, val1, val2, val3) values ('g1', 'gh1','gh11', 'gh111')");
+ newConn.createStatement().execute("upsert into " + dataTableName
+ + " (id, val1, val2, val3) values ('l', 'lm','lmn', 'lmno')");
+ newConn.createStatement().execute("upsert into " + dataTableName
+ + " (id, val1, val2, val3) values ('m', 'mn','mno', 'mnop')");
+ newConn.commit();
+ }
+ assertRowsWithOldScanner(dataTableName, indexTableName, rs, true);
+ }
+
+ try (ResultSet rs = conn.createStatement().executeQuery(selectSql)) {
+ assertRowsWithNewScanner(dataTableName, indexTableName, rs, true);
+ }
+ }
+ }
+
+ @Test
+ public void testNoIndex() throws Exception {
+ hasTestStarted = true;
+ String dataTableName = generateUniqueName();
+ populateTable(dataTableName);
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ populateTableWithAdditionalRows(dataTableName, conn);
+ conn.commit();
+ TABLE_NAMES.add(dataTableName);
+ String selectSql;
+
+ // Verify that an index hint is not necessary for an uncovered index
+ selectSql = "SELECT val2, val3 from " + dataTableName +
+ " WHERE val1 IS NOT NULL";
+
+ try (ResultSet rs = conn.createStatement().executeQuery(selectSql)) {
+ assertTrue(rs.next());
+ assertEquals("abc", rs.getString(1));
+ assertEquals("abcd", rs.getString(2));
+ moveRegionsOfTable(dataTableName);
+ try (Connection newConn = DriverManager.getConnection(getUrl())) {
+ newConn.createStatement().execute("upsert into " + dataTableName
+ + " (id, val1, val2, val3) values ('c1', 'cd1','cde1', 'cdef1')");
+ newConn.createStatement().execute("upsert into " + dataTableName
+ + " (id, val1, val2, val3) values ('g1', 'gh1','gh11', 'gh111')");
+ newConn.createStatement().execute("upsert into " + dataTableName
+ + " (id, val1, val2, val3) values ('l', 'lm','lmn', 'lmno')");
+ newConn.createStatement().execute("upsert into " + dataTableName
+ + " (id, val1, val2, val3) values ('m', 'mn','mno', 'mnop')");
+ newConn.commit();
+ }
+ assertRowsWithOldScanner(dataTableName, "", rs, true);
+ }
+
+ try (ResultSet rs = conn.createStatement().executeQuery(selectSql)) {
+ assertRowsWithNewScanner(dataTableName, "", rs, true);
+ }
+ }
+ }
+
+ @Test
+ public void testCoveredIndex2() throws Exception {
+ hasTestStarted = true;
+ String dataTableName = generateUniqueName();
+ populateTable(dataTableName);
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ String indexTableName = generateUniqueName();
+ conn.createStatement().execute("CREATE INDEX "
+ + indexTableName + " on " + dataTableName + " (val1) INCLUDE (val2, val3)");
+ populateTableWithAdditionalRows(dataTableName, conn);
+ conn.commit();
+ String selectSql;
+
+ // Verify that an index hint is not necessary for an uncovered index
+ selectSql = "SELECT val2, val3 from " + dataTableName +
+ " WHERE val1 IS NOT NULL";
+
+ try (ResultSet rs = conn.createStatement().executeQuery(selectSql)) {
+ assertTrue(rs.next());
+ assertEquals("abc", rs.getString(1));
+ assertEquals("abcd", rs.getString(2));
+ try (Connection newConn = DriverManager.getConnection(getUrl())) {
+ newConn.createStatement().execute("upsert into " + dataTableName
+ + " (id, val1, val2, val3) values ('c1', 'cd1','cde1', 'cdef1')");
+ newConn.createStatement().execute("upsert into " + dataTableName
+ + " (id, val1, val2, val3) values ('g1', 'gh1','gh11', 'gh111')");
+ newConn.createStatement().execute("upsert into " + dataTableName
+ + " (id, val1, val2, val3) values ('l', 'lm','lmn', 'lmno')");
+ newConn.createStatement().execute("upsert into " + dataTableName
+ + " (id, val1, val2, val3) values ('m', 'mn','mno', 'mnop')");
+ newConn.commit();
+ }
+ assertRowsWithOldScanner(dataTableName, indexTableName, rs, false);
+ }
+
+ try (ResultSet rs = conn.createStatement().executeQuery(selectSql)) {
+ assertRowsWithNewScanner(dataTableName, indexTableName, rs, false);
+ }
+ }
+ }
+
+ @Test
+ public void testNoIndex2() throws Exception {
+ hasTestStarted = true;
+ String dataTableName = generateUniqueName();
+ populateTable(dataTableName);
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ populateTableWithAdditionalRows(dataTableName, conn);
+ conn.commit();
+ TABLE_NAMES.add(dataTableName);
+ String selectSql;
+
+ // Verify that an index hint is not necessary for an uncovered index
+ selectSql = "SELECT val2, val3 from " + dataTableName +
+ " WHERE val1 IS NOT NULL";
+
+ try (ResultSet rs = conn.createStatement().executeQuery(selectSql)) {
+ assertTrue(rs.next());
+ assertEquals("abc", rs.getString(1));
+ assertEquals("abcd", rs.getString(2));
+ try (Connection newConn = DriverManager.getConnection(getUrl())) {
+ newConn.createStatement().execute("upsert into " + dataTableName
+ + " (id, val1, val2, val3) values ('c1', 'cd1','cde1', 'cdef1')");
+ newConn.createStatement().execute("upsert into " + dataTableName
+ + " (id, val1, val2, val3) values ('g1', 'gh1','gh11', 'gh111')");
+ newConn.createStatement().execute("upsert into " + dataTableName
+ + " (id, val1, val2, val3) values ('l', 'lm','lmn', 'lmno')");
+ newConn.createStatement().execute("upsert into " + dataTableName
+ + " (id, val1, val2, val3) values ('m', 'mn','mno', 'mnop')");
+ newConn.commit();
+ }
+ assertRowsWithOldScanner(dataTableName, "", rs, false);
+ }
+
+ try (ResultSet rs = conn.createStatement().executeQuery(selectSql)) {
+ assertRowsWithNewScanner(dataTableName, "", rs, false);
+ }
+ }
+ }
+
+ private static void populateTableWithAdditionalRows(String dataTableName, Connection conn)
+ throws SQLException {
+ conn.createStatement().execute("upsert into " + dataTableName
+ + " (id, val1, val2, val3) values ('c', 'cd','cde', 'cdef')");
+ conn.createStatement().execute("upsert into " + dataTableName
+ + " (id, val1, val2, val3) values ('d', 'de','de1', 'de11')");
+ conn.createStatement().execute("upsert into " + dataTableName
+ + " (id, val1, val2, val3) values ('e', 'ef','ef1', 'ef11')");
+ conn.createStatement().execute("upsert into " + dataTableName
+ + " (id, val1, val2, val3) values ('f', 'fg','fg1', 'fg11')");
+ conn.createStatement().execute("upsert into " + dataTableName
+ + " (id, val1, val2, val3) values ('g', 'gh','gh1', 'gh11')");
+ conn.createStatement().execute("upsert into " + dataTableName
+ + " (id, val1, val2, val3) values ('h', 'hi','hi1', 'hi11')");
+ conn.createStatement().execute("upsert into " + dataTableName
+ + " (id, val1, val2, val3) values ('i', 'ij','ij1', 'ij11')");
+ conn.createStatement().execute("upsert into " + dataTableName
+ + " (id, val1, val2, val3) values ('j', 'jk','jk1', 'jk11')");
+ conn.createStatement().execute("upsert into " + dataTableName
+ + " (id, val1, val2, val3) values ('k', 'kl','kl1', 'kl11')");
+ }
+
+ private static void assertRowsWithNewScanner(String dataTableName, String indexTableName,
+ ResultSet rs, boolean moveRegions)
+ throws SQLException, IOException {
+ assertTrue(rs.next());
+ assertEquals("abc", rs.getString(1));
+ assertEquals("abcd", rs.getString(2));
+ if (moveRegions) {
+ moveRegionsOfTable(dataTableName);
+ moveRegionsOfTable(indexTableName);
+ }
+ assertTrue(rs.next());
+ assertEquals("bcd", rs.getString(1));
+ assertEquals("bcde", rs.getString(2));
+ if (moveRegions) {
+ moveRegionsOfTable(dataTableName);
+ moveRegionsOfTable(indexTableName);
+ }
+ assertTrue(rs.next());
+ assertEquals("cde", rs.getString(1));
+ assertEquals("cdef", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("cde1", rs.getString(1));
+ assertEquals("cdef1", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("de1", rs.getString(1));
+ assertEquals("de11", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("ef1", rs.getString(1));
+ assertEquals("ef11", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("fg1", rs.getString(1));
+ assertEquals("fg11", rs.getString(2));
+ if (moveRegions) {
+ moveRegionsOfTable(dataTableName);
+ moveRegionsOfTable(indexTableName);
+ }
+ assertTrue(rs.next());
+ assertEquals("gh1", rs.getString(1));
+ assertEquals("gh11", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("gh11", rs.getString(1));
+ assertEquals("gh111", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("hi1", rs.getString(1));
+ assertEquals("hi11", rs.getString(2));
+ if (moveRegions) {
+ moveRegionsOfTable(dataTableName);
+ moveRegionsOfTable(indexTableName);
+ }
+ assertTrue(rs.next());
+ assertEquals("ij1", rs.getString(1));
+ assertEquals("ij11", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("jk1", rs.getString(1));
+ assertEquals("jk11", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("kl1", rs.getString(1));
+ assertEquals("kl11", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("lmn", rs.getString(1));
+ assertEquals("lmno", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("mno", rs.getString(1));
+ assertEquals("mnop", rs.getString(2));
+ assertFalse(rs.next());
+ }
+
+ private static void assertRowsWithOldScanner(String dataTableName, String indexTableName,
+ ResultSet rs, boolean moveRegions)
+ throws SQLException, IOException {
+ assertTrue(rs.next());
+ assertEquals("bcd", rs.getString(1));
+ assertEquals("bcde", rs.getString(2));
+ if (moveRegions) {
+ moveRegionsOfTable(dataTableName);
+ moveRegionsOfTable(indexTableName);
+ }
+ assertTrue(rs.next());
+ assertEquals("cde", rs.getString(1));
+ assertEquals("cdef", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("de1", rs.getString(1));
+ assertEquals("de11", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("ef1", rs.getString(1));
+ assertEquals("ef11", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("fg1", rs.getString(1));
+ assertEquals("fg11", rs.getString(2));
+ if (moveRegions) {
+ moveRegionsOfTable(dataTableName);
+ moveRegionsOfTable(indexTableName);
+ }
+ assertTrue(rs.next());
+ assertEquals("gh1", rs.getString(1));
+ assertEquals("gh11", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("hi1", rs.getString(1));
+ assertEquals("hi11", rs.getString(2));
+ if (moveRegions) {
+ moveRegionsOfTable(dataTableName);
+ moveRegionsOfTable(indexTableName);
+ }
+ assertTrue(rs.next());
+ assertEquals("ij1", rs.getString(1));
+ assertEquals("ij11", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("jk1", rs.getString(1));
+ assertEquals("jk11", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("kl1", rs.getString(1));
+ assertEquals("kl11", rs.getString(2));
+ assertFalse(rs.next());
+ }
+
+ private void populateTable(String tableName) throws Exception {
+ Connection conn = DriverManager.getConnection(getUrl());
+ conn.createStatement().execute("create table " + tableName +
+ " (id varchar(10) not null primary key, val1 varchar(10), val2 varchar(10)," +
+ " val3 varchar(10)) SPLIT ON ('d', 'h', 'k')");
+ conn.createStatement().execute("upsert into " + tableName
+ + " values ('a', 'ab', 'abc', 'abcd')");
+ conn.commit();
+ conn.createStatement().execute("upsert into " + tableName
+ + " values ('b', 'bc', 'bcd', 'bcde')");
+ conn.commit();
+ conn.close();
+ }
+}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ServerPagingIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ServerPagingIT.java
index d4368ee1d7..4c03131d60 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ServerPagingIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ServerPagingIT.java
@@ -223,8 +223,7 @@ public class ServerPagingIT extends ParallelStatsDisabledIT {
assertEquals("Expected string didn't match for i = " + i, STRINGS[offset + i], rs.getString(1));
i++;
}
- // no paging when serial offset
- assertServerPagingMetric(tablename, rs, false);
+ assertServerPagingMetric(tablename, rs, true);
// Testing query with offset + filter
int filterCond = 10;
@@ -239,8 +238,7 @@ public class ServerPagingIT extends ParallelStatsDisabledIT {
STRINGS[offset + filterCond + i], rs.getString(1));
i++;
}
- // no paging when serial offset
- assertServerPagingMetric(tablename, rs, false);
+ assertServerPagingMetric(tablename, rs, true);
limit = 35;
rs = conn.createStatement().executeQuery("SELECT t_id from " + tablename + " union all SELECT t_id from "
@@ -255,8 +253,7 @@ public class ServerPagingIT extends ParallelStatsDisabledIT {
assertTrue(rs.next());
assertEquals(STRINGS[i - 1], rs.getString(1));
}
- // no paging when serial offset
- assertServerPagingMetric(tablename, rs, false);
+ assertServerPagingMetric(tablename, rs, true);
limit = 1;
offset = 1;
rs = conn.createStatement()
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ServerPagingWithRegionMovesIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ServerPagingWithRegionMovesIT.java
new file mode 100644
index 0000000000..76aea5b681
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ServerPagingWithRegionMovesIT.java
@@ -0,0 +1,840 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.iterate.ScanningResultPostDummyResultCaller;
+import org.apache.phoenix.iterate.ScanningResultPostValidResultCaller;
+import org.apache.phoenix.monitoring.MetricType;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.schema.types.PDate;
+import org.apache.phoenix.thirdparty.com.google.common.collect.Maps;
+import org.apache.phoenix.util.ByteUtil;
+import org.apache.phoenix.util.DateUtil;
+import org.apache.phoenix.util.PhoenixRuntime;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.phoenix.util.TestUtil;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.Date;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import static org.apache.phoenix.monitoring.GlobalClientMetrics.GLOBAL_PAGED_ROWS_COUNTER;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * ServerPagingIT tests that include some region moves while performing rs#next.
+ */
+@Category(NeedsOwnMiniClusterTest.class)
+public class ServerPagingWithRegionMovesIT extends ParallelStatsDisabledIT {
+
+ private static final Logger LOGGER =
+ LoggerFactory.getLogger(ServerPagingWithRegionMovesIT.class);
+
+ private static boolean hasTestStarted = false;
+ private static int countOfDummyResults = 0;
+ private static int countOfValidResults = 0;
+ private static final List<String> TABLE_NAMES = Collections.synchronizedList(new ArrayList<>());
+
+ private static class TestScanningResultPostValidResultCaller extends
+ ScanningResultPostValidResultCaller {
+
+ @Override
+ public void postValidRowProcess() {
+ if (hasTestStarted && (countOfValidResults++ % 2) == 0 &&
+ (countOfValidResults < 17 ||
+ countOfValidResults > 28 && countOfValidResults < 40)) {
+ LOGGER.info("Moving regions of tables {}. current count of valid results: {}",
+ TABLE_NAMES, countOfValidResults);
+ TABLE_NAMES.forEach(table -> {
+ try {
+ moveRegionsOfTable(table);
+ } catch (Exception e) {
+ LOGGER.error("Unable to move regions of table: {}", table);
+ }
+ });
+ }
+ }
+ }
+
+ private static class TestScanningResultPostDummyResultCaller extends
+ ScanningResultPostDummyResultCaller {
+
+ @Override
+ public void postDummyProcess() {
+ if (hasTestStarted && (countOfDummyResults++ % 3) == 0 &&
+ (countOfDummyResults < 17 ||
+ countOfDummyResults > 28 && countOfDummyResults < 40)) {
+ LOGGER.info("Moving regions of tables {}. current count of dummy results: {}",
+ TABLE_NAMES, countOfDummyResults);
+ TABLE_NAMES.forEach(table -> {
+ try {
+ moveRegionsOfTable(table);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+ }
+ }
+
+ @BeforeClass
+ public static synchronized void doSetup() throws Exception {
+ Map<String, String> props = Maps.newHashMapWithExpectedSize(2);
+ props.put(QueryServices.PHOENIX_SERVER_PAGE_SIZE_MS, Long.toString(0));
+ props.put(QueryServices.COLLECT_REQUEST_LEVEL_METRICS, String.valueOf(true));
+ props.put(QueryServices.TESTS_MINI_CLUSTER_NUM_REGION_SERVERS, String.valueOf(2));
+ props.put(HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY, String.valueOf(1));
+ props.put(QueryServices.PHOENIX_POST_DUMMY_PROCESS,
+ TestScanningResultPostDummyResultCaller.class.getName());
+ props.put(QueryServices.PHOENIX_POST_VALID_PROCESS,
+ TestScanningResultPostValidResultCaller.class.getName());
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ TABLE_NAMES.clear();
+ hasTestStarted = false;
+ countOfDummyResults = 0;
+ countOfValidResults = 0;
+ }
+
+ private void assertServerPagingMetric(String tableName, ResultSet rs, boolean isPaged)
+ throws SQLException {
+ Map<String, Map<MetricType, Long>> metrics = PhoenixRuntime.getRequestReadMetricInfo(rs);
+ for (Map.Entry<String, Map<MetricType, Long>> entry : metrics.entrySet()) {
+ assertEquals(String.format("Got %s", entry.getKey()), tableName, entry.getKey());
+ Map<MetricType, Long> metricValues = entry.getValue();
+ Long pagedRowsCntr = metricValues.get(MetricType.PAGED_ROWS_COUNTER);
+ assertNotNull(pagedRowsCntr);
+ if (isPaged) {
+ assertTrue(String.format("Got %d", pagedRowsCntr), pagedRowsCntr > 0);
+ } else {
+ assertEquals(String.format("Got %d", pagedRowsCntr), 0, (long) pagedRowsCntr);
+ }
+ }
+ assertTrue(GLOBAL_PAGED_ROWS_COUNTER.getMetric().getValue() > 0);
+ }
+
+ @Test
+ public void testOrderByNonAggregation() throws Exception {
+ hasTestStarted = true;
+ final String tablename = generateUniqueName();
+ final String tenantId = getOrganizationId();
+
+ final Date D1 = DateUtil.parseDate("1970-01-01 00:58:00");
+ final Date D2 = DateUtil.parseDate("1970-01-01 01:02:00");
+ final Date D3 = DateUtil.parseDate("1970-01-01 01:30:00");
+ final Date D4 = DateUtil.parseDate("1970-01-01 01:45:00");
+ final Date D5 = DateUtil.parseDate("1970-01-01 02:00:00");
+ final Date D6 = DateUtil.parseDate("1970-01-01 04:00:00");
+ final String F1 = "A";
+ final String F2 = "B";
+ final String F3 = "C";
+ final String R1 = "R1";
+ final String R2 = "R2";
+ byte[][] splits = new byte[][] {
+ ByteUtil.concat(Bytes.toBytes(tenantId), PDate.INSTANCE.toBytes(D3)),
+ ByteUtil.concat(Bytes.toBytes(tenantId), PDate.INSTANCE.toBytes(D5)),
+ };
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String ddl = "create table " + tablename +
+ " (organization_id char(15) not null," +
+ " date date not null," +
+ " feature char(1) not null," +
+ " unique_users integer not null,\n" +
+ " transactions bigint,\n" +
+ " region varchar,\n" +
+ " CONSTRAINT pk PRIMARY KEY (organization_id, \"DATE\", feature, unique_users))";
+ TABLE_NAMES.add(tablename);
+ StringBuilder buf = new StringBuilder(ddl);
+ if (splits != null) {
+ buf.append(" SPLIT ON (");
+ for (int i = 0; i < splits.length; i++) {
+ buf.append("'").append(Bytes.toString(splits[i])).append("'").append(",");
+ }
+ buf.setCharAt(buf.length()-1, ')');
+ }
+ ddl = buf.toString();
+ conn.createStatement().execute(ddl);
+
+ PreparedStatement stmt = conn.prepareStatement(
+ "upsert into " + tablename +
+ " (" +
+ " ORGANIZATION_ID, " +
+ " \"DATE\", " +
+ " FEATURE, " +
+ " UNIQUE_USERS, " +
+ " TRANSACTIONS, " +
+ " REGION) " +
+ "VALUES (?, ?, ?, ?, ?, ?)");
+ stmt.setString(1, tenantId);
+ stmt.setDate(2, D1);
+ stmt.setString(3, F1);
+ stmt.setInt(4, 10);
+ stmt.setLong(5, 100L);
+ stmt.setString(6, R2);
+ stmt.execute();
+
+ stmt.setString(1, tenantId);
+ stmt.setDate(2, D2);
+ stmt.setString(3, F1);
+ stmt.setInt(4, 20);
+ stmt.setLong(5, 200);
+ stmt.setString(6, null);
+ stmt.execute();
+
+ stmt.setString(1, tenantId);
+ stmt.setDate(2, D3);
+ stmt.setString(3, F1);
+ stmt.setInt(4, 30);
+ stmt.setLong(5, 300);
+ stmt.setString(6, R1);
+ stmt.execute();
+
+ stmt.setString(1, tenantId);
+ stmt.setDate(2, D4);
+ stmt.setString(3, F2);
+ stmt.setInt(4, 40);
+ stmt.setLong(5, 400);
+ stmt.setString(6, R1);
+ stmt.execute();
+
+ stmt.setString(1, tenantId);
+ stmt.setDate(2, D5);
+ stmt.setString(3, F3);
+ stmt.setInt(4, 50);
+ stmt.setLong(5, 500);
+ stmt.setString(6, R2);
+ stmt.execute();
+
+ stmt.setString(1, tenantId);
+ stmt.setDate(2, D6);
+ stmt.setString(3, F1);
+ stmt.setInt(4, 60);
+ stmt.setLong(5, 600);
+ stmt.setString(6, null);
+ stmt.execute();
+ conn.commit();
+ }
+
+ String query = "SELECT \"DATE\", transactions t FROM "+tablename+
+ " WHERE organization_id=? AND unique_users <= 30 ORDER BY t DESC LIMIT 2";
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ PreparedStatement statement = conn.prepareStatement(query)) {
+ TestUtil.dumpTable(conn, TableName.valueOf(tablename));
+ statement.setString(1, tenantId);
+ try (ResultSet rs = statement.executeQuery()) {
+ moveRegionsOfTable(tablename);
+ assertTrue(rs.next());
+ assertEquals(D3.getTime(), rs.getDate(1).getTime());
+ moveRegionsOfTable(tablename);
+ assertTrue(rs.next());
+ moveRegionsOfTable(tablename);
+ assertEquals(D2.getTime(), rs.getDate(1).getTime());
+ assertFalse(rs.next());
+ assertServerPagingMetric(tablename, rs, true);
+ }
+ }
+ }
+
+ private static void moveRegionsOfTable(String tableName)
+ throws IOException {
+ Admin admin = getUtility().getAdmin();
+ List<ServerName> servers =
+ new ArrayList<>(admin.getRegionServers());
+ ServerName server1 = servers.get(0);
+ ServerName server2 = servers.get(1);
+ List<RegionInfo> regionsOnServer1;
+ try {
+ regionsOnServer1 = admin.getRegions(server1);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ List<RegionInfo> regionsOnServer2;
+ try {
+ regionsOnServer2 = admin.getRegions(server2);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ regionsOnServer1.forEach(regionInfo -> {
+ if (regionInfo.getTable().equals(TableName.valueOf(tableName))) {
+ try {
+ admin.move(regionInfo.getEncodedNameAsBytes(), server2);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ });
+ regionsOnServer2.forEach(regionInfo -> {
+ if (regionInfo.getTable().equals(TableName.valueOf(tableName))) {
+ try {
+ admin.move(regionInfo.getEncodedNameAsBytes(), server1);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ });
+ }
+
+ private static void moveAllRegions() throws IOException {
+ Admin admin = getUtility().getAdmin();
+ List<ServerName> servers =
+ new ArrayList<>(admin.getRegionServers());
+ ServerName server1 = servers.get(0);
+ ServerName server2 = servers.get(1);
+ List<RegionInfo> regionsOnServer1;
+ try {
+ regionsOnServer1 = admin.getRegions(server1);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ List<RegionInfo> regionsOnServer2;
+ try {
+ regionsOnServer2 = admin.getRegions(server2);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ regionsOnServer1.forEach(regionInfo -> {
+ try {
+ admin.move(regionInfo.getEncodedNameAsBytes(), server2);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ regionsOnServer2.forEach(regionInfo -> {
+ try {
+ admin.move(regionInfo.getEncodedNameAsBytes(), server1);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
+ }
+
+ @Test
+ public void testLimitOffsetWithSplit() throws Exception {
+ hasTestStarted = true;
+ final String tablename = generateUniqueName();
+ final String[] STRINGS = { "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l",
+ "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z" };
+ String ddl = "CREATE TABLE " + tablename + " (t_id VARCHAR NOT NULL,\n"
+ + "k1 INTEGER NOT NULL,\n" + "k2 INTEGER NOT NULL,\n" + "C3.k3 INTEGER,\n"
+ + "C2.v1 VARCHAR,\n" + "CONSTRAINT pk PRIMARY KEY (t_id, k1, k2)) "
+ + "SPLIT ON ('e','i','o')";
+ TABLE_NAMES.add(tablename);
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ createTestTable(getUrl(), ddl);
+ for (int i = 0; i < 26; i++) {
+ conn.createStatement().execute("UPSERT INTO " + tablename + " values('"
+ + STRINGS[i] + "'," + i + ","
+ + (i + 1) + "," + (i + 2) + ",'" + STRINGS[25 - i] + "')");
+ }
+ conn.commit();
+ int limit = 10;
+ // Testing 0 as remaining offset after 4 rows in first region, 4 rows in second region
+ int offset = 8;
+ ResultSet rs;
+ rs = conn.createStatement()
+ .executeQuery("SELECT t_id from " + tablename + " order by t_id limit "
+ + limit + " offset " + offset);
+ int i = 0;
+ while (i < limit) {
+ if (i % 3 == 0) {
+ moveRegionsOfTable(tablename);
+ }
+ assertTrue(rs.next());
+ assertEquals("Expected string didn't match for i = " + i, STRINGS[offset + i],
+ rs.getString(1));
+ i++;
+ }
+ assertServerPagingMetric(tablename, rs, true);
+
+ // Testing query with offset + filter
+ int filterCond = 10;
+ rs = conn.createStatement().executeQuery(
+ "SELECT t_id from " + tablename + " where k2 > " + filterCond +
+ " order by t_id limit " + limit + " offset " + offset);
+ i = 0;
+ limit = 5;
+ while (i < limit) {
+ if (i % 4 == 0) {
+ moveRegionsOfTable(tablename);
+ }
+ assertTrue(rs.next());
+ assertEquals("Expected string didn't match for i = " + i,
+ STRINGS[offset + filterCond + i], rs.getString(1));
+ i++;
+ }
+ assertServerPagingMetric(tablename, rs, true);
+
+ limit = 35;
+ rs = conn.createStatement().executeQuery("SELECT t_id from " + tablename
+ + " union all SELECT t_id from "
+ + tablename + " offset " + offset + " FETCH FIRST " + limit + " rows only");
+ i = 0;
+ while (i++ < STRINGS.length - offset) {
+ if (i % 3 == 0) {
+ moveRegionsOfTable(tablename);
+ }
+ assertTrue(rs.next());
+ assertEquals(STRINGS[offset + i - 1], rs.getString(1));
+ }
+ i = 0;
+ while (i++ < limit - STRINGS.length - offset) {
+ if (i % 3 == 0) {
+ moveRegionsOfTable(tablename);
+ }
+ assertTrue(rs.next());
+ assertEquals(STRINGS[i - 1], rs.getString(1));
+ }
+ // no paging when serial offset
+ assertServerPagingMetric(tablename, rs, true);
+ limit = 1;
+ offset = 1;
+ rs = conn.createStatement()
+ .executeQuery("SELECT k2 from " + tablename + " order by k2 desc limit "
+ + limit + " offset " + offset);
+ moveRegionsOfTable(tablename);
+ assertTrue(rs.next());
+ assertEquals(25, rs.getInt(1));
+ assertFalse(rs.next());
+ assertServerPagingMetric(tablename, rs, true);
+ }
+ }
+
+ @Test
+ public void testLimitOffsetWithoutSplit() throws Exception {
+ hasTestStarted = true;
+ final String tablename = generateUniqueName();
+ final String[] STRINGS = { "a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l",
+ "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z" };
+ String ddl = "CREATE TABLE " + tablename + " (t_id VARCHAR NOT NULL,\n"
+ + "k1 INTEGER NOT NULL,\n" + "k2 INTEGER NOT NULL,\n" + "C3.k3 INTEGER,\n"
+ + "C2.v1 VARCHAR,\n" + "CONSTRAINT pk PRIMARY KEY (t_id, k1, k2))";
+ TABLE_NAMES.add(tablename);
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ createTestTable(getUrl(), ddl);
+ for (int i = 0; i < 26; i++) {
+ conn.createStatement().execute("UPSERT INTO " + tablename + " values('"
+ + STRINGS[i] + "'," + i + ","
+ + (i + 1) + "," + (i + 2) + ",'" + STRINGS[25 - i] + "')");
+ }
+ conn.commit();
+ int limit = 10;
+ // Testing 0 as remaining offset after 4 rows in first region, 4 rows in second region
+ int offset = 8;
+ ResultSet rs;
+ rs = conn.createStatement()
+ .executeQuery("SELECT t_id from " + tablename + " order by t_id limit "
+ + limit + " offset " + offset);
+ int i = 0;
+ while (i < limit) {
+ if (i % 3 == 0) {
+ moveRegionsOfTable(tablename);
+ }
+ assertTrue(rs.next());
+ assertEquals("Expected string didn't match for i = " + i, STRINGS[offset + i],
+ rs.getString(1));
+ i++;
+ }
+ assertServerPagingMetric(tablename, rs, false);
+
+ // Testing query with offset + filter
+ int filterCond = 10;
+ rs = conn.createStatement().executeQuery(
+ "SELECT t_id from " + tablename + " where k2 > " + filterCond +
+ " order by t_id limit " + limit + " offset " + offset);
+ i = 0;
+ limit = 5;
+ while (i < limit) {
+ if (i % 4 == 0) {
+ moveRegionsOfTable(tablename);
+ }
+ assertTrue(rs.next());
+ assertEquals("Expected string didn't match for i = " + i,
+ STRINGS[offset + filterCond + i], rs.getString(1));
+ i++;
+ }
+ assertServerPagingMetric(tablename, rs, false);
+
+ limit = 35;
+ rs = conn.createStatement().executeQuery("SELECT t_id from " + tablename
+ + " union all SELECT t_id from "
+ + tablename + " offset " + offset + " FETCH FIRST " + limit + " rows only");
+ i = 0;
+ while (i++ < STRINGS.length - offset) {
+ if (i % 3 == 0) {
+ moveRegionsOfTable(tablename);
+ }
+ assertTrue(rs.next());
+ assertEquals(STRINGS[offset + i - 1], rs.getString(1));
+ }
+ i = 0;
+ while (i++ < limit - STRINGS.length - offset) {
+ if (i % 3 == 0) {
+ moveRegionsOfTable(tablename);
+ }
+ assertTrue(rs.next());
+ assertEquals(STRINGS[i - 1], rs.getString(1));
+ }
+ assertServerPagingMetric(tablename, rs, false);
+ limit = 1;
+ offset = 1;
+ rs = conn.createStatement()
+ .executeQuery("SELECT k2 from " + tablename + " order by k2 desc limit "
+ + limit + " offset " + offset);
+ moveRegionsOfTable(tablename);
+ assertTrue(rs.next());
+ assertEquals(25, rs.getInt(1));
+ assertFalse(rs.next());
+ assertServerPagingMetric(tablename, rs, true);
+ }
+ }
+
+ @Test
+ public void testLimitOffsetWithSplit2() throws Exception {
+ hasTestStarted = true;
+ final String tablename = generateUniqueName();
+ final String[] STRINGS = {"a_xyz", "b_xyz", "c_xyz", "d_xyz", "e_xyz", "f_xyz", "g_xyz",
+ "h_xyz", "i_xyz", "j_xyz", "k_xyz", "l_xyz",
+ "m_xyz", "n_xyz", "o_xyz", "p_xyz", "q_xyz", "r_xyz", "s_xyz", "t_xyz", "u_xyz",
+ "v_xyz", "w_xyz", "x_xyz", "y_xyz", "z_xyz"};
+ String ddl = "CREATE TABLE " + tablename + " (t_id VARCHAR,\n"
+ + "k1 INTEGER,\n" + "k2 INTEGER,\n" + "C3.k3 INTEGER,\n"
+ + "C2.v1 VARCHAR,\n" + "CONSTRAINT pk PRIMARY KEY (t_id)) "
+ + "SPLIT ON ('e123','i123','o123')";
+ TABLE_NAMES.add(tablename);
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ createTestTable(getUrl(), ddl);
+ for (int i = 0; i < 26; i++) {
+ conn.createStatement().execute("UPSERT INTO " + tablename + " values('"
+ + STRINGS[i] + "'," + (i + 1) * 2 + ","
+ + (i + 1) * 3 + "," + (i + 2) * 2 + ",'" + STRINGS[25 - i] + "')");
+ }
+ conn.commit();
+ int limit = 10;
+ // Testing 0 as remaining offset after 4 rows in first region, 4 rows in second region
+ int offset = 8;
+ ResultSet rs;
+ rs = conn.createStatement()
+ .executeQuery("SELECT t_id from " + tablename + " order by t_id limit "
+ + limit + " offset " + offset);
+ int i = 0;
+ while (i < limit) {
+ if (i % 3 == 0) {
+ moveRegionsOfTable(tablename);
+ }
+ assertTrue(rs.next());
+ assertEquals("Expected string didn't match for i = " + i, STRINGS[offset + i],
+ rs.getString(1));
+ i++;
+ }
+ assertServerPagingMetric(tablename, rs, true);
+
+ // Testing query with offset + filter
+ int filterCond = 30;
+ rs = conn.createStatement().executeQuery(
+ "SELECT t_id from " + tablename + " where k2 > " + filterCond +
+ " order by t_id limit " + limit + " offset " + offset);
+ i = 0;
+ limit = 5;
+ while (i < limit) {
+ if (i % 4 == 0) {
+ moveRegionsOfTable(tablename);
+ }
+ assertTrue(rs.next());
+ assertEquals("Expected string didn't match for i = " + i,
+ STRINGS[offset + 10 + i], rs.getString(1));
+ i++;
+ }
+ assertServerPagingMetric(tablename, rs, true);
+
+ limit = 35;
+ rs = conn.createStatement().executeQuery("SELECT t_id from " + tablename
+ + " union all SELECT t_id from "
+ + tablename + " offset " + offset + " FETCH FIRST " + limit + " rows only");
+ i = 0;
+ while (i++ < STRINGS.length - offset) {
+ if (i % 3 == 0) {
+ moveRegionsOfTable(tablename);
+ }
+ assertTrue(rs.next());
+ assertEquals(STRINGS[offset + i - 1], rs.getString(1));
+ }
+ i = 0;
+ while (i++ < limit - STRINGS.length - offset) {
+ if (i % 3 == 0) {
+ moveRegionsOfTable(tablename);
+ }
+ assertTrue(rs.next());
+ assertEquals(STRINGS[i - 1], rs.getString(1));
+ }
+ // no paging when serial offset
+ assertServerPagingMetric(tablename, rs, true);
+ limit = 1;
+ offset = 1;
+ rs = conn.createStatement()
+ .executeQuery("SELECT k2 from " + tablename + " order by k2 desc limit "
+ + limit + " offset " + offset);
+ moveRegionsOfTable(tablename);
+ assertTrue(rs.next());
+ assertEquals(75, rs.getInt(1));
+ assertFalse(rs.next());
+ assertServerPagingMetric(tablename, rs, true);
+ }
+ }
+
+ @Test
+ public void testLimitOffsetWithoutSplit2() throws Exception {
+ hasTestStarted = true;
+ final String tablename = generateUniqueName();
+ final String[] STRINGS = {"a_xyz", "b_xyz", "c_xyz", "d_xyz", "e_xyz", "f_xyz", "g_xyz",
+ "h_xyz", "i_xyz", "j_xyz", "k_xyz", "l_xyz",
+ "m_xyz", "n_xyz", "o_xyz", "p_xyz", "q_xyz", "r_xyz", "s_xyz", "t_xyz", "u_xyz",
+ "v_xyz", "w_xyz", "x_xyz", "y_xyz", "z_xyz"};
+ String ddl = "CREATE TABLE " + tablename + " (t_id VARCHAR,\n"
+ + "k1 INTEGER,\n" + "k2 INTEGER,\n" + "C3.k3 INTEGER,\n"
+ + "C2.v1 VARCHAR,\n" + "CONSTRAINT pk PRIMARY KEY (t_id))";
+ TABLE_NAMES.add(tablename);
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ createTestTable(getUrl(), ddl);
+ for (int i = 0; i < 26; i++) {
+ conn.createStatement().execute("UPSERT INTO " + tablename + " values('"
+ + STRINGS[i] + "'," + (i + 1) * 2 + ","
+ + (i + 1) * 3 + "," + (i + 2) * 2 + ",'" + STRINGS[25 - i] + "')");
+ }
+ conn.commit();
+ int limit = 10;
+ // Testing 0 as remaining offset after 4 rows in first region, 4 rows in second region
+ int offset = 8;
+ ResultSet rs;
+ rs = conn.createStatement()
+ .executeQuery("SELECT t_id from " + tablename + " order by t_id limit "
+ + limit + " offset " + offset);
+ int i = 0;
+ while (i < limit) {
+ if (i % 3 == 0) {
+ moveRegionsOfTable(tablename);
+ }
+ assertTrue(rs.next());
+ assertEquals("Expected string didn't match for i = " + i, STRINGS[offset + i],
+ rs.getString(1));
+ i++;
+ }
+ assertServerPagingMetric(tablename, rs, false);
+
+ // Testing query with offset + filter
+ int filterCond = 30;
+ rs = conn.createStatement().executeQuery(
+ "SELECT t_id from " + tablename + " where k2 > " + filterCond +
+ " order by t_id limit " + limit + " offset " + offset);
+ i = 0;
+ limit = 5;
+ while (i < limit) {
+ if (i % 4 == 0) {
+ moveRegionsOfTable(tablename);
+ }
+ assertTrue(rs.next());
+ assertEquals("Expected string didn't match for i = " + i,
+ STRINGS[offset + 10 + i], rs.getString(1));
+ i++;
+ }
+ assertServerPagingMetric(tablename, rs, false);
+
+ limit = 35;
+ rs = conn.createStatement().executeQuery("SELECT t_id from " + tablename
+ + " union all SELECT t_id from "
+ + tablename + " offset " + offset + " FETCH FIRST " + limit + " rows only");
+ i = 0;
+ while (i++ < STRINGS.length - offset) {
+ if (i % 3 == 0) {
+ moveRegionsOfTable(tablename);
+ }
+ assertTrue(rs.next());
+ assertEquals(STRINGS[offset + i - 1], rs.getString(1));
+ }
+ i = 0;
+ while (i++ < limit - STRINGS.length - offset) {
+ if (i % 3 == 0) {
+ moveRegionsOfTable(tablename);
+ }
+ assertTrue(rs.next());
+ assertEquals(STRINGS[i - 1], rs.getString(1));
+ }
+ // no paging when serial offset
+ assertServerPagingMetric(tablename, rs, false);
+ limit = 1;
+ offset = 1;
+ rs = conn.createStatement()
+ .executeQuery("SELECT k2 from " + tablename + " order by k2 desc limit "
+ + limit + " offset " + offset);
+ moveRegionsOfTable(tablename);
+ assertTrue(rs.next());
+ assertEquals(75, rs.getInt(1));
+ assertFalse(rs.next());
+ assertServerPagingMetric(tablename, rs, true);
+ }
+ }
+
+ @Test
+ public void testGroupBy() throws Exception {
+ hasTestStarted = true;
+ final String tablename = generateUniqueName();
+ TABLE_NAMES.add(tablename);
+ String ddl = "CREATE TABLE " + tablename + " (t_id VARCHAR NOT NULL,\n"
+ + "k1 INTEGER NOT NULL,\n"
+ + "k2 INTEGER CONSTRAINT pk PRIMARY KEY (t_id, k1)) ";
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ createTestTable(getUrl(), ddl);
+ for (int i = 0; i < 8; i++) {
+ conn.createStatement().execute("UPSERT INTO " + tablename
+ + " values('tenant1'," + i + ","
+ + (i + 1) + ")");
+ }
+ conn.createStatement()
+ .execute("UPSERT INTO " + tablename + " values('tenant1', 10, 2)");
+ conn.createStatement()
+ .execute("UPSERT INTO " + tablename + " values('tenant1', 11, 2)");
+ conn.createStatement()
+ .execute("UPSERT INTO " + tablename + " values('tenant1', 12, 3)");
+ conn.createStatement()
+ .execute("UPSERT INTO " + tablename + " values('tenant1', 13, 3)");
+ conn.createStatement()
+ .execute("UPSERT INTO " + tablename + " values('tenant1', 14, 4)");
+ conn.createStatement()
+ .execute("UPSERT INTO " + tablename + " values('tenant1', 15, 4)");
+ conn.createStatement()
+ .execute("UPSERT INTO " + tablename + " values('tenant1', 16, 4)");
+ conn.createStatement()
+ .execute("UPSERT INTO " + tablename + " values('tenant1', 17, 5)");
+ conn.createStatement()
+ .execute("UPSERT INTO " + tablename + " values('tenant1', 18, 5)");
+ conn.commit();
+ TestUtil.dumpTable(conn, TableName.valueOf(tablename));
+ ResultSet rs =
+ conn.createStatement().executeQuery("SELECT k2, count(*) FROM " + tablename
+ + " where t_id = 'tenant1' group by k2");
+ moveRegionsOfTable(tablename);
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(1, rs.getInt(1));
+ Assert.assertEquals(1, rs.getInt(2));
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(2, rs.getInt(1));
+ Assert.assertEquals(3, rs.getInt(2));
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(3, rs.getInt(1));
+ Assert.assertEquals(3, rs.getInt(2));
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(4, rs.getInt(1));
+ Assert.assertEquals(4, rs.getInt(2));
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(5, rs.getInt(1));
+ Assert.assertEquals(3, rs.getInt(2));
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(6, rs.getInt(1));
+ Assert.assertEquals(1, rs.getInt(2));
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(7, rs.getInt(1));
+ Assert.assertEquals(1, rs.getInt(2));
+ Assert.assertTrue(rs.next());
+ Assert.assertEquals(8, rs.getInt(1));
+ Assert.assertEquals(1, rs.getInt(2));
+ Assert.assertFalse(rs.next());
+ assertServerPagingMetric(tablename, rs, true);
+ }
+ }
+
+ @Test
+ public void testGroupByWithIndex() throws Exception {
+ hasTestStarted = true;
+ final String tablename = generateUniqueName();
+ final String indexName = generateUniqueName();
+ TABLE_NAMES.add(tablename);
+ TABLE_NAMES.add(indexName);
+
+ String ddl = "CREATE TABLE " + tablename + " (t_id VARCHAR NOT NULL,\n"
+ + "k1 INTEGER NOT NULL,\n"
+ + "k2 INTEGER CONSTRAINT pk PRIMARY KEY (t_id, k1)) ";
+ String indexDDl = "CREATE INDEX IF NOT EXISTS " + indexName + " ON "
+ + tablename + "(k2)";
+
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ createTestTable(getUrl(), ddl);
+ createTestTable(getUrl(), indexDDl);
+ for (int i = 0; i < 8; i++) {
+ conn.createStatement().execute("UPSERT INTO " + tablename
+ + " values('tenant1'," + i + ","
+ + (i + 1) + ")");
+ }
+ conn.commit();
+ ResultSet rs = conn.createStatement().executeQuery("SELECT count(*) FROM " + tablename
+ + " where t_id = 'tenant1' AND (k2 IN (5,6) or k2 is null) group by k2=6");
+ boolean moveRegions = true;
+ moveRegionsOfTable(tablename);
+ moveRegionsOfTable(indexName);
+ while (rs.next()) {
+ if (moveRegions) {
+ moveRegionsOfTable(tablename);
+ moveRegionsOfTable(indexName);
+ moveRegions = false;
+ } else {
+ moveRegions = true;
+ }
+ Assert.assertEquals(1, rs.getInt(1));
+ }
+ Assert.assertFalse(rs.next());
+ assertServerPagingMetric(indexName, rs, true);
+ }
+ }
+
+}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortOrderIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortOrderIT.java
index 86a0666d31..e52dbb4e23 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortOrderIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortOrderIT.java
@@ -34,8 +34,10 @@ import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collections;
import java.util.GregorianCalendar;
+import java.util.HashSet;
import java.util.List;
import java.util.Properties;
+import java.util.Set;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.phoenix.schema.SortOrder;
@@ -44,6 +46,7 @@ import org.apache.phoenix.schema.types.PDecimal;
import org.apache.phoenix.schema.types.PDouble;
import org.apache.phoenix.schema.types.PFloat;
import org.apache.phoenix.util.PropertiesUtil;
+import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -57,16 +60,23 @@ import org.apache.phoenix.thirdparty.com.google.common.collect.Lists;
@Category(ParallelStatsDisabledTest.class)
public class SortOrderIT extends ParallelStatsDisabledIT {
- private String baseTableName;
-
+ protected String baseTableName;
+ protected static final Set<String> TABLE_NAMES = new HashSet<>();
+
@Before
public void generateTableName() {
baseTableName = generateUniqueName();
}
-
+
+ @After
+ public void tearDown() throws Exception {
+ TABLE_NAMES.clear();
+ }
+
@Test
public void noOrder() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (pk VARCHAR NOT NULL PRIMARY KEY)";
runQueryTest(ddl, "pk", new Object[][]{{"a"}, {"b"}, {"c"}}, new Object[][]{{"a"}, {"b"}, {"c"}},
table);
@@ -75,6 +85,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void noOrderCompositePK() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (oid CHAR(2) NOT NULL, code INTEGER NOT NULL constraint pk primary key (oid, code))";
Object[][] rows = new Object[][]{{"o1", 1}, {"o2", 2}, {"o3", 3}};
runQueryTest(ddl, upsert("oid", "code"), rows, rows, table);
@@ -83,6 +94,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void ascOrderInlinePK() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (pk VARCHAR NOT NULL PRIMARY KEY ASC)";
runQueryTest(ddl, "pk", new Object[][]{{"a"}, {"b"}, {"c"}}, new Object[][]{{"a"}, {"b"}, {"c"}},
table);
@@ -91,6 +103,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void ascOrderCompositePK() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (oid CHAR(2) NOT NULL, code INTEGER NOT NULL constraint pk primary key (oid ASC, code DESC))";
Object[][] insertedRows = new Object[][]{{"o1", 1}, {"o1", 2}, {"o1", 3}};
Object[][] expectedRows = new Object[][]{{"o1", 3}, {"o1", 2}, {"o1", 1}};
@@ -100,6 +113,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void descOrderInlinePK() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
for (String type : new String[]{"CHAR(2)", "VARCHAR"}) {
String ddl = "CREATE table " + table + " (pk ${type} NOT NULL PRIMARY KEY DESC)".replace("${type}", type);
runQueryTest(ddl, "pk", new Object[][]{{"aa"}, {"bb"}, {"cc"}}, new Object[][]{{"cc"}, {"bb"}, {"aa"}},
@@ -110,6 +124,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void descOrderCompositePK1() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (oid CHAR(2) NOT NULL, code INTEGER NOT NULL constraint pk primary key (oid DESC, code))";
Object[][] insertedRows = new Object[][]{{"o1", 1}, {"o2", 2}, {"o3", 3}};
Object[][] expectedRows = new Object[][]{{"o3", 3}, {"o2", 2}, {"o1", 1}};
@@ -119,6 +134,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void descOrderCompositePK2() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (oid CHAR(2) NOT NULL, code INTEGER NOT NULL constraint pk primary key (oid DESC, code DESC))";
Object[][] insertedRows = new Object[][]{{"o1", 1}, {"o1", 2}, {"o1", 3}};
Object[][] expectedRows = new Object[][]{{"o1", 3}, {"o1", 2}, {"o1", 1}};
@@ -128,6 +144,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void equalityDescInlinePK() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (pk VARCHAR NOT NULL PRIMARY KEY DESC)";
runQueryTest(ddl, upsert("pk"), new Object[][]{{"a"}, {"b"}, {"c"}}, new Object[][]{{"b"}}, new WhereCondition("pk", "=", "'b'"),
table);
@@ -136,6 +153,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void equalityDescCompositePK1() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (oid CHAR(2) NOT NULL, code INTEGER NOT NULL constraint pk primary key (oid DESC, code DESC))";
Object[][] insertedRows = new Object[][]{{"o1", 1}, {"o2", 2}, {"o3", 3}};
runQueryTest(ddl, upsert("oid", "code"), insertedRows, new Object[][]{{"o2", 2}}, new WhereCondition("oid", "=", "'o2'"),
@@ -145,6 +163,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void equalityDescCompositePK2() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (oid CHAR(2) NOT NULL, code INTEGER NOT NULL constraint pk primary key (oid DESC, code DESC))";
Object[][] insertedRows = new Object[][]{{"o1", 1}, {"o1", 2}, {"o1", 3}};
runQueryTest(ddl, upsert("oid", "code"), insertedRows, new Object[][]{{"o1", 2}}, new WhereCondition("code", "=", "2"),
@@ -154,6 +173,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void inDescCompositePK1() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (oid CHAR(2) NOT NULL, code INTEGER NOT NULL constraint pk primary key (oid DESC, code DESC))";
Object[][] insertedRows = new Object[][]{{"o1", 1}, {"o1", 2}, {"o1", 3}};
runQueryTest(ddl, upsert("oid", "code"), insertedRows, new Object[][]{{"o1", 2}}, new WhereCondition("code", "IN", "(2)"),
@@ -163,6 +183,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void inDescCompositePK2() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (oid CHAR(2) NOT NULL, code INTEGER NOT NULL constraint pk primary key (oid DESC, code DESC))";
Object[][] insertedRows = new Object[][]{{"o1", 1}, {"o2", 2}, {"o3", 3}};
runQueryTest(ddl, upsert("oid", "code"), insertedRows, new Object[][]{{"o2", 2}}, new WhereCondition("oid", "IN", "('o2')"),
@@ -172,6 +193,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void inDescCompositePK3() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (oid VARCHAR NOT NULL, code VARCHAR NOT NULL constraint pk primary key (oid DESC, code DESC))";
Object[][] insertedRows = new Object[][]{{"o1", "1"}, {"o2", "2"}, {"o3", "3"}};
runQueryTest(ddl, upsert("oid", "code"), insertedRows, new Object[][]{{"o2", "2"}, {"o1", "1"}}, new WhereCondition("(oid, code)", "IN", "(('o2', '2'), ('o1', '1'))"),
@@ -181,6 +203,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void likeDescCompositePK1() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (oid CHAR(2) NOT NULL, code INTEGER NOT NULL constraint pk primary key (oid DESC, code DESC))";
Object[][] insertedRows = new Object[][]{{"a1", 1}, {"b2", 2}, {"c3", 3}};
runQueryTest(ddl, upsert("oid", "code"), insertedRows, new Object[][]{{"b2", 2}}, new WhereCondition("oid", "LIKE", "('b%')"),
@@ -190,6 +213,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void likeDescCompositePK2() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (oid CHAR(2) NOT NULL, code CHAR(2) NOT NULL constraint pk primary key (oid DESC, code DESC))";
Object[][] insertedRows = new Object[][]{{"a1", "11"}, {"b2", "22"}, {"c3", "33"}};
runQueryTest(ddl, upsert("oid", "code"), insertedRows, new Object[][]{{"b2", "22"}}, new WhereCondition("code", "LIKE", "('2%')"),
@@ -199,6 +223,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void greaterThanDescCompositePK3() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (oid CHAR(2) NOT NULL, code INTEGER NOT NULL constraint pk primary key (oid DESC, code DESC))";
Object[][] insertedRows = new Object[][]{{"o1", 1}, {"o1", 2}, {"o1", 3}};
Object[][] expectedRows = new Object[][]{{"o1", 2}, {"o1", 1}};
@@ -209,6 +234,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void substrDescCompositePK1() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (oid CHAR(3) NOT NULL, code INTEGER NOT NULL constraint pk primary key (oid DESC, code ASC))";
Object[][] insertedRows = new Object[][]{{"ao1", 1}, {"bo2", 2}, {"co3", 3}};
Object[][] expectedRows = new Object[][]{{"co3", 3}, {"bo2", 2}};
@@ -219,6 +245,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void substrDescCompositePK2() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (oid CHAR(4) NOT NULL, code INTEGER NOT NULL constraint pk primary key (oid DESC, code ASC))";
Object[][] insertedRows = new Object[][]{{"aaaa", 1}, {"bbbb", 2}, {"cccd", 3}};
Object[][] expectedRows = new Object[][]{{"cccd", 3}};
@@ -229,6 +256,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void substrFixedLengthDescPK1() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (oid CHAR(3) PRIMARY KEY DESC)";
Object[][] insertedRows = new Object[][]{{"a"}, {"ab"}};
Object[][] expectedRows = new Object[][]{{"ab"}, {"a"} };
@@ -239,6 +267,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void substrVarLengthDescPK1() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (oid VARCHAR PRIMARY KEY DESC)";
Object[][] insertedRows = new Object[][]{{"a"}, {"ab"}};
Object[][] expectedRows = new Object[][]{{"ab"}, {"a"} };
@@ -249,6 +278,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void likeVarLengthDescPK1() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (oid VARCHAR PRIMARY KEY DESC)";
Object[][] insertedRows = new Object[][]{{"a"}, {"ab"}};
Object[][] expectedRows = new Object[][]{{"ab"}, {"a"} };
@@ -259,6 +289,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void likeFixedLengthDescPK1() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (oid CHAR(3) PRIMARY KEY DESC)";
Object[][] insertedRows = new Object[][]{{"a"}, {"ab"}};
Object[][] expectedRows = new Object[][]{{"ab"}, {"a"} };
@@ -269,6 +300,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void decimalRangeDescPK1() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (oid DECIMAL PRIMARY KEY DESC)";
Connection conn = DriverManager.getConnection(getUrl());
conn.createStatement().execute(ddl);
@@ -295,6 +327,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void lTrimDescCompositePK() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (oid VARCHAR NOT NULL, code INTEGER NOT NULL constraint pk primary key (oid DESC, code DESC))";
Object[][] insertedRows = new Object[][]{{" o1 ", 1}, {" o2", 2}, {" o3", 3}};
Object[][] expectedRows = new Object[][]{{" o2", 2}};
@@ -305,6 +338,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void lPadDescCompositePK() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (oid VARCHAR NOT NULL, code INTEGER NOT NULL constraint pk primary key (oid DESC, code DESC))";
Object[][] insertedRows = new Object[][]{{"aaaa", 1}, {"bbbb", 2}, {"cccc", 3}};
Object[][] expectedRows = new Object[][]{{"bbbb", 2}};
@@ -315,6 +349,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void countDescCompositePK() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (oid CHAR(2) NOT NULL, code INTEGER NOT NULL constraint pk primary key (oid DESC, code ASC))";
Object[][] insertedRows = new Object[][]{{"o1", 1}, {"o2", 2}, {"o3", 3}};
Object[][] expectedRows = new Object[][]{{3l}};
@@ -325,6 +360,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void sumDescCompositePK() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (n1 INTEGER NOT NULL, n2 DECIMAL(10, 2) NOT NULL, n3 BIGINT NOT NULL " +
"constraint pk primary key (n1 DESC, n2 DESC, n3 DESC))";
Object[][] insertedRows = new Object[][]{{10, bdec(10.2), 21l}, {20, bdec(20.2), 32l}, {30, bdec(30.2), 43l}};
@@ -336,6 +372,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void avgDescCompositePK() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (n1 INTEGER NOT NULL, n2 DECIMAL(10, 2) NOT NULL, n3 BIGINT NOT NULL " +
"constraint pk primary key (n1 DESC, n2 DESC, n3 DESC))";
Object[][] insertedRows = new Object[][]{{10, bdec(10.2), 21l}, {20, bdec(20.2), 32l}, {30, bdec(30.2), 43l}};
@@ -347,6 +384,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void minDescCompositePK() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (n1 INTEGER NOT NULL, n2 DECIMAL(10, 2) NOT NULL, n3 BIGINT NOT NULL " +
"constraint pk primary key (n1 DESC, n2 DESC, n3 DESC))";
Object[][] insertedRows = new Object[][]{{10, bdec(10.2), 21l}, {20, bdec(20.2), 32l}, {30, bdec(30.2), 43l}};
@@ -358,6 +396,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void maxDescCompositePK() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (n1 INTEGER NOT NULL, n2 DECIMAL(10, 2) NOT NULL, n3 BIGINT NOT NULL " +
"constraint pk primary key (n1 DESC, n2 DESC, n3 DESC))";
Object[][] insertedRows = new Object[][]{{10, bdec(10.2), 21l}, {20, bdec(20.2), 32l}, {30, bdec(30.2), 43l}};
@@ -369,6 +408,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void havingSumDescCompositePK() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (name CHAR(1) NOT NULL, code INTEGER NOT NULL " +
"constraint pk primary key (name DESC, code DESC))";
Object[][] insertedRows = new Object[][]{{"a", 10}, {"a", 20}, {"b", 100}};
@@ -380,6 +420,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void queryDescDateWithExplicitOrderBy() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (c1 CHAR(1) NOT NULL, c2 CHAR(1) NOT NULL, d1 \"DATE\" NOT NULL, c3 CHAR(1) NOT NULL " +
"constraint pk primary key (c1, c2, d1 DESC, c3))";
Object[] row1 = {"1", "2", date(10, 11, 2001), "3"};
@@ -392,6 +433,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void additionOnDescCompositePK() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (n1 INTEGER NOT NULL, n2 DECIMAL(10, 2) NOT NULL, n3 BIGINT NOT NULL, d1 DATE NOT NULL " +
"constraint pk primary key (n1 DESC, n2 DESC, n3 DESC, d1 DESC))";
Object[][] insertedRows = new Object[][]{
@@ -405,6 +447,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void subtractionOnDescCompositePK() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (n1 INTEGER NOT NULL, n2 DECIMAL(10, 2) NOT NULL, n3 BIGINT NOT NULL, d1 DATE NOT NULL " +
"constraint pk primary key (n1 DESC, n2 DESC, n3 DESC, d1 DESC))";
Object[][] insertedRows = new Object[][]{
@@ -418,6 +461,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void lessThanLeadingDescCompositePK() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (id INTEGER NOT NULL, \"DATE\" DATE NOT NULL constraint pk primary key (id DESC, \"DATE\"))";
Object[][] insertedRows = new Object[][]{{1, date(1, 1, 2012)}, {3, date(1, 1, 2013)}, {2, date(1, 1, 2011)}};
Object[][] expectedRows = new Object[][]{{1, date(1, 1, 2012)}};
@@ -428,6 +472,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void lessThanTrailingDescCompositePK() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (id INTEGER NOT NULL, \"DATE\" DATE NOT NULL constraint pk primary key (id DESC, \"DATE\"))";
Object[][] insertedRows = new Object[][]{{1, date(1, 1, 2002)}, {3, date(1, 1, 2003)}, {2, date(1, 1, 2001)}};
Object[][] expectedRows = new Object[][]{{2, date(1, 1, 2001)}};
@@ -438,6 +483,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void descVarLengthPK() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (id VARCHAR PRIMARY KEY DESC)";
Object[][] insertedRows = new Object[][]{{"a"}, {"ab"}, {"abc"}};
Object[][] expectedRows = new Object[][]{{"abc"}, {"ab"}, {"a"}};
@@ -448,6 +494,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void descVarLengthAscPKGT() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (k1 INTEGER NOT NULL, k2 VARCHAR, CONSTRAINT pk PRIMARY KEY (k1, k2))";
Object[][] insertedRows = new Object[][]{{0, null}, {1, "a"}, {2, "b"}, {3, "ba"}, {4, "baa"}, {5, "c"}, {6, "d"}};
Object[][] expectedRows = new Object[][]{{3}, {4}, {5}, {6}};
@@ -458,6 +505,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void descVarLengthDescPKGT() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (k1 INTEGER NOT NULL, k2 VARCHAR, CONSTRAINT pk PRIMARY KEY (k1, k2 desc))";
Object[][] insertedRows = new Object[][]{{0, null}, {1, "a"}, {2, "b"}, {3, "ba"}, {4, "baa"}, {5, "c"}, {6, "d"}};
Object[][] expectedRows = new Object[][]{{3}, {4}, {5}, {6}};
@@ -468,6 +516,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void descVarLengthDescPKLTE() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (k1 INTEGER NOT NULL, k2 VARCHAR, CONSTRAINT pk PRIMARY KEY (k1, k2 desc))";
Object[][] insertedRows = new Object[][]{{0, null}, {1, "a"}, {2, "b"}, {3, "ba"}, {4, "bb"}, {5, "bc"}, {6, "bba"}, {7, "c"}};
Object[][] expectedRows = new Object[][]{{1}, {2}, {3}, {4}};
@@ -478,6 +527,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void descVarLengthAscPKLTE() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (k1 INTEGER NOT NULL, k2 VARCHAR, CONSTRAINT pk PRIMARY KEY (k1, k2))";
Object[][] insertedRows = new Object[][]{{0, null}, {1, "a"}, {2, "b"}, {3, "ba"}, {4, "bb"}, {5, "bc"}, {6, "bba"}, {7, "c"}};
Object[][] expectedRows = new Object[][]{{1}, {2}, {3}, {4}};
@@ -488,6 +538,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void varLengthAscLT() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (k1 VARCHAR NOT NULL, k2 VARCHAR, CONSTRAINT pk PRIMARY KEY (k1, k2))";
Object[][] insertedRows = new Object[][]{{"a", ""}, {"b",""}, {"b","a"}};
Object[][] expectedRows = new Object[][]{{"a"}};
@@ -498,6 +549,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void varLengthDescLT() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (k1 VARCHAR NOT NULL, k2 VARCHAR, CONSTRAINT pk PRIMARY KEY (k1 desc, k2))";
Object[][] insertedRows = new Object[][]{{"a", ""}, {"b",""}, {"b","a"}};
Object[][] expectedRows = new Object[][]{{"a"}};
@@ -508,6 +560,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
@Test
public void varLengthDescGT() throws Exception {
String table = generateUniqueName();
+ TABLE_NAMES.add(table);
String ddl = "CREATE table " + table + " (k1 VARCHAR NOT NULL, k2 VARCHAR, CONSTRAINT pk PRIMARY KEY (k1 desc, k2))";
Object[][] insertedRows = new Object[][]{{"a", ""}, {"b",""}, {"b","a"}, {"ba","a"}};
Object[][] expectedRows = new Object[][]{{"ba"}};
@@ -548,6 +601,7 @@ public class SortOrderIT extends ParallelStatsDisabledIT {
private void testCompareCompositeKey(Integer saltBuckets, PDataType dataType, SortOrder sortOrder, String whereClause, List<Integer> expectedResults, String orderBy) throws SQLException {
String tableName = "t_" + saltBuckets + "_" + dataType + "_" + sortOrder + "_" + baseTableName;
+ TABLE_NAMES.add(tableName);
String ddl = "create table if not exists " + tableName + " (k1 bigint not null, k2 " + dataType.getSqlTypeName() + (dataType.isFixedWidth() ? " not null" : "") + ", constraint pk primary key (k1,k2 " + sortOrder + "))" + (saltBuckets == null ? "" : (" SALT_BUCKETS= " + saltBuckets));
Connection conn = DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES));
conn.createStatement().execute(ddl);
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortOrderWithRegionMoves2IT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortOrderWithRegionMoves2IT.java
new file mode 100644
index 0000000000..56515f9292
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortOrderWithRegionMoves2IT.java
@@ -0,0 +1,200 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.phoenix.coprocessor.BaseScannerRegionObserver;
+import org.apache.phoenix.iterate.ScanningResultPostValidResultCaller;
+import org.apache.phoenix.query.BaseTest;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.schema.StaleRegionBoundaryCacheException;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.HashMap;
+import java.util.Map;
+
+
+@Category(NeedsOwnMiniClusterTest.class)
+public class SortOrderWithRegionMoves2IT extends SortOrderIT {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(SortOrderWithRegionMoves2IT.class);
+
+ private static int countOfValidResults = 0;
+ protected static boolean hasTestStarted = false;
+
+ @Before
+ public void setUp() throws Exception {
+ hasTestStarted = true;
+ baseTableName = generateUniqueName();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ countOfValidResults = 0;
+ TABLE_NAMES.clear();
+ hasTestStarted = false;
+ }
+
+ private static class TestScanningResultPostValidResultCaller extends
+ ScanningResultPostValidResultCaller {
+
+ @Override
+ public void postValidRowProcess() {
+ if (hasTestStarted && countOfValidResults <= 1) {
+ LOGGER.info("Splitting regions of tables {}. current count of valid results: {}",
+ TABLE_NAMES, countOfValidResults);
+ countOfValidResults++;
+ TABLE_NAMES.forEach(table -> {
+ try {
+ ParallelStatsDisabledWithRegionMovesIT.splitAllRegionsOfTable(table, 2);
+ } catch (Exception e) {
+ LOGGER.error("Unable to move regions of table: {}", table);
+ }
+ });
+ }
+ }
+
+ }
+
+ @BeforeClass
+ public static synchronized void doSetup() throws Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put(BaseScannerRegionObserver.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY,
+ Integer.toString(60 * 60));
+ props.put(QueryServices.USE_STATS_FOR_PARALLELIZATION, Boolean.toString(false));
+ props.put(QueryServices.PHOENIX_SERVER_PAGE_SIZE_MS, Long.toString(0));
+ props.put(QueryServices.TESTS_MINI_CLUSTER_NUM_REGION_SERVERS, String.valueOf(2));
+ props.put(HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY, String.valueOf(1));
+ props.put(QueryServices.PHOENIX_POST_VALID_PROCESS,
+ TestScanningResultPostValidResultCaller.class.getName());
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ }
+
+ @AfterClass
+ public static synchronized void freeResources() throws Exception {
+ BaseTest.freeResourcesIfBeyondThreshold();
+ }
+
+ @Test
+ public void sumDescCompositePK() throws Exception {
+ try {
+ super.sumDescCompositePK();
+ } catch (StaleRegionBoundaryCacheException e) {
+ hasTestStarted = false;
+ LOGGER.error("Rows could not be scanned because of stale region boundary. Try again.");
+ super.sumDescCompositePK();
+ }
+ }
+
+ @Test
+ public void queryDescDateWithExplicitOrderBy() throws Exception {
+ try {
+ super.queryDescDateWithExplicitOrderBy();
+ } catch (StaleRegionBoundaryCacheException e) {
+ hasTestStarted = false;
+ LOGGER.error("Rows could not be scanned because of stale region boundary. Try again.");
+ super.queryDescDateWithExplicitOrderBy();
+ }
+ }
+
+ @Test
+ public void avgDescCompositePK() throws Exception {
+ try {
+ super.avgDescCompositePK();
+ } catch (StaleRegionBoundaryCacheException e) {
+ hasTestStarted = false;
+ LOGGER.error("Rows could not be scanned because of stale region boundary. Try again.");
+ super.avgDescCompositePK();
+ }
+ }
+
+ @Test
+ public void countDescCompositePK() throws Exception {
+ try {
+ super.countDescCompositePK();
+ } catch (StaleRegionBoundaryCacheException e) {
+ hasTestStarted = false;
+ LOGGER.error("Rows could not be scanned because of stale region boundary. Try again.");
+ super.countDescCompositePK();
+ }
+ }
+
+ @Test
+ public void descVarLengthDescPKGT() throws Exception {
+ try {
+ super.descVarLengthDescPKGT();
+ } catch (StaleRegionBoundaryCacheException e) {
+ hasTestStarted = false;
+ LOGGER.error("Rows could not be scanned because of stale region boundary. Try again.");
+ super.descVarLengthDescPKGT();
+ }
+ }
+
+ @Test
+ public void havingSumDescCompositePK() throws Exception {
+ try {
+ super.havingSumDescCompositePK();
+ } catch (StaleRegionBoundaryCacheException e) {
+ hasTestStarted = false;
+ LOGGER.error("Rows could not be scanned because of stale region boundary. Try again.");
+ super.havingSumDescCompositePK();
+ }
+ }
+
+ @Test
+ public void minDescCompositePK() throws Exception {
+ try {
+ super.minDescCompositePK();
+ } catch (StaleRegionBoundaryCacheException e) {
+ hasTestStarted = false;
+ LOGGER.error("Rows could not be scanned because of stale region boundary. Try again.");
+ super.minDescCompositePK();
+ }
+ }
+
+ @Test
+ public void descVarLengthAscPKGT() throws Exception {
+ try {
+ super.descVarLengthAscPKGT();
+ } catch (StaleRegionBoundaryCacheException e) {
+ hasTestStarted = false;
+ LOGGER.error("Rows could not be scanned because of stale region boundary. Try again.");
+ super.descVarLengthAscPKGT();
+ }
+ }
+
+ @Test
+ public void maxDescCompositePK() throws Exception {
+ try {
+ super.maxDescCompositePK();
+ } catch (StaleRegionBoundaryCacheException e) {
+ hasTestStarted = false;
+ LOGGER.error("Rows could not be scanned because of stale region boundary. Try again.");
+ super.maxDescCompositePK();
+ }
+ }
+
+}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortOrderWithRegionMovesIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortOrderWithRegionMovesIT.java
new file mode 100644
index 0000000000..925eb7f3a4
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SortOrderWithRegionMovesIT.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.AsyncAdmin;
+import org.apache.hadoop.hbase.client.AsyncConnection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.RegionStatesCount;
+import org.apache.phoenix.coprocessor.BaseScannerRegionObserver;
+import org.apache.phoenix.iterate.ScanningResultPostDummyResultCaller;
+import org.apache.phoenix.iterate.ScanningResultPostValidResultCaller;
+import org.apache.phoenix.query.BaseTest;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.experimental.categories.Category;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+@Category(NeedsOwnMiniClusterTest.class)
+public class SortOrderWithRegionMovesIT extends SortOrderIT {
+
+ private static final Logger LOGGER = LoggerFactory.getLogger(SortOrderWithRegionMovesIT.class);
+
+ private static int countOfValidResults = 0;
+ protected static boolean hasTestStarted = false;
+ protected static int countOfDummyResults = 0;
+
+ @Before
+ public void setUp() throws Exception {
+ hasTestStarted = true;
+ baseTableName = generateUniqueName();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ countOfDummyResults = 0;
+ countOfValidResults = 0;
+ TABLE_NAMES.clear();
+ hasTestStarted = false;
+ }
+
+ private static class TestScanningResultPostDummyResultCaller extends
+ ScanningResultPostDummyResultCaller {
+
+ @Override
+ public void postDummyProcess() {
+ if (hasTestStarted && (countOfDummyResults++ % 3) == 0 &&
+ (countOfDummyResults < 17 ||
+ countOfDummyResults > 28 && countOfDummyResults < 40)) {
+ LOGGER.info("Moving regions of tables {}. current count of dummy results: {}",
+ TABLE_NAMES, countOfDummyResults);
+ TABLE_NAMES.forEach(table -> {
+ try {
+ moveRegionsOfTable(table);
+ } catch (Exception e) {
+ LOGGER.error("Unable to move regions of table: {}", table);
+ }
+ });
+ }
+ }
+ }
+
+ private static class TestScanningResultPostValidResultCaller extends
+ ScanningResultPostValidResultCaller {
+
+ @Override
+ public void postValidRowProcess() {
+ if (hasTestStarted &&
+ (countOfValidResults < 17 ||
+ countOfValidResults > 28 && countOfValidResults < 40)) {
+ LOGGER.info("Moving regions of tables {}. current count of valid results: {}",
+ TABLE_NAMES, countOfValidResults);
+ countOfValidResults++;
+ TABLE_NAMES.forEach(table -> {
+ try {
+ moveRegionsOfTable(table);
+ } catch (Exception e) {
+ LOGGER.error("Unable to move regions of table: {}", table);
+ }
+ });
+ }
+ }
+ }
+
+ @BeforeClass
+ public static synchronized void doSetup() throws Exception {
+ Map<String, String> props = new HashMap<>();
+ props.put(BaseScannerRegionObserver.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY,
+ Integer.toString(60 * 60));
+ props.put(QueryServices.USE_STATS_FOR_PARALLELIZATION, Boolean.toString(false));
+ props.put(QueryServices.PHOENIX_SERVER_PAGE_SIZE_MS, Long.toString(0));
+ props.put(QueryServices.TESTS_MINI_CLUSTER_NUM_REGION_SERVERS, String.valueOf(2));
+ props.put(HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY, String.valueOf(1));
+ props.put(QueryServices.PHOENIX_POST_DUMMY_PROCESS,
+ TestScanningResultPostDummyResultCaller.class.getName());
+ props.put(QueryServices.PHOENIX_POST_VALID_PROCESS,
+ TestScanningResultPostValidResultCaller.class.getName());
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ }
+
+ @AfterClass
+ public static synchronized void freeResources() throws Exception {
+ BaseTest.freeResourcesIfBeyondThreshold();
+ }
+
+ private static void moveRegionsOfTable(String tableName)
+ throws IOException {
+ try (AsyncConnection asyncConnection =
+ ConnectionFactory.createAsyncConnection(getUtility().getConfiguration())
+ .get()) {
+ AsyncAdmin admin = asyncConnection.getAdmin();
+ List<ServerName> servers =
+ new ArrayList<>(admin.getRegionServers().get());
+ ServerName server1 = servers.get(0);
+ ServerName server2 = servers.get(1);
+ List<RegionInfo> regionsOnServer1;
+ regionsOnServer1 = admin.getRegions(server1).get();
+ List<RegionInfo> regionsOnServer2;
+ regionsOnServer2 = admin.getRegions(server2).get();
+ regionsOnServer1.forEach(regionInfo -> {
+ if (regionInfo.getTable().equals(TableName.valueOf(tableName))) {
+ try {
+ for (int i = 0; i < 2; i++) {
+ RegionStatesCount regionStatesCount =
+ admin.getClusterMetrics().get().getTableRegionStatesCount()
+ .get(TableName.valueOf(tableName));
+ if (regionStatesCount.getRegionsInTransition() == 0 &&
+ regionStatesCount.getOpenRegions() ==
+ regionStatesCount.getTotalRegions()) {
+ LOGGER.info("Moving region {} to {}",
+ regionInfo.getRegionNameAsString(), server2);
+ admin.move(regionInfo.getEncodedNameAsBytes(), server2).get(3,
+ TimeUnit.SECONDS);
+ break;
+ } else {
+ LOGGER.info("Table {} has some region(s) in RIT or not online",
+ tableName);
+ }
+ }
+ } catch (InterruptedException | ExecutionException | TimeoutException e) {
+ LOGGER.error("Something went wrong", e);
+ throw new RuntimeException(e);
+ }
+ }
+ });
+ regionsOnServer2.forEach(regionInfo -> {
+ if (regionInfo.getTable().equals(TableName.valueOf(tableName))) {
+ try {
+ for (int i = 0; i < 2; i++) {
+ RegionStatesCount regionStatesCount =
+ admin.getClusterMetrics().get().getTableRegionStatesCount()
+ .get(TableName.valueOf(tableName));
+ if (regionStatesCount.getRegionsInTransition() == 0 &&
+ regionStatesCount.getOpenRegions() ==
+ regionStatesCount.getTotalRegions()) {
+ admin.move(regionInfo.getEncodedNameAsBytes(), server1)
+ .get(3, TimeUnit.SECONDS);
+ LOGGER.info("Moving region {} to {}",
+ regionInfo.getRegionNameAsString(), server1);
+ break;
+ } else {
+ LOGGER.info("Table {} has some region(s) in RIT or not online",
+ tableName);
+ }
+ }
+ } catch (InterruptedException | ExecutionException | TimeoutException e) {
+ LOGGER.error("Something went wrong", e);
+ throw new RuntimeException(e);
+ }
+ }
+ });
+ } catch (Exception e) {
+ LOGGER.error("Something went wrong..", e);
+ }
+ }
+
+}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectWithRegionMovesIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectWithRegionMovesIT.java
new file mode 100644
index 0000000000..93d31348de
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UpsertSelectWithRegionMovesIT.java
@@ -0,0 +1,1927 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.phoenix.compile.QueryPlan;
+import org.apache.phoenix.coprocessor.BaseScannerRegionObserver;
+import org.apache.phoenix.exception.SQLExceptionCode;
+import org.apache.phoenix.jdbc.PhoenixResultSet;
+import org.apache.phoenix.jdbc.PhoenixStatement;
+import org.apache.phoenix.monitoring.GlobalMetric;
+import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.schema.types.PInteger;
+import org.apache.phoenix.thirdparty.com.google.common.collect.Maps;
+import org.apache.phoenix.util.EnvironmentEdgeManager;
+import org.apache.phoenix.util.PhoenixRuntime;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.QueryUtil;
+import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.phoenix.util.TestUtil;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+import java.math.BigDecimal;
+import java.sql.Connection;
+import java.sql.Date;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.sql.Timestamp;
+import java.util.Map;
+import java.util.Properties;
+
+import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB;
+import static org.apache.phoenix.util.PhoenixRuntime.UPSERT_BATCH_SIZE_ATTRIB;
+import static org.apache.phoenix.util.TestUtil.A_VALUE;
+import static org.apache.phoenix.util.TestUtil.B_VALUE;
+import static org.apache.phoenix.util.TestUtil.C_VALUE;
+import static org.apache.phoenix.util.TestUtil.PTSDB_NAME;
+import static org.apache.phoenix.util.TestUtil.ROW6;
+import static org.apache.phoenix.util.TestUtil.ROW7;
+import static org.apache.phoenix.util.TestUtil.ROW8;
+import static org.apache.phoenix.util.TestUtil.ROW9;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+@Category(NeedsOwnMiniClusterTest.class)
+@RunWith(Parameterized.class)
+public class UpsertSelectWithRegionMovesIT extends ParallelStatsDisabledWithRegionMovesIT {
+ private final String allowServerSideMutations;
+
+ @BeforeClass
+ public static synchronized void doSetup() throws Exception {
+ Map<String, String> props = Maps.newHashMapWithExpectedSize(1);
+ // An hour - inherited from ParallelStatsDisabledIT
+ props.put(BaseScannerRegionObserver.PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY,
+ Integer.toString(60 * 60));
+ // Postpone scans of SYSTEM.TASK indefinitely so as to prevent
+ // any addition to GLOBAL_OPEN_PHOENIX_CONNECTIONS
+ props.put(QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB,
+ Long.toString(Long.MAX_VALUE));
+ props.put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB,
+ Long.toString(Long.MAX_VALUE));
+ props.put(QueryServices.PHOENIX_SERVER_PAGE_SIZE_MS, Long.toString(0));
+ props.put(QueryServices.TESTS_MINI_CLUSTER_NUM_REGION_SERVERS, String.valueOf(2));
+ props.put(HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY, String.valueOf(1));
+ props.put(QueryServices.PHOENIX_POST_DUMMY_PROCESS,
+ TestScanningResultPostDummyResultCaller.class.getName());
+ setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
+ }
+
+ public UpsertSelectWithRegionMovesIT(String allowServerSideMutations) {
+ this.allowServerSideMutations = allowServerSideMutations;
+ }
+
+ @Before
+ public void setup() {
+ hasTestStarted = true;
+ assertTrue(PhoenixRuntime.areGlobalClientMetricsBeingCollected());
+ for (GlobalMetric m : PhoenixRuntime.getGlobalPhoenixClientMetrics()) {
+ m.reset();
+ }
+ }
+
+ @After
+ public void assertNoConnLeak() throws Exception {
+ countOfDummyResults = 0;
+ TABLE_NAMES.clear();
+ hasTestStarted = false;
+ }
+
+ // name is used by failsafe as file name in reports
+ @Parameters(name = "UpsertSelecttIT_allowServerSideMutations={0}")
+ public static synchronized Object[] data() {
+ return new Object[]{"true", "false"};
+ }
+
+ @Test
+ public void testUpsertSelectWithNoIndex() throws Exception {
+ testUpsertSelect(false, false);
+ }
+
+ @Test
+ public void testUpsertSelectWithIndex() throws Exception {
+ testUpsertSelect(true, false);
+ }
+
+ @Test
+ public void testUpsertSelectWithIndexWithSalt() throws Exception {
+ testUpsertSelect(true, true);
+ }
+
+ @Test
+ public void testUpsertSelectWithNoIndexWithSalt() throws Exception {
+ testUpsertSelect(false, true);
+ }
+
+ private void testUpsertSelect(boolean createIndex, boolean saltTable) throws Exception {
+ String tenantId = getOrganizationId();
+ byte[][] splits = getDefaultSplits(tenantId);
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ props.setProperty(QueryServices.ENABLE_SERVER_SIDE_UPSERT_MUTATIONS,
+ allowServerSideMutations);
+ String aTable = initATableValues(tenantId, saltTable ? null : splits, null,
+ null, getUrl(), saltTable ? "salt_buckets = 2" : null);
+
+ String customEntityTable = generateUniqueName();
+ TABLE_NAMES.add(customEntityTable);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ String ddl = "create table " + customEntityTable +
+ " (organization_id char(15) not null, \n" +
+ " key_prefix char(3) not null,\n" +
+ " custom_entity_data_id char(12) not null,\n" +
+ " created_by varchar,\n" +
+ " created_date date,\n" +
+ " currency_iso_code char(3),\n" +
+ " deleted char(1),\n" +
+ " division decimal(31,10),\n" +
+ " last_activity date,\n" +
+ " last_update date,\n" +
+ " last_update_by varchar,\n" +
+ " name varchar(240),\n" +
+ " owner varchar,\n" +
+ " record_type_id char(15),\n" +
+ " setup_owner varchar,\n" +
+ " system_modstamp date,\n" +
+ " b.val0 varchar,\n" +
+ " b.val1 varchar,\n" +
+ " b.val2 varchar,\n" +
+ " b.val3 varchar,\n" +
+ " b.val4 varchar,\n" +
+ " b.val5 varchar,\n" +
+ " b.val6 varchar,\n" +
+ " b.val7 varchar,\n" +
+ " b.val8 varchar,\n" +
+ " b.val9 varchar\n" +
+ " CONSTRAINT pk PRIMARY KEY " +
+ "(organization_id, key_prefix, custom_entity_data_id)) " +
+ (saltTable ? "salt_buckets = 2" : "");
+ stmt.execute(ddl);
+ }
+
+ String indexName = generateUniqueName();
+ if (createIndex) {
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ stmt.execute("CREATE INDEX IF NOT EXISTS " + indexName +
+ " ON " + aTable + "(a_string)");
+ TABLE_NAMES.add(indexName);
+ }
+ }
+ // Trigger multiple batches
+ props.setProperty(UPSERT_BATCH_SIZE_ATTRIB, Integer.toString(3));
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ conn.setAutoCommit(true);
+ String upsert = "UPSERT INTO " + customEntityTable +
+ "(custom_entity_data_id, key_prefix, organization_id, created_by) " +
+ "SELECT substr(entity_id, 4), substr(entity_id, 1, 3), organization_id, " +
+ "a_string FROM " + aTable + " WHERE ?=a_string";
+ if (createIndex) { // Confirm index is used
+ try (PreparedStatement upsertStmt =
+ conn.prepareStatement("EXPLAIN " + upsert)) {
+ upsertStmt.setString(1, tenantId);
+ ResultSet ers = upsertStmt.executeQuery();
+ assertTrue(ers.next());
+ String explainPlan = QueryUtil.getExplainPlan(ers);
+ assertTrue(explainPlan.contains(" SCAN OVER " + indexName));
+ }
+ }
+
+ try (PreparedStatement upsertStmt = conn.prepareStatement(upsert)) {
+ upsertStmt.setString(1, A_VALUE);
+ int rowsInserted = upsertStmt.executeUpdate();
+ assertEquals(4, rowsInserted);
+ }
+ conn.commit();
+ }
+
+ String query = "SELECT key_prefix, substr(custom_entity_data_id, 1, 1), created_by FROM " +
+ customEntityTable + " WHERE organization_id = ? ";
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ PreparedStatement statement = conn.prepareStatement(query)) {
+ statement.setString(1, tenantId);
+ ResultSet rs = statement.executeQuery();
+
+ assertTrue(rs.next());
+ assertEquals("00A", rs.getString(1));
+ assertEquals("1", rs.getString(2));
+ assertEquals(A_VALUE, rs.getString(3));
+
+ assertTrue(rs.next());
+ assertEquals("00A", rs.getString(1));
+ assertEquals("2", rs.getString(2));
+ assertEquals(A_VALUE, rs.getString(3));
+
+ assertTrue(rs.next());
+ assertEquals("00A", rs.getString(1));
+ assertEquals("3", rs.getString(2));
+ assertEquals(A_VALUE, rs.getString(3));
+
+ assertTrue(rs.next());
+ assertEquals("00A", rs.getString(1));
+ assertEquals("4", rs.getString(2));
+ assertEquals(A_VALUE, rs.getString(3));
+
+ assertFalse(rs.next());
+ }
+
+ // Test UPSERT through coprocessor
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ conn.setAutoCommit(true);
+ String upsert = "UPSERT INTO " + customEntityTable +
+ "(custom_entity_data_id, key_prefix, organization_id, last_update_by, " +
+ "division) SELECT custom_entity_data_id, key_prefix, organization_id, " +
+ "created_by, 1.0 FROM " + customEntityTable + " WHERE organization_id = ?" +
+ " and created_by >= 'a'";
+
+ try (PreparedStatement upsertStmt = conn.prepareStatement(upsert)) {
+ upsertStmt.setString(1, tenantId);
+ assertEquals(4, upsertStmt.executeUpdate());
+ }
+ conn.commit();
+ }
+
+ query = "SELECT key_prefix, substr(custom_entity_data_id, 1, 1), created_by, " +
+ "last_update_by, division FROM " + customEntityTable + " WHERE organization_id = ?";
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ PreparedStatement statement = conn.prepareStatement(query)) {
+ statement.setString(1, tenantId);
+ ResultSet rs = statement.executeQuery();
+
+ assertTrue(rs.next());
+ assertEquals("00A", rs.getString(1));
+ assertEquals("1", rs.getString(2));
+ assertEquals(A_VALUE, rs.getString(3));
+ assertEquals(A_VALUE, rs.getString(4));
+ assertTrue(BigDecimal.valueOf(1.0).compareTo(rs.getBigDecimal(5)) == 0);
+
+ assertTrue(rs.next());
+ assertEquals("00A", rs.getString(1));
+ assertEquals("2", rs.getString(2));
+ assertEquals(A_VALUE, rs.getString(3));
+ assertEquals(A_VALUE, rs.getString(4));
+ assertTrue(BigDecimal.valueOf(1.0).compareTo(rs.getBigDecimal(5)) == 0);
+
+ assertTrue(rs.next());
+ assertEquals("00A", rs.getString(1));
+ assertEquals("3", rs.getString(2));
+ assertEquals(A_VALUE, rs.getString(3));
+ assertEquals(A_VALUE, rs.getString(4));
+ assertTrue(BigDecimal.valueOf(1.0).compareTo(rs.getBigDecimal(5)) == 0);
+
+ assertTrue(rs.next());
+ assertEquals("00A", rs.getString(1));
+ assertEquals("4", rs.getString(2));
+ assertEquals(A_VALUE, rs.getString(3));
+ assertEquals(A_VALUE, rs.getString(4));
+ assertTrue(BigDecimal.valueOf(1.0).compareTo(rs.getBigDecimal(5)) == 0);
+
+ assertFalse(rs.next());
+ }
+ }
+
+ // TODO: more tests - nullable fixed length last PK column
+ @Test
+ public void testUpsertSelectEmptyPKColumn() throws Exception {
+ String tenantId = getOrganizationId();
+ String aTable = initATableValues(tenantId, getDefaultSplits(tenantId));
+ String ptsdbTable = generateUniqueName();
+ TABLE_NAMES.add(ptsdbTable);
+ ensureTableCreated(getUrl(), ptsdbTable, PTSDB_NAME);
+ Properties props = new Properties();
+ props.setProperty(QueryServices.ENABLE_SERVER_SIDE_UPSERT_MUTATIONS,
+ allowServerSideMutations);
+ String upsert;
+ ResultSet rs;
+ int rowsInserted;
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ conn.setAutoCommit(false);
+ upsert = "UPSERT INTO " + ptsdbTable + "(\"DATE\", val, host) " +
+ "SELECT current_date(), x_integer+2, entity_id FROM " + aTable +
+ " WHERE a_integer >= ?";
+ try (PreparedStatement upsertStmt = conn.prepareStatement(upsert)) {
+ upsertStmt.setInt(1, 6);
+ rowsInserted = upsertStmt.executeUpdate();
+ assertEquals(4, rowsInserted);
+ }
+ conn.commit();
+ }
+
+ String query = "SELECT inst,host,\"DATE\",val FROM " + ptsdbTable;
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ PreparedStatement statement = conn.prepareStatement(query)) {
+ rs = statement.executeQuery();
+
+ Date now = new Date(EnvironmentEdgeManager.currentTimeMillis());
+ assertTrue(rs.next());
+ moveRegionsOfTable(ptsdbTable);
+ moveRegionsOfTable(aTable);
+ assertEquals(null, rs.getString(1));
+ assertEquals(ROW6, rs.getString(2));
+ assertTrue(rs.getDate(3).before(now));
+ assertEquals(null, rs.getBigDecimal(4));
+
+ assertTrue(rs.next());
+ moveRegionsOfTable(ptsdbTable);
+ moveRegionsOfTable(aTable);
+ assertEquals(null, rs.getString(1));
+ assertEquals(ROW7, rs.getString(2));
+ assertTrue(rs.getDate(3).before(now));
+ assertTrue(BigDecimal.valueOf(7).compareTo(rs.getBigDecimal(4)) == 0);
+
+ assertTrue(rs.next());
+ assertEquals(null, rs.getString(1));
+ assertEquals(ROW8, rs.getString(2));
+ assertTrue(rs.getDate(3).before(now));
+ assertTrue(BigDecimal.valueOf(6).compareTo(rs.getBigDecimal(4)) == 0);
+
+ assertTrue(rs.next());
+ assertEquals(null, rs.getString(1));
+ assertEquals(ROW9, rs.getString(2));
+ assertTrue(rs.getDate(3).before(now));
+ assertTrue(BigDecimal.valueOf(5).compareTo(rs.getBigDecimal(4)) == 0);
+
+ assertFalse(rs.next());
+ }
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ conn.setAutoCommit(true);
+ upsert = "UPSERT INTO " + ptsdbTable + "(\"DATE\", val, inst) " +
+ "SELECT \"DATE\"+1, val*10, host FROM " + ptsdbTable;
+ try (PreparedStatement upsertStmt = conn.prepareStatement(upsert)) {
+ rowsInserted = upsertStmt.executeUpdate();
+ assertEquals(4, rowsInserted);
+ }
+ conn.commit();
+ }
+ Date now = new Date(EnvironmentEdgeManager.currentTimeMillis());
+ Date then = new Date(now.getTime() + QueryConstants.MILLIS_IN_DAY);
+ query = "SELECT host,inst, \"DATE\",val FROM " + ptsdbTable + " where inst is not null";
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ PreparedStatement statement = conn.prepareStatement(query)) {
+ rs = statement.executeQuery();
+ assertTrue(rs.next());
+ moveRegionsOfTable(ptsdbTable);
+ moveRegionsOfTable(aTable);
+ assertEquals(null, rs.getString(1));
+ assertEquals(ROW6, rs.getString(2));
+ assertTrue(rs.getDate(3).after(now) && rs.getDate(3).before(then));
+ assertEquals(null, rs.getBigDecimal(4));
+
+ assertTrue(rs.next());
+ moveRegionsOfTable(ptsdbTable);
+ moveRegionsOfTable(aTable);
+ assertEquals(null, rs.getString(1));
+ assertEquals(ROW7, rs.getString(2));
+ assertTrue(rs.getDate(3).after(now) && rs.getDate(3).before(then));
+ assertTrue(BigDecimal.valueOf(70).compareTo(rs.getBigDecimal(4)) == 0);
+
+ assertTrue(rs.next());
+ moveRegionsOfTable(ptsdbTable);
+ moveRegionsOfTable(aTable);
+ assertEquals(null, rs.getString(1));
+ assertEquals(ROW8, rs.getString(2));
+ assertTrue(rs.getDate(3).after(now) && rs.getDate(3).before(then));
+ assertTrue(BigDecimal.valueOf(60).compareTo(rs.getBigDecimal(4)) == 0);
+
+ assertTrue(rs.next());
+ assertEquals(null, rs.getString(1));
+ assertEquals(ROW9, rs.getString(2));
+ assertTrue(rs.getDate(3).after(now) && rs.getDate(3).before(then));
+ assertTrue(BigDecimal.valueOf(50).compareTo(rs.getBigDecimal(4)) == 0);
+
+ assertFalse(rs.next());
+ }
+
+ // Should just update all values with the same value, essentially just updating the timestamp
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ conn.setAutoCommit(true);
+ upsert = "UPSERT INTO " + ptsdbTable + " SELECT * FROM " + ptsdbTable;
+ try (PreparedStatement upsertStmt = conn.prepareStatement(upsert)) {
+ TestUtil.dumpTable(conn, TableName.valueOf(ptsdbTable));
+ rowsInserted = upsertStmt.executeUpdate();
+ assertEquals(8, rowsInserted);
+ }
+ conn.commit();
+ }
+
+ query = "SELECT * FROM " + ptsdbTable;
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ PreparedStatement statement = conn.prepareStatement(query)) {
+ rs = statement.executeQuery();
+ assertTrue(rs.next());
+ moveRegionsOfTable(ptsdbTable);
+ moveRegionsOfTable(aTable);
+ assertEquals(null, rs.getString(1));
+ assertEquals(ROW6, rs.getString(2));
+ assertTrue(rs.getDate(3).before(now));
+ assertEquals(null, rs.getBigDecimal(4));
+
+ assertTrue(rs.next());
+ assertEquals(null, rs.getString(1));
+ assertEquals(ROW7, rs.getString(2));
+ assertTrue(rs.getDate(3).before(now));
+ assertTrue(BigDecimal.valueOf(7).compareTo(rs.getBigDecimal(4)) == 0);
+
+ assertTrue(rs.next());
+ assertEquals(null, rs.getString(1));
+ assertEquals(ROW8, rs.getString(2));
+ assertTrue(rs.getDate(3).before(now));
+ assertTrue(BigDecimal.valueOf(6).compareTo(rs.getBigDecimal(4)) == 0);
+
+ assertTrue(rs.next());
+ moveRegionsOfTable(ptsdbTable);
+ moveRegionsOfTable(aTable);
+ assertEquals(null, rs.getString(1));
+ assertEquals(ROW9, rs.getString(2));
+ assertTrue(rs.getDate(3).before(now));
+ assertTrue(BigDecimal.valueOf(5).compareTo(rs.getBigDecimal(4)) == 0);
+
+ assertTrue(rs.next());
+ assertEquals(ROW6, rs.getString(1));
+ assertEquals(null, rs.getString(2));
+ assertTrue(rs.getDate(3).after(now) && rs.getDate(3).before(then));
+ assertEquals(null, rs.getBigDecimal(4));
+
+ assertTrue(rs.next());
+ assertEquals(ROW7, rs.getString(1));
+ assertEquals(null, rs.getString(2));
+ assertTrue(rs.getDate(3).after(now) && rs.getDate(3).before(then));
+ assertTrue(BigDecimal.valueOf(70).compareTo(rs.getBigDecimal(4)) == 0);
+
+ assertTrue(rs.next());
+ moveRegionsOfTable(ptsdbTable);
+ moveRegionsOfTable(aTable);
+ assertEquals(ROW8, rs.getString(1));
+ assertEquals(null, rs.getString(2));
+ assertTrue(rs.getDate(3).after(now) && rs.getDate(3).before(then));
+ assertTrue(BigDecimal.valueOf(60).compareTo(rs.getBigDecimal(4)) == 0);
+
+ assertTrue(rs.next());
+ assertEquals(ROW9, rs.getString(1));
+ assertEquals(null, rs.getString(2));
+ assertTrue(rs.getDate(3).after(now) && rs.getDate(3).before(then));
+ assertTrue(BigDecimal.valueOf(50).compareTo(rs.getBigDecimal(4)) == 0);
+
+ assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testUpsertSelectForAggAutoCommit() throws Exception {
+ testUpsertSelectForAgg(true);
+ }
+
+ @Test
+ public void testUpsertSelectForAgg() throws Exception {
+ testUpsertSelectForAgg(false);
+ }
+
+ private void testUpsertSelectForAgg(boolean autoCommit) throws Exception {
+ String tenantId = getOrganizationId();
+ String aTable = initATableValues(tenantId, getDefaultSplits(tenantId));
+ String ptsdbTable = generateUniqueName();
+ TABLE_NAMES.add(aTable);
+ TABLE_NAMES.add(ptsdbTable);
+ ensureTableCreated(getUrl(), ptsdbTable, PTSDB_NAME);
+ Properties props = new Properties();
+ props.setProperty(QueryServices.ENABLE_SERVER_SIDE_UPSERT_MUTATIONS,
+ allowServerSideMutations);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ conn.setAutoCommit(autoCommit);
+ String upsert = "UPSERT INTO " + ptsdbTable + "(\"DATE\", val, host) " +
+ "SELECT current_date(), sum(a_integer), a_string FROM " + aTable +
+ " GROUP BY a_string";
+ try (PreparedStatement upsertStmt = conn.prepareStatement(upsert)) {
+ int rowsInserted = upsertStmt.executeUpdate();
+ assertEquals(3, rowsInserted);
+ }
+ if (!autoCommit) {
+ conn.commit();
+ }
+ }
+
+ String query = "SELECT inst,host,\"DATE\",val FROM " + ptsdbTable;
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ PreparedStatement statement = conn.prepareStatement(query)) {
+ ResultSet rs = statement.executeQuery();
+ Date now = new Date(EnvironmentEdgeManager.currentTimeMillis());
+
+ assertTrue(rs.next());
+ moveRegionsOfTable(ptsdbTable);
+ moveRegionsOfTable(aTable);
+ assertEquals(null, rs.getString(1));
+ assertEquals(A_VALUE, rs.getString(2));
+ assertTrue(rs.getDate(3).before(now));
+ assertTrue(BigDecimal.valueOf(10).compareTo(rs.getBigDecimal(4)) == 0);
+
+ assertTrue(rs.next());
+ assertEquals(null, rs.getString(1));
+ assertEquals(B_VALUE, rs.getString(2));
+ assertTrue(rs.getDate(3).before(now));
+ assertTrue(BigDecimal.valueOf(26).compareTo(rs.getBigDecimal(4)) == 0);
+
+ assertTrue(rs.next());
+ assertEquals(null, rs.getString(1));
+ assertEquals(C_VALUE, rs.getString(2));
+ assertTrue(rs.getDate(3).before(now));
+ assertTrue(BigDecimal.valueOf(9).compareTo(rs.getBigDecimal(4)) == 0);
+ assertFalse(rs.next());
+ }
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ conn.setAutoCommit(true);
+ String upsert = "UPSERT INTO " + ptsdbTable + "(\"DATE\", val, host, inst) " +
+ "SELECT current_date(), max(val), max(host), 'x' FROM " + ptsdbTable;
+ try (PreparedStatement upsertStmt = conn.prepareStatement(upsert)) {
+ int rowsInserted = upsertStmt.executeUpdate();
+ assertEquals(1, rowsInserted);
+ }
+ if (!autoCommit) {
+ conn.commit();
+ }
+ }
+
+ query = "SELECT inst,host,\"DATE\",val FROM " + ptsdbTable + " WHERE inst='x'";
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ PreparedStatement statement = conn.prepareStatement(query)) {
+ ResultSet rs = statement.executeQuery();
+ Date now = new Date(EnvironmentEdgeManager.currentTimeMillis());
+
+ assertTrue(rs.next());
+ assertEquals("x", rs.getString(1));
+ assertEquals(C_VALUE, rs.getString(2));
+ assertTrue(rs.getDate(3).before(now));
+ assertTrue(BigDecimal.valueOf(26).compareTo(rs.getBigDecimal(4)) == 0);
+ assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testUpsertSelectLongToInt() throws Exception {
+ byte[][] splits = new byte[][]{PInteger.INSTANCE.toBytes(1),
+ PInteger.INSTANCE.toBytes(2), PInteger.INSTANCE.toBytes(3),
+ PInteger.INSTANCE.toBytes(4)};
+ String tableName = generateUniqueName();
+ TABLE_NAMES.add(tableName);
+ ensureTableCreated(getUrl(), tableName, "IntKeyTest", splits, null);
+ Properties props = new Properties();
+ props.setProperty(QueryServices.ENABLE_SERVER_SIDE_UPSERT_MUTATIONS,
+ allowServerSideMutations);
+ String upsert = "UPSERT INTO " + tableName + " VALUES(1)";
+ int rowsInserted;
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ PreparedStatement upsertStmt = conn.prepareStatement(upsert)) {
+ rowsInserted = upsertStmt.executeUpdate();
+ assertEquals(1, rowsInserted);
+ conn.commit();
+ }
+
+ upsert = "UPSERT INTO " + tableName + " select i+1 from " + tableName;
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ PreparedStatement upsertStmt = conn.prepareStatement(upsert)) {
+ rowsInserted = upsertStmt.executeUpdate();
+ assertEquals(1, rowsInserted);
+ conn.commit();
+ }
+
+ String select = "SELECT i FROM " + tableName;
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ ResultSet rs = stmt.executeQuery(select);
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals(1, rs.getInt(1));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals(2, rs.getInt(1));
+ assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testUpsertSelectRunOnServer() throws Exception {
+ byte[][] splits = new byte[][]{PInteger.INSTANCE.toBytes(1),
+ PInteger.INSTANCE.toBytes(2), PInteger.INSTANCE.toBytes(3),
+ PInteger.INSTANCE.toBytes(4)};
+ String tableName = generateUniqueName();
+ TABLE_NAMES.add(tableName);
+ createTestTable(getUrl(), "create table " + tableName +
+ " (i integer not null primary key desc, j integer)", splits, null);
+ Properties props = new Properties();
+ props.setProperty(QueryServices.ENABLE_SERVER_SIDE_UPSERT_MUTATIONS,
+ allowServerSideMutations);
+ ResultSet rs;
+ int rowsInserted;
+ String upsert = "UPSERT INTO " + tableName + " VALUES(1, 1)";
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ PreparedStatement upsertStmt = conn.prepareStatement(upsert)) {
+ rowsInserted = upsertStmt.executeUpdate();
+ assertEquals(1, rowsInserted);
+ conn.commit();
+ }
+
+ String select = "SELECT i,j+1 FROM " + tableName;
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ rs = stmt.executeQuery(select);
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals(1, rs.getInt(1));
+ assertEquals(2, rs.getInt(2));
+ assertFalse(rs.next());
+ }
+
+ upsert = "UPSERT INTO " + tableName + "(i,j) select i, j+1 from " + tableName;
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ conn.setAutoCommit(true); // Force to run on server side.
+ try (PreparedStatement upsertStmt = conn.prepareStatement(upsert)) {
+ rowsInserted = upsertStmt.executeUpdate();
+ assertEquals(1, rowsInserted);
+ }
+ }
+
+ select = "SELECT j FROM " + tableName;
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ rs = stmt.executeQuery(select);
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals(2, rs.getInt(1));
+ assertFalse(rs.next());
+ }
+
+ upsert = "UPSERT INTO " + tableName + "(i,j) select i, i from " + tableName;
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ conn.setAutoCommit(true); // Force to run on server side.
+ try (PreparedStatement upsertStmt = conn.prepareStatement(upsert)) {
+ rowsInserted = upsertStmt.executeUpdate();
+ assertEquals(1, rowsInserted);
+ }
+ }
+
+ select = "SELECT j FROM " + tableName;
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ rs = stmt.executeQuery(select);
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals(1, rs.getInt(1));
+ assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testUpsertSelectOnDescToAsc() throws Exception {
+ byte[][] splits = new byte[][]{PInteger.INSTANCE.toBytes(1),
+ PInteger.INSTANCE.toBytes(2), PInteger.INSTANCE.toBytes(3),
+ PInteger.INSTANCE.toBytes(4)};
+ String tableName = generateUniqueName();
+ TABLE_NAMES.add(tableName);
+ createTestTable(getUrl(), "create table " + tableName +
+ " (i integer not null primary key desc, j integer)", splits, null);
+ Properties props = new Properties();
+ props.setProperty(QueryServices.ENABLE_SERVER_SIDE_UPSERT_MUTATIONS,
+ allowServerSideMutations);
+ ResultSet rs;
+ int rowsInserted;
+ String upsert = "UPSERT INTO " + tableName + " VALUES(1, 1)";
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ PreparedStatement upsertStmt = conn.prepareStatement(upsert)) {
+ rowsInserted = upsertStmt.executeUpdate();
+ assertEquals(1, rowsInserted);
+ conn.commit();
+ }
+
+ upsert = "UPSERT INTO " + tableName + " (i,j) select i+1, j+1 from " + tableName;
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ conn.setAutoCommit(true); // Force to run on server side.
+ try (PreparedStatement upsertStmt = conn.prepareStatement(upsert)) {
+ rowsInserted = upsertStmt.executeUpdate();
+ assertEquals(1, rowsInserted);
+ }
+ conn.commit();
+ }
+
+ String select = "SELECT i,j FROM " + tableName;
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ rs = stmt.executeQuery(select);
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals(2, rs.getInt(1));
+ assertEquals(2, rs.getInt(2));
+ assertTrue(rs.next());
+ assertEquals(1, rs.getInt(1));
+ assertEquals(1, rs.getInt(2));
+ assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testUpsertSelectRowKeyMutationOnSplitedTable() throws Exception {
+ byte[][] splits = new byte[][]{PInteger.INSTANCE.toBytes(1),
+ PInteger.INSTANCE.toBytes(2), PInteger.INSTANCE.toBytes(3),
+ PInteger.INSTANCE.toBytes(4)};
+ String tableName = generateUniqueName();
+ TABLE_NAMES.add(tableName);
+ ensureTableCreated(getUrl(), tableName, "IntKeyTest", splits, null, null);
+ Properties props = new Properties();
+ props.setProperty(QueryServices.ENABLE_SERVER_SIDE_UPSERT_MUTATIONS,
+ allowServerSideMutations);
+ int rowsInserted;
+ ResultSet rs;
+
+ String upsert = "UPSERT INTO " + tableName + " VALUES(?)";
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ PreparedStatement upsertStmt = conn.prepareStatement(upsert)) {
+ upsertStmt.setInt(1, 1);
+ upsertStmt.executeUpdate();
+ upsertStmt.setInt(1, 3);
+ upsertStmt.executeUpdate();
+ conn.commit();
+ }
+
+ upsert = "UPSERT INTO " + tableName + " (i) SELECT i+1 from " + tableName;
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ // Normally this would force a server side update. But since this changes the
+ // PK column, it would for to run on the client side.
+ conn.setAutoCommit(true);
+ try (PreparedStatement upsertStmt = conn.prepareStatement(upsert)) {
+ rowsInserted = upsertStmt.executeUpdate();
+ assertEquals(2, rowsInserted);
+ }
+ conn.commit();
+ }
+
+ String select = "SELECT i FROM " + tableName;
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ rs = stmt.executeQuery(select);
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals(1, rs.getInt(1));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertTrue(rs.next());
+ assertTrue(rs.next());
+ assertEquals(4, rs.getInt(1));
+ assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testUpsertSelectWithLimit() throws Exception {
+ Properties props = new Properties();
+ props.setProperty(QueryServices.ENABLE_SERVER_SIDE_UPSERT_MUTATIONS,
+ allowServerSideMutations);
+ String tableName = generateUniqueName();
+ TABLE_NAMES.add(tableName);
+ ResultSet rs;
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ stmt.execute("create table " + tableName +
+ " (id varchar(10) not null primary key, val varchar(10), ts timestamp)");
+ }
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ stmt.execute("upsert into " + tableName +
+ " values ('aaa', 'abc', current_date())");
+ stmt.execute("upsert into " + tableName +
+ " values ('bbb', 'bcd', current_date())");
+ stmt.execute("upsert into " + tableName +
+ " values ('ccc', 'cde', current_date())");
+ conn.commit();
+ }
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ rs = stmt.executeQuery("select * from " + tableName);
+
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("aaa", rs.getString(1));
+ assertEquals("abc", rs.getString(2));
+ assertNotNull(rs.getDate(3));
+
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("bbb", rs.getString(1));
+ assertEquals("bcd", rs.getString(2));
+ assertNotNull(rs.getDate(3));
+
+ assertTrue(rs.next());
+ assertEquals("ccc", rs.getString(1));
+ assertEquals("cde", rs.getString(2));
+ assertNotNull(rs.getDate(3));
+
+ assertFalse(rs.next());
+ }
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ stmt.execute("upsert into " + tableName +
+ " (id, ts) select id, CAST(null AS timestamp) from " + tableName +
+ " where id <= 'bbb' limit 1");
+ conn.commit();
+ }
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ rs = stmt.executeQuery("select * from " + tableName);
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("aaa", rs.getString(1));
+ assertEquals("abc", rs.getString(2));
+ assertNull(rs.getDate(3));
+
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals("bbb", rs.getString(1));
+ assertEquals("bcd", rs.getString(2));
+ assertNotNull(rs.getDate(3));
+
+ assertTrue(rs.next());
+ assertEquals("ccc", rs.getString(1));
+ assertEquals("cde", rs.getString(2));
+ assertNotNull(rs.getDate(3));
+
+ assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testUpsertSelectWithSequence() throws Exception {
+ Properties props = new Properties();
+ props.setProperty(QueryServices.ENABLE_SERVER_SIDE_UPSERT_MUTATIONS,
+ allowServerSideMutations);
+ String t1 = generateUniqueName();
+ String t2 = generateUniqueName();
+ String seq = generateUniqueName();
+ TABLE_NAMES.add(t1);
+ TABLE_NAMES.add(t2);
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ stmt.execute("create table " + t1 +
+ " (id bigint not null primary key, v varchar)");
+ stmt.execute("create table " + t2 + " (k varchar primary key)");
+ stmt.execute("create sequence " + seq);
+ }
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ stmt.execute("upsert into " + t2 + " values ('a')");
+ stmt.execute("upsert into " + t2 + " values ('b')");
+ stmt.execute("upsert into " + t2 + " values ('c')");
+ conn.commit();
+ }
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ stmt.execute("upsert into " + t1 + " select next value for " +
+ seq + " , k from " + t2);
+ conn.commit();
+ }
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ ResultSet rs = stmt.executeQuery("select * from " + t1);
+
+ assertTrue(rs.next());
+ moveRegionsOfTable(t1);
+ moveRegionsOfTable(t2);
+ assertEquals(1, rs.getLong(1));
+ assertEquals("a", rs.getString(2));
+
+ assertTrue(rs.next());
+ assertEquals(2, rs.getLong(1));
+ assertEquals("b", rs.getString(2));
+
+ assertTrue(rs.next());
+ moveRegionsOfTable(t1);
+ moveRegionsOfTable(t2);
+ assertEquals(3, rs.getLong(1));
+ assertEquals("c", rs.getString(2));
+
+ assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testUpsertSelectWithSequenceAndOrderByWithSalting() throws Exception {
+ int numOfRecords = 200;
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ props.setProperty(QueryServices.ENABLE_SERVER_SIDE_UPSERT_MUTATIONS,
+ allowServerSideMutations);
+ String t1 = generateUniqueName();
+ String t2 = generateUniqueName();
+ TABLE_NAMES.add(t1);
+ TABLE_NAMES.add(t2);
+ String ddl = "CREATE TABLE IF NOT EXISTS " + t1 + "(ORGANIZATION_ID CHAR(15) NOT NULL, " +
+ "QUERY_ID CHAR(15) NOT NULL, CURSOR_ORDER BIGINT NOT NULL, K1 INTEGER, " +
+ "V1 INTEGER " + "CONSTRAINT MAIN_PK PRIMARY KEY (ORGANIZATION_ID, QUERY_ID, " +
+ "CURSOR_ORDER) " + ") SALT_BUCKETS = 4";
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ stmt.execute(ddl);
+ stmt.execute(
+ "CREATE TABLE " + t2 + "(ORGANIZATION_ID CHAR(15) NOT NULL, k1 integer " +
+ "NOT NULL, v1 integer NOT NULL CONSTRAINT PK PRIMARY KEY " +
+ "(ORGANIZATION_ID, k1, v1) ) VERSIONS=1, SALT_BUCKETS = 4");
+ stmt.execute("create sequence s cache " + Integer.MAX_VALUE);
+ }
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ for (int i = 0; i < numOfRecords; i++) {
+ stmt.execute("UPSERT INTO " + t2 +
+ " values ('00Dxx0000001gEH'," + i + "," + (i + 2) + ")");
+ }
+ conn.commit();
+ }
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ conn.setAutoCommit(true);
+ stmt.execute("UPSERT INTO " + t1 +
+ " SELECT '00Dxx0000001gEH', 'MyQueryId', NEXT VALUE FOR S, k1, v1 FROM " +
+ t2 + " ORDER BY K1, V1");
+
+ }
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ ResultSet rs = stmt.executeQuery("select count(*) from " + t1);
+
+ assertTrue(rs.next());
+ moveRegionsOfTable(t1);
+ moveRegionsOfTable(t2);
+ assertEquals(numOfRecords, rs.getLong(1));
+
+ ResultSet rs2 = stmt.executeQuery("select cursor_order, k1, v1 from " + t1 +
+ " order by cursor_order");
+ long seq = 1;
+ while (rs2.next()) {
+ if (seq % 75 == 0) {
+ moveRegionsOfTable(t1);
+ moveRegionsOfTable(t2);
+ }
+ assertEquals(seq, rs2.getLong("cursor_order"));
+ // This value should be the sequence - 1 as we said order by k1 in the
+ // UPSERT...SELECT, but is not because of sequence processing.
+ assertEquals(seq - 1, rs2.getLong("k1"));
+ seq++;
+ }
+ // cleanup afrer ourselves
+ stmt.execute("drop sequence s");
+ }
+ }
+
+ @Test
+ public void testUpsertSelectWithRowtimeStampColumn() throws Exception {
+ String t1 = generateUniqueName();
+ String t2 = generateUniqueName();
+ String t3 = generateUniqueName();
+ TABLE_NAMES.add(t1);
+ TABLE_NAMES.add(t2);
+ TABLE_NAMES.add(t3);
+ try (Connection conn = DriverManager.getConnection(getUrl());
+ Statement stmt = conn.createStatement()) {
+ stmt.execute("CREATE TABLE " + t1 +
+ " (PK1 VARCHAR NOT NULL, PK2 DATE NOT NULL, KV1 VARCHAR CONSTRAINT PK " +
+ "PRIMARY KEY(PK1, PK2 DESC ROW_TIMESTAMP " + ")) ");
+ stmt.execute("CREATE TABLE " + t2 +
+ " (PK1 VARCHAR NOT NULL, PK2 DATE NOT NULL, KV1 VARCHAR CONSTRAINT PK " +
+ "PRIMARY KEY(PK1, PK2 ROW_TIMESTAMP)) ");
+ stmt.execute("CREATE TABLE " + t3 + " (PK1 VARCHAR NOT NULL, " +
+ "PK2 DATE NOT NULL, KV1 VARCHAR CONSTRAINT PK " +
+ "PRIMARY KEY(PK1, PK2 DESC ROW_TIMESTAMP " + ")) ");
+ }
+
+ // The timestamp of the put will be the value of the row_timestamp column.
+ long rowTimestamp = EnvironmentEdgeManager.currentTimeMillis();
+ Date rowTimestampDate = new Date(rowTimestamp);
+ Properties props = new Properties();
+ props.setProperty(QueryServices.ENABLE_SERVER_SIDE_UPSERT_MUTATIONS,
+ allowServerSideMutations);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + t1 +
+ " (PK1, PK2, KV1) VALUES(?, ?, ?)")) {
+ stmt.setString(1, "PK1");
+ stmt.setDate(2, rowTimestampDate);
+ stmt.setString(3, "KV1");
+ stmt.executeUpdate();
+ conn.commit();
+ }
+
+ // Upsert select data into table T2. The connection needs to be at a timestamp beyond the
+ // row timestamp. Otherwise it won't see the data from table T1.
+ try (Connection conn = DriverManager.getConnection(getUrl());
+ Statement stmt = conn.createStatement()) {
+ stmt.executeUpdate("UPSERT INTO " + t2 + " SELECT * FROM " + t1);
+ conn.commit();
+ // Verify the data upserted in T2. Note that we can use the same connection here because
+ // the data was inserted with a timestamp of rowTimestamp and the connection uses
+ // the latest timestamp
+ try (PreparedStatement prepStmt = conn.prepareStatement("SELECT * FROM " + t2 +
+ " WHERE PK1 = ? AND PK2 = ?")) {
+ prepStmt.setString(1, "PK1");
+ prepStmt.setDate(2, rowTimestampDate);
+ ResultSet rs = prepStmt.executeQuery();
+ assertTrue(rs.next());
+ assertEquals("PK1", rs.getString("PK1"));
+ assertEquals(rowTimestampDate, rs.getDate("PK2"));
+ assertEquals("KV1", rs.getString("KV1"));
+ }
+
+ }
+
+ // Verify that you can't see the data in T2 if the connection is at a timestamp
+ // lower than the row timestamp.
+ props.setProperty(PhoenixRuntime.CURRENT_SCN_ATTRIB, Long.toString(rowTimestamp - 1));
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ PreparedStatement stmt = conn.prepareStatement("SELECT * FROM " + t2 +
+ " WHERE PK1 = ? AND PK2 = ?")) {
+ stmt.setString(1, "PK1");
+ stmt.setDate(2, rowTimestampDate);
+ ResultSet rs = stmt.executeQuery();
+ assertFalse(rs.next());
+ }
+
+ // Upsert select data into table T3. The connection needs to be at a timestamp beyond the
+ // row timestamp. Otherwise it won't see the data from table T1.
+ try (Connection conn = DriverManager.getConnection(getUrl());
+ Statement stmt = conn.createStatement()) {
+ stmt.executeUpdate("UPSERT INTO " + t3 + " SELECT * FROM " + t1);
+ conn.commit();
+ // Verify the data upserted in T3. Note that we can use the same connection here
+ // because the data was inserted with a timestamp of rowTimestamp and the connection
+ // uses the latest timestamp
+ try (PreparedStatement prepStmt = conn.prepareStatement("SELECT * FROM " + t3 +
+ " WHERE PK1 = ? AND PK2 = ?")) {
+ prepStmt.setString(1, "PK1");
+ prepStmt.setDate(2, rowTimestampDate);
+ ResultSet rs = prepStmt.executeQuery();
+ assertTrue(rs.next());
+ assertEquals("PK1", rs.getString("PK1"));
+ assertEquals(rowTimestampDate, rs.getDate("PK2"));
+ assertEquals("KV1", rs.getString("KV1"));
+ }
+ }
+
+ // Verify that you can't see the data in T2 if the connection is at next timestamp
+ // (which is lower than the row timestamp).
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ PreparedStatement stmt = conn.prepareStatement("SELECT * FROM " + t3 +
+ " WHERE PK1 = ? AND PK2 = ?")) {
+ stmt.setString(1, "PK1");
+ stmt.setDate(2, rowTimestampDate);
+ ResultSet rs = stmt.executeQuery();
+ assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testUpsertSelectSameTableWithRowTimestampColumn() throws Exception {
+ String tableName = generateUniqueName();
+ TABLE_NAMES.add(tableName);
+ Properties props = new Properties();
+ props.setProperty(QueryServices.ENABLE_SERVER_SIDE_UPSERT_MUTATIONS,
+ allowServerSideMutations);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ stmt.execute("CREATE TABLE " + tableName +
+ " (PK1 INTEGER NOT NULL, PK2 DATE NOT NULL, KV1 VARCHAR CONSTRAINT PK " +
+ "PRIMARY KEY(PK1, PK2 ROW_TIMESTAMP)) ");
+ }
+
+ // The timestamp of the put will be the value of the row_timestamp column.
+ long rowTimestamp = 100;
+ Date rowTimestampDate = new Date(rowTimestamp);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + tableName +
+ " (PK1, PK2, KV1) VALUES(?, ?, ?)")) {
+ stmt.setInt(1, 1);
+ stmt.setDate(2, rowTimestampDate);
+ stmt.setString(3, "KV1");
+ stmt.executeUpdate();
+ conn.commit();
+ }
+ String seq = generateUniqueName();
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ stmt.execute("CREATE SEQUENCE " + seq);
+ }
+ // Upsert select data into table. The connection needs to be at a timestamp beyond the
+ // row timestamp. Otherwise it won't see the data from table.
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ stmt.executeUpdate("UPSERT INTO " + tableName +
+ " SELECT NEXT VALUE FOR " + seq + ", PK2 FROM " + tableName);
+ conn.commit();
+ }
+
+ // Upsert select using sequences.
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ conn.setAutoCommit(true);
+ for (int i = 0; i < 10; i++) {
+ int count = stmt.executeUpdate("UPSERT INTO " + tableName +
+ " SELECT NEXT VALUE FOR " + seq + ", PK2 FROM " + tableName);
+ assertEquals((int) Math.pow(2, i), count);
+ }
+ }
+ }
+
+ @Test
+ public void testAutomaticallySettingRowtimestamp() throws Exception {
+ String table1 = generateUniqueName();
+ String table2 = generateUniqueName();
+ String table3 = generateUniqueName();
+ TABLE_NAMES.add(table1);
+ TABLE_NAMES.add(table2);
+ TABLE_NAMES.add(table3);
+ try (Connection conn = DriverManager.getConnection(getUrl());
+ Statement stmt = conn.createStatement()) {
+ stmt.execute("CREATE TABLE " + table1 +
+ " (T1PK1 VARCHAR NOT NULL, T1PK2 DATE NOT NULL, T1KV1 VARCHAR, T1KV2 VARCHAR " +
+ "CONSTRAINT PK PRIMARY KEY(T1PK1, T1PK2 DESC ROW_TIMESTAMP)) ");
+ stmt.execute("CREATE TABLE " + table2 +
+ " (T2PK1 VARCHAR NOT NULL, T2PK2 DATE NOT NULL, T2KV1 VARCHAR, T2KV2 VARCHAR" +
+ " CONSTRAINT PK PRIMARY KEY(T2PK1, T2PK2 ROW_TIMESTAMP)) ");
+ stmt.execute("CREATE TABLE " + table3 +
+ " (T3PK1 VARCHAR NOT NULL, T3PK2 DATE NOT NULL, T3KV1 VARCHAR, T3KV2 VARCHAR" +
+ " CONSTRAINT PK PRIMARY KEY(T3PK1, T3PK2 DESC ROW_TIMESTAMP)) ");
+ }
+ long startTime = EnvironmentEdgeManager.currentTimeMillis();
+ Properties props = new Properties();
+ props.setProperty(QueryServices.ENABLE_SERVER_SIDE_UPSERT_MUTATIONS,
+ allowServerSideMutations);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + table1 +
+ " (T1PK1, T1KV1, T1KV2) VALUES (?, ?, ?)")) {
+ // Upsert values where row_timestamp column PK2 is not set and the column names
+ // are specified. This should upsert data with the value for PK2 as server timestamp
+ stmt.setString(1, "PK1");
+ stmt.setString(2, "KV1");
+ stmt.setString(3, "KV2");
+ stmt.executeUpdate();
+ conn.commit();
+ }
+ long endTime = EnvironmentEdgeManager.currentTimeMillis();
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ PreparedStatement stmt = conn.prepareStatement("SELECT T1KV1, T1KV2 FROM " +
+ table1 + " WHERE T1PK1 = ? AND T1PK2 >= ? AND T1PK2 <= ?")) {
+ // Now query for data that was upserted above. If the row key was generated correctly
+ // then we should be able to see the data in this query.
+ stmt.setString(1, "PK1");
+ stmt.setDate(2, new Date(startTime));
+ stmt.setDate(3, new Date(endTime));
+ ResultSet rs = stmt.executeQuery();
+ assertTrue(rs.next());
+ moveRegionsOfTable(table1);
+ moveRegionsOfTable(table2);
+ moveRegionsOfTable(table3);
+ assertEquals("KV1", rs.getString(1));
+ assertEquals("KV2", rs.getString(2));
+ assertFalse(rs.next());
+ }
+
+ startTime = EnvironmentEdgeManager.currentTimeMillis();
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + table2 +
+ " (T2PK1, T2KV1, T2KV2) SELECT T1PK1, T1KV1, T1KV2 FROM " + table1)) {
+ // Upsert select into table2 by not selecting the row timestamp column. In this case,
+ // the rowtimestamp column would end up being set to the server timestamp
+ stmt.executeUpdate();
+ conn.commit();
+ }
+ endTime = EnvironmentEdgeManager.currentTimeMillis();
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ PreparedStatement stmt = conn.prepareStatement("SELECT T2KV1, T2KV2 FROM " +
+ table2 + " WHERE T2PK1 = ? AND T2PK2 >= ? AND T2PK2 <= ?")) {
+ // Now query for data that was upserted above. If the row key was generated correctly
+ // then we should be able to see the data in this query.
+ stmt.setString(1, "PK1");
+ stmt.setDate(2, new Date(startTime));
+ stmt.setDate(3, new Date(endTime));
+ ResultSet rs = stmt.executeQuery();
+ assertTrue(rs.next());
+ assertEquals("KV1", rs.getString(1));
+ assertEquals("KV2", rs.getString(2));
+ assertFalse(rs.next());
+ }
+
+ startTime = EnvironmentEdgeManager.currentTimeMillis();
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + table3 +
+ " (T3PK1, T3KV1, T3KV2) SELECT T2PK1, T2KV1, T2KV2 FROM " + table2)) {
+ // Upsert select into table3 by not selecting the row timestamp column. In this case,
+ // the rowtimestamp column would end up being set to the server timestamp
+ stmt.executeUpdate();
+ conn.commit();
+ }
+ endTime = EnvironmentEdgeManager.currentTimeMillis();
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ PreparedStatement stmt = conn.prepareStatement("SELECT T3KV1, T3KV2 FROM " +
+ table3 + " WHERE T3PK1 = ? AND T3PK2 >= ? AND T3PK2 <= ?")) {
+ // Now query for data that was upserted above. If the row key was generated correctly
+ // then we should be able to see the data in this query.
+ stmt.setString(1, "PK1");
+ stmt.setDate(2, new Date(startTime));
+ stmt.setDate(3, new Date(endTime));
+ ResultSet rs = stmt.executeQuery();
+ assertTrue(rs.next());
+ moveRegionsOfTable(table1);
+ moveRegionsOfTable(table2);
+ moveRegionsOfTable(table3);
+ assertEquals("KV1", rs.getString(1));
+ assertEquals("KV2", rs.getString(2));
+ assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testUpsertSelectAutoCommitWithRowTimestampColumn() throws Exception {
+ String tableName1 = generateUniqueName();
+ String tableName2 = generateUniqueName();
+ TABLE_NAMES.add(tableName1);
+ TABLE_NAMES.add(tableName2);
+ Properties props = new Properties();
+ props.setProperty(QueryServices.ENABLE_SERVER_SIDE_UPSERT_MUTATIONS,
+ allowServerSideMutations);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ stmt.execute("CREATE TABLE " + tableName1 +
+ " (PK1 INTEGER NOT NULL, PK2 DATE NOT NULL, PK3 INTEGER NOT NULL, KV1 VARCHAR" +
+ " CONSTRAINT PK PRIMARY KEY(PK1, PK2 ROW_TIMESTAMP, PK3)) ");
+ stmt.execute("CREATE TABLE " + tableName2 +
+ " (PK1 INTEGER NOT NULL, PK2 DATE NOT NULL, PK3 INTEGER NOT NULL, KV1 VARCHAR" +
+ " CONSTRAINT PK PRIMARY KEY(PK1, PK2 DESC ROW_TIMESTAMP, PK3)) ");
+ }
+
+ String[] tableNames = {tableName1, tableName2};
+ for (String tableName : tableNames) {
+ // Upsert data with the row timestamp value set
+ long rowTimestamp1 = 100;
+ Date rowTimestampDate = new Date(rowTimestamp1);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " +
+ tableName + " (PK1, PK2, PK3, KV1) VALUES(?, ?, ?, ?)")) {
+ stmt.setInt(1, 1);
+ stmt.setDate(2, rowTimestampDate);
+ stmt.setInt(3, 3);
+ stmt.setString(4, "KV1");
+ stmt.executeUpdate();
+ conn.commit();
+ }
+
+ long startTime = EnvironmentEdgeManager.currentTimeMillis();
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ conn.setAutoCommit(true);
+ // Upsert select in the same table with the row_timestamp column PK2 not specified.
+ // This will end up creating a new row whose timestamp is the server time stamp
+ // which is also used for the row key
+ stmt.executeUpdate("UPSERT INTO " + tableName +
+ " (PK1, PK3, KV1) SELECT PK1, PK3, KV1 FROM " + tableName);
+ }
+ long endTime = EnvironmentEdgeManager.currentTimeMillis();
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ PreparedStatement stmt = conn.prepareStatement("SELECT * FROM " + tableName +
+ " WHERE PK1 = ? AND PK2 >= ? AND PK2<= ? AND PK3 = ?")) {
+ // Verify the row that was upserted above
+ stmt.setInt(1, 1);
+ stmt.setDate(2, new Date(startTime));
+ stmt.setDate(3, new Date(endTime));
+ stmt.setInt(4, 3);
+ ResultSet rs = stmt.executeQuery();
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals(1, rs.getInt("PK1"));
+ assertEquals(3, rs.getInt("PK3"));
+ assertEquals("KV1", rs.getString("KV1"));
+ assertFalse(rs.next());
+ // Number of rows in the table should be 2.
+ try (Statement newStmt = conn.createStatement()) {
+ rs = newStmt.executeQuery("SELECT COUNT(*) FROM " + tableName);
+ assertTrue(rs.next());
+ assertEquals(2, rs.getInt(1));
+ }
+
+ }
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ conn.setAutoCommit(true);
+ // Upsert select in the same table with the row_timestamp column PK2 specified.
+ // This will not end up creating a new row because the destination pk columns,
+ // including the row timestamp column PK2, are the same as the source column.
+ stmt.executeUpdate("UPSERT INTO " + tableName +
+ " (PK1, PK2, PK3, KV1) SELECT PK1, PK2, PK3, KV1 FROM " + tableName);
+ }
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ // Verify that two rows were created. One with rowtimestamp1 and the other
+ // with rowtimestamp2
+ ResultSet rs = stmt.executeQuery("SELECT COUNT(*) FROM " + tableName);
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ assertEquals(2, rs.getInt(1));
+ assertFalse(rs.next());
+ }
+
+ }
+ }
+
+ @Test
+ public void testRowTimestampColWithViewsIndexesAndSaltedTables() throws Exception {
+ String baseTable = generateUniqueName();
+ String tenantView = generateUniqueName();
+ String globalView = generateUniqueName();
+ String baseTableIdx = generateUniqueName();
+ String tenantViewIdx = generateUniqueName();
+ TABLE_NAMES.add(baseTable);
+ TABLE_NAMES.add(baseTableIdx);
+ TABLE_NAMES.add("_IDX_" + baseTable);
+
+ Properties props = new Properties();
+ props.setProperty(QueryServices.ENABLE_SERVER_SIDE_UPSERT_MUTATIONS,
+ allowServerSideMutations);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ stmt.execute("CREATE IMMUTABLE TABLE " + baseTable +
+ " (TENANT_ID CHAR(15) NOT NULL, PK2 DATE NOT NULL, PK3 INTEGER NOT NULL, " +
+ "KV1 VARCHAR, KV2 VARCHAR, KV3 VARCHAR CONSTRAINT PK PRIMARY KEY(TENANT_ID, " +
+ "PK2 ROW_TIMESTAMP, PK3)) MULTI_TENANT = true, SALT_BUCKETS = 8");
+ stmt.execute("CREATE INDEX " + baseTableIdx + " ON " +
+ baseTable + " (PK2, KV3) INCLUDE (KV1)");
+ stmt.execute("CREATE VIEW " + globalView + " AS SELECT * FROM " +
+ baseTable + " WHERE KV1 = 'KV1'");
+ }
+
+ String tenantId = "tenant1";
+ try (Connection conn = getTenantConnection(tenantId);
+ Statement stmt = conn.createStatement()) {
+ stmt.execute("CREATE VIEW " + tenantView + " AS SELECT * FROM " +
+ baseTable);
+ stmt.execute("CREATE INDEX " + tenantViewIdx + " ON " +
+ tenantView + " (PK2, KV2) INCLUDE (KV1)");
+ }
+
+ // upsert data into base table without specifying the row timestamp column PK2
+ long startTime = EnvironmentEdgeManager.currentTimeMillis();
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + baseTable +
+ " (TENANT_ID, PK3, KV1, KV2, KV3) VALUES (?, ?, ?, ?, ?)")) {
+ // Upsert select in the same table with the row_timestamp column PK2 not specified.
+ // This will end up creating a new row whose timestamp is the latest timestamp
+ // (which will be used for the row key too)
+ stmt.setString(1, tenantId);
+ stmt.setInt(2, 3);
+ stmt.setString(3, "KV1");
+ stmt.setString(4, "KV2");
+ stmt.setString(5, "KV3");
+ stmt.executeUpdate();
+ conn.commit();
+ }
+ long endTime = EnvironmentEdgeManager.currentTimeMillis();
+
+ // Verify that we can see data when querying through base table, global view and index on
+ // the base table
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ // Query the base table
+ try (PreparedStatement stmt = conn.prepareStatement("SELECT * FROM " + baseTable +
+ " WHERE TENANT_ID = ? AND PK2 >= ? AND PK2 <= ? AND PK3 = ?")) {
+ stmt.setString(1, tenantId);
+ stmt.setDate(2, new Date(startTime));
+ stmt.setDate(3, new Date(endTime));
+ stmt.setInt(4, 3);
+ ResultSet rs = stmt.executeQuery();
+ assertTrue(rs.next());
+ moveRegionsOfTable(baseTable);
+ moveRegionsOfTable(baseTableIdx);
+ moveRegionsOfTable("_IDX_" + baseTable);
+ assertEquals(tenantId, rs.getString("TENANT_ID"));
+ assertEquals("KV1", rs.getString("KV1"));
+ assertEquals("KV2", rs.getString("KV2"));
+ assertEquals("KV3", rs.getString("KV3"));
+ assertFalse(rs.next());
+ }
+
+ // Query the globalView
+ try (PreparedStatement stmt = conn.prepareStatement(
+ "SELECT /*+ NO_INDEX */ * FROM " + globalView +
+ " WHERE TENANT_ID = ? AND PK2 >= ? AND PK2 <= ? AND PK3 = ?")) {
+ stmt.setString(1, tenantId);
+ stmt.setDate(2, new Date(startTime));
+ stmt.setDate(3, new Date(endTime));
+ stmt.setInt(4, 3);
+ ResultSet rs = stmt.executeQuery();
+ assertTrue(rs.next());
+ moveRegionsOfTable(baseTable);
+ moveRegionsOfTable(baseTableIdx);
+ moveRegionsOfTable("_IDX_" + baseTable);
+ assertEquals(tenantId, rs.getString("TENANT_ID"));
+ assertEquals("KV1", rs.getString("KV1"));
+ assertEquals("KV2", rs.getString("KV2"));
+ assertEquals("KV3", rs.getString("KV3"));
+ assertFalse(rs.next());
+ }
+
+ // Query using the index on base table
+ try (PreparedStatement stmt = conn.prepareStatement("SELECT KV1 FROM " +
+ baseTable + " WHERE PK2 >= ? AND PK2 <= ? AND KV3 = ?")) {
+ stmt.setDate(1, new Date(startTime));
+ stmt.setDate(2, new Date(endTime));
+ stmt.setString(3, "KV3");
+ ResultSet rs = stmt.executeQuery();
+ QueryPlan plan = stmt.unwrap(PhoenixStatement.class).getQueryPlan();
+ assertEquals(plan.getTableRef().getTable().getName().getString(), baseTableIdx);
+ assertTrue(rs.next());
+ moveRegionsOfTable(baseTable);
+ moveRegionsOfTable(baseTableIdx);
+ moveRegionsOfTable("_IDX_" + baseTable);
+ assertEquals("KV1", rs.getString("KV1"));
+ assertFalse(rs.next());
+ }
+ }
+
+ // Verify that data can be queried using tenant view and tenant view index
+ try (Connection tenantConn = getTenantConnection(tenantId);
+ PreparedStatement stmt = tenantConn.prepareStatement("SELECT * FROM " +
+ tenantView + " WHERE PK2 >= ? AND PK2 <= ? AND PK3 = ?")) {
+ // Query the tenant view
+ stmt.setDate(1, new Date(startTime));
+ stmt.setDate(2, new Date(endTime));
+ stmt.setInt(3, 3);
+ ResultSet rs = stmt.executeQuery();
+ assertTrue(rs.next());
+ moveRegionsOfTable(baseTable);
+ moveRegionsOfTable(baseTableIdx);
+ moveRegionsOfTable("_IDX_" + baseTable);
+ assertEquals("KV1", rs.getString("KV1"));
+ assertEquals("KV2", rs.getString("KV2"));
+ assertEquals("KV3", rs.getString("KV3"));
+ assertFalse(rs.next());
+
+ // Query using the index on the tenantView
+ //TODO: uncomment the code after PHOENIX-2277 is fixed
+// stmt = tenantConn.prepareStatement("SELECT KV1 FROM " + tenantView +
+// " WHERE PK2 = ? AND KV2 = ?");
+// stmt.setDate(1, new Date(upsertedTs));
+// stmt.setString(2, "KV2");
+// rs = stmt.executeQuery();
+// QueryPlan plan = stmt.unwrap(PhoenixStatement.class).getQueryPlan();
+// assertTrue(plan.getTableRef().getTable().getName().getString().equals(tenantViewIdx));
+// assertTrue(rs.next());
+// assertEquals("KV1", rs.getString("KV1"));
+// assertFalse(rs.next());
+ }
+
+ long upsertedTs;
+ try (Connection tenantConn = getTenantConnection(tenantId)) {
+ // Upsert into tenant view where the row_timestamp column PK2 is not specified
+ startTime = EnvironmentEdgeManager.currentTimeMillis();
+ try (PreparedStatement stmt = tenantConn.prepareStatement("UPSERT INTO " +
+ tenantView + " (PK3, KV1, KV2, KV3) VALUES (?, ?, ?, ?)")) {
+ stmt.setInt(1, 33);
+ stmt.setString(2, "KV13");
+ stmt.setString(3, "KV23");
+ stmt.setString(4, "KV33");
+ stmt.executeUpdate();
+ }
+ tenantConn.commit();
+ upsertedTs = endTime = EnvironmentEdgeManager.currentTimeMillis();
+
+ // Upsert into tenant view where the row_timestamp column PK2 is specified
+ try (PreparedStatement stmt = tenantConn.prepareStatement("UPSERT INTO " +
+ tenantView + " (PK2, PK3, KV1, KV2, KV3) VALUES (?, ?, ?, ?, ?)")) {
+ stmt.setDate(1, new Date(upsertedTs));
+ stmt.setInt(2, 44);
+ stmt.setString(3, "KV14");
+ stmt.setString(4, "KV24");
+ stmt.setString(5, "KV34");
+ stmt.executeUpdate();
+ }
+ tenantConn.commit();
+ }
+
+ // Verify that the data upserted using the tenant view can now be queried using base table
+ // and the base table index
+ Date upsertedDate;
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ // Query the base table
+ try (PreparedStatement stmt = conn.prepareStatement("SELECT * FROM " + baseTable +
+ " WHERE TENANT_ID = ? AND PK2 >= ? AND PK2 <= ? AND PK3 = ? ")) {
+ stmt.setString(1, tenantId);
+ stmt.setDate(2, new Date(startTime));
+ stmt.setDate(3, new Date(endTime));
+ stmt.setInt(4, 33);
+ ResultSet rs = stmt.executeQuery();
+ assertTrue(rs.next());
+ moveRegionsOfTable(baseTable);
+ moveRegionsOfTable(baseTableIdx);
+ moveRegionsOfTable("_IDX_" + baseTable);
+ assertEquals(tenantId, rs.getString("TENANT_ID"));
+ assertEquals("KV13", rs.getString("KV1"));
+ assertEquals("KV23", rs.getString("KV2"));
+ assertEquals("KV33", rs.getString("KV3"));
+ upsertedDate = rs.getDate("PK2");
+ assertFalse(rs.next());
+ }
+
+ try (PreparedStatement stmt = conn.prepareStatement("SELECT * FROM " + baseTable +
+ " WHERE TENANT_ID = ? AND PK2 = ? AND PK3 = ? ")) {
+ stmt.setString(1, tenantId);
+ stmt.setDate(2, new Date(upsertedTs));
+ stmt.setInt(3, 44);
+ ResultSet rs = stmt.executeQuery();
+ assertTrue(rs.next());
+ moveRegionsOfTable(baseTable);
+ moveRegionsOfTable(baseTableIdx);
+ moveRegionsOfTable("_IDX_" + baseTable);
+ assertEquals(tenantId, rs.getString("TENANT_ID"));
+ assertEquals("KV14", rs.getString("KV1"));
+ assertEquals("KV24", rs.getString("KV2"));
+ assertEquals("KV34", rs.getString("KV3"));
+ assertFalse(rs.next());
+ }
+
+ // Query using the index on base table
+ try (PreparedStatement stmt = conn.prepareStatement("SELECT KV1 FROM " + baseTable +
+ " WHERE (PK2, KV3) IN ((?, ?), (?, ?)) ORDER BY KV1")) {
+ stmt.setDate(1, upsertedDate);
+ stmt.setString(2, "KV33");
+ stmt.setDate(3, new Date(upsertedTs));
+ stmt.setString(4, "KV34");
+ ResultSet rs = stmt.executeQuery();
+ QueryPlan plan = stmt.unwrap(PhoenixStatement.class).getQueryPlan();
+ assertTrue(
+ plan.getTableRef().getTable().getName().getString().equals(baseTableIdx));
+ assertTrue(rs.next());
+ moveRegionsOfTable(baseTable);
+ moveRegionsOfTable(baseTableIdx);
+ moveRegionsOfTable("_IDX_" + baseTable);
+ assertEquals("KV13", rs.getString("KV1"));
+ assertTrue(rs.next());
+ moveRegionsOfTable(baseTable);
+ moveRegionsOfTable(baseTableIdx);
+ moveRegionsOfTable("_IDX_" + baseTable);
+ assertEquals("KV14", rs.getString("KV1"));
+ assertFalse(rs.next());
+ }
+ }
+
+ // Verify that the data upserted using the tenant view can now be queried using tenant view
+ try (Connection tenantConn = getTenantConnection(tenantId);
+ PreparedStatement stmt = tenantConn.prepareStatement("SELECT * FROM " +
+ tenantView + " WHERE (PK2, PK3) IN ((?, ?), (?, ?)) ORDER BY KV1")) {
+ // Query the base table
+ stmt.setDate(1, upsertedDate);
+ stmt.setInt(2, 33);
+ stmt.setDate(3, new Date(upsertedTs));
+ stmt.setInt(4, 44);
+ ResultSet rs = stmt.executeQuery();
+ assertTrue(rs.next());
+ moveRegionsOfTable(baseTable);
+ moveRegionsOfTable(baseTableIdx);
+ moveRegionsOfTable("_IDX_" + baseTable);
+ assertEquals("KV13", rs.getString("KV1"));
+ assertTrue(rs.next());
+ moveRegionsOfTable(baseTable);
+ moveRegionsOfTable(baseTableIdx);
+ moveRegionsOfTable("_IDX_" + baseTable);
+ assertEquals("KV14", rs.getString("KV1"));
+ assertFalse(rs.next());
+
+ //TODO: uncomment the code after PHOENIX-2277 is fixed
+// // Query using the index on the tenantView
+// stmt = tenantConn.prepareStatement("SELECT KV1 FROM " + tenantView +
+// " WHERE (PK2, KV2) IN (?, ?, ?, ?) ORDER BY KV1");
+// stmt.setDate(1, new Date(upsertedTs));
+// stmt.setString(2, "KV23");
+// stmt.setDate(3, new Date(upsertedTs));
+// stmt.setString(4, "KV24");
+// rs = stmt.executeQuery();
+// QueryPlan plan = stmt.unwrap(PhoenixStatement.class).getQueryPlan();
+// assertTrue(plan.getTableRef().getTable().getName().getString().equals(tenantViewIdx));
+// assertTrue(rs.next());
+// assertEquals("KV13", rs.getString("KV1"));
+// assertTrue(rs.next());
+// assertEquals("KV14", rs.getString("KV1"));
+// assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testDisallowNegativeValuesForRowTsColumn() throws Exception {
+ String tableName = generateUniqueName();
+ String tableName2 = generateUniqueName();
+ TABLE_NAMES.add(tableName);
+ TABLE_NAMES.add(tableName2);
+ Properties props = new Properties();
+ props.setProperty(QueryServices.ENABLE_SERVER_SIDE_UPSERT_MUTATIONS,
+ allowServerSideMutations);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ stmt.execute("CREATE TABLE " + tableName +
+ " (PK1 BIGINT NOT NULL PRIMARY KEY ROW_TIMESTAMP, KV1 VARCHAR)");
+ stmt.execute("CREATE TABLE " + tableName2 +
+ " (PK1 BIGINT NOT NULL PRIMARY KEY ROW_TIMESTAMP, KV1 VARCHAR)");
+ }
+ long upsertedTs = 100;
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + tableName +
+ " VALUES (?, ?)")) {
+ stmt.setLong(1, upsertedTs);
+ stmt.setString(2, "KV1");
+ stmt.executeUpdate();
+ conn.commit();
+ }
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + tableName2 +
+ " SELECT (PK1 - 500), KV1 FROM " + tableName)) {
+ stmt.executeUpdate();
+ fail();
+ } catch (SQLException e) {
+ assertEquals(SQLExceptionCode.ILLEGAL_DATA.getErrorCode(), e.getErrorCode());
+ }
+ }
+
+ @Test
+ public void testUpsertSelectWithMultiByteCharsNoAutoCommit() throws Exception {
+ testUpsertSelectWithMultiByteChars(false);
+ }
+
+ @Test
+ public void testUpsertSelectWithMultiByteCharsAutoCommit() throws Exception {
+ testUpsertSelectWithMultiByteChars(true);
+ }
+
+ private void testUpsertSelectWithMultiByteChars(boolean autoCommit) throws Exception {
+ Properties props = new Properties();
+ props.setProperty(QueryServices.ENABLE_SERVER_SIDE_UPSERT_MUTATIONS,
+ allowServerSideMutations);
+ String t1 = generateUniqueName();
+ TABLE_NAMES.add(t1);
+ String validValue = "澴粖蟤य褻酃岤豦팑薰鄩脼ժ끦碉碉碉碉碉碉";
+ String invalidValue = "澴粖蟤य褻酃岤豦팑薰鄩脼ժ끦碉碉碉碉碉碉碉";
+ String columnTypeInfo = "VARCHAR(20)";
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ conn.setAutoCommit(autoCommit);
+ stmt.execute("create table " + t1 +
+ " (id bigint not null primary key, v varchar(20))");
+ }
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ conn.setAutoCommit(autoCommit);
+ stmt.execute("upsert into " + t1 + " values (1, 'foo')");
+ conn.commit();
+ }
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ conn.setAutoCommit(autoCommit);
+ stmt.execute("upsert into " + t1 + " (id, v) select id, "
+ + "'" + validValue + "' from " + t1 + " WHERE id = 1");
+ conn.commit();
+ }
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ conn.setAutoCommit(autoCommit);
+ ResultSet rs = stmt.executeQuery("select * from " + t1);
+ assertTrue(rs.next());
+ moveRegionsOfTable(t1);
+ assertEquals(1, rs.getLong(1));
+ assertEquals(validValue, rs.getString(2));
+ }
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ conn.setAutoCommit(autoCommit);
+ stmt.execute("upsert into " + t1 + " (id, v) select id, "
+ + "'" + invalidValue + "' from " + t1 + " WHERE id = 1");
+ conn.commit();
+ fail();
+ } catch (SQLException e) {
+ assertEquals(SQLExceptionCode.DATA_EXCEEDS_MAX_CAPACITY.getErrorCode(),
+ e.getErrorCode());
+ assertTrue(e.getMessage().contains(columnTypeInfo));
+ }
+ }
+
+ @Test
+ public void testParallelUpsertSelect() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ props.setProperty(QueryServices.MUTATE_BATCH_SIZE_BYTES_ATTRIB, Integer.toString(512));
+ props.setProperty(QueryServices.SCAN_CACHE_SIZE_ATTRIB, Integer.toString(3));
+ props.setProperty(QueryServices.SCAN_RESULT_CHUNK_SIZE, Integer.toString(3));
+ props.setProperty(QueryServices.ENABLE_SERVER_SIDE_UPSERT_MUTATIONS,
+ allowServerSideMutations);
+ String t1 = generateUniqueName();
+ String t2 = generateUniqueName();
+ String seq = generateUniqueName();
+ TABLE_NAMES.add(t1);
+ TABLE_NAMES.add(t2);
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ conn.setAutoCommit(false);
+
+ stmt.execute("CREATE SEQUENCE " + seq);
+ stmt.execute("CREATE TABLE " + t1 +
+ " (pk INTEGER PRIMARY KEY, val INTEGER) SALT_BUCKETS=4");
+ stmt.execute("CREATE TABLE " + t2 +
+ " (pk INTEGER PRIMARY KEY, val INTEGER)");
+ }
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ for (int i = 0; i < 100; i++) {
+ stmt.execute("UPSERT INTO " + t1 +
+ " VALUES (NEXT VALUE FOR " + seq + ", " + (i % 10) + ")");
+ }
+ conn.commit();
+ }
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ conn.setAutoCommit(true);
+ try (Statement stmt = conn.createStatement()) {
+ int upsertCount = stmt.executeUpdate("UPSERT INTO " + t2 +
+ " SELECT pk, val FROM " + t1);
+ assertEquals(100, upsertCount);
+ }
+ }
+ }
+
+ @Test // See https://issues.apache.org/jira/browse/PHOENIX-4265
+ public void testLongCodecUsedForRowTimestamp() throws Exception {
+ String tableName = generateUniqueName();
+ String indexName = generateUniqueName();
+ TABLE_NAMES.add(tableName);
+ TABLE_NAMES.add(indexName);
+ Properties props = new Properties();
+ props.setProperty(QueryServices.ENABLE_SERVER_SIDE_UPSERT_MUTATIONS,
+ allowServerSideMutations);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement statement = conn.createStatement()) {
+ statement.execute("CREATE IMMUTABLE TABLE " + tableName +
+ " (k1 TIMESTAMP not null, k2 bigint not null, v bigint, constraint pk " +
+ "primary key (k1 row_timestamp, k2)) SALT_BUCKETS = 9");
+ statement.execute("CREATE INDEX " + indexName + " ON " + tableName +
+ " (v) INCLUDE (k2)");
+ try (PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + tableName +
+ " VALUES (?, ?, ?) ")) {
+ stmt.setTimestamp(1, new Timestamp(1000));
+ stmt.setLong(2, 2000);
+ stmt.setLong(3, 1000);
+ stmt.executeUpdate();
+ stmt.setTimestamp(1, new Timestamp(2000));
+ stmt.setLong(2, 5000);
+ stmt.setLong(3, 5);
+ stmt.executeUpdate();
+ stmt.setTimestamp(1, new Timestamp(3000));
+ stmt.setLong(2, 5000);
+ stmt.setLong(3, 5);
+ stmt.executeUpdate();
+ stmt.setTimestamp(1, new Timestamp(4000));
+ stmt.setLong(2, 5000);
+ stmt.setLong(3, 5);
+ stmt.executeUpdate();
+ stmt.setTimestamp(1, new Timestamp(5000));
+ stmt.setLong(2, 2000);
+ stmt.setLong(3, 10);
+ stmt.executeUpdate();
+ stmt.setTimestamp(1, new Timestamp(6000));
+ stmt.setLong(2, 2000);
+ stmt.setLong(3, 20);
+ stmt.executeUpdate();
+ }
+ conn.commit();
+
+ ResultSet rs = statement.executeQuery("SELECT " +
+ " K2 FROM " + tableName + " WHERE V = 5");
+ assertTrue("Index " + indexName + " should have been used",
+ rs.unwrap(PhoenixResultSet.class).getStatement().getQueryPlan().getTableRef()
+ .getTable().getName().getString().equals(indexName));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ moveRegionsOfTable(indexName);
+ assertEquals(5000, rs.getLong("k2"));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ moveRegionsOfTable(indexName);
+ assertEquals(5000, rs.getLong("k2"));
+ assertTrue(rs.next());
+ assertEquals(5000, rs.getLong("k2"));
+ assertFalse(rs.next());
+
+ rs = statement.executeQuery("SELECT /*+ INDEX(" + tableName + " "
+ + indexName + ") */ " + " K2 FROM " + tableName + " WHERE V = 5");
+ assertTrue("Index " + indexName + " should have been used",
+ rs.unwrap(PhoenixResultSet.class).getStatement().getQueryPlan()
+ .getTableRef().getTable().getName().getString().equals(indexName));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ moveRegionsOfTable(indexName);
+ assertEquals(5000, rs.getLong("k2"));
+ assertTrue(rs.next());
+ assertEquals(5000, rs.getLong("k2"));
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName);
+ moveRegionsOfTable(indexName);
+ assertEquals(5000, rs.getLong("k2"));
+ assertFalse(rs.next());
+
+ }
+ }
+
+ @Test // See https://issues.apache.org/jira/browse/PHOENIX-4646
+ public void testLengthLimitedVarchar() throws Exception {
+ String tableName1 = generateUniqueName();
+ String tableName2 = generateUniqueName();
+ TABLE_NAMES.add(tableName1);
+ TABLE_NAMES.add(tableName2);
+ Properties props = new Properties();
+ props.setProperty(QueryServices.ENABLE_SERVER_SIDE_UPSERT_MUTATIONS,
+ allowServerSideMutations);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props);
+ Statement stmt = conn.createStatement()) {
+ conn.setAutoCommit(true);
+ stmt.execute("create table " + tableName1 +
+ "(name varchar(160) primary key, id varchar(120), address varchar(160))");
+ stmt.execute("create table " + tableName2 +
+ "(name varchar(160) primary key, id varchar(10), address varchar(10))");
+ stmt.execute("upsert into " + tableName1 +
+ " values('test','test','test')");
+ stmt.execute("upsert into " + tableName2 + " select * from " +
+ tableName1);
+ ResultSet rs = stmt.executeQuery("select * from " +
+ tableName2);
+ assertTrue(rs.next());
+ moveRegionsOfTable(tableName1);
+ moveRegionsOfTable(tableName2);
+ assertEquals("test", rs.getString(1));
+ assertEquals("test", rs.getString(2));
+ assertEquals("test", rs.getString(2));
+ assertFalse(rs.next());
+ }
+ }
+
+ private Connection getTenantConnection(String tenantId) throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TestUtil.TEST_PROPERTIES);
+ props.setProperty(QueryServices.ENABLE_SERVER_SIDE_UPSERT_MUTATIONS,
+ allowServerSideMutations);
+ props.setProperty(TENANT_ID_ATTRIB, tenantId);
+ return DriverManager.getConnection(getUrl(), props);
+ }
+
+}
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseIndexWithRegionMovesIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseIndexWithRegionMovesIT.java
new file mode 100644
index 0000000000..6f4c270591
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/BaseIndexWithRegionMovesIT.java
@@ -0,0 +1,1847 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.phoenix.end2end.index;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellScanner;
+import org.apache.hadoop.hbase.CellUtil;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Result;
+import org.apache.hadoop.hbase.client.ResultScanner;
+import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
+import org.apache.phoenix.compile.ColumnResolver;
+import org.apache.phoenix.compile.ExplainPlan;
+import org.apache.phoenix.compile.ExplainPlanAttributes;
+import org.apache.phoenix.compile.FromCompiler;
+import org.apache.phoenix.end2end.CreateTableIT;
+import org.apache.phoenix.end2end.IndexToolIT;
+import org.apache.phoenix.end2end.ParallelStatsDisabledWithRegionMovesIT;
+import org.apache.phoenix.exception.SQLExceptionCode;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixPreparedStatement;
+import org.apache.phoenix.jdbc.PhoenixResultSet;
+import org.apache.phoenix.jdbc.PhoenixStatement;
+import org.apache.phoenix.parse.NamedTableNode;
+import org.apache.phoenix.parse.TableName;
+import org.apache.phoenix.query.BaseTest;
+import org.apache.phoenix.query.QueryConstants;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.schema.PIndexState;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTableImpl;
+import org.apache.phoenix.schema.PTableKey;
+import org.apache.phoenix.schema.PTableType;
+import org.apache.phoenix.transaction.PhoenixTransactionProvider.Feature;
+import org.apache.phoenix.transaction.TransactionFactory;
+import org.apache.phoenix.util.DateUtil;
+import org.apache.phoenix.util.EnvironmentEdgeManager;
+import org.apache.phoenix.util.PhoenixRuntime;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.QueryUtil;
+import org.apache.phoenix.util.SchemaUtil;
+import org.apache.phoenix.util.TestUtil;
+import org.apache.phoenix.util.TransactionUtil;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import java.io.IOException;
+import java.math.BigDecimal;
+import java.sql.Connection;
+import java.sql.Date;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Properties;
+import java.util.Random;
+
+import static org.apache.phoenix.query.QueryConstants.MILLIS_IN_DAY;
+import static org.apache.phoenix.util.TestUtil.ROW5;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.junit.Assume.assumeFalse;
+
+@RunWith(Parameterized.class)
+public abstract class BaseIndexWithRegionMovesIT extends ParallelStatsDisabledWithRegionMovesIT {
+ private static final Random RAND = new Random();
+
+ private final boolean localIndex;
+ private final boolean uncovered;
+ private final boolean transactional;
+ private final TransactionFactory.Provider transactionProvider;
+ private final boolean mutable;
+ private final boolean columnEncoded;
+ private final String tableDDLOptions;
+
+ protected BaseIndexWithRegionMovesIT(boolean localIndex, boolean uncovered, boolean mutable,
+ String transactionProvider, boolean columnEncoded) {
+ this.localIndex = localIndex;
+ this.uncovered = uncovered;
+ this.mutable = mutable;
+ this.columnEncoded = columnEncoded;
+ StringBuilder optionBuilder = new StringBuilder();
+ if (!columnEncoded) {
+ if (optionBuilder.length() != 0)
+ optionBuilder.append(",");
+ optionBuilder.append("COLUMN_ENCODED_BYTES=0");
+ }
+ if (!mutable) {
+ if (optionBuilder.length() != 0)
+ optionBuilder.append(",");
+ optionBuilder.append("IMMUTABLE_ROWS=true");
+ if (!columnEncoded) {
+ optionBuilder.append(",IMMUTABLE_STORAGE_SCHEME=" +
+ PTableImpl.ImmutableStorageScheme.ONE_CELL_PER_COLUMN);
+ }
+ }
+ transactional = transactionProvider != null;
+ if (transactional) {
+ if (optionBuilder.length() != 0)
+ optionBuilder.append(",");
+ optionBuilder.append(
+ " TRANSACTIONAL=true,TRANSACTION_PROVIDER='" + transactionProvider + "'");
+ this.transactionProvider = TransactionFactory.Provider.valueOf(transactionProvider);
+ } else {
+ this.transactionProvider = null;
+ }
+ this.tableDDLOptions = optionBuilder.toString();
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ hasTestStarted = true;
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ countOfDummyResults = 0;
+ TABLE_NAMES.clear();
+ hasTestStarted = false;
+ }
+
+ @Test
+ public void testIndexWithNullableFixedWithCols() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ String tableName = "TBL_" + generateUniqueName();
+ String indexName = "IND_" + generateUniqueName();
+ String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
+ String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ conn.setAutoCommit(false);
+ String ddl =
+ "CREATE TABLE " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + tableDDLOptions;
+ Statement stmt = conn.createStatement();
+ stmt.execute(ddl);
+ BaseTest.populateTestTable(fullTableName);
+ ddl = "CREATE " + (localIndex ? "LOCAL" : "") + (uncovered ? "UNCOVERED" : "")
+ + " INDEX " + indexName + " ON " + fullTableName
+ + " (char_col1 ASC, int_col1 ASC)"
+ + (uncovered ? "" : " INCLUDE (long_col1, long_col2)");
+ stmt.execute(ddl);
+ TABLE_NAMES.add(fullTableName);
+ TABLE_NAMES.add(fullIndexName);
+
+ String query = "SELECT d.char_col1, int_col1 from " + fullTableName + " as d";
+
+ ExplainPlan plan = conn.prepareStatement(query)
+ .unwrap(PhoenixPreparedStatement.class).optimizeQuery()
+ .getExplainPlan();
+ ExplainPlanAttributes explainPlanAttributes =
+ plan.getPlanStepsAsAttributes();
+ assertEquals("PARALLEL 1-WAY",
+ explainPlanAttributes.getIteratorTypeAndScanSize());
+ if (!uncovered) {
+ // Optimizer would not select the uncovered index for this query
+ assertEquals(columnEncoded ? "SERVER FILTER BY FIRST KEY ONLY" :
+ "SERVER FILTER BY EMPTY COLUMN ONLY",
+ explainPlanAttributes.getServerWhereFilter());
+ }
+
+ if (localIndex) {
+ assertEquals(fullTableName, explainPlanAttributes.getTableName());
+ assertEquals(" [1]", explainPlanAttributes.getKeyRanges());
+ assertEquals("CLIENT MERGE SORT",
+ explainPlanAttributes.getClientSortAlgo());
+ assertEquals("RANGE SCAN ",
+ explainPlanAttributes.getExplainScanType());
+ } else if (!uncovered) {
+ assertEquals(fullIndexName, explainPlanAttributes.getTableName());
+ assertNull(explainPlanAttributes.getClientSortAlgo());
+ assertEquals("FULL SCAN ",
+ explainPlanAttributes.getExplainScanType());
+ }
+
+ ResultSet rs = conn.createStatement().executeQuery(query);
+ assertTrue(rs.next());
+ moveRegionsOfTable(fullTableName);
+ moveRegionsOfTable(fullIndexName);
+ assertEquals("chara", rs.getString(1));
+ assertEquals("chara", rs.getString("char_col1"));
+ assertEquals(2, rs.getInt(2));
+ assertTrue(rs.next());
+ moveRegionsOfTable(fullTableName);
+ moveRegionsOfTable(fullIndexName);
+ assertEquals("chara", rs.getString(1));
+ assertEquals(3, rs.getInt(2));
+ assertTrue(rs.next());
+ moveRegionsOfTable(fullTableName);
+ moveRegionsOfTable(fullIndexName);
+ assertEquals("chara", rs.getString(1));
+ assertEquals(4, rs.getInt(2));
+ assertFalse(rs.next());
+
+ conn.createStatement().execute("DROP INDEX " + indexName + " ON " + fullTableName);
+
+ query = "SELECT char_col1, int_col1 from " + fullTableName;
+ rs = conn.createStatement().executeQuery(query);
+ assertTrue(rs.next());
+
+ query = "SELECT char_col1, int_col1 from " + fullIndexName;
+ try {
+ rs = conn.createStatement().executeQuery(query);
+ fail();
+ } catch (SQLException e) {
+ assertEquals(SQLExceptionCode.TABLE_UNDEFINED.getErrorCode(), e.getErrorCode());
+ }
+ }
+ }
+
+ @Test
+ public void testDeleteFromAllPKColumnIndex() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ String tableName = "TBL_" + generateUniqueName();
+ String indexName = "IND_" + generateUniqueName();
+ String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
+ String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ conn.setAutoCommit(false);
+ String ddl =
+ "CREATE TABLE " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + tableDDLOptions;
+ Statement stmt = conn.createStatement();
+ stmt.execute(ddl);
+ BaseTest.populateTestTable(fullTableName);
+ ddl = "CREATE " + (localIndex ? "LOCAL" : "") + (uncovered ? "UNCOVERED" : "")
+ + " INDEX " + indexName + " ON " + fullTableName
+ + " (long_pk, varchar_pk)"
+ + (uncovered ? "" : " INCLUDE (long_col1, long_col2)");
+ stmt.execute(ddl);
+ TABLE_NAMES.add(fullTableName);
+ TABLE_NAMES.add(fullIndexName);
+
+ ResultSet rs;
+
+ rs = conn.createStatement()
+ .executeQuery("SELECT /*+ NO_INDEX */ COUNT(*) FROM " + fullTableName);
+ assertTrue(rs.next());
+ assertEquals(3, rs.getInt(1));
+ rs = conn.createStatement().executeQuery("SELECT COUNT(*) FROM " + fullIndexName);
+ assertTrue(rs.next());
+ assertEquals(3, rs.getInt(1));
+
+ String dml = "DELETE from " + fullTableName + " WHERE long_col2 = 4";
+ assertEquals(1, conn.createStatement().executeUpdate(dml));
+ assertNoClientSideIndexMutations(conn);
+ conn.commit();
+
+ String query = "SELECT /*+ NO_INDEX */ long_pk FROM " + fullTableName;
+ rs = conn.createStatement().executeQuery(query);
+ assertTrue(rs.next());
+ moveRegionsOfTable(fullTableName);
+ moveRegionsOfTable(fullIndexName);
+ assertEquals(1L, rs.getLong(1));
+ assertTrue(rs.next());
+ assertEquals(3L, rs.getLong(1));
+ assertFalse(rs.next());
+
+ query = "SELECT long_pk FROM " + fullTableName;
+ rs = conn.createStatement().executeQuery(query);
+ assertTrue(rs.next());
+ moveRegionsOfTable(fullTableName);
+ moveRegionsOfTable(fullIndexName);
+ assertEquals(1L, rs.getLong(1));
+ assertTrue(rs.next());
+ assertEquals(3L, rs.getLong(1));
+ assertFalse(rs.next());
+
+ query = "SELECT * FROM " + fullIndexName;
+ rs = conn.createStatement().executeQuery(query);
+ assertTrue(rs.next());
+ moveRegionsOfTable(fullTableName);
+ moveRegionsOfTable(fullIndexName);
+ assertEquals(1L, rs.getLong(1));
+ assertTrue(rs.next());
+ assertEquals(3L, rs.getLong(1));
+ assertFalse(rs.next());
+ TABLE_NAMES.clear();
+
+ conn.createStatement().execute("DROP INDEX " + indexName + " ON " + fullTableName);
+ }
+ }
+
+ private void assertNoClientSideIndexMutations(Connection conn) throws SQLException {
+ Iterator<Pair<byte[], List<Cell>>> iterator =
+ PhoenixRuntime.getUncommittedDataIterator(conn);
+ if (iterator.hasNext()) {
+ byte[] tableName = iterator.next().getFirst(); // skip data table mutations
+ PTable table = PhoenixRuntime.getTable(conn, Bytes.toString(tableName));
+ boolean clientSideUpdate = (!localIndex || (transactional &&
+ table.getTransactionProvider().getTransactionProvider()
+ .isUnsupported(Feature.MAINTAIN_LOCAL_INDEX_ON_SERVER)))
+ && (!mutable || transactional);
+ if (!clientSideUpdate) {
+ assertTrue(table.getType() == PTableType.TABLE); // should be data table
+ }
+ boolean hasIndexData = iterator.hasNext();
+ // global immutable and global transactional tables are processed client side
+ assertEquals(clientSideUpdate, hasIndexData);
+ }
+ }
+
+ @Test
+ public void testCreateIndexAfterUpsertStarted() throws Exception {
+ testCreateIndexAfterUpsertStarted(transactional,
+ SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, generateUniqueName()),
+ SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, generateUniqueName()));
+ }
+
+ private void testCreateIndexAfterUpsertStarted(boolean readOwnWrites, String fullTableName,
+ String fullIndexName) throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn1 = DriverManager.getConnection(getUrl(), props)) {
+ conn1.setAutoCommit(true);
+ String ddl =
+ "CREATE TABLE " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + tableDDLOptions;
+ Statement stmt1 = conn1.createStatement();
+ stmt1.execute(ddl);
+ BaseTest.populateTestTable(fullTableName);
+ TABLE_NAMES.add(fullTableName);
+
+ ResultSet rs;
+
+ rs = conn1.createStatement().executeQuery("SELECT COUNT(*) FROM " + fullTableName);
+ assertTrue(rs.next());
+ assertEquals(3, rs.getInt(1));
+
+ try (Connection conn2 = DriverManager.getConnection(getUrl(), props)) {
+
+ String upsert = "UPSERT INTO " + fullTableName
+ + " VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
+ PreparedStatement pstmt2 = conn2.prepareStatement(upsert);
+ pstmt2.setString(1, "varchar4");
+ pstmt2.setString(2, "char4");
+ pstmt2.setInt(3, 4);
+ pstmt2.setLong(4, 4L);
+ pstmt2.setBigDecimal(5, new BigDecimal(4.0));
+ Date date = DateUtil.parseDate("2015-01-01 00:00:00");
+ pstmt2.setDate(6, date);
+ pstmt2.setString(7, "varchar_a");
+ pstmt2.setString(8, "chara");
+ pstmt2.setInt(9, 2);
+ pstmt2.setLong(10, 2L);
+ pstmt2.setBigDecimal(11, new BigDecimal(2.0));
+ pstmt2.setDate(12, date);
+ pstmt2.setString(13, "varchar_b");
+ pstmt2.setString(14, "charb");
+ pstmt2.setInt(15, 3);
+ pstmt2.setLong(16, 3L);
+ pstmt2.setBigDecimal(17, new BigDecimal(3.0));
+ pstmt2.setDate(18, date);
+ pstmt2.executeUpdate();
+
+ if (readOwnWrites) {
+ String query = "SELECT long_pk FROM " + fullTableName + " WHERE long_pk=4";
+ rs = conn2.createStatement().executeQuery(query);
+ assertTrue(rs.next());
+ assertFalse(rs.next());
+ }
+
+ String indexName = SchemaUtil.getTableNameFromFullName(fullIndexName);
+ ddl = "CREATE " + (localIndex ? "LOCAL" : "") + (uncovered ? "UNCOVERED" : "")
+ + " INDEX " + indexName + " ON " + fullTableName
+ + " (long_pk, varchar_pk)"
+ + (uncovered ? "" : " INCLUDE (long_col1, long_col2)");
+ stmt1.execute(ddl);
+
+ /*
+ * Commit upsert after index created through different connection.
+ * This forces conn2 (which doesn't know about the index yet) to update the metadata
+ * at commit time, recognize the new index, and generate the correct metadata (or index
+ * rows for immutable indexes).
+ *
+ * For transactional data, this is problematic because the index
+ * gets a timestamp *after* the commit timestamp of conn2 and thus won't be seen during
+ * the commit. Also, when the index is being built, the data hasn't yet been committed
+ * and thus won't be part of the initial index build (fixed by PHOENIX-2446).
+ */
+ conn2.commit();
+
+ stmt1 = conn1.createStatement();
+ rs = stmt1.executeQuery("SELECT COUNT(*) FROM " + fullTableName);
+ assertTrue(rs.next());
+ assertEquals(4, rs.getInt(1));
+ assertEquals(fullIndexName,
+ stmt1.unwrap(PhoenixStatement.class).getQueryPlan().getTableRef().getTable()
+ .getName().getString());
+
+ String query = "SELECT /*+ NO_INDEX */ long_pk FROM " + fullTableName;
+ rs = conn1.createStatement().executeQuery(query);
+ assertTrue(rs.next());
+ moveRegionsOfTable(fullTableName);
+ moveRegionsOfTable(fullIndexName);
+ assertEquals(1L, rs.getLong(1));
+ assertTrue(rs.next());
+ assertEquals(2L, rs.getLong(1));
+ assertTrue(rs.next());
+ assertEquals(3L, rs.getLong(1));
+ assertTrue(rs.next());
+ assertEquals(4L, rs.getLong(1));
+ assertFalse(rs.next());
+ }
+ } catch (Exception e) {
+ if (!transactional) {
+ fail("Should not fail for non-transactional tables");
+ }
+ }
+ }
+
+ @Test
+ public void testDeleteFromNonPKColumnIndex() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ String tableName = "TBL_" + generateUniqueName();
+ String indexName = "IND_" + generateUniqueName();
+ String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
+ String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
+
+ String ddl = "CREATE TABLE " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + tableDDLOptions;
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ conn.setAutoCommit(false);
+ Statement stmt = conn.createStatement();
+ stmt.execute(ddl);
+ BaseTest.populateTestTable(fullTableName);
+ ddl = "CREATE " + (localIndex ? "LOCAL" : "") + (uncovered ? "UNCOVERED" : "")
+ + " INDEX " + indexName + " ON " + fullTableName
+ + " (long_col1, long_col2)"
+ + (uncovered ? "" : " INCLUDE (decimal_col1, decimal_col2)");
+ stmt.execute(ddl);
+ TABLE_NAMES.add(fullTableName);
+ TABLE_NAMES.add(fullIndexName);
+ }
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ ResultSet rs;
+
+ rs = conn.createStatement().executeQuery("SELECT COUNT(*) FROM " + fullTableName);
+ assertTrue(rs.next());
+ assertEquals(3, rs.getInt(1));
+ rs = conn.createStatement().executeQuery("SELECT COUNT(*) FROM " + fullIndexName);
+ assertTrue(rs.next());
+ assertEquals(3, rs.getInt(1));
+
+ String dml = "DELETE from " + fullTableName + " WHERE long_col2 = 4";
+ assertEquals(1, conn.createStatement().executeUpdate(dml));
+ assertNoClientSideIndexMutations(conn);
+ conn.commit();
+
+ // query the data table
+ String query = "SELECT /*+ NO_INDEX */ long_pk FROM " + fullTableName;
+ rs = conn.createStatement().executeQuery(query);
+ assertTrue(rs.next());
+ moveRegionsOfTable(fullTableName);
+ moveRegionsOfTable(fullIndexName);
+ assertEquals(1L, rs.getLong(1));
+ assertTrue(rs.next());
+ assertEquals(3L, rs.getLong(1));
+ assertFalse(rs.next());
+
+ // query the index table
+ query = "SELECT long_pk FROM " + fullTableName + " ORDER BY long_col1";
+ rs = conn.createStatement().executeQuery(query);
+ assertTrue(rs.next());
+ moveRegionsOfTable(fullTableName);
+ moveRegionsOfTable(fullIndexName);
+ assertEquals(1L, rs.getLong(1));
+ assertTrue(rs.next());
+ assertEquals(3L, rs.getLong(1));
+ assertFalse(rs.next());
+
+ conn.createStatement().execute("DROP INDEX " + indexName + " ON " + fullTableName);
+ }
+ }
+
+ @Test
+ public void testGroupByCount() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ String tableName = "TBL_" + generateUniqueName();
+ String indexName = "IND_" + generateUniqueName();
+ String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
+ String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ conn.setAutoCommit(false);
+ String ddl =
+ "CREATE TABLE " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + tableDDLOptions;
+ Statement stmt = conn.createStatement();
+ stmt.execute(ddl);
+ BaseTest.populateTestTable(fullTableName);
+ ddl = "CREATE " + (localIndex ? "LOCAL" : "") + (uncovered ? "UNCOVERED" : "")
+ + " INDEX " + indexName + " ON " + fullTableName + " (int_col2)";
+ stmt.execute(ddl);
+ TABLE_NAMES.add(fullTableName);
+ TABLE_NAMES.add(fullIndexName);
+ ResultSet rs;
+ rs = conn.createStatement().executeQuery(
+ "SELECT int_col2, COUNT(*) FROM " + fullTableName + " GROUP BY int_col2");
+ assertTrue(rs.next());
+ assertEquals(1, rs.getInt(2));
+ }
+ }
+
+ @Test
+ public void testSelectDistinctOnTableWithSecondaryImmutableIndex() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ String tableName = "TBL_" + generateUniqueName();
+ String indexName = "IND_" + generateUniqueName();
+ String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
+ String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ conn.setAutoCommit(false);
+ String ddl =
+ "CREATE TABLE " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + tableDDLOptions;
+ Statement stmt = conn.createStatement();
+ stmt.execute(ddl);
+ BaseTest.populateTestTable(fullTableName);
+ ddl = "CREATE " + (localIndex ? "LOCAL" : "") + (uncovered ? "UNCOVERED" : "")
+ + " INDEX " + indexName + " ON " + fullTableName + " (int_col2)";
+ conn.createStatement().execute(ddl);
+ TABLE_NAMES.add(fullTableName);
+ TABLE_NAMES.add(fullIndexName);
+ ResultSet rs = conn.createStatement().executeQuery(
+ "SELECT distinct int_col2 FROM " + fullTableName + " where int_col2 > 0");
+ assertTrue(rs.next());
+ moveRegionsOfTable(fullTableName);
+ moveRegionsOfTable(fullIndexName);
+ assertEquals(3, rs.getInt(1));
+ assertTrue(rs.next());
+ assertEquals(4, rs.getInt(1));
+ assertTrue(rs.next());
+ moveRegionsOfTable(fullTableName);
+ moveRegionsOfTable(fullIndexName);
+ assertEquals(5, rs.getInt(1));
+ assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testInClauseWithIndexOnColumnOfUsignedIntType() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ String tableName = "TBL_" + generateUniqueName();
+ String indexName = "IND_" + generateUniqueName();
+ String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
+ String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ conn.setAutoCommit(false);
+ String ddl =
+ "CREATE TABLE " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + tableDDLOptions;
+ Statement stmt = conn.createStatement();
+ stmt.execute(ddl);
+ BaseTest.populateTestTable(fullTableName);
+ ddl = "CREATE " + (localIndex ? "LOCAL" : "") + (uncovered ? "UNCOVERED" : "")
+ + " INDEX " + indexName + " ON " + fullTableName + " (int_col1)";
+ stmt.execute(ddl);
+ TABLE_NAMES.add(fullTableName);
+ TABLE_NAMES.add(fullIndexName);
+ ResultSet rs = conn.createStatement().executeQuery(
+ "SELECT int_col1 FROM " + fullTableName + " where int_col1 IN (1, 2, 3, 4)");
+ assertTrue(rs.next());
+ moveRegionsOfTable(fullTableName);
+ moveRegionsOfTable(fullIndexName);
+ assertEquals(2, rs.getInt(1));
+ assertTrue(rs.next());
+ moveRegionsOfTable(fullTableName);
+ moveRegionsOfTable(fullIndexName);
+ assertEquals(3, rs.getInt(1));
+ assertTrue(rs.next());
+ assertEquals(4, rs.getInt(1));
+ assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void createIndexOnTableWithSpecifiedDefaultCF() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ String tableName = "TBL_" + generateUniqueName();
+ String indexName = "IND_" + generateUniqueName();
+ String fullTableName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, tableName);
+ String fullIndexName = SchemaUtil.getTableName(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ conn.setAutoCommit(false);
+ String query;
+ ResultSet rs;
+ String ddl = "CREATE TABLE " + fullTableName
+ +
+ " (k VARCHAR NOT NULL PRIMARY KEY, v1 VARCHAR, v2 VARCHAR) DEFAULT_COLUMN_FAMILY='A'" +
+ (!tableDDLOptions.isEmpty() ? "," + tableDDLOptions : "");
+ Statement stmt = conn.createStatement();
+ stmt.execute(ddl);
+ TABLE_NAMES.add(fullTableName);
+
+ query = "SELECT * FROM " + fullTableName;
+ rs = conn.createStatement().executeQuery(query);
+ assertFalse(rs.next());
+
+ String options = localIndex ?
+ "SALT_BUCKETS=10, MULTI_TENANT=true, IMMUTABLE_ROWS=true, DISABLE_WAL=true" :
+ "";
+ conn.createStatement().execute(
+ "CREATE " + (uncovered ? "UNCOVERED" : "") + " INDEX " + indexName
+ + " ON " + fullTableName + " (v1)"
+ + (uncovered ? " " : "INCLUDE (v2) ") + options);
+ query = "SELECT * FROM " + fullIndexName;
+ rs = conn.createStatement().executeQuery(query);
+ assertFalse(rs.next());
+
+ //check options set correctly on index
+ TableName indexTableName = TableName.create(TestUtil.DEFAULT_SCHEMA_NAME, indexName);
+ TABLE_NAMES.add(fullIndexName);
+ NamedTableNode indexNode = NamedTableNode.create(null, indexTableName, null);
+ ColumnResolver resolver =
+ FromCompiler.getResolver(indexNode, conn.unwrap(PhoenixConnection.class));
+ PTable indexTable = resolver.getTables().get(0).getTable();
+ // Can't set IMMUTABLE_ROWS, MULTI_TENANT or DEFAULT_COLUMN_FAMILY_NAME on an index
+ assertNull(indexTable.getDefaultFamilyName());
+ assertFalse(indexTable.isMultiTenant());
+ assertEquals(mutable, !indexTable.isImmutableRows()); // Should match table
+ if (localIndex) {
+ assertEquals(10, indexTable.getBucketNum().intValue());
+ assertTrue(indexTable.isWALDisabled());
+ }
+ }
+ }
+
+ @Test
+ public void testIndexWithNullableDateCol() throws Exception {
+ String tableName = "TBL_" + generateUniqueName();
+ String indexName = "IND_" + generateUniqueName();
... 6837 lines suppressed ...