You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ja...@apache.org on 2018/11/10 20:59:04 UTC
[01/22] phoenix git commit: PHOENIX-4977 Make
KillServerOnFailurePolicy a configurable option in PhoenixIndexFailurePolicy
Repository: phoenix
Updated Branches:
refs/heads/omid2 0f2b1b8ba -> d03eadc23
PHOENIX-4977 Make KillServerOnFailurePolicy a configurable option in PhoenixIndexFailurePolicy
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/a48e42e7
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/a48e42e7
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/a48e42e7
Branch: refs/heads/omid2
Commit: a48e42e7f639c8194db84155ce73ca9d4df25d91
Parents: 2ff316a
Author: Vincent Poon <vi...@apache.org>
Authored: Tue Oct 23 16:03:22 2018 -0700
Committer: Vincent Poon <vi...@apache.org>
Committed: Thu Oct 25 12:06:31 2018 -0700
----------------------------------------------------------------------
.../phoenix/hbase/index/write/DelegateIndexFailurePolicy.java | 5 ++++-
.../org/apache/phoenix/index/PhoenixIndexFailurePolicy.java | 6 ++++++
.../src/main/java/org/apache/phoenix/query/QueryServices.java | 1 +
3 files changed, 11 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a48e42e7/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/DelegateIndexFailurePolicy.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/DelegateIndexFailurePolicy.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/DelegateIndexFailurePolicy.java
index a7fb7ec..caf2b38 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/DelegateIndexFailurePolicy.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/write/DelegateIndexFailurePolicy.java
@@ -28,7 +28,7 @@ import com.google.common.collect.Multimap;
public class DelegateIndexFailurePolicy implements IndexFailurePolicy {
- private final IndexFailurePolicy delegate;
+ private IndexFailurePolicy delegate;
public DelegateIndexFailurePolicy(IndexFailurePolicy delegate) {
this.delegate = delegate;
@@ -55,4 +55,7 @@ public class DelegateIndexFailurePolicy implements IndexFailurePolicy {
delegate.stop(arg0);
}
+ public void setDelegate(IndexFailurePolicy delegate) {
+ this.delegate = delegate;
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a48e42e7/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexFailurePolicy.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexFailurePolicy.java b/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexFailurePolicy.java
index 4c2013b..f0af6f6 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexFailurePolicy.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/index/PhoenixIndexFailurePolicy.java
@@ -60,6 +60,7 @@ import org.apache.phoenix.hbase.index.table.HTableInterfaceReference;
import org.apache.phoenix.hbase.index.util.ImmutableBytesPtr;
import org.apache.phoenix.hbase.index.write.DelegateIndexFailurePolicy;
import org.apache.phoenix.hbase.index.write.KillServerOnFailurePolicy;
+import org.apache.phoenix.hbase.index.write.LeaveIndexActiveFailurePolicy;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.query.QueryServices;
@@ -133,6 +134,11 @@ public class PhoenixIndexFailurePolicy extends DelegateIndexFailurePolicy {
} else {
throwIndexWriteFailure = Boolean.parseBoolean(value);
}
+
+ boolean killServer = env.getConfiguration().getBoolean(QueryServices.INDEX_FAILURE_KILL_SERVER, true);
+ if (!killServer) {
+ setDelegate(new LeaveIndexActiveFailurePolicy());
+ } // else, default in constructor is KillServerOnFailurePolicy
}
/**
http://git-wip-us.apache.org/repos/asf/phoenix/blob/a48e42e7/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index d3c8dc0..337bb05 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -154,6 +154,7 @@ public interface QueryServices extends SQLCloseable {
public static final String INDEX_FAILURE_BLOCK_WRITE = "phoenix.index.failure.block.write";
public static final String INDEX_FAILURE_DISABLE_INDEX = "phoenix.index.failure.disable.index";
public static final String INDEX_FAILURE_THROW_EXCEPTION_ATTRIB = "phoenix.index.failure.throw.exception";
+ public static final String INDEX_FAILURE_KILL_SERVER = "phoenix.index.failure.unhandled.killserver";
// Index will be partially re-built from index disable time stamp - following overlap time
@Deprecated
[06/22] phoenix git commit: PHOENIX-4764 Cleanup metadata of child
views for a base table that has been dropped (addendum)
Posted by ja...@apache.org.
PHOENIX-4764 Cleanup metadata of child views for a base table that has been dropped (addendum)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/c936d815
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/c936d815
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/c936d815
Branch: refs/heads/omid2
Commit: c936d815119b73dc927a7b3bb6a2b577b4a2f91c
Parents: 6478879
Author: Kadir <ko...@salesforce.com>
Authored: Tue Oct 30 17:47:26 2018 -0700
Committer: Karan Mehta <ka...@gmail.com>
Committed: Tue Oct 30 18:33:10 2018 -0700
----------------------------------------------------------------------
.../end2end/MigrateSystemTablesToSystemNamespaceIT.java | 4 ++--
.../phoenix/end2end/SystemCatalogCreationOnConnectionIT.java | 4 ++--
.../org/apache/phoenix/monitoring/BasePhoenixMetricsIT.java | 4 +++-
.../org/apache/phoenix/coprocessor/TaskRegionObserver.java | 7 +++++--
.../org/apache/phoenix/query/ConnectionQueryServicesImpl.java | 2 +-
.../src/main/java/org/apache/phoenix/query/QueryServices.java | 2 ++
.../java/org/apache/phoenix/query/QueryServicesOptions.java | 3 ++-
7 files changed, 17 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c936d815/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
index ffac4d6..b6f061e 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java
@@ -62,10 +62,10 @@ public class MigrateSystemTablesToSystemNamespaceIT extends BaseTest {
private static final Set<String> PHOENIX_SYSTEM_TABLES = new HashSet<>(Arrays.asList(
"SYSTEM.CATALOG", "SYSTEM.SEQUENCE", "SYSTEM.STATS", "SYSTEM.FUNCTION",
- "SYSTEM.MUTEX","SYSTEM.LOG", "SYSTEM.CHILD_LINK"));
+ "SYSTEM.MUTEX","SYSTEM.LOG", "SYSTEM.CHILD_LINK", "SYSTEM.TASK"));
private static final Set<String> PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES = new HashSet<>(
Arrays.asList("SYSTEM:CATALOG", "SYSTEM:SEQUENCE", "SYSTEM:STATS", "SYSTEM:FUNCTION",
- "SYSTEM:MUTEX","SYSTEM:LOG", "SYSTEM:CHILD_LINK"));
+ "SYSTEM:MUTEX","SYSTEM:LOG", "SYSTEM:CHILD_LINK", "SYSTEM:TASK"));
private static final String SCHEMA_NAME = "MIGRATETEST";
private static final String TABLE_NAME =
SCHEMA_NAME + "." + MigrateSystemTablesToSystemNamespaceIT.class.getSimpleName().toUpperCase();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c936d815/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemCatalogCreationOnConnectionIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemCatalogCreationOnConnectionIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemCatalogCreationOnConnectionIT.java
index aa2d971..a1685c44 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemCatalogCreationOnConnectionIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemCatalogCreationOnConnectionIT.java
@@ -72,11 +72,11 @@ public class SystemCatalogCreationOnConnectionIT {
private static final Set<String> PHOENIX_SYSTEM_TABLES = new HashSet<>(Arrays.asList(
"SYSTEM.CATALOG", "SYSTEM.SEQUENCE", "SYSTEM.STATS", "SYSTEM.FUNCTION",
- "SYSTEM.MUTEX", "SYSTEM.LOG", "SYSTEM.CHILD_LINK"));
+ "SYSTEM.MUTEX", "SYSTEM.LOG", "SYSTEM.CHILD_LINK", "SYSTEM.TASK"));
private static final Set<String> PHOENIX_NAMESPACE_MAPPED_SYSTEM_TABLES = new HashSet<>(
Arrays.asList("SYSTEM:CATALOG", "SYSTEM:SEQUENCE", "SYSTEM:STATS", "SYSTEM:FUNCTION",
- "SYSTEM:MUTEX", "SYSTEM:LOG", "SYSTEM:CHILD_LINK"));
+ "SYSTEM:MUTEX", "SYSTEM:LOG", "SYSTEM:CHILD_LINK", "SYSTEM:TASK"));
private static class PhoenixSysCatCreationServices extends ConnectionQueryServicesImpl {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c936d815/phoenix-core/src/it/java/org/apache/phoenix/monitoring/BasePhoenixMetricsIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/monitoring/BasePhoenixMetricsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/monitoring/BasePhoenixMetricsIT.java
index 5c016f6..9c2b763 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/monitoring/BasePhoenixMetricsIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/monitoring/BasePhoenixMetricsIT.java
@@ -19,6 +19,7 @@ package org.apache.phoenix.monitoring;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
+import org.apache.phoenix.coprocessor.TaskRegionObserver;
import org.apache.phoenix.end2end.BaseUniqueNamesOwnClusterIT;
import org.apache.phoenix.jdbc.PhoenixDriver;
import org.apache.phoenix.query.QueryServices;
@@ -53,7 +54,8 @@ public class BasePhoenixMetricsIT extends BaseUniqueNamesOwnClusterIT {
@BeforeClass
public static void doSetup() throws Exception {
- Map<String, String> props = Maps.newHashMapWithExpectedSize(1);
+ Map<String, String> props = Maps.newHashMapWithExpectedSize(3);
+ props.put(QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB, Long.toString(Long.MAX_VALUE));
// Phoenix Global client metrics are enabled by default
// Enable request metric collection at the driver level
props.put(QueryServices.COLLECT_REQUEST_LEVEL_METRICS, String.valueOf(true));
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c936d815/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/TaskRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/TaskRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/TaskRegionObserver.java
index ca71961..87f7414 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/TaskRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/TaskRegionObserver.java
@@ -68,8 +68,7 @@ public class TaskRegionObserver extends BaseRegionObserver {
private long timeInterval = QueryServicesOptions.DEFAULT_TASK_HANDLING_INTERVAL_MS;
private long timeMaxInterval = QueryServicesOptions.DEFAULT_TASK_HANDLING_MAX_INTERVAL_MS;
@GuardedBy("TaskRegionObserver.class")
- // initial delay before the first task is handled
- private static final long initialDelay = 10000; // 10 secs
+ private long initialDelay;
@Override
public void preClose(final ObserverContext<RegionCoprocessorEnvironment> c,
@@ -88,6 +87,10 @@ public class TaskRegionObserver extends BaseRegionObserver {
config.getLong(
QueryServices.TASK_HANDLING_MAX_INTERVAL_MS_ATTRIB,
QueryServicesOptions.DEFAULT_TASK_HANDLING_MAX_INTERVAL_MS);
+ initialDelay =
+ config.getLong(
+ QueryServices.TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB,
+ QueryServicesOptions.DEFAULT_TASK_HANDLING_INITIAL_DELAY_MS);
}
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c936d815/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index 1381247..8c09833 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -1177,7 +1177,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
splits, isNamespaceMapped);
if (!tableExist) {
- if (isMetaTable && !isUpgradeRequired() && (!isAutoUpgradeEnabled || isDoNotUpgradePropSet)) {
+ if (SchemaUtil.isSystemTable(physicalTableName) && !isUpgradeRequired() && (!isAutoUpgradeEnabled || isDoNotUpgradePropSet)) {
// Disallow creating the SYSTEM.CATALOG or SYSTEM:CATALOG HBase table
throw new UpgradeRequiredException();
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c936d815/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index d06f07c..a8bbc22 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -325,6 +325,8 @@ public interface QueryServices extends SQLCloseable {
public static final String TASK_HANDLING_INTERVAL_MS_ATTRIB = "phoenix.task.handling.interval.ms";
// The maximum time for a task to stay in table SYSTEM.TASK
public static final String TASK_HANDLING_MAX_INTERVAL_MS_ATTRIB = "phoenix.task.handling.maxInterval.ms";
+ // The initial delay before the first task from table SYSTEM.TASK is handled
+ public static final String TASK_HANDLING_INITIAL_DELAY_MS_ATTRIB = "phoenix.task.handling.initial.delay.ms";
/**
* Get executor service used for parallel scans
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c936d815/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index a6a73f7..d2f5bce 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -342,7 +342,8 @@ public class QueryServicesOptions {
// default system task handling interval in milliseconds
public static final long DEFAULT_TASK_HANDLING_INTERVAL_MS = 60*1000; // 1 min
- public static final long DEFAULT_TASK_HANDLING_MAX_INTERVAL_MS = 30*60*1000; // 30 minutes
+ public static final long DEFAULT_TASK_HANDLING_MAX_INTERVAL_MS = 30*60*1000; // 30 min
+ public static final long DEFAULT_TASK_HANDLING_INITIAL_DELAY_MS = 10*1000; // 10 sec
@SuppressWarnings("serial")
public static final Set<String> DEFAULT_QUERY_SERVER_SKIP_WORDS = new HashSet<String>() {
[12/22] phoenix git commit: PHOENIX-4981 Add tests for ORDER BY,
GROUP BY and salted tables using phoenix-spark
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrderByIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrderByIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrderByIT.java
index 578a3af..792d08f 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrderByIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/OrderByIT.java
@@ -17,17 +17,7 @@
*/
package org.apache.phoenix.end2end;
-import static org.apache.phoenix.util.TestUtil.ROW1;
-import static org.apache.phoenix.util.TestUtil.ROW2;
-import static org.apache.phoenix.util.TestUtil.ROW3;
-import static org.apache.phoenix.util.TestUtil.ROW4;
-import static org.apache.phoenix.util.TestUtil.ROW5;
-import static org.apache.phoenix.util.TestUtil.ROW6;
-import static org.apache.phoenix.util.TestUtil.ROW7;
-import static org.apache.phoenix.util.TestUtil.ROW8;
-import static org.apache.phoenix.util.TestUtil.ROW9;
import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
-import static org.apache.phoenix.util.TestUtil.assertResultSet;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -40,83 +30,10 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.Properties;
-import org.apache.phoenix.jdbc.PhoenixStatement;
import org.apache.phoenix.util.PropertiesUtil;
import org.junit.Test;
-
-public class OrderByIT extends ParallelStatsDisabledIT {
-
- @Test
- public void testMultiOrderByExpr() throws Exception {
- String tenantId = getOrganizationId();
- String tableName = initATableValues(tenantId, getDefaultSplits(tenantId), getUrl());
- String query = "SELECT entity_id FROM " + tableName + " ORDER BY b_string, entity_id";
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = DriverManager.getConnection(getUrl(), props);
- try {
- PreparedStatement statement = conn.prepareStatement(query);
- ResultSet rs = statement.executeQuery();
- assertTrue (rs.next());
- assertEquals(ROW1,rs.getString(1));
- assertTrue (rs.next());
- assertEquals(ROW4,rs.getString(1));
- assertTrue (rs.next());
- assertEquals(ROW7,rs.getString(1));
- assertTrue (rs.next());
- assertEquals(ROW2,rs.getString(1));
- assertTrue (rs.next());
- assertEquals(ROW5,rs.getString(1));
- assertTrue (rs.next());
- assertEquals(ROW8,rs.getString(1));
- assertTrue (rs.next());
- assertEquals(ROW3,rs.getString(1));
- assertTrue (rs.next());
- assertEquals(ROW6,rs.getString(1));
- assertTrue (rs.next());
- assertEquals(ROW9,rs.getString(1));
-
- assertFalse(rs.next());
- } finally {
- conn.close();
- }
- }
-
-
- @Test
- public void testDescMultiOrderByExpr() throws Exception {
- String tenantId = getOrganizationId();
- String tableName = initATableValues(tenantId, getDefaultSplits(tenantId), getUrl());
- String query = "SELECT entity_id FROM " + tableName + " ORDER BY b_string || entity_id desc";
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = DriverManager.getConnection(getUrl(), props);
- try {
- PreparedStatement statement = conn.prepareStatement(query);
- ResultSet rs = statement.executeQuery();
- assertTrue (rs.next());
- assertEquals(ROW9,rs.getString(1));
- assertTrue (rs.next());
- assertEquals(ROW6,rs.getString(1));
- assertTrue (rs.next());
- assertEquals(ROW3,rs.getString(1));
- assertTrue (rs.next());
- assertEquals(ROW8,rs.getString(1));
- assertTrue (rs.next());
- assertEquals(ROW5,rs.getString(1));
- assertTrue (rs.next());
- assertEquals(ROW2,rs.getString(1));
- assertTrue (rs.next());
- assertEquals(ROW7,rs.getString(1));
- assertTrue (rs.next());
- assertEquals(ROW4,rs.getString(1));
- assertTrue (rs.next());
- assertEquals(ROW1,rs.getString(1));
-
- assertFalse(rs.next());
- } finally {
- conn.close();
- }
- }
+public class OrderByIT extends BaseOrderByIT {
@Test
public void testOrderByWithPosition() throws Exception {
@@ -151,8 +68,8 @@ public class OrderByIT extends ParallelStatsDisabledIT {
assertTrue(rs.next());
assertEquals(1,rs.getInt(1));
assertTrue(rs.next());
- assertEquals(1,rs.getInt(1));
- assertFalse(rs.next());
+ assertEquals(1,rs.getInt(1));
+ assertFalse(rs.next());
query = "select a_string x, col1 y from " + tableName + " order by x";
rs = conn.createStatement().executeQuery(query);
@@ -163,9 +80,9 @@ public class OrderByIT extends ParallelStatsDisabledIT {
assertEquals("b",rs.getString(1));
assertEquals(20,rs.getInt(2));
assertTrue(rs.next());
- assertEquals("c",rs.getString(1));
+ assertEquals("c",rs.getString(1));
assertEquals(30,rs.getInt(2));
- assertFalse(rs.next());
+ assertFalse(rs.next());
query = "select * from " + tableName + " order by 2";
rs = conn.createStatement().executeQuery(query);
@@ -173,113 +90,12 @@ public class OrderByIT extends ParallelStatsDisabledIT {
assertEquals("b",rs.getString(1));
assertEquals(20,rs.getInt(2));
assertTrue(rs.next());
- assertEquals("c",rs.getString(1));
- assertEquals(30,rs.getInt(2));
- assertTrue(rs.next());
- assertEquals("a",rs.getString(1));
- assertEquals(40,rs.getInt(2));
- assertFalse(rs.next());
- } finally {
- conn.close();
- }
- }
-
-
- @Test
- public void testColumnFamily() throws Exception {
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = DriverManager.getConnection(getUrl(), props);
- conn.setAutoCommit(false);
-
- try {
- String tableName = generateUniqueName();
- String ddl = "CREATE TABLE " + tableName +
- " (a_string varchar not null, cf1.a integer, cf1.b varchar, col1 integer, cf2.c varchar, cf2.d integer, col2 integer" +
- " CONSTRAINT pk PRIMARY KEY (a_string))\n";
- createTestTable(getUrl(), ddl);
- String dml = "UPSERT INTO " + tableName + " VALUES(?,?,?,?,?,?,?)";
- PreparedStatement stmt = conn.prepareStatement(dml);
- stmt.setString(1, "a");
- stmt.setInt(2, 40);
- stmt.setString(3, "aa");
- stmt.setInt(4, 10);
- stmt.setString(5, "bb");
- stmt.setInt(6, 20);
- stmt.setInt(7, 1);
- stmt.execute();
- stmt.setString(1, "c");
- stmt.setInt(2, 30);
- stmt.setString(3, "cc");
- stmt.setInt(4, 50);
- stmt.setString(5, "dd");
- stmt.setInt(6, 60);
- stmt.setInt(7, 3);
- stmt.execute();
- stmt.setString(1, "b");
- stmt.setInt(2, 40);
- stmt.setString(3, "bb");
- stmt.setInt(4, 5);
- stmt.setString(5, "aa");
- stmt.setInt(6, 80);
- stmt.setInt(7, 2);
- stmt.execute();
- conn.commit();
-
- String query = "select * from " + tableName + " order by 2, 5";
- ResultSet rs = conn.createStatement().executeQuery(query);
- assertTrue(rs.next());
assertEquals("c",rs.getString(1));
assertEquals(30,rs.getInt(2));
- assertEquals("cc",rs.getString(3));
- assertEquals(50,rs.getInt(4));
- assertEquals("dd",rs.getString(5));
- assertEquals(60,rs.getInt(6));
- assertEquals(3,rs.getInt(7));
- assertTrue(rs.next());
- assertEquals("b",rs.getString(1));
- assertEquals(40,rs.getInt(2));
- assertEquals("bb",rs.getString(3));
- assertEquals(5,rs.getInt(4));
- assertEquals("aa",rs.getString(5));
- assertEquals(80,rs.getInt(6));
- assertEquals(2,rs.getInt(7));
- assertTrue(rs.next());
- assertEquals("a",rs.getString(1));
- assertEquals(40,rs.getInt(2));
- assertEquals("aa",rs.getString(3));
- assertEquals(10,rs.getInt(4));
- assertEquals("bb",rs.getString(5));
- assertEquals(20,rs.getInt(6));
- assertEquals(1,rs.getInt(7));
- assertFalse(rs.next());
-
- query = "select * from " + tableName + " order by 7";
- rs = conn.createStatement().executeQuery(query);
assertTrue(rs.next());
- assertEquals("a",rs.getString(1));
- assertEquals(40,rs.getInt(2));
- assertEquals("aa",rs.getString(3));
- assertEquals(10,rs.getInt(4));
- assertEquals("bb",rs.getString(5));
- assertEquals(20,rs.getInt(6));
- assertEquals(1,rs.getInt(7));
- assertTrue(rs.next());
- assertEquals("b",rs.getString(1));
+ assertEquals("a",rs.getString(1));
assertEquals(40,rs.getInt(2));
- assertEquals("bb",rs.getString(3));
- assertEquals(5,rs.getInt(4));
- assertEquals("aa",rs.getString(5));
- assertEquals(80,rs.getInt(6));
- assertEquals(2,rs.getInt(7));
- assertTrue(rs.next());
- assertEquals("c",rs.getString(1));
- assertEquals(30,rs.getInt(2));
- assertEquals("cc",rs.getString(3));
- assertEquals(50,rs.getInt(4));
- assertEquals("dd",rs.getString(5));
- assertEquals(60,rs.getInt(6));
- assertEquals(3,rs.getInt(7));
- assertFalse(rs.next());
+ assertFalse(rs.next());
} finally {
conn.close();
}
@@ -343,19 +159,19 @@ public class OrderByIT extends ParallelStatsDisabledIT {
String query = "select t1.* from " + tableName1 + " t1 join " + tableName2 + " t2 on t1.a_string = t2.a_string order by 3";
ResultSet rs = conn.createStatement().executeQuery(query);
assertTrue(rs.next());
- assertEquals("a",rs.getString(1));
+ assertEquals("a",rs.getString(1));
assertEquals(40,rs.getInt(2));
assertEquals("aa",rs.getString(3));
assertEquals(10,rs.getInt(4));
assertEquals("bb",rs.getString(5));
assertEquals(20,rs.getInt(6));
assertTrue(rs.next());
- assertEquals("b",rs.getString(1));
+ assertEquals("b",rs.getString(1));
assertEquals(40,rs.getInt(2));
assertEquals("bb",rs.getString(3));
assertEquals(5,rs.getInt(4));
assertEquals("aa",rs.getString(5));
- assertEquals(80,rs.getInt(6));
+ assertEquals(80,rs.getInt(6));
assertTrue(rs.next());
assertEquals("c",rs.getString(1));
assertEquals(30,rs.getInt(2));
@@ -363,20 +179,20 @@ public class OrderByIT extends ParallelStatsDisabledIT {
assertEquals(50,rs.getInt(4));
assertEquals("dd",rs.getString(5));
assertEquals(60,rs.getInt(6));
- assertFalse(rs.next());
+ assertFalse(rs.next());
query = "select t1.a_string, t2.col1 from " + tableName1 + " t1 join " + tableName2 + " t2 on t1.a_string = t2.a_string order by 2";
rs = conn.createStatement().executeQuery(query);
assertTrue(rs.next());
- assertEquals("b",rs.getString(1));
+ assertEquals("b",rs.getString(1));
assertEquals(20,rs.getInt(2));
assertTrue(rs.next());
- assertEquals("c",rs.getString(1));
+ assertEquals("c",rs.getString(1));
assertEquals(30,rs.getInt(2));
assertTrue(rs.next());
- assertEquals("a",rs.getString(1));
+ assertEquals("a",rs.getString(1));
assertEquals(40,rs.getInt(2));
- assertFalse(rs.next());
+ assertFalse(rs.next());
} finally {
conn.close();
}
@@ -441,24 +257,24 @@ public class OrderByIT extends ParallelStatsDisabledIT {
String query = "select a_string, cf2.d from " + tableName1 + " union all select * from " + tableName2 + " order by 2";
ResultSet rs = conn.createStatement().executeQuery(query);
assertTrue(rs.next());
- assertEquals("bb",rs.getString(1));
+ assertEquals("bb",rs.getString(1));
assertEquals(10,rs.getInt(2));
assertTrue(rs.next());
- assertEquals("a",rs.getString(1));
- assertEquals(20,rs.getInt(2));
+ assertEquals("a",rs.getString(1));
+ assertEquals(20,rs.getInt(2));
assertTrue(rs.next());
assertEquals("cc",rs.getString(1));
assertEquals(30,rs.getInt(2));
assertTrue(rs.next());
- assertEquals("aa",rs.getString(1));
+ assertEquals("aa",rs.getString(1));
assertEquals(40,rs.getInt(2));
assertTrue(rs.next());
- assertEquals("c",rs.getString(1));
- assertEquals(60,rs.getInt(2));
+ assertEquals("c",rs.getString(1));
+ assertEquals(60,rs.getInt(2));
assertTrue(rs.next());
assertEquals("b",rs.getString(1));
assertEquals(80,rs.getInt(2));
- assertFalse(rs.next());
+ assertFalse(rs.next());
} finally {
conn.close();
}
@@ -514,7 +330,8 @@ public class OrderByIT extends ParallelStatsDisabledIT {
conn.close();
}
}
-
+
+
@Test
public void testOrderByRVC() throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
@@ -546,636 +363,102 @@ public class OrderByIT extends ParallelStatsDisabledIT {
}
@Test
- public void testAggregateOrderBy() throws Exception {
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = DriverManager.getConnection(getUrl(), props);
- String tableName = generateUniqueName();
- String ddl = "create table " + tableName + " (ID VARCHAR NOT NULL PRIMARY KEY, VAL1 VARCHAR, VAL2 INTEGER)";
- conn.createStatement().execute(ddl);
-
- conn.createStatement().execute("upsert into " + tableName + " values ('ABC','aa123', 11)");
- conn.createStatement().execute("upsert into " + tableName + " values ('ABD','ba124', 1)");
- conn.createStatement().execute("upsert into " + tableName + " values ('ABE','cf125', 13)");
- conn.createStatement().execute("upsert into " + tableName + " values ('ABF','dan126', 4)");
- conn.createStatement().execute("upsert into " + tableName + " values ('ABG','elf127', 15)");
- conn.createStatement().execute("upsert into " + tableName + " values ('ABH','fan128', 6)");
- conn.createStatement().execute("upsert into " + tableName + " values ('AAA','get211', 100)");
- conn.createStatement().execute("upsert into " + tableName + " values ('AAB','hat212', 7)");
- conn.createStatement().execute("upsert into " + tableName + " values ('AAC','aap12', 2)");
- conn.createStatement().execute("upsert into " + tableName + " values ('AAD','ball12', 3)");
- conn.createStatement().execute("upsert into " + tableName + " values ('AAE','inn2110', 13)");
- conn.createStatement().execute("upsert into " + tableName + " values ('AAF','key2112', 40)");
- conn.commit();
-
- ResultSet rs;
- PhoenixStatement stmt = conn.createStatement().unwrap(PhoenixStatement.class);
- rs = stmt.executeQuery("select distinct ID, VAL1, VAL2 from " + tableName + " where ID in ('ABC','ABD','ABE','ABF','ABG','ABH','AAA', 'AAB', 'AAC','AAD','AAE','AAF') order by VAL1");
- assertFalse(stmt.getQueryPlan().getOrderBy().getOrderByExpressions().isEmpty());
- assertTrue(rs.next());
- assertEquals("ABC", rs.getString(1));
- assertEquals("aa123", rs.getString(2));
- assertTrue(rs.next());
- assertEquals("aap12", rs.getString(2));
- assertTrue(rs.next());
- assertEquals("ba124", rs.getString(2));
- assertTrue(rs.next());
- assertEquals("ball12", rs.getString(2));
- assertTrue(rs.next());
- assertEquals("cf125", rs.getString(2));
- assertTrue(rs.next());
- assertEquals("dan126", rs.getString(2));
- assertTrue(rs.next());
- assertEquals("elf127", rs.getString(2));
- assertTrue(rs.next());
- assertEquals("fan128", rs.getString(2));
- assertTrue(rs.next());
- assertEquals("get211", rs.getString(2));
- assertTrue(rs.next());
- assertEquals("hat212", rs.getString(2));
- assertTrue(rs.next());
- assertEquals("inn2110", rs.getString(2));
- assertTrue(rs.next());
- assertEquals("AAF", rs.getString(1));
- assertEquals("key2112", rs.getString(2));
- assertFalse(rs.next());
- }
-
- @Test
- public void testAggregateOptimizedOutOrderBy() throws Exception {
+ public void testColumnFamily() throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
Connection conn = DriverManager.getConnection(getUrl(), props);
- String tableName = generateUniqueName();
- String ddl = "create table " + tableName + " (K1 VARCHAR NOT NULL, K2 VARCHAR NOT NULL, VAL1 VARCHAR, VAL2 INTEGER, CONSTRAINT pk PRIMARY KEY(K1,K2))";
- conn.createStatement().execute(ddl);
-
- conn.createStatement().execute("upsert into " + tableName + " values ('ABC','ABC','aa123', 11)");
- conn.createStatement().execute("upsert into " + tableName + " values ('ABD','ABC','ba124', 1)");
- conn.createStatement().execute("upsert into " + tableName + " values ('ABE','ABC','cf125', 13)");
- conn.createStatement().execute("upsert into " + tableName + " values ('ABF','ABC','dan126', 4)");
- conn.createStatement().execute("upsert into " + tableName + " values ('ABG','ABC','elf127', 15)");
- conn.createStatement().execute("upsert into " + tableName + " values ('ABH','ABC','fan128', 6)");
- conn.createStatement().execute("upsert into " + tableName + " values ('AAA','ABC','get211', 100)");
- conn.createStatement().execute("upsert into " + tableName + " values ('AAB','ABC','hat212', 7)");
- conn.createStatement().execute("upsert into " + tableName + " values ('AAC','ABC','aap12', 2)");
- conn.createStatement().execute("upsert into " + tableName + " values ('AAD','ABC','ball12', 3)");
- conn.createStatement().execute("upsert into " + tableName + " values ('AAE','ABC','inn2110', 13)");
- conn.createStatement().execute("upsert into " + tableName + " values ('AAF','ABC','key2112', 40)");
- conn.commit();
-
- ResultSet rs;
- PhoenixStatement stmt = conn.createStatement().unwrap(PhoenixStatement.class);
- rs = stmt.executeQuery("select distinct K2, VAL1, VAL2 from " + tableName + " where K2 = 'ABC' order by VAL1");
- assertTrue(stmt.getQueryPlan().getOrderBy().getOrderByExpressions().isEmpty());
- assertTrue(rs.next());
- assertEquals("ABC", rs.getString(1));
- assertEquals("aa123", rs.getString(2));
- assertTrue(rs.next());
- assertEquals("aap12", rs.getString(2));
- assertTrue(rs.next());
- assertEquals("ba124", rs.getString(2));
- assertTrue(rs.next());
- assertEquals("ball12", rs.getString(2));
- assertTrue(rs.next());
- assertEquals("cf125", rs.getString(2));
- assertTrue(rs.next());
- assertEquals("dan126", rs.getString(2));
- assertTrue(rs.next());
- assertEquals("elf127", rs.getString(2));
- assertTrue(rs.next());
- assertEquals("fan128", rs.getString(2));
- assertTrue(rs.next());
- assertEquals("get211", rs.getString(2));
- assertTrue(rs.next());
- assertEquals("hat212", rs.getString(2));
- assertTrue(rs.next());
- assertEquals("inn2110", rs.getString(2));
- assertTrue(rs.next());
- assertEquals("ABC", rs.getString(1));
- assertEquals("key2112", rs.getString(2));
- assertFalse(rs.next());
- }
-
- @Test
- public void testNullsLastWithDesc() throws Exception {
- Connection conn=null;
- try {
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- conn = DriverManager.getConnection(getUrl(), props);
-
- String tableName=generateUniqueName();
- String sql="CREATE TABLE "+tableName+" ( "+
- "ORGANIZATION_ID VARCHAR,"+
- "CONTAINER_ID VARCHAR,"+
- "ENTITY_ID VARCHAR NOT NULL,"+
- "CONSTRAINT TEST_PK PRIMARY KEY ( "+
- "ORGANIZATION_ID DESC,"+
- "CONTAINER_ID DESC,"+
- "ENTITY_ID"+
- "))";
- conn.createStatement().execute(sql);
-
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('a',null,'11')");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,'2','22')");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('c','3','33')");
- conn.commit();
-
- //-----ORGANIZATION_ID
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID ASC NULLS FIRST";
- ResultSet rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"2",null},{null,"a"},{"3","c"},});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID ASC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{"2",null}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,"a"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{"2",null}});
-
- //----CONTAINER_ID
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID ASC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,"a"},{"2",null},{"3","c"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID ASC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,"a"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{"2",null}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,"a"}});
-
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,null,'44')");
- conn.commit();
-
- //-----ORGANIZATION_ID ASC CONTAINER_ID ASC
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS FIRST,CONTAINER_ID NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,null},{"2",null},{null,"a"},{"3","c"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS FIRST,CONTAINER_ID NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"2",null},{null,null},{null,"a"},{"3","c"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS LAST,CONTAINER_ID NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{null,null},{"2",null}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS LAST,CONTAINER_ID NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{"2",null},{null,null}});
-
-
- //-----ORGANIZATION_ID ASC CONTAINER_ID DESC
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS FIRST,CONTAINER_ID DESC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,null},{"2",null},{null,"a"},{"3","c"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS FIRST,CONTAINER_ID DESC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"2",null},{null,null},{null,"a"},{"3","c"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS LAST,CONTAINER_ID DESC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{null,null},{"2",null}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS LAST,CONTAINER_ID DESC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{"2",null},{null,null}});
-
- //-----ORGANIZATION_ID DESC CONTAINER_ID ASC
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,null},{"2",null},{"3","c"},{null,"a"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"2",null},{null,null},{"3","c"},{null,"a"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{null,null},{"2",null}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{"2",null},{null,null}});
-
- //-----ORGANIZATION_ID DESC CONTAINER_ID DESC
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID DESC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,null},{"2",null},{"3","c"},{null,"a"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID DESC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"2",null},{null,null},{"3","c"},{null,"a"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID DESC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{null,null},{"2",null}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID DESC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{"2",null},{null,null}});
-
- //-----CONTAINER_ID ASC ORGANIZATION_ID ASC
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID NULLS FIRST,ORGANIZATION_ID NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,null},{null,"a"},{"2",null},{"3","c"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID NULLS FIRST,ORGANIZATION_ID NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,"a"},{null,null},{"2",null},{"3","c"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID NULLS LAST,ORGANIZATION_ID NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,null},{null,"a"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID NULLS LAST,ORGANIZATION_ID NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,"a"},{null,null}});
-
- //-----CONTAINER_ID ASC ORGANIZATION_ID DESC
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID ASC NULLS FIRST,ORGANIZATION_ID DESC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,null},{null,"a"},{"2",null},{"3","c"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID ASC NULLS FIRST,ORGANIZATION_ID DESC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,"a"},{null,null},{"2",null},{"3","c"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID ASC NULLS LAST,ORGANIZATION_ID DESC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,null},{null,"a"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID ASC NULLS LAST,ORGANIZATION_ID DESC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,"a"},{null,null}});
-
- //-----CONTAINER_ID DESC ORGANIZATION_ID ASC
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID ASC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,null},{null,"a"},{"3","c"},{"2",null}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID ASC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,"a"},{null,null},{"3","c"},{"2",null}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID ASC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,null},{null,"a"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID ASC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,"a"},{null,null}});
-
- //-----CONTAINER_ID DESC ORGANIZATION_ID DESC
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID DESC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,null},{null,"a"},{"3","c"},{"2",null}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID DESC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,"a"},{null,null},{"3","c"},{"2",null}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID DESC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,null},{null,"a"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID DESC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,"a"},{null,null}});
- } finally {
- if(conn!=null) {
- conn.close();
- }
- }
- }
-
- @Test
- public void testOrderByReverseOptimization() throws Exception {
- for(boolean salted: new boolean[]{true,false}) {
- doTestOrderByReverseOptimization(salted,true,true,true);
- doTestOrderByReverseOptimization(salted,true,true,false);
- doTestOrderByReverseOptimization(salted,true,false,true);
- doTestOrderByReverseOptimization(salted,true,false,false);
- doTestOrderByReverseOptimization(salted,false,true,true);
- doTestOrderByReverseOptimization(salted,false,true,false);
- doTestOrderByReverseOptimization(salted,false,false,true);
- doTestOrderByReverseOptimization(salted,false,false,false);
- }
- }
-
- private void doTestOrderByReverseOptimization(boolean salted,boolean desc1,boolean desc2,boolean desc3) throws Exception {
- Connection conn = null;
- try {
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- conn = DriverManager.getConnection(getUrl(), props);
- String tableName=generateUniqueName();
- String sql="CREATE TABLE "+tableName+" ( "+
- "ORGANIZATION_ID INTEGER NOT NULL,"+
- "CONTAINER_ID INTEGER NOT NULL,"+
- "SCORE INTEGER NOT NULL,"+
- "ENTITY_ID INTEGER NOT NULL,"+
- "CONSTRAINT TEST_PK PRIMARY KEY ( "+
- "ORGANIZATION_ID" +(desc1 ? " DESC" : "" )+","+
- "CONTAINER_ID"+(desc2 ? " DESC" : "" )+","+
- "SCORE"+(desc3 ? " DESC" : "" )+","+
- "ENTITY_ID"+
- ")) "+(salted ? "SALT_BUCKETS =4" : "split on(4)");
- conn.createStatement().execute(sql);
-
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,1,1,1)");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (2,2,2,2)");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (3,3,3,3)");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (4,4,4,4)");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (5,5,5,5)");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (6,6,6,6)");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,1,1,11)");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (2,2,2,22)");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (3,3,3,33)");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (4,4,4,44)");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (5,5,5,55)");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (6,6,6,66)");
- conn.commit();
-
- //groupBy orderPreserving orderBy asc asc
- sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC, CONTAINER_ID ASC";
- ResultSet rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{1,1},{2,2},{3,3},{4,4},{5,5},{6,6}});
-
- //groupBy orderPreserving orderBy asc desc
- sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC, CONTAINER_ID desc";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{1,1},{2,2},{3,3},{4,4},{5,5},{6,6}});
-
- //groupBy orderPreserving orderBy desc asc
- sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC, CONTAINER_ID DESC";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{6,6},{5,5},{4,4},{3,3},{2,2},{1,1}});
-
- //groupBy orderPreserving orderBy desc desc
- sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC, CONTAINER_ID DESC";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{6,6},{5,5},{4,4},{3,3},{2,2},{1,1}});
-
- //groupBy not orderPreserving orderBy asc asc
- sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC, SCORE ASC";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{1,1},{2,2},{3,3},{4,4},{5,5},{6,6}});
-
- //groupBy not orderPreserving orderBy asc desc
- sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC, SCORE ASC";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{1,1},{2,2},{3,3},{4,4},{5,5},{6,6}});
-
- //groupBy not orderPreserving orderBy desc asc
- sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC, SCORE DESC";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{6,6},{5,5},{4,4},{3,3},{2,2},{1,1}});
-
- //groupBy not orderPreserving orderBy desc desc
- sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC, SCORE DESC";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{6,6},{5,5},{4,4},{3,3},{2,2},{1,1}});
- } finally {
- if(conn!=null) {
- conn.close();
- }
- }
- }
-
- @Test
- public void testOrderByReverseOptimizationWithNullsLast() throws Exception{
- for(boolean salted: new boolean[]{true,false}) {
- doTestOrderByReverseOptimizationWithNullsLast(salted,true,true,true);
- doTestOrderByReverseOptimizationWithNullsLast(salted,true,true,false);
- doTestOrderByReverseOptimizationWithNullsLast(salted,true,false,true);
- doTestOrderByReverseOptimizationWithNullsLast(salted,true,false,false);
- doTestOrderByReverseOptimizationWithNullsLast(salted,false,true,true);
- doTestOrderByReverseOptimizationWithNullsLast(salted,false,true,false);
- doTestOrderByReverseOptimizationWithNullsLast(salted,false,false,true);
- doTestOrderByReverseOptimizationWithNullsLast(salted,false,false,false);
- }
- }
+ conn.setAutoCommit(false);
- private void doTestOrderByReverseOptimizationWithNullsLast(boolean salted,boolean desc1,boolean desc2,boolean desc3) throws Exception {
- Connection conn = null;
try {
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- conn = DriverManager.getConnection(getUrl(), props);
- String tableName=generateUniqueName();
- String sql="CREATE TABLE "+tableName+" ( "+
- "ORGANIZATION_ID VARCHAR,"+
- "CONTAINER_ID VARCHAR,"+
- "SCORE VARCHAR,"+
- "ENTITY_ID VARCHAR NOT NULL,"+
- "CONSTRAINT TEST_PK PRIMARY KEY ( "+
- "ORGANIZATION_ID" +(desc1 ? " DESC" : "" )+","+
- "CONTAINER_ID"+(desc2 ? " DESC" : "" )+","+
- "SCORE"+(desc3 ? " DESC" : "" )+","+
- "ENTITY_ID"+
- ")) "+(salted ? "SALT_BUCKETS =4" : "split on('4')");
- conn.createStatement().execute(sql);
-
- for(int i=1;i<=6;i++) {
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,'"+i+"','"+i+"','"+i+"')");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,'"+i+"',null,'"+i+"')");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,null,'"+i+"','"+i+"')");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,null,null,'"+i+"')");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('"+i+"','"+i+"','"+i+"','"+i+"')");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('"+i+"','"+i+"',null,'"+i+"')");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('"+i+"',null,'"+i+"','"+i+"')");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('"+i+"',null,null,'"+i+"')");
- }
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,null,null,'66')");
+ String tableName = generateUniqueName();
+ String ddl = "CREATE TABLE " + tableName +
+ " (a_string varchar not null, cf1.a integer, cf1.b varchar, col1 integer, cf2.c varchar, cf2.d integer, col2 integer" +
+ " CONSTRAINT pk PRIMARY KEY (a_string))\n";
+ createTestTable(getUrl(), ddl);
+ String dml = "UPSERT INTO " + tableName + " VALUES(?,?,?,?,?,?,?)";
+ PreparedStatement stmt = conn.prepareStatement(dml);
+ stmt.setString(1, "a");
+ stmt.setInt(2, 40);
+ stmt.setString(3, "aa");
+ stmt.setInt(4, 10);
+ stmt.setString(5, "bb");
+ stmt.setInt(6, 20);
+ stmt.setInt(7, 1);
+ stmt.execute();
+ stmt.setString(1, "c");
+ stmt.setInt(2, 30);
+ stmt.setString(3, "cc");
+ stmt.setInt(4, 50);
+ stmt.setString(5, "dd");
+ stmt.setInt(6, 60);
+ stmt.setInt(7, 3);
+ stmt.execute();
+ stmt.setString(1, "b");
+ stmt.setInt(2, 40);
+ stmt.setString(3, "bb");
+ stmt.setInt(4, 5);
+ stmt.setString(5, "aa");
+ stmt.setInt(6, 80);
+ stmt.setInt(7, 2);
+ stmt.execute();
conn.commit();
- //groupBy orderPreserving orderBy asc asc
-
- sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID ASC NULLS FIRST";
- ResultSet rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{null,null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{"1",null},{"1","1"},{"2",null},{"2","2"},{"3",null},{"3","3"},{"4",null},{"4","4"},{"5",null},{"5","5"},{"6",null},{"6","6"}});
-
- sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID ASC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{null,null},{"1","1"},{"1",null},{"2","2"},{"2",null},{"3","3"},{"3",null},{"4","4"},{"4",null},{"5","5"},{"5",null},{"6","6"},{"6",null}});
-
- sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS LAST, CONTAINER_ID ASC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{"1",null},{"1","1"},{"2",null},{"2","2"},{"3",null},{"3","3"},{"4",null},{"4","4"},{"5",null},{"5","5"},{"6",null},{"6","6"},{null,null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"}});
-
- sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS LAST, CONTAINER_ID ASC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{"1","1"},{"1",null},{"2","2"},{"2",null},{"3","3"},{"3",null},{"4","4"},{"4",null},{"5","5"},{"5",null},{"6","6"},{"6",null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{null,null}});
-
- //groupBy orderPreserving orderBy asc desc
-
- sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID DESC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{null,null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{"1",null},{"1","1"},{"2",null},{"2","2"},{"3",null},{"3","3"},{"4",null},{"4","4"},{"5",null},{"5","5"},{"6",null},{"6","6"}});
-
- sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID DESC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{null,null},{"1","1"},{"1",null},{"2","2"},{"2",null},{"3","3"},{"3",null},{"4","4"},{"4",null},{"5","5"},{"5",null},{"6","6"},{"6",null}});
-
- sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS LAST, CONTAINER_ID DESC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{"1",null},{"1","1"},{"2",null},{"2","2"},{"3",null},{"3","3"},{"4",null},{"4","4"},{"5",null},{"5","5"},{"6",null},{"6","6"},{null,null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"}});
-
- sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS LAST, CONTAINER_ID DESC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{"1","1"},{"1",null},{"2","2"},{"2",null},{"3","3"},{"3",null},{"4","4"},{"4",null},{"5","5"},{"5",null},{"6","6"},{"6",null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{null,null}});
-
- //groupBy orderPreserving orderBy desc asc
-
- sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS FIRST, CONTAINER_ID ASC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{null,null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{"6",null},{"6","6"},{"5",null},{"5","5"},{"4",null},{"4","4"},{"3",null},{"3","3"},{"2",null},{"2","2"},{"1",null},{"1","1"}});
-
- sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS FIRST, CONTAINER_ID ASC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{null,null},{"6","6"},{"6",null},{"5","5"},{"5",null},{"4","4"},{"4",null},{"3","3"},{"3",null},{"2","2"},{"2",null},{"1","1"},{"1",null}});
-
- sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS LAST, CONTAINER_ID ASC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{"6",null},{"6","6"},{"5",null},{"5","5"},{"4",null},{"4","4"},{"3",null},{"3","3"},{"2",null},{"2","2"},{"1",null},{"1","1"},{null,null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"}});
-
- sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS LAST, CONTAINER_ID ASC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{"6","6"},{"6",null},{"5","5"},{"5",null},{"4","4"},{"4",null},{"3","3"},{"3",null},{"2","2"},{"2",null},{"1","1"},{"1",null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{null,null}});
-
- //groupBy orderPreserving orderBy desc desc
-
- sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS FIRST, CONTAINER_ID DESC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{null,null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{"6",null},{"6","6"},{"5",null},{"5","5"},{"4",null},{"4","4"},{"3",null},{"3","3"},{"2",null},{"2","2"},{"1",null},{"1","1"}});
-
- sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS FIRST, CONTAINER_ID DESC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{null,null},{"6","6"},{"6",null},{"5","5"},{"5",null},{"4","4"},{"4",null},{"3","3"},{"3",null},{"2","2"},{"2",null},{"1","1"},{"1",null}});
-
- sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS LAST, CONTAINER_ID DESC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{"6",null},{"6","6"},{"5",null},{"5","5"},{"4",null},{"4","4"},{"3",null},{"3","3"},{"2",null},{"2","2"},{"1",null},{"1","1"},{null,null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"}});
-
- sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS LAST, CONTAINER_ID DESC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{"6","6"},{"6",null},{"5","5"},{"5",null},{"4","4"},{"4",null},{"3","3"},{"3",null},{"2","2"},{"2",null},{"1","1"},{"1",null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{null,null}});
-
- //-----groupBy not orderPreserving--
-
- //groupBy not orderPreserving orderBy asc asc
-
- sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS FIRST, SCORE ASC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{null,null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{"1",null},{"1","1"},{"2",null},{"2","2"},{"3",null},{"3","3"},{"4",null},{"4","4"},{"5",null},{"5","5"},{"6",null},{"6","6"}});
-
- sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS FIRST, SCORE ASC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{null,null},{"1","1"},{"1",null},{"2","2"},{"2",null},{"3","3"},{"3",null},{"4","4"},{"4",null},{"5","5"},{"5",null},{"6","6"},{"6",null}});
-
- sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS LAST, SCORE ASC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{"1",null},{"1","1"},{"2",null},{"2","2"},{"3",null},{"3","3"},{"4",null},{"4","4"},{"5",null},{"5","5"},{"6",null},{"6","6"},{null,null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"}});
-
- sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS LAST, SCORE ASC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{"1","1"},{"1",null},{"2","2"},{"2",null},{"3","3"},{"3",null},{"4","4"},{"4",null},{"5","5"},{"5",null},{"6","6"},{"6",null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{null,null}});
-
- //groupBy not orderPreserving orderBy asc desc
-
- sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS FIRST, SCORE DESC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{null,null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{"1",null},{"1","1"},{"2",null},{"2","2"},{"3",null},{"3","3"},{"4",null},{"4","4"},{"5",null},{"5","5"},{"6",null},{"6","6"}});
-
- sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS FIRST, SCORE DESC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{null,null},{"1","1"},{"1",null},{"2","2"},{"2",null},{"3","3"},{"3",null},{"4","4"},{"4",null},{"5","5"},{"5",null},{"6","6"},{"6",null}});
-
- sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS LAST, SCORE DESC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{"1",null},{"1","1"},{"2",null},{"2","2"},{"3",null},{"3","3"},{"4",null},{"4","4"},{"5",null},{"5","5"},{"6",null},{"6","6"},{null,null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"}});
-
- sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS LAST, SCORE DESC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{"1","1"},{"1",null},{"2","2"},{"2",null},{"3","3"},{"3",null},{"4","4"},{"4",null},{"5","5"},{"5",null},{"6","6"},{"6",null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{null,null}});
-
- //groupBy not orderPreserving orderBy desc asc
-
- sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS FIRST, SCORE ASC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{null,null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{"6",null},{"6","6"},{"5",null},{"5","5"},{"4",null},{"4","4"},{"3",null},{"3","3"},{"2",null},{"2","2"},{"1",null},{"1","1"}});
-
- sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS FIRST, SCORE ASC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{null,null},{"6","6"},{"6",null},{"5","5"},{"5",null},{"4","4"},{"4",null},{"3","3"},{"3",null},{"2","2"},{"2",null},{"1","1"},{"1",null}});
-
- sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS LAST, SCORE ASC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{"6",null},{"6","6"},{"5",null},{"5","5"},{"4",null},{"4","4"},{"3",null},{"3","3"},{"2",null},{"2","2"},{"1",null},{"1","1"},{null,null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"}});
-
- sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS LAST, SCORE ASC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{"6","6"},{"6",null},{"5","5"},{"5",null},{"4","4"},{"4",null},{"3","3"},{"3",null},{"2","2"},{"2",null},{"1","1"},{"1",null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{null,null}});
-
- //groupBy not orderPreserving orderBy desc desc
-
- sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS FIRST, SCORE DESC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{null,null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{"6",null},{"6","6"},{"5",null},{"5","5"},{"4",null},{"4","4"},{"3",null},{"3","3"},{"2",null},{"2","2"},{"1",null},{"1","1"}});
-
- sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS FIRST, SCORE DESC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{null,null},{"6","6"},{"6",null},{"5","5"},{"5",null},{"4","4"},{"4",null},{"3","3"},{"3",null},{"2","2"},{"2",null},{"1","1"},{"1",null}});
-
- sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS LAST, SCORE DESC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{"6",null},{"6","6"},{"5",null},{"5","5"},{"4",null},{"4","4"},{"3",null},{"3","3"},{"2",null},{"2","2"},{"1",null},{"1","1"},{null,null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"}});
-
- sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS LAST, SCORE DESC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{"6","6"},{"6",null},{"5","5"},{"5",null},{"4","4"},{"4",null},{"3","3"},{"3",null},{"2","2"},{"2",null},{"1","1"},{"1",null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{null,null}});
-
- //-------test only one return column----------------------------------
-
- sql="SELECT SCORE FROM "+tableName+" group by SCORE ORDER BY SCORE ASC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{null},{"1"},{"2"},{"3"},{"4"},{"5"},{"6"}});
-
- sql="SELECT SCORE FROM "+tableName+" group by SCORE ORDER BY SCORE ASC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{"1"},{"2"},{"3"},{"4"},{"5"},{"6"},{null}});
-
- sql="SELECT SCORE FROM "+tableName+" group by SCORE ORDER BY SCORE DESC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{null},{"6"},{"5"},{"4"},{"3"},{"2"},{"1"}});
+ String query = "select * from " + tableName + " order by 2, 5";
+ ResultSet rs = conn.createStatement().executeQuery(query);
+ assertTrue(rs.next());
+ assertEquals("c",rs.getString(1));
+ assertEquals(30,rs.getInt(2));
+ assertEquals("cc",rs.getString(3));
+ assertEquals(50,rs.getInt(4));
+ assertEquals("dd",rs.getString(5));
+ assertEquals(60,rs.getInt(6));
+ assertEquals(3,rs.getInt(7));
+ assertTrue(rs.next());
+ assertEquals("b",rs.getString(1));
+ assertEquals(40,rs.getInt(2));
+ assertEquals("bb",rs.getString(3));
+ assertEquals(5,rs.getInt(4));
+ assertEquals("aa",rs.getString(5));
+ assertEquals(80,rs.getInt(6));
+ assertEquals(2,rs.getInt(7));
+ assertTrue(rs.next());
+ assertEquals("a",rs.getString(1));
+ assertEquals(40,rs.getInt(2));
+ assertEquals("aa",rs.getString(3));
+ assertEquals(10,rs.getInt(4));
+ assertEquals("bb",rs.getString(5));
+ assertEquals(20,rs.getInt(6));
+ assertEquals(1,rs.getInt(7));
+ assertFalse(rs.next());
- sql="SELECT SCORE FROM "+tableName+" group by SCORE ORDER BY SCORE DESC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{"6"},{"5"},{"4"},{"3"},{"2"},{"1"},{null}});
+ query = "select * from " + tableName + " order by 7";
+ rs = conn.createStatement().executeQuery(query);
+ assertTrue(rs.next());
+ assertEquals("a",rs.getString(1));
+ assertEquals(40,rs.getInt(2));
+ assertEquals("aa",rs.getString(3));
+ assertEquals(10,rs.getInt(4));
+ assertEquals("bb",rs.getString(5));
+ assertEquals(20,rs.getInt(6));
+ assertEquals(1,rs.getInt(7));
+ assertTrue(rs.next());
+ assertEquals("b",rs.getString(1));
+ assertEquals(40,rs.getInt(2));
+ assertEquals("bb",rs.getString(3));
+ assertEquals(5,rs.getInt(4));
+ assertEquals("aa",rs.getString(5));
+ assertEquals(80,rs.getInt(6));
+ assertEquals(2,rs.getInt(7));
+ assertTrue(rs.next());
+ assertEquals("c",rs.getString(1));
+ assertEquals(30,rs.getInt(2));
+ assertEquals("cc",rs.getString(3));
+ assertEquals(50,rs.getInt(4));
+ assertEquals("dd",rs.getString(5));
+ assertEquals(60,rs.getInt(6));
+ assertEquals(3,rs.getInt(7));
+ assertFalse(rs.next());
} finally {
- if(conn!=null) {
- conn.close();
- }
+ conn.close();
}
}
-
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParallelStatsDisabledIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParallelStatsDisabledIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParallelStatsDisabledIT.java
index 98939da..561aee5 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParallelStatsDisabledIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/ParallelStatsDisabledIT.java
@@ -18,12 +18,26 @@
package org.apache.phoenix.end2end;
+import org.apache.commons.lang.StringUtils;
import org.apache.phoenix.query.BaseTest;
+import org.apache.phoenix.util.QueryBuilder;
+import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.ReadOnlyProps;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.experimental.categories.Category;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+
+
/**
* Base class for tests whose methods run in parallel with statistics disabled.
* You must create unique names using {@link #generateUniqueName()} for each
@@ -41,4 +55,30 @@ public abstract class ParallelStatsDisabledIT extends BaseTest {
public static void tearDownMiniCluster() throws Exception {
BaseTest.tearDownMiniClusterIfBeyondThreshold();
}
+
+ protected ResultSet executeQuery(Connection conn, QueryBuilder queryBuilder) throws SQLException {
+ PreparedStatement statement = conn.prepareStatement(queryBuilder.build());
+ ResultSet rs = statement.executeQuery();
+ return rs;
+ }
+
+ protected ResultSet executeQueryThrowsException(Connection conn, QueryBuilder queryBuilder,
+ String expectedPhoenixExceptionMsg, String expectedSparkExceptionMsg) {
+ ResultSet rs = null;
+ try {
+ rs = executeQuery(conn, queryBuilder);
+ fail();
+ }
+ catch(Exception e) {
+ assertTrue(e.getMessage().contains(expectedPhoenixExceptionMsg));
+ }
+ return rs;
+ }
+
+ protected void validateQueryPlan(Connection conn, QueryBuilder queryBuilder, String expectedPhoenixPlan, String expectedSparkPlan) throws SQLException {
+ if (StringUtils.isNotBlank(expectedPhoenixPlan)) {
+ ResultSet rs = conn.createStatement().executeQuery("EXPLAIN " + queryBuilder.build());
+ assertEquals(expectedPhoenixPlan, QueryUtil.getExplainPlan(rs));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/BaseSaltedTableIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/BaseSaltedTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/BaseSaltedTableIT.java
new file mode 100644
index 0000000..3051cd6
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/BaseSaltedTableIT.java
@@ -0,0 +1,474 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end.salted;
+
+import static org.apache.phoenix.util.TestUtil.TABLE_WITH_SALTING;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.util.Properties;
+
+import org.apache.phoenix.end2end.ParallelStatsDisabledIT;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.QueryBuilder;
+import org.apache.phoenix.util.QueryUtil;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+public abstract class BaseSaltedTableIT extends ParallelStatsDisabledIT {
+
+ protected static String initTableValues(byte[][] splits) throws Exception {
+ String tableName = generateUniqueName();
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+
+ // Rows we inserted:
+ // 1ab123abc111
+ // 1abc456abc111
+ // 1de123abc111
+ // 2abc123def222
+ // 3abc123ghi333
+ // 4abc123jkl444
+ try {
+ // Upsert with no column specifies.
+ ensureTableCreated(getUrl(), tableName, TABLE_WITH_SALTING, splits, null, null);
+ String query = "UPSERT INTO " + tableName + " VALUES(?,?,?,?,?)";
+ PreparedStatement stmt = conn.prepareStatement(query);
+ stmt.setInt(1, 1);
+ stmt.setString(2, "ab");
+ stmt.setString(3, "123");
+ stmt.setString(4, "abc");
+ stmt.setInt(5, 111);
+ stmt.execute();
+ conn.commit();
+
+ stmt.setInt(1, 1);
+ stmt.setString(2, "abc");
+ stmt.setString(3, "456");
+ stmt.setString(4, "abc");
+ stmt.setInt(5, 111);
+ stmt.execute();
+ conn.commit();
+
+ // Test upsert when statement explicitly specifies the columns to upsert into.
+ query = "UPSERT INTO " + tableName +
+ " (a_integer, a_string, a_id, b_string, b_integer) " +
+ " VALUES(?,?,?,?,?)";
+ stmt = conn.prepareStatement(query);
+
+ stmt.setInt(1, 1);
+ stmt.setString(2, "de");
+ stmt.setString(3, "123");
+ stmt.setString(4, "abc");
+ stmt.setInt(5, 111);
+ stmt.execute();
+ conn.commit();
+
+ stmt.setInt(1, 2);
+ stmt.setString(2, "abc");
+ stmt.setString(3, "123");
+ stmt.setString(4, "def");
+ stmt.setInt(5, 222);
+ stmt.execute();
+ conn.commit();
+
+ // Test upsert when order of column is shuffled.
+ query = "UPSERT INTO " + tableName +
+ " (a_string, a_integer, a_id, b_string, b_integer) " +
+ " VALUES(?,?,?,?,?)";
+ stmt = conn.prepareStatement(query);
+ stmt.setString(1, "abc");
+ stmt.setInt(2, 3);
+ stmt.setString(3, "123");
+ stmt.setString(4, "ghi");
+ stmt.setInt(5, 333);
+ stmt.execute();
+ conn.commit();
+
+ stmt.setString(1, "abc");
+ stmt.setInt(2, 4);
+ stmt.setString(3, "123");
+ stmt.setString(4, "jkl");
+ stmt.setInt(5, 444);
+ stmt.execute();
+ conn.commit();
+ } finally {
+ conn.close();
+ }
+ return tableName;
+ }
+
+ @Test
+ public void testSelectValueNoWhereClause() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ try {
+ String tableName = initTableValues(null);
+ // "SELECT * FROM " + tableName;
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectColumns(
+ Lists.newArrayList("A_INTEGER", "A_STRING", "A_ID", "B_STRING", "B_INTEGER"))
+ .setFullTableName(tableName);
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals(1, rs.getInt(1));
+ assertEquals("ab", rs.getString(2));
+ assertEquals("123", rs.getString(3));
+ assertEquals("abc", rs.getString(4));
+ assertEquals(111, rs.getInt(5));
+
+ assertTrue(rs.next());
+ assertEquals(1, rs.getInt(1));
+ assertEquals("abc", rs.getString(2));
+ assertEquals("456", rs.getString(3));
+ assertEquals("abc", rs.getString(4));
+ assertEquals(111, rs.getInt(5));
+
+ assertTrue(rs.next());
+ assertEquals(1, rs.getInt(1));
+ assertEquals("de", rs.getString(2));
+ assertEquals("123", rs.getString(3));
+ assertEquals("abc", rs.getString(4));
+ assertEquals(111, rs.getInt(5));
+
+ assertTrue(rs.next());
+ assertEquals(2, rs.getInt(1));
+ assertEquals("abc", rs.getString(2));
+ assertEquals("123", rs.getString(3));
+ assertEquals("def", rs.getString(4));
+ assertEquals(222, rs.getInt(5));
+
+ assertTrue(rs.next());
+ assertEquals(3, rs.getInt(1));
+ assertEquals("abc", rs.getString(2));
+ assertEquals("123", rs.getString(3));
+ assertEquals("ghi", rs.getString(4));
+ assertEquals(333, rs.getInt(5));
+
+ assertTrue(rs.next());
+ assertEquals(4, rs.getInt(1));
+ assertEquals("abc", rs.getString(2));
+ assertEquals("123", rs.getString(3));
+ assertEquals("jkl", rs.getString(4));
+ assertEquals(444, rs.getInt(5));
+
+ assertFalse(rs.next());
+ } finally {
+ conn.close();
+ }
+ }
+
+ @Test
+ public void testSelectValueWithFullyQualifiedWhereClause() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ try {
+ String tableName = initTableValues(null);
+ PreparedStatement stmt;
+ ResultSet rs;
+
+ // Variable length slot with bounded ranges.
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectColumns(
+ Lists.newArrayList("A_INTEGER", "A_STRING", "A_ID", "B_STRING", "B_INTEGER"))
+ .setFullTableName(tableName)
+ .setWhereClause("a_integer = 1 AND a_string >= 'ab' AND a_string < 'de' AND a_id = '123'");
+ rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals(1, rs.getInt(1));
+ assertEquals("ab", rs.getString(2));
+ assertEquals("123", rs.getString(3));
+ assertEquals("abc", rs.getString(4));
+ assertEquals(111, rs.getInt(5));
+ assertFalse(rs.next());
+
+ // all single slots with one value.
+ queryBuilder.setWhereClause("a_integer = 1 AND a_string = 'ab' AND a_id = '123'");
+ rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals(1, rs.getInt(1));
+ assertEquals("ab", rs.getString(2));
+ assertEquals("123", rs.getString(3));
+ assertEquals("abc", rs.getString(4));
+ assertEquals(111, rs.getInt(5));
+ assertFalse(rs.next());
+
+ // all single slots with multiple values.
+ queryBuilder.setWhereClause("a_integer in (2, 4) AND a_string = 'abc' AND a_id = '123'");
+ rs = executeQuery(conn, queryBuilder);
+
+ assertTrue(rs.next());
+ assertEquals(2, rs.getInt(1));
+ assertEquals("abc", rs.getString(2));
+ assertEquals("123", rs.getString(3));
+ assertEquals("def", rs.getString(4));
+ assertEquals(222, rs.getInt(5));
+
+ assertTrue(rs.next());
+ assertEquals(4, rs.getInt(1));
+ assertEquals("abc", rs.getString(2));
+ assertEquals("123", rs.getString(3));
+ assertEquals("jkl", rs.getString(4));
+ assertEquals(444, rs.getInt(5));
+ assertFalse(rs.next());
+
+ queryBuilder.setWhereClause("A_INTEGER in (1,2,3,4) AND A_STRING in ('a', 'abc', 'de') AND A_ID = '123'");
+ queryBuilder.setSelectColumns(Lists.newArrayList("A_INTEGER", "A_STRING", "A_ID"));
+ rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals(1, rs.getInt(1));
+ assertEquals("de", rs.getString(2));
+
+ assertTrue(rs.next());
+ assertEquals(2, rs.getInt(1));
+ assertEquals("abc", rs.getString(2));
+
+ assertTrue(rs.next());
+ assertEquals(3, rs.getInt(1));
+ assertEquals("abc", rs.getString(2));
+
+ assertTrue(rs.next());
+ assertEquals(4, rs.getInt(1));
+ assertEquals("abc", rs.getString(2));
+ assertFalse(rs.next());
+
+ // fixed length slot with bounded ranges.
+ queryBuilder.setWhereClause("A_INTEGER > 1 AND A_INTEGER < 4 AND A_STRING = 'abc' AND A_ID = '123'");
+ queryBuilder.setSelectColumns(Lists.newArrayList("A_STRING", "A_ID", "A_INTEGER"));
+ rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals("abc", rs.getString(1));
+ assertEquals("123", rs.getString(2));
+
+ assertTrue(rs.next());
+ assertEquals("abc", rs.getString(1));
+ assertEquals("123", rs.getString(2));
+ assertFalse(rs.next());
+
+ // fixed length slot with unbound ranges.
+ queryBuilder.setWhereClause("A_INTEGER > 1 AND A_STRING = 'abc' AND A_ID = '123'");
+ queryBuilder.setSelectColumns(Lists.newArrayList("B_STRING", "B_INTEGER", "A_INTEGER", "A_STRING", "A_ID"));
+ rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals("def", rs.getString(1));
+ assertEquals(222, rs.getInt(2));
+
+ assertTrue(rs.next());
+ assertEquals("ghi", rs.getString(1));
+ assertEquals(333, rs.getInt(2));
+
+ assertTrue(rs.next());
+ assertEquals("jkl", rs.getString(1));
+ assertEquals(444, rs.getInt(2));
+ assertFalse(rs.next());
+
+ // Variable length slot with unbounded ranges.
+ queryBuilder.setWhereClause("A_INTEGER = 1 AND A_STRING > 'ab' AND A_ID = '123'");
+ queryBuilder.setSelectColumns(
+ Lists.newArrayList("A_INTEGER", "A_STRING", "A_ID", "B_STRING", "B_INTEGER"));
+ rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals(1, rs.getInt(1));
+ assertEquals("de", rs.getString(2));
+ assertEquals("123", rs.getString(3));
+ assertEquals("abc", rs.getString(4));
+ assertEquals(111, rs.getInt(5));
+ assertFalse(rs.next());
+
+ } finally {
+ conn.close();
+ }
+ }
+
+ @Test
+ public void testSelectValueWithNotFullyQualifiedWhereClause() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ try {
+ String tableName = initTableValues(null);
+
+ // Where without fully qualified key, point query.
+ String query = "SELECT * FROM " + tableName + " WHERE a_integer = ? AND a_string = ?";
+ PreparedStatement stmt = conn.prepareStatement(query);
+
+ stmt.setInt(1, 1);
+ stmt.setString(2, "abc");
+ ResultSet rs = stmt.executeQuery();
+ assertTrue(rs.next());
+ assertEquals(1, rs.getInt(1));
+ assertEquals("abc", rs.getString(2));
+ assertEquals("456", rs.getString(3));
+ assertEquals("abc", rs.getString(4));
+ assertEquals(111, rs.getInt(5));
+ assertFalse(rs.next());
+
+ // Where without fully qualified key, range query.
+ query = "SELECT * FROM " + tableName + " WHERE a_integer >= 2";
+ stmt = conn.prepareStatement(query);
+ rs = stmt.executeQuery();
+ assertTrue(rs.next());
+ assertEquals(2, rs.getInt(1));
+ assertEquals("abc", rs.getString(2));
+ assertEquals("123", rs.getString(3));
+ assertEquals("def", rs.getString(4));
+ assertEquals(222, rs.getInt(5));
+
+ assertTrue(rs.next());
+ assertEquals(3, rs.getInt(1));
+ assertEquals("abc", rs.getString(2));
+ assertEquals("123", rs.getString(3));
+ assertEquals("ghi", rs.getString(4));
+ assertEquals(333, rs.getInt(5));
+
+ assertTrue(rs.next());
+ assertEquals(4, rs.getInt(1));
+ assertEquals("abc", rs.getString(2));
+ assertEquals("123", rs.getString(3));
+ assertEquals("jkl", rs.getString(4));
+ assertEquals(444, rs.getInt(5));
+ assertFalse(rs.next());
+
+ // With point query.
+ query = "SELECT a_string FROM " + tableName + " WHERE a_string = ?";
+ stmt = conn.prepareStatement(query);
+ stmt.setString(1, "de");
+ rs = stmt.executeQuery();
+ assertTrue(rs.next());
+ assertEquals("de", rs.getString(1));
+ assertFalse(rs.next());
+
+ query = "SELECT a_id FROM " + tableName + " WHERE a_id = ?";
+ stmt = conn.prepareStatement(query);
+ stmt.setString(1, "456");
+ rs = stmt.executeQuery();
+ assertTrue(rs.next());
+ assertEquals("456", rs.getString(1));
+ assertFalse(rs.next());
+ } finally {
+ conn.close();
+ }
+ }
+
+ @Test
+ public void testSelectWithGroupBy() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ try {
+ String tableName = initTableValues(null);
+
+ String query = "SELECT a_integer FROM " + tableName + " GROUP BY a_integer";
+ PreparedStatement stmt = conn.prepareStatement(query);
+ ResultSet rs = stmt.executeQuery();
+ int count = 0;
+ while (rs.next()) {
+ count++;
+ }
+ assertEquals("Group by does not return the right count.", count, 4);
+ } finally {
+ conn.close();
+ }
+ }
+
+ @Test
+ public void testLimitScan() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ try {
+ String tableName = initTableValues(null);
+
+ String query = "SELECT a_integer FROM " + tableName + " WHERE a_string='abc' LIMIT 1";
+ PreparedStatement stmt = conn.prepareStatement(query);
+ ResultSet rs = stmt.executeQuery();
+ assertTrue(rs.next());
+ assertEquals(1, rs.getInt(1));
+ assertFalse(rs.next());
+ } finally {
+ conn.close();
+ }
+ }
+
+ @Test
+ public void testSelectWithOrderByRowKey() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ try {
+ String tableName = initTableValues(null);
+
+ String query = "SELECT * FROM " + tableName + " ORDER BY a_integer, a_string, a_id";
+ PreparedStatement statement = conn.prepareStatement(query);
+ ResultSet explainPlan = statement.executeQuery("EXPLAIN " + query);
+ // Confirm that ORDER BY in row key order will be optimized out for salted table
+ assertEquals("CLIENT PARALLEL 4-WAY FULL SCAN OVER " + tableName + "\n" +
+ "CLIENT MERGE SORT", QueryUtil.getExplainPlan(explainPlan));
+ ResultSet rs = statement.executeQuery();
+
+ assertTrue(rs.next());
+ assertEquals(1, rs.getInt(1));
+ assertEquals("ab", rs.getString(2));
+ assertEquals("123", rs.getString(3));
+ assertEquals("abc", rs.getString(4));
+ assertEquals(111, rs.getInt(5));
+
+ assertTrue(rs.next());
+ assertEquals(1, rs.getInt(1));
+ assertEquals("abc", rs.getString(2));
+ assertEquals("456", rs.getString(3));
+ assertEquals("abc", rs.getString(4));
+ assertEquals(111, rs.getInt(5));
+
+ assertTrue(rs.next());
+ assertEquals(1, rs.getInt(1));
+ assertEquals("de", rs.getString(2));
+ assertEquals("123", rs.getString(3));
+ assertEquals("abc", rs.getString(4));
+ assertEquals(111, rs.getInt(5));
+
+ assertTrue(rs.next());
+ assertEquals(2, rs.getInt(1));
+ assertEquals("abc", rs.getString(2));
+ assertEquals("123", rs.getString(3));
+ assertEquals("def", rs.getString(4));
+ assertEquals(222, rs.getInt(5));
+
+ assertTrue(rs.next());
+ assertEquals(3, rs.getInt(1));
+ assertEquals("abc", rs.getString(2));
+ assertEquals("123", rs.getString(3));
+ assertEquals("ghi", rs.getString(4));
+ assertEquals(333, rs.getInt(5));
+
+ assertTrue(rs.next());
+ assertEquals(4, rs.getInt(1));
+ assertEquals("abc", rs.getString(2));
+ assertEquals("123", rs.getString(3));
+ assertEquals("jkl", rs.getString(4));
+ assertEquals(444, rs.getInt(5));
+
+ assertFalse(rs.next());
+ } finally {
+ conn.close();
+ }
+ }
+}
[07/22] phoenix git commit: PHOENIX-4909 Missing the
phoenix-loadbalancer module dependency in phoenix-assembly/pom.xml
Posted by ja...@apache.org.
PHOENIX-4909 Missing the phoenix-loadbalancer module dependency in phoenix-assembly/pom.xml
Closes #352
Signed-off-by: Josh Elser <el...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/fc35ccec
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/fc35ccec
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/fc35ccec
Branch: refs/heads/omid2
Commit: fc35ccecc6fa497da506aff48d601657e41d2a70
Parents: c936d81
Author: Vitaliy <vi...@mail.ru>
Authored: Wed Sep 19 01:09:01 2018 +0300
Committer: Josh Elser <el...@apache.org>
Committed: Wed Oct 31 12:20:52 2018 -0400
----------------------------------------------------------------------
bin/phoenix_utils.py | 2 ++
phoenix-assembly/pom.xml | 4 ++++
phoenix-assembly/src/build/components/all-common-jars.xml | 8 ++++++++
3 files changed, 14 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc35ccec/bin/phoenix_utils.py
----------------------------------------------------------------------
diff --git a/bin/phoenix_utils.py b/bin/phoenix_utils.py
index aa04a5b..98a0896 100755
--- a/bin/phoenix_utils.py
+++ b/bin/phoenix_utils.py
@@ -166,6 +166,8 @@ def setPath():
global phoenix_loadbalancer_jar
phoenix_loadbalancer_jar = find(PHOENIX_LOADBALANCER_JAR_PATTERN, os.path.join(current_dir, "..", "phoenix-loadbalancer", "target", "*"))
if phoenix_loadbalancer_jar == "":
+ phoenix_loadbalancer_jar = findFileInPathWithoutRecursion(PHOENIX_LOADBALANCER_JAR_PATTERN, os.path.join(current_dir, "..", "lib"))
+ if phoenix_loadbalancer_jar == "":
phoenix_loadbalancer_jar = findFileInPathWithoutRecursion(PHOENIX_LOADBALANCER_JAR_PATTERN, os.path.join(current_dir, ".."))
global phoenix_traceserver_jar
http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc35ccec/phoenix-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-assembly/pom.xml b/phoenix-assembly/pom.xml
index f11645a..e41d7e6 100644
--- a/phoenix-assembly/pom.xml
+++ b/phoenix-assembly/pom.xml
@@ -126,5 +126,9 @@
<groupId>org.apache.phoenix</groupId>
<artifactId>phoenix-queryserver-client</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.phoenix</groupId>
+ <artifactId>phoenix-load-balancer</artifactId>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/fc35ccec/phoenix-assembly/src/build/components/all-common-jars.xml
----------------------------------------------------------------------
diff --git a/phoenix-assembly/src/build/components/all-common-jars.xml b/phoenix-assembly/src/build/components/all-common-jars.xml
index 3d27b26..08ca29a 100644
--- a/phoenix-assembly/src/build/components/all-common-jars.xml
+++ b/phoenix-assembly/src/build/components/all-common-jars.xml
@@ -158,5 +158,13 @@
</includes>
<fileMode>0644</fileMode>
</fileSet>
+ <fileSet>
+ <directory>${project.basedir}/../phoenix-load-balancer/target/</directory>
+ <outputDirectory>lib</outputDirectory>
+ <includes>
+ <include>phoenix-*.jar</include>
+ </includes>
+ <fileMode>0644</fileMode>
+ </fileSet>
</fileSets>
</component>
[15/22] phoenix git commit: PHOENIX-4981 Add tests for ORDER BY,
GROUP BY and salted tables using phoenix-spark
Posted by ja...@apache.org.
PHOENIX-4981 Add tests for ORDER BY, GROUP BY and salted tables using phoenix-spark
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/9bfaf183
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/9bfaf183
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/9bfaf183
Branch: refs/heads/omid2
Commit: 9bfaf183a3d092bce6b2dfcacd77ba46aa1b078b
Parents: 1b2a3d5
Author: Thomas D'Silva <td...@apache.org>
Authored: Thu Oct 18 22:00:01 2018 -0700
Committer: Thomas D'Silva <td...@apache.org>
Committed: Tue Nov 6 14:51:13 2018 -0800
----------------------------------------------------------------------
.../org/apache/phoenix/end2end/AggregateIT.java | 987 +---------------
.../apache/phoenix/end2end/BaseAggregateIT.java | 1022 +++++++++++++++++
.../apache/phoenix/end2end/BaseOrderByIT.java | 940 ++++++++++++++++
.../org/apache/phoenix/end2end/OrderByIT.java | 943 ++--------------
.../end2end/ParallelStatsDisabledIT.java | 40 +
.../end2end/salted/BaseSaltedTableIT.java | 474 ++++++++
.../phoenix/end2end/salted/SaltedTableIT.java | 450 +-------
.../org/apache/phoenix/util/QueryBuilder.java | 211 ++++
.../java/org/apache/phoenix/util/QueryUtil.java | 38 +-
.../index/IndexScrutinyTableOutputTest.java | 6 +-
.../util/PhoenixConfigurationUtilTest.java | 6 +-
.../org/apache/phoenix/util/QueryUtilTest.java | 10 +-
phoenix-spark/pom.xml | 8 +
.../org/apache/phoenix/spark/AggregateIT.java | 91 ++
.../org/apache/phoenix/spark/OrderByIT.java | 460 ++++++++
.../org/apache/phoenix/spark/SaltedTableIT.java | 53 +
.../org/apache/phoenix/spark/SparkUtil.java | 87 ++
.../apache/phoenix/spark/PhoenixSparkIT.scala | 9 +-
.../apache/phoenix/spark/SparkResultSet.java | 1056 ++++++++++++++++++
.../org/apache/phoenix/spark/PhoenixRDD.scala | 27 +-
pom.xml | 2 +-
21 files changed, 4650 insertions(+), 2270 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-core/src/it/java/org/apache/phoenix/end2end/AggregateIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AggregateIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AggregateIT.java
index 2059311..8916d4d 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/AggregateIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/AggregateIT.java
@@ -18,506 +18,28 @@
package org.apache.phoenix.end2end;
import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.apache.phoenix.util.TestUtil.assertResultSet;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
-import static org.apache.phoenix.util.TestUtil.assertResultSet;
-import java.io.IOException;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
-import java.sql.Statement;
-import java.util.List;
import java.util.Properties;
-import org.apache.hadoop.hbase.util.Bytes;
-import org.apache.phoenix.compile.QueryPlan;
-import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
-import org.apache.phoenix.jdbc.PhoenixStatement;
-import org.apache.phoenix.query.KeyRange;
import org.apache.phoenix.schema.AmbiguousColumnException;
-import org.apache.phoenix.schema.types.PChar;
-import org.apache.phoenix.schema.types.PInteger;
-import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.PropertiesUtil;
-import org.apache.phoenix.util.QueryUtil;
+import org.apache.phoenix.util.QueryBuilder;
import org.apache.phoenix.util.TestUtil;
import org.junit.Test;
+public class AggregateIT extends BaseAggregateIT {
-public class AggregateIT extends ParallelStatsDisabledIT {
- private static void initData(Connection conn, String tableName) throws SQLException {
- conn.createStatement().execute("create table " + tableName +
- " (id varchar not null primary key,\n" +
- " uri varchar, appcpu integer)");
- insertRow(conn, tableName, "Report1", 10, 1);
- insertRow(conn, tableName, "Report2", 10, 2);
- insertRow(conn, tableName, "Report3", 30, 3);
- insertRow(conn, tableName, "Report4", 30, 4);
- insertRow(conn, tableName, "SOQL1", 10, 5);
- insertRow(conn, tableName, "SOQL2", 10, 6);
- insertRow(conn, tableName, "SOQL3", 30, 7);
- insertRow(conn, tableName, "SOQL4", 30, 8);
- conn.commit();
- }
-
- private static void insertRow(Connection conn, String tableName, String uri, int appcpu, int id) throws SQLException {
- PreparedStatement statement = conn.prepareStatement("UPSERT INTO " + tableName + "(id, uri, appcpu) values (?,?,?)");
- statement.setString(1, "id" + id);
- statement.setString(2, uri);
- statement.setInt(3, appcpu);
- statement.executeUpdate();
- }
-
- @Test
- public void testDuplicateTrailingAggExpr() throws Exception {
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = DriverManager.getConnection(getUrl(), props);
- String tableName = generateUniqueName();
- conn.createStatement().execute("create table " + tableName +
- " (nam VARCHAR(20), address VARCHAR(20), id BIGINT "
- + "constraint my_pk primary key (id))");
- PreparedStatement statement = conn.prepareStatement("UPSERT INTO " + tableName + "(nam, address, id) values (?,?,?)");
- statement.setString(1, "pulkit");
- statement.setString(2, "badaun");
- statement.setInt(3, 1);
- statement.executeUpdate();
- conn.commit();
- Statement stmt = conn.createStatement();
- ResultSet rs = stmt.executeQuery("select distinct 'harshit' as \"test_column\", trim(nam), trim(nam) from " + tableName);
- assertTrue(rs.next());
- assertEquals("harshit", rs.getString(1));
- assertEquals("pulkit", rs.getString(2));
- assertEquals("pulkit", rs.getString(3));
- conn.close();
- }
-
- @Test
- public void testExpressionInGroupBy() throws Exception {
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = DriverManager.getConnection(getUrl(), props);
- String tableName = generateUniqueName();
- String ddl = " create table " + tableName + "(tgb_id integer NOT NULL,utc_date_epoch integer NOT NULL,tgb_name varchar(40),ack_success_count integer" +
- ",ack_success_one_ack_count integer, CONSTRAINT pk_tgb_counter PRIMARY KEY(tgb_id, utc_date_epoch))";
- String query = "SELECT tgb_id, tgb_name, (utc_date_epoch/10)*10 AS utc_epoch_hour,SUM(ack_success_count + ack_success_one_ack_count) AS ack_tx_sum" +
- " FROM " + tableName + " GROUP BY tgb_id, tgb_name, utc_epoch_hour";
-
- createTestTable(getUrl(), ddl);
- String dml = "UPSERT INTO " + tableName + " VALUES(?,?,?,?,?)";
- PreparedStatement stmt = conn.prepareStatement(dml);
- stmt.setInt(1, 1);
- stmt.setInt(2, 1000);
- stmt.setString(3, "aaa");
- stmt.setInt(4, 1);
- stmt.setInt(5, 1);
- stmt.execute();
- stmt.setInt(1, 2);
- stmt.setInt(2, 2000);
- stmt.setString(3, "bbb");
- stmt.setInt(4, 2);
- stmt.setInt(5, 2);
- stmt.execute();
- conn.commit();
-
- ResultSet rs = conn.createStatement().executeQuery(query);
- assertTrue(rs.next());
- assertEquals(1,rs.getInt(1));
- assertEquals("aaa",rs.getString(2));
- assertEquals(1000,rs.getInt(3));
- assertEquals(2,rs.getInt(4));
- assertTrue(rs.next());
- assertEquals(2,rs.getInt(1));
- assertEquals("bbb",rs.getString(2));
- assertEquals(2000,rs.getInt(3));
- assertEquals(4,rs.getInt(4));
- assertFalse(rs.next());
- rs.close();
- conn.close();
- }
-
- @Test
- public void testBooleanInGroupBy() throws Exception {
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = DriverManager.getConnection(getUrl(), props);
- String tableName = generateUniqueName();
- String ddl = " create table " + tableName + "(id varchar primary key,v1 boolean, v2 integer, v3 integer)";
-
- createTestTable(getUrl(), ddl);
- PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + tableName + "(id,v2,v3) VALUES(?,?,?)");
- stmt.setString(1, "a");
- stmt.setInt(2, 1);
- stmt.setInt(3, 1);
- stmt.execute();
- stmt.close();
- stmt = conn.prepareStatement("UPSERT INTO " + tableName + " VALUES(?,?,?,?)");
- stmt.setString(1, "b");
- stmt.setBoolean(2, false);
- stmt.setInt(3, 2);
- stmt.setInt(4, 2);
- stmt.execute();
- stmt.setString(1, "c");
- stmt.setBoolean(2, true);
- stmt.setInt(3, 3);
- stmt.setInt(4, 3);
- stmt.execute();
- conn.commit();
-
- String[] gbs = {"v1,v2,v3","v1,v3,v2","v2,v1,v3"};
- for (String gb : gbs) {
- ResultSet rs = conn.createStatement().executeQuery("SELECT v1, v2, v3 from " + tableName + " group by " + gb);
- assertTrue(rs.next());
- assertEquals(false,rs.getBoolean("v1"));
- assertTrue(rs.wasNull());
- assertEquals(1,rs.getInt("v2"));
- assertEquals(1,rs.getInt("v3"));
- assertTrue(rs.next());
- assertEquals(false,rs.getBoolean("v1"));
- assertFalse(rs.wasNull());
- assertEquals(2,rs.getInt("v2"));
- assertEquals(2,rs.getInt("v3"));
- assertTrue(rs.next());
- assertEquals(true,rs.getBoolean("v1"));
- assertEquals(3,rs.getInt("v2"));
- assertEquals(3,rs.getInt("v3"));
- assertFalse(rs.next());
- rs.close();
- }
- conn.close();
- }
-
- @Test
- public void testScanUri() throws Exception {
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = DriverManager.getConnection(getUrl(), props);
- String tableName = generateUniqueName();
- initData(conn, tableName);
- Statement stmt = conn.createStatement();
- ResultSet rs = stmt.executeQuery("select uri from " + tableName);
- assertTrue(rs.next());
- assertEquals("Report1", rs.getString(1));
- assertTrue(rs.next());
- assertEquals("Report2", rs.getString(1));
- assertTrue(rs.next());
- assertEquals("Report3", rs.getString(1));
- assertTrue(rs.next());
- assertEquals("Report4", rs.getString(1));
- assertTrue(rs.next());
- assertEquals("SOQL1", rs.getString(1));
- assertTrue(rs.next());
- assertEquals("SOQL2", rs.getString(1));
- assertTrue(rs.next());
- assertEquals("SOQL3", rs.getString(1));
- assertTrue(rs.next());
- assertEquals("SOQL4", rs.getString(1));
- assertFalse(rs.next());
- conn.close();
- }
-
- @Test
- public void testCount() throws Exception {
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = DriverManager.getConnection(getUrl(), props);
- String tableName = generateUniqueName();
- initData(conn, tableName);
- Statement stmt = conn.createStatement();
- ResultSet rs = stmt.executeQuery("select count(1) from " + tableName);
- assertTrue(rs.next());
- assertEquals(8, rs.getInt(1));
- assertFalse(rs.next());
- conn.close();
- }
-
- @Test
- public void testGroupByCase() throws Exception {
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = DriverManager.getConnection(getUrl(), props);
- String tableName = generateUniqueName();
- String groupBy1 = "select " +
- "case when uri LIKE 'Report%' then 'Reports' else 'Other' END category" +
- ", avg(appcpu) from " + tableName +
- " group by category";
-
- String groupBy2 = "select " +
- "case uri when 'Report%' then 'Reports' else 'Other' END category" +
- ", avg(appcpu) from " + tableName +
- " group by appcpu, category";
-
- String groupBy3 = "select " +
- "case uri when 'Report%' then 'Reports' else 'Other' END category" +
- ", avg(appcpu) from " + tableName +
- " group by avg(appcpu), category";
- initData(conn, tableName);
- conn.createStatement().executeQuery(groupBy1);
- conn.createStatement().executeQuery(groupBy2);
- // TODO: validate query results
- try {
- conn.createStatement().executeQuery(groupBy3);
- fail();
- } catch (SQLException e) {
- assertTrue(e.getMessage().contains("Aggregate expressions may not be used in GROUP BY"));
- }
- conn.close();
- }
-
-
- @Test
- public void testGroupByArray() throws Exception {
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = DriverManager.getConnection(getUrl(), props);
-
- String tableName = generateUniqueName();
- conn.createStatement().execute("CREATE TABLE " + tableName + "(\n" +
- " a VARCHAR NOT NULL,\n" +
- " b VARCHAR,\n" +
- " c INTEGER,\n" +
- " d VARCHAR,\n" +
- " e VARCHAR ARRAY,\n" +
- " f BIGINT,\n" +
- " g BIGINT,\n" +
- " CONSTRAINT pk PRIMARY KEY(a)\n" +
- ")");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('1', 'val', 100, 'a', ARRAY ['b'], 1, 2)");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('2', 'val', 100, 'a', ARRAY ['b'], 3, 4)");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('3', 'val', 100, 'a', ARRAY ['b','c'], 5, 6)");
- conn.commit();
-
- ResultSet rs = conn.createStatement().executeQuery("SELECT c, SUM(f + g) AS sumone, d, e\n" +
- "FROM " + tableName + "\n" +
- "WHERE b = 'val'\n" +
- " AND a IN ('1','2','3')\n" +
- "GROUP BY c, d, e\n" +
- "ORDER BY sumone DESC");
- assertTrue(rs.next());
- assertEquals(100, rs.getInt(1));
- assertEquals(11, rs.getLong(2));
- assertTrue(rs.next());
- assertEquals(100, rs.getInt(1));
- assertEquals(10, rs.getLong(2));
- assertFalse(rs.next());
- conn.close();
- }
-
- @Test
- public void testGroupByOrderPreserving() throws Exception {
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = DriverManager.getConnection(getUrl(), props);
- String tableName = generateUniqueName();
-
- conn.createStatement().execute("CREATE TABLE " + tableName + "(ORGANIZATION_ID char(15) not null, \n" +
- "JOURNEY_ID char(15) not null, \n" +
- "DATASOURCE SMALLINT not null, \n" +
- "MATCH_STATUS TINYINT not null, \n" +
- "EXTERNAL_DATASOURCE_KEY varchar(30), \n" +
- "ENTITY_ID char(15) not null, \n" +
- "CONSTRAINT PK PRIMARY KEY (\n" +
- " ORGANIZATION_ID, \n" +
- " JOURNEY_ID, \n" +
- " DATASOURCE, \n" +
- " MATCH_STATUS,\n" +
- " EXTERNAL_DATASOURCE_KEY,\n" +
- " ENTITY_ID))");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('000001111122222', '333334444455555', 0, 0, 'abc', '666667777788888')");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('000001111122222', '333334444455555', 0, 0, 'abcd', '666667777788889')");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('000001111122222', '333334444455555', 0, 0, 'abc', '666667777788899')");
- conn.commit();
- String query =
- "SELECT COUNT(1), EXTERNAL_DATASOURCE_KEY As DUP_COUNT\n" +
- " FROM " + tableName + " \n" +
- " WHERE JOURNEY_ID='333334444455555' AND \n" +
- " DATASOURCE=0 AND MATCH_STATUS <= 1 and \n" +
- " ORGANIZATION_ID='000001111122222' \n" +
- " GROUP BY MATCH_STATUS, EXTERNAL_DATASOURCE_KEY \n" +
- " HAVING COUNT(1) > 1";
- ResultSet rs = conn.createStatement().executeQuery(query);
- assertTrue(rs.next());
- assertEquals(2,rs.getInt(1));
- assertEquals("abc", rs.getString(2));
- assertFalse(rs.next());
-
- rs = conn.createStatement().executeQuery("EXPLAIN " + query);
- assertEquals(
- "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + tableName + " ['000001111122222','333334444455555',0,*] - ['000001111122222','333334444455555',0,1]\n" +
- " SERVER FILTER BY FIRST KEY ONLY\n" +
- " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [MATCH_STATUS, EXTERNAL_DATASOURCE_KEY]\n" +
- "CLIENT FILTER BY COUNT(1) > 1",QueryUtil.getExplainPlan(rs));
- }
-
- @Test
- public void testGroupByOrderPreservingDescSort() throws Exception {
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = DriverManager.getConnection(getUrl(), props);
- String tableName = generateUniqueName();
- conn.createStatement().execute("CREATE TABLE " + tableName + " (k1 char(1) not null, k2 char(1) not null, constraint pk primary key (k1,k2)) split on ('ac','jc','nc')");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 'a')");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 'b')");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 'c')");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 'd')");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 'a')");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 'b')");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 'c')");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 'd')");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 'a')");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 'b')");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 'c')");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 'd')");
- conn.commit();
- String query = "SELECT k1,count(*) FROM " + tableName + " GROUP BY k1 ORDER BY k1 DESC";
- ResultSet rs = conn.createStatement().executeQuery(query);
- assertTrue(rs.next());
- assertEquals("n", rs.getString(1));
- assertEquals(4, rs.getInt(2));
- assertTrue(rs.next());
- assertEquals("j", rs.getString(1));
- assertEquals(4, rs.getInt(2));
- assertTrue(rs.next());
- assertEquals("a", rs.getString(1));
- assertEquals(4, rs.getInt(2));
- assertFalse(rs.next());
- rs = conn.createStatement().executeQuery("EXPLAIN " + query);
- assertEquals(
- "CLIENT PARALLEL 1-WAY REVERSE FULL SCAN OVER " + tableName + "\n" +
- " SERVER FILTER BY FIRST KEY ONLY\n" +
- " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]", QueryUtil.getExplainPlan(rs));
- }
-
- @Test
- public void testSumGroupByOrderPreservingDesc() throws Exception {
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = DriverManager.getConnection(getUrl(), props);
- String tableName = generateUniqueName();
-
- PreparedStatement stmt = conn.prepareStatement("CREATE TABLE " + tableName + " (k1 char(1) not null, k2 integer not null, constraint pk primary key (k1,k2)) split on (?,?,?)");
- stmt.setBytes(1, ByteUtil.concat(PChar.INSTANCE.toBytes("a"), PInteger.INSTANCE.toBytes(3)));
- stmt.setBytes(2, ByteUtil.concat(PChar.INSTANCE.toBytes("j"), PInteger.INSTANCE.toBytes(3)));
- stmt.setBytes(3, ByteUtil.concat(PChar.INSTANCE.toBytes("n"), PInteger.INSTANCE.toBytes(3)));
- stmt.execute();
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 1)");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 2)");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 3)");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 4)");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('b', 5)");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 1)");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 2)");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 3)");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 4)");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 1)");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 2)");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 3)");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 4)");
- conn.commit();
- String query = "SELECT k1,sum(k2) FROM " + tableName + " GROUP BY k1 ORDER BY k1 DESC";
- ResultSet rs = conn.createStatement().executeQuery(query);
- assertTrue(rs.next());
- assertEquals("n", rs.getString(1));
- assertEquals(10, rs.getInt(2));
- assertTrue(rs.next());
- assertEquals("j", rs.getString(1));
- assertEquals(10, rs.getInt(2));
- assertTrue(rs.next());
- assertEquals("b", rs.getString(1));
- assertEquals(5, rs.getInt(2));
- assertTrue(rs.next());
- assertEquals("a", rs.getString(1));
- assertEquals(10, rs.getInt(2));
- assertFalse(rs.next());
- rs = conn.createStatement().executeQuery("EXPLAIN " + query);
- assertEquals(
- "CLIENT PARALLEL 1-WAY REVERSE FULL SCAN OVER " + tableName + "\n" +
- " SERVER FILTER BY FIRST KEY ONLY\n" +
- " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]", QueryUtil.getExplainPlan(rs));
- }
-
- @Test
- public void testAvgGroupByOrderPreservingWithStats() throws Exception {
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = DriverManager.getConnection(getUrl(), props);
- String tableName = generateUniqueName();
- ResultSet rs = conn.createStatement().executeQuery("SELECT COUNT(*) FROM " + "\"SYSTEM\".\"STATS\"" + " WHERE " + PhoenixDatabaseMetaData.PHYSICAL_NAME + " ='" + tableName + "'");
- assertTrue(rs.next());
- assertEquals(0,rs.getInt(1));
- initAvgGroupTable(conn, tableName, PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + "=20 ");
- testAvgGroupByOrderPreserving(conn, tableName, 13);
- rs = conn.createStatement().executeQuery("SELECT COUNT(*) FROM " + "\"SYSTEM\".\"STATS\"" + " WHERE " + PhoenixDatabaseMetaData.PHYSICAL_NAME + " ='" + tableName + "'");
- assertTrue(rs.next());
- assertEquals(13,rs.getInt(1));
- conn.setAutoCommit(true);
- conn.createStatement().execute("DELETE FROM " + "\"SYSTEM\".\"STATS\"");
- rs = conn.createStatement().executeQuery("SELECT COUNT(*) FROM " + "\"SYSTEM\".\"STATS\"" + " WHERE " + PhoenixDatabaseMetaData.PHYSICAL_NAME + " ='" + tableName + "'");
- assertTrue(rs.next());
- assertEquals(0,rs.getInt(1));
- TestUtil.doMajorCompaction(conn, tableName);
- rs = conn.createStatement().executeQuery("SELECT COUNT(*) FROM " + "\"SYSTEM\".\"STATS\"" + " WHERE " + PhoenixDatabaseMetaData.PHYSICAL_NAME + " ='" + tableName + "'");
- assertTrue(rs.next());
- assertEquals(13,rs.getInt(1));
- testAvgGroupByOrderPreserving(conn, tableName, 13);
- conn.createStatement().execute("ALTER TABLE " + tableName + " SET " + PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + "=100");
- testAvgGroupByOrderPreserving(conn, tableName, 6);
- conn.createStatement().execute("ALTER TABLE " + tableName + " SET " + PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + "=null");
- testAvgGroupByOrderPreserving(conn, tableName, 4);
- }
-
- @Test
- public void testAvgGroupByOrderPreservingWithNoStats() throws Exception {
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = DriverManager.getConnection(getUrl(), props);
- String tableName = generateUniqueName();
- initAvgGroupTable(conn, tableName, "");
- testAvgGroupByOrderPreserving(conn, tableName, 4);
- }
-
- private void initAvgGroupTable(Connection conn, String tableName, String tableProps) throws SQLException {
- PreparedStatement stmt = conn.prepareStatement("CREATE TABLE " + tableName + " (k1 char(1) not null, k2 integer not null, constraint pk primary key (k1,k2)) " + tableProps + " split on (?,?,?)");
- stmt.setBytes(1, ByteUtil.concat(PChar.INSTANCE.toBytes("a"), PInteger.INSTANCE.toBytes(3)));
- stmt.setBytes(2, ByteUtil.concat(PChar.INSTANCE.toBytes("j"), PInteger.INSTANCE.toBytes(3)));
- stmt.setBytes(3, ByteUtil.concat(PChar.INSTANCE.toBytes("n"), PInteger.INSTANCE.toBytes(3)));
- stmt.execute();
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 1)");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 2)");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 3)");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 6)");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('b', 5)");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 1)");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 2)");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 3)");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 10)");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 1)");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 2)");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 3)");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 2)");
- conn.commit();
- }
-
- private void testAvgGroupByOrderPreserving(Connection conn, String tableName, int nGuidePosts) throws SQLException, IOException {
- String query = "SELECT k1,avg(k2) FROM " + tableName + " GROUP BY k1";
- ResultSet rs = conn.createStatement().executeQuery(query);
- assertTrue(rs.next());
- assertEquals("a", rs.getString(1));
- assertEquals(3, rs.getInt(2));
- assertTrue(rs.next());
- assertEquals("b", rs.getString(1));
- assertEquals(5, rs.getInt(2));
- assertTrue(rs.next());
- assertEquals("j", rs.getString(1));
- assertEquals(4, rs.getInt(2));
- assertTrue(rs.next());
- assertEquals("n", rs.getString(1));
- assertEquals(2, rs.getInt(2));
- assertFalse(rs.next());
- rs = conn.createStatement().executeQuery("EXPLAIN " + query);
- assertEquals(
- "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + tableName + "\n" +
- " SERVER FILTER BY FIRST KEY ONLY\n" +
- " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]", QueryUtil.getExplainPlan(rs));
- TestUtil.analyzeTable(conn, tableName);
- List<KeyRange> splits = TestUtil.getAllSplits(conn, tableName);
- assertEquals(nGuidePosts, splits.size());
- }
-
@Test
public void testGroupByWithAliasWithSameColumnName() throws SQLException {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
@@ -557,344 +79,6 @@ public class AggregateIT extends ParallelStatsDisabledIT {
}
@Test
- public void testDistinctGroupByBug3452WithoutMultiTenant() throws Exception {
- doTestDistinctGroupByBug3452("");
- }
-
- @Test
- public void testDistinctGroupByBug3452WithMultiTenant() throws Exception {
- doTestDistinctGroupByBug3452("VERSIONS=1, MULTI_TENANT=TRUE, REPLICATION_SCOPE=1, TTL=31536000");
- }
-
- private void doTestDistinctGroupByBug3452(String options) throws Exception {
- Connection conn=null;
- try {
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- conn = DriverManager.getConnection(getUrl(), props);
-
- String tableName=generateUniqueName();
- conn.createStatement().execute("DROP TABLE if exists "+tableName);
- String sql="CREATE TABLE "+ tableName +" ( "+
- "ORGANIZATION_ID CHAR(15) NOT NULL,"+
- "CONTAINER_ID CHAR(15) NOT NULL,"+
- "ENTITY_ID CHAR(15) NOT NULL,"+
- "SCORE DOUBLE,"+
- "CONSTRAINT TEST_PK PRIMARY KEY ( "+
- "ORGANIZATION_ID,"+
- "CONTAINER_ID,"+
- "ENTITY_ID"+
- ")) "+options;
- conn.createStatement().execute(sql);
-
- String indexTableName=generateUniqueName();
- conn.createStatement().execute("DROP INDEX IF EXISTS "+indexTableName+" ON "+tableName);
- conn.createStatement().execute("CREATE INDEX "+indexTableName+" ON "+tableName+" (CONTAINER_ID, SCORE DESC, ENTITY_ID DESC)");
-
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('org1','container1','entityId6',1.1)");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('org1','container1','entityId5',1.2)");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('org1','container1','entityId4',1.3)");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('org1','container1','entityId3',1.4)");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('org1','container1','entityId2',1.5)");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('org1','container1','entityId1',1.6)");
- conn.commit();
-
- sql="SELECT DISTINCT entity_id,score FROM "+tableName+" WHERE organization_id = 'org1' AND container_id = 'container1' ORDER BY score DESC";
- ResultSet rs=conn.createStatement().executeQuery(sql);
- assertTrue(rs.next());
- assertTrue(rs.getString(1).equals("entityId1"));
- assertEquals(rs.getDouble(2),1.6,0.0001);
-
- assertTrue(rs.next());
- assertTrue(rs.getString(1).equals("entityId2"));
- assertEquals(rs.getDouble(2),1.5,0.0001);
-
- assertTrue(rs.next());
- assertTrue(rs.getString(1).equals("entityId3"));
- assertEquals(rs.getDouble(2),1.4,0.0001);
-
- assertTrue(rs.next());
- assertTrue(rs.getString(1).equals("entityId4"));
- assertEquals(rs.getDouble(2),1.3,0.0001);
-
- assertTrue(rs.next());
- assertTrue(rs.getString(1).equals("entityId5"));
- assertEquals(rs.getDouble(2),1.2,0.0001);
-
- assertTrue(rs.next());
- assertTrue(rs.getString(1).equals("entityId6"));
- assertEquals(rs.getDouble(2),1.1,0.0001);
- assertTrue(!rs.next());
- } finally {
- if(conn!=null) {
- conn.close();
- }
- }
- }
-
- @Test
- public void testGroupByOrderByDescBug3451() throws Exception {
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
- String tableName=generateUniqueName();
- String sql="CREATE TABLE " + tableName + " (\n" +
- " ORGANIZATION_ID CHAR(15) NOT NULL,\n" +
- " CONTAINER_ID CHAR(15) NOT NULL,\n" +
- " ENTITY_ID CHAR(15) NOT NULL,\n" +
- " SCORE DOUBLE,\n" +
- " CONSTRAINT TEST_PK PRIMARY KEY (\n" +
- " ORGANIZATION_ID,\n" +
- " CONTAINER_ID,\n" +
- " ENTITY_ID\n" +
- " )\n" +
- " )";
- conn.createStatement().execute(sql);
- String indexName=generateUniqueName();
- conn.createStatement().execute("CREATE INDEX " + indexName + " ON " + tableName + "(ORGANIZATION_ID,CONTAINER_ID, SCORE DESC, ENTITY_ID DESC)");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES ('org2','container2','entityId6',1.1)");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES ('org2','container1','entityId5',1.2)");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES ('org2','container2','entityId4',1.3)");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES ('org2','container1','entityId5',1.2)");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES ('org2','container1','entityId3',1.4)");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES ('org2','container3','entityId7',1.35)");
- conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES ('org2','container3','entityId8',1.45)");
- conn.commit();
- String query = "SELECT DISTINCT entity_id, score\n" +
- " FROM " + tableName + "\n" +
- " WHERE organization_id = 'org2'\n" +
- " AND container_id IN ( 'container1','container2','container3' )\n" +
- " ORDER BY score DESC\n" +
- " LIMIT 2";
- Statement stmt = conn.createStatement();
- ResultSet rs = stmt.executeQuery(query);
- QueryPlan plan = stmt.unwrap(PhoenixStatement.class).getQueryPlan();
- assertEquals(indexName, plan.getContext().getCurrentTable().getTable().getName().getString());
- assertFalse(plan.getOrderBy().getOrderByExpressions().isEmpty());
- assertTrue(rs.next());
- assertEquals("entityId8", rs.getString(1));
- assertEquals(1.45, rs.getDouble(2),0.001);
- assertTrue(rs.next());
- assertEquals("entityId3", rs.getString(1));
- assertEquals(1.4, rs.getDouble(2),0.001);
- assertFalse(rs.next());
- }
- }
-
- @Test
- public void testGroupByDescColumnWithNullsLastBug3452() throws Exception {
-
- Connection conn=null;
- try
- {
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- conn = DriverManager.getConnection(getUrl(), props);
-
- String tableName=generateUniqueName();
- String sql="CREATE TABLE "+tableName+" ( "+
- "ORGANIZATION_ID VARCHAR,"+
- "CONTAINER_ID VARCHAR,"+
- "ENTITY_ID VARCHAR NOT NULL,"+
- "CONSTRAINT TEST_PK PRIMARY KEY ( "+
- "ORGANIZATION_ID DESC,"+
- "CONTAINER_ID DESC,"+
- "ENTITY_ID"+
- "))";
- conn.createStatement().execute(sql);
-
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('a',null,'11')");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,'2','22')");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('c','3','33')");
- conn.commit();
-
- //-----ORGANIZATION_ID
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID ASC NULLS FIRST";
- ResultSet rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"2",null},{null,"a"},{"3","c"},});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID ASC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{"2",null}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID DESC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,"a"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID DESC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{"2",null}});
-
- //----CONTAINER_ID
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID ASC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,"a"},{"2",null},{"3","c"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID ASC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,"a"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID DESC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{"2",null}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID DESC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,"a"}});
-
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,null,'44')");
- conn.commit();
-
- //-----ORGANIZATION_ID ASC CONTAINER_ID ASC
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID NULLS FIRST,CONTAINER_ID NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,null},{"2",null},{null,"a"},{"3","c"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID NULLS FIRST,CONTAINER_ID NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"2",null},{null,null},{null,"a"},{"3","c"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID NULLS LAST,CONTAINER_ID NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{null,null},{"2",null}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID NULLS LAST,CONTAINER_ID NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{"2",null},{null,null}});
-
- //-----ORGANIZATION_ID ASC CONTAINER_ID DESC
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID NULLS FIRST,CONTAINER_ID DESC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,null},{"2",null},{null,"a"},{"3","c"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID NULLS FIRST,CONTAINER_ID DESC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"2",null},{null,null},{null,"a"},{"3","c"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID NULLS LAST,CONTAINER_ID DESC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{null,null},{"2",null}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID NULLS LAST,CONTAINER_ID DESC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{"2",null},{null,null}});
-
- //-----ORGANIZATION_ID DESC CONTAINER_ID ASC
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,null},{"2",null},{"3","c"},{null,"a"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"2",null},{null,null},{"3","c"},{null,"a"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{null,null},{"2",null}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{"2",null},{null,null}});
-
- //-----ORGANIZATION_ID DESC CONTAINER_ID DESC
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID DESC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,null},{"2",null},{"3","c"},{null,"a"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID DESC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"2",null},{null,null},{"3","c"},{null,"a"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID DESC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{null,null},{"2",null}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID DESC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{"2",null},{null,null}});
-
- //-----CONTAINER_ID ASC ORGANIZATION_ID ASC
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID NULLS FIRST,ORGANIZATION_ID NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,null},{null,"a"},{"2",null},{"3","c"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID NULLS FIRST,ORGANIZATION_ID NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,"a"},{null,null},{"2",null},{"3","c"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID NULLS LAST,ORGANIZATION_ID NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,null},{null,"a"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID NULLS LAST,ORGANIZATION_ID NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,"a"},{null,null}});
-
- //-----CONTAINER_ID ASC ORGANIZATION_ID DESC
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID ASC NULLS FIRST,ORGANIZATION_ID DESC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,null},{null,"a"},{"2",null},{"3","c"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID ASC NULLS FIRST,ORGANIZATION_ID DESC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,"a"},{null,null},{"2",null},{"3","c"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID ASC NULLS LAST,ORGANIZATION_ID DESC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,null},{null,"a"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID ASC NULLS LAST,ORGANIZATION_ID DESC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,"a"},{null,null}});
-
- //-----CONTAINER_ID DESC ORGANIZATION_ID ASC
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID ASC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,null},{null,"a"},{"3","c"},{"2",null}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID ASC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,"a"},{null,null},{"3","c"},{"2",null}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID ASC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,null},{null,"a"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID ASC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,"a"},{null,null}});
-
- //-----CONTAINER_ID DESC ORGANIZATION_ID DESC
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID DESC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,null},{null,"a"},{"3","c"},{"2",null}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID DESC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{null,"a"},{null,null},{"3","c"},{"2",null}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID DESC NULLS FIRST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,null},{null,"a"}});
-
- sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID order by CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID DESC NULLS LAST";
- rs=conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,"a"},{null,null}});
- } finally {
- if(conn!=null) {
- conn.close();
- }
- }
- }
-
- @Test
public void testGroupByCoerceExpressionBug3453() throws Exception {
final Connection conn = DriverManager.getConnection(getUrl());
try {
@@ -955,16 +139,6 @@ public class AggregateIT extends ParallelStatsDisabledIT {
}
@Test
- public void testCountNullInEncodedNonEmptyKeyValueCF() throws Exception {
- testCountNullInNonEmptyKeyValueCF(1);
- }
-
- @Test
- public void testCountNullInNonEncodedNonEmptyKeyValueCF() throws Exception {
- testCountNullInNonEmptyKeyValueCF(0);
- }
-
- @Test
public void testNestedGroupedAggregationWithBigInt() throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
String tableName = generateUniqueName();
@@ -983,7 +157,41 @@ public class AggregateIT extends ParallelStatsDisabledIT {
}
}
- private void testCountNullInNonEmptyKeyValueCF(int columnEncodedBytes) throws Exception {
+ @Test
+ public void testAvgGroupByOrderPreservingWithStats() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectExpression("COUNT(*)")
+ .setFullTableName(PhoenixDatabaseMetaData.SYSTEM_STATS_NAME)
+ .setWhereClause(PhoenixDatabaseMetaData.PHYSICAL_NAME + " ='" + tableName + "'");
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals(0,rs.getInt(1));
+ initAvgGroupTable(conn, tableName, PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + "=20 ");
+ testAvgGroupByOrderPreserving(conn, tableName, 13);
+ rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals(13,rs.getInt(1));
+ conn.setAutoCommit(true);
+ conn.createStatement().execute("DELETE FROM " + "\"SYSTEM\".\"STATS\"");
+ rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals(0,rs.getInt(1));
+ TestUtil.doMajorCompaction(conn, tableName);
+ rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals(13,rs.getInt(1));
+ testAvgGroupByOrderPreserving(conn, tableName, 13);
+ conn.createStatement().execute("ALTER TABLE " + tableName + " SET " + PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + "=100");
+ testAvgGroupByOrderPreserving(conn, tableName, 6);
+ conn.createStatement().execute("ALTER TABLE " + tableName + " SET " + PhoenixDatabaseMetaData.GUIDE_POSTS_WIDTH + "=null");
+ testAvgGroupByOrderPreserving(conn, tableName, 4);
+ }
+
+ @Override
+ protected void testCountNullInNonEmptyKeyValueCF(int columnEncodedBytes) throws Exception {
try (Connection conn = DriverManager.getConnection(getUrl())) {
//Type is INT
String intTableName=generateUniqueName();
@@ -998,119 +206,26 @@ public class AggregateIT extends ParallelStatsDisabledIT {
conn.createStatement().execute("UPSERT INTO "+intTableName+" VALUES (5,1)");
conn.commit();
- TestUtil.dumpTable(conn.unwrap(PhoenixConnection.class).getQueryServices().getTable(Bytes.toBytes(intTableName)));
-
sql="select count(*) from "+intTableName;
- ResultSet rs=conn.createStatement().executeQuery(sql);
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectExpression("COUNT(*)")
+ .setFullTableName(intTableName);
+ ResultSet rs = executeQuery(conn, queryBuilder);
assertTrue(rs.next());
- assertEquals(5, rs.getInt(1));
-
+ assertEquals(5, rs.getLong(1));
+
sql="select count(*) from "+intTableName + " where b.colb is not null";
- rs=conn.createStatement().executeQuery(sql);
+ queryBuilder.setWhereClause("B.COLB IS NOT NULL");
+ rs = executeQuery(conn, queryBuilder);
assertTrue(rs.next());
- assertEquals(1, rs.getInt(1));
+ assertEquals(1, rs.getLong(1));
sql="select count(*) from "+intTableName + " where b.colb is null";
- rs=conn.createStatement().executeQuery(sql);
+ queryBuilder.setWhereClause("B.COLB IS NULL");
+ rs = executeQuery(conn, queryBuilder);
assertTrue(rs.next());
- assertEquals(4, rs.getInt(1));
- }
- }
-
- @Test
- public void testGroupByOrderMatchPkColumnOrderBug4690() throws Exception {
- this.doTestGroupByOrderMatchPkColumnOrderBug4690(false, false);
- this.doTestGroupByOrderMatchPkColumnOrderBug4690(false, true);
- this.doTestGroupByOrderMatchPkColumnOrderBug4690(true, false);
- this.doTestGroupByOrderMatchPkColumnOrderBug4690(true, true);
- }
-
- private void doTestGroupByOrderMatchPkColumnOrderBug4690(boolean desc ,boolean salted) throws Exception {
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = null;
- try {
- conn = DriverManager.getConnection(getUrl(), props);
- String tableName = generateUniqueName();
- String sql = "create table " + tableName + "( "+
- " pk1 integer not null , " +
- " pk2 integer not null, " +
- " pk3 integer not null," +
- " pk4 integer not null,"+
- " v integer, " +
- " CONSTRAINT TEST_PK PRIMARY KEY ( "+
- "pk1 "+(desc ? "desc" : "")+", "+
- "pk2 "+(desc ? "desc" : "")+", "+
- "pk3 "+(desc ? "desc" : "")+", "+
- "pk4 "+(desc ? "desc" : "")+
- " )) "+(salted ? "SALT_BUCKETS =4" : "split on(2)");
- conn.createStatement().execute(sql);
-
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,8,10,20,30)");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,8,11,21,31)");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,9,5 ,22,32)");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,9,6 ,12,33)");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,9,6 ,13,34)");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,9,7 ,8,35)");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (2,3,15,25,35)");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (2,7,16,26,36)");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (2,7,17,27,37)");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (3,2,18,28,38)");
- conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (3,2,19,29,39)");
- conn.commit();
-
- sql = "select pk2,pk1,count(v) from " + tableName + " group by pk2,pk1 order by pk2,pk1";
- ResultSet rs = conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{2,3,2L},{3,2,1L},{7,2,2L},{8,1,2L},{9,1,4L}});
-
- sql = "select pk1,pk2,count(v) from " + tableName + " group by pk2,pk1 order by pk1,pk2";
- rs = conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{1,8,2L},{1,9,4L},{2,3,1L},{2,7,2L},{3,2,2L}});
-
- sql = "select pk2,pk1,count(v) from " + tableName + " group by pk2,pk1 order by pk2 desc,pk1 desc";
- rs = conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{9,1,4L},{8,1,2L},{7,2,2L},{3,2,1L},{2,3,2L}});
-
- sql = "select pk1,pk2,count(v) from " + tableName + " group by pk2,pk1 order by pk1 desc,pk2 desc";
- rs = conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{3,2,2L},{2,7,2L},{2,3,1L},{1,9,4L},{1,8,2L}});
-
-
- sql = "select pk3,pk2,count(v) from " + tableName + " where pk1=1 group by pk3,pk2 order by pk3,pk2";
- rs = conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{5,9,1L},{6,9,2L},{7,9,1L},{10,8,1L},{11,8,1L}});
-
- sql = "select pk2,pk3,count(v) from " + tableName + " where pk1=1 group by pk3,pk2 order by pk2,pk3";
- rs = conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{8,10,1L},{8,11,1L},{9,5,1L},{9,6,2L},{9,7,1L}});
-
- sql = "select pk3,pk2,count(v) from " + tableName + " where pk1=1 group by pk3,pk2 order by pk3 desc,pk2 desc";
- rs = conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{11,8,1L},{10,8,1L},{7,9,1L},{6,9,2L},{5,9,1L}});
-
- sql = "select pk2,pk3,count(v) from " + tableName + " where pk1=1 group by pk3,pk2 order by pk2 desc,pk3 desc";
- rs = conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{9,7,1L},{9,6,2L},{9,5,1L},{8,11,1L},{8,10,1L}});
-
-
- sql = "select pk4,pk3,pk1,count(v) from " + tableName + " where pk2=9 group by pk4,pk3,pk1 order by pk4,pk3,pk1";
- rs = conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{8,7,1,1L},{12,6,1,1L},{13,6,1,1L},{22,5,1,1L}});
-
- sql = "select pk1,pk3,pk4,count(v) from " + tableName + " where pk2=9 group by pk4,pk3,pk1 order by pk1,pk3,pk4";
- rs = conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{1,5,22,1L},{1,6,12,1L},{1,6,13,1L},{1,7,8,1L}});
-
- sql = "select pk4,pk3,pk1,count(v) from " + tableName + " where pk2=9 group by pk4,pk3,pk1 order by pk4 desc,pk3 desc,pk1 desc";
- rs = conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{22,5,1,1L},{13,6,1,1L},{12,6,1,1L},{8,7,1,1L}});
-
- sql = "select pk1,pk3,pk4,count(v) from " + tableName + " where pk2=9 group by pk4,pk3,pk1 order by pk1 desc,pk3 desc,pk4 desc";
- rs = conn.prepareStatement(sql).executeQuery();
- assertResultSet(rs, new Object[][]{{1,7,8,1L},{1,6,13,1L},{1,6,12,1L},{1,5,22,1L}});
- } finally {
- if(conn != null) {
- conn.close();
- }
+ assertEquals(4, rs.getLong(1));
}
}
}
+
[19/22] phoenix git commit: PHOENIX-5012 Don't derive IndexToolIT
from ParallelStatsEnabled
Posted by ja...@apache.org.
PHOENIX-5012 Don't derive IndexToolIT from ParallelStatsEnabled
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/d1efd665
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/d1efd665
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/d1efd665
Branch: refs/heads/omid2
Commit: d1efd665fa84f611e1ad544ed0859f1fecb4f616
Parents: fd7ae94
Author: James Taylor <ja...@apache.org>
Authored: Sat Nov 10 11:04:48 2018 -0800
Committer: James Taylor <ja...@apache.org>
Committed: Sat Nov 10 11:04:48 2018 -0800
----------------------------------------------------------------------
.../src/it/java/org/apache/phoenix/end2end/IndexToolIT.java | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d1efd665/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
index c99f145..e096bb5 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexToolIT.java
@@ -58,7 +58,6 @@ import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.TestUtil;
import org.junit.BeforeClass;
import org.junit.Test;
-import org.junit.experimental.categories.Category;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
@@ -67,8 +66,7 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@RunWith(Parameterized.class)
-@Category(NeedsOwnMiniClusterTest.class)
-public class IndexToolIT extends ParallelStatsEnabledIT {
+public class IndexToolIT extends BaseUniqueNamesOwnClusterIT {
private final boolean localIndex;
private final boolean transactional;
@@ -99,9 +97,13 @@ public class IndexToolIT extends ParallelStatsEnabledIT {
@BeforeClass
public static void setup() throws Exception {
Map<String, String> serverProps = Maps.newHashMapWithExpectedSize(2);
+ serverProps.put(QueryServices.STATS_GUIDEPOST_WIDTH_BYTES_ATTRIB, Long.toString(20));
+ serverProps.put(QueryServices.MAX_SERVER_METADATA_CACHE_TIME_TO_LIVE_MS_ATTRIB, Long.toString(5));
serverProps.put(QueryServices.EXTRA_JDBC_ARGUMENTS_ATTRIB,
QueryServicesOptions.DEFAULT_EXTRA_JDBC_ARGUMENTS);
Map<String, String> clientProps = Maps.newHashMapWithExpectedSize(2);
+ clientProps.put(QueryServices.USE_STATS_FOR_PARALLELIZATION, Boolean.toString(true));
+ clientProps.put(QueryServices.STATS_UPDATE_FREQ_MS_ATTRIB, Long.toString(5));
clientProps.put(QueryServices.TRANSACTIONS_ENABLED, Boolean.TRUE.toString());
clientProps.put(QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB, Boolean.TRUE.toString());
setUpTestDriver(new ReadOnlyProps(serverProps.entrySet().iterator()),
[17/22] phoenix git commit: PHOENIX-4996: Refactor PTableImpl to use
Builder Pattern
Posted by ja...@apache.org.
PHOENIX-4996: Refactor PTableImpl to use Builder Pattern
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/d6083ae5
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/d6083ae5
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/d6083ae5
Branch: refs/heads/omid2
Commit: d6083ae5c598f5293adbfb24370b8e40962bc3d7
Parents: 9bfaf18
Author: Chinmay Kulkarni <ch...@gmail.com>
Authored: Fri Nov 2 14:00:09 2018 -0700
Committer: Thomas D'Silva <td...@apache.org>
Committed: Tue Nov 6 15:19:09 2018 -0800
----------------------------------------------------------------------
.../apache/phoenix/compile/DeleteCompiler.java | 5 +-
.../apache/phoenix/compile/FromCompiler.java | 66 +-
.../apache/phoenix/compile/JoinCompiler.java | 53 +-
.../compile/TupleProjectionCompiler.java | 60 +-
.../apache/phoenix/compile/UnionCompiler.java | 41 +-
.../apache/phoenix/compile/UpsertCompiler.java | 12 +-
.../coprocessor/MetaDataEndpointImpl.java | 96 +-
.../UngroupedAggregateRegionObserver.java | 6 +-
.../coprocessor/WhereConstantParser.java | 3 +-
.../query/ConnectionlessQueryServicesImpl.java | 9 +-
.../apache/phoenix/schema/MetaDataClient.java | 215 ++-
.../apache/phoenix/schema/PMetaDataImpl.java | 28 +-
.../org/apache/phoenix/schema/PTableImpl.java | 1259 +++++++++++-------
.../org/apache/phoenix/schema/TableRef.java | 17 +-
.../phoenix/execute/CorrelatePlanTest.java | 32 +-
.../execute/LiteralResultIteratorPlanTest.java | 33 +-
16 files changed, 1303 insertions(+), 632 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6083ae5/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
index 583085e..8c9a930 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/DeleteCompiler.java
@@ -89,7 +89,6 @@ import org.apache.phoenix.schema.types.PLong;
import org.apache.phoenix.transaction.PhoenixTransactionProvider.Feature;
import org.apache.phoenix.util.ByteUtil;
import org.apache.phoenix.util.IndexUtil;
-import org.apache.phoenix.util.MetaDataUtil;
import org.apache.phoenix.util.ScanUtil;
import com.google.common.base.Preconditions;
@@ -615,7 +614,9 @@ public class DeleteCompiler {
}
});
}
- PTable projectedTable = PTableImpl.makePTable(table, PTableType.PROJECTED, adjustedProjectedColumns);
+ PTable projectedTable = PTableImpl.builderWithColumns(table, adjustedProjectedColumns)
+ .setType(PTableType.PROJECTED)
+ .build();
final TableRef projectedTableRef = new TableRef(projectedTable, targetTableRef.getLowerBoundTimeStamp(), targetTableRef.getTimeStamp());
QueryPlan bestPlanToBe = dataPlan;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6083ae5/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
index efc66a9..2701af0 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/FromCompiler.java
@@ -32,8 +32,6 @@ import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.coprocessor.MetaDataProtocol;
import org.apache.phoenix.coprocessor.MetaDataProtocol.MetaDataMutationResult;
import org.apache.phoenix.coprocessor.MetaDataProtocol.MutationCode;
-import org.apache.phoenix.exception.SQLExceptionCode;
-import org.apache.phoenix.exception.SQLExceptionInfo;
import org.apache.phoenix.expression.Expression;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.parse.AliasedNode;
@@ -82,6 +80,7 @@ import org.apache.phoenix.schema.PTableImpl;
import org.apache.phoenix.schema.PTableKey;
import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.schema.ProjectedColumn;
+import org.apache.phoenix.schema.RowKeySchema;
import org.apache.phoenix.schema.SchemaNotFoundException;
import org.apache.phoenix.schema.SortOrder;
import org.apache.phoenix.schema.TableNotFoundException;
@@ -284,7 +283,8 @@ public class FromCompiler {
column.getTimestamp());
projectedColumns.add(projectedColumn);
}
- PTable t = PTableImpl.makePTable(table, projectedColumns);
+ PTable t = PTableImpl.builderWithColumns(table, projectedColumns)
+ .build();
return new SingleTableColumnResolver(connection, new TableRef(tableRef.getTableAlias(), t, tableRef.getLowerBoundTimeStamp(), tableRef.hasDynamicCols()));
}
@@ -369,10 +369,30 @@ public class FromCompiler {
if (connection.getSchema() != null) {
schema = schema != null ? schema : connection.getSchema();
}
+
// Storage scheme and encoding scheme don't matter here since the PTable is being used only for the purposes of create table.
// The actual values of these two will be determined by the metadata client.
- PTable theTable = new PTableImpl(connection.getTenantId(), schema, table.getName().getTableName(),
- scn == null ? HConstants.LATEST_TIMESTAMP : scn, families, isNamespaceMapped);
+ PName tenantId = connection.getTenantId();
+ PTableImpl.checkTenantId(tenantId);
+ String tableName = table.getName().getTableName();
+ PName name = PNameFactory.newName(SchemaUtil.getTableName(schema, tableName));
+ PTable theTable = new PTableImpl.Builder()
+ .setTenantId(tenantId)
+ .setName(name)
+ .setKey(new PTableKey(tenantId, name.getString()))
+ .setSchemaName(PNameFactory.newName(schema))
+ .setTableName(PNameFactory.newName(tableName))
+ .setType(PTableType.VIEW)
+ .setViewType(PTable.ViewType.MAPPED)
+ .setTimeStamp(scn == null ? HConstants.LATEST_TIMESTAMP : scn)
+ .setPkColumns(Collections.<PColumn>emptyList())
+ .setAllColumns(Collections.<PColumn>emptyList())
+ .setRowKeySchema(RowKeySchema.EMPTY_SCHEMA)
+ .setIndexes(Collections.<PTable>emptyList())
+ .setFamilyAttributes(families)
+ .setPhysicalNames(Collections.<PName>emptyList())
+ .setNamespaceMapped(isNamespaceMapped)
+ .build();
theTable = this.addDynamicColumns(table.getDynamicColumns(), theTable);
alias = null;
tableRefs = ImmutableList.of(new TableRef(alias, theTable, timeStamp, !table.getDynamicColumns().isEmpty()));
@@ -706,7 +726,7 @@ public class FromCompiler {
if (!dynColumns.isEmpty()) {
List<PColumn> allcolumns = new ArrayList<PColumn>();
List<PColumn> existingColumns = theTable.getColumns();
- // Need to skip the salting column, as it's added in the makePTable call below
+ // Need to skip the salting column, as it's handled in the PTable builder call below
allcolumns.addAll(theTable.getBucketNum() == null ? existingColumns : existingColumns.subList(1, existingColumns.size()));
// Position still based on with the salting columns
int position = existingColumns.size();
@@ -724,7 +744,8 @@ public class FromCompiler {
HConstants.LATEST_TIMESTAMP));
position++;
}
- theTable = PTableImpl.makePTable(theTable, allcolumns);
+ theTable = PTableImpl.builderWithColumns(theTable, allcolumns)
+ .build();
}
return theTable;
}
@@ -830,11 +851,32 @@ public class FromCompiler {
HConstants.LATEST_TIMESTAMP);
columns.add(column);
}
- PTable t = PTableImpl.makePTable(null, PName.EMPTY_NAME, PName.EMPTY_NAME, PTableType.SUBQUERY, null,
- MetaDataProtocol.MIN_TABLE_TIMESTAMP, PTable.INITIAL_SEQ_NUM, null, null, columns, null, null,
- Collections.<PTable> emptyList(), false, Collections.<PName> emptyList(), null, null, false, false,
- false, null, null, null, null, false, null, 0, 0L, SchemaUtil
- .isNamespaceMappingEnabled(PTableType.SUBQUERY, connection.getQueryServices().getProps()), null, false, ImmutableStorageScheme.ONE_CELL_PER_COLUMN, QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, PTable.EncodedCQCounter.NULL_COUNTER, true);
+ PTable t = new PTableImpl.Builder()
+ .setType(PTableType.SUBQUERY)
+ .setTimeStamp(MetaDataProtocol.MIN_TABLE_TIMESTAMP)
+ .setIndexDisableTimestamp(0L)
+ .setSequenceNumber(PTable.INITIAL_SEQ_NUM)
+ .setImmutableRows(false)
+ .setDisableWAL(false)
+ .setMultiTenant(false)
+ .setStoreNulls(false)
+ .setUpdateCacheFrequency(0)
+ .setNamespaceMapped(SchemaUtil.isNamespaceMappingEnabled(PTableType.SUBQUERY,
+ connection.getQueryServices().getProps()))
+ .setAppendOnlySchema(false)
+ .setImmutableStorageScheme(ImmutableStorageScheme.ONE_CELL_PER_COLUMN)
+ .setQualifierEncodingScheme(QualifierEncodingScheme.NON_ENCODED_QUALIFIERS)
+ .setBaseColumnCount(QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT)
+ .setEncodedCQCounter(PTable.EncodedCQCounter.NULL_COUNTER)
+ .setUseStatsForParallelization(true)
+ .setExcludedColumns(ImmutableList.<PColumn>of())
+ .setSchemaName(PName.EMPTY_NAME)
+ .setTableName(PName.EMPTY_NAME)
+ .setRowKeyOrderOptimizable(false)
+ .setIndexes(Collections.<PTable>emptyList())
+ .setPhysicalNames(ImmutableList.<PName>of())
+ .setColumns(columns)
+ .build();
String alias = subselectNode.getAlias();
TableRef tableRef = new TableRef(alias, t, MetaDataProtocol.MIN_TABLE_TIMESTAMP, false);
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6083ae5/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java
index 880fa72..807e54b 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/JoinCompiler.java
@@ -17,6 +17,7 @@
*/
package org.apache.phoenix.compile;
+import static org.apache.phoenix.query.QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT;
import static org.apache.phoenix.schema.PTable.ImmutableStorageScheme.ONE_CELL_PER_COLUMN;
import static org.apache.phoenix.schema.PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS;
@@ -32,6 +33,7 @@ import java.util.ListIterator;
import java.util.Map;
import java.util.Set;
+import com.google.common.collect.ImmutableList;
import org.apache.hadoop.hbase.client.Scan;
import org.apache.hadoop.hbase.util.Pair;
import org.apache.phoenix.exception.SQLExceptionCode;
@@ -71,8 +73,8 @@ import org.apache.phoenix.schema.ColumnRef;
import org.apache.phoenix.schema.LocalIndexDataColumnRef;
import org.apache.phoenix.schema.MetaDataEntityNotFoundException;
import org.apache.phoenix.schema.PColumn;
-import org.apache.phoenix.schema.PName;
import org.apache.phoenix.schema.PNameFactory;
+import org.apache.phoenix.schema.PName;
import org.apache.phoenix.schema.PTable;
import org.apache.phoenix.schema.PTable.IndexType;
import org.apache.phoenix.schema.PTableImpl;
@@ -1270,17 +1272,44 @@ public class JoinCompiler {
if (left.getBucketNum() != null) {
merged.remove(0);
}
- return PTableImpl.makePTable(left.getTenantId(), left.getSchemaName(),
- PNameFactory.newName(SchemaUtil.getTableName(left.getName().getString(), right.getName().getString())),
- left.getType(), left.getIndexState(), left.getTimeStamp(), left.getSequenceNumber(), left.getPKName(),
- left.getBucketNum(), merged, left.getParentSchemaName(), left.getParentTableName(), left.getIndexes(),
- left.isImmutableRows(), Collections.<PName> emptyList(), null, null, PTable.DEFAULT_DISABLE_WAL,
- left.isMultiTenant(), left.getStoreNulls(), left.getViewType(), left.getViewIndexType(), left.getViewIndexId(),
- left.getIndexType(), left.rowKeyOrderOptimizable(), left.getTransactionProvider(),
- left.getUpdateCacheFrequency(), left.getIndexDisableTimestamp(), left.isNamespaceMapped(),
- left.getAutoPartitionSeqName(), left.isAppendOnlySchema(), ONE_CELL_PER_COLUMN, NON_ENCODED_QUALIFIERS, PTable.EncodedCQCounter.NULL_COUNTER, left.useStatsForParallelization());
+ return new PTableImpl.Builder()
+ .setType(left.getType())
+ .setState(left.getIndexState())
+ .setTimeStamp(left.getTimeStamp())
+ .setIndexDisableTimestamp(left.getIndexDisableTimestamp())
+ .setSequenceNumber(left.getSequenceNumber())
+ .setImmutableRows(left.isImmutableRows())
+ .setDisableWAL(PTable.DEFAULT_DISABLE_WAL)
+ .setMultiTenant(left.isMultiTenant())
+ .setStoreNulls(left.getStoreNulls())
+ .setViewType(left.getViewType())
+ .setViewIndexType(left.getViewIndexType())
+ .setViewIndexId(left.getViewIndexId())
+ .setIndexType(left.getIndexType())
+ .setTransactionProvider(left.getTransactionProvider())
+ .setUpdateCacheFrequency(left.getUpdateCacheFrequency())
+ .setNamespaceMapped(left.isNamespaceMapped())
+ .setAutoPartitionSeqName(left.getAutoPartitionSeqName())
+ .setAppendOnlySchema(left.isAppendOnlySchema())
+ .setImmutableStorageScheme(ONE_CELL_PER_COLUMN)
+ .setQualifierEncodingScheme(NON_ENCODED_QUALIFIERS)
+ .setBaseColumnCount(BASE_TABLE_BASE_COLUMN_COUNT)
+ .setEncodedCQCounter(PTable.EncodedCQCounter.NULL_COUNTER)
+ .setUseStatsForParallelization(left.useStatsForParallelization())
+ .setExcludedColumns(ImmutableList.<PColumn>of())
+ .setTenantId(left.getTenantId())
+ .setSchemaName(left.getSchemaName())
+ .setTableName(PNameFactory.newName(SchemaUtil.getTableName(left.getName().getString(),
+ right.getName().getString())))
+ .setPkName(left.getPKName())
+ .setRowKeyOrderOptimizable(left.rowKeyOrderOptimizable())
+ .setBucketNum(left.getBucketNum())
+ .setIndexes(left.getIndexes() == null ? Collections.<PTable>emptyList() : left.getIndexes())
+ .setParentSchemaName(left.getParentSchemaName())
+ .setParentTableName(left.getParentTableName())
+ .setPhysicalNames(ImmutableList.<PName>of())
+ .setColumns(merged)
+ .build();
}
}
-
-
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6083ae5/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
index dc85bd3..5b92a5d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/TupleProjectionCompiler.java
@@ -17,6 +17,8 @@
*/
package org.apache.phoenix.compile;
import static org.apache.phoenix.query.QueryConstants.VALUE_COLUMN_FAMILY;
+import static org.apache.phoenix.query.QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT;
+import static org.apache.phoenix.schema.PTable.ImmutableStorageScheme;
import java.sql.SQLException;
import java.util.ArrayList;
@@ -26,6 +28,7 @@ import java.util.LinkedHashSet;
import java.util.List;
import java.util.Set;
+import com.google.common.collect.ImmutableList;
import org.apache.phoenix.parse.AliasedNode;
import org.apache.phoenix.parse.ColumnParseNode;
import org.apache.phoenix.parse.FamilyWildcardParseNode;
@@ -160,15 +163,13 @@ public class TupleProjectionCompiler {
sourceColumnRef.getColumn().isNullable(), sourceColumnRef, sourceColumnRef.getColumn().getColumnQualifierBytes());
projectedColumns.add(column);
}
-
- return PTableImpl.makePTable(table.getTenantId(), table.getSchemaName(), table.getTableName(),
- PTableType.PROJECTED, table.getIndexState(), table.getTimeStamp(), table.getSequenceNumber(),
- table.getPKName(), table.getBucketNum(), projectedColumns, table.getParentSchemaName(),
- table.getParentTableName(), table.getIndexes(), table.isImmutableRows(), Collections.<PName> emptyList(),
- table.getDefaultFamilyName(), table.getViewStatement(), table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(),
- table.getViewIndexType(), table.getViewIndexId(),
- table.getIndexType(), table.rowKeyOrderOptimizable(), table.getTransactionProvider(), table.getUpdateCacheFrequency(),
- table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.getEncodedCQCounter(), table.useStatsForParallelization());
+ return PTableImpl.builderWithColumns(table, projectedColumns)
+ .setType(PTableType.PROJECTED)
+ .setBaseColumnCount(BASE_TABLE_BASE_COLUMN_COUNT)
+ .setExcludedColumns(ImmutableList.<PColumn>of())
+ .setPhysicalNames(ImmutableList.<PName>of())
+ .setColumns(projectedColumns)
+ .build();
}
public static PTable createProjectedTable(TableRef tableRef, List<ColumnRef> sourceColumnRefs, boolean retainPKColumns) throws SQLException {
@@ -192,14 +193,39 @@ public class TupleProjectionCompiler {
if (EncodedColumnsUtil.usesEncodedColumnNames(table)) {
cqCounter = EncodedCQCounter.copy(table.getEncodedCQCounter());
}
-
- return PTableImpl.makePTable(table.getTenantId(), PROJECTED_TABLE_SCHEMA, table.getName(), PTableType.PROJECTED,
- null, table.getTimeStamp(), table.getSequenceNumber(), table.getPKName(),
- table.getBucketNum(), projectedColumns, null, null,
- Collections.<PTable> emptyList(), table.isImmutableRows(), Collections.<PName> emptyList(), null, null,
- table.isWALDisabled(), table.isMultiTenant(), table.getStoreNulls(), table.getViewType(),
- table.getViewIndexType(), table.getViewIndexId(), null, table.rowKeyOrderOptimizable(), table.getTransactionProvider(),
- table.getUpdateCacheFrequency(), table.getIndexDisableTimestamp(), table.isNamespaceMapped(), table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(), table.getEncodingScheme(), cqCounter, table.useStatsForParallelization());
+ return new PTableImpl.Builder()
+ .setType(PTableType.PROJECTED)
+ .setTimeStamp(table.getTimeStamp())
+ .setIndexDisableTimestamp(table.getIndexDisableTimestamp())
+ .setSequenceNumber(table.getSequenceNumber())
+ .setImmutableRows(table.isImmutableRows())
+ .setDisableWAL(table.isWALDisabled())
+ .setMultiTenant(table.isMultiTenant())
+ .setStoreNulls(table.getStoreNulls())
+ .setViewType(table.getViewType())
+ .setViewIndexType(table.getViewIndexType())
+ .setViewIndexId(table.getViewIndexId())
+ .setTransactionProvider(table.getTransactionProvider())
+ .setUpdateCacheFrequency(table.getUpdateCacheFrequency())
+ .setNamespaceMapped(table.isNamespaceMapped())
+ .setAutoPartitionSeqName(table.getAutoPartitionSeqName())
+ .setAppendOnlySchema(table.isAppendOnlySchema())
+ .setImmutableStorageScheme(table.getImmutableStorageScheme())
+ .setQualifierEncodingScheme(table.getEncodingScheme())
+ .setBaseColumnCount(BASE_TABLE_BASE_COLUMN_COUNT)
+ .setEncodedCQCounter(cqCounter)
+ .setUseStatsForParallelization(table.useStatsForParallelization())
+ .setExcludedColumns(ImmutableList.<PColumn>of())
+ .setTenantId(table.getTenantId())
+ .setSchemaName(PROJECTED_TABLE_SCHEMA)
+ .setTableName(table.getTableName())
+ .setPkName(table.getPKName())
+ .setRowKeyOrderOptimizable(table.rowKeyOrderOptimizable())
+ .setBucketNum(table.getBucketNum())
+ .setIndexes(Collections.<PTable>emptyList())
+ .setPhysicalNames(ImmutableList.<PName>of())
+ .setColumns(projectedColumns)
+ .build();
}
// For extracting column references from single select statement
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6083ae5/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java
index c0c63eb..a54fbdd 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/UnionCompiler.java
@@ -17,10 +17,14 @@
*/
package org.apache.phoenix.compile;
+import static org.apache.phoenix.query.QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT;
+
import java.sql.SQLException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
+import com.google.common.collect.ImmutableList;
import org.apache.hadoop.hbase.HConstants;
import org.apache.phoenix.exception.SQLExceptionCode;
import org.apache.phoenix.exception.SQLExceptionInfo;
@@ -96,15 +100,34 @@ public class UnionCompiler {
projectedColumns.add(projectedColumn);
}
Long scn = statement.getConnection().getSCN();
- PTable tempTable = PTableImpl.makePTable(statement.getConnection().getTenantId(),
- UNION_SCHEMA_NAME, UNION_TABLE_NAME, PTableType.SUBQUERY, null,
- HConstants.LATEST_TIMESTAMP, scn == null ? HConstants.LATEST_TIMESTAMP : scn,
- null, null, projectedColumns, null, null, null, true, null, null, null, true,
- true, true, null, null, null, null, false, null, 0, 0L,
- SchemaUtil.isNamespaceMappingEnabled(PTableType.SUBQUERY,
- statement.getConnection().getQueryServices().getProps()), null, false, ImmutableStorageScheme.ONE_CELL_PER_COLUMN, QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, PTable.EncodedCQCounter.NULL_COUNTER, true);
- TableRef tableRef = new TableRef(null, tempTable, 0, false);
- return tableRef;
+ PTable tempTable = new PTableImpl.Builder()
+ .setType(PTableType.SUBQUERY)
+ .setTimeStamp(HConstants.LATEST_TIMESTAMP)
+ .setIndexDisableTimestamp(0L)
+ .setSequenceNumber(scn == null ? HConstants.LATEST_TIMESTAMP : scn)
+ .setImmutableRows(true)
+ .setDisableWAL(true)
+ .setMultiTenant(true)
+ .setStoreNulls(true)
+ .setUpdateCacheFrequency(0)
+ .setNamespaceMapped(SchemaUtil.isNamespaceMappingEnabled(PTableType.SUBQUERY,
+ statement.getConnection().getQueryServices().getProps()))
+ .setAppendOnlySchema(false)
+ .setImmutableStorageScheme(ImmutableStorageScheme.ONE_CELL_PER_COLUMN)
+ .setQualifierEncodingScheme(QualifierEncodingScheme.NON_ENCODED_QUALIFIERS)
+ .setBaseColumnCount(BASE_TABLE_BASE_COLUMN_COUNT)
+ .setEncodedCQCounter(PTable.EncodedCQCounter.NULL_COUNTER)
+ .setUseStatsForParallelization(true)
+ .setExcludedColumns(ImmutableList.<PColumn>of())
+ .setTenantId(statement.getConnection().getTenantId())
+ .setSchemaName(UNION_SCHEMA_NAME)
+ .setTableName(UNION_TABLE_NAME)
+ .setRowKeyOrderOptimizable(false)
+ .setIndexes(Collections.<PTable>emptyList())
+ .setPhysicalNames(ImmutableList.<PName>of())
+ .setColumns(projectedColumns)
+ .build();
+ return new TableRef(null, tempTable, 0, false);
}
private static void compareExperssions(int i, Expression expression,
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6083ae5/phoenix-core/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java b/phoenix-core/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java
index 61be561..4ed0c9a 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/compile/UpsertCompiler.java
@@ -32,6 +32,7 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import com.google.common.collect.ImmutableList;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.client.Scan;
@@ -701,9 +702,11 @@ public class UpsertCompiler {
}
// Build table from projectedColumns
// Hack to add default column family to be used on server in case no value column is projected.
- PTable projectedTable = PTableImpl.makePTable(table, projectedColumns,
- PNameFactory.newName(SchemaUtil.getEmptyColumnFamily(table)));
-
+ PTable projectedTable = PTableImpl.builderWithColumns(table, projectedColumns)
+ .setExcludedColumns(ImmutableList.<PColumn>of())
+ .setDefaultFamilyName(PNameFactory.newName(SchemaUtil.getEmptyColumnFamily(table)))
+ .setColumns(projectedColumns)
+ .build();
SelectStatement select = SelectStatement.create(SelectStatement.COUNT_ONE, upsert.getHint());
StatementContext statementContext = queryPlan.getContext();
@@ -856,7 +859,8 @@ public class UpsertCompiler {
}
updateExpressions.add(updateExpression);
}
- PTable onDupKeyTable = PTableImpl.makePTable(table, updateColumns);
+ PTable onDupKeyTable = PTableImpl.builderWithColumns(table, updateColumns)
+ .build();
onDupKeyBytesToBe = PhoenixIndexBuilder.serializeOnDupKeyUpdate(onDupKeyTable, updateExpressions);
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6083ae5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index 424b6d6..d899e32 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@ -78,6 +78,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID_DATA
import static org.apache.phoenix.query.QueryConstants.DIVERGED_VIEW_BASE_COLUMN_COUNT;
import static org.apache.phoenix.schema.PTableType.INDEX;
import static org.apache.phoenix.schema.PTableType.TABLE;
+import static org.apache.phoenix.schema.PTableImpl.getColumnsToClone;
import static org.apache.phoenix.util.SchemaUtil.getVarCharLength;
import static org.apache.phoenix.util.SchemaUtil.getVarChars;
@@ -101,6 +102,7 @@ import java.util.NavigableMap;
import java.util.Properties;
import java.util.Set;
+import com.google.common.collect.ImmutableList;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.CellUtil;
@@ -215,6 +217,7 @@ import org.apache.phoenix.schema.PTableImpl;
import org.apache.phoenix.schema.PTableKey;
import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.schema.ParentTableNotFoundException;
+import org.apache.phoenix.schema.RowKeySchema;
import org.apache.phoenix.schema.SaltingUtil;
import org.apache.phoenix.schema.SequenceAllocation;
import org.apache.phoenix.schema.SequenceAlreadyExistsException;
@@ -689,7 +692,9 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
}
indexes.add(latestIndex);
}
- table = PTableImpl.makePTable(table, table.getTimeStamp(), indexes);
+ table = PTableImpl.builderWithColumns(table, getColumnsToClone(table))
+ .setIndexes(indexes == null ? Collections.<PTable>emptyList() : indexes)
+ .build();
}
}
@@ -939,11 +944,31 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
int baseTableColumnCount =
isDiverged ? QueryConstants.DIVERGED_VIEW_BASE_COLUMN_COUNT
: columnsToAdd.size() - myColumns.size() + (isSalted ? 1 : 0);
+
+ // When creating a PTable for views or view indexes, use the baseTable PTable for attributes
+ // inherited from the physical base table.
+ // if a TableProperty is not valid on a view we set it to the base table value
+ // if a TableProperty is valid on a view and is not mutable on a view we set it to the base table value
+ // if a TableProperty is valid on a view and is mutable on a view we use the value set on the view
// TODO Implement PHOENIX-4763 to set the view properties correctly instead of just
// setting them same as the base table
- PTableImpl pTable =
- PTableImpl.makePTable(table, baseTable, columnsToAdd, maxTableTimestamp,
- baseTableColumnCount, excludedColumns);
+ PTableImpl pTable = PTableImpl.builderWithColumns(table, columnsToAdd)
+ .setImmutableRows(baseTable.isImmutableRows())
+ .setDisableWAL(baseTable.isWALDisabled())
+ .setMultiTenant(baseTable.isMultiTenant())
+ .setStoreNulls(baseTable.getStoreNulls())
+ .setTransactionProvider(baseTable.getTransactionProvider())
+ .setAutoPartitionSeqName(baseTable.getAutoPartitionSeqName())
+ .setAppendOnlySchema(baseTable.isAppendOnlySchema())
+ .setImmutableStorageScheme(baseTable.getImmutableStorageScheme() == null ?
+ ImmutableStorageScheme.ONE_CELL_PER_COLUMN : baseTable.getImmutableStorageScheme())
+ .setQualifierEncodingScheme(baseTable.getEncodingScheme() == null ?
+ QualifierEncodingScheme.NON_ENCODED_QUALIFIERS : baseTable.getEncodingScheme())
+ .setBaseColumnCount(baseTableColumnCount)
+ .setTimeStamp(maxTableTimestamp)
+ .setExcludedColumns(excludedColumns == null ?
+ ImmutableList.<PColumn>of() : ImmutableList.copyOf(excludedColumns))
+ .build();
return WhereConstantParser.addViewInfoToPColumnsIfNeeded(pTable);
}
@@ -1486,11 +1511,48 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
}
// Avoid querying the stats table because we're holding the rowLock here. Issuing an RPC to a remote
// server while holding this lock is a bad idea and likely to cause contention.
- return PTableImpl.makePTable(tenantId, schemaName, tableName, tableType, indexState, timeStamp, tableSeqNum,
- pkName, saltBucketNum, columns, parentSchemaName, parentTableName, indexes, isImmutableRows, physicalTables, defaultFamilyName,
- viewStatement, disableWAL, multiTenant, storeNulls, viewType, viewIndexType, viewIndexId, indexType,
- rowKeyOrderOptimizable, transactionProvider, updateCacheFrequency, baseColumnCount,
- indexDisableTimestamp, isNamespaceMapped, autoPartitionSeq, isAppendOnlySchema, storageScheme, encodingScheme, cqCounter, useStatsForParallelization);
+ return new PTableImpl.Builder()
+ .setType(tableType)
+ .setState(indexState)
+ .setTimeStamp(timeStamp)
+ .setIndexDisableTimestamp(indexDisableTimestamp)
+ .setSequenceNumber(tableSeqNum)
+ .setImmutableRows(isImmutableRows)
+ .setViewStatement(viewStatement)
+ .setDisableWAL(disableWAL)
+ .setMultiTenant(multiTenant)
+ .setStoreNulls(storeNulls)
+ .setViewType(viewType)
+ .setViewIndexType(viewIndexType)
+ .setViewIndexId(viewIndexId)
+ .setIndexType(indexType)
+ .setTransactionProvider(transactionProvider)
+ .setUpdateCacheFrequency(updateCacheFrequency)
+ .setNamespaceMapped(isNamespaceMapped)
+ .setAutoPartitionSeqName(autoPartitionSeq)
+ .setAppendOnlySchema(isAppendOnlySchema)
+ .setImmutableStorageScheme(storageScheme == null ?
+ ImmutableStorageScheme.ONE_CELL_PER_COLUMN : storageScheme)
+ .setQualifierEncodingScheme(encodingScheme == null ?
+ QualifierEncodingScheme.NON_ENCODED_QUALIFIERS : encodingScheme)
+ .setBaseColumnCount(baseColumnCount)
+ .setEncodedCQCounter(cqCounter)
+ .setUseStatsForParallelization(useStatsForParallelization)
+ .setExcludedColumns(ImmutableList.<PColumn>of())
+ .setTenantId(tenantId)
+ .setSchemaName(schemaName)
+ .setTableName(tableName)
+ .setPkName(pkName)
+ .setDefaultFamilyName(defaultFamilyName)
+ .setRowKeyOrderOptimizable(rowKeyOrderOptimizable)
+ .setBucketNum(saltBucketNum)
+ .setIndexes(indexes == null ? Collections.<PTable>emptyList() : indexes)
+ .setParentSchemaName(parentSchemaName)
+ .setParentTableName(parentTableName)
+ .setPhysicalNames(physicalTables == null ?
+ ImmutableList.<PName>of() : ImmutableList.copyOf(physicalTables))
+ .setColumns(columns)
+ .build();
}
private Long getViewIndexId(Cell[] tableKeyValues, PDataType viewIndexType) {
Cell viewIndexIdKv = tableKeyValues[VIEW_INDEX_ID_INDEX];
@@ -1740,7 +1802,21 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
}
private static PTable newDeletedTableMarker(long timestamp) {
- return new PTableImpl(timestamp);
+ try {
+ return new PTableImpl.Builder()
+ .setType(PTableType.TABLE)
+ .setTimeStamp(timestamp)
+ .setPkColumns(Collections.<PColumn>emptyList())
+ .setAllColumns(Collections.<PColumn>emptyList())
+ .setFamilyAttributes(Collections.<PColumnFamily>emptyList())
+ .setRowKeySchema(RowKeySchema.EMPTY_SCHEMA)
+ .setIndexes(Collections.<PTable>emptyList())
+ .setPhysicalNames(Collections.<PName>emptyList())
+ .build();
+ } catch (SQLException e) {
+ // Should never happen
+ return null;
+ }
}
private static PFunction newDeletedFunctionMarker(long timestamp) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6083ae5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
index a667316..73386a2 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/UngroupedAggregateRegionObserver.java
@@ -23,6 +23,7 @@ import static org.apache.phoenix.query.QueryConstants.SINGLE_COLUMN_FAMILY;
import static org.apache.phoenix.query.QueryConstants.UNGROUPED_AGG_ROW_KEY;
import static org.apache.phoenix.query.QueryServices.MUTATE_BATCH_SIZE_ATTRIB;
import static org.apache.phoenix.query.QueryServices.MUTATE_BATCH_SIZE_BYTES_ATTRIB;
+import static org.apache.phoenix.schema.PTableImpl.getColumnsToClone;
import static org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.COMPACTION_UPDATE_STATS_ROW_COUNT;
import static org.apache.phoenix.schema.stats.StatisticsCollectionRunTracker.CONCURRENT_UPDATE_STATS_ROW_COUNT;
@@ -411,7 +412,10 @@ public class UngroupedAggregateRegionObserver extends BaseScannerRegionObserver
logger.debug("Upgrading row key for " + region.getRegionInfo().getTable().getNameAsString());
projectedTable = deserializeTable(descRowKeyTableBytes);
try {
- writeToTable = PTableImpl.makePTable(projectedTable, true);
+ writeToTable = PTableImpl.builderWithColumns(projectedTable,
+ getColumnsToClone(projectedTable))
+ .setRowKeyOrderOptimizable(true)
+ .build();
} catch (SQLException e) {
ServerUtil.throwIOException("Upgrade failed", e); // Impossible
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6083ae5/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/WhereConstantParser.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/WhereConstantParser.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/WhereConstantParser.java
index 4528f55..9fc030c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/WhereConstantParser.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/WhereConstantParser.java
@@ -94,7 +94,8 @@ public class WhereConstantParser {
result.add(column);
}
}
- return PTableImpl.makePTable(view, result);
+ return PTableImpl.builderWithColumns(view, result)
+ .build();
}
private static PhoenixConnection getConnectionlessConnection() throws SQLException {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6083ae5/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
index 655de0d..d7e7d6d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
@@ -18,6 +18,7 @@
package org.apache.phoenix.query;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.INDEX_STATE_BYTES;
+import static org.apache.phoenix.schema.PTableImpl.getColumnsToClone;
import java.sql.SQLException;
import java.util.ArrayList;
@@ -316,7 +317,8 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
public MetaDataMutationResult addColumn(List<Mutation> tableMetaData, PTable table, Map<String, List<Pair<String,Object>>> properties, Set<String> colFamiliesForPColumnsToBeAdded, List<PColumn> columnsToBeAdded) throws SQLException {
List<PColumn> columns = Lists.newArrayList(table.getColumns());
columns.addAll(columnsToBeAdded);
- return new MetaDataMutationResult(MutationCode.TABLE_ALREADY_EXISTS, 0, PTableImpl.makePTable(table, columns));
+ return new MetaDataMutationResult(MutationCode.TABLE_ALREADY_EXISTS, 0,
+ PTableImpl.builderWithColumns(table, columns).build());
}
@Override
@@ -453,7 +455,10 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
String indexTableName = SchemaUtil.getTableName(schemaName, indexName);
PName tenantId = tenantIdBytes.length == 0 ? null : PNameFactory.newName(tenantIdBytes);
PTable index = metaData.getTableRef(new PTableKey(tenantId, indexTableName)).getTable();
- index = PTableImpl.makePTable(index,newState == PIndexState.USABLE ? PIndexState.ACTIVE : newState == PIndexState.UNUSABLE ? PIndexState.INACTIVE : newState);
+ index = PTableImpl.builderWithColumns(index, getColumnsToClone(index))
+ .setState(newState == PIndexState.USABLE ? PIndexState.ACTIVE :
+ newState == PIndexState.UNUSABLE ? PIndexState.INACTIVE : newState)
+ .build();
return new MetaDataMutationResult(MutationCode.TABLE_ALREADY_EXISTS, 0, index);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6083ae5/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index df0e5d4..aae9fc4 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -104,6 +104,7 @@ import static org.apache.phoenix.schema.PTable.ImmutableStorageScheme.ONE_CELL_P
import static org.apache.phoenix.schema.PTable.ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS;
import static org.apache.phoenix.schema.PTable.QualifierEncodingScheme.NON_ENCODED_QUALIFIERS;
import static org.apache.phoenix.schema.PTable.ViewType.MAPPED;
+import static org.apache.phoenix.schema.PTableImpl.getColumnsToClone;
import static org.apache.phoenix.schema.PTableType.TABLE;
import static org.apache.phoenix.schema.PTableType.VIEW;
import static org.apache.phoenix.schema.types.PDataType.FALSE_BYTES;
@@ -132,6 +133,7 @@ import java.util.Properties;
import java.util.Set;
import org.apache.hadoop.hbase.HColumnDescriptor;
+import com.google.common.base.Objects;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.client.ClusterConnection;
@@ -247,6 +249,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.ListMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -931,10 +934,21 @@ public class MetaDataClient {
+ QueryConstants.CHILD_VIEW_INDEX_NAME_SEPARATOR + index.getName().getString());
// add the index table with a new name so that it does not conflict with the existing index table
// and set update cache frequency to that of the view
- indexesToAdd.add(PTableImpl.makePTable(index, modifiedIndexName, viewStatement, view.getUpdateCacheFrequency(), view.getTenantId()));
+ if (Objects.equal(viewStatement, index.getViewStatement())) {
+ indexesToAdd.add(index);
+ } else {
+ indexesToAdd.add(PTableImpl.builderWithColumns(index, getColumnsToClone(index))
+ .setTableName(modifiedIndexName)
+ .setViewStatement(viewStatement)
+ .setUpdateCacheFrequency(view.getUpdateCacheFrequency())
+ .setTenantId(view.getTenantId())
+ .build());
+ }
}
}
- PTable allIndexesTable = PTableImpl.makePTable(view, view.getTimeStamp(), indexesToAdd);
+ PTable allIndexesTable = PTableImpl.builderWithColumns(view, getColumnsToClone(view))
+ .setIndexes(indexesToAdd == null ? Collections.<PTable>emptyList() : indexesToAdd)
+ .build();
result.setTable(allIndexesTable);
return true;
}
@@ -2650,13 +2664,38 @@ public class MetaDataClient {
// TODO: what about stats for system catalog?
PName newSchemaName = PNameFactory.newName(schemaName);
// Column names and qualifiers and hardcoded for system tables.
- PTable table = PTableImpl.makePTable(tenantId,newSchemaName, PNameFactory.newName(tableName), tableType,
- null, MetaDataProtocol.MIN_TABLE_TIMESTAMP, PTable.INITIAL_SEQ_NUM,
- PNameFactory.newName(QueryConstants.SYSTEM_TABLE_PK_NAME), null, columns.values(), null, null,
- Collections.<PTable>emptyList(), isImmutableRows,
- Collections.<PName>emptyList(), defaultFamilyName == null ? null :
- PNameFactory.newName(defaultFamilyName), null,
- Boolean.TRUE.equals(disableWAL), false, false, null, viewIndexType, null, indexType, true, null, 0, 0L, isNamespaceMapped, autoPartitionSeq, isAppendOnlySchema, ONE_CELL_PER_COLUMN, NON_ENCODED_QUALIFIERS, PTable.EncodedCQCounter.NULL_COUNTER, true);
+ PTable table = new PTableImpl.Builder()
+ .setType(tableType)
+ .setTimeStamp(MetaDataProtocol.MIN_TABLE_TIMESTAMP)
+ .setIndexDisableTimestamp(0L)
+ .setSequenceNumber(PTable.INITIAL_SEQ_NUM)
+ .setImmutableRows(isImmutableRows)
+ .setDisableWAL(Boolean.TRUE.equals(disableWAL))
+ .setMultiTenant(false)
+ .setStoreNulls(false)
+ .setViewIndexType(viewIndexType)
+ .setIndexType(indexType)
+ .setUpdateCacheFrequency(0)
+ .setNamespaceMapped(isNamespaceMapped)
+ .setAutoPartitionSeqName(autoPartitionSeq)
+ .setAppendOnlySchema(isAppendOnlySchema)
+ .setImmutableStorageScheme(ONE_CELL_PER_COLUMN)
+ .setQualifierEncodingScheme(NON_ENCODED_QUALIFIERS)
+ .setBaseColumnCount(QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT)
+ .setEncodedCQCounter(PTable.EncodedCQCounter.NULL_COUNTER)
+ .setUseStatsForParallelization(true)
+ .setExcludedColumns(ImmutableList.<PColumn>of())
+ .setTenantId(tenantId)
+ .setSchemaName(newSchemaName)
+ .setTableName(PNameFactory.newName(tableName))
+ .setPkName(PNameFactory.newName(QueryConstants.SYSTEM_TABLE_PK_NAME))
+ .setDefaultFamilyName(defaultFamilyName == null ? null :
+ PNameFactory.newName(defaultFamilyName))
+ .setRowKeyOrderOptimizable(true)
+ .setIndexes(Collections.<PTable>emptyList())
+ .setPhysicalNames(ImmutableList.<PName>of())
+ .setColumns(columns.values())
+ .build();
connection.addTable(table, MetaDataProtocol.MIN_TABLE_TIMESTAMP);
}
@@ -2930,12 +2969,49 @@ public class MetaDataClient {
* for extra safety.
*/
EncodedCQCounter cqCounterToBe = tableType == PTableType.VIEW ? NULL_COUNTER : cqCounter;
- PTable table = PTableImpl.makePTable(
- tenantId, newSchemaName, PNameFactory.newName(tableName), tableType, indexState, timestamp!=null ? timestamp : result.getMutationTime(),
- PTable.INITIAL_SEQ_NUM, pkName == null ? null : PNameFactory.newName(pkName), saltBucketNum, columns.values(),
- parent == null ? null : parent.getSchemaName(), parent == null ? null : parent.getTableName(), Collections.<PTable>emptyList(), isImmutableRows,
- physicalNames, defaultFamilyName == null ? null : PNameFactory.newName(defaultFamilyName), viewStatement, Boolean.TRUE.equals(disableWAL), multiTenant, storeNulls, viewType,
- viewIndexType, result.getViewIndexId(), indexType, rowKeyOrderOptimizable, transactionProvider, updateCacheFrequency, 0L, isNamespaceMapped, autoPartitionSeq, isAppendOnlySchema, immutableStorageScheme, encodingScheme, cqCounterToBe, useStatsForParallelizationProp);
+ PTable table = new PTableImpl.Builder()
+ .setType(tableType)
+ .setState(indexState)
+ .setTimeStamp(timestamp != null ? timestamp : result.getMutationTime())
+ .setIndexDisableTimestamp(0L)
+ .setSequenceNumber(PTable.INITIAL_SEQ_NUM)
+ .setImmutableRows(isImmutableRows)
+ .setViewStatement(viewStatement)
+ .setDisableWAL(Boolean.TRUE.equals(disableWAL))
+ .setMultiTenant(multiTenant)
+ .setStoreNulls(storeNulls)
+ .setViewType(viewType)
+ .setViewIndexType(viewIndexType)
+ .setViewIndexId(result.getViewIndexId())
+ .setIndexType(indexType)
+ .setTransactionProvider(transactionProvider)
+ .setUpdateCacheFrequency(updateCacheFrequency)
+ .setNamespaceMapped(isNamespaceMapped)
+ .setAutoPartitionSeqName(autoPartitionSeq)
+ .setAppendOnlySchema(isAppendOnlySchema)
+ .setImmutableStorageScheme(immutableStorageScheme == null ?
+ ImmutableStorageScheme.ONE_CELL_PER_COLUMN : immutableStorageScheme)
+ .setQualifierEncodingScheme(encodingScheme == null ?
+ QualifierEncodingScheme.NON_ENCODED_QUALIFIERS : encodingScheme)
+ .setBaseColumnCount(QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT)
+ .setEncodedCQCounter(cqCounterToBe)
+ .setUseStatsForParallelization(useStatsForParallelizationProp)
+ .setExcludedColumns(ImmutableList.<PColumn>of())
+ .setTenantId(tenantId)
+ .setSchemaName(newSchemaName)
+ .setTableName(PNameFactory.newName(tableName))
+ .setPkName(pkName == null ? null : PNameFactory.newName(pkName))
+ .setDefaultFamilyName(defaultFamilyName == null ?
+ null : PNameFactory.newName(defaultFamilyName))
+ .setRowKeyOrderOptimizable(rowKeyOrderOptimizable)
+ .setBucketNum(saltBucketNum)
+ .setIndexes(Collections.<PTable>emptyList())
+ .setParentSchemaName((parent == null) ? null : parent.getSchemaName())
+ .setParentTableName((parent == null) ? null : parent.getTableName())
+ .setPhysicalNames(physicalNames == null ?
+ ImmutableList.<PName>of() : ImmutableList.copyOf(physicalNames))
+ .setColumns(columns.values())
+ .build();
result = new MetaDataMutationResult(code, result.getMutationTime(), table, true);
addTableToCache(result);
return table;
@@ -3131,10 +3207,29 @@ public class MetaDataClient {
&& (table.isMultiTenant() || hasViewIndexTable)) {
if (hasViewIndexTable) {
byte[] viewIndexPhysicalName = MetaDataUtil.getViewIndexPhysicalName(table.getPhysicalName().getBytes());
- PTable viewIndexTable = new PTableImpl(null,
- SchemaUtil.getSchemaNameFromFullName(viewIndexPhysicalName),
- SchemaUtil.getTableNameFromFullName(viewIndexPhysicalName), ts,
- table.getColumnFamilies(),table.isNamespaceMapped(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.useStatsForParallelization());
+ String viewIndexSchemaName = SchemaUtil.getSchemaNameFromFullName(viewIndexPhysicalName);
+ String viewIndexTableName = SchemaUtil.getTableNameFromFullName(viewIndexPhysicalName);
+ PName viewIndexName = PNameFactory.newName(SchemaUtil.getTableName(viewIndexSchemaName, viewIndexTableName));
+
+ PTable viewIndexTable = new PTableImpl.Builder()
+ .setName(viewIndexName)
+ .setKey(new PTableKey(tenantId, viewIndexName.getString()))
+ .setSchemaName(PNameFactory.newName(viewIndexSchemaName))
+ .setTableName(PNameFactory.newName(viewIndexTableName))
+ .setType(PTableType.VIEW)
+ .setViewType(ViewType.MAPPED)
+ .setTimeStamp(ts)
+ .setPkColumns(Collections.<PColumn>emptyList())
+ .setAllColumns(Collections.<PColumn>emptyList())
+ .setRowKeySchema(RowKeySchema.EMPTY_SCHEMA)
+ .setIndexes(Collections.<PTable>emptyList())
+ .setFamilyAttributes(table.getColumnFamilies())
+ .setPhysicalNames(Collections.<PName>emptyList())
+ .setNamespaceMapped(table.isNamespaceMapped())
+ .setImmutableStorageScheme(table.getImmutableStorageScheme())
+ .setQualifierEncodingScheme(table.getEncodingScheme())
+ .setUseStatsForParallelization(table.useStatsForParallelization())
+ .build();
tableRefs.add(new TableRef(null, viewIndexTable, ts, false));
}
}
@@ -3754,13 +3849,32 @@ public class MetaDataClient {
long ts = (scn == null ? result.getMutationTime() : scn);
byte[] viewIndexPhysicalName = MetaDataUtil
.getViewIndexPhysicalName(table.getPhysicalName().getBytes());
- PTable viewIndexTable = new PTableImpl(null,
- SchemaUtil.getSchemaNameFromFullName(viewIndexPhysicalName),
- SchemaUtil.getTableNameFromFullName(viewIndexPhysicalName), ts,
- table.getColumnFamilies(), table.isNamespaceMapped(), table.getImmutableStorageScheme(), table.getEncodingScheme(), table.useStatsForParallelization());
+ String viewIndexSchemaName = SchemaUtil.getSchemaNameFromFullName(viewIndexPhysicalName);
+ String viewIndexTableName = SchemaUtil.getTableNameFromFullName(viewIndexPhysicalName);
+ PName viewIndexName = PNameFactory.newName(SchemaUtil.getTableName(viewIndexSchemaName, viewIndexTableName));
+
+ PTable viewIndexTable = new PTableImpl.Builder()
+ .setName(viewIndexName)
+ .setKey(new PTableKey(tenantId, viewIndexName.getString()))
+ .setSchemaName(PNameFactory.newName(viewIndexSchemaName))
+ .setTableName(PNameFactory.newName(viewIndexTableName))
+ .setType(PTableType.VIEW)
+ .setViewType(ViewType.MAPPED)
+ .setTimeStamp(ts)
+ .setPkColumns(Collections.<PColumn>emptyList())
+ .setAllColumns(Collections.<PColumn>emptyList())
+ .setRowKeySchema(RowKeySchema.EMPTY_SCHEMA)
+ .setIndexes(Collections.<PTable>emptyList())
+ .setFamilyAttributes(table.getColumnFamilies())
+ .setPhysicalNames(Collections.<PName>emptyList())
+ .setNamespaceMapped(table.isNamespaceMapped())
+ .setImmutableStorageScheme(table.getImmutableStorageScheme())
+ .setQualifierEncodingScheme(table.getEncodingScheme())
+ .setUseStatsForParallelization(table.useStatsForParallelization())
+ .build();
List<TableRef> tableRefs = Collections.singletonList(new TableRef(null, viewIndexTable, ts, false));
MutationPlan plan = new PostDDLCompiler(connection).compile(tableRefs, null, null,
- Collections.<PColumn> emptyList(), ts);
+ Collections.<PColumn>emptyList(), ts);
connection.getQueryServices().updateData(plan);
}
}
@@ -4029,21 +4143,44 @@ public class MetaDataClient {
Map<String, List<TableRef>> tenantIdTableRefMap = Maps.newHashMap();
if (result.getSharedTablesToDelete() != null) {
for (SharedTableState sharedTableState : result.getSharedTablesToDelete()) {
- PTableImpl viewIndexTable =
- new PTableImpl(sharedTableState.getTenantId(),
- sharedTableState.getSchemaName(),
- sharedTableState.getTableName(), ts,
- table.getColumnFamilies(),
- sharedTableState.getColumns(),
- sharedTableState.getPhysicalNames(),
- sharedTableState.getViewIndexType(),
- sharedTableState.getViewIndexId(),
- table.isMultiTenant(), table.isNamespaceMapped(),
- table.getImmutableStorageScheme(),
- table.getEncodingScheme(),
- table.getEncodedCQCounter(),
- table.useStatsForParallelization(),
- table.getBucketNum());
+ ImmutableStorageScheme storageScheme = table.getImmutableStorageScheme();
+ QualifierEncodingScheme qualifierEncodingScheme = table.getEncodingScheme();
+ List<PColumn> columns = sharedTableState.getColumns();
+ if (table.getBucketNum() != null) {
+ columns = columns.subList(1, columns.size());
+ }
+
+ PTableImpl viewIndexTable = new PTableImpl.Builder()
+ .setPkColumns(Collections.<PColumn>emptyList())
+ .setAllColumns(Collections.<PColumn>emptyList())
+ .setRowKeySchema(RowKeySchema.EMPTY_SCHEMA)
+ .setIndexes(Collections.<PTable>emptyList())
+ .setFamilyAttributes(table.getColumnFamilies())
+ .setType(PTableType.INDEX)
+ .setTimeStamp(ts)
+ .setMultiTenant(table.isMultiTenant())
+ .setViewIndexType(sharedTableState.getViewIndexType())
+ .setViewIndexId(sharedTableState.getViewIndexId())
+ .setNamespaceMapped(table.isNamespaceMapped())
+ .setAppendOnlySchema(false)
+ .setImmutableStorageScheme(storageScheme == null ?
+ ImmutableStorageScheme.ONE_CELL_PER_COLUMN : storageScheme)
+ .setQualifierEncodingScheme(qualifierEncodingScheme == null ?
+ QualifierEncodingScheme.NON_ENCODED_QUALIFIERS : qualifierEncodingScheme)
+ .setEncodedCQCounter(table.getEncodedCQCounter())
+ .setUseStatsForParallelization(table.useStatsForParallelization())
+ .setExcludedColumns(ImmutableList.<PColumn>of())
+ .setTenantId(sharedTableState.getTenantId())
+ .setSchemaName(sharedTableState.getSchemaName())
+ .setTableName(sharedTableState.getTableName())
+ .setRowKeyOrderOptimizable(false)
+ .setBucketNum(table.getBucketNum())
+ .setIndexes(Collections.<PTable>emptyList())
+ .setPhysicalNames(sharedTableState.getPhysicalNames() == null ?
+ ImmutableList.<PName>of() :
+ ImmutableList.copyOf(sharedTableState.getPhysicalNames()))
+ .setColumns(columns)
+ .build();
TableRef indexTableRef = new TableRef(viewIndexTable);
PName indexTableTenantId = sharedTableState.getTenantId();
if (indexTableTenantId==null) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6083ae5/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaDataImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaDataImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaDataImpl.java
index 598cc79..3ce70ea 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaDataImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PMetaDataImpl.java
@@ -17,7 +17,10 @@
*/
package org.apache.phoenix.schema;
+import static org.apache.phoenix.schema.PTableImpl.getColumnsToClone;
+
import java.sql.SQLException;
+import java.util.Collections;
import java.util.Iterator;
import java.util.List;
@@ -27,7 +30,6 @@ import org.apache.phoenix.parse.PSchema;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.query.QueryServicesOptions;
import org.apache.phoenix.util.ReadOnlyProps;
-import org.apache.phoenix.util.SchemaUtil;
import org.apache.phoenix.util.TimeKeeper;
import com.google.common.collect.Lists;
@@ -121,7 +123,11 @@ public class PMetaDataImpl implements PMetaData {
}
newIndexes.add(table);
netGain -= oldParentRef.getEstimatedSize();
- newParentTable = PTableImpl.makePTable(oldParentRef.getTable(), table.getTimeStamp(), newIndexes);
+ newParentTable = PTableImpl.builderWithColumns(oldParentRef.getTable(),
+ getColumnsToClone(oldParentRef.getTable()))
+ .setIndexes(newIndexes)
+ .setTimeStamp(table.getTimeStamp())
+ .build();
newParentTableRef = tableRefFactory.makePTableRef(newParentTable, this.timeKeeper.getCurrentTime(), parentResolvedTimestamp);
netGain += newParentTableRef.getEstimatedSize();
}
@@ -173,10 +179,14 @@ public class PMetaDataImpl implements PMetaData {
PTable index = newIndexes.get(i);
if (index.getName().getString().equals(tableName)) {
newIndexes.remove(i);
- PTable parentTable = PTableImpl.makePTable(
- parentTableRef.getTable(),
- tableTimeStamp == HConstants.LATEST_TIMESTAMP ? parentTableRef.getTable().getTimeStamp() : tableTimeStamp,
- newIndexes);
+ PTableImpl.Builder parentTableBuilder =
+ PTableImpl.builderWithColumns(parentTableRef.getTable(),
+ getColumnsToClone(parentTableRef.getTable()))
+ .setIndexes(newIndexes == null ? Collections.<PTable>emptyList() : newIndexes);
+ if (tableTimeStamp != HConstants.LATEST_TIMESTAMP) {
+ parentTableBuilder.setTimeStamp(tableTimeStamp);
+ }
+ PTable parentTable = parentTableBuilder.build();
metaData.put(parentTable.getKey(), tableRefFactory.makePTableRef(parentTable, this.timeKeeper.getCurrentTime(), parentTableRef.getResolvedTimeStamp()));
break;
}
@@ -218,8 +228,10 @@ public class PMetaDataImpl implements PMetaData {
oldColumn.getTimestamp());
columns.add(newColumn);
}
-
- table = PTableImpl.makePTable(table, tableTimeStamp, tableSeqNum, columns);
+ table = PTableImpl.builderWithColumns(table, columns)
+ .setTimeStamp(tableTimeStamp)
+ .setSequenceNumber(tableSeqNum)
+ .build();
}
tables.put(table.getKey(), tableRefFactory.makePTableRef(table, this.timeKeeper.getCurrentTime(), resolvedTime));
}
[10/22] phoenix git commit: PHOENIX-4981 Add tests for ORDER BY,
GROUP BY and salted tables using phoenix-spark
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-spark/src/main/java/org/apache/phoenix/spark/SparkResultSet.java
----------------------------------------------------------------------
diff --git a/phoenix-spark/src/main/java/org/apache/phoenix/spark/SparkResultSet.java b/phoenix-spark/src/main/java/org/apache/phoenix/spark/SparkResultSet.java
new file mode 100644
index 0000000..0cb8009
--- /dev/null
+++ b/phoenix-spark/src/main/java/org/apache/phoenix/spark/SparkResultSet.java
@@ -0,0 +1,1056 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.spark;
+
+import org.apache.phoenix.exception.SQLExceptionCode;
+import org.apache.phoenix.exception.SQLExceptionInfo;
+import org.apache.phoenix.util.SQLCloseable;
+import org.apache.spark.sql.Row;
+
+import java.io.InputStream;
+import java.io.Reader;
+import java.math.BigDecimal;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.sql.Array;
+import java.sql.Blob;
+import java.sql.Clob;
+import java.sql.Date;
+import java.sql.NClob;
+import java.sql.Ref;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.RowId;
+import java.sql.SQLException;
+import java.sql.SQLFeatureNotSupportedException;
+import java.sql.SQLWarning;
+import java.sql.SQLXML;
+import java.sql.Statement;
+import java.sql.Time;
+import java.sql.Timestamp;
+import java.util.Arrays;
+import java.util.Calendar;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Helper class to convert a List of Rows returned from a dataset to a sql ResultSet
+ */
+public class SparkResultSet implements ResultSet, SQLCloseable {
+
+ private int index = -1;
+ private List<Row> dataSetRows;
+ private List<String> columnNames;
+ private boolean wasNull = false;
+
+ public SparkResultSet(List<Row> rows, String[] columnNames) {
+ this.dataSetRows = rows;
+ this.columnNames = Arrays.asList(columnNames);
+ }
+
+ private Row getCurrentRow() {
+ return dataSetRows.get(index);
+ }
+
+ @Override
+ public boolean absolute(int row) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void afterLast() throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void beforeFirst() throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void cancelRowUpdates() throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void clearWarnings() throws SQLException {
+ }
+
+ @Override
+ public void close() throws SQLException {
+ }
+
+ @Override
+ public void deleteRow() throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public int findColumn(String columnLabel) throws SQLException {
+ return columnNames.indexOf(columnLabel.toUpperCase())+1;
+ }
+
+ @Override
+ public boolean first() throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public Array getArray(int columnIndex) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public Array getArray(String columnLabel) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public InputStream getAsciiStream(int columnIndex) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public InputStream getAsciiStream(String columnLabel) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ private void checkOpen() throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ private void checkCursorState() throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public BigDecimal getBigDecimal(int columnIndex) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public BigDecimal getBigDecimal(String columnLabel) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public BigDecimal getBigDecimal(int columnIndex, int scale) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public BigDecimal getBigDecimal(String columnLabel, int scale) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public InputStream getBinaryStream(int columnIndex) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public InputStream getBinaryStream(String columnLabel) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public Blob getBlob(int columnIndex) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public Blob getBlob(String columnLabel) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public boolean getBoolean(int columnIndex) throws SQLException {
+ wasNull = getCurrentRow().isNullAt(columnIndex-1);
+ return wasNull ? false : getCurrentRow().getBoolean(columnIndex-1);
+ }
+
+ @Override
+ public boolean getBoolean(String columnLabel) throws SQLException {
+ return getBoolean(findColumn(columnLabel));
+ }
+
+ @Override
+ public byte[] getBytes(int columnIndex) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public byte[] getBytes(String columnLabel) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public byte getByte(int columnIndex) throws SQLException {
+ wasNull = getCurrentRow().isNullAt(columnIndex-1);
+ return wasNull ? 0 : getCurrentRow().getByte(columnIndex-1);
+ }
+
+ @Override
+ public byte getByte(String columnLabel) throws SQLException {
+ return getByte(findColumn(columnLabel));
+ }
+
+ @Override
+ public Reader getCharacterStream(int columnIndex) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public Reader getCharacterStream(String columnLabel) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public Clob getClob(int columnIndex) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public Clob getClob(String columnLabel) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public int getConcurrency() throws SQLException {
+ return ResultSet.CONCUR_READ_ONLY;
+ }
+
+ @Override
+ public String getCursorName() throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public Date getDate(int columnIndex) throws SQLException {
+ wasNull = getCurrentRow().isNullAt(columnIndex-1);
+ return getCurrentRow().getDate(columnIndex-1);
+ }
+
+ @Override
+ public Date getDate(String columnLabel) throws SQLException {
+ return getDate(findColumn(columnLabel));
+ }
+
+ @Override
+ public Date getDate(int columnIndex, Calendar cal) throws SQLException {
+ cal.setTime(getCurrentRow().getDate(columnIndex-1));
+ return new Date(cal.getTimeInMillis());
+ }
+
+ @Override
+ public Date getDate(String columnLabel, Calendar cal) throws SQLException {
+ return getDate(findColumn(columnLabel), cal);
+ }
+
+ @Override
+ public double getDouble(int columnIndex) throws SQLException {
+ wasNull = getCurrentRow().isNullAt(columnIndex-1);
+ return wasNull ? 0 : getCurrentRow().getDouble(columnIndex-1);
+ }
+
+ @Override
+ public double getDouble(String columnLabel) throws SQLException {
+ return getDouble(findColumn(columnLabel));
+ }
+
+ @Override
+ public int getFetchDirection() throws SQLException {
+ return ResultSet.FETCH_FORWARD;
+ }
+
+ @Override
+ public int getFetchSize() throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public float getFloat(int columnIndex) throws SQLException {
+ wasNull = getCurrentRow().isNullAt(columnIndex-1);
+ return wasNull ? 0 : getCurrentRow().getFloat(columnIndex-1);
+ }
+
+ @Override
+ public float getFloat(String columnLabel) throws SQLException {
+ return getFloat(findColumn(columnLabel));
+ }
+
+ @Override
+ public int getHoldability() throws SQLException {
+ return ResultSet.CLOSE_CURSORS_AT_COMMIT;
+ }
+
+ @Override
+ public int getInt(int columnIndex) throws SQLException {
+ wasNull = getCurrentRow().isNullAt(columnIndex-1);
+ return wasNull ? 0 : getCurrentRow().getInt(columnIndex-1);
+ }
+
+ @Override
+ public int getInt(String columnLabel) throws SQLException {
+ return getInt(findColumn(columnLabel));
+ }
+
+ @Override
+ public long getLong(int columnIndex) throws SQLException {
+ wasNull = getCurrentRow().isNullAt(columnIndex-1);
+ return wasNull ? 0 : getCurrentRow().getLong(columnIndex-1);
+ }
+
+ @Override
+ public long getLong(String columnLabel) throws SQLException {
+ return getLong(findColumn(columnLabel));
+ }
+
+ @Override
+ public ResultSetMetaData getMetaData() throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public Reader getNCharacterStream(int columnIndex) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public Reader getNCharacterStream(String columnLabel) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public NClob getNClob(int columnIndex) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public NClob getNClob(String columnLabel) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public String getNString(int columnIndex) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public String getNString(String columnLabel) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public Object getObject(int columnIndex) throws SQLException {
+ return getCurrentRow().get(columnIndex-1);
+ }
+
+ @Override
+ public Object getObject(String columnLabel) throws SQLException {
+ return getObject(findColumn(columnLabel));
+ }
+
+ @Override
+ public Object getObject(int columnIndex, Map<String, Class<?>> map) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public Object getObject(String columnLabel, Map<String, Class<?>> map) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public Ref getRef(int columnIndex) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public Ref getRef(String columnLabel) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public int getRow() throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public RowId getRowId(int columnIndex) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public RowId getRowId(String columnLabel) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public SQLXML getSQLXML(int columnIndex) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public SQLXML getSQLXML(String columnLabel) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public short getShort(int columnIndex) throws SQLException {
+ return getCurrentRow().getShort(columnIndex-1);
+ }
+
+ @Override
+ public short getShort(String columnLabel) throws SQLException {
+ return getShort(findColumn(columnLabel));
+ }
+
+ @Override
+ public Statement getStatement() throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public String getString(int columnIndex) throws SQLException {
+ wasNull = getCurrentRow().isNullAt(columnIndex-1);
+ return wasNull ? null : getCurrentRow().getString(columnIndex-1);
+ }
+
+ @Override
+ public String getString(String columnLabel) throws SQLException {
+ return getString(findColumn(columnLabel));
+ }
+
+ @Override
+ public Time getTime(int columnIndex) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public Time getTime(String columnLabel) throws SQLException {
+ return getTime(findColumn(columnLabel));
+ }
+
+ @Override
+ public Time getTime(int columnIndex, Calendar cal) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public Time getTime(String columnLabel, Calendar cal) throws SQLException {
+ return getTime(findColumn(columnLabel),cal);
+ }
+
+ @Override
+ public Timestamp getTimestamp(int columnIndex) throws SQLException {
+ return getCurrentRow().getTimestamp(columnIndex-1);
+ }
+
+ @Override
+ public Timestamp getTimestamp(String columnLabel) throws SQLException {
+ return getTimestamp(findColumn(columnLabel));
+ }
+
+ @Override
+ public Timestamp getTimestamp(int columnIndex, Calendar cal) throws SQLException {
+ return getTimestamp(columnIndex-1);
+ }
+
+ @Override
+ public Timestamp getTimestamp(String columnLabel, Calendar cal) throws SQLException {
+ return getTimestamp(findColumn(columnLabel),cal);
+ }
+
+ @Override
+ public int getType() throws SQLException {
+ return ResultSet.TYPE_FORWARD_ONLY;
+ }
+
+ @Override
+ public URL getURL(int columnIndex) throws SQLException {
+ try {
+ return new URL(getCurrentRow().getString(columnIndex-1));
+ } catch (MalformedURLException e) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.MALFORMED_URL).setRootCause(e)
+ .build().buildException();
+ }
+ }
+
+ @Override
+ public URL getURL(String columnLabel) throws SQLException {
+ return getURL(findColumn(columnLabel));
+ }
+
+ @Override
+ public InputStream getUnicodeStream(int columnIndex) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public InputStream getUnicodeStream(String columnLabel) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public SQLWarning getWarnings() throws SQLException {
+ return null;
+ }
+
+ @Override
+ public void insertRow() throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public boolean isAfterLast() throws SQLException {
+ return index >= dataSetRows.size();
+ }
+
+ @Override
+ public boolean isBeforeFirst() throws SQLException {
+ return index == -1;
+ }
+
+ @Override
+ public boolean isClosed() throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public boolean isFirst() throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public boolean isLast() throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public boolean last() throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void moveToCurrentRow() throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void moveToInsertRow() throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public boolean next() throws SQLException {
+ index++;
+ return index < dataSetRows.size();
+ }
+
+ @Override
+ public boolean previous() throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void refreshRow() throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public boolean relative(int rows) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public boolean rowDeleted() throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public boolean rowInserted() throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public boolean rowUpdated() throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void setFetchDirection(int direction) throws SQLException {
+ if (direction != ResultSet.FETCH_FORWARD) {
+ throw new SQLFeatureNotSupportedException();
+ }
+ }
+
+ @Override
+ public void setFetchSize(int rows) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateArray(int columnIndex, Array x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateArray(String columnLabel, Array x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateAsciiStream(int columnIndex, InputStream x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateAsciiStream(String columnLabel, InputStream x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateAsciiStream(int columnIndex, InputStream x, int length) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateAsciiStream(String columnLabel, InputStream x, int length) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateAsciiStream(int columnIndex, InputStream x, long length) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateAsciiStream(String columnLabel, InputStream x, long length) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateBigDecimal(int columnIndex, BigDecimal x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateBigDecimal(String columnLabel, BigDecimal x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateBinaryStream(int columnIndex, InputStream x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateBinaryStream(String columnLabel, InputStream x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateBinaryStream(int columnIndex, InputStream x, int length) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateBinaryStream(String columnLabel, InputStream x, int length) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateBinaryStream(int columnIndex, InputStream x, long length) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateBinaryStream(String columnLabel, InputStream x, long length) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateBlob(int columnIndex, Blob x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateBlob(String columnLabel, Blob x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateBlob(int columnIndex, InputStream inputStream) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateBlob(String columnLabel, InputStream inputStream) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateBlob(int columnIndex, InputStream inputStream, long length) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateBlob(String columnLabel, InputStream inputStream, long length) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateBoolean(int columnIndex, boolean x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateBoolean(String columnLabel, boolean x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateByte(int columnIndex, byte x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateByte(String columnLabel, byte x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateBytes(int columnIndex, byte[] x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateBytes(String columnLabel, byte[] x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateCharacterStream(int columnIndex, Reader x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateCharacterStream(String columnLabel, Reader reader) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateCharacterStream(int columnIndex, Reader x, int length) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateCharacterStream(String columnLabel, Reader reader, int length) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateCharacterStream(int columnIndex, Reader x, long length) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateCharacterStream(String columnLabel, Reader reader, long length) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateClob(int columnIndex, Clob x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateClob(String columnLabel, Clob x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateClob(int columnIndex, Reader reader) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateClob(String columnLabel, Reader reader) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateClob(int columnIndex, Reader reader, long length) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateClob(String columnLabel, Reader reader, long length) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateDate(int columnIndex, Date x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateDate(String columnLabel, Date x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateDouble(int columnIndex, double x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateDouble(String columnLabel, double x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateFloat(int columnIndex, float x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateFloat(String columnLabel, float x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateInt(int columnIndex, int x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateInt(String columnLabel, int x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateLong(int columnIndex, long x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateLong(String columnLabel, long x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateNCharacterStream(int columnIndex, Reader x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateNCharacterStream(String columnLabel, Reader reader) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateNCharacterStream(int columnIndex, Reader x, long length) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateNCharacterStream(String columnLabel, Reader reader, long length) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateNClob(int columnIndex, NClob nClob) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateNClob(String columnLabel, NClob nClob) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateNClob(int columnIndex, Reader reader) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateNClob(String columnLabel, Reader reader) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateNClob(int columnIndex, Reader reader, long length) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateNClob(String columnLabel, Reader reader, long length) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateNString(int columnIndex, String nString) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateNString(String columnLabel, String nString) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateNull(int columnIndex) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateNull(String columnLabel) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateObject(int columnIndex, Object x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateObject(String columnLabel, Object x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateObject(int columnIndex, Object x, int scaleOrLength) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateObject(String columnLabel, Object x, int scaleOrLength) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateRef(int columnIndex, Ref x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateRef(String columnLabel, Ref x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateRow() throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateRowId(int columnIndex, RowId x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateRowId(String columnLabel, RowId x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateSQLXML(int columnIndex, SQLXML xmlObject) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateSQLXML(String columnLabel, SQLXML xmlObject) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateShort(int columnIndex, short x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateShort(String columnLabel, short x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateString(int columnIndex, String x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateString(String columnLabel, String x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateTime(int columnIndex, Time x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateTime(String columnLabel, Time x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateTimestamp(int columnIndex, Timestamp x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public void updateTimestamp(String columnLabel, Timestamp x) throws SQLException {
+ throw new SQLFeatureNotSupportedException();
+ }
+
+ @Override
+ public boolean wasNull() throws SQLException {
+ return wasNull;
+ }
+
+ @Override
+ public boolean isWrapperFor(Class<?> iface) throws SQLException {
+ return iface.isInstance(this);
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <T> T unwrap(Class<T> iface) throws SQLException {
+ if (!iface.isInstance(this)) {
+ throw new SQLExceptionInfo.Builder(SQLExceptionCode.CLASS_NOT_UNWRAPPABLE)
+ .setMessage(this.getClass().getName() + " not unwrappable from " + iface.getName())
+ .build().buildException();
+ }
+ return (T)this;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <T> T getObject(int columnIndex, Class<T> type) throws SQLException {
+ return (T) getObject(columnIndex-1); // Just ignore type since we only support built-in types
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public <T> T getObject(String columnLabel, Class<T> type) throws SQLException {
+ return (T) getObject(columnLabel); // Just ignore type since we only support built-in types
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-spark/src/main/scala/org/apache/phoenix/spark/PhoenixRDD.scala
----------------------------------------------------------------------
diff --git a/phoenix-spark/src/main/scala/org/apache/phoenix/spark/PhoenixRDD.scala b/phoenix-spark/src/main/scala/org/apache/phoenix/spark/PhoenixRDD.scala
index 2c2c6e1..d604e0e 100644
--- a/phoenix-spark/src/main/scala/org/apache/phoenix/spark/PhoenixRDD.scala
+++ b/phoenix-spark/src/main/scala/org/apache/phoenix/spark/PhoenixRDD.scala
@@ -21,8 +21,9 @@ import org.apache.hadoop.io.NullWritable
import org.apache.phoenix.jdbc.PhoenixDriver
import org.apache.phoenix.mapreduce.PhoenixInputFormat
import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil
+import org.apache.phoenix.query.QueryConstants
import org.apache.phoenix.schema.types._
-import org.apache.phoenix.util.ColumnInfo
+import org.apache.phoenix.util.{ColumnInfo, SchemaUtil}
import org.apache.spark._
import org.apache.spark.annotation.DeveloperApi
import org.apache.spark.rdd.RDD
@@ -148,13 +149,25 @@ class PhoenixRDD(sc: SparkContext, table: String, columns: Seq[String],
}), new StructType(structFields))
}
- def phoenixSchemaToCatalystSchema(columnList: Seq[ColumnInfo]) = {
- columnList.map(ci => {
- val structType = phoenixTypeToCatalystType(ci)
- StructField(ci.getDisplayName, structType)
- })
+ def normalizeColumnName(columnName: String) = {
+ val unescapedColumnName = SchemaUtil.getUnEscapedFullColumnName(columnName)
+ var normalizedColumnName = ""
+ if (unescapedColumnName.indexOf(QueryConstants.NAME_SEPARATOR) < 0) {
+ normalizedColumnName = unescapedColumnName
+ }
+ else {
+ // split by separator to get the column family and column name
+ val tokens = unescapedColumnName.split(QueryConstants.NAME_SEPARATOR_REGEX)
+ normalizedColumnName = if (tokens(0) == "0") tokens(1) else unescapedColumnName
+ }
+ normalizedColumnName
}
+ def phoenixSchemaToCatalystSchema(columnList: Seq[ColumnInfo]) = columnList.map(ci => {
+ val structType = phoenixTypeToCatalystType(ci)
+ StructField(normalizeColumnName(ci.getColumnName), structType)
+ })
+
// Lookup table for Phoenix types to Spark catalyst types
def phoenixTypeToCatalystType(columnInfo: ColumnInfo): DataType = columnInfo.getPDataType match {
@@ -166,7 +179,7 @@ class PhoenixRDD(sc: SparkContext, table: String, columns: Seq[String],
case t if t.isInstanceOf[PFloat] || t.isInstanceOf[PUnsignedFloat] => FloatType
case t if t.isInstanceOf[PDouble] || t.isInstanceOf[PUnsignedDouble] => DoubleType
// Use Spark system default precision for now (explicit to work with < 1.5)
- case t if t.isInstanceOf[PDecimal] =>
+ case t if t.isInstanceOf[PDecimal] =>
if (columnInfo.getPrecision == null || columnInfo.getPrecision < 0) DecimalType(38, 18) else DecimalType(columnInfo.getPrecision, columnInfo.getScale)
case t if t.isInstanceOf[PTimestamp] || t.isInstanceOf[PUnsignedTimestamp] => TimestampType
case t if t.isInstanceOf[PTime] || t.isInstanceOf[PUnsignedTime] => TimestampType
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 143f285..eba69a9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -101,7 +101,7 @@
<avatica.version>1.12.0</avatica.version>
<jettyVersion>8.1.7.v20120910</jettyVersion>
<tephra.version>0.15.0-incubating</tephra.version>
- <spark.version>2.0.2</spark.version>
+ <spark.version>2.3.2</spark.version>
<scala.version>2.11.8</scala.version>
<scala.binary.version>2.11</scala.binary.version>
<stream.version>2.9.5</stream.version>
[09/22] phoenix git commit: PHOENIX-4997 Phoenix MR on snapshots can
produce duplicate rows
Posted by ja...@apache.org.
PHOENIX-4997 Phoenix MR on snapshots can produce duplicate rows
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/1b2a3d5c
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/1b2a3d5c
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/1b2a3d5c
Branch: refs/heads/omid2
Commit: 1b2a3d5c7f25864158cd5062f2cd58d834fb5e1f
Parents: c3ed4f8
Author: Karan Mehta <ka...@gmail.com>
Authored: Thu Nov 1 17:15:26 2018 -0700
Committer: Karan Mehta <ka...@gmail.com>
Committed: Thu Nov 1 17:54:37 2018 -0700
----------------------------------------------------------------------
.../end2end/TableSnapshotReadsMapReduceIT.java | 122 +++++++++++--------
.../iterate/MapReduceParallelScanGrouper.java | 32 ++++-
.../iterate/TableSnapshotResultIterator.java | 28 +++--
.../java/org/apache/phoenix/query/BaseTest.java | 14 +--
4 files changed, 122 insertions(+), 74 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/1b2a3d5c/phoenix-core/src/it/java/org/apache/phoenix/end2end/TableSnapshotReadsMapReduceIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TableSnapshotReadsMapReduceIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TableSnapshotReadsMapReduceIT.java
index cae91a3..e35e159 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TableSnapshotReadsMapReduceIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TableSnapshotReadsMapReduceIT.java
@@ -36,6 +36,7 @@ import java.util.UUID;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.io.ImmutableBytesWritable;
@@ -49,12 +50,18 @@ import org.apache.phoenix.mapreduce.index.PhoenixIndexDBWritable;
import org.apache.phoenix.mapreduce.util.PhoenixMapReduceUtil;
import org.apache.phoenix.util.ReadOnlyProps;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import com.google.common.collect.Maps;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class TableSnapshotReadsMapReduceIT extends BaseUniqueNamesOwnClusterIT {
+
+ private static final Logger logger = LoggerFactory.getLogger(TableSnapshotReadsMapReduceIT.class);
+
private final static String SNAPSHOT_NAME = "FOO";
private static final String FIELD1 = "FIELD1";
private static final String FIELD2 = "FIELD2";
@@ -66,6 +73,9 @@ public class TableSnapshotReadsMapReduceIT extends BaseUniqueNamesOwnClusterIT {
private static List<List<Object>> result;
private long timestamp;
private String tableName;
+ private Job job;
+ private Path tmpDir;
+ private Configuration conf;
@BeforeClass
public static void doSetup() throws Exception {
@@ -73,8 +83,8 @@ public class TableSnapshotReadsMapReduceIT extends BaseUniqueNamesOwnClusterIT {
setUpTestDriver(new ReadOnlyProps(props.entrySet().iterator()));
}
- @Test
- public void testMapReduceSnapshots() throws Exception {
+ @Before
+ public void before() throws SQLException, IOException {
// create table
Connection conn = DriverManager.getConnection(getUrl());
tableName = generateUniqueName();
@@ -82,58 +92,43 @@ public class TableSnapshotReadsMapReduceIT extends BaseUniqueNamesOwnClusterIT {
conn.commit();
// configure Phoenix M/R job to read snapshot
- final Configuration conf = getUtility().getConfiguration();
- Job job = Job.getInstance(conf);
- Path tmpDir = getUtility().getRandomDir();
+ conf = getUtility().getConfiguration();
+ job = Job.getInstance(conf);
+ tmpDir = getUtility().getRandomDir();
+ }
- PhoenixMapReduceUtil.setInput(job,PhoenixIndexDBWritable.class,SNAPSHOT_NAME,tableName,tmpDir, null, FIELD1, FIELD2, FIELD3);
+ @Test
+ public void testMapReduceSnapshots() throws Exception {
+ PhoenixMapReduceUtil.setInput(job,PhoenixIndexDBWritable.class,
+ SNAPSHOT_NAME, tableName, tmpDir, null, FIELD1, FIELD2, FIELD3);
+ configureJob(job, tableName, null, null, false);
+ }
- // configure and test job
- configureJob(job, tableName, null, null);
+ @Test
+ public void testMapReduceSnapshotsMultiRegion() throws Exception {
+ PhoenixMapReduceUtil.setInput(job,PhoenixIndexDBWritable.class,
+ SNAPSHOT_NAME, tableName, tmpDir, null, FIELD1, FIELD2, FIELD3);
+ configureJob(job, tableName, null, null, true);
}
@Test
public void testMapReduceSnapshotsWithCondition() throws Exception {
- // create table
- Connection conn = DriverManager.getConnection(getUrl());
- tableName = generateUniqueName();
- conn.createStatement().execute(String.format(CREATE_TABLE, tableName));
- conn.commit();
-
- // configure Phoenix M/R job to read snapshot
- final Configuration conf = getUtility().getConfiguration();
- Job job = Job.getInstance(conf);
- Path tmpDir = getUtility().getRandomDir();
- PhoenixMapReduceUtil.setInput(job,PhoenixIndexDBWritable.class,SNAPSHOT_NAME,tableName,tmpDir, FIELD3 + " > 0001", FIELD1, FIELD2, FIELD3);
-
- // configure and test job
- configureJob(job, tableName, null, "FIELD3 > 0001");
-
+ PhoenixMapReduceUtil.setInput(job,PhoenixIndexDBWritable.class,
+ SNAPSHOT_NAME, tableName, tmpDir, FIELD3 + " > 0001", FIELD1, FIELD2, FIELD3);
+ configureJob(job, tableName, null, "FIELD3 > 0001", false);
}
@Test
public void testMapReduceSnapshotWithLimit() throws Exception {
- // create table
- Connection conn = DriverManager.getConnection(getUrl());
- tableName = generateUniqueName();
- conn.createStatement().execute(String.format(CREATE_TABLE, tableName));
- conn.commit();
-
- // configure Phoenix M/R job to read snapshot
- final Configuration conf = getUtility().getConfiguration();
- Job job = Job.getInstance(conf);
- Path tmpDir = getUtility().getRandomDir();
- // Running limit with order by on non pk column
String inputQuery = "SELECT * FROM " + tableName + " ORDER BY FIELD2 LIMIT 1";
- PhoenixMapReduceUtil.setInput(job,PhoenixIndexDBWritable.class,SNAPSHOT_NAME,tableName,tmpDir,inputQuery);
-
- // configure and test job
- configureJob(job, tableName, inputQuery, null);
+ PhoenixMapReduceUtil.setInput(job,PhoenixIndexDBWritable.class,
+ SNAPSHOT_NAME, tableName, tmpDir, inputQuery);
+ configureJob(job, tableName, inputQuery, null, false);
}
- private void configureJob(Job job, String tableName, String inputQuery, String condition) throws Exception {
+ private void configureJob(Job job, String tableName, String inputQuery, String condition, boolean shouldSplit) throws Exception {
try {
- upsertAndSnapshot(tableName);
+ upsertAndSnapshot(tableName, shouldSplit);
result = new ArrayList<>();
job.setMapperClass(TableSnapshotMapper.class);
@@ -151,6 +146,7 @@ public class TableSnapshotReadsMapReduceIT extends BaseUniqueNamesOwnClusterIT {
if (condition != null) {
selectQuery.append(" WHERE " + condition);
}
+
if (inputQuery == null)
inputQuery = selectQuery.toString();
@@ -176,12 +172,13 @@ public class TableSnapshotReadsMapReduceIT extends BaseUniqueNamesOwnClusterIT {
private void upsertData(String tableName) throws SQLException {
Connection conn = DriverManager.getConnection(getUrl());
PreparedStatement stmt = conn.prepareStatement(String.format(UPSERT, tableName));
- upsertData(stmt, "CCCC", "SSDD", 0001);
- upsertData(stmt, "CCCC", "HDHG", 0005);
- upsertData(stmt, "BBBB", "JSHJ", 0002);
- upsertData(stmt, "AAAA", "JHHD", 0003);
+ upsertData(stmt, "AAAA", "JHHD", 37);
+ upsertData(stmt, "BBBB", "JSHJ", 224);
+ upsertData(stmt, "CCCC", "SSDD", 15);
+ upsertData(stmt, "PPPP", "AJDG", 53);
+ upsertData(stmt, "SSSS", "HSDG", 59);
+ upsertData(stmt, "XXXX", "HDPP", 22);
conn.commit();
- timestamp = System.currentTimeMillis();
}
private void upsertData(PreparedStatement stmt, String field1, String field2, int field3) throws SQLException {
@@ -191,31 +188,52 @@ public class TableSnapshotReadsMapReduceIT extends BaseUniqueNamesOwnClusterIT {
stmt.execute();
}
- public void upsertAndSnapshot(String tableName) throws Exception {
+ private void upsertAndSnapshot(String tableName, boolean shouldSplit) throws Exception {
upsertData(tableName);
+ TableName hbaseTableName = TableName.valueOf(tableName);
Connection conn = DriverManager.getConnection(getUrl());
HBaseAdmin admin = conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin();
- admin.snapshot(SNAPSHOT_NAME, TableName.valueOf(tableName));
- // call flush to create new files in the region
- admin.flush(tableName);
+
+ if (shouldSplit) {
+ splitTableSync(admin, hbaseTableName, "BBBB".getBytes(), 2);
+ }
+
+ admin.snapshot(SNAPSHOT_NAME, hbaseTableName);
List<HBaseProtos.SnapshotDescription> snapshots = admin.listSnapshots();
Assert.assertEquals(tableName, snapshots.get(0).getTable());
+ // Capture the snapshot timestamp to use as SCN while reading the table later
+ // Assigning the timestamp value here will make tests less flaky
+ timestamp = System.currentTimeMillis();
+
// upsert data after snapshot
PreparedStatement stmt = conn.prepareStatement(String.format(UPSERT, tableName));
- upsertData(stmt, "DDDD", "SNFB", 0004);
+ upsertData(stmt, "DDDD", "SNFB", 45);
conn.commit();
}
- public void deleteSnapshot(String tableName) throws Exception {
+ private void splitTableSync(HBaseAdmin admin, TableName hbaseTableName,
+ byte[] splitPoint , int expectedRegions) throws IOException, InterruptedException {
+ admin.split(hbaseTableName, splitPoint);
+ for (int i = 0; i < 100; i++) {
+ List<HRegionInfo> hRegionInfoList = admin.getTableRegions(hbaseTableName);
+ if (hRegionInfoList.size() >= expectedRegions) {
+ break;
+ }
+ logger.info("Sleeping for 1000 ms while waiting for " + hbaseTableName.getNameAsString() + " to split");
+ Thread.sleep(1000);
+ }
+ }
+
+ private void deleteSnapshot(String tableName) throws Exception {
try (Connection conn = DriverManager.getConnection(getUrl());
HBaseAdmin admin =
conn.unwrap(PhoenixConnection.class).getQueryServices().getAdmin();) {
admin.deleteSnapshot(SNAPSHOT_NAME);
}
- }
+ }
public static class TableSnapshotMapper extends Mapper<NullWritable, PhoenixIndexDBWritable, ImmutableBytesWritable, NullWritable> {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/1b2a3d5c/phoenix-core/src/main/java/org/apache/phoenix/iterate/MapReduceParallelScanGrouper.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/MapReduceParallelScanGrouper.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/MapReduceParallelScanGrouper.java
index 593608f..b4f81ae 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/MapReduceParallelScanGrouper.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/MapReduceParallelScanGrouper.java
@@ -18,6 +18,7 @@
package org.apache.phoenix.iterate;
import java.sql.SQLException;
+import java.util.Collections;
import java.util.List;
import com.google.common.base.Preconditions;
@@ -45,7 +46,7 @@ public class MapReduceParallelScanGrouper implements ParallelScanGrouper {
private static final MapReduceParallelScanGrouper INSTANCE = new MapReduceParallelScanGrouper();
- public static MapReduceParallelScanGrouper getInstance() {
+ public static MapReduceParallelScanGrouper getInstance() {
return INSTANCE;
}
@@ -79,18 +80,39 @@ public class MapReduceParallelScanGrouper implements ParallelScanGrouper {
}
}
+ /**
+ * Get list of region locations from SnapshotManifest
+ * BaseResultIterators assume that regions are sorted using RegionInfo.COMPARATOR
+ */
private List<HRegionLocation> getRegionLocationsFromManifest(SnapshotManifest manifest) {
List<SnapshotProtos.SnapshotRegionManifest> regionManifests = manifest.getRegionManifests();
Preconditions.checkNotNull(regionManifests);
- List<HRegionLocation> regionLocations = Lists.newArrayListWithCapacity(regionManifests.size());
+ List<HRegionInfo> regionInfos = Lists.newArrayListWithCapacity(regionManifests.size());
+ List<HRegionLocation> hRegionLocations = Lists.newArrayListWithCapacity(regionManifests.size());
for (SnapshotProtos.SnapshotRegionManifest regionManifest : regionManifests) {
- regionLocations.add(new HRegionLocation(
- HRegionInfo.convert(regionManifest.getRegionInfo()), null));
+ HRegionInfo regionInfo = HRegionInfo.convert(regionManifest.getRegionInfo());
+ if (isValidRegion(regionInfo)) {
+ regionInfos.add(regionInfo);
+ }
+ }
+
+ Collections.sort(regionInfos);
+
+ for (HRegionInfo regionInfo : regionInfos) {
+ hRegionLocations.add(new HRegionLocation(regionInfo, null));
}
- return regionLocations;
+ return hRegionLocations;
+ }
+
+ // Exclude offline split parent regions
+ private boolean isValidRegion(HRegionInfo hri) {
+ if (hri.isOffline() && (hri.isSplit() || hri.isSplitParent())) {
+ return false;
+ }
+ return true;
}
private String getSnapshotName(Configuration conf) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/1b2a3d5c/phoenix-core/src/main/java/org/apache/phoenix/iterate/TableSnapshotResultIterator.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/TableSnapshotResultIterator.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/TableSnapshotResultIterator.java
index 016d3be..c3d75f7 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/TableSnapshotResultIterator.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/TableSnapshotResultIterator.java
@@ -37,7 +37,6 @@ import java.io.IOException;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Collections;
-import java.util.Iterator;
import java.util.List;
import java.util.UUID;
@@ -79,22 +78,31 @@ public class TableSnapshotResultIterator implements ResultIterator {
RestoreSnapshotHelper.RestoreMetaChanges meta =
RestoreSnapshotHelper.copySnapshotForScanner(this.configuration, this.fs,
this.rootDir, this.restoreDir, this.snapshotName);
- List restoredRegions = meta.getRegionsToAdd();
+ List<HRegionInfo> restoredRegions = meta.getRegionsToAdd();
this.htd = meta.getTableDescriptor();
- this.regions = new ArrayList(restoredRegions.size());
- Iterator i$ = restoredRegions.iterator();
-
- while(i$.hasNext()) {
- HRegionInfo hri = (HRegionInfo)i$.next();
- if(CellUtil.overlappingKeys(this.scan.getStartRow(), this.scan.getStopRow(),
- hri.getStartKey(), hri.getEndKey())) {
- this.regions.add(hri);
+ this.regions = new ArrayList<>(restoredRegions.size());
+
+ for (HRegionInfo restoredRegion : restoredRegions) {
+ if (isValidRegion(restoredRegion)) {
+ this.regions.add(restoredRegion);
}
}
Collections.sort(this.regions);
}
+ /**
+ * Exclude offline split parent regions and
+ * regions that don't intersect with provided scan
+ */
+ private boolean isValidRegion(HRegionInfo hri) {
+ if (hri.isOffline() && (hri.isSplit() || hri.isSplitParent())) {
+ return false;
+ }
+ return CellUtil.overlappingKeys(scan.getStartRow(), scan.getStopRow(),
+ hri.getStartKey(), hri.getEndKey());
+ }
+
public boolean initSnapshotScanner() throws SQLException {
if (closed) {
return true;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/1b2a3d5c/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
index e6e3936..42fa085 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
@@ -1782,15 +1782,15 @@ public abstract class BaseTest {
/**
- * Split SYSTEM.CATALOG at the given split point
+ * Synchronously split table at the given split point
*/
- protected static void splitRegion(byte[] splitPoint) throws SQLException, IOException, InterruptedException {
- HBaseAdmin admin =
+ protected static void splitRegion(TableName fullTableName, byte[] splitPoint) throws SQLException, IOException, InterruptedException {
+ HBaseAdmin admin =
driver.getConnectionQueryServices(getUrl(), TestUtil.TEST_PROPERTIES).getAdmin();
- admin.split(PhoenixDatabaseMetaData.SYSTEM_CATALOG_HBASE_TABLE_NAME, splitPoint);
+ admin.split(fullTableName, splitPoint);
// make sure the split finishes (there's no synchronous splitting before HBase 2.x)
- admin.disableTable(PhoenixDatabaseMetaData.SYSTEM_CATALOG_HBASE_TABLE_NAME);
- admin.enableTable(PhoenixDatabaseMetaData.SYSTEM_CATALOG_HBASE_TABLE_NAME);
+ admin.disableTable(fullTableName);
+ admin.enableTable(fullTableName);
}
/**
@@ -1819,7 +1819,7 @@ public abstract class BaseTest {
AssignmentManager am = master.getAssignmentManager();
// No need to split on the first splitPoint since the end key of region boundaries are exclusive
for (int i=1; i<splitPoints.size(); ++i) {
- splitRegion(splitPoints.get(i));
+ splitRegion(fullTableName, splitPoints.get(i));
}
HashMap<ServerName, List<HRegionInfo>> serverToRegionsList = Maps.newHashMapWithExpectedSize(NUM_SLAVES_BASE);
Deque<ServerName> availableRegionServers = new ArrayDeque<ServerName>(NUM_SLAVES_BASE);
[02/22] phoenix git commit: PHOENIX-4764 Cleanup metadata of child
views for a base table that has been dropped
Posted by ja...@apache.org.
PHOENIX-4764 Cleanup metadata of child views for a base table that has been dropped
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/3903ad76
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/3903ad76
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/3903ad76
Branch: refs/heads/omid2
Commit: 3903ad768496d0a6517298a8f196863b87290e72
Parents: a48e42e
Author: Kadir <ko...@salesforce.com>
Authored: Wed Sep 26 23:32:31 2018 -0700
Committer: Karan Mehta <ka...@gmail.com>
Committed: Fri Oct 26 14:40:48 2018 -0700
----------------------------------------------------------------------
.../phoenix/end2end/BasePermissionsIT.java | 4 +-
.../phoenix/end2end/DropTableWithViewsIT.java | 151 ++++++++++
.../end2end/QueryDatabaseMetaDataIT.java | 4 +
.../end2end/TenantSpecificTablesDDLIT.java | 4 +-
.../coprocessor/MetaDataEndpointImpl.java | 46 ++-
.../phoenix/coprocessor/TaskRegionObserver.java | 292 +++++++++++++++++++
.../phoenix/jdbc/PhoenixDatabaseMetaData.java | 9 +-
.../query/ConnectionQueryServicesImpl.java | 20 +-
.../query/ConnectionlessQueryServicesImpl.java | 9 +
.../apache/phoenix/query/QueryConstants.java | 17 +-
.../org/apache/phoenix/query/QueryServices.java | 6 +
.../phoenix/query/QueryServicesOptions.java | 4 +
.../java/org/apache/phoenix/schema/PTable.java | 31 +-
.../phoenix/schema/stats/StatisticsUtil.java | 2 +
.../org/apache/phoenix/util/SchemaUtil.java | 10 +
.../java/org/apache/phoenix/query/BaseTest.java | 1 +
16 files changed, 589 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3903ad76/phoenix-core/src/it/java/org/apache/phoenix/end2end/BasePermissionsIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BasePermissionsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BasePermissionsIT.java
index 88a942e..932ce9f 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BasePermissionsIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BasePermissionsIT.java
@@ -428,7 +428,7 @@ public class BasePermissionsIT extends BaseTest {
@Override
public Object run() throws Exception {
try (Connection conn = getConnection(); Statement stmt = conn.createStatement();) {
- assertFalse(stmt.execute("DROP TABLE IF EXISTS " + tableName));
+ assertFalse(stmt.execute(String.format("DROP TABLE IF EXISTS %s CASCADE", tableName)));
}
return null;
}
@@ -653,7 +653,7 @@ public class BasePermissionsIT extends BaseTest {
@Override
public Object run() throws Exception {
try (Connection conn = getConnection(); Statement stmt = conn.createStatement();) {
- assertFalse(stmt.execute("DROP VIEW " + viewName));
+ assertFalse(stmt.execute(String.format("DROP VIEW %s CASCADE", viewName)));
}
return null;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3903ad76/phoenix-core/src/it/java/org/apache/phoenix/end2end/DropTableWithViewsIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/DropTableWithViewsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DropTableWithViewsIT.java
new file mode 100644
index 0000000..9502218
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/DropTableWithViewsIT.java
@@ -0,0 +1,151 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import static org.apache.phoenix.util.PhoenixRuntime.TENANT_ID_ATTRIB;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.fail;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.phoenix.coprocessor.TableViewFinderResult;
+import org.apache.phoenix.coprocessor.ViewFinder;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
+
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.query.QueryServicesOptions;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.util.SchemaUtil;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+
+@RunWith(Parameterized.class)
+public class DropTableWithViewsIT extends SplitSystemCatalogIT {
+
+ private final boolean isMultiTenant;
+ private final boolean columnEncoded;
+ private final String TENANT_SPECIFIC_URL1 = getUrl() + ';' + TENANT_ID_ATTRIB + "=" + TENANT1;
+
+ public DropTableWithViewsIT(boolean isMultiTenant, boolean columnEncoded) {
+ this.isMultiTenant = isMultiTenant;
+ this.columnEncoded = columnEncoded;
+ }
+
+ @Parameters(name="DropTableWithViewsIT_multiTenant={0}, columnEncoded={1}") // name is used by failsafe as file name in reports
+ public static Collection<Boolean[]> data() {
+ return Arrays.asList(new Boolean[][] {
+ { false, false }, { false, true },
+ { true, false }, { true, true } });
+ }
+
+ private String generateDDL(String format) {
+ return generateDDL("", format);
+ }
+
+ private String generateDDL(String options, String format) {
+ StringBuilder optionsBuilder = new StringBuilder(options);
+ if (!columnEncoded) {
+ if (optionsBuilder.length() != 0)
+ optionsBuilder.append(",");
+ optionsBuilder.append("COLUMN_ENCODED_BYTES=0");
+ }
+ if (isMultiTenant) {
+ if (optionsBuilder.length() !=0 )
+ optionsBuilder.append(",");
+ optionsBuilder.append("MULTI_TENANT=true");
+ }
+ return String.format(format, isMultiTenant ? "TENANT_ID VARCHAR NOT NULL, " : "",
+ isMultiTenant ? "TENANT_ID, " : "", optionsBuilder.toString());
+ }
+
+ @Test
+ public void testDropTableWithChildViews() throws Exception {
+ String baseTable = SchemaUtil.getTableName(SCHEMA1, generateUniqueName());
+ try (Connection conn = DriverManager.getConnection(getUrl());
+ Connection viewConn =
+ isMultiTenant ? DriverManager.getConnection(TENANT_SPECIFIC_URL1) : conn) {
+ String ddlFormat =
+ "CREATE TABLE IF NOT EXISTS " + baseTable + " ("
+ + " %s PK2 VARCHAR NOT NULL, V1 VARCHAR, V2 VARCHAR "
+ + " CONSTRAINT NAME_PK PRIMARY KEY (%s PK2)" + " ) %s";
+ conn.createStatement().execute(generateDDL(ddlFormat));
+ conn.commit();
+ // Create a view tree (i.e., tree of views) with depth of 2 and fanout factor of 4
+ for (int i = 0; i < 4; i++) {
+ String childView = SchemaUtil.getTableName(SCHEMA2, generateUniqueName());
+ String childViewDDL = "CREATE VIEW " + childView + " AS SELECT * FROM " + baseTable;
+ viewConn.createStatement().execute(childViewDDL);
+ for (int j = 0; j < 4; j++) {
+ String grandChildView = SchemaUtil.getTableName(SCHEMA2, generateUniqueName());
+ String grandChildViewDDL = "CREATE VIEW " + grandChildView + " AS SELECT * FROM " + childView;
+ viewConn.createStatement().execute(grandChildViewDDL);
+ }
+ }
+ // Drop the base table
+ String dropTable = String.format("DROP TABLE IF EXISTS %s CASCADE", baseTable);
+ conn.createStatement().execute(dropTable);
+
+ // Wait for the tasks for dropping child views to complete. The depth of the view tree is 2, so we expect that
+ // this will be done in two task handling runs, i.e., in tree task handling interval at most in general
+ // by assuming that each non-root level will be processed in one interval. To be on the safe side, we will
+ // wait at most 10 intervals.
+ long halfTimeInterval = config.getLong(QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB,
+ QueryServicesOptions.DEFAULT_TASK_HANDLING_INTERVAL_MS)/2;
+ ResultSet rs = null;
+ boolean timedOut = true;
+ Thread.sleep(3 * halfTimeInterval);
+ for (int i = 3; i < 20; i++) {
+ rs = conn.createStatement().executeQuery("SELECT * " +
+ " FROM " + PhoenixDatabaseMetaData.SYSTEM_TASK_NAME +
+ " WHERE " + PhoenixDatabaseMetaData.TASK_TYPE + " = " +
+ PTable.TaskType.DROP_CHILD_VIEWS.getSerializedValue());
+ Thread.sleep(halfTimeInterval);
+ if (!rs.next()) {
+ timedOut = false;
+ break;
+ }
+ }
+ if (timedOut) {
+ fail("Drop child view task execution timed out!");
+ }
+ // Views should be dropped by now
+ TableName linkTable = TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_NAME_BYTES);
+ TableViewFinderResult childViewsResult = new TableViewFinderResult();
+ ViewFinder.findAllRelatives(getUtility().getConnection().getTable(linkTable),
+ HConstants.EMPTY_BYTE_ARRAY,
+ SchemaUtil.getSchemaNameFromFullName(baseTable).getBytes(),
+ SchemaUtil.getTableNameFromFullName(baseTable).getBytes(),
+ PTable.LinkType.CHILD_TABLE,
+ childViewsResult);
+ assertTrue(childViewsResult.getLinks().size() == 0);
+ // There should not be any orphan views
+ rs = conn.createStatement().executeQuery("SELECT * FROM " + PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME +
+ " WHERE " + PhoenixDatabaseMetaData.TABLE_SCHEM + " = '" + SCHEMA2 +"'");
+ assertFalse(rs.next());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3903ad76/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java
index ab22b6d..226a2cc 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java
@@ -186,6 +186,10 @@ public class QueryDatabaseMetaDataIT extends ParallelStatsDisabledIT {
assertEquals(PhoenixDatabaseMetaData.SYSTEM_STATS_TABLE, rs.getString("TABLE_NAME"));
assertEquals(PTableType.SYSTEM.toString(), rs.getString("TABLE_TYPE"));
assertTrue(rs.next());
+ assertEquals(SYSTEM_CATALOG_SCHEMA, rs.getString("TABLE_SCHEM"));
+ assertEquals(PhoenixDatabaseMetaData.SYSTEM_TASK_TABLE, rs.getString("TABLE_NAME"));
+ assertEquals(PTableType.SYSTEM.toString(), rs.getString("TABLE_TYPE"));
+ assertTrue(rs.next());
assertEquals(null, rs.getString("TABLE_SCHEM"));
assertEquals(tableAName, rs.getString("TABLE_NAME"));
assertEquals(PTableType.TABLE.toString(), rs.getString("TABLE_TYPE"));
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3903ad76/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java
index 956b43c..85c9128 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/TenantSpecificTablesDDLIT.java
@@ -503,8 +503,10 @@ public class TenantSpecificTablesDDLIT extends BaseTenantSpecificTablesIT {
assertTableMetaData(rs, PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA, PhoenixDatabaseMetaData.TYPE_SEQUENCE, PTableType.SYSTEM);
assertTrue(rs.next());
assertTableMetaData(rs, SYSTEM_CATALOG_SCHEMA, PhoenixDatabaseMetaData.SYSTEM_STATS_TABLE, PTableType.SYSTEM);
+ assertTrue(rs.next());
+ assertTableMetaData(rs, SYSTEM_CATALOG_SCHEMA, PhoenixDatabaseMetaData.SYSTEM_TASK_TABLE, PTableType.SYSTEM);
assertFalse(rs.next());
-
+
rs = meta.getTables(null, "", StringUtil.escapeLike(tenantTable2), new String[] {TABLE.getValue().getString()});
assertFalse(rs.next());
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3903ad76/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
index aa78b1b..424b6d6 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/MetaDataEndpointImpl.java
@@ -86,6 +86,7 @@ import java.security.PrivilegedExceptionAction;
import java.sql.ResultSetMetaData;
import java.sql.SQLException;
import java.sql.Statement;
+import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -1936,7 +1937,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
table = loadTable(env, tableKey, cacheKey, clientTimeStamp, HConstants.LATEST_TIMESTAMP,
clientVersion);
} catch (ParentTableNotFoundException e) {
- dropChildMetadata(e.getParentSchemaName(), e.getParentTableName(), e.getParentTenantId());
+ dropChildViews(env, e.getParentTenantId(), e.getParentSchemaName(), e.getParentTableName());
}
if (table != null) {
if (table.getTimeStamp() < clientTimeStamp) {
@@ -1961,7 +1962,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
// check if the table was dropped, but had child views that were have not yet
// been cleaned up by compaction
if (!Bytes.toString(schemaName).equals(QueryConstants.SYSTEM_SCHEMA_NAME)) {
- dropChildMetadata(schemaName, tableName, tenantIdBytes);
+ dropChildViews(env, tenantIdBytes, schemaName, tableName);
}
byte[] parentTableKey = null;
@@ -2328,19 +2329,31 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
}
}
- private void dropChildMetadata(byte[] schemaName, byte[] tableName, byte[] tenantIdBytes)
+ public static void dropChildViews(RegionCoprocessorEnvironment env, byte[] tenantIdBytes, byte[] schemaName, byte[] tableName)
throws IOException, SQLException, ClassNotFoundException {
- TableViewFinderResult childViewsResult = new TableViewFinderResult();
- findAllChildViews(tenantIdBytes, schemaName, tableName, childViewsResult);
+ Table hTable =
+ ServerUtil.getHTableForCoprocessorScan(env,
+ SchemaUtil.getPhysicalTableName(
+ PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_NAME_BYTES,
+ env.getConfiguration()).getName());
+ TableViewFinderResult childViewsResult = ViewFinder.findRelatedViews(hTable, tenantIdBytes, schemaName, tableName,
+ PTable.LinkType.CHILD_TABLE, HConstants.LATEST_TIMESTAMP);
+
if (childViewsResult.hasLinks()) {
+
for (TableInfo viewInfo : childViewsResult.getLinks()) {
byte[] viewTenantId = viewInfo.getTenantId();
byte[] viewSchemaName = viewInfo.getSchemaName();
byte[] viewName = viewInfo.getTableName();
+ if (logger.isDebugEnabled()) {
+ logger.debug("dropChildViews :" + Bytes.toString(schemaName) + "." + Bytes.toString(tableName) +
+ " -> " + Bytes.toString(viewSchemaName) + "." + Bytes.toString(viewName) +
+ "with tenant id :" + Bytes.toString(viewTenantId));
+ }
Properties props = new Properties();
if (viewTenantId != null && viewTenantId.length != 0)
props.setProperty(PhoenixRuntime.TENANT_ID_ATTRIB, Bytes.toString(viewTenantId));
- try (PhoenixConnection connection = QueryUtil.getConnectionOnServer(env.getConfiguration())
+ try (PhoenixConnection connection = QueryUtil.getConnectionOnServer(props, env.getConfiguration())
.unwrap(PhoenixConnection.class)) {
MetaDataClient client = new MetaDataClient(connection);
org.apache.phoenix.parse.TableName viewTableName = org.apache.phoenix.parse.TableName
@@ -2351,7 +2364,6 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
}
}
}
-
private boolean execeededIndexQuota(PTableType tableType, PTable parentTable) {
return PTableType.INDEX == tableType && parentTable.getIndexes().size() >= maxIndexesPerTable;
}
@@ -2514,6 +2526,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
}
throw new IllegalStateException(msg);
}
+
// drop rows from catalog on this region
mutateRowsWithLocks(region, localMutations, Collections.<byte[]> emptySet(), HConstants.NO_NONCE,
HConstants.NO_NONCE);
@@ -2631,7 +2644,7 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
EnvironmentEdgeManager.currentTimeMillis(), null);
}
- if (tableType == PTableType.TABLE || tableType == PTableType.SYSTEM) {
+ if (tableType == PTableType.TABLE || tableType == PTableType.VIEW || tableType == PTableType.SYSTEM) {
// check to see if the table has any child views
try (Table hTable =
env.getTable(SchemaUtil.getPhysicalTableName(
@@ -2640,10 +2653,19 @@ public class MetaDataEndpointImpl extends MetaDataProtocol implements Coprocesso
boolean hasChildViews =
ViewFinder.hasChildViews(hTable, tenantId, schemaName, tableName,
clientTimeStamp);
- if (hasChildViews && !isCascade) {
- // DROP without CASCADE on tables with child views is not permitted
- return new MetaDataMutationResult(MutationCode.UNALLOWED_TABLE_MUTATION,
- EnvironmentEdgeManager.currentTimeMillis(), null);
+ if (hasChildViews) {
+ if (!isCascade) {
+ // DROP without CASCADE on tables with child views is not permitted
+ return new MetaDataMutationResult(MutationCode.UNALLOWED_TABLE_MUTATION,
+ EnvironmentEdgeManager.currentTimeMillis(), null);
+ }
+ try {
+ PhoenixConnection conn = QueryUtil.getConnectionOnServer(env.getConfiguration()).unwrap(PhoenixConnection.class);
+ TaskRegionObserver.addTask(conn, PTable.TaskType.DROP_CHILD_VIEWS, Bytes.toString(tenantId),
+ Bytes.toString(schemaName), Bytes.toString(tableName), this.accessCheckEnabled);
+ } catch (Throwable t) {
+ logger.error("Adding a task to drop child views failed!", t);
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3903ad76/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/TaskRegionObserver.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/TaskRegionObserver.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/TaskRegionObserver.java
new file mode 100644
index 0000000..ca71961
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/TaskRegionObserver.java
@@ -0,0 +1,292 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.coprocessor;
+
+import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Timestamp;
+import java.sql.Types;
+
+import java.util.Properties;
+import java.util.TimerTask;
+import java.util.concurrent.ScheduledThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import javax.annotation.concurrent.GuardedBy;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.CoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.BaseRegionObserver;
+import org.apache.hadoop.hbase.coprocessor.ObserverContext;
+import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment;
+import org.apache.hadoop.hbase.coprocessor.RegionObserver;
+import org.apache.hadoop.hbase.ipc.RpcServer.Call;
+import org.apache.hadoop.hbase.ipc.RpcUtil;
+import org.apache.hadoop.hbase.security.User;
+import org.apache.log4j.Level;
+import org.apache.log4j.Logger;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.query.QueryServicesOptions;
+import org.apache.phoenix.schema.MetaDataClient;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTable.TaskType;
+
+import org.apache.phoenix.util.PhoenixRuntime;
+import org.apache.phoenix.util.QueryUtil;
+
+
+/**
+ * Coprocessor for task related operations. This coprocessor would only be registered
+ * to SYSTEM.TASK table
+ */
+
+public class TaskRegionObserver extends BaseRegionObserver {
+ public static final Log LOG = LogFactory.getLog(TaskRegionObserver.class);
+ protected ScheduledThreadPoolExecutor executor = new ScheduledThreadPoolExecutor(TaskType.values().length);
+ private long timeInterval = QueryServicesOptions.DEFAULT_TASK_HANDLING_INTERVAL_MS;
+ private long timeMaxInterval = QueryServicesOptions.DEFAULT_TASK_HANDLING_MAX_INTERVAL_MS;
+ @GuardedBy("TaskRegionObserver.class")
+ // initial delay before the first task is handled
+ private static final long initialDelay = 10000; // 10 secs
+
+ @Override
+ public void preClose(final ObserverContext<RegionCoprocessorEnvironment> c,
+ boolean abortRequested) {
+ executor.shutdownNow();
+ }
+
+ @Override
+ public void start(CoprocessorEnvironment env) throws IOException {
+ Configuration config = env.getConfiguration();
+ timeInterval =
+ config.getLong(
+ QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB,
+ QueryServicesOptions.DEFAULT_TASK_HANDLING_INTERVAL_MS);
+ timeMaxInterval =
+ config.getLong(
+ QueryServices.TASK_HANDLING_MAX_INTERVAL_MS_ATTRIB,
+ QueryServicesOptions.DEFAULT_TASK_HANDLING_MAX_INTERVAL_MS);
+ }
+
+ @Override
+ public void postOpen(ObserverContext<RegionCoprocessorEnvironment> e) {
+ final RegionCoprocessorEnvironment env = e.getEnvironment();
+
+ // turn off verbose deprecation logging
+ Logger deprecationLogger = Logger.getLogger("org.apache.hadoop.conf.Configuration.deprecation");
+ if (deprecationLogger != null) {
+ deprecationLogger.setLevel(Level.WARN);
+ }
+
+ DropChildViewsTask task = new DropChildViewsTask(e.getEnvironment(), timeMaxInterval);
+ executor.scheduleWithFixedDelay(task, initialDelay, timeInterval, TimeUnit.MILLISECONDS);
+ }
+
+ private static void mutateSystemTaskTable(final PhoenixConnection conn, final PreparedStatement stmt, boolean accessCheckEnabled)
+ throws IOException {
+ // we need to mutate SYSTEM.TASK with HBase/login user if access is enabled.
+ if (accessCheckEnabled) {
+ User.runAsLoginUser(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ final Call rpcContext = RpcUtil.getRpcContext();
+ // setting RPC context as null so that user can be reset
+ try {
+ RpcUtil.setRpcContext(null);
+ stmt.execute();
+ conn.commit();
+ } catch (SQLException e) {
+ throw new IOException(e);
+ } finally {
+ // setting RPC context back to original context of the RPC
+ RpcUtil.setRpcContext(rpcContext);
+ }
+ return null;
+ }
+ });
+ }
+ else {
+ try {
+ stmt.execute();
+ conn.commit();
+ } catch (SQLException e) {
+ throw new IOException(e);
+ }
+ }
+ }
+
+ public static void addTask(PhoenixConnection conn, TaskType taskType, String tenantId, String schemaName,
+ String tableName, boolean accessCheckEnabled)
+ throws IOException {
+ PreparedStatement stmt = null;
+ try {
+ stmt = conn.prepareStatement("UPSERT INTO " +
+ PhoenixDatabaseMetaData.SYSTEM_TASK_NAME + " ( " +
+ PhoenixDatabaseMetaData.TASK_TYPE + ", " +
+ PhoenixDatabaseMetaData.TENANT_ID + ", " +
+ PhoenixDatabaseMetaData.TABLE_SCHEM + ", " +
+ PhoenixDatabaseMetaData.TABLE_NAME + " ) VALUES(?,?,?,?)");
+ stmt.setByte(1, taskType.getSerializedValue());
+ if (tenantId != null) {
+ stmt.setString(2, tenantId);
+ } else {
+ stmt.setNull(2, Types.VARCHAR);
+ }
+ if (schemaName != null) {
+ stmt.setString(3, schemaName);
+ } else {
+ stmt.setNull(3, Types.VARCHAR);
+ }
+ stmt.setString(4, tableName);
+ } catch (SQLException e) {
+ throw new IOException(e);
+ }
+ mutateSystemTaskTable(conn, stmt, accessCheckEnabled);
+ }
+
+ public static void deleteTask(PhoenixConnection conn, TaskType taskType, Timestamp ts, String tenantId,
+ String schemaName, String tableName, boolean accessCheckEnabled) throws IOException {
+ PreparedStatement stmt = null;
+ try {
+ stmt = conn.prepareStatement("DELETE FROM " +
+ PhoenixDatabaseMetaData.SYSTEM_TASK_NAME +
+ " WHERE " + PhoenixDatabaseMetaData.TASK_TYPE + " = ? AND " +
+ PhoenixDatabaseMetaData.TASK_TS + " = ? AND " +
+ PhoenixDatabaseMetaData.TENANT_ID + (tenantId == null ? " IS NULL " : " = '" + tenantId + "'") + " AND " +
+ PhoenixDatabaseMetaData.TABLE_SCHEM + (schemaName == null ? " IS NULL " : " = '" + schemaName + "'") + " AND " +
+ PhoenixDatabaseMetaData.TABLE_NAME + " = ?");
+ stmt.setByte(1, taskType.getSerializedValue());
+ stmt.setTimestamp(2, ts);
+ stmt.setString(3, tableName);
+ } catch (SQLException e) {
+ throw new IOException(e);
+ }
+ mutateSystemTaskTable(conn, stmt, accessCheckEnabled);
+ }
+
+ /**
+ * Task runs periodically to clean up task of child views whose parent is dropped
+ *
+ */
+ public static class DropChildViewsTask extends TimerTask {
+ private RegionCoprocessorEnvironment env;
+ private long timeMaxInterval;
+ private boolean accessCheckEnabled;
+
+ public DropChildViewsTask(RegionCoprocessorEnvironment env, long timeMaxInterval) {
+ this.env = env;
+ this.accessCheckEnabled = env.getConfiguration().getBoolean(QueryServices.PHOENIX_ACLS_ENABLED,
+ QueryServicesOptions.DEFAULT_PHOENIX_ACLS_ENABLED);
+ this.timeMaxInterval = timeMaxInterval;
+ }
+
+ @Override
+ public void run() {
+ PhoenixConnection connForTask = null;
+ Timestamp timestamp = null;
+ String tenantId = null;
+ byte[] tenantIdBytes;
+ String schemaName= null;
+ byte[] schemaNameBytes;
+ String tableName = null;
+ byte[] tableNameBytes;
+ PhoenixConnection pconn;
+ try {
+ String taskQuery = "SELECT " +
+ PhoenixDatabaseMetaData.TASK_TS + ", " +
+ PhoenixDatabaseMetaData.TENANT_ID + ", " +
+ PhoenixDatabaseMetaData.TABLE_SCHEM + ", " +
+ PhoenixDatabaseMetaData.TABLE_NAME +
+ " FROM " + PhoenixDatabaseMetaData.SYSTEM_TASK_NAME +
+ " WHERE "+ PhoenixDatabaseMetaData.TASK_TYPE + " = " + PTable.TaskType.DROP_CHILD_VIEWS.getSerializedValue();
+
+ connForTask = QueryUtil.getConnectionOnServer(env.getConfiguration()).unwrap(PhoenixConnection.class);
+ PreparedStatement taskStatement = connForTask.prepareStatement(taskQuery);
+ ResultSet rs = taskStatement.executeQuery();
+ while (rs.next()) {
+ try {
+ // delete child views only if the parent table is deleted from the system catalog
+ timestamp = rs.getTimestamp(1);
+ tenantId = rs.getString(2);
+ tenantIdBytes= rs.getBytes(2);
+ schemaName= rs.getString(3);
+ schemaNameBytes = rs.getBytes(3);
+ tableName= rs.getString(4);
+ tableNameBytes = rs.getBytes(4);
+
+ if (tenantId != null) {
+ Properties tenantProps = new Properties();
+ tenantProps.setProperty(PhoenixRuntime.TENANT_ID_ATTRIB, tenantId);
+ pconn = QueryUtil.getConnectionOnServer(tenantProps, env.getConfiguration()).unwrap(PhoenixConnection.class);
+
+ }
+ else {
+ pconn = QueryUtil.getConnectionOnServer(env.getConfiguration()).unwrap(PhoenixConnection.class);
+ }
+
+ MetaDataProtocol.MetaDataMutationResult result = new MetaDataClient(pconn).updateCache(pconn.getTenantId(),
+ schemaName, tableName, true);
+ if (result.getMutationCode() != MetaDataProtocol.MutationCode.TABLE_ALREADY_EXISTS) {
+ MetaDataEndpointImpl.dropChildViews(env, tenantIdBytes, schemaNameBytes, tableNameBytes);
+ } else if (System.currentTimeMillis() < timeMaxInterval + timestamp.getTime()) {
+ // skip this task as it has not been expired and its parent table has not been dropped yet
+ LOG.info("Skipping a child view drop task. The parent table has not been dropped yet : " +
+ schemaName + "." + tableName +
+ " with tenant id " + (tenantId == null ? " IS NULL" : tenantId) +
+ " and timestamp " + timestamp.toString());
+ continue;
+ }
+ else {
+ LOG.warn(" A drop child view task has expired and will be removed from the system task table : " +
+ schemaName + "." + tableName +
+ " with tenant id " + (tenantId == null ? " IS NULL" : tenantId) +
+ " and timestamp " + timestamp.toString());
+ }
+
+ deleteTask(connForTask, PTable.TaskType.DROP_CHILD_VIEWS, timestamp, tenantId, schemaName,
+ tableName, this.accessCheckEnabled);
+ }
+ catch (Throwable t) {
+ LOG.warn("Exception while dropping a child view task. " +
+ "It will be retried in the next system task table scan : " +
+ schemaName + "." + tableName +
+ " with tenant id " + (tenantId == null ? " IS NULL" : tenantId) +
+ " and timestamp " + timestamp.toString(), t);
+ }
+ }
+ } catch (Throwable t) {
+ LOG.error("DropChildViewsTask failed!", t);
+ } finally {
+ if (connForTask != null) {
+ try {
+ connForTask.close();
+ } catch (SQLException ignored) {
+ LOG.debug("DropChildViewsTask can't close connection", ignored);
+ }
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3903ad76/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
index 52dfe99..3ff62e2 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/jdbc/PhoenixDatabaseMetaData.java
@@ -213,6 +213,10 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData {
public static final byte[] INDEX_TYPE_BYTES = Bytes.toBytes(INDEX_TYPE);
public static final String LINK_TYPE = "LINK_TYPE";
public static final byte[] LINK_TYPE_BYTES = Bytes.toBytes(LINK_TYPE);
+ public static final String TASK_TYPE = "TASK_TYPE";
+ public static final byte[] TASK_TYPE_BYTES = Bytes.toBytes(TASK_TYPE);
+ public static final String TASK_TS = "TASK_TS";
+ public static final byte[] TASK_TS_BYTES = Bytes.toBytes(TASK_TS);
public static final String ARRAY_SIZE = "ARRAY_SIZE";
public static final byte[] ARRAY_SIZE_BYTES = Bytes.toBytes(ARRAY_SIZE);
public static final String VIEW_CONSTANT = "VIEW_CONSTANT";
@@ -361,7 +365,10 @@ public class PhoenixDatabaseMetaData implements DatabaseMetaData {
public static final byte[] SYSTEM_CHILD_LINK_NAME_BYTES = Bytes.toBytes(SYSTEM_CHILD_LINK_NAME);
public static final TableName SYSTEM_LINK_HBASE_TABLE_NAME = TableName.valueOf(SYSTEM_CHILD_LINK_NAME);
-
+ public static final String SYSTEM_TASK_TABLE = "TASK";
+ public static final String SYSTEM_TASK_NAME = SchemaUtil.getTableName(SYSTEM_CATALOG_SCHEMA, SYSTEM_TASK_TABLE);
+ public static final byte[] SYSTEM_TASK_NAME_BYTES = Bytes.toBytes(SYSTEM_TASK_NAME);
+ public static final TableName SYSTEM_TASK_HBASE_TABLE_NAME = TableName.valueOf(SYSTEM_TASK_NAME);
//SYSTEM:LOG
public static final String SYSTEM_LOG_TABLE = "LOG";
public static final String QUERY_ID = "QUERY_ID";
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3903ad76/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index b68637a..ebcb7b9 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -152,6 +152,7 @@ import org.apache.phoenix.coprocessor.MetaDataRegionObserver;
import org.apache.phoenix.coprocessor.ScanRegionObserver;
import org.apache.phoenix.coprocessor.SequenceRegionObserver;
import org.apache.phoenix.coprocessor.ServerCachingEndpointImpl;
+import org.apache.phoenix.coprocessor.TaskRegionObserver;
import org.apache.phoenix.coprocessor.UngroupedAggregateRegionObserver;
import org.apache.phoenix.coprocessor.generated.MetaDataProtos;
import org.apache.phoenix.coprocessor.generated.MetaDataProtos.AddColumnRequest;
@@ -958,7 +959,11 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
if (!descriptor.hasCoprocessor(SequenceRegionObserver.class.getName())) {
descriptor.addCoprocessor(SequenceRegionObserver.class.getName(), null, priority, null);
}
+ } else if (SchemaUtil.isTaskTable(tableName)) {
+ if(!descriptor.hasCoprocessor(TaskRegionObserver.class.getName())) {
+ descriptor.addCoprocessor(TaskRegionObserver.class.getName(), null, priority, null);
}
+ }
if (isTransactional) {
Class<? extends RegionObserver> coprocessorClass = provider.getTransactionProvider().getCoprocessor();
@@ -2812,6 +2817,10 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
return setSystemDDLProperties(QueryConstants.CREATE_MUTEX_METADTA);
}
+ protected String getTaskDDL() {
+ return setSystemDDLProperties(QueryConstants.CREATE_TASK_METADATA);
+ }
+
private String setSystemDDLProperties(String ddl) {
return String.format(ddl,
props.getInt(DEFAULT_SYSTEM_MAX_VERSIONS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_MAX_VERSIONS),
@@ -3038,6 +3047,10 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
try {
metaConnection.createStatement().executeUpdate(getMutexDDL());
} catch (TableAlreadyExistsException e) {}
+ try {
+ metaConnection.createStatement().executeUpdate(getTaskDDL());
+ } catch (TableAlreadyExistsException e) {}
+
// Catch the IOException to log the error message and then bubble it up for the client to retry.
}
@@ -3496,6 +3509,9 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
try {
metaConnection.createStatement().executeUpdate(getMutexDDL());
} catch (NewerTableAlreadyExistsException e) {} catch (TableAlreadyExistsException e) {}
+ try {
+ metaConnection.createStatement().executeUpdate(getTaskDDL());
+ } catch (NewerTableAlreadyExistsException e) {} catch (TableAlreadyExistsException e) {}
// In case namespace mapping is enabled and system table to system namespace mapping is also enabled,
// create an entry for the SYSTEM namespace in the SYSCAT table, so that GRANT/REVOKE commands can work
@@ -3733,8 +3749,8 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
// No tables exist matching "SYSTEM\..*", they are all already in "SYSTEM:.*"
if (tableNames.size() == 0) { return; }
// Try to move any remaining tables matching "SYSTEM\..*" into "SYSTEM:"
- if (tableNames.size() > 7) {
- logger.warn("Expected 7 system tables but found " + tableNames.size() + ":" + tableNames);
+ if (tableNames.size() > 8) {
+ logger.warn("Expected 8 system tables but found " + tableNames.size() + ":" + tableNames);
}
byte[] mappedSystemTable = SchemaUtil
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3903ad76/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
index bbfacc0..655de0d 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionlessQueryServicesImpl.java
@@ -185,6 +185,10 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
return setSystemDDLProperties(QueryConstants.CREATE_MUTEX_METADTA);
}
+ protected String getTaskDDL() {
+ return setSystemDDLProperties(QueryConstants.CREATE_TASK_METADATA);
+ }
+
private String setSystemDDLProperties(String ddl) {
return String.format(ddl,
props.getInt(DEFAULT_SYSTEM_MAX_VERSIONS_ATTRIB, QueryServicesOptions.DEFAULT_SYSTEM_MAX_VERSIONS),
@@ -388,6 +392,11 @@ public class ConnectionlessQueryServicesImpl extends DelegateQueryServices imple
.executeUpdate(getMutexDDL());
} catch (NewerTableAlreadyExistsException ignore) {
}
+ try {
+ metaConnection.createStatement()
+ .executeUpdate(getTaskDDL());
+ } catch (NewerTableAlreadyExistsException ignore) {
+ }
} catch (SQLException e) {
sqlE = e;
} finally {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3903ad76/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
index bbff343..c45fec9 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryConstants.java
@@ -347,8 +347,8 @@ public interface QueryConstants {
// Links from parent to child views are stored in a separate table for
// scalability
- public static final String CREATE_CHILD_LINK_METADATA = "CREATE TABLE " + SYSTEM_CATALOG_SCHEMA + ".\""
- + SYSTEM_CHILD_LINK_TABLE + "\"(\n" +
+ public static final String CREATE_CHILD_LINK_METADATA = "CREATE TABLE " + SYSTEM_CATALOG_SCHEMA + ".\"" +
+ SYSTEM_CHILD_LINK_TABLE + "\"(\n" +
// PK columns
TENANT_ID + " VARCHAR NULL," + TABLE_SCHEM + " VARCHAR NULL," + TABLE_NAME + " VARCHAR NOT NULL,"
+ COLUMN_NAME + " VARCHAR NULL," + COLUMN_FAMILY + " VARCHAR NULL," + LINK_TYPE + " UNSIGNED_TINYINT,\n"
@@ -371,4 +371,17 @@ public interface QueryConstants {
HColumnDescriptor.KEEP_DELETED_CELLS + "=%s,\n" +
PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE;
+ public static final String CREATE_TASK_METADATA =
+ "CREATE TABLE " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_TASK_TABLE + "\"(\n" +
+ // PK columns
+ TASK_TYPE + " UNSIGNED_TINYINT NOT NULL," +
+ TASK_TS + " TIMESTAMP NOT NULL," +
+ TENANT_ID + " VARCHAR NULL," +
+ TABLE_SCHEM + " VARCHAR NULL," +
+ TABLE_NAME + " VARCHAR NOT NULL,\n" +
+ "CONSTRAINT " + SYSTEM_TABLE_PK_NAME + " PRIMARY KEY (" + TASK_TYPE + "," + TASK_TS + " ROW_TIMESTAMP," + TENANT_ID + "," + TABLE_SCHEM + "," +
+ TABLE_NAME + "))\n" +
+ HConstants.VERSIONS + "=%s,\n" +
+ HColumnDescriptor.KEEP_DELETED_CELLS + "=%s,\n" +
+ PhoenixDatabaseMetaData.TRANSACTIONAL + "=" + Boolean.FALSE;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3903ad76/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
index 337bb05..d06f07c 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServices.java
@@ -320,6 +320,12 @@ public interface QueryServices extends SQLCloseable {
public static final String SYSTEM_CATALOG_SPLITTABLE = "phoenix.system.catalog.splittable";
+ // The parameters defined for handling task stored in table SYSTEM.TASK
+ // The time interval between periodic scans of table SYSTEM.TASK
+ public static final String TASK_HANDLING_INTERVAL_MS_ATTRIB = "phoenix.task.handling.interval.ms";
+ // The maximum time for a task to stay in table SYSTEM.TASK
+ public static final String TASK_HANDLING_MAX_INTERVAL_MS_ATTRIB = "phoenix.task.handling.maxInterval.ms";
+
/**
* Get executor service used for parallel scans
*/
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3903ad76/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
index 35dbe3a..a6a73f7 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/QueryServicesOptions.java
@@ -340,6 +340,10 @@ public class QueryServicesOptions {
public static final int DEFAULT_UPDATE_CACHE_FREQUENCY = 0;
public static final int DEFAULT_SMALL_SCAN_THRESHOLD = 100;
+ // default system task handling interval in milliseconds
+ public static final long DEFAULT_TASK_HANDLING_INTERVAL_MS = 60*1000; // 1 min
+ public static final long DEFAULT_TASK_HANDLING_MAX_INTERVAL_MS = 30*60*1000; // 30 minutes
+
@SuppressWarnings("serial")
public static final Set<String> DEFAULT_QUERY_SERVER_SKIP_WORDS = new HashSet<String>() {
{
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3903ad76/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
index 8cbf757..6dfe411 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTable.java
@@ -192,7 +192,36 @@ public interface PTable extends PMetaDataEntity {
return LinkType.values()[serializedValue-1];
}
}
-
+
+ public enum TaskType {
+ DROP_CHILD_VIEWS((byte)1);
+
+ private final byte[] byteValue;
+ private final byte serializedValue;
+
+ TaskType(byte serializedValue) {
+ this.serializedValue = serializedValue;
+ this.byteValue = Bytes.toBytes(this.name());
+ }
+
+ public byte[] getBytes() {
+ return byteValue;
+ }
+
+ public byte getSerializedValue() {
+ return this.serializedValue;
+ }
+ public static TaskType getDefault() {
+ return DROP_CHILD_VIEWS;
+ }
+ public static TaskType fromSerializedValue(byte serializedValue) {
+ if (serializedValue < 1 || serializedValue > TaskType.values().length) {
+ throw new IllegalArgumentException("Invalid TaskType " + serializedValue);
+ }
+ return TaskType.values()[serializedValue-1];
+ }
+ }
+
public enum ImmutableStorageScheme implements ColumnValueEncoderDecoderSupplier {
ONE_CELL_PER_COLUMN((byte)1) {
@Override
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3903ad76/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
index 4a758b7..23b1fcc 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/stats/StatisticsUtil.java
@@ -62,10 +62,12 @@ public class StatisticsUtil {
DISABLE_STATS.add(TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_FUNCTION_NAME));
DISABLE_STATS.add(TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_SEQUENCE_NAME));
DISABLE_STATS.add(TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_STATS_NAME));
+ DISABLE_STATS.add(TableName.valueOf(PhoenixDatabaseMetaData.SYSTEM_TASK_NAME));
DISABLE_STATS.add(SchemaUtil.getPhysicalTableName(PhoenixDatabaseMetaData.SYSTEM_CATALOG_NAME_BYTES,true));
DISABLE_STATS.add(SchemaUtil.getPhysicalTableName(PhoenixDatabaseMetaData.SYSTEM_FUNCTION_NAME_BYTES,true));
DISABLE_STATS.add(SchemaUtil.getPhysicalTableName(PhoenixDatabaseMetaData.SYSTEM_SEQUENCE_NAME_BYTES,true));
DISABLE_STATS.add(SchemaUtil.getPhysicalTableName(PhoenixDatabaseMetaData.SYSTEM_STATS_NAME_BYTES,true));
+ DISABLE_STATS.add(SchemaUtil.getPhysicalTableName(PhoenixDatabaseMetaData.SYSTEM_TASK_NAME_BYTES,true));
}
private StatisticsUtil() {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3903ad76/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
index ec2eb14..24a0e12 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/SchemaUtil.java
@@ -540,6 +540,12 @@ public class SchemaUtil {
|| Bytes.compareTo(tableName, SchemaUtil
.getPhysicalTableName(PhoenixDatabaseMetaData.SYSTEM_SEQUENCE_NAME_BYTES, true).getName()) == 0;
}
+
+ public static boolean isTaskTable(byte[] tableName) {
+ return Bytes.compareTo(tableName, PhoenixDatabaseMetaData.SYSTEM_TASK_NAME_BYTES) == 0
+ || Bytes.compareTo(tableName, SchemaUtil
+ .getPhysicalTableName(PhoenixDatabaseMetaData.SYSTEM_TASK_NAME_BYTES, true).getName()) == 0;
+ }
public static boolean isChildLinkTable(byte[] tableName) {
return Bytes.compareTo(tableName, SYSTEM_CHILD_LINK_NAME_BYTES) == 0 || Bytes.compareTo(tableName,
@@ -550,6 +556,10 @@ public class SchemaUtil {
return PhoenixDatabaseMetaData.SYSTEM_SEQUENCE_NAME.equals(table.getName().getString());
}
+ public static boolean isTaskTable(PTable table) {
+ return PhoenixDatabaseMetaData.SYSTEM_TASK_NAME.equals(table.getName().getString());
+ }
+
public static boolean isMetaTable(PTable table) {
return PhoenixDatabaseMetaData.SYSTEM_CATALOG_SCHEMA.equals(table.getSchemaName().getString()) && PhoenixDatabaseMetaData.SYSTEM_CATALOG_TABLE.equals(table.getTableName().getString());
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/3903ad76/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
index 5ca247b..e6e3936 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/BaseTest.java
@@ -620,6 +620,7 @@ public abstract class BaseTest {
conf.setInt("hbase.assignment.zkevent.workers", 5);
conf.setInt("hbase.assignment.threads.max", 5);
conf.setInt("hbase.catalogjanitor.interval", 5000);
+ conf.setInt(QueryServices.TASK_HANDLING_INTERVAL_MS_ATTRIB, 1000);
conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2);
conf.setInt(NUM_CONCURRENT_INDEX_WRITER_THREADS_CONF_KEY, 1);
return conf;
[04/22] phoenix git commit: PHOENIX-4872: BulkLoad has bug when
loading on single-cell-array-with-offsets table.
Posted by ja...@apache.org.
PHOENIX-4872: BulkLoad has bug when loading on single-cell-array-with-offsets table.
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/fe466ebe
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/fe466ebe
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/fe466ebe
Branch: refs/heads/omid2
Commit: fe466ebe8e546c24f686232e7f3be03f96f1448a
Parents: 527098e
Author: s.kadam <s....@gus.com>
Authored: Mon Oct 29 14:47:21 2018 -0700
Committer: Thomas D'Silva <td...@apache.org>
Committed: Tue Oct 30 14:56:22 2018 -0700
----------------------------------------------------------------------
.../phoenix/end2end/CsvBulkLoadToolIT.java | 51 ++++++++++++++++++++
.../mapreduce/FormatToBytesWritableMapper.java | 10 ++--
2 files changed, 56 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/fe466ebe/phoenix-core/src/it/java/org/apache/phoenix/end2end/CsvBulkLoadToolIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CsvBulkLoadToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CsvBulkLoadToolIT.java
index 40fe900..7e4226d 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/CsvBulkLoadToolIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/CsvBulkLoadToolIT.java
@@ -35,7 +35,10 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.FileAlreadyExistsException;
+import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.mapreduce.CsvBulkLoadTool;
+import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTableKey;
import org.apache.phoenix.util.DateUtil;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.ReadOnlyProps;
@@ -446,4 +449,52 @@ public class CsvBulkLoadToolIT extends BaseOwnClusterIT {
rs.close();
stmt.close();
}
+
+ /**
+ * This test case validates the import using CsvBulkLoadTool in
+ * SingleCellArrayWithOffsets table.
+ * PHOENIX-4872
+ */
+
+ @Test
+ public void testImportInSingleCellArrayWithOffsetsTable() throws Exception {
+ Statement stmt = conn.createStatement();
+ stmt.execute("CREATE IMMUTABLE TABLE S.TABLE12 (ID INTEGER NOT NULL PRIMARY KEY," +
+ " CF0.NAME VARCHAR, CF0.T DATE, CF1.T2 DATE, CF2.T3 DATE) " +
+ "IMMUTABLE_STORAGE_SCHEME=SINGLE_CELL_ARRAY_WITH_OFFSETS");
+ PhoenixConnection phxConn = conn.unwrap(PhoenixConnection.class);
+ PTable table = phxConn.getTable(new PTableKey(null, "S.TABLE12"));
+
+ assertEquals(PTable.ImmutableStorageScheme.SINGLE_CELL_ARRAY_WITH_OFFSETS,
+ table.getImmutableStorageScheme());
+
+ FileSystem fs = FileSystem.get(getUtility().getConfiguration());
+ FSDataOutputStream outputStream = fs.create(new Path("/tmp/inputSCAWO.csv"));
+ PrintWriter printWriter = new PrintWriter(outputStream);
+ printWriter.println("1,Name 1,1970/01/01,1970/02/01,1970/03/01");
+ printWriter.println("2,Name 2,1970/01/02,1970/02/02,1970/03/02");
+ printWriter.println("3,Name 1,1970/01/01,1970/02/03,1970/03/01");
+ printWriter.println("4,Name 2,1970/01/02,1970/02/04,1970/03/02");
+ printWriter.println("5,Name 1,1970/01/01,1970/02/05,1970/03/01");
+ printWriter.println("6,Name 2,1970/01/02,1970/02/06,1970/03/02");
+ printWriter.close();
+
+ CsvBulkLoadTool csvBulkLoadTool = new CsvBulkLoadTool();
+ csvBulkLoadTool.setConf(new Configuration(getUtility().getConfiguration()));
+ csvBulkLoadTool.getConf().set(DATE_FORMAT_ATTRIB,"yyyy/MM/dd");
+ int exitCode = csvBulkLoadTool.run(new String[] {
+ "--input", "/tmp/inputSCAWO.csv",
+ "--table", "table12",
+ "--schema", "s",
+ "--zookeeper", zkQuorum});
+ assertEquals(0, exitCode);
+
+ ResultSet rs = stmt.executeQuery("SELECT COUNT(1) FROM S.TABLE12");
+ assertTrue(rs.next());
+ assertEquals(6, rs.getInt(1));
+
+ rs.close();
+ stmt.close();
+
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/fe466ebe/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToBytesWritableMapper.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToBytesWritableMapper.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToBytesWritableMapper.java
index 360859e..93ab188 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToBytesWritableMapper.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/FormatToBytesWritableMapper.java
@@ -242,12 +242,12 @@ public abstract class FormatToBytesWritableMapper<RECORD> extends Mapper<LongWri
columnIndex++;
}
}
- byte[] emptyColumnFamily = SchemaUtil.getEmptyColumnFamily(table);
- byte[] emptyKeyValue = EncodedColumnsUtil.getEmptyKeyValueInfo(table).getFirst();
- byte[] cfn = Bytes.add(emptyColumnFamily, QueryConstants.NAMESPACE_SEPARATOR_BYTES, emptyKeyValue);
- columnIndexes.put(cfn, new Integer(columnIndex));
- columnIndex++;
}
+ byte[] emptyColumnFamily = SchemaUtil.getEmptyColumnFamily(table);
+ byte[] emptyKeyValue = EncodedColumnsUtil.getEmptyKeyValueInfo(table).getFirst();
+ byte[] cfn = Bytes.add(emptyColumnFamily, QueryConstants.NAMESPACE_SEPARATOR_BYTES, emptyKeyValue);
+ columnIndexes.put(cfn, new Integer(columnIndex));
+ columnIndex++;
}
}
[21/22] phoenix git commit: Merge branch '4.x-HBase-1.3' into omid2
Posted by ja...@apache.org.
Merge branch '4.x-HBase-1.3' into omid2
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/cf087158
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/cf087158
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/cf087158
Branch: refs/heads/omid2
Commit: cf087158170dbf7c9d18c02bedb76f5ec91d7b23
Parents: 0f2b1b8 169270d
Author: James Taylor <ja...@apache.org>
Authored: Sat Nov 10 12:14:49 2018 -0800
Committer: James Taylor <ja...@apache.org>
Committed: Sat Nov 10 12:14:49 2018 -0800
----------------------------------------------------------------------
bin/phoenix_utils.py | 2 +
phoenix-assembly/pom.xml | 4 +
.../src/build/components/all-common-jars.xml | 8 +
.../org/apache/phoenix/end2end/AggregateIT.java | 987 +-------------
.../apache/phoenix/end2end/BaseAggregateIT.java | 1022 ++++++++++++++
.../apache/phoenix/end2end/BaseOrderByIT.java | 940 +++++++++++++
.../phoenix/end2end/BasePermissionsIT.java | 4 +-
.../phoenix/end2end/CsvBulkLoadToolIT.java | 51 +
.../phoenix/end2end/DropTableWithViewsIT.java | 151 +++
.../MigrateSystemTablesToSystemNamespaceIT.java | 4 +-
.../org/apache/phoenix/end2end/OrderByIT.java | 943 ++-----------
.../end2end/ParallelStatsDisabledIT.java | 40 +
.../phoenix/end2end/PropertiesInSyncIT.java | 172 ++-
.../end2end/QueryDatabaseMetaDataIT.java | 4 +
.../SystemCatalogCreationOnConnectionIT.java | 4 +-
.../end2end/TableSnapshotReadsMapReduceIT.java | 122 +-
.../end2end/TenantSpecificTablesDDLIT.java | 4 +-
.../phoenix/end2end/index/IndexMetadataIT.java | 145 +-
.../end2end/salted/BaseSaltedTableIT.java | 474 +++++++
.../phoenix/end2end/salted/SaltedTableIT.java | 450 +------
.../monitoring/BasePhoenixMetricsIT.java | 4 +-
.../org/apache/phoenix/rpc/UpdateCacheIT.java | 134 +-
.../apache/phoenix/compile/DeleteCompiler.java | 5 +-
.../apache/phoenix/compile/FromCompiler.java | 66 +-
.../apache/phoenix/compile/JoinCompiler.java | 53 +-
.../compile/TupleProjectionCompiler.java | 60 +-
.../apache/phoenix/compile/UnionCompiler.java | 41 +-
.../apache/phoenix/compile/UpsertCompiler.java | 12 +-
.../coprocessor/MetaDataEndpointImpl.java | 142 +-
.../phoenix/coprocessor/TaskRegionObserver.java | 295 ++++
.../UngroupedAggregateRegionObserver.java | 6 +-
.../coprocessor/WhereConstantParser.java | 3 +-
.../phoenix/exception/SQLExceptionCode.java | 7 +-
.../index/write/DelegateIndexFailurePolicy.java | 5 +-
.../index/PhoenixIndexFailurePolicy.java | 6 +
.../iterate/MapReduceParallelScanGrouper.java | 32 +-
.../iterate/TableSnapshotResultIterator.java | 28 +-
.../phoenix/jdbc/PhoenixDatabaseMetaData.java | 9 +-
.../mapreduce/FormatToBytesWritableMapper.java | 10 +-
.../query/ConnectionQueryServicesImpl.java | 28 +-
.../query/ConnectionlessQueryServicesImpl.java | 18 +-
.../apache/phoenix/query/QueryConstants.java | 17 +-
.../org/apache/phoenix/query/QueryServices.java | 9 +
.../phoenix/query/QueryServicesOptions.java | 5 +
.../apache/phoenix/schema/MetaDataClient.java | 248 +++-
.../apache/phoenix/schema/PMetaDataImpl.java | 28 +-
.../java/org/apache/phoenix/schema/PTable.java | 31 +-
.../org/apache/phoenix/schema/PTableImpl.java | 1259 +++++++++++-------
.../org/apache/phoenix/schema/TableRef.java | 17 +-
.../phoenix/schema/stats/StatisticsUtil.java | 2 +
.../org/apache/phoenix/util/MetaDataUtil.java | 6 +-
.../org/apache/phoenix/util/QueryBuilder.java | 211 +++
.../java/org/apache/phoenix/util/QueryUtil.java | 38 +-
.../org/apache/phoenix/util/SchemaUtil.java | 10 +
.../org/apache/phoenix/util/UpgradeUtil.java | 79 +-
.../phoenix/execute/CorrelatePlanTest.java | 32 +-
.../execute/LiteralResultIteratorPlanTest.java | 33 +-
.../index/IndexScrutinyTableOutputTest.java | 6 +-
.../util/PhoenixConfigurationUtilTest.java | 6 +-
.../java/org/apache/phoenix/query/BaseTest.java | 15 +-
.../org/apache/phoenix/util/QueryUtilTest.java | 10 +-
phoenix-server/pom.xml | 1 +
phoenix-spark/pom.xml | 8 +
.../org/apache/phoenix/spark/AggregateIT.java | 91 ++
.../org/apache/phoenix/spark/OrderByIT.java | 460 +++++++
.../org/apache/phoenix/spark/SaltedTableIT.java | 53 +
.../org/apache/phoenix/spark/SparkUtil.java | 87 ++
.../apache/phoenix/spark/PhoenixSparkIT.scala | 9 +-
.../apache/phoenix/spark/SparkResultSet.java | 1056 +++++++++++++++
.../org/apache/phoenix/spark/PhoenixRDD.scala | 27 +-
70 files changed, 7235 insertions(+), 3114 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cf087158/phoenix-assembly/pom.xml
----------------------------------------------------------------------
diff --cc phoenix-assembly/pom.xml
index f9d6d92,e41d7e6..9d3c891
--- a/phoenix-assembly/pom.xml
+++ b/phoenix-assembly/pom.xml
@@@ -127,19 -127,8 +127,23 @@@
<artifactId>phoenix-queryserver-client</artifactId>
</dependency>
<dependency>
+ <groupId>com.google.inject</groupId>
+ <artifactId>guice</artifactId>
+ <version>3.0</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.omid</groupId>
+ <artifactId>omid-hbase-tools-hbase1.x</artifactId>
+ <version>${omid.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.htrace</groupId>
+ <artifactId>htrace-core</artifactId>
+ <version>3.2.0-incubating</version>
+ </dependency>
++ <dependency>
+ <groupId>org.apache.phoenix</groupId>
+ <artifactId>phoenix-load-balancer</artifactId>
+ </dependency>
</dependencies>
</project>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/cf087158/phoenix-server/pom.xml
----------------------------------------------------------------------
[14/22] phoenix git commit: PHOENIX-4981 Add tests for ORDER BY,
GROUP BY and salted tables using phoenix-spark
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseAggregateIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseAggregateIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseAggregateIT.java
new file mode 100644
index 0000000..5b466df
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseAggregateIT.java
@@ -0,0 +1,1022 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+import static org.apache.phoenix.util.TestUtil.assertResultSet;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.sql.Statement;
+import java.util.List;
+import java.util.Properties;
+
+import com.google.common.collect.Lists;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.phoenix.compile.QueryPlan;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
+import org.apache.phoenix.jdbc.PhoenixStatement;
+import org.apache.phoenix.query.KeyRange;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.schema.AmbiguousColumnException;
+import org.apache.phoenix.schema.types.PChar;
+import org.apache.phoenix.schema.types.PInteger;
+import org.apache.phoenix.util.ByteUtil;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.QueryBuilder;
+import org.apache.phoenix.util.QueryUtil;
+import org.apache.phoenix.util.TestUtil;
+import org.junit.Test;
+
+
+public abstract class BaseAggregateIT extends ParallelStatsDisabledIT {
+
+ private static void initData(Connection conn, String tableName) throws SQLException {
+ conn.createStatement().execute("create table " + tableName +
+ " (id varchar not null primary key,\n" +
+ " uri varchar, appcpu integer)");
+ insertRow(conn, tableName, "Report1", 10, 1);
+ insertRow(conn, tableName, "Report2", 10, 2);
+ insertRow(conn, tableName, "Report3", 30, 3);
+ insertRow(conn, tableName, "Report4", 30, 4);
+ insertRow(conn, tableName, "SOQL1", 10, 5);
+ insertRow(conn, tableName, "SOQL2", 10, 6);
+ insertRow(conn, tableName, "SOQL3", 30, 7);
+ insertRow(conn, tableName, "SOQL4", 30, 8);
+ conn.commit();
+ }
+
+ private static void insertRow(Connection conn, String tableName, String uri, int appcpu, int id) throws SQLException {
+ PreparedStatement statement = conn.prepareStatement("UPSERT INTO " + tableName + "(id, uri, appcpu) values (?,?,?)");
+ statement.setString(1, "id" + id);
+ statement.setString(2, uri);
+ statement.setInt(3, appcpu);
+ statement.executeUpdate();
+ }
+
+ @Test
+ public void testDuplicateTrailingAggExpr() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ props.put(QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB, Boolean.FALSE.toString());
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+
+ conn.createStatement().execute("create table " + tableName +
+ " (nam VARCHAR(20), address VARCHAR(20), id BIGINT "
+ + "constraint my_pk primary key (id))");
+ PreparedStatement statement = conn.prepareStatement("UPSERT INTO " + tableName + "(nam, address, id) values (?,?,?)");
+ statement.setString(1, "pulkit");
+ statement.setString(2, "badaun");
+ statement.setInt(3, 1);
+ statement.executeUpdate();
+ conn.commit();
+
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setDistinct(true)
+ .setSelectExpression("'harshit' as TEST_COLUMN, trim(NAM), trim(NAM)")
+ .setSelectExpressionColumns(Lists.newArrayList("NAM"))
+ .setFullTableName(tableName);
+
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals("harshit", rs.getString(1));
+ assertEquals("pulkit", rs.getString(2));
+ assertEquals("pulkit", rs.getString(3));
+ conn.close();
+ }
+
+ @Test
+ public void testExpressionInGroupBy() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ String ddl = "create table " + tableName + "(tgb_id integer NOT NULL,utc_date_epoch integer NOT NULL,tgb_name varchar(40),ack_success_count integer" +
+ ",ack_success_one_ack_count integer, CONSTRAINT pk_tgb_counter PRIMARY KEY(tgb_id, utc_date_epoch))";
+
+ createTestTable(getUrl(), ddl);
+ String dml = "UPSERT INTO " + tableName + " VALUES(?,?,?,?,?)";
+ PreparedStatement stmt = conn.prepareStatement(dml);
+ stmt.setInt(1, 1);
+ stmt.setInt(2, 1000);
+ stmt.setString(3, "aaa");
+ stmt.setInt(4, 1);
+ stmt.setInt(5, 1);
+ stmt.execute();
+ stmt.setInt(1, 2);
+ stmt.setInt(2, 2000);
+ stmt.setString(3, "bbb");
+ stmt.setInt(4, 2);
+ stmt.setInt(5, 2);
+ stmt.execute();
+ conn.commit();
+
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectExpression("TGB_ID, TGB_NAME, (UTC_DATE_EPOCH/10)*10 AS UTC_EPOCH_HOUR,SUM(ACK_SUCCESS_COUNT + " +
+ "ACK_SUCCESS_ONE_ACK_COUNT) AS ACK_TX_SUM")
+ .setSelectExpressionColumns(Lists.newArrayList("TGB_ID", "TGB_NAME",
+ "UTC_DATE_EPOCH", "ACK_SUCCESS_COUNT", "ACK_SUCCESS_ONE_ACK_COUNT"))
+ .setGroupByClause("TGB_ID, TGB_NAME, UTC_EPOCH_HOUR")
+ .setFullTableName(tableName)
+ .setOrderByClause("TGB_ID, UTC_EPOCH_HOUR");
+
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals(1,rs.getInt(1));
+ assertEquals("aaa",rs.getString(2));
+ assertEquals(1000,rs.getDouble(3), 1e-6);
+ assertEquals(2,rs.getLong(4));
+ assertTrue(rs.next());
+ assertEquals(2,rs.getInt(1));
+ assertEquals("bbb",rs.getString(2));
+ assertEquals(2000,rs.getDouble(3), 1e-6);
+ assertEquals(4,rs.getLong(4));
+ assertFalse(rs.next());
+ rs.close();
+ conn.close();
+ }
+
+ @Test
+ public void testBooleanInGroupBy() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ String ddl = " create table " + tableName + "(id varchar primary key,v1 boolean, v2 integer, v3 integer)";
+
+ createTestTable(getUrl(), ddl);
+ PreparedStatement stmt = conn.prepareStatement("UPSERT INTO " + tableName + "(id,v2,v3) VALUES(?,?,?)");
+ stmt.setString(1, "a");
+ stmt.setInt(2, 1);
+ stmt.setInt(3, 1);
+ stmt.execute();
+ stmt.close();
+ stmt = conn.prepareStatement("UPSERT INTO " + tableName + " VALUES(?,?,?,?)");
+ stmt.setString(1, "b");
+ stmt.setBoolean(2, false);
+ stmt.setInt(3, 2);
+ stmt.setInt(4, 2);
+ stmt.execute();
+ stmt.setString(1, "c");
+ stmt.setBoolean(2, true);
+ stmt.setInt(3, 3);
+ stmt.setInt(4, 3);
+ stmt.execute();
+ conn.commit();
+
+ String[] gbs = {"V1,V2,V3","V1,V3,V2","V2,V1,V3"};
+ for (String gb : gbs) {
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectColumns(Lists.newArrayList("V1", "V2", "V3"))
+ .setFullTableName(tableName)
+ .setGroupByClause(gb);
+
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals(false,rs.getBoolean("v1"));
+ assertTrue(rs.wasNull());
+ assertEquals(1,rs.getInt("v2"));
+ assertEquals(1,rs.getInt("v3"));
+ assertTrue(rs.next());
+ assertEquals(false,rs.getBoolean("v1"));
+ assertFalse(rs.wasNull());
+ assertEquals(2,rs.getInt("v2"));
+ assertEquals(2,rs.getInt("v3"));
+ assertTrue(rs.next());
+ assertEquals(true,rs.getBoolean("v1"));
+ assertEquals(3,rs.getInt("v2"));
+ assertEquals(3,rs.getInt("v3"));
+ assertFalse(rs.next());
+ rs.close();
+ }
+ conn.close();
+ }
+
+ @Test
+ public void testScanUri() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ initData(conn, tableName);
+ Statement stmt = conn.createStatement();
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectColumns(Lists.newArrayList("URI"))
+ .setFullTableName(tableName);
+
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals("Report1", rs.getString(1));
+ assertTrue(rs.next());
+ assertEquals("Report2", rs.getString(1));
+ assertTrue(rs.next());
+ assertEquals("Report3", rs.getString(1));
+ assertTrue(rs.next());
+ assertEquals("Report4", rs.getString(1));
+ assertTrue(rs.next());
+ assertEquals("SOQL1", rs.getString(1));
+ assertTrue(rs.next());
+ assertEquals("SOQL2", rs.getString(1));
+ assertTrue(rs.next());
+ assertEquals("SOQL3", rs.getString(1));
+ assertTrue(rs.next());
+ assertEquals("SOQL4", rs.getString(1));
+ assertFalse(rs.next());
+ conn.close();
+ }
+
+ @Test
+ public void testCount() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ initData(conn, tableName);
+ Statement stmt = conn.createStatement();
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectExpression("count(1)")
+ .setFullTableName(tableName);
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals(8, rs.getLong(1));
+ assertFalse(rs.next());
+ conn.close();
+ }
+
+ @Test
+ public void testGroupByCase() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ props.put(QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB, Boolean.FALSE.toString());
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ initData(conn, tableName);
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectExpression("CASE WHEN URI LIKE 'REPORT%' THEN 'REPORTS' ELSE 'OTHER' END CATEGORY, AVG(APPCPU)")
+ .setSelectExpressionColumns(Lists.newArrayList("URI", "APPCPU"))
+ .setFullTableName(tableName)
+ .setGroupByClause("CATEGORY");
+ executeQuery(conn, queryBuilder);
+
+ queryBuilder.setSelectExpression(
+ "CASE URI WHEN 'REPORT%' THEN 'REPORTS' ELSE 'OTHER' END CATEGORY, AVG(APPCPU)")
+ .setSelectExpressionColumns(Lists.newArrayList("URI", "APPCPU"))
+ .setFullTableName(tableName)
+ .setGroupByClause("APPCPU, CATEGORY");
+ executeQuery(conn, queryBuilder);
+
+ queryBuilder.setSelectExpression(
+ "CASE URI WHEN 'Report%' THEN 'Reports' ELSE 'Other' END CATEGORY, AVG(APPCPU)")
+ .setSelectColumns(Lists.newArrayList("URI", "APPCPU"))
+ .setFullTableName(tableName)
+ .setGroupByClause("AVG(APPCPU), CATEGORY");
+ executeQueryThrowsException(conn, queryBuilder, "Aggregate expressions may not be used in GROUP BY", "");
+ conn.close();
+ }
+
+
+ @Test
+ public void testGroupByArray() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+
+ String tableName = generateUniqueName();
+ conn.createStatement().execute("CREATE TABLE " + tableName + "(\n" +
+ " a VARCHAR NOT NULL,\n" +
+ " b VARCHAR,\n" +
+ " c INTEGER,\n" +
+ " d VARCHAR,\n" +
+ " e VARCHAR ARRAY,\n" +
+ " f BIGINT,\n" +
+ " g BIGINT,\n" +
+ " CONSTRAINT pk PRIMARY KEY(a)\n" +
+ ")");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('1', 'val', 100, 'a', ARRAY ['b'], 1, 2)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('2', 'val', 100, 'a', ARRAY ['b'], 3, 4)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('3', 'val', 100, 'a', ARRAY ['b','c'], 5, 6)");
+ conn.commit();
+
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectExpression("C, SUM(F + G) AS SUMONE, D, E")
+ .setSelectExpressionColumns(Lists.newArrayList("A", "B", "C", "F", "G", "D", "E"))
+ .setWhereClause("B = 'val' AND A IN ('1','2','3')")
+ .setFullTableName(tableName)
+ .setGroupByClause("C, D, E")
+ .setOrderByClause("SUMONE desc");
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals(100, rs.getInt(1));
+ assertEquals(11, rs.getLong(2));
+ assertTrue(rs.next());
+ assertEquals(100, rs.getInt(1));
+ assertEquals(10, rs.getLong(2));
+ assertFalse(rs.next());
+ conn.close();
+ }
+
+ @Test
+ public void testGroupByOrderPreserving() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ props.put(QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB, Boolean.FALSE.toString());
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+
+ conn.createStatement().execute("CREATE TABLE " + tableName + "(ORGANIZATION_ID char(15) not null, \n" +
+ "JOURNEY_ID char(15) not null, \n" +
+ "DATASOURCE SMALLINT not null, \n" +
+ "MATCH_STATUS TINYINT not null, \n" +
+ "EXTERNAL_DATASOURCE_KEY varchar(30), \n" +
+ "ENTITY_ID char(15) not null, \n" +
+ "CONSTRAINT PK PRIMARY KEY (\n" +
+ " ORGANIZATION_ID, \n" +
+ " JOURNEY_ID, \n" +
+ " DATASOURCE, \n" +
+ " MATCH_STATUS,\n" +
+ " EXTERNAL_DATASOURCE_KEY,\n" +
+ " ENTITY_ID))");
+ conn.createStatement().execute("UPSERT INTO " + tableName
+ + " VALUES('000001111122222', '333334444455555', 0, 0, 'abc', '666667777788888')");
+ conn.createStatement().execute("UPSERT INTO " + tableName
+ + " VALUES('000001111122222', '333334444455555', 0, 0, 'abcd', '666667777788889')");
+ conn.createStatement().execute("UPSERT INTO " + tableName
+ + " VALUES('000001111122222', '333334444455555', 0, 0, 'abc', '666667777788899')");
+ conn.commit();
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectExpression("COUNT(1), EXTERNAL_DATASOURCE_KEY As DUP_COUNT")
+ .setSelectExpressionColumns(Lists.newArrayList("EXTERNAL_DATASOURCE_KEY", "MATCH_STATUS",
+ "JOURNEY_ID", "DATASOURCE", "ORGANIZATION_ID"))
+ .setWhereClause(
+ "JOURNEY_ID='333334444455555' AND DATASOURCE=0 AND MATCH_STATUS <= 1 and ORGANIZATION_ID='000001111122222'")
+ .setFullTableName(tableName)
+ .setGroupByClause("MATCH_STATUS, EXTERNAL_DATASOURCE_KEY")
+ .setHavingClause("COUNT(1) > 1");
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals(2,rs.getLong(1));
+ assertEquals("abc", rs.getString(2));
+ assertFalse(rs.next());
+
+ String expectedPhoenixPlan = "CLIENT PARALLEL 1-WAY RANGE SCAN OVER " + tableName
+ + " ['000001111122222','333334444455555',0,*] - ['000001111122222','333334444455555',0,1]\n" +
+ " SERVER FILTER BY FIRST KEY ONLY\n" +
+ " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [MATCH_STATUS, EXTERNAL_DATASOURCE_KEY]\n" +
+ "CLIENT FILTER BY COUNT(1) > 1";
+ validateQueryPlan(conn, queryBuilder, expectedPhoenixPlan, null);
+ }
+
+ @Test
+ public void testGroupByOrderPreservingDescSort() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ conn.createStatement().execute("CREATE TABLE " + tableName + " (k1 char(1) not null, k2 char(1) not null," +
+ " constraint pk primary key (k1,k2)) split on ('ac','jc','nc')");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 'a')");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 'b')");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 'c')");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 'd')");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 'a')");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 'b')");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 'c')");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 'd')");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 'a')");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 'b')");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 'c')");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 'd')");
+ conn.commit();
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectExpression("K1,COUNT(*)")
+ .setSelectColumns(Lists.newArrayList("K1"))
+ .setFullTableName(tableName)
+ .setGroupByClause("K1")
+ .setOrderByClause("K1 DESC");
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals("n", rs.getString(1));
+ assertEquals(4, rs.getLong(2));
+ assertTrue(rs.next());
+ assertEquals("j", rs.getString(1));
+ assertEquals(4, rs.getLong(2));
+ assertTrue(rs.next());
+ assertEquals("a", rs.getString(1));
+ assertEquals(4, rs.getLong(2));
+ assertFalse(rs.next());
+ String expectedPhoenixPlan = "CLIENT PARALLEL 1-WAY REVERSE FULL SCAN OVER " + tableName + "\n" +
+ " SERVER FILTER BY FIRST KEY ONLY\n" +
+ " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]";
+ validateQueryPlan(conn, queryBuilder, expectedPhoenixPlan, null);
+ }
+
+ @Test
+ public void testSumGroupByOrderPreservingDesc() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+
+ PreparedStatement stmt = conn.prepareStatement("CREATE TABLE " + tableName + " (k1 char(1) not null, k2 integer not null, constraint pk primary key (k1,k2)) split on (?,?,?)");
+ stmt.setBytes(1, ByteUtil.concat(PChar.INSTANCE.toBytes("a"), PInteger.INSTANCE.toBytes(3)));
+ stmt.setBytes(2, ByteUtil.concat(PChar.INSTANCE.toBytes("j"), PInteger.INSTANCE.toBytes(3)));
+ stmt.setBytes(3, ByteUtil.concat(PChar.INSTANCE.toBytes("n"), PInteger.INSTANCE.toBytes(3)));
+ stmt.execute();
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 1)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 2)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 3)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 4)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('b', 5)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 1)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 2)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 3)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 4)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 1)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 2)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 3)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 4)");
+ conn.commit();
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectExpression("K1,SUM(K2)")
+ .setSelectExpressionColumns(Lists.newArrayList("K1", "K2"))
+ .setFullTableName(tableName)
+ .setGroupByClause("K1")
+ .setOrderByClause("K1 DESC");
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals("n", rs.getString(1));
+ assertEquals(10, rs.getLong(2));
+ assertTrue(rs.next());
+ assertEquals("j", rs.getString(1));
+ assertEquals(10, rs.getLong(2));
+ assertTrue(rs.next());
+ assertEquals("b", rs.getString(1));
+ assertEquals(5, rs.getLong(2));
+ assertTrue(rs.next());
+ assertEquals("a", rs.getString(1));
+ assertEquals(10, rs.getLong(2));
+ assertFalse(rs.next());
+ String expectedPhoenixPlan = "CLIENT PARALLEL 1-WAY REVERSE FULL SCAN OVER " + tableName + "\n" +
+ " SERVER FILTER BY FIRST KEY ONLY\n" +
+ " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]";
+ validateQueryPlan(conn, queryBuilder, expectedPhoenixPlan, null);
+ }
+
+ @Test
+ public void testAvgGroupByOrderPreservingWithNoStats() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ initAvgGroupTable(conn, tableName, "");
+ testAvgGroupByOrderPreserving(conn, tableName, 4);
+ }
+
+ protected void initAvgGroupTable(Connection conn, String tableName, String tableProps) throws SQLException {
+ PreparedStatement stmt = conn.prepareStatement("CREATE TABLE " + tableName + " (k1 char(1) not null, k2 integer not null, constraint pk primary key (k1,k2)) " + tableProps + " split on (?,?,?)");
+ stmt.setBytes(1, ByteUtil.concat(PChar.INSTANCE.toBytes("a"), PInteger.INSTANCE.toBytes(3)));
+ stmt.setBytes(2, ByteUtil.concat(PChar.INSTANCE.toBytes("j"), PInteger.INSTANCE.toBytes(3)));
+ stmt.setBytes(3, ByteUtil.concat(PChar.INSTANCE.toBytes("n"), PInteger.INSTANCE.toBytes(3)));
+ stmt.execute();
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 1)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 2)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 3)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('a', 6)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('b', 5)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 1)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 2)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 3)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('j', 10)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 1)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 2)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 3)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES('n', 2)");
+ conn.commit();
+ }
+
+ protected void testAvgGroupByOrderPreserving(Connection conn, String tableName, int nGuidePosts) throws SQLException, IOException {
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectExpression("K1, AVG(K2)")
+ .setSelectExpressionColumns(Lists.newArrayList("K1", "K2"))
+ .setFullTableName(tableName)
+ .setGroupByClause("K1")
+ .setOrderByClause("K1");
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals("a", rs.getString(1));
+ assertEquals(3, rs.getDouble(2), 1e-6);
+ assertTrue(rs.next());
+ assertEquals("b", rs.getString(1));
+ assertEquals(5, rs.getDouble(2), 1e-6);
+ assertTrue(rs.next());
+ assertEquals("j", rs.getString(1));
+ assertEquals(4, rs.getDouble(2), 1e-6);
+ assertTrue(rs.next());
+ assertEquals("n", rs.getString(1));
+ assertEquals(2, rs.getDouble(2), 1e-6);
+ assertFalse(rs.next());
+ String expectedPhoenixPlan = "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + tableName + "\n" +
+ " SERVER FILTER BY FIRST KEY ONLY\n" +
+ " SERVER AGGREGATE INTO ORDERED DISTINCT ROWS BY [K1]";
+ validateQueryPlan(conn, queryBuilder, expectedPhoenixPlan, null);
+ TestUtil.analyzeTable(conn, tableName);
+ List<KeyRange> splits = TestUtil.getAllSplits(conn, tableName);
+ assertEquals(nGuidePosts, splits.size());
+ }
+
+ @Test
+ public void testDistinctGroupByBug3452WithoutMultiTenant() throws Exception {
+ doTestDistinctGroupByBug3452("");
+ }
+
+ @Test
+ public void testDistinctGroupByBug3452WithMultiTenant() throws Exception {
+ doTestDistinctGroupByBug3452("VERSIONS=1, MULTI_TENANT=TRUE, REPLICATION_SCOPE=1, TTL=31536000");
+ }
+
+ private void doTestDistinctGroupByBug3452(String options) throws Exception {
+ Connection conn=null;
+ try {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ conn = DriverManager.getConnection(getUrl(), props);
+
+ String tableName=generateUniqueName();
+ conn.createStatement().execute("DROP TABLE if exists "+tableName);
+ String sql="CREATE TABLE "+ tableName +" ( "+
+ "ORGANIZATION_ID CHAR(15) NOT NULL,"+
+ "CONTAINER_ID CHAR(15) NOT NULL,"+
+ "ENTITY_ID CHAR(15) NOT NULL,"+
+ "SCORE DOUBLE,"+
+ "CONSTRAINT TEST_PK PRIMARY KEY ( "+
+ "ORGANIZATION_ID,"+
+ "CONTAINER_ID,"+
+ "ENTITY_ID"+
+ ")) "+options;
+ conn.createStatement().execute(sql);
+
+ String indexTableName=generateUniqueName();
+ conn.createStatement().execute("DROP INDEX IF EXISTS "+indexTableName+" ON "+tableName);
+ conn.createStatement().execute("CREATE INDEX "+indexTableName+" ON "+tableName+" (CONTAINER_ID, SCORE DESC, ENTITY_ID DESC)");
+
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('org1','container1','entityId6',1.1)");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('org1','container1','entityId5',1.2)");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('org1','container1','entityId4',1.3)");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('org1','container1','entityId3',1.4)");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('org1','container1','entityId2',1.5)");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('org1','container1','entityId1',1.6)");
+ conn.commit();
+
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setDistinct(true)
+ .setSelectColumns(Lists.newArrayList("ENTITY_ID", "SCORE", "ORGANIZATION_ID", "CONTAINER_ID"))
+ .setFullTableName(tableName)
+ .setWhereClause("ORGANIZATION_ID = 'org1' AND CONTAINER_ID = 'container1'")
+ .setOrderByClause("SCORE DESC");
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertTrue(rs.getString(1).equals("entityId1"));
+ assertEquals(rs.getDouble(2),1.6,0.0001);
+
+ assertTrue(rs.next());
+ assertTrue(rs.getString(1).equals("entityId2"));
+ assertEquals(rs.getDouble(2),1.5,0.0001);
+
+ assertTrue(rs.next());
+ assertTrue(rs.getString(1).equals("entityId3"));
+ assertEquals(rs.getDouble(2),1.4,0.0001);
+
+ assertTrue(rs.next());
+ assertTrue(rs.getString(1).equals("entityId4"));
+ assertEquals(rs.getDouble(2),1.3,0.0001);
+
+ assertTrue(rs.next());
+ assertTrue(rs.getString(1).equals("entityId5"));
+ assertEquals(rs.getDouble(2),1.2,0.0001);
+
+ assertTrue(rs.next());
+ assertTrue(rs.getString(1).equals("entityId6"));
+ assertEquals(rs.getDouble(2),1.1,0.0001);
+ assertTrue(!rs.next());
+ } finally {
+ if(conn!=null) {
+ conn.close();
+ }
+ }
+ }
+
+ @Test
+ public void testGroupByOrderByDescBug3451() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String tableName=generateUniqueName();
+ String sql="CREATE TABLE " + tableName + " (\n" +
+ " ORGANIZATION_ID CHAR(15) NOT NULL,\n" +
+ " CONTAINER_ID CHAR(15) NOT NULL,\n" +
+ " ENTITY_ID CHAR(15) NOT NULL,\n" +
+ " SCORE DOUBLE,\n" +
+ " CONSTRAINT TEST_PK PRIMARY KEY (\n" +
+ " ORGANIZATION_ID,\n" +
+ " CONTAINER_ID,\n" +
+ " ENTITY_ID\n" +
+ " )\n" +
+ " )";
+ conn.createStatement().execute(sql);
+ String indexName=generateUniqueName();
+ conn.createStatement().execute("CREATE INDEX " + indexName + " ON " + tableName + "(ORGANIZATION_ID,CONTAINER_ID, SCORE DESC, ENTITY_ID DESC)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES ('org2','container2','entityId6',1.1)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES ('org2','container1','entityId5',1.2)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES ('org2','container2','entityId4',1.3)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES ('org2','container1','entityId5',1.2)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES ('org2','container1','entityId3',1.4)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES ('org2','container3','entityId7',1.35)");
+ conn.createStatement().execute("UPSERT INTO " + tableName + " VALUES ('org2','container3','entityId8',1.45)");
+ conn.commit();
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setDistinct(true)
+ .setSelectColumns(Lists.newArrayList("ENTITY_ID", "SCORE", "ORGANIZATION_ID", "CONTAINER_ID"))
+ .setFullTableName(tableName)
+ .setWhereClause(
+ "ORGANIZATION_ID = 'org2' AND CONTAINER_ID IN ('container1','container2','container3')")
+ .setOrderByClause("SCORE DESC")
+ .setLimit(2);
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals("entityId8", rs.getString(1));
+ assertEquals(1.45, rs.getDouble(2),0.001);
+ assertTrue(rs.next());
+ assertEquals("entityId3", rs.getString(1));
+ assertEquals(1.4, rs.getDouble(2),0.001);
+ assertFalse(rs.next());
+
+ String expectedPhoenixPlan = "";
+ validateQueryPlan(conn, queryBuilder, expectedPhoenixPlan, null);
+ }
+ }
+
+ @Test
+ public void testGroupByDescColumnWithNullsLastBug3452() throws Exception {
+
+ Connection conn=null;
+ try
+ {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ conn = DriverManager.getConnection(getUrl(), props);
+
+ String tableName=generateUniqueName();
+ String sql="CREATE TABLE "+tableName+" ( "+
+ "ORGANIZATION_ID VARCHAR,"+
+ "CONTAINER_ID VARCHAR,"+
+ "ENTITY_ID VARCHAR NOT NULL,"+
+ "CONSTRAINT TEST_PK PRIMARY KEY ( "+
+ "ORGANIZATION_ID DESC,"+
+ "CONTAINER_ID DESC,"+
+ "ENTITY_ID"+
+ "))";
+ conn.createStatement().execute(sql);
+
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('a',null,'11')");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,'2','22')");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('c','3','33')");
+ conn.commit();
+
+ //-----ORGANIZATION_ID
+
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectColumns(
+ Lists.newArrayList("CONTAINER_ID", "ORGANIZATION_ID"))
+ .setFullTableName(tableName)
+ .setGroupByClause("ORGANIZATION_ID, CONTAINER_ID")
+ .setOrderByClause("ORGANIZATION_ID ASC NULLS FIRST");
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2",null},{null,"a"},{"3","c"},});
+
+ queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{"2",null}});
+
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,"a"}});
+
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{"2",null}});
+
+ //----CONTAINER_ID
+
+ queryBuilder.setOrderByClause("CONTAINER_ID ASC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,"a"},{"2",null},{"3","c"}});
+
+ queryBuilder.setOrderByClause("CONTAINER_ID ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,"a"}});
+
+ queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{"2",null}});
+
+ queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,"a"}});
+
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,null,'44')");
+ conn.commit();
+
+ //-----ORGANIZATION_ID ASC CONTAINER_ID ASC
+
+ queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS FIRST,CONTAINER_ID NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,null},{"2",null},{null,"a"},{"3","c"}});
+
+ queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS FIRST,CONTAINER_ID NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2",null},{null,null},{null,"a"},{"3","c"}});
+
+ queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS LAST,CONTAINER_ID NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{null,null},{"2",null}});
+
+ queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS LAST,CONTAINER_ID NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{"2",null},{null,null}});
+
+ //-----ORGANIZATION_ID ASC CONTAINER_ID DESC
+
+ queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS FIRST,CONTAINER_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,null},{"2",null},{null,"a"},{"3","c"}});
+
+ queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS FIRST,CONTAINER_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2",null},{null,null},{null,"a"},{"3","c"}});
+
+ queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS LAST,CONTAINER_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{null,null},{"2",null}});
+
+ queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS LAST,CONTAINER_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{"2",null},{null,null}});
+
+ //-----ORGANIZATION_ID DESC CONTAINER_ID ASC
+
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,null},{"2",null},{"3","c"},{null,"a"}});
+
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2",null},{null,null},{"3","c"},{null,"a"}});
+
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{null,null},{"2",null}});
+
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{"2",null},{null,null}});
+
+ //-----ORGANIZATION_ID DESC CONTAINER_ID DESC
+
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,null},{"2",null},{"3","c"},{null,"a"}});
+
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2",null},{null,null},{"3","c"},{null,"a"}});
+
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{null,null},{"2",null}});
+
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{"2",null},{null,null}});
+
+ //-----CONTAINER_ID ASC ORGANIZATION_ID ASC
+
+ queryBuilder.setOrderByClause("CONTAINER_ID NULLS FIRST,ORGANIZATION_ID NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,null},{null,"a"},{"2",null},{"3","c"}});
+
+ queryBuilder.setOrderByClause("CONTAINER_ID NULLS FIRST,ORGANIZATION_ID NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,"a"},{null,null},{"2",null},{"3","c"}});
+
+ queryBuilder.setOrderByClause("CONTAINER_ID NULLS LAST,ORGANIZATION_ID NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,null},{null,"a"}});
+
+ queryBuilder.setOrderByClause("CONTAINER_ID NULLS LAST,ORGANIZATION_ID NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,"a"},{null,null}});
+
+ //-----CONTAINER_ID ASC ORGANIZATION_ID DESC
+
+ queryBuilder.setOrderByClause("CONTAINER_ID ASC NULLS FIRST,ORGANIZATION_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,null},{null,"a"},{"2",null},{"3","c"}});
+
+ queryBuilder.setOrderByClause("CONTAINER_ID ASC NULLS FIRST,ORGANIZATION_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,"a"},{null,null},{"2",null},{"3","c"}});
+
+ queryBuilder.setOrderByClause("CONTAINER_ID ASC NULLS LAST,ORGANIZATION_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,null},{null,"a"}});
+
+ queryBuilder.setOrderByClause("CONTAINER_ID ASC NULLS LAST,ORGANIZATION_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,"a"},{null,null}});
+
+ //-----CONTAINER_ID DESC ORGANIZATION_ID ASC
+
+ queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID ASC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,null},{null,"a"},{"3","c"},{"2",null}});
+
+ queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,"a"},{null,null},{"3","c"},{"2",null}});
+
+ queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID ASC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,null},{null,"a"}});
+
+ queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,"a"},{null,null}});
+
+ //-----CONTAINER_ID DESC ORGANIZATION_ID DESC
+
+ queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,null},{null,"a"},{"3","c"},{"2",null}});
+
+ queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,"a"},{null,null},{"3","c"},{"2",null}});
+
+ queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,null},{null,"a"}});
+
+ queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,"a"},{null,null}});
+ } finally {
+ if(conn!=null) {
+ conn.close();
+ }
+ }
+ }
+
+ @Test
+ public void testCountNullInEncodedNonEmptyKeyValueCF() throws Exception {
+ testCountNullInNonEmptyKeyValueCF(1);
+ }
+
+ @Test
+ public void testCountNullInNonEncodedNonEmptyKeyValueCF() throws Exception {
+ testCountNullInNonEmptyKeyValueCF(0);
+ }
+
+ protected abstract void testCountNullInNonEmptyKeyValueCF(int columnEncodedBytes) throws Exception;
+
+ @Test
+ public void testGroupByOrderMatchPkColumnOrderBug4690() throws Exception {
+ this.doTestGroupByOrderMatchPkColumnOrderBug4690(false, false);
+ this.doTestGroupByOrderMatchPkColumnOrderBug4690(false, true);
+ this.doTestGroupByOrderMatchPkColumnOrderBug4690(true, false);
+ this.doTestGroupByOrderMatchPkColumnOrderBug4690(true, true);
+ }
+
+ private void doTestGroupByOrderMatchPkColumnOrderBug4690(boolean desc ,boolean salted) throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = null;
+ try {
+ conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ String sql = "create table " + tableName + "( "+
+ " pk1 integer not null , " +
+ " pk2 integer not null, " +
+ " pk3 integer not null," +
+ " pk4 integer not null,"+
+ " v integer, " +
+ " CONSTRAINT TEST_PK PRIMARY KEY ( "+
+ "pk1 "+(desc ? "desc" : "")+", "+
+ "pk2 "+(desc ? "desc" : "")+", "+
+ "pk3 "+(desc ? "desc" : "")+", "+
+ "pk4 "+(desc ? "desc" : "")+
+ " )) "+(salted ? "SALT_BUCKETS =4" : "split on(2)");
+ conn.createStatement().execute(sql);
+
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,8,10,20,30)");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,8,11,21,31)");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,9,5 ,22,32)");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,9,6 ,12,33)");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,9,6 ,13,34)");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,9,7 ,8,35)");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (2,3,15,25,35)");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (2,7,16,26,36)");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (2,7,17,27,37)");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (3,2,18,28,38)");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (3,2,19,29,39)");
+ conn.commit();
+
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectExpression("PK2,PK1,COUNT(V)")
+ .setSelectExpressionColumns(Lists.newArrayList("PK1", "PK2", "V"))
+ .setFullTableName(tableName)
+ .setGroupByClause("PK2, PK1")
+ .setOrderByClause("PK2, PK1");
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{2,3,2L},{3,2,1L},{7,2,2L},{8,1,2L},{9,1,4L}});
+
+ queryBuilder.setSelectExpression("PK1, PK2, COUNT(V)");
+ queryBuilder.setOrderByClause("PK1, PK2");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{1,8,2L},{1,9,4L},{2,3,1L},{2,7,2L},{3,2,2L}});
+
+ queryBuilder.setSelectExpression("PK2,PK1,COUNT(V)");
+ queryBuilder.setOrderByClause("PK2 DESC,PK1 DESC");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{9,1,4L},{8,1,2L},{7,2,2L},{3,2,1L},{2,3,2L}});
+
+ queryBuilder.setSelectExpression("PK1,PK2,COUNT(V)");
+ queryBuilder.setOrderByClause("PK1 DESC,PK2 DESC");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{3,2,2L},{2,7,2L},{2,3,1L},{1,9,4L},{1,8,2L}});
+
+
+ queryBuilder.setSelectExpression("PK3,PK2,COUNT(V)");
+ queryBuilder.setSelectExpressionColumns(Lists.newArrayList("PK1", "PK2", "PK3", "V"));
+ queryBuilder.setFullTableName(tableName);
+ queryBuilder.setGroupByClause("PK3,PK2");
+ queryBuilder.setOrderByClause("PK3,PK2");
+ queryBuilder.setWhereClause("PK1=1");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{5,9,1L},{6,9,2L},{7,9,1L},{10,8,1L},{11,8,1L}});
+
+ queryBuilder.setSelectExpression("PK2,PK3,COUNT(V)");
+ queryBuilder.setOrderByClause("PK2,PK3");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{8,10,1L},{8,11,1L},{9,5,1L},{9,6,2L},{9,7,1L}});
+
+ queryBuilder.setSelectExpression("PK3,PK2,COUNT(V)");
+ queryBuilder.setOrderByClause("PK3 DESC,PK2 DESC");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{11,8,1L},{10,8,1L},{7,9,1L},{6,9,2L},{5,9,1L}});
+
+ queryBuilder.setSelectExpression("PK2,PK3,COUNT(V)");
+ queryBuilder.setOrderByClause("PK2 DESC,PK3 DESC");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{9,7,1L},{9,6,2L},{9,5,1L},{8,11,1L},{8,10,1L}});
+
+
+ queryBuilder.setSelectExpression("PK4,PK3,PK1,COUNT(V)");
+ queryBuilder.setSelectExpressionColumns(Lists.newArrayList("PK1", "PK2", "PK3", "PK4", "V"));
+ queryBuilder.setFullTableName(tableName);
+ queryBuilder.setWhereClause("PK2=9 ");
+ queryBuilder.setGroupByClause("PK4,PK3,PK1");
+ queryBuilder.setOrderByClause("PK4,PK3,PK1");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{8,7,1,1L},{12,6,1,1L},{13,6,1,1L},{22,5,1,1L}});
+
+ queryBuilder.setSelectExpression("PK1,PK3,PK4,COUNT(V)");
+ queryBuilder.setOrderByClause("PK1,PK3,PK4");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{1,5,22,1L},{1,6,12,1L},{1,6,13,1L},{1,7,8,1L}});
+
+ queryBuilder.setSelectExpression("PK4,PK3,PK1,COUNT(V)");
+ queryBuilder.setOrderByClause("PK4 DESC,PK3 DESC,PK1 DESC");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{22,5,1,1L},{13,6,1,1L},{12,6,1,1L},{8,7,1,1L}});
+
+ queryBuilder.setSelectExpression("PK1,PK3,PK4,COUNT(V)");
+ queryBuilder.setOrderByClause("PK1 DESC,PK3 DESC,PK4 DESC");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{1,7,8,1L},{1,6,13,1L},{1,6,12,1L},{1,5,22,1L}});
+ } finally {
+ if(conn != null) {
+ conn.close();
+ }
+ }
+ }
+}
[22/22] phoenix git commit: Remove unnecessary dependencies in
assembly
Posted by ja...@apache.org.
Remove unnecessary dependencies in assembly
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/d03eadc2
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/d03eadc2
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/d03eadc2
Branch: refs/heads/omid2
Commit: d03eadc23f7da4bcfa36c7add810ff43707e5700
Parents: cf08715
Author: James Taylor <ja...@apache.org>
Authored: Sat Nov 10 12:56:59 2018 -0800
Committer: James Taylor <ja...@apache.org>
Committed: Sat Nov 10 12:58:32 2018 -0800
----------------------------------------------------------------------
phoenix-assembly/pom.xml | 14 ++------------
1 file changed, 2 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d03eadc2/phoenix-assembly/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-assembly/pom.xml b/phoenix-assembly/pom.xml
index 9d3c891..e2543c5 100644
--- a/phoenix-assembly/pom.xml
+++ b/phoenix-assembly/pom.xml
@@ -127,23 +127,13 @@
<artifactId>phoenix-queryserver-client</artifactId>
</dependency>
<dependency>
- <groupId>com.google.inject</groupId>
- <artifactId>guice</artifactId>
- <version>3.0</version>
+ <groupId>org.apache.phoenix</groupId>
+ <artifactId>phoenix-load-balancer</artifactId>
</dependency>
<dependency>
<groupId>org.apache.omid</groupId>
<artifactId>omid-hbase-tools-hbase1.x</artifactId>
<version>${omid.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.htrace</groupId>
- <artifactId>htrace-core</artifactId>
- <version>3.2.0-incubating</version>
- </dependency>
- <dependency>
- <groupId>org.apache.phoenix</groupId>
- <artifactId>phoenix-load-balancer</artifactId>
- </dependency>
</dependencies>
</project>
[13/22] phoenix git commit: PHOENIX-4981 Add tests for ORDER BY,
GROUP BY and salted tables using phoenix-spark
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseOrderByIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseOrderByIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseOrderByIT.java
new file mode 100644
index 0000000..31bf050
--- /dev/null
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BaseOrderByIT.java
@@ -0,0 +1,940 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.end2end;
+
+import static org.apache.phoenix.util.TestUtil.ROW1;
+import static org.apache.phoenix.util.TestUtil.ROW2;
+import static org.apache.phoenix.util.TestUtil.ROW3;
+import static org.apache.phoenix.util.TestUtil.ROW4;
+import static org.apache.phoenix.util.TestUtil.ROW5;
+import static org.apache.phoenix.util.TestUtil.ROW6;
+import static org.apache.phoenix.util.TestUtil.ROW7;
+import static org.apache.phoenix.util.TestUtil.ROW8;
+import static org.apache.phoenix.util.TestUtil.ROW9;
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.apache.phoenix.util.TestUtil.assertResultSet;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.util.Properties;
+
+import com.google.common.collect.Lists;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.QueryBuilder;
+import org.junit.Test;
+
+
+public abstract class BaseOrderByIT extends ParallelStatsDisabledIT {
+
+ @Test
+ public void testMultiOrderByExpr() throws Exception {
+ String tenantId = getOrganizationId();
+ String tableName = initATableValues(tenantId, getDefaultSplits(tenantId), getUrl());
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectColumns(
+ Lists.newArrayList("ENTITY_ID", "B_STRING"))
+ .setFullTableName(tableName)
+ .setOrderByClause("B_STRING, ENTITY_ID");
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue (rs.next());
+ assertEquals(ROW1,rs.getString(1));
+ assertTrue (rs.next());
+ assertEquals(ROW4,rs.getString(1));
+ assertTrue (rs.next());
+ assertEquals(ROW7,rs.getString(1));
+ assertTrue (rs.next());
+ assertEquals(ROW2,rs.getString(1));
+ assertTrue (rs.next());
+ assertEquals(ROW5,rs.getString(1));
+ assertTrue (rs.next());
+ assertEquals(ROW8,rs.getString(1));
+ assertTrue (rs.next());
+ assertEquals(ROW3,rs.getString(1));
+ assertTrue (rs.next());
+ assertEquals(ROW6,rs.getString(1));
+ assertTrue (rs.next());
+ assertEquals(ROW9,rs.getString(1));
+
+ assertFalse(rs.next());
+ }
+ }
+
+
+ @Test
+ public void testDescMultiOrderByExpr() throws Exception {
+ String tenantId = getOrganizationId();
+ String tableName = initATableValues(tenantId, getDefaultSplits(tenantId), getUrl());
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectColumns(
+ Lists.newArrayList("ENTITY_ID", "B_STRING"))
+ .setFullTableName(tableName)
+ .setOrderByClause("B_STRING || ENTITY_ID DESC");
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue (rs.next());
+ assertEquals(ROW9,rs.getString(1));
+ assertTrue (rs.next());
+ assertEquals(ROW6,rs.getString(1));
+ assertTrue (rs.next());
+ assertEquals(ROW3,rs.getString(1));
+ assertTrue (rs.next());
+ assertEquals(ROW8,rs.getString(1));
+ assertTrue (rs.next());
+ assertEquals(ROW5,rs.getString(1));
+ assertTrue (rs.next());
+ assertEquals(ROW2,rs.getString(1));
+ assertTrue (rs.next());
+ assertEquals(ROW7,rs.getString(1));
+ assertTrue (rs.next());
+ assertEquals(ROW4,rs.getString(1));
+ assertTrue (rs.next());
+ assertEquals(ROW1,rs.getString(1));
+
+ assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testOrderByDifferentColumns() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ conn.setAutoCommit(false);
+ String tableName = generateUniqueName();
+ String ddl = "CREATE TABLE " + tableName +
+ " (a_string varchar not null, col1 integer" +
+ " CONSTRAINT pk PRIMARY KEY (a_string))\n";
+ createTestTable(getUrl(), ddl);
+
+ String dml = "UPSERT INTO " + tableName + " VALUES(?, ?)";
+ PreparedStatement stmt = conn.prepareStatement(dml);
+ stmt.setString(1, "a");
+ stmt.setInt(2, 40);
+ stmt.execute();
+ stmt.setString(1, "b");
+ stmt.setInt(2, 20);
+ stmt.execute();
+ stmt.setString(1, "c");
+ stmt.setInt(2, 30);
+ stmt.execute();
+ conn.commit();
+
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectColumns(
+ Lists.newArrayList("COL1"))
+ .setFullTableName(tableName)
+ .setSelectExpression("count(*)")
+ .setGroupByClause("COL1")
+ .setOrderByClause("COL1");
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals(1,rs.getLong(1));
+ assertTrue(rs.next());
+ assertEquals(1,rs.getLong(1));
+ assertTrue(rs.next());
+ assertEquals(1,rs.getLong(1));
+ assertFalse(rs.next());
+
+ queryBuilder = new QueryBuilder();
+ queryBuilder.setSelectColumns(
+ Lists.newArrayList("A_STRING", "COL1"));
+ queryBuilder.setFullTableName(tableName);
+ queryBuilder.setOrderByClause("A_STRING");
+ rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals("a",rs.getString(1));
+ assertEquals(40,rs.getInt(2));
+ assertTrue(rs.next());
+ assertEquals("b",rs.getString(1));
+ assertEquals(20,rs.getInt(2));
+ assertTrue(rs.next());
+ assertEquals("c",rs.getString(1));
+ assertEquals(30,rs.getInt(2));
+ assertFalse(rs.next());
+
+ queryBuilder.setSelectColumns(
+ Lists.newArrayList("A_STRING", "COL1"));
+ queryBuilder.setFullTableName(tableName);
+ queryBuilder.setOrderByClause("COL1");
+ rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals("b",rs.getString(1));
+ assertEquals(20,rs.getInt(2));
+ assertTrue(rs.next());
+ assertEquals("c",rs.getString(1));
+ assertEquals(30,rs.getInt(2));
+ assertTrue(rs.next());
+ assertEquals("a",rs.getString(1));
+ assertEquals(40,rs.getInt(2));
+ assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testAggregateOrderBy() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ String ddl = "create table " + tableName + " (ID VARCHAR NOT NULL PRIMARY KEY, VAL1 VARCHAR, VAL2 INTEGER)";
+ conn.createStatement().execute(ddl);
+
+ conn.createStatement().execute("upsert into " + tableName + " values ('ABC','aa123', 11)");
+ conn.createStatement().execute("upsert into " + tableName + " values ('ABD','ba124', 1)");
+ conn.createStatement().execute("upsert into " + tableName + " values ('ABE','cf125', 13)");
+ conn.createStatement().execute("upsert into " + tableName + " values ('ABF','dan126', 4)");
+ conn.createStatement().execute("upsert into " + tableName + " values ('ABG','elf127', 15)");
+ conn.createStatement().execute("upsert into " + tableName + " values ('ABH','fan128', 6)");
+ conn.createStatement().execute("upsert into " + tableName + " values ('AAA','get211', 100)");
+ conn.createStatement().execute("upsert into " + tableName + " values ('AAB','hat212', 7)");
+ conn.createStatement().execute("upsert into " + tableName + " values ('AAC','aap12', 2)");
+ conn.createStatement().execute("upsert into " + tableName + " values ('AAD','ball12', 3)");
+ conn.createStatement().execute("upsert into " + tableName + " values ('AAE','inn2110', 13)");
+ conn.createStatement().execute("upsert into " + tableName + " values ('AAF','key2112', 40)");
+ conn.commit();
+
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectColumns(
+ Lists.newArrayList("VAL1", "VAL2"))
+ .setFullTableName(tableName)
+ .setOrderByClause("VAL1")
+ .setSelectExpression("DISTINCT(ID)")
+ .setSelectExpressionColumns(Lists.newArrayList("ID"))
+ .setWhereClause(
+ "ID in ('ABC','ABD','ABE','ABF','ABG','ABH','AAA', 'AAB', 'AAC','AAD','AAE','AAF')");
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals("ABC", rs.getString(1));
+ assertEquals("aa123", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("aap12", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("ba124", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("ball12", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("cf125", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("dan126", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("elf127", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("fan128", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("get211", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("hat212", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("inn2110", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("AAF", rs.getString(1));
+ assertEquals("key2112", rs.getString(2));
+ assertFalse(rs.next());
+ }
+
+ @Test
+ public void testAggregateOptimizedOutOrderBy() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String tableName = generateUniqueName();
+ String ddl = "create table " + tableName + " (K1 VARCHAR NOT NULL, K2 VARCHAR NOT NULL, VAL1 VARCHAR, VAL2 INTEGER, CONSTRAINT pk PRIMARY KEY(K1,K2))";
+ conn.createStatement().execute(ddl);
+
+ conn.createStatement().execute("upsert into " + tableName + " values ('ABC','ABC','aa123', 11)");
+ conn.createStatement().execute("upsert into " + tableName + " values ('ABD','ABC','ba124', 1)");
+ conn.createStatement().execute("upsert into " + tableName + " values ('ABE','ABC','cf125', 13)");
+ conn.createStatement().execute("upsert into " + tableName + " values ('ABF','ABC','dan126', 4)");
+ conn.createStatement().execute("upsert into " + tableName + " values ('ABG','ABC','elf127', 15)");
+ conn.createStatement().execute("upsert into " + tableName + " values ('ABH','ABC','fan128', 6)");
+ conn.createStatement().execute("upsert into " + tableName + " values ('AAA','ABC','get211', 100)");
+ conn.createStatement().execute("upsert into " + tableName + " values ('AAB','ABC','hat212', 7)");
+ conn.createStatement().execute("upsert into " + tableName + " values ('AAC','ABC','aap12', 2)");
+ conn.createStatement().execute("upsert into " + tableName + " values ('AAD','ABC','ball12', 3)");
+ conn.createStatement().execute("upsert into " + tableName + " values ('AAE','ABC','inn2110', 13)");
+ conn.createStatement().execute("upsert into " + tableName + " values ('AAF','ABC','key2112', 40)");
+ conn.commit();
+
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectColumns(
+ Lists.newArrayList("VAL1", "VAL2"))
+ .setFullTableName(tableName)
+ .setOrderByClause("VAL1")
+ .setSelectExpressionColumns(Lists.newArrayList("K2"))
+ .setSelectExpression("DISTINCT(K2)")
+ .setWhereClause("K2 = 'ABC'");
+
+ // verify that the phoenix query plan doesn't contain an order by
+ String expectedPhoenixPlan = "CLIENT PARALLEL 1-WAY FULL SCAN OVER " + tableName +"\n" +
+ " SERVER FILTER BY K2 = 'ABC'\n" +
+ " SERVER AGGREGATE INTO DISTINCT ROWS BY [K2, VAL1, VAL2]\n" +
+ "CLIENT MERGE SORT";
+ validateQueryPlan(conn, queryBuilder, expectedPhoenixPlan, null);
+
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals("ABC", rs.getString(1));
+ assertEquals("aa123", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("aap12", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("ba124", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("ball12", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("cf125", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("dan126", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("elf127", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("fan128", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("get211", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("hat212", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("inn2110", rs.getString(2));
+ assertTrue(rs.next());
+ assertEquals("ABC", rs.getString(1));
+ assertEquals("key2112", rs.getString(2));
+ assertFalse(rs.next());
+ }
+
+ @Test
+ public void testNullsLastWithDesc() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String tableName=generateUniqueName();
+ String sql="CREATE TABLE "+tableName+" ( "+
+ "ORGANIZATION_ID VARCHAR,"+
+ "CONTAINER_ID VARCHAR,"+
+ "ENTITY_ID VARCHAR NOT NULL,"+
+ "CONSTRAINT TEST_PK PRIMARY KEY ( "+
+ "ORGANIZATION_ID DESC,"+
+ "CONTAINER_ID DESC,"+
+ "ENTITY_ID"+
+ "))";
+ conn.createStatement().execute(sql);
+
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('a',null,'11')");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,'2','22')");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('c','3','33')");
+ conn.commit();
+
+ //-----ORGANIZATION_ID
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID ASC NULLS FIRST";
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectColumns(
+ Lists.newArrayList("CONTAINER_ID", "ORGANIZATION_ID"))
+ .setFullTableName(tableName)
+ .setOrderByClause("ORGANIZATION_ID ASC NULLS FIRST");
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2",null},{null,"a"},{"3","c"},});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID ASC NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{"2",null}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS FIRST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,"a"}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{"2",null}});
+
+ //----CONTAINER_ID
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID ASC NULLS FIRST";
+ queryBuilder.setOrderByClause("CONTAINER_ID ASC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,"a"},{"2",null},{"3","c"}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID ASC NULLS LAST";
+ queryBuilder.setOrderByClause("CONTAINER_ID ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,"a"}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS FIRST";
+ queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{"2",null}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS LAST";
+ queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,"a"}});
+
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,null,'44')");
+ conn.commit();
+
+ //-----ORGANIZATION_ID ASC CONTAINER_ID ASC
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS FIRST,CONTAINER_ID NULLS FIRST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS FIRST,CONTAINER_ID NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,null},{"2",null},{null,"a"},{"3","c"}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS FIRST,CONTAINER_ID NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS FIRST,CONTAINER_ID NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2",null},{null,null},{null,"a"},{"3","c"}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS LAST,CONTAINER_ID NULLS FIRST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS LAST,CONTAINER_ID NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{null,null},{"2",null}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS LAST,CONTAINER_ID NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS LAST,CONTAINER_ID NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{"2",null},{null,null}});
+
+
+ //-----ORGANIZATION_ID ASC CONTAINER_ID DESC
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS FIRST,CONTAINER_ID DESC NULLS FIRST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS FIRST,CONTAINER_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,null},{"2",null},{null,"a"},{"3","c"}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS FIRST,CONTAINER_ID DESC NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS FIRST,CONTAINER_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2",null},{null,null},{null,"a"},{"3","c"}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS LAST,CONTAINER_ID DESC NULLS FIRST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS LAST,CONTAINER_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{null,null},{"2",null}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID NULLS LAST,CONTAINER_ID DESC NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID NULLS LAST,CONTAINER_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,"a"},{"3","c"},{"2",null},{null,null}});
+
+ //-----ORGANIZATION_ID DESC CONTAINER_ID ASC
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID NULLS FIRST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,null},{"2",null},{"3","c"},{null,"a"}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2",null},{null,null},{"3","c"},{null,"a"}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID NULLS FIRST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{null,null},{"2",null}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{"2",null},{null,null}});
+
+ //-----ORGANIZATION_ID DESC CONTAINER_ID DESC
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID DESC NULLS FIRST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,null},{"2",null},{"3","c"},{null,"a"}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID DESC NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST,CONTAINER_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2",null},{null,null},{"3","c"},{null,"a"}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID DESC NULLS FIRST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{null,null},{"2",null}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID DESC NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST,CONTAINER_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3","c"},{null,"a"},{"2",null},{null,null}});
+
+ //-----CONTAINER_ID ASC ORGANIZATION_ID ASC
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID NULLS FIRST,ORGANIZATION_ID NULLS FIRST";
+ queryBuilder.setOrderByClause("CONTAINER_ID NULLS FIRST,ORGANIZATION_ID NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,null},{null,"a"},{"2",null},{"3","c"}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID NULLS FIRST,ORGANIZATION_ID NULLS LAST";
+ queryBuilder.setOrderByClause("CONTAINER_ID NULLS FIRST,ORGANIZATION_ID NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,"a"},{null,null},{"2",null},{"3","c"}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID NULLS LAST,ORGANIZATION_ID NULLS FIRST";
+ queryBuilder.setOrderByClause("CONTAINER_ID NULLS LAST,ORGANIZATION_ID NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,null},{null,"a"}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID NULLS LAST,ORGANIZATION_ID NULLS LAST";
+ queryBuilder.setOrderByClause("CONTAINER_ID NULLS LAST,ORGANIZATION_ID NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,"a"},{null,null}});
+
+ //-----CONTAINER_ID ASC ORGANIZATION_ID DESC
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID ASC NULLS FIRST,ORGANIZATION_ID DESC NULLS FIRST";
+ queryBuilder.setOrderByClause("CONTAINER_ID ASC NULLS FIRST,ORGANIZATION_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,null},{null,"a"},{"2",null},{"3","c"}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID ASC NULLS FIRST,ORGANIZATION_ID DESC NULLS LAST";
+ queryBuilder.setOrderByClause("CONTAINER_ID ASC NULLS FIRST,ORGANIZATION_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,"a"},{null,null},{"2",null},{"3","c"}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID ASC NULLS LAST,ORGANIZATION_ID DESC NULLS FIRST";
+ queryBuilder.setOrderByClause("CONTAINER_ID ASC NULLS LAST,ORGANIZATION_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,null},{null,"a"}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID ASC NULLS LAST,ORGANIZATION_ID DESC NULLS LAST";
+ queryBuilder.setOrderByClause("CONTAINER_ID ASC NULLS LAST,ORGANIZATION_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"2",null},{"3","c"},{null,"a"},{null,null}});
+
+ //-----CONTAINER_ID DESC ORGANIZATION_ID ASC
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID ASC NULLS FIRST";
+ queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID ASC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,null},{null,"a"},{"3","c"},{"2",null}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID ASC NULLS LAST";
+ queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,"a"},{null,null},{"3","c"},{"2",null}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID ASC NULLS FIRST";
+ queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID ASC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,null},{null,"a"}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID ASC NULLS LAST";
+ queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,"a"},{null,null}});
+
+ //-----CONTAINER_ID DESC ORGANIZATION_ID DESC
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID DESC NULLS FIRST";
+ queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,null},{null,"a"},{"3","c"},{"2",null}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID DESC NULLS LAST";
+ queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS FIRST,ORGANIZATION_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{null,"a"},{null,null},{"3","c"},{"2",null}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID DESC NULLS FIRST";
+ queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,null},{null,"a"}});
+
+// sql="SELECT CONTAINER_ID,ORGANIZATION_ID FROM "+tableName+" order by CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID DESC NULLS LAST";
+ queryBuilder.setOrderByClause("CONTAINER_ID DESC NULLS LAST,ORGANIZATION_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new String[][]{{"3","c"},{"2",null},{null,"a"},{null,null}});
+ }
+ }
+
+ @Test
+ public void testOrderByReverseOptimization() throws Exception {
+ for(boolean salted: new boolean[]{true,false}) {
+ doTestOrderByReverseOptimization(salted,true,true,true);
+ doTestOrderByReverseOptimization(salted,true,true,false);
+ doTestOrderByReverseOptimization(salted,true,false,true);
+ doTestOrderByReverseOptimization(salted,true,false,false);
+ doTestOrderByReverseOptimization(salted,false,true,true);
+ doTestOrderByReverseOptimization(salted,false,true,false);
+ doTestOrderByReverseOptimization(salted,false,false,true);
+ doTestOrderByReverseOptimization(salted,false,false,false);
+ }
+ }
+
+ private void doTestOrderByReverseOptimization(boolean salted,boolean desc1,boolean desc2,boolean desc3) throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String tableName=generateUniqueName();
+ String sql="CREATE TABLE "+tableName+" ( "+
+ "ORGANIZATION_ID INTEGER NOT NULL,"+
+ "CONTAINER_ID INTEGER NOT NULL,"+
+ "SCORE INTEGER NOT NULL,"+
+ "ENTITY_ID INTEGER NOT NULL,"+
+ "CONSTRAINT TEST_PK PRIMARY KEY ( "+
+ "ORGANIZATION_ID" +(desc1 ? " DESC" : "" )+","+
+ "CONTAINER_ID"+(desc2 ? " DESC" : "" )+","+
+ "SCORE"+(desc3 ? " DESC" : "" )+","+
+ "ENTITY_ID"+
+ ")) "+(salted ? "SALT_BUCKETS =4" : "split on(4)");
+ conn.createStatement().execute(sql);
+
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,1,1,1)");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (2,2,2,2)");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (3,3,3,3)");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (4,4,4,4)");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (5,5,5,5)");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (6,6,6,6)");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (1,1,1,11)");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (2,2,2,22)");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (3,3,3,33)");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (4,4,4,44)");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (5,5,5,55)");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (6,6,6,66)");
+ conn.commit();
+
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectColumns(
+ Lists.newArrayList("CONTAINER_ID", "ORGANIZATION_ID"))
+ .setFullTableName(tableName)
+ .setGroupByClause("ORGANIZATION_ID, CONTAINER_ID")
+ .setOrderByClause( "ORGANIZATION_ID ASC, CONTAINER_ID ASC");
+ //groupBy orderPreserving orderBy asc asc
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC, CONTAINER_ID ASC";
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{1,1},{2,2},{3,3},{4,4},{5,5},{6,6}});
+
+ //groupBy orderPreserving orderBy asc desc
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC, CONTAINER_ID desc";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID ASC, CONTAINER_ID DESC");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{1,1},{2,2},{3,3},{4,4},{5,5},{6,6}});
+
+ //groupBy orderPreserving orderBy desc asc
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC, CONTAINER_ID ASC";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC, CONTAINER_ID ASC");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{6,6},{5,5},{4,4},{3,3},{2,2},{1,1}});
+
+ //groupBy orderPreserving orderBy desc desc
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC, CONTAINER_ID DESC";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC, CONTAINER_ID DESC");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{6,6},{5,5},{4,4},{3,3},{2,2},{1,1}});
+
+ //groupBy not orderPreserving orderBy asc asc
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC, SCORE ASC";
+ queryBuilder.setSelectColumns(
+ Lists.newArrayList("ORGANIZATION_ID", "SCORE"));
+ queryBuilder.setGroupByClause("ORGANIZATION_ID, SCORE");
+ queryBuilder.setOrderByClause("ORGANIZATION_ID ASC, SCORE ASC");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{1,1},{2,2},{3,3},{4,4},{5,5},{6,6}});
+
+ //groupBy not orderPreserving orderBy asc desc
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC, SCORE DESC";
+ queryBuilder.setGroupByClause("ORGANIZATION_ID, SCORE");
+ queryBuilder.setOrderByClause("ORGANIZATION_ID ASC, SCORE DESC");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{1,1},{2,2},{3,3},{4,4},{5,5},{6,6}});
+
+ //groupBy not orderPreserving orderBy desc asc
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC, SCORE ASC";
+ queryBuilder.setGroupByClause("ORGANIZATION_ID, SCORE");
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC, SCORE ASC");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{6,6},{5,5},{4,4},{3,3},{2,2},{1,1}});
+
+ //groupBy not orderPreserving orderBy desc desc
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC, SCORE DESC";
+ queryBuilder.setGroupByClause("ORGANIZATION_ID, SCORE");
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC, SCORE DESC");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{6,6},{5,5},{4,4},{3,3},{2,2},{1,1}});
+ }
+ }
+
+ @Test
+ public void testOrderByReverseOptimizationWithNullsLast() throws Exception{
+ for(boolean salted: new boolean[]{true,false}) {
+ doTestOrderByReverseOptimizationWithNullsLast(salted,true,true,true);
+ doTestOrderByReverseOptimizationWithNullsLast(salted,true,true,false);
+ doTestOrderByReverseOptimizationWithNullsLast(salted,true,false,true);
+ doTestOrderByReverseOptimizationWithNullsLast(salted,true,false,false);
+ doTestOrderByReverseOptimizationWithNullsLast(salted,false,true,true);
+ doTestOrderByReverseOptimizationWithNullsLast(salted,false,true,false);
+ doTestOrderByReverseOptimizationWithNullsLast(salted,false,false,true);
+ doTestOrderByReverseOptimizationWithNullsLast(salted,false,false,false);
+ }
+ }
+
+ private void doTestOrderByReverseOptimizationWithNullsLast(boolean salted,boolean desc1,boolean desc2,boolean desc3) throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ String tableName=generateUniqueName();
+ String sql="CREATE TABLE "+tableName+" ( "+
+ "ORGANIZATION_ID VARCHAR,"+
+ "CONTAINER_ID VARCHAR,"+
+ "SCORE VARCHAR,"+
+ "ENTITY_ID VARCHAR NOT NULL,"+
+ "CONSTRAINT TEST_PK PRIMARY KEY ( "+
+ "ORGANIZATION_ID" +(desc1 ? " DESC" : "" )+","+
+ "CONTAINER_ID"+(desc2 ? " DESC" : "" )+","+
+ "SCORE"+(desc3 ? " DESC" : "" )+","+
+ "ENTITY_ID"+
+ ")) "+(salted ? "SALT_BUCKETS =4" : "split on('4')");
+ conn.createStatement().execute(sql);
+
+ for(int i=1;i<=6;i++) {
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,'"+i+"','"+i+"','"+i+"')");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,'"+i+"',null,'"+i+"')");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,null,'"+i+"','"+i+"')");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,null,null,'"+i+"')");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('"+i+"','"+i+"','"+i+"','"+i+"')");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('"+i+"','"+i+"',null,'"+i+"')");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('"+i+"',null,'"+i+"','"+i+"')");
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES ('"+i+"',null,null,'"+i+"')");
+ }
+ conn.createStatement().execute("UPSERT INTO "+tableName+" VALUES (null,null,null,'66')");
+ conn.commit();
+
+ //groupBy orderPreserving orderBy asc asc
+
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID ASC NULLS FIRST";
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectColumns(
+ Lists.newArrayList("ORGANIZATION_ID", "CONTAINER_ID"))
+ .setFullTableName(tableName)
+ .setGroupByClause("ORGANIZATION_ID, CONTAINER_ID")
+ .setOrderByClause("ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID ASC NULLS FIRST");
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{null,null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{"1",null},{"1","1"},{"2",null},{"2","2"},{"3",null},{"3","3"},{"4",null},{"4","4"},{"5",null},{"5","5"},{"6",null},{"6","6"}});
+
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID ASC NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{null,null},{"1","1"},{"1",null},{"2","2"},{"2",null},{"3","3"},{"3",null},{"4","4"},{"4",null},{"5","5"},{"5",null},{"6","6"},{"6",null}});
+
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORDER BY ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID ASC NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{null,null},{"1","1"},{"1",null},{"2","2"},{"2",null},{"3","3"},{"3",null},{"4","4"},{"4",null},{"5","5"},{"5",null},{"6","6"},{"6",null}});
+
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS LAST, CONTAINER_ID ASC NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS LAST, CONTAINER_ID ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{"1","1"},{"1",null},{"2","2"},{"2",null},{"3","3"},{"3",null},{"4","4"},{"4",null},{"5","5"},{"5",null},{"6","6"},{"6",null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{null,null}});
+
+ //groupBy orderPreserving orderBy asc desc
+
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID DESC NULLS FIRST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{null,null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{"1",null},{"1","1"},{"2",null},{"2","2"},{"3",null},{"3","3"},{"4",null},{"4","4"},{"5",null},{"5","5"},{"6",null},{"6","6"}});
+
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID DESC NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS FIRST, CONTAINER_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{null,null},{"1","1"},{"1",null},{"2","2"},{"2",null},{"3","3"},{"3",null},{"4","4"},{"4",null},{"5","5"},{"5",null},{"6","6"},{"6",null}});
+
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS LAST, CONTAINER_ID DESC NULLS FIRST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS LAST, CONTAINER_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{"1",null},{"1","1"},{"2",null},{"2","2"},{"3",null},{"3","3"},{"4",null},{"4","4"},{"5",null},{"5","5"},{"6",null},{"6","6"},{null,null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"}});
+
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID ASC NULLS LAST, CONTAINER_ID DESC NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS LAST, CONTAINER_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{"1","1"},{"1",null},{"2","2"},{"2",null},{"3","3"},{"3",null},{"4","4"},{"4",null},{"5","5"},{"5",null},{"6","6"},{"6",null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{null,null}});
+
+ //groupBy orderPreserving orderBy desc asc
+
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS FIRST, CONTAINER_ID ASC NULLS FIRST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST, CONTAINER_ID ASC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{null,null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{"6",null},{"6","6"},{"5",null},{"5","5"},{"4",null},{"4","4"},{"3",null},{"3","3"},{"2",null},{"2","2"},{"1",null},{"1","1"}});
+
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS FIRST, CONTAINER_ID ASC NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST, CONTAINER_ID ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{null,null},{"6","6"},{"6",null},{"5","5"},{"5",null},{"4","4"},{"4",null},{"3","3"},{"3",null},{"2","2"},{"2",null},{"1","1"},{"1",null}});
+
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS LAST, CONTAINER_ID ASC NULLS FIRST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST, CONTAINER_ID ASC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{"6",null},{"6","6"},{"5",null},{"5","5"},{"4",null},{"4","4"},{"3",null},{"3","3"},{"2",null},{"2","2"},{"1",null},{"1","1"},{null,null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"}});
+
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS LAST, CONTAINER_ID ASC NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST, CONTAINER_ID ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{"6","6"},{"6",null},{"5","5"},{"5",null},{"4","4"},{"4",null},{"3","3"},{"3",null},{"2","2"},{"2",null},{"1","1"},{"1",null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{null,null}});
+
+ //groupBy orderPreserving orderBy desc desc
+
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS FIRST, CONTAINER_ID DESC NULLS FIRST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST, CONTAINER_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{null,null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{"6",null},{"6","6"},{"5",null},{"5","5"},{"4",null},{"4","4"},{"3",null},{"3","3"},{"2",null},{"2","2"},{"1",null},{"1","1"}});
+
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS FIRST, CONTAINER_ID DESC NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST, CONTAINER_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{null,null},{"6","6"},{"6",null},{"5","5"},{"5",null},{"4","4"},{"4",null},{"3","3"},{"3",null},{"2","2"},{"2",null},{"1","1"},{"1",null}});
+
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS LAST, CONTAINER_ID DESC NULLS FIRST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST, CONTAINER_ID DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{"6",null},{"6","6"},{"5",null},{"5","5"},{"4",null},{"4","4"},{"3",null},{"3","3"},{"2",null},{"2","2"},{"1",null},{"1","1"},{null,null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"}});
+
+// sql="SELECT ORGANIZATION_ID,CONTAINER_ID FROM "+tableName+" group by ORGANIZATION_ID, CONTAINER_ID ORDER BY ORGANIZATION_ID DESC NULLS LAST, CONTAINER_ID DESC NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST, CONTAINER_ID DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{"6","6"},{"6",null},{"5","5"},{"5",null},{"4","4"},{"4",null},{"3","3"},{"3",null},{"2","2"},{"2",null},{"1","1"},{"1",null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{null,null}});
+
+ //-----groupBy not orderPreserving--
+
+ //groupBy not orderPreserving orderBy asc asc
+
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS FIRST, SCORE ASC NULLS FIRST";
+ queryBuilder.setSelectColumns(
+ Lists.newArrayList("ORGANIZATION_ID", "SCORE"))
+ .setFullTableName(tableName)
+ .setGroupByClause("ORGANIZATION_ID, SCORE")
+ .setOrderByClause("ORGANIZATION_ID ASC NULLS FIRST, SCORE ASC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{null,null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{"1",null},{"1","1"},{"2",null},{"2","2"},{"3",null},{"3","3"},{"4",null},{"4","4"},{"5",null},{"5","5"},{"6",null},{"6","6"}});
+
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS FIRST, SCORE ASC NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS FIRST, SCORE ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{null,null},{"1","1"},{"1",null},{"2","2"},{"2",null},{"3","3"},{"3",null},{"4","4"},{"4",null},{"5","5"},{"5",null},{"6","6"},{"6",null}});
+
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS LAST, SCORE ASC NULLS FIRST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS LAST, SCORE ASC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{"1",null},{"1","1"},{"2",null},{"2","2"},{"3",null},{"3","3"},{"4",null},{"4","4"},{"5",null},{"5","5"},{"6",null},{"6","6"},{null,null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"}});
+
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS LAST, SCORE ASC NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS LAST, SCORE ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{"1","1"},{"1",null},{"2","2"},{"2",null},{"3","3"},{"3",null},{"4","4"},{"4",null},{"5","5"},{"5",null},{"6","6"},{"6",null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{null,null}});
+
+ //groupBy not orderPreserving orderBy asc desc
+
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS FIRST, SCORE DESC NULLS FIRST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS FIRST, SCORE DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{null,null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{"1",null},{"1","1"},{"2",null},{"2","2"},{"3",null},{"3","3"},{"4",null},{"4","4"},{"5",null},{"5","5"},{"6",null},{"6","6"}});
+
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS FIRST, SCORE DESC NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS FIRST, SCORE DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{null,null},{"1","1"},{"1",null},{"2","2"},{"2",null},{"3","3"},{"3",null},{"4","4"},{"4",null},{"5","5"},{"5",null},{"6","6"},{"6",null}});
+
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS LAST, SCORE DESC NULLS FIRST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS LAST, SCORE DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{"1",null},{"1","1"},{"2",null},{"2","2"},{"3",null},{"3","3"},{"4",null},{"4","4"},{"5",null},{"5","5"},{"6",null},{"6","6"},{null,null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"}});
+
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID ASC NULLS LAST, SCORE DESC NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID ASC NULLS LAST, SCORE DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{"1","1"},{"1",null},{"2","2"},{"2",null},{"3","3"},{"3",null},{"4","4"},{"4",null},{"5","5"},{"5",null},{"6","6"},{"6",null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{null,null}});
+
+ //groupBy not orderPreserving orderBy desc asc
+
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS FIRST, SCORE ASC NULLS FIRST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST, SCORE ASC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{null,null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{"6",null},{"6","6"},{"5",null},{"5","5"},{"4",null},{"4","4"},{"3",null},{"3","3"},{"2",null},{"2","2"},{"1",null},{"1","1"}});
+
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS FIRST, SCORE ASC NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST, SCORE ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{null,null},{"6","6"},{"6",null},{"5","5"},{"5",null},{"4","4"},{"4",null},{"3","3"},{"3",null},{"2","2"},{"2",null},{"1","1"},{"1",null}});
+
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS LAST, SCORE ASC NULLS FIRST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST, SCORE ASC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{"6",null},{"6","6"},{"5",null},{"5","5"},{"4",null},{"4","4"},{"3",null},{"3","3"},{"2",null},{"2","2"},{"1",null},{"1","1"},{null,null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"}});
+
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS LAST, SCORE ASC NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST, SCORE ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{"6","6"},{"6",null},{"5","5"},{"5",null},{"4","4"},{"4",null},{"3","3"},{"3",null},{"2","2"},{"2",null},{"1","1"},{"1",null},{null,"1"},{null,"2"},{null,"3"},{null,"4"},{null,"5"},{null,"6"},{null,null}});
+
+ //groupBy not orderPreserving orderBy desc desc
+
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS FIRST, SCORE DESC NULLS FIRST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST, SCORE DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{null,null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{"6",null},{"6","6"},{"5",null},{"5","5"},{"4",null},{"4","4"},{"3",null},{"3","3"},{"2",null},{"2","2"},{"1",null},{"1","1"}});
+
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS FIRST, SCORE DESC NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS FIRST, SCORE DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{null,null},{"6","6"},{"6",null},{"5","5"},{"5",null},{"4","4"},{"4",null},{"3","3"},{"3",null},{"2","2"},{"2",null},{"1","1"},{"1",null}});
+
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS LAST, SCORE DESC NULLS FIRST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST, SCORE DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{"6",null},{"6","6"},{"5",null},{"5","5"},{"4",null},{"4","4"},{"3",null},{"3","3"},{"2",null},{"2","2"},{"1",null},{"1","1"},{null,null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"}});
+
+// sql="SELECT ORGANIZATION_ID,SCORE FROM "+tableName+" group by ORGANIZATION_ID, SCORE ORDER BY ORGANIZATION_ID DESC NULLS LAST, SCORE DESC NULLS LAST";
+ queryBuilder.setOrderByClause("ORGANIZATION_ID DESC NULLS LAST, SCORE DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{"6","6"},{"6",null},{"5","5"},{"5",null},{"4","4"},{"4",null},{"3","3"},{"3",null},{"2","2"},{"2",null},{"1","1"},{"1",null},{null,"6"},{null,"5"},{null,"4"},{null,"3"},{null,"2"},{null,"1"},{null,null}});
+
+ //-------test only one return column----------------------------------
+
+// sql="SELECT SCORE FROM "+tableName+" group by SCORE ORDER BY SCORE ASC NULLS FIRST";
+ queryBuilder.setSelectColumns(
+ Lists.newArrayList("SCORE"))
+ .setFullTableName(tableName)
+ .setGroupByClause("SCORE")
+ .setOrderByClause("SCORE ASC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{null},{"1"},{"2"},{"3"},{"4"},{"5"},{"6"}});
+
+// sql="SELECT SCORE FROM "+tableName+" group by SCORE ORDER BY SCORE ASC NULLS LAST";
+ queryBuilder.setOrderByClause("SCORE ASC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{"1"},{"2"},{"3"},{"4"},{"5"},{"6"},{null}});
+
+// sql="SELECT SCORE FROM "+tableName+" group by SCORE ORDER BY SCORE DESC NULLS FIRST";
+ queryBuilder.setOrderByClause("SCORE DESC NULLS FIRST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{null},{"6"},{"5"},{"4"},{"3"},{"2"},{"1"}});
+
+// sql="SELECT SCORE FROM "+tableName+" group by SCORE ORDER BY SCORE DESC NULLS LAST";
+ queryBuilder.setOrderByClause("SCORE DESC NULLS LAST");
+ rs = executeQuery(conn, queryBuilder);
+ assertResultSet(rs, new Object[][]{{"6"},{"5"},{"4"},{"3"},{"2"},{"1"},{null}});
+ }
+ }
+
+}
\ No newline at end of file
[05/22] phoenix git commit: PHOENIX-4891: An index should inherit
UPDATE_CACHE_FREQUENCY setting rom parent table
Posted by ja...@apache.org.
PHOENIX-4891: An index should inherit UPDATE_CACHE_FREQUENCY setting rom parent table
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/64788794
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/64788794
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/64788794
Branch: refs/heads/omid2
Commit: 647887947f631d624d6897d729bce3acb07a94bf
Parents: fe466eb
Author: Chinmay Kulkarni <ch...@gmail.com>
Authored: Tue Oct 30 14:52:26 2018 -0700
Committer: Thomas D'Silva <td...@apache.org>
Committed: Tue Oct 30 16:23:42 2018 -0700
----------------------------------------------------------------------
.../phoenix/end2end/PropertiesInSyncIT.java | 172 ++++++++++++++-----
.../phoenix/end2end/index/IndexMetadataIT.java | 145 +++++++++++++++-
.../org/apache/phoenix/rpc/UpdateCacheIT.java | 134 ++++++++++-----
.../phoenix/exception/SQLExceptionCode.java | 7 +-
.../query/ConnectionQueryServicesImpl.java | 6 +-
.../apache/phoenix/schema/MetaDataClient.java | 33 +++-
.../org/apache/phoenix/util/MetaDataUtil.java | 6 +-
.../org/apache/phoenix/util/UpgradeUtil.java | 79 ++++++++-
8 files changed, 476 insertions(+), 106 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/64788794/phoenix-core/src/it/java/org/apache/phoenix/end2end/PropertiesInSyncIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/PropertiesInSyncIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/PropertiesInSyncIT.java
index db44735..348b195 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/PropertiesInSyncIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/PropertiesInSyncIT.java
@@ -23,9 +23,12 @@ import org.apache.hadoop.hbase.KeepDeletedCells;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
import org.apache.phoenix.exception.SQLExceptionCode;
import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.schema.PName;
import org.apache.phoenix.schema.PTable;
+import org.apache.phoenix.schema.PTableType;
import org.apache.phoenix.util.MetaDataUtil;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.SchemaUtil;
@@ -33,17 +36,23 @@ import org.junit.Test;
import java.sql.Connection;
import java.sql.DriverManager;
+import java.sql.PreparedStatement;
import java.sql.SQLException;
+import java.util.HashMap;
import java.util.HashSet;
+import java.util.Map;
import java.util.Properties;
import java.util.Set;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.fail;
import static org.apache.phoenix.query.QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES;
-import static org.apache.phoenix.util.MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES;
+import static org.apache.phoenix.util.MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES;
import static org.apache.phoenix.util.MetaDataUtil.VIEW_INDEX_TABLE_PREFIX;
+import static org.apache.phoenix.util.UpgradeUtil.UPSERT_UPDATE_CACHE_FREQUENCY;
import static org.apache.phoenix.util.UpgradeUtil.syncTableAndIndexProperties;
+import static org.apache.phoenix.util.UpgradeUtil.syncUpdateCacheFreqAllIndexes;
+import static org.apache.phoenix.end2end.index.IndexMetadataIT.assertUpdateCacheFreq;
/**
* Test properties that need to be kept in sync amongst all column families and indexes of a table
@@ -56,12 +65,16 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
private static final int INITIAL_TTL_VALUE = 700;
private static final KeepDeletedCells INITIAL_KEEP_DELETED_CELLS_VALUE = KeepDeletedCells.TRUE;
private static final int INITIAL_REPLICATION_SCOPE_VALUE = 1;
+ private static final int INITIAL_UPDATE_CACHE_FREQUENCY = 100;
+ private static final int INITIAL_UPDATE_CACHE_FREQUENCY_VIEWS = 900;
private static final int MODIFIED_TTL_VALUE = INITIAL_TTL_VALUE + 300;
private static final KeepDeletedCells MODIFIED_KEEP_DELETED_CELLS_VALUE =
- (INITIAL_KEEP_DELETED_CELLS_VALUE == KeepDeletedCells.TRUE)
- ? KeepDeletedCells.FALSE: KeepDeletedCells.TRUE;
+ (INITIAL_KEEP_DELETED_CELLS_VALUE == KeepDeletedCells.TRUE) ?
+ KeepDeletedCells.FALSE: KeepDeletedCells.TRUE;
private static final int MODIFIED_REPLICATION_SCOPE_VALUE =
(INITIAL_REPLICATION_SCOPE_VALUE == 1) ? 0 : 1;
+ private static final int MODIFIED_UPDATE_CACHE_FREQUENCY = INITIAL_UPDATE_CACHE_FREQUENCY + 300;
+ private static final int MODIFIED_UPDATE_CACHE_FREQUENCY_VIEWS = INITIAL_UPDATE_CACHE_FREQUENCY_VIEWS + 300;
// Test that we disallow specifying synced properties to be set per column family
@@ -70,7 +83,7 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
public void testDisallowSyncedPropsToBeSetColFamSpecificCreateTable() throws Exception {
Connection conn = DriverManager.getConnection(getUrl(), new Properties());
String tableName = generateUniqueName();
- for (String propName: SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES) {
+ for (String propName: SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES) {
try {
conn.createStatement().execute("create table " + tableName
+ " (id INTEGER not null primary key, "
@@ -107,7 +120,7 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
String viewName = "VIEW_" + tableName;
conn.createStatement().execute("create view " + viewName
+ " (new_col SMALLINT) as select * from " + tableName + " where id > 1");
- for (String propName: SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES) {
+ for (String propName: SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES) {
try {
conn.createStatement().execute("create local index " + localIndexName
+ " on " + tableName + "(name) "
@@ -148,8 +161,8 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
public void testSyncedPropsBaseTableCreateIndex() throws Exception {
Connection conn = DriverManager.getConnection(getUrl(), new Properties());
String tableName = createBaseTableWithProps(conn);
- createIndexTable(conn, tableName, PTable.IndexType.LOCAL);
- String globalIndexName = createIndexTable(conn, tableName, PTable.IndexType.GLOBAL);
+ createIndexTable(conn, tableName, PTable.IndexType.LOCAL).getSecond();
+ String globalIndexName = createIndexTable(conn, tableName, PTable.IndexType.GLOBAL).getSecond();
// We pass the base table as the physical HBase table since our check includes checking
// the local index column family too
@@ -164,7 +177,7 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
public void testSyncedPropsBaseTableCreateViewIndex() throws Exception {
Connection conn = DriverManager.getConnection(getUrl(), new Properties());
String tableName = createBaseTableWithProps(conn);
- String viewIndexName = createIndexTable(conn, tableName, null);
+ String viewIndexName = createIndexTable(conn, tableName, null).getSecond();
verifyHBaseColumnFamilyProperties(tableName, conn, false, false);
verifyHBaseColumnFamilyProperties(viewIndexName, conn, false, false);
@@ -179,7 +192,7 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
String tableName = createBaseTableWithProps(conn);
StringBuilder alterAllSyncedPropsString = new StringBuilder();
String modPropString = COL_FAM1 + ".%s=" + DUMMY_PROP_VALUE + ",";
- for (String propName: SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES) {
+ for (String propName: SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES) {
try {
conn.createStatement().execute("alter table " + tableName
+ " set " + COL_FAM1 + "." + propName + "=" + DUMMY_PROP_VALUE);
@@ -216,11 +229,11 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
Set<String> tablesToCheck = new HashSet<>();
tablesToCheck.add(tableName);
for (int i=0; i<2; i++) {
- tablesToCheck.add(createIndexTable(conn, tableName, PTable.IndexType.LOCAL));
- tablesToCheck.add(createIndexTable(conn, tableName, PTable.IndexType.GLOBAL));
+ tablesToCheck.add(createIndexTable(conn, tableName, PTable.IndexType.LOCAL).getSecond());
+ tablesToCheck.add(createIndexTable(conn, tableName, PTable.IndexType.GLOBAL).getSecond());
}
// Create a view and view index
- tablesToCheck.add(createIndexTable(conn, tableName, null));
+ tablesToCheck.add(createIndexTable(conn, tableName, null).getSecond());
// Now alter the base table's properties. This should get propagated to all index tables
conn.createStatement().execute("alter table " + tableName + " set TTL=" + MODIFIED_TTL_VALUE
@@ -232,8 +245,8 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
}
// Any indexes created henceforth should have the modified properties
- String newGlobalIndex = createIndexTable(conn, tableName, PTable.IndexType.GLOBAL);
- String newViewIndex = createIndexTable(conn, tableName, null);
+ String newGlobalIndex = createIndexTable(conn, tableName, PTable.IndexType.GLOBAL).getSecond();
+ String newViewIndex = createIndexTable(conn, tableName, null).getSecond();
verifyHBaseColumnFamilyProperties(newGlobalIndex, conn, true, false);
verifyHBaseColumnFamilyProperties(newViewIndex, conn, true, false);
conn.close();
@@ -245,9 +258,8 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
Connection conn = DriverManager.getConnection(getUrl(), new Properties());
String tableName = createBaseTableWithProps(conn);
- // Test that we are not allowed to set any property to be kept in sync, specific
- // to the new column family to be added
- for (String propName: SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES) {
+ // Test that we are not allowed to set any property to be kept in sync, specific to the new column family to be added
+ for (String propName: SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES) {
try {
conn.createStatement().execute(
"alter table " + tableName + " add " + NEW_CF + ".new_column varchar(2) "
@@ -268,11 +280,11 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
Set<String> tablesToCheck = new HashSet<>();
tablesToCheck.add(tableName);
for (int i=0; i<2; i++) {
- tablesToCheck.add(createIndexTable(conn, tableName, PTable.IndexType.LOCAL));
- tablesToCheck.add(createIndexTable(conn, tableName, PTable.IndexType.GLOBAL));
+ tablesToCheck.add(createIndexTable(conn, tableName, PTable.IndexType.LOCAL).getSecond());
+ tablesToCheck.add(createIndexTable(conn, tableName, PTable.IndexType.GLOBAL).getSecond());
}
// Create a view and view index
- tablesToCheck.add(createIndexTable(conn, tableName, null));
+ tablesToCheck.add(createIndexTable(conn, tableName, null).getSecond());
// Now add a new column family while simultaneously modifying properties to be kept in sync,
// as well as a property which does not need to be kept in sync. Properties to be kept
@@ -307,8 +319,8 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
public void testDisallowAlterGlobalIndexTable() throws Exception {
Connection conn = DriverManager.getConnection(getUrl(), new Properties());
String tableName = createBaseTableWithProps(conn);
- String globalIndexName = createIndexTable(conn, tableName, PTable.IndexType.GLOBAL);
- for (String propName: SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES) {
+ String globalIndexName = createIndexTable(conn, tableName, PTable.IndexType.GLOBAL).getSecond();
+ for (String propName: SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES) {
try {
conn.createStatement().execute("alter table " + globalIndexName + " set "
+ propName + "=" + DUMMY_PROP_VALUE);
@@ -336,12 +348,12 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
createdTables.add(baseTableName1);
// Create different indexes on the base table
for (int i=0; i<2; i++) {
- createdTables.add(createIndexTable(conn, baseTableName, PTable.IndexType.GLOBAL));
- createdTables.add(createIndexTable(conn, baseTableName, PTable.IndexType.LOCAL));
- createdTables.add(createIndexTable(conn, baseTableName, null));
- createdTables.add(createIndexTable(conn, baseTableName1, PTable.IndexType.GLOBAL));
- createdTables.add(createIndexTable(conn, baseTableName1, PTable.IndexType.LOCAL));
- createdTables.add(createIndexTable(conn, baseTableName1, null));
+ createdTables.add(createIndexTable(conn, baseTableName, PTable.IndexType.GLOBAL).getSecond());
+ createdTables.add(createIndexTable(conn, baseTableName, PTable.IndexType.LOCAL).getSecond());
+ createdTables.add(createIndexTable(conn, baseTableName, null).getSecond());
+ createdTables.add(createIndexTable(conn, baseTableName1, PTable.IndexType.GLOBAL).getSecond());
+ createdTables.add(createIndexTable(conn, baseTableName1, PTable.IndexType.LOCAL).getSecond());
+ createdTables.add(createIndexTable(conn, baseTableName1, null).getSecond());
}
for (String t: createdTables) {
verifyHBaseColumnFamilyProperties(t, conn, false, false);
@@ -394,6 +406,77 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
conn.close();
}
+ @Test
+ public void testOldClientSyncUpdateCacheFreqUpgradePath() throws Exception {
+ PTable base, index;
+ String baseTableName, viewName, viewName2;
+ Map<String, Set<String>> createdTablesAndViews = new HashMap<>();
+
+ try (Connection conn = DriverManager.getConnection(getUrl(), new Properties())) {
+ baseTableName = createBaseTableWithProps(conn);
+ createdTablesAndViews.put(baseTableName, new HashSet<String>());
+ Set<String> indexes = createdTablesAndViews.get(baseTableName);
+ indexes.add(createIndexTable(conn, baseTableName, PTable.IndexType.GLOBAL).getSecond());
+ indexes.add(createIndexTable(conn, baseTableName, PTable.IndexType.LOCAL).getFirst());
+
+ viewName = createViewOnBaseTableOrView(conn, baseTableName);
+ createdTablesAndViews.put(viewName, new HashSet<String>());
+ indexes = createdTablesAndViews.get(viewName);
+ indexes.add(createIndexTable(conn, viewName, PTable.IndexType.GLOBAL).getSecond());
+
+ viewName2 = createViewOnBaseTableOrView(conn, viewName);
+ createdTablesAndViews.put(viewName2, new HashSet<String>());
+ indexes = createdTablesAndViews.get(viewName2);
+ indexes.add(createIndexTable(conn, viewName2, PTable.IndexType.LOCAL).getFirst());
+
+ // Intentionally make UPDATE_CACHE_FREQUENCY out of sync for indexes
+ PreparedStatement stmt = conn.prepareStatement(UPSERT_UPDATE_CACHE_FREQUENCY);
+ for (String tableOrViewName : createdTablesAndViews.keySet()) {
+ base = PhoenixRuntime.getTable(conn, tableOrViewName);
+ for (String indexTableName : createdTablesAndViews.get(tableOrViewName)) {
+ index = PhoenixRuntime.getTable(conn, indexTableName);
+ PName tenantId = index.getTenantId();
+ stmt.setString(1, tenantId == null ? null : tenantId.getString());
+ stmt.setString(2, index.getSchemaName().getString());
+ stmt.setString(3, index.getTableName().getString());
+ stmt.setLong(4, base.getType() == PTableType.TABLE ?
+ MODIFIED_UPDATE_CACHE_FREQUENCY : MODIFIED_UPDATE_CACHE_FREQUENCY_VIEWS);
+ stmt.addBatch();
+ }
+ }
+ stmt.executeBatch();
+ conn.commit();
+
+ // Clear the server-side cache so that we get the latest built PTables
+ conn.unwrap(PhoenixConnection.class).getQueryServices().clearCache();
+ // Verify that the modified values are reflected
+ for (String tableOrViewName : createdTablesAndViews.keySet()) {
+ assertUpdateCacheFreq(conn, tableOrViewName, baseTableName.equals(tableOrViewName) ?
+ INITIAL_UPDATE_CACHE_FREQUENCY : INITIAL_UPDATE_CACHE_FREQUENCY_VIEWS);
+ for (String indexName : createdTablesAndViews.get(tableOrViewName)) {
+ assertUpdateCacheFreq(conn, indexName, baseTableName.equals(tableOrViewName) ?
+ MODIFIED_UPDATE_CACHE_FREQUENCY : MODIFIED_UPDATE_CACHE_FREQUENCY_VIEWS);
+ }
+ }
+
+ PhoenixConnection upgradeConn = conn.unwrap(PhoenixConnection.class);
+ upgradeConn.setRunningUpgrade(true);
+ syncUpdateCacheFreqAllIndexes(upgradeConn,
+ PhoenixRuntime.getTableNoCache(conn, baseTableName));
+
+ conn.unwrap(PhoenixConnection.class).getQueryServices().clearCache();
+ // Verify that indexes have the synced values for UPDATE_CACHE_FREQUENCY
+ for (String tableOrViewName : createdTablesAndViews.keySet()) {
+ long expectedVal = baseTableName.equals(tableOrViewName) ?
+ INITIAL_UPDATE_CACHE_FREQUENCY : INITIAL_UPDATE_CACHE_FREQUENCY_VIEWS;
+ assertUpdateCacheFreq(conn, tableOrViewName, expectedVal);
+ for (String indexOnTableOrView : createdTablesAndViews.get(tableOrViewName)) {
+ assertUpdateCacheFreq(conn, indexOnTableOrView, expectedVal);
+ }
+ }
+ }
+ }
+
/**
* Helper method to modify the synced properties for a column family descriptor
* @param cfd The column family descriptor object
@@ -417,9 +500,9 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
conn.createStatement().execute("create table " + tableName
+ " (id INTEGER not null primary key, type varchar(5), "
+ COL_FAM1 + ".name varchar(10), " + COL_FAM2 + ".flag boolean) "
- + "TTL=" + INITIAL_TTL_VALUE + ",KEEP_DELETED_CELLS="
- + INITIAL_KEEP_DELETED_CELLS_VALUE
- + ",REPLICATION_SCOPE=" + INITIAL_REPLICATION_SCOPE_VALUE);
+ + "TTL=" + INITIAL_TTL_VALUE + ",KEEP_DELETED_CELLS=" + INITIAL_KEEP_DELETED_CELLS_VALUE
+ + ",REPLICATION_SCOPE=" + INITIAL_REPLICATION_SCOPE_VALUE
+ + ",UPDATE_CACHE_FREQUENCY=" + INITIAL_UPDATE_CACHE_FREQUENCY);
return tableName;
}
@@ -429,34 +512,43 @@ public class PropertiesInSyncIT extends ParallelStatsDisabledIT {
* @param baseTableName Name of the HBase base table on which to create an index
* @param indexType LOCAL, GLOBAL or if we pass in null as the indexType,
* we create a view and an index on that view for the given base table
- * @return The physical HBase table corresponding to the index created
+ * @return A pair consisting of the index name and the name of the physical HBase table
+ * corresponding to the index created
* @throws SQLException
*/
- private String createIndexTable(Connection conn, String baseTableName,
+ private Pair<String,String> createIndexTable(Connection conn, String baseTableName,
PTable.IndexType indexType) throws SQLException {
// Create a view on top of the base table and then an index on that view
if (indexType == null) {
- String viewName = "VIEW_" + baseTableName + "_" + generateUniqueName();
+ String viewName = createViewOnBaseTableOrView(conn, baseTableName);
String viewIndexName = VIEW_INDEX_TABLE_PREFIX + baseTableName;
- conn.createStatement().execute("create view " + viewName
- + " (new_col SMALLINT) as select * from " + baseTableName + " where id > 1");
conn.createStatement().execute("create index view_index_" + generateUniqueName()
+ " on " + viewName + " (flag)");
- return viewIndexName;
+ return new Pair<>(viewIndexName, viewIndexName);
}
switch(indexType) {
case LOCAL:
String localIndexName = baseTableName + "_LOCAL_" + generateUniqueName();
conn.createStatement().execute(
"create local index " + localIndexName + " on " + baseTableName + "(flag)");
- return baseTableName;
+ return new Pair<>(localIndexName, baseTableName);
case GLOBAL:
String globalIndexName = baseTableName + "_GLOBAL_" + generateUniqueName();
conn.createStatement()
.execute("create index " + globalIndexName + " on " + baseTableName + "(name)");
- return globalIndexName;
+ return new Pair<>(globalIndexName, globalIndexName);
+ default:
+ return new Pair<>(baseTableName, baseTableName);
}
- return baseTableName;
+ }
+
+ private String createViewOnBaseTableOrView(Connection conn, String baseTableOrView) throws SQLException {
+ String viewName = "VIEW_" + baseTableOrView + "_" + generateUniqueName();
+ conn.createStatement().execute("create view " + viewName
+ + " (" + generateUniqueName() + " SMALLINT) as select * from "
+ + baseTableOrView + " where id > 1 UPDATE_CACHE_FREQUENCY="
+ + INITIAL_UPDATE_CACHE_FREQUENCY_VIEWS);
+ return viewName;
}
/**
http://git-wip-us.apache.org/repos/asf/phoenix/blob/64788794/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexMetadataIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexMetadataIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexMetadataIT.java
index 21fa312..f797b70 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexMetadataIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/IndexMetadataIT.java
@@ -19,6 +19,8 @@ package org.apache.phoenix.end2end.index;
import static org.apache.phoenix.util.TestUtil.INDEX_DATA_SCHEMA;
import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.apache.phoenix.exception.SQLExceptionCode.CANNOT_SET_OR_ALTER_UPDATE_CACHE_FREQ_FOR_INDEX;
+import static org.apache.phoenix.query.QueryServicesOptions.DEFAULT_UPDATE_CACHE_FREQUENCY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
@@ -675,15 +677,12 @@ public class IndexMetadataIT extends ParallelStatsDisabledIT {
}
}
-
-
@Test
public void testIndexAlterPhoenixProperty() throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
Connection conn = DriverManager.getConnection(getUrl(), props);
String testTable = generateUniqueName();
-
String ddl = "create table " + testTable + " (k varchar primary key, v1 varchar)";
Statement stmt = conn.createStatement();
stmt.execute(ddl);
@@ -703,6 +702,123 @@ public class IndexMetadataIT extends ParallelStatsDisabledIT {
assertEquals(20,rs.getInt(1));
}
+ @Test
+ public void testCreateIndexSetUpdateCacheFreqFails() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String testTable = generateUniqueName();
+
+ String ddl = "CREATE TABLE " + testTable + " (k varchar primary key, v1 varchar)";
+ Statement stmt = conn.createStatement();
+ stmt.execute(ddl);
+ String indexName = "IDX_" + generateUniqueName();
+
+ ddl = "CREATE INDEX " + indexName + " ON " + testTable + " (v1) " +
+ "UPDATE_CACHE_FREQUENCY=10000";
+ try {
+ stmt.execute(ddl);
+ fail("Should fail trying to set UPDATE_CACHE_FREQUENCY when creating an index");
+ } catch (SQLException sqlE) {
+ assertEquals("Unexpected error occurred",
+ CANNOT_SET_OR_ALTER_UPDATE_CACHE_FREQ_FOR_INDEX.getErrorCode(), sqlE.getErrorCode());
+ }
+ }
+
+ @Test
+ public void testIndexGetsUpdateCacheFreqFromBaseTable() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String testTable = generateUniqueName();
+
+ long updateCacheFreq = 10000;
+ String ddl = "CREATE TABLE " + testTable + " (k varchar primary key, v1 varchar) " +
+ "UPDATE_CACHE_FREQUENCY=" + updateCacheFreq;
+ Statement stmt = conn.createStatement();
+ stmt.execute(ddl);
+
+ String localIndex = "LOCAL_" + generateUniqueName();
+ String globalIndex = "GLOBAL_" + generateUniqueName();
+
+ ddl = "CREATE LOCAL INDEX " + localIndex + " ON " + testTable + " (v1) ";
+ stmt.execute(ddl);
+ ddl = "CREATE INDEX " + globalIndex + " ON " + testTable + " (v1) ";
+ stmt.execute(ddl);
+
+ // Check that local and global index both have the propagated UPDATE_CACHE_FREQUENCY value
+ assertUpdateCacheFreq(conn, testTable, updateCacheFreq);
+ assertUpdateCacheFreq(conn, localIndex, updateCacheFreq);
+ assertUpdateCacheFreq(conn, globalIndex, updateCacheFreq);
+ }
+
+ @Test
+ public void testAlterTablePropagatesUpdateCacheFreqToIndexes() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String testTable = generateUniqueName();
+
+ String ddl = "CREATE TABLE " + testTable + " (k varchar primary key, v1 varchar) ";
+ Statement stmt = conn.createStatement();
+ stmt.execute(ddl);
+
+ String localIndex = "LOCAL_" + generateUniqueName();
+ String globalIndex = "GLOBAL_" + generateUniqueName();
+
+ ddl = "CREATE LOCAL INDEX " + localIndex + " ON " + testTable + " (v1) ";
+ stmt.execute(ddl);
+ ddl = "CREATE INDEX " + globalIndex + " ON " + testTable + " (v1) ";
+ stmt.execute(ddl);
+
+ assertUpdateCacheFreq(conn, testTable, DEFAULT_UPDATE_CACHE_FREQUENCY);
+ assertUpdateCacheFreq(conn, localIndex, DEFAULT_UPDATE_CACHE_FREQUENCY);
+ assertUpdateCacheFreq(conn, globalIndex, DEFAULT_UPDATE_CACHE_FREQUENCY);
+
+ // Alter UPDATE_CACHE_FREQUENCY on the base table
+ long updateCacheFreq = 10000;
+ ddl = "ALTER TABLE " + testTable + " SET UPDATE_CACHE_FREQUENCY=" + updateCacheFreq;
+ stmt.execute(ddl);
+
+ // Check that local and global index both have the propagated UPDATE_CACHE_FREQUENCY value
+ assertUpdateCacheFreq(conn, testTable, updateCacheFreq);
+ assertUpdateCacheFreq(conn, localIndex, updateCacheFreq);
+ assertUpdateCacheFreq(conn, globalIndex, updateCacheFreq);
+ }
+
+ @Test
+ public void testIndexAlterUpdateCacheFreqFails() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ String testTable = generateUniqueName();
+
+ String ddl = "CREATE TABLE " + testTable + " (k varchar primary key, v1 varchar)";
+ Statement stmt = conn.createStatement();
+ stmt.execute(ddl);
+
+ String localIndex = "LOCAL_" + generateUniqueName();
+ String globalIndex = "GLOBAL_" + generateUniqueName();
+
+ ddl = "CREATE LOCAL INDEX " + localIndex + " ON " + testTable + " (v1) ";
+ stmt.execute(ddl);
+ ddl = "CREATE INDEX " + globalIndex + " ON " + testTable + " (v1) ";
+ stmt.execute(ddl);
+
+ try {
+ stmt.execute("ALTER INDEX " + localIndex + " ON " + testTable +
+ " ACTIVE SET UPDATE_CACHE_FREQUENCY=NEVER");
+ fail("Should fail trying to alter UPDATE_CACHE_FREQUENCY on index");
+ } catch (SQLException sqlE) {
+ assertEquals("Unexpected error occurred",
+ CANNOT_SET_OR_ALTER_UPDATE_CACHE_FREQ_FOR_INDEX.getErrorCode(), sqlE.getErrorCode());
+ }
+
+ try {
+ stmt.execute("ALTER INDEX " + globalIndex + " ON " + testTable +
+ " ACTIVE SET UPDATE_CACHE_FREQUENCY=NEVER");
+ fail("Should fail trying to alter UPDATE_CACHE_FREQUENCY on index");
+ } catch (SQLException sqlE) {
+ assertEquals("Unexpected error occurred",
+ CANNOT_SET_OR_ALTER_UPDATE_CACHE_FREQ_FOR_INDEX.getErrorCode(), sqlE.getErrorCode());
+ }
+ }
@Test
public void testIndexAlterHBaseProperty() throws Exception {
@@ -718,10 +834,10 @@ public class IndexMetadataIT extends ParallelStatsDisabledIT {
ddl = "CREATE INDEX " + indexName + " ON " + testTable + " (v1) ";
stmt.execute(ddl);
- conn.createStatement().execute("ALTER INDEX "+indexName+" ON " + testTable +" ACTIVE SET DISABLE_WAL=false");
- asssertIsWALDisabled(conn,indexName,false);
conn.createStatement().execute("ALTER INDEX "+indexName+" ON " + testTable +" ACTIVE SET DISABLE_WAL=true");
asssertIsWALDisabled(conn,indexName,true);
+ conn.createStatement().execute("ALTER INDEX "+indexName+" ON " + testTable +" ACTIVE SET DISABLE_WAL=false");
+ asssertIsWALDisabled(conn,indexName,false);
}
private static void asssertIsWALDisabled(Connection conn, String fullTableName, boolean expectedValue) throws SQLException {
@@ -729,4 +845,23 @@ public class IndexMetadataIT extends ParallelStatsDisabledIT {
assertEquals(expectedValue, pconn.getTable(new PTableKey(pconn.getTenantId(), fullTableName)).isWALDisabled());
}
+ /**
+ * Helper method to assert the value of UPDATE_CACHE_FREQUENCY for a table/index/view
+ * @param conn Phoenix connection
+ * @param name table/view/index name
+ * @param expectedUpdateCacheFreq expected value of UPDATE_CACHE_FREQUENCY
+ * @throws SQLException
+ */
+ public static void assertUpdateCacheFreq(Connection conn, String name,
+ long expectedUpdateCacheFreq) throws SQLException {
+ ResultSet rs = conn.createStatement().executeQuery(
+ "select UPDATE_CACHE_FREQUENCY from SYSTEM.\"CATALOG\" where TABLE_NAME='" +
+ name + "'");
+ assertTrue(rs.next());
+ assertEquals("Mismatch found for " + name, expectedUpdateCacheFreq, rs.getLong(1));
+ assertEquals("Mismatch in UPDATE_CACHE_FREQUENCY for PTable of " + name,
+ expectedUpdateCacheFreq, PhoenixRuntime.getTableNoCache(conn, name)
+ .getUpdateCacheFrequency());
+ }
+
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/64788794/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
index 0ddbed3..2959b99 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/rpc/UpdateCacheIT.java
@@ -19,6 +19,7 @@ package org.apache.phoenix.rpc;
import static org.apache.phoenix.util.TestUtil.INDEX_DATA_SCHEMA;
import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertFalse;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Matchers.eq;
@@ -73,7 +74,7 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT {
String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + tableName;
Connection conn = DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES));
conn.createStatement().execute("create table " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + "TRANSACTIONAL=true,TRANSACTION_PROVIDER='" + provider + "'");
- helpTestUpdateCache(fullTableName, new int[] {1, 1});
+ helpTestUpdateCache(fullTableName, new int[] {1, 1}, false);
}
}
}
@@ -84,14 +85,14 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT {
String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + tableName;
Connection conn = DriverManager.getConnection(getUrl(), PropertiesUtil.deepCopy(TEST_PROPERTIES));
conn.createStatement().execute("create table " + fullTableName + TestUtil.TEST_TABLE_SCHEMA);
- helpTestUpdateCache(fullTableName, new int[] {1, 3});
+ helpTestUpdateCache(fullTableName, new int[] {1, 3}, false);
}
@Test
public void testUpdateCacheForNonTxnSystemTable() throws Exception {
String fullTableName = "\""+ QueryConstants.SYSTEM_SCHEMA_NAME + "\""+ QueryConstants.NAME_SEPARATOR + generateUniqueName();
setupSystemTable(fullTableName);
- helpTestUpdateCache(fullTableName, new int[] {0, 0});
+ helpTestUpdateCache(fullTableName, new int[] {0, 0}, false);
}
@Test
@@ -104,7 +105,7 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT {
conn.createStatement().execute(
"alter table " + fullTableName + " SET UPDATE_CACHE_FREQUENCY=NEVER");
}
- helpTestUpdateCache(fullTableName, new int[] {0, 0});
+ helpTestUpdateCache(fullTableName, new int[] {0, 0}, false);
}
@Test
@@ -114,7 +115,7 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT {
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.createStatement().execute("CREATE TABLE " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + " UPDATE_CACHE_FREQUENCY=always");
}
- helpTestUpdateCache(fullTableName, new int[] {1, 3});
+ helpTestUpdateCache(fullTableName, new int[] {1, 3}, false);
}
@Test
@@ -124,9 +125,9 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT {
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.createStatement().execute("CREATE TABLE " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + " UPDATE_CACHE_FREQUENCY=" + 10000);
}
- helpTestUpdateCache(fullTableName, new int[] {0, 0});
+ helpTestUpdateCache(fullTableName, new int[] {0, 0}, false);
Thread.sleep(10000);
- helpTestUpdateCache(fullTableName, new int[] {1, 0});
+ helpTestUpdateCache(fullTableName, new int[] {1, 0}, false);
}
@Test
@@ -136,14 +137,55 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT {
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.createStatement().execute("CREATE TABLE " + fullTableName + TestUtil.TEST_TABLE_SCHEMA + " UPDATE_CACHE_FREQUENCY=never");
}
- helpTestUpdateCache(fullTableName, new int[] {0, 0});
+ helpTestUpdateCache(fullTableName, new int[] {0, 0}, false);
try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
conn.createStatement().execute("ALTER TABLE " + fullTableName + " SET UPDATE_CACHE_FREQUENCY=ALWAYS");
}
- helpTestUpdateCache(fullTableName, new int[] {1, 3});
+ helpTestUpdateCache(fullTableName, new int[] {1, 3}, false);
+ }
+
+ @Test
+ public void testUpdateCacheFreqPropagatedToIndexes() throws Exception {
+ String baseTableName = generateUniqueName();
+ String fullTableName = INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + baseTableName;
+ String localIndex = "LOCAL_" + baseTableName;
+ String globalIndex = "GLOBAL_" + baseTableName;
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ conn.createStatement().execute("CREATE TABLE " + fullTableName +
+ TestUtil.TEST_TABLE_SCHEMA + " UPDATE_CACHE_FREQUENCY=never");
+
+ // Create local and global indexes on the base table
+ conn.createStatement().execute("CREATE LOCAL INDEX " + localIndex
+ + " on " + fullTableName + " (a.date1, b.varchar_col2)");
+ conn.createStatement().execute("CREATE INDEX " + globalIndex + " on "
+ + fullTableName + " (a.int_col1, a.long_col1)");
+ }
+
+ // The indexes should have got the UPDATE_CACHE_FREQUENCY value of their base table
+ helpTestUpdateCache(fullTableName, new int[] {0, 0}, false);
+ helpTestUpdateCache(INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + localIndex,
+ new int[] {0}, true);
+ helpTestUpdateCache(INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + globalIndex,
+ new int[] {0}, true);
+
+ // Now alter the UPDATE_CACHE_FREQUENCY value of the base table
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ conn.createStatement()
+ .execute("ALTER TABLE " + fullTableName + " SET UPDATE_CACHE_FREQUENCY=ALWAYS");
+ }
+ // Even the indexes should now have the modified value of UPDATE_CACHE_FREQUENCY
+ // Note that when we query the base table, during query plan generation, we make 2 getTable
+ // requests (to retrieve the base table) for each index of the base table
+ helpTestUpdateCache(fullTableName, new int[] {1, 15}, false);
+ helpTestUpdateCache(INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + localIndex,
+ new int[] {3}, true);
+ helpTestUpdateCache(INDEX_DATA_SCHEMA + QueryConstants.NAME_SEPARATOR + globalIndex,
+ new int[] {3}, true);
}
- private static void helpTestUpdateCache(String fullTableName, int[] expectedRPCs) throws Exception {
+ private static void helpTestUpdateCache(String fullTableName, int[] expectedRPCs,
+ boolean skipUpsertForIndexes) throws Exception {
String tableName = SchemaUtil.getTableNameFromFullName(fullTableName);
String schemaName = SchemaUtil.getSchemaNameFromFullName(fullTableName);
String selectSql = "SELECT * FROM "+fullTableName;
@@ -154,46 +196,31 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT {
Connection conn = connectionQueryServices.connect(getUrl(), props);
try {
conn.setAutoCommit(false);
- String upsert = "UPSERT INTO " + fullTableName + "(varchar_pk, char_pk, int_pk, long_pk, decimal_pk, date_pk) VALUES(?, ?, ?, ?, ?, ?)";
- PreparedStatement stmt = conn.prepareStatement(upsert);
- // upsert three rows
- TestUtil.setRowKeyColumns(stmt, 1);
- stmt.execute();
- TestUtil.setRowKeyColumns(stmt, 2);
- stmt.execute();
- TestUtil.setRowKeyColumns(stmt, 3);
- stmt.execute();
- conn.commit();
- int numUpsertRpcs = expectedRPCs[0];
- // verify only 0 or 1 rpc to fetch table metadata,
- verify(connectionQueryServices, times(numUpsertRpcs)).getTable((PName) isNull(),
- eq(PVarchar.INSTANCE.toBytes(schemaName)), eq(PVarchar.INSTANCE.toBytes(tableName)),
- anyLong(), anyLong(), eq(false), eq(false), (PTable)isNull());
- reset(connectionQueryServices);
-
- ResultSet rs = conn.createStatement().executeQuery(selectSql);
- TestUtil.validateRowKeyColumns(rs, 1);
- TestUtil.validateRowKeyColumns(rs, 2);
- TestUtil.validateRowKeyColumns(rs, 3);
- assertFalse(rs.next());
-
- rs = conn.createStatement().executeQuery(selectSql);
- TestUtil.validateRowKeyColumns(rs, 1);
- TestUtil.validateRowKeyColumns(rs, 2);
- TestUtil.validateRowKeyColumns(rs, 3);
- assertFalse(rs.next());
-
- rs = conn.createStatement().executeQuery(selectSql);
- TestUtil.validateRowKeyColumns(rs, 1);
- TestUtil.validateRowKeyColumns(rs, 2);
- TestUtil.validateRowKeyColumns(rs, 3);
- assertFalse(rs.next());
-
+ if (!skipUpsertForIndexes) {
+ String upsert = "UPSERT INTO " + fullTableName + "(varchar_pk, char_pk, int_pk, long_pk, decimal_pk, date_pk) VALUES(?, ?, ?, ?, ?, ?)";
+ PreparedStatement stmt = conn.prepareStatement(upsert);
+ // upsert three rows
+ for (int i=0; i<3; i++) {
+ TestUtil.setRowKeyColumns(stmt, i);
+ stmt.execute();
+ }
+ conn.commit();
+ int numUpsertRpcs = expectedRPCs[0];
+ // verify only 0 or 1 rpc to fetch table metadata,
+ verify(connectionQueryServices, times(numUpsertRpcs)).getTable((PName) isNull(),
+ eq(PVarchar.INSTANCE.toBytes(schemaName)), eq(PVarchar.INSTANCE.toBytes(tableName)),
+ anyLong(), anyLong(), eq(false), eq(false), (PTable)isNull());
+ reset(connectionQueryServices);
+ }
+ validateSelectRowKeyCols(conn, selectSql, skipUpsertForIndexes);
+ validateSelectRowKeyCols(conn, selectSql, skipUpsertForIndexes);
+ validateSelectRowKeyCols(conn, selectSql, skipUpsertForIndexes);
+
// for non-transactional tables without a scn : verify one rpc to getTable occurs *per* query
// for non-transactional tables with a scn : verify *only* one rpc occurs
// for transactional tables : verify *only* one rpc occurs
// for non-transactional, system tables : verify no rpc occurs
- int numRpcs = expectedRPCs[1];
+ int numRpcs = skipUpsertForIndexes ? expectedRPCs[0] : expectedRPCs[1];
verify(connectionQueryServices, times(numRpcs)).getTable((PName) isNull(),
eq(PVarchar.INSTANCE.toBytes(schemaName)), eq(PVarchar.INSTANCE.toBytes(tableName)),
anyLong(), anyLong(), eq(false), eq(false), (PTable)isNull());
@@ -202,4 +229,19 @@ public class UpdateCacheIT extends ParallelStatsDisabledIT {
conn.close();
}
}
+
+ private static void validateSelectRowKeyCols(Connection conn, String selectSql,
+ boolean skipUpsertForIndexes) throws SQLException {
+ ResultSet rs = conn.createStatement().executeQuery(selectSql);
+ if (skipUpsertForIndexes) {
+ for (int i=0; i<3; i++) {
+ assertTrue(rs.next());
+ }
+ } else {
+ for (int i=0; i<3; i++) {
+ TestUtil.validateRowKeyColumns(rs, i);
+ }
+ }
+ assertFalse(rs.next());
+ }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/64788794/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
index 6696521..5bffed5 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
@@ -23,6 +23,7 @@ import java.util.Map;
import org.apache.phoenix.coprocessor.MetaDataProtocol;
import org.apache.phoenix.hbase.index.util.IndexManagementUtil;
+import org.apache.phoenix.jdbc.PhoenixDatabaseMetaData;
import org.apache.phoenix.query.QueryConstants;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.schema.AmbiguousColumnException;
@@ -263,7 +264,7 @@ public enum SQLExceptionCode {
NO_LOCAL_INDEX_ON_TABLE_WITH_IMMUTABLE_ROWS(1048,"43A05","Local indexes aren't allowed on tables with immutable rows."),
COLUMN_FAMILY_NOT_ALLOWED_TABLE_PROPERTY(1049, "43A06", "Column family not allowed for table properties."),
COLUMN_FAMILY_NOT_ALLOWED_FOR_PROPERTY(1050, "43A07", "Setting or altering any of the following properties: "
- + MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES.toString()
+ + MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES.toString()
+ " for a column family is not supported since they must be kept in sync. You can only set these properties for the entire table."),
CANNOT_ALTER_PROPERTY(1051, "43A08", "Property can be specified or changed only when creating a table."),
CANNOT_SET_PROPERTY_FOR_COLUMN_NOT_ADDED(1052, "43A09", "Property cannot be specified for a column family that is not being added or modified."),
@@ -308,7 +309,9 @@ public enum SQLExceptionCode {
TTL_UNSUPPORTED_FOR_TXN_TABLE(10947, "44A28", "TTL is not supported for"),
CANNOT_CREATE_LOCAL_INDEX_FOR_TXN_TABLE(10948, "44A29", "Local indexes cannot be created for"),
CANNOT_SET_OR_ALTER_PROPERTY_FOR_INDEX(10949, "44A30", "Cannot set or alter the following properties on an index: "
- + MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES.toString()),
+ + MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES.toString()),
+ CANNOT_SET_OR_ALTER_UPDATE_CACHE_FREQ_FOR_INDEX(10950, "44A31", "Cannot set or alter "
+ + PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY + " on an index"),
/** Sequence related */
SEQUENCE_ALREADY_EXIST(1200, "42Z00", "Sequence already exists.", new Factory() {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/64788794/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
index ebcb7b9..1381247 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/query/ConnectionQueryServicesImpl.java
@@ -2519,7 +2519,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
* Keep the TTL, KEEP_DELETED_CELLS and REPLICATION_SCOPE properties of new column families
* in sync with the existing column families. Note that we use the new values for these properties in case they
* are passed from our alter table command, if not, we use the default column family's value for each property
- * See {@link MetaDataUtil#SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES}
+ * See {@link MetaDataUtil#SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES}
* @param allFamiliesProps Map of all column family properties
* @param table original table
* @param tableDesc new table descriptor
@@ -2565,7 +2565,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
/**
* Set the new values for properties that are to be kept in sync amongst those column families of the table which are
* not referenced in the context of our alter table command, including the local index column family if it exists
- * See {@link MetaDataUtil#SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES}
+ * See {@link MetaDataUtil#SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES}
* @param tableDesc original table descriptor
* @param allFamiliesProps Map of all column family properties
* @param newTTL new value of TTL
@@ -2587,7 +2587,7 @@ public class ConnectionQueryServicesImpl extends DelegateQueryServices implement
/**
* Set properties to be kept in sync for global indexes of a table, as well as
* the physical table corresponding to indexes created on views of a table
- * See {@link MetaDataUtil#SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES} and
+ * See {@link MetaDataUtil#SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES} and
* @param table base table
* @param tableAndIndexDescriptorMappings old to new table descriptor mappings
* @param applyPropsToAllIndexesDefaultCF new properties to apply to all index column families
http://git-wip-us.apache.org/repos/asf/phoenix/blob/64788794/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index 6563c40..df0e5d4 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -272,6 +272,7 @@ public class MetaDataClient {
TABLE_NAME + "," +
SYNC_INDEX_CREATED_DATE + " " + PDate.INSTANCE.getSqlTypeName() +
") VALUES (?, ?, ?, ?)";
+
private static final String CREATE_TABLE =
"UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"( " +
TENANT_ID + "," +
@@ -1147,7 +1148,7 @@ public class MetaDataClient {
/**
* Populate properties for the table and common properties for all column families of the table
* @param statementProps Properties specified in SQL statement
- * @param tableProps Properties for an HTableDescriptor
+ * @param tableProps Properties for an HTableDescriptor and Phoenix Table Properties
* @param commonFamilyProps Properties common to all column families
* @param tableType Used to distinguish between index creation vs. base table creation paths
* @throws SQLException
@@ -1164,9 +1165,17 @@ public class MetaDataClient {
.setMessage("Property: " + prop.getFirst()).build()
.buildException();
}
+ // HTableDescriptor property or Phoenix Table Property
if (defaultDescriptor.getValue(Bytes.toBytes(prop.getFirst())) == null) {
+ // See PHOENIX-4891
+ if (tableType == PTableType.INDEX && UPDATE_CACHE_FREQUENCY.equals(prop.getFirst())) {
+ throw new SQLExceptionInfo.Builder(
+ SQLExceptionCode.CANNOT_SET_OR_ALTER_UPDATE_CACHE_FREQ_FOR_INDEX)
+ .build()
+ .buildException();
+ }
tableProps.put(prop.getFirst(), prop.getSecond());
- } else {
+ } else { // HColumnDescriptor property
commonFamilyProps.put(prop.getFirst(), prop.getSecond());
}
}
@@ -2127,8 +2136,11 @@ public class MetaDataClient {
}
long updateCacheFrequency = connection.getQueryServices().getProps().getLong(
QueryServices.DEFAULT_UPDATE_CACHE_FREQUENCY_ATRRIB, QueryServicesOptions.DEFAULT_UPDATE_CACHE_FREQUENCY);
+ if (tableType == PTableType.INDEX && parent != null) {
+ updateCacheFrequency = parent.getUpdateCacheFrequency();
+ }
Long updateCacheFrequencyProp = (Long) TableProperty.UPDATE_CACHE_FREQUENCY.getValue(tableProps);
- if (updateCacheFrequencyProp != null) {
+ if (tableType != PTableType.INDEX && updateCacheFrequencyProp != null) {
updateCacheFrequency = updateCacheFrequencyProp;
}
String autoPartitionSeq = (String) TableProperty.AUTO_PARTITION_SEQ.getValue(tableProps);
@@ -3613,9 +3625,13 @@ public class MetaDataClient {
}
}
- if (!table.getIndexes().isEmpty() && (numPkColumnsAdded>0 || metaProperties.getNonTxToTx())) {
+ if (!table.getIndexes().isEmpty() &&
+ (numPkColumnsAdded>0 || metaProperties.getNonTxToTx() ||
+ metaPropertiesEvaluated.getUpdateCacheFrequency() != null)) {
for (PTable index : table.getIndexes()) {
- incrementTableSeqNum(index, index.getType(), numPkColumnsAdded, metaProperties.getNonTxToTx() ? Boolean.TRUE : null, metaPropertiesEvaluated.getUpdateCacheFrequency());
+ incrementTableSeqNum(index, index.getType(), numPkColumnsAdded,
+ metaProperties.getNonTxToTx() ? Boolean.TRUE : null,
+ metaPropertiesEvaluated.getUpdateCacheFrequency());
}
tableMetaData.addAll(connection.getMutationState().toMutations(timeStamp).next().getSecond());
connection.rollback();
@@ -4573,6 +4589,13 @@ public class MetaDataClient {
}
if (metaProperties.getUpdateCacheFrequencyProp() != null) {
+ // See PHOENIX-4891
+ if (table.getType() == PTableType.INDEX) {
+ throw new SQLExceptionInfo.Builder(
+ SQLExceptionCode.CANNOT_SET_OR_ALTER_UPDATE_CACHE_FREQ_FOR_INDEX)
+ .build()
+ .buildException();
+ }
if (metaProperties.getUpdateCacheFrequencyProp().longValue() != table.getUpdateCacheFrequency()) {
metaPropertiesEvaluated.setUpdateCacheFrequency(metaProperties.getUpdateCacheFrequencyProp());
changingPhoenixTableProperty = true;
http://git-wip-us.apache.org/repos/asf/phoenix/blob/64788794/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
index df7530c..8cf514f 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/MetaDataUtil.java
@@ -109,7 +109,7 @@ public class MetaDataUtil {
public static final byte[] DATA_TABLE_NAME_PROP_BYTES = Bytes.toBytes(DATA_TABLE_NAME_PROP_NAME);
// See PHOENIX-3955
- public static final List<String> SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES = ImmutableList.of(
+ public static final List<String> SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES = ImmutableList.of(
HColumnDescriptor.TTL,
HColumnDescriptor.KEEP_DELETED_CELLS,
HColumnDescriptor.REPLICATION_SCOPE);
@@ -725,13 +725,13 @@ public class MetaDataUtil {
}
public static boolean propertyNotAllowedToBeOutOfSync(String colFamProp) {
- return SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES.contains(colFamProp);
+ return SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES.contains(colFamProp);
}
public static Map<String, Object> getSyncedProps(HColumnDescriptor defaultCFDesc) {
Map<String, Object> syncedProps = new HashMap<>();
if (defaultCFDesc != null) {
- for (String propToKeepInSync: SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES) {
+ for (String propToKeepInSync: SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES) {
syncedProps.put(propToKeepInSync, Bytes.toString(
defaultCFDesc.getValue(Bytes.toBytes(propToKeepInSync))));
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/64788794/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
index ede14b8..8d0aca1 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/UpgradeUtil.java
@@ -46,6 +46,7 @@ import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SCHEM;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_SEQ_NUM;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TABLE_TYPE;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.TENANT_ID;
+import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.UPDATE_CACHE_FREQUENCY;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.VIEW_INDEX_ID;
import static org.apache.phoenix.query.QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT;
import static org.apache.phoenix.query.QueryConstants.DIVERGED_VIEW_BASE_COLUMN_COUNT;
@@ -140,6 +141,14 @@ public class UpgradeUtil {
+ "(TENANT_ID, TABLE_SCHEM, TABLE_NAME, COLUMN_NAME, COLUMN_FAMILY, BASE_COLUMN_COUNT) "
+ "VALUES (?, ?, ?, ?, ?, ?) ";
+ public static final String UPSERT_UPDATE_CACHE_FREQUENCY =
+ "UPSERT INTO " + SYSTEM_CATALOG_SCHEMA + ".\"" + SYSTEM_CATALOG_TABLE + "\"( " +
+ TENANT_ID + "," +
+ TABLE_SCHEM + "," +
+ TABLE_NAME + "," +
+ UPDATE_CACHE_FREQUENCY +
+ ") VALUES (?, ?, ?, ?)";
+
public static String SELECT_BASE_COLUMN_COUNT_FROM_HEADER_ROW = "SELECT "
+ "BASE_COLUMN_COUNT "
+ "FROM \"SYSTEM\".CATALOG "
@@ -1227,7 +1236,7 @@ public class UpgradeUtil {
for (HColumnDescriptor currentColFam: tableDesc.getColumnFamilies()) {
if (!currentColFam.equals(defaultColFam)) {
HColumnDescriptor newColFamDesc = new HColumnDescriptor(currentColFam);
- for (String prop: MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_PROPERTIES) {
+ for (String prop: MetaDataUtil.SYNCED_DATA_TABLE_AND_INDEX_COL_FAM_PROPERTIES) {
String existingPropVal = Bytes.toString(currentColFam.getValue(Bytes.toBytes(prop)));
String expectedPropVal = syncedProps.get(prop).toString();
if (existingPropVal == null || !existingPropVal.toLowerCase().equals(expectedPropVal.toLowerCase())) {
@@ -1299,6 +1308,71 @@ public class UpgradeUtil {
}
}
+ private static void syncUpdateCacheFreqForIndexesOfTable(PTable baseTable,
+ PreparedStatement stmt) throws SQLException {
+ for (PTable index : baseTable.getIndexes()) {
+ if (index.getUpdateCacheFrequency() == baseTable.getUpdateCacheFrequency()) {
+ continue;
+ }
+ stmt.setString(2, index.getSchemaName().getString());
+ stmt.setString(3, index.getTableName().getString());
+ stmt.setLong(4, baseTable.getUpdateCacheFrequency());
+ stmt.addBatch();
+ }
+ }
+
+ /**
+ * See PHOENIX-4891. We set the UPDATE_CACHE_FREQUENCY of indexes to be same as their parent.
+ * We do this for both physical base tables as well as views
+ * @param conn Phoenix Connection object
+ * @param table PTable corresponding to a physical base table
+ * @throws SQLException
+ * @throws IOException
+ */
+ public static void syncUpdateCacheFreqAllIndexes(PhoenixConnection conn, PTable table)
+ throws SQLException, IOException {
+ // Use own connection with max time stamp to be able to read all data from SYSTEM.CATALOG
+ try(PhoenixConnection newConn = new PhoenixConnection(conn, HConstants.LATEST_TIMESTAMP)) {
+ // Clear the server-side cache so that we get the latest built PTables
+ newConn.unwrap(PhoenixConnection.class).getQueryServices().clearCache();
+ byte[] tenantId = newConn.getTenantId() != null ?
+ newConn.getTenantId().getBytes() : null;
+
+ PreparedStatement stmt =
+ newConn.prepareStatement(UPSERT_UPDATE_CACHE_FREQUENCY);
+ stmt.setString(1, Bytes.toString(tenantId));
+ syncUpdateCacheFreqForIndexesOfTable(table, stmt);
+
+ TableViewFinderResult childViewsResult = new TableViewFinderResult();
+ try (Table childLinkTable = newConn.getQueryServices()
+ .getTable(SchemaUtil.getPhysicalName(
+ PhoenixDatabaseMetaData.SYSTEM_CHILD_LINK_NAME_BYTES,
+ newConn.getQueryServices().getProps())
+ .getName())) {
+ ViewFinder.findAllRelatives(childLinkTable, tenantId,
+ table.getSchemaName().getBytes(), table.getTableName().getBytes(),
+ LinkType.CHILD_TABLE, childViewsResult);
+
+ // Iterate over the chain of child views
+ for (TableInfo tableInfo: childViewsResult.getLinks()) {
+ PTable view;
+ String viewName = SchemaUtil.getTableName(tableInfo.getSchemaName(),
+ tableInfo.getTableName());
+ try {
+ view = PhoenixRuntime.getTable(newConn, viewName);
+ } catch (TableNotFoundException e) {
+ // Ignore
+ logger.warn("Error getting PTable for view: " + viewName);
+ continue;
+ }
+ syncUpdateCacheFreqForIndexesOfTable(view, stmt);
+ }
+ }
+ stmt.executeBatch();
+ newConn.commit();
+ }
+ }
+
/**
* Make sure that all tables have necessary column family properties in sync
* with each other and also in sync with all the table's indexes
@@ -1316,7 +1390,7 @@ public class UpgradeUtil {
// Ignore physical view index tables since we handle them for each base table already
continue;
}
- PTable table = null;
+ PTable table;
String tableName = origTableDesc.getTableName().getNameAsString();
try {
table = PhoenixRuntime.getTable(conn, tableName);
@@ -1329,6 +1403,7 @@ public class UpgradeUtil {
// Ignore global index tables since we handle them for each base table already
continue;
}
+ syncUpdateCacheFreqAllIndexes(conn, table);
HColumnDescriptor defaultColFam = origTableDesc.getFamily(SchemaUtil.getEmptyColumnFamily(table));
Map<String, Object> syncedProps = MetaDataUtil.getSyncedProps(defaultColFam);
[16/22] phoenix git commit: PHOENIX-4996: Refactor PTableImpl to use
Builder Pattern
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6083ae5/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
index 9f06e04..7939b97 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/PTableImpl.java
@@ -36,6 +36,7 @@ import java.util.Map.Entry;
import javax.annotation.Nonnull;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.client.Delete;
@@ -69,7 +70,6 @@ import org.apache.phoenix.schema.types.PChar;
import org.apache.phoenix.schema.types.PDataType;
import org.apache.phoenix.schema.types.PDouble;
import org.apache.phoenix.schema.types.PFloat;
-import org.apache.phoenix.schema.types.PLong;
import org.apache.phoenix.schema.types.PVarchar;
import org.apache.phoenix.transaction.TransactionFactory;
import org.apache.phoenix.util.ByteUtil;
@@ -102,164 +102,661 @@ import com.google.common.collect.Maps;
public class PTableImpl implements PTable {
private static final Integer NO_SALTING = -1;
- private PTableKey key;
- private PName name;
- private PName schemaName = PName.EMPTY_NAME;
- private PName tableName = PName.EMPTY_NAME;
- private PName tenantId;
- private PTableType type;
- private PIndexState state;
- private long sequenceNumber;
- private long timeStamp;
- private long indexDisableTimestamp;
+ private IndexMaintainer indexMaintainer;
+ private ImmutableBytesWritable indexMaintainersPtr;
+
+ private final PTableKey key;
+ private final PName name;
+ private final PName schemaName;
+ private final PName tableName;
+ private final PName tenantId;
+ private final PTableType type;
+ private final PIndexState state;
+ private final long sequenceNumber;
+ private final long timeStamp;
+ private final long indexDisableTimestamp;
// Have MultiMap for String->PColumn (may need family qualifier)
- private List<PColumn> pkColumns;
- private List<PColumn> allColumns;
+ private final List<PColumn> pkColumns;
+ private final List<PColumn> allColumns;
// columns that were inherited from a parent table but that were dropped in the view
- private List<PColumn> excludedColumns;
- private List<PColumnFamily> families;
- private Map<byte[], PColumnFamily> familyByBytes;
- private Map<String, PColumnFamily> familyByString;
- private ListMultimap<String, PColumn> columnsByName;
- private Map<KVColumnFamilyQualifier, PColumn> kvColumnsByQualifiers;
- private PName pkName;
- private Integer bucketNum;
- private RowKeySchema rowKeySchema;
+ private final List<PColumn> excludedColumns;
+ private final List<PColumnFamily> families;
+ private final Map<byte[], PColumnFamily> familyByBytes;
+ private final Map<String, PColumnFamily> familyByString;
+ private final ListMultimap<String, PColumn> columnsByName;
+ private final Map<KVColumnFamilyQualifier, PColumn> kvColumnsByQualifiers;
+ private final PName pkName;
+ private final Integer bucketNum;
+ private final RowKeySchema rowKeySchema;
// Indexes associated with this table.
- private List<PTable> indexes;
+ private final List<PTable> indexes;
// Data table name that the index is created on.
- private PName parentName;
- private PName parentSchemaName;
- private PName parentTableName;
- private List<PName> physicalNames;
- private boolean isImmutableRows;
- private IndexMaintainer indexMaintainer;
- private ImmutableBytesWritable indexMaintainersPtr;
- private PName defaultFamilyName;
- private String viewStatement;
- private boolean disableWAL;
- private boolean multiTenant;
- private boolean storeNulls;
- private TransactionFactory.Provider transactionProvider;
- private ViewType viewType;
- private PDataType viewIndexType;
- private Long viewIndexId;
- private int estimatedSize;
- private IndexType indexType;
- private int baseColumnCount;
- private boolean rowKeyOrderOptimizable; // TODO: remove when required that tables have been upgrade for PHOENIX-2067
- private boolean hasColumnsRequiringUpgrade; // TODO: remove when required that tables have been upgrade for PHOENIX-2067
- private int rowTimestampColPos;
- private long updateCacheFrequency;
- private boolean isNamespaceMapped;
- private String autoPartitionSeqName;
- private boolean isAppendOnlySchema;
- private ImmutableStorageScheme immutableStorageScheme;
- private QualifierEncodingScheme qualifierEncodingScheme;
- private EncodedCQCounter encodedCQCounter;
- private Boolean useStatsForParallelization;
-
- public PTableImpl() {
- this.indexes = Collections.emptyList();
- this.physicalNames = Collections.emptyList();
- this.rowKeySchema = RowKeySchema.EMPTY_SCHEMA;
- }
-
- // Constructor used at table creation time
- public PTableImpl(PName tenantId, String schemaName, String tableName, long timestamp, List<PColumnFamily> families, boolean isNamespaceMapped) {
- Preconditions.checkArgument(tenantId==null || tenantId.getBytes().length > 0); // tenantId should be null or not empty
- this.tenantId = tenantId;
- this.name = PNameFactory.newName(SchemaUtil.getTableName(schemaName, tableName));
- this.key = new PTableKey(tenantId, name.getString());
- this.schemaName = PNameFactory.newName(schemaName);
- this.tableName = PNameFactory.newName(tableName);
- this.type = PTableType.VIEW;
- this.viewType = ViewType.MAPPED;
- this.timeStamp = timestamp;
- this.pkColumns = this.allColumns = Collections.emptyList();
- this.rowKeySchema = RowKeySchema.EMPTY_SCHEMA;
- this.indexes = Collections.emptyList();
- this.familyByBytes = Maps.newHashMapWithExpectedSize(families.size());
- this.familyByString = Maps.newHashMapWithExpectedSize(families.size());
- for (PColumnFamily family : families) {
- familyByBytes.put(family.getName().getBytes(), family);
- familyByString.put(family.getName().getString(), family);
- }
- this.families = families;
- this.physicalNames = Collections.emptyList();
- this.isNamespaceMapped = isNamespaceMapped;
- }
-
- public PTableImpl(PName tenantId, String schemaName, String tableName, long timestamp, List<PColumnFamily> families, boolean isNamespaceMapped, ImmutableStorageScheme storageScheme, QualifierEncodingScheme encodingScheme, Boolean useStatsForParallelization) { // For base table of mapped VIEW
- Preconditions.checkArgument(tenantId==null || tenantId.getBytes().length > 0); // tenantId should be null or not empty
- this.tenantId = tenantId;
- this.name = PNameFactory.newName(SchemaUtil.getTableName(schemaName, tableName));
- this.key = new PTableKey(tenantId, name.getString());
- this.schemaName = PNameFactory.newName(schemaName);
- this.tableName = PNameFactory.newName(tableName);
- this.type = PTableType.VIEW;
- this.viewType = ViewType.MAPPED;
- this.timeStamp = timestamp;
- this.pkColumns = this.allColumns = Collections.emptyList();
- this.rowKeySchema = RowKeySchema.EMPTY_SCHEMA;
- this.indexes = Collections.emptyList();
- this.familyByBytes = Maps.newHashMapWithExpectedSize(families.size());
- this.familyByString = Maps.newHashMapWithExpectedSize(families.size());
- for (PColumnFamily family : families) {
- familyByBytes.put(family.getName().getBytes(), family);
- familyByString.put(family.getName().getString(), family);
- }
- this.families = families;
- this.physicalNames = Collections.emptyList();
- this.isNamespaceMapped = isNamespaceMapped;
- this.immutableStorageScheme = storageScheme;
- this.qualifierEncodingScheme = encodingScheme;
- this.useStatsForParallelization = useStatsForParallelization;
- }
-
- // For indexes stored in shared physical tables
- public PTableImpl(PName tenantId, PName schemaName, PName tableName, long timestamp, List<PColumnFamily> families,
- List<PColumn> columns, List<PName> physicalNames, PDataType viewIndexType, Long viewIndexId, boolean multiTenant, boolean isNamespaceMpped, ImmutableStorageScheme storageScheme, QualifierEncodingScheme qualifierEncodingScheme,
- EncodedCQCounter encodedCQCounter, Boolean useStatsForParallelization, Integer bucketNum) throws SQLException {
- this.pkColumns = this.allColumns = Collections.emptyList();
- this.rowKeySchema = RowKeySchema.EMPTY_SCHEMA;
- this.indexes = Collections.emptyList();
- this.familyByBytes = Maps.newHashMapWithExpectedSize(families.size());
- this.familyByString = Maps.newHashMapWithExpectedSize(families.size());
- for (PColumnFamily family : families) {
- familyByBytes.put(family.getName().getBytes(), family);
- familyByString.put(family.getName().getString(), family);
- }
- this.families = families;
- if (bucketNum!=null) {
- columns = columns.subList(1, columns.size());
- }
- init(tenantId, this.schemaName, this.tableName, PTableType.INDEX, state, timeStamp, sequenceNumber, pkName, bucketNum, columns,
- this.schemaName, parentTableName, indexes, isImmutableRows, physicalNames, defaultFamilyName,
- null, disableWAL, multiTenant, storeNulls, viewType, viewIndexType, viewIndexId, indexType, baseColumnCount, rowKeyOrderOptimizable,
- transactionProvider, updateCacheFrequency, indexDisableTimestamp, isNamespaceMpped, null,
- false, storageScheme, qualifierEncodingScheme, encodedCQCounter, useStatsForParallelization, null);
- }
-
- public PTableImpl(long timeStamp) { // For delete marker
- this(timeStamp, false);
- }
-
- public PTableImpl(long timeStamp, boolean isIndex) { // For index delete marker
- if (isIndex) {
- this.type = PTableType.INDEX;
- this.state = PIndexState.INACTIVE;
- } else {
- this.type = PTableType.TABLE;
+ private final PName parentName;
+ private final PName parentSchemaName;
+ private final PName parentTableName;
+ private final List<PName> physicalNames;
+ private final boolean isImmutableRows;
+ private final PName defaultFamilyName;
+ private final String viewStatement;
+ private final boolean disableWAL;
+ private final boolean multiTenant;
+ private final boolean storeNulls;
+ private final TransactionFactory.Provider transactionProvider;
+ private final ViewType viewType;
+ private final PDataType viewIndexType;
+ private final Long viewIndexId;
+ private final int estimatedSize;
+ private final IndexType indexType;
+ private final int baseColumnCount;
+ private final boolean rowKeyOrderOptimizable; // TODO: remove when required that tables have been upgrade for PHOENIX-2067
+ private final boolean hasColumnsRequiringUpgrade; // TODO: remove when required that tables have been upgrade for PHOENIX-2067
+ private final int rowTimestampColPos;
+ private final long updateCacheFrequency;
+ private final boolean isNamespaceMapped;
+ private final String autoPartitionSeqName;
+ private final boolean isAppendOnlySchema;
+ private final ImmutableStorageScheme immutableStorageScheme;
+ private final QualifierEncodingScheme qualifierEncodingScheme;
+ private final EncodedCQCounter encodedCQCounter;
+ private final Boolean useStatsForParallelization;
+
+ public static class Builder {
+ private PTableKey key;
+ private PName name;
+ private PName schemaName = PName.EMPTY_NAME;
+ private PName tableName = PName.EMPTY_NAME;
+ private PName tenantId;
+ private PTableType type;
+ private PIndexState state;
+ private long sequenceNumber;
+ private long timeStamp;
+ private long indexDisableTimestamp;
+ private List<PColumn> pkColumns;
+ private List<PColumn> allColumns;
+ private List<PColumn> excludedColumns;
+ private List<PColumnFamily> families;
+ private Map<byte[], PColumnFamily> familyByBytes;
+ private Map<String, PColumnFamily> familyByString;
+ private ListMultimap<String, PColumn> columnsByName;
+ private Map<KVColumnFamilyQualifier, PColumn> kvColumnsByQualifiers;
+ private PName pkName;
+ private Integer bucketNum;
+ private RowKeySchema rowKeySchema;
+ private List<PTable> indexes;
+ private PName parentName;
+ private PName parentSchemaName;
+ private PName parentTableName;
+ private List<PName> physicalNames;
+ private boolean isImmutableRows;
+ private IndexMaintainer indexMaintainer;
+ private ImmutableBytesWritable indexMaintainersPtr;
+ private PName defaultFamilyName;
+ private String viewStatement;
+ private boolean disableWAL;
+ private boolean multiTenant;
+ private boolean storeNulls;
+ private TransactionFactory.Provider transactionProvider;
+ private ViewType viewType;
+ private PDataType viewIndexType;
+ private Long viewIndexId;
+ private int estimatedSize;
+ private IndexType indexType;
+ private int baseColumnCount;
+ private boolean rowKeyOrderOptimizable;
+ private boolean hasColumnsRequiringUpgrade;
+ private int rowTimestampColPos;
+ private long updateCacheFrequency;
+ private boolean isNamespaceMapped;
+ private String autoPartitionSeqName;
+ private boolean isAppendOnlySchema;
+ private ImmutableStorageScheme immutableStorageScheme;
+ private QualifierEncodingScheme qualifierEncodingScheme;
+ private EncodedCQCounter encodedCQCounter;
+ private Boolean useStatsForParallelization;
+ // Optionally set columns for the builder, but not for the actual PTable
+ private Collection<PColumn> columns;
+
+ public Builder setKey(PTableKey key) {
+ this.key = key;
+ return this;
+ }
+
+ public Builder setName(PName name) {
+ this.name = name;
+ return this;
+ }
+
+ public Builder setSchemaName(PName schemaName) {
+ this.schemaName = schemaName;
+ return this;
+ }
+
+ public Builder setTableName(PName tableName) {
+ this.tableName = tableName;
+ return this;
+ }
+
+ public Builder setTenantId(PName tenantId) {
+ this.tenantId = tenantId;
+ return this;
+ }
+
+ public Builder setType(PTableType type) {
+ this.type = type;
+ return this;
+ }
+
+ public Builder setState(PIndexState state) {
+ this.state = state;
+ return this;
+ }
+
+ public Builder setSequenceNumber(long sequenceNumber) {
+ this.sequenceNumber = sequenceNumber;
+ return this;
+ }
+
+ public Builder setTimeStamp(long timeStamp) {
+ this.timeStamp = timeStamp;
+ return this;
+ }
+
+ public Builder setIndexDisableTimestamp(long indexDisableTimestamp) {
+ this.indexDisableTimestamp = indexDisableTimestamp;
+ return this;
+ }
+
+ public Builder setPkColumns(List<PColumn> pkColumns) {
+ this.pkColumns = pkColumns;
+ return this;
+ }
+
+ public Builder setAllColumns(List<PColumn> allColumns) {
+ this.allColumns = allColumns;
+ return this;
+ }
+
+ public Builder setExcludedColumns(List<PColumn> excludedColumns) {
+ this.excludedColumns = excludedColumns;
+ return this;
+ }
+
+ public Builder setFamilyAttributes(List<PColumnFamily> families) {
+ this.familyByBytes = Maps.newHashMapWithExpectedSize(families.size());
+ this.familyByString = Maps.newHashMapWithExpectedSize(families.size());
+ for (PColumnFamily family : families) {
+ familyByBytes.put(family.getName().getBytes(), family);
+ familyByString.put(family.getName().getString(), family);
+ }
+ this.families = families;
+ return this;
+ }
+
+ public Builder setFamilies(List<PColumnFamily> families) {
+ this.families = families;
+ return this;
+ }
+
+ public Builder setFamilyByBytes(Map<byte[], PColumnFamily> familyByBytes) {
+ this.familyByBytes = familyByBytes;
+ return this;
+ }
+
+ public Builder setFamilyByString(Map<String, PColumnFamily> familyByString) {
+ this.familyByString = familyByString;
+ return this;
+ }
+
+ public Builder setColumnsByName(ListMultimap<String, PColumn> columnsByName) {
+ this.columnsByName = columnsByName;
+ return this;
+ }
+
+ public Builder setKvColumnsByQualifiers(Map<KVColumnFamilyQualifier, PColumn> kvColumnsByQualifiers) {
+ this.kvColumnsByQualifiers = kvColumnsByQualifiers;
+ return this;
+ }
+
+ public Builder setPkName(PName pkName) {
+ this.pkName = pkName;
+ return this;
+ }
+
+ public Builder setBucketNum(Integer bucketNum) {
+ this.bucketNum = bucketNum;
+ return this;
+ }
+
+ public Builder setRowKeySchema(RowKeySchema rowKeySchema) {
+ this.rowKeySchema = rowKeySchema;
+ return this;
+ }
+
+ public Builder setIndexes(List<PTable> indexes) {
+ this.indexes = indexes;
+ return this;
+ }
+
+ public Builder setParentName(PName parentName) {
+ this.parentName = parentName;
+ return this;
+ }
+
+ public Builder setParentSchemaName(PName parentSchemaName) {
+ this.parentSchemaName = parentSchemaName;
+ return this;
+ }
+
+ public Builder setParentTableName(PName parentTableName) {
+ this.parentTableName = parentTableName;
+ return this;
+ }
+
+ public Builder setPhysicalNames(List<PName> physicalNames) {
+ this.physicalNames = physicalNames;
+ return this;
+ }
+
+ public Builder setImmutableRows(boolean immutableRows) {
+ isImmutableRows = immutableRows;
+ return this;
+ }
+
+ public Builder setIndexMaintainer(IndexMaintainer indexMaintainer) {
+ this.indexMaintainer = indexMaintainer;
+ return this;
+ }
+
+ public Builder setIndexMaintainersPtr(ImmutableBytesWritable indexMaintainersPtr) {
+ this.indexMaintainersPtr = indexMaintainersPtr;
+ return this;
+ }
+
+ public Builder setDefaultFamilyName(PName defaultFamilyName) {
+ this.defaultFamilyName = defaultFamilyName;
+ return this;
+ }
+
+ public Builder setViewStatement(String viewStatement) {
+ this.viewStatement = viewStatement;
+ return this;
+ }
+
+ public Builder setDisableWAL(boolean disableWAL) {
+ this.disableWAL = disableWAL;
+ return this;
+ }
+
+ public Builder setMultiTenant(boolean multiTenant) {
+ this.multiTenant = multiTenant;
+ return this;
+ }
+
+ public Builder setStoreNulls(boolean storeNulls) {
+ this.storeNulls = storeNulls;
+ return this;
+ }
+
+ public Builder setTransactionProvider(TransactionFactory.Provider transactionProvider) {
+ this.transactionProvider = transactionProvider;
+ return this;
+ }
+
+ public Builder setViewType(ViewType viewType) {
+ this.viewType = viewType;
+ return this;
+ }
+
+ public Builder setViewIndexType(PDataType viewIndexType) {
+ this.viewIndexType = viewIndexType;
+ return this;
+ }
+
+ public Builder setViewIndexId(Long viewIndexId) {
+ this.viewIndexId = viewIndexId;
+ return this;
+ }
+
+ public Builder setEstimatedSize(int estimatedSize) {
+ this.estimatedSize = estimatedSize;
+ return this;
+ }
+
+ public Builder setIndexType(IndexType indexType) {
+ this.indexType = indexType;
+ return this;
+ }
+
+ public Builder setBaseColumnCount(int baseColumnCount) {
+ this.baseColumnCount = baseColumnCount;
+ return this;
+ }
+
+ public Builder setRowKeyOrderOptimizable(boolean rowKeyOrderOptimizable) {
+ this.rowKeyOrderOptimizable = rowKeyOrderOptimizable;
+ return this;
+ }
+
+ public Builder setHasColumnsRequiringUpgrade(boolean hasColumnsRequiringUpgrade) {
+ this.hasColumnsRequiringUpgrade = hasColumnsRequiringUpgrade;
+ return this;
+ }
+
+ public Builder setRowTimestampColPos(int rowTimestampColPos) {
+ this.rowTimestampColPos = rowTimestampColPos;
+ return this;
+ }
+
+ public Builder setUpdateCacheFrequency(long updateCacheFrequency) {
+ this.updateCacheFrequency = updateCacheFrequency;
+ return this;
+ }
+
+ public Builder setNamespaceMapped(boolean namespaceMapped) {
+ isNamespaceMapped = namespaceMapped;
+ return this;
+ }
+
+ public Builder setAutoPartitionSeqName(String autoPartitionSeqName) {
+ this.autoPartitionSeqName = autoPartitionSeqName;
+ return this;
+ }
+
+ public Builder setAppendOnlySchema(boolean appendOnlySchema) {
+ isAppendOnlySchema = appendOnlySchema;
+ return this;
+ }
+
+ public Builder setImmutableStorageScheme(ImmutableStorageScheme immutableStorageScheme) {
+ this.immutableStorageScheme = immutableStorageScheme;
+ return this;
+ }
+
+ public Builder setQualifierEncodingScheme(QualifierEncodingScheme qualifierEncodingScheme) {
+ this.qualifierEncodingScheme = qualifierEncodingScheme;
+ return this;
+ }
+
+ public Builder setEncodedCQCounter(EncodedCQCounter encodedCQCounter) {
+ this.encodedCQCounter = encodedCQCounter;
+ return this;
+ }
+
+ public Builder setUseStatsForParallelization(Boolean useStatsForParallelization) {
+ this.useStatsForParallelization = useStatsForParallelization;
+ return this;
+ }
+
+ /**
+ * Note: When set in the builder, we must call {@link Builder#initDerivedAttributes()}
+ * before building the PTable in order to correctly populate other attributes of the PTable
+ * @param columns PColumns to be set in the builder
+ * @return PTableImpl.Builder object
+ */
+ public Builder setColumns(Collection<PColumn> columns) {
+ this.columns = columns;
+ return this;
+ }
+
+ /**
+ * Populate derivable attributes of the PTable
+ * @return PTableImpl.Builder object
+ * @throws SQLException
+ */
+ private Builder initDerivedAttributes() throws SQLException {
+ checkTenantId(this.tenantId);
+ Preconditions.checkNotNull(this.schemaName);
+ Preconditions.checkNotNull(this.tableName);
+ Preconditions.checkNotNull(this.columns);
+ Preconditions.checkNotNull(this.indexes);
+ Preconditions.checkNotNull(this.physicalNames);
+ Preconditions.checkNotNull(this.hasColumnsRequiringUpgrade);
+ Preconditions.checkNotNull(this.rowKeyOrderOptimizable);
+
+ PName fullName = PNameFactory.newName(SchemaUtil.getTableName(
+ this.schemaName.getString(), this.tableName.getString()));
+ int estimatedSize = SizedUtil.OBJECT_SIZE * 2 + 23 * SizedUtil.POINTER_SIZE +
+ 4 * SizedUtil.INT_SIZE + 2 * SizedUtil.LONG_SIZE + 2 * SizedUtil.INT_OBJECT_SIZE +
+ PNameFactory.getEstimatedSize(this.tenantId) +
+ PNameFactory.getEstimatedSize(this.schemaName) +
+ PNameFactory.getEstimatedSize(this.tableName) +
+ PNameFactory.getEstimatedSize(this.pkName) +
+ PNameFactory.getEstimatedSize(this.parentTableName) +
+ PNameFactory.getEstimatedSize(this.defaultFamilyName);
+ int numPKColumns = 0;
+ List<PColumn> pkColumns;
+ PColumn[] allColumns;
+ if (this.bucketNum != null) {
+ // Add salt column to allColumns and pkColumns, but don't add to
+ // columnsByName, since it should not be addressable via name.
+ allColumns = new PColumn[this.columns.size()+1];
+ allColumns[SALTING_COLUMN.getPosition()] = SALTING_COLUMN;
+ pkColumns = Lists.newArrayListWithExpectedSize(this.columns.size()+1);
+ ++numPKColumns;
+ } else {
+ allColumns = new PColumn[this.columns.size()];
+ pkColumns = Lists.newArrayListWithExpectedSize(this.columns.size());
+ }
+ // Must do this as with the new method of storing diffs, we just care about
+ // ordinal position relative order and not the true ordinal value itself.
+ List<PColumn> sortedColumns = Lists.newArrayList(this.columns);
+ Collections.sort(sortedColumns, new Comparator<PColumn>() {
+ @Override
+ public int compare(PColumn o1, PColumn o2) {
+ return Integer.valueOf(o1.getPosition()).compareTo(o2.getPosition());
+ }
+ });
+
+ int position = 0;
+ if (this.bucketNum != null) {
+ position = 1;
+ }
+ ListMultimap<String, PColumn> populateColumnsByName =
+ ArrayListMultimap.create(this.columns.size(), 1);
+ Map<KVColumnFamilyQualifier, PColumn> populateKvColumnsByQualifiers =
+ Maps.newHashMapWithExpectedSize(this.columns.size());
+ for (PColumn column : sortedColumns) {
+ allColumns[position] = column;
+ position++;
+ PName familyName = column.getFamilyName();
+ if (familyName == null) {
+ ++numPKColumns;
+ }
+ String columnName = column.getName().getString();
+ if (populateColumnsByName.put(columnName, column)) {
+ int count = 0;
+ for (PColumn dupColumn : populateColumnsByName.get(columnName)) {
+ if (Objects.equal(familyName, dupColumn.getFamilyName())) {
+ count++;
+ if (count > 1) {
+ throw new ColumnAlreadyExistsException(this.schemaName.getString(),
+ fullName.getString(), columnName);
+ }
+ }
+ }
+ }
+ byte[] cq = column.getColumnQualifierBytes();
+ String cf = column.getFamilyName() != null ?
+ column.getFamilyName().getString() : null;
+ if (cf != null && cq != null) {
+ KVColumnFamilyQualifier info = new KVColumnFamilyQualifier(cf, cq);
+ if (populateKvColumnsByQualifiers.get(info) != null) {
+ throw new ColumnAlreadyExistsException(this.schemaName.getString(),
+ fullName.getString(), columnName);
+ }
+ populateKvColumnsByQualifiers.put(info, column);
+ }
+ }
+ estimatedSize += SizedUtil.sizeOfMap(allColumns.length, SizedUtil.POINTER_SIZE,
+ SizedUtil.sizeOfArrayList(1)); // for multi-map
+ estimatedSize += SizedUtil.sizeOfMap(numPKColumns) +
+ SizedUtil.sizeOfMap(allColumns.length);
+
+ RowKeySchemaBuilder builder = new RowKeySchemaBuilder(numPKColumns);
+ // Two pass so that column order in column families matches overall column order
+ // and to ensure that column family order is constant
+ int maxExpectedSize = allColumns.length - numPKColumns;
+ // Maintain iteration order so that column families are ordered as they are listed
+ Map<PName, List<PColumn>> familyMap = Maps.newLinkedHashMap();
+ PColumn rowTimestampCol = null;
+ boolean hasColsRequiringUpgrade = false;
+ for (PColumn column : allColumns) {
+ PName familyName = column.getFamilyName();
+ if (familyName == null) {
+ hasColsRequiringUpgrade |=
+ (column.getSortOrder() == SortOrder.DESC
+ && (!column.getDataType().isFixedWidth()
+ || column.getDataType() == PChar.INSTANCE
+ || column.getDataType() == PFloat.INSTANCE
+ || column.getDataType() == PDouble.INSTANCE
+ || column.getDataType() == PBinary.INSTANCE) )
+ || (column.getSortOrder() == SortOrder.ASC
+ && column.getDataType() == PBinary.INSTANCE
+ && column.getMaxLength() != null
+ && column.getMaxLength() > 1);
+ pkColumns.add(column);
+ if (column.isRowTimestamp()) {
+ rowTimestampCol = column;
+ }
+ }
+ if (familyName == null) {
+ estimatedSize += column.getEstimatedSize(); // PK columns
+ builder.addField(column, column.isNullable(), column.getSortOrder());
+ } else {
+ List<PColumn> columnsInFamily = familyMap.get(familyName);
+ if (columnsInFamily == null) {
+ columnsInFamily = Lists.newArrayListWithExpectedSize(maxExpectedSize);
+ familyMap.put(familyName, columnsInFamily);
+ }
+ columnsInFamily.add(column);
+ }
+ }
+ int rowTimestampColPos;
+ if (rowTimestampCol != null) {
+ rowTimestampColPos = pkColumns.indexOf(rowTimestampCol);
+ } else {
+ rowTimestampColPos = -1;
+ }
+
+ Iterator<Map.Entry<PName,List<PColumn>>> iterator = familyMap.entrySet().iterator();
+ PColumnFamily[] families = new PColumnFamily[familyMap.size()];
+ ImmutableMap.Builder<String, PColumnFamily> familyByString = ImmutableMap.builder();
+ ImmutableSortedMap.Builder<byte[], PColumnFamily> familyByBytes = ImmutableSortedMap
+ .orderedBy(Bytes.BYTES_COMPARATOR);
+ for (int i = 0; i < families.length; i++) {
+ Map.Entry<PName,List<PColumn>> entry = iterator.next();
+ PColumnFamily family = new PColumnFamilyImpl(entry.getKey(), entry.getValue());
+ families[i] = family;
+ familyByString.put(family.getName().getString(), family);
+ familyByBytes.put(family.getName().getBytes(), family);
+ estimatedSize += family.getEstimatedSize();
+ }
+ estimatedSize += SizedUtil.sizeOfArrayList(families.length);
+ estimatedSize += SizedUtil.sizeOfMap(families.length) * 2;
+ for (PTable index : this.indexes) {
+ estimatedSize += index.getEstimatedSize();
+ }
+
+ estimatedSize += PNameFactory.getEstimatedSize(this.parentName);
+ for (PName physicalName : this.physicalNames) {
+ estimatedSize += physicalName.getEstimatedSize();
+ }
+ // Populate the derived fields and return the builder
+ return this.setName(fullName)
+ .setKey(new PTableKey(this.tenantId, fullName.getString()))
+ .setParentName(this.parentTableName == null ? null :
+ PNameFactory.newName(SchemaUtil.getTableName(
+ this.parentSchemaName != null ?
+ this.parentSchemaName.getString() : null,
+ this.parentTableName.getString())))
+ .setColumnsByName(populateColumnsByName)
+ .setKvColumnsByQualifiers(populateKvColumnsByQualifiers)
+ .setAllColumns(ImmutableList.copyOf(allColumns))
+ .setHasColumnsRequiringUpgrade(hasColsRequiringUpgrade
+ | this.hasColumnsRequiringUpgrade)
+ .setPkColumns(ImmutableList.copyOf(pkColumns))
+ .setRowTimestampColPos(rowTimestampColPos)
+ // after hasDescVarLengthColumns is calculated
+ .setRowKeySchema(builder.rowKeyOrderOptimizable(
+ this.rowKeyOrderOptimizable || !this.hasColumnsRequiringUpgrade)
+ .build())
+ .setFamilies(ImmutableList.copyOf(families))
+ .setFamilyByBytes(familyByBytes.build())
+ .setFamilyByString(familyByString.build())
+ .setEstimatedSize(estimatedSize + this.rowKeySchema.getEstimatedSize());
}
- this.timeStamp = timeStamp;
- this.pkColumns = this.allColumns = Collections.emptyList();
- this.families = Collections.emptyList();
- this.familyByBytes = Collections.emptyMap();
- this.familyByString = Collections.emptyMap();
- this.rowKeySchema = RowKeySchema.EMPTY_SCHEMA;
- this.indexes = Collections.emptyList();
- this.physicalNames = Collections.emptyList();;
+
+ public PTableImpl build() throws SQLException {
+ // Note that we call initDerivedAttributes to populate derivable attributes if
+ // this.columns is set in the PTableImpl.Builder object
+ return (this.columns == null) ? new PTableImpl(this) :
+ new PTableImpl(this.initDerivedAttributes());
+ }
+
+ }
+
+ @VisibleForTesting
+ PTableImpl() {
+ this(new PTableImpl.Builder()
+ .setIndexes(Collections.<PTable>emptyList())
+ .setPhysicalNames(Collections.<PName>emptyList())
+ .setRowKeySchema(RowKeySchema.EMPTY_SCHEMA));
+ }
+
+ // Private constructor used by the builder
+ private PTableImpl(Builder builder) {
+ this.key = builder.key;
+ this.name = builder.name;
+ this.schemaName = builder.schemaName;
+ this.tableName = builder.tableName;
+ this.tenantId = builder.tenantId;
+ this.type = builder.type;
+ this.state = builder.state;
+ this.sequenceNumber = builder.sequenceNumber;
+ this.timeStamp = builder.timeStamp;
+ this.indexDisableTimestamp = builder.indexDisableTimestamp;
+ this.pkColumns = builder.pkColumns;
+ this.allColumns = builder.allColumns;
+ this.excludedColumns = builder.excludedColumns;
+ this.families = builder.families;
+ this.familyByBytes = builder.familyByBytes;
+ this.familyByString = builder.familyByString;
+ this.columnsByName = builder.columnsByName;
+ this.kvColumnsByQualifiers = builder.kvColumnsByQualifiers;
+ this.pkName = builder.pkName;
+ this.bucketNum = builder.bucketNum;
+ this.rowKeySchema = builder.rowKeySchema;
+ this.indexes = builder.indexes;
+ this.parentName = builder.parentName;
+ this.parentSchemaName = builder.parentSchemaName;
+ this.parentTableName = builder.parentTableName;
+ this.physicalNames = builder.physicalNames;
+ this.isImmutableRows = builder.isImmutableRows;
+ this.indexMaintainer = builder.indexMaintainer;
+ this.indexMaintainersPtr = builder.indexMaintainersPtr;
+ this.defaultFamilyName = builder.defaultFamilyName;
+ this.viewStatement = builder.viewStatement;
+ this.disableWAL = builder.disableWAL;
+ this.multiTenant = builder.multiTenant;
+ this.storeNulls = builder.storeNulls;
+ this.transactionProvider = builder.transactionProvider;
+ this.viewType = builder.viewType;
+ this.viewIndexType = builder.viewIndexType;
+ this.viewIndexId = builder.viewIndexId;
+ this.estimatedSize = builder.estimatedSize;
+ this.indexType = builder.indexType;
+ this.baseColumnCount = builder.baseColumnCount;
+ this.rowKeyOrderOptimizable = builder.rowKeyOrderOptimizable;
+ this.hasColumnsRequiringUpgrade = builder.hasColumnsRequiringUpgrade;
+ this.rowTimestampColPos = builder.rowTimestampColPos;
+ this.updateCacheFrequency = builder.updateCacheFrequency;
+ this.isNamespaceMapped = builder.isNamespaceMapped;
+ this.autoPartitionSeqName = builder.autoPartitionSeqName;
+ this.isAppendOnlySchema = builder.isAppendOnlySchema;
+ this.immutableStorageScheme = builder.immutableStorageScheme;
+ this.qualifierEncodingScheme = builder.qualifierEncodingScheme;
+ this.encodedCQCounter = builder.encodedCQCounter;
+ this.useStatsForParallelization = builder.useStatsForParallelization;
}
// When cloning table, ignore the salt column as it will be added back in the constructor
@@ -267,153 +764,65 @@ public class PTableImpl implements PTable {
return table.getBucketNum() == null ? table.getColumns() : table.getColumns().subList(1, table.getColumns().size());
}
- public static PTableImpl makePTable(PTable table, long timeStamp, List<PTable> indexes) throws SQLException {
- return new PTableImpl(table, table.rowKeyOrderOptimizable(), table.getIndexState(), timeStamp,
- table.getSequenceNumber(), getColumnsToClone(table), table.getDefaultFamilyName(), table.getType(),
- table.getBaseColumnCount(), table.getSchemaName(), table.getTableName(), table.getViewStatement(),
- table.getUpdateCacheFrequency(), table.getTenantId(), indexes);
- }
-
- public static PTable makePTable(PTable index, PName indexName, String viewStatement, long updateCacheFrequency,
- PName tenantId) throws SQLException {
- return Objects.equal(viewStatement, index.getViewStatement()) ? index
- : new PTableImpl(index, index.rowKeyOrderOptimizable(), index.getIndexState(), index.getTimeStamp(),
- index.getSequenceNumber(), index.getColumns(), index.getDefaultFamilyName(), index.getType(),
- index.getBaseColumnCount(), index.getSchemaName(), indexName,
- viewStatement, updateCacheFrequency, tenantId,
- index.getIndexes());
- }
-
- public static PTableImpl makePTable(PTable table, Collection<PColumn> columns) throws SQLException {
- return new PTableImpl(table, table.rowKeyOrderOptimizable(), table.getIndexState(), table.getTimeStamp(),
- table.getSequenceNumber(), columns, table.getDefaultFamilyName(), table.getType(),
- table.getBaseColumnCount(), table.getSchemaName(), table.getTableName(), table.getViewStatement(),
- table.getUpdateCacheFrequency(), table.getTenantId(), table.getIndexes());
+ /**
+ * Get a PTableImpl.Builder from an existing PTable and set the builder columns
+ * @param table Original PTable
+ * @param columns Columns to set in the builder for the new PTable to be constructed
+ * @return PTable builder object based on an existing PTable
+ */
+ public static PTableImpl.Builder builderWithColumns(PTable table, Collection<PColumn> columns) {
+ return builderFromExisting(table).setColumns(columns);
}
/**
- * Used to create a PTable for views or view indexes, the basePTable is for attributes we inherit from the physical table
+ * Get a PTableImpl.Builder from an existing PTable
+ * @param table Original PTable
*/
- public static PTableImpl makePTable(PTable view, PTable baseTable, Collection<PColumn> columns, long timestamp, int baseTableColumnCount, Collection<PColumn> excludedColumns) throws SQLException {
- // if a TableProperty is not valid on a view we set it to the base table value
- // if a TableProperty is valid on a view and is not mutable on a view we set it to the base table value
- // if a TableProperty is valid on a view and is mutable on a view we use the value set on the view
- return new PTableImpl(
- view.getTenantId(), view.getSchemaName(), view.getTableName(), view.getType(), view.getIndexState(), timestamp,
- view.getSequenceNumber(), view.getPKName(), view.getBucketNum(), columns, view.getParentSchemaName(), view.getParentTableName(),
- view.getIndexes(), baseTable.isImmutableRows(), view.getPhysicalNames(), view.getDefaultFamilyName(), view.getViewStatement(),
- baseTable.isWALDisabled(), baseTable.isMultiTenant(), baseTable.getStoreNulls(), view.getViewType(),
- view.getViewIndexType(), view.getViewIndexId(), view.getIndexType(),
- baseTableColumnCount, view.rowKeyOrderOptimizable(), baseTable.getTransactionProvider(), view.getUpdateCacheFrequency(),
- view.getIndexDisableTimestamp(), view.isNamespaceMapped(), baseTable.getAutoPartitionSeqName(), baseTable.isAppendOnlySchema(),
- baseTable.getImmutableStorageScheme(), baseTable.getEncodingScheme(), view.getEncodedCQCounter(), view.useStatsForParallelization(), excludedColumns);
+ private static PTableImpl.Builder builderFromExisting(PTable table) {
+ return new PTableImpl.Builder()
+ .setType(table.getType())
+ .setState(table.getIndexState())
+ .setTimeStamp(table.getTimeStamp())
+ .setIndexDisableTimestamp(table.getIndexDisableTimestamp())
+ .setSequenceNumber(table.getSequenceNumber())
+ .setImmutableRows(table.isImmutableRows())
+ .setViewStatement(table.getViewStatement())
+ .setDisableWAL(table.isWALDisabled())
+ .setMultiTenant(table.isMultiTenant())
+ .setStoreNulls(table.getStoreNulls())
+ .setViewType(table.getViewType())
+ .setViewIndexType(table.getViewIndexType())
+ .setViewIndexId(table.getViewIndexId())
+ .setIndexType(table.getIndexType())
+ .setTransactionProvider(table.getTransactionProvider())
+ .setUpdateCacheFrequency(table.getUpdateCacheFrequency())
+ .setNamespaceMapped(table.isNamespaceMapped())
+ .setAutoPartitionSeqName(table.getAutoPartitionSeqName())
+ .setAppendOnlySchema(table.isAppendOnlySchema())
+ .setImmutableStorageScheme(table.getImmutableStorageScheme() == null ?
+ ImmutableStorageScheme.ONE_CELL_PER_COLUMN : table.getImmutableStorageScheme())
+ .setQualifierEncodingScheme(table.getEncodingScheme() == null ?
+ QualifierEncodingScheme.NON_ENCODED_QUALIFIERS : table.getEncodingScheme())
+ .setBaseColumnCount(table.getBaseColumnCount())
+ .setEncodedCQCounter(table.getEncodedCQCounter())
+ .setUseStatsForParallelization(table.useStatsForParallelization())
+ .setExcludedColumns(table.getExcludedColumns() == null ?
+ ImmutableList.<PColumn>of() : ImmutableList.copyOf(table.getExcludedColumns()))
+ .setTenantId(table.getTenantId())
+ .setSchemaName(table.getSchemaName())
+ .setTableName(table.getTableName())
+ .setPkName(table.getPKName())
+ .setDefaultFamilyName(table.getDefaultFamilyName())
+ .setRowKeyOrderOptimizable(table.rowKeyOrderOptimizable())
+ .setBucketNum(table.getBucketNum())
+ .setIndexes(table.getIndexes() == null ?
+ Collections.<PTable>emptyList() : table.getIndexes())
+ .setParentSchemaName(table.getParentSchemaName())
+ .setParentTableName(table.getParentTableName())
+ .setPhysicalNames(table.getPhysicalNames() == null ?
+ ImmutableList.<PName>of() : ImmutableList.copyOf(table.getPhysicalNames()));
}
-
- public static PTableImpl makePTable(PTable table, PTableType type, Collection<PColumn> columns) throws SQLException {
- return new PTableImpl(table, table.rowKeyOrderOptimizable(), table.getIndexState(), table.getTimeStamp(),
- table.getSequenceNumber(), columns, table.getDefaultFamilyName(), type,
- table.getBaseColumnCount(), table.getSchemaName(), table.getTableName(), table.getViewStatement(),
- table.getUpdateCacheFrequency(), table.getTenantId(), table.getIndexes());
- }
-
- public static PTableImpl makePTable(PTable table, Collection<PColumn> columns, PName defaultFamily)
- throws SQLException {
- return new PTableImpl(table, table.rowKeyOrderOptimizable(), table.getIndexState(), table.getTimeStamp(),
- table.getSequenceNumber(), columns, defaultFamily, table.getType(),
- table.getBaseColumnCount(), table.getSchemaName(), table.getTableName(), table.getViewStatement(),
- table.getUpdateCacheFrequency(), table.getTenantId(), table.getIndexes());
- }
-
- public static PTableImpl makePTable(PTable table, long timeStamp, long sequenceNumber, Collection<PColumn> columns)
- throws SQLException {
- return new PTableImpl(table, table.rowKeyOrderOptimizable(), table.getIndexState(), timeStamp,
- sequenceNumber, columns, table.getDefaultFamilyName(), table.getType(),
- table.getBaseColumnCount(), table.getSchemaName(), table.getTableName(), table.getViewStatement(),
- table.getUpdateCacheFrequency(), table.getTenantId(), table.getIndexes());
- }
-
- public static PTableImpl makePTable(PTable table, PIndexState state) throws SQLException {
- return new PTableImpl(table, table.rowKeyOrderOptimizable(), state, table.getTimeStamp(),
- table.getSequenceNumber(), getColumnsToClone(table), table.getDefaultFamilyName(), table.getType(),
- table.getBaseColumnCount(), table.getSchemaName(), table.getTableName(), table.getViewStatement(),
- table.getUpdateCacheFrequency(), table.getTenantId(), table.getIndexes());
- }
-
- public static PTableImpl makePTable(PTable table, boolean rowKeyOrderOptimizable) throws SQLException {
- return new PTableImpl(table, rowKeyOrderOptimizable, table.getIndexState(), table.getTimeStamp(),
- table.getSequenceNumber(), getColumnsToClone(table), table.getDefaultFamilyName(), table.getType(),
- table.getBaseColumnCount(), table.getSchemaName(), table.getTableName(), table.getViewStatement(),
- table.getUpdateCacheFrequency(), table.getTenantId(), table.getIndexes());
- }
-
- public static PTableImpl makePTable(PName tenantId, PName schemaName, PName tableName, PTableType type,
- PIndexState state, long timeStamp, long sequenceNumber, PName pkName, Integer bucketNum,
- Collection<PColumn> columns, PName dataSchemaName, PName dataTableName, List<PTable> indexes,
- boolean isImmutableRows, List<PName> physicalNames, PName defaultFamilyName, String viewExpression,
- boolean disableWAL, boolean multiTenant, boolean storeNulls, ViewType viewType, PDataType viewIndexType, Long viewIndexId,
- IndexType indexType, boolean rowKeyOrderOptimizable, TransactionFactory.Provider transactionProvider,
- long updateCacheFrequency, long indexDisableTimestamp, boolean isNamespaceMapped,
- String autoPartitionSeqName, boolean isAppendOnlySchema, ImmutableStorageScheme storageScheme,
- QualifierEncodingScheme qualifierEncodingScheme, EncodedCQCounter encodedCQCounter,
- Boolean useStatsForParallelization) throws SQLException {
- return new PTableImpl(tenantId, schemaName, tableName, type, state, timeStamp, sequenceNumber, pkName,
- bucketNum, columns, dataSchemaName, dataTableName, indexes, isImmutableRows, physicalNames,
- defaultFamilyName, viewExpression, disableWAL, multiTenant, storeNulls, viewType, viewIndexType, viewIndexId,
- indexType, QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT, rowKeyOrderOptimizable, transactionProvider,
- updateCacheFrequency, indexDisableTimestamp, isNamespaceMapped, autoPartitionSeqName,
- isAppendOnlySchema, storageScheme, qualifierEncodingScheme, encodedCQCounter,
- useStatsForParallelization, null);
- }
-
- public static PTableImpl makePTable(PName tenantId, PName schemaName, PName tableName, PTableType type,
- PIndexState state, long timeStamp, long sequenceNumber, PName pkName, Integer bucketNum,
- Collection<PColumn> columns, PName dataSchemaName, PName dataTableName, List<PTable> indexes,
- boolean isImmutableRows, List<PName> physicalNames, PName defaultFamilyName, String viewExpression,
- boolean disableWAL, boolean multiTenant, boolean storeNulls, ViewType viewType, PDataType viewIndexType, Long viewIndexId,
- IndexType indexType, boolean rowKeyOrderOptimizable, TransactionFactory.Provider transactionProvider,
- long updateCacheFrequency, int baseColumnCount, long indexDisableTimestamp, boolean isNamespaceMapped,
- String autoPartitionSeqName, boolean isAppendOnlySchema, ImmutableStorageScheme storageScheme,
- QualifierEncodingScheme qualifierEncodingScheme, EncodedCQCounter encodedCQCounter,
- Boolean useStatsForParallelization) throws SQLException {
- return new PTableImpl(tenantId, schemaName, tableName, type, state, timeStamp, sequenceNumber, pkName,
- bucketNum, columns, dataSchemaName, dataTableName, indexes, isImmutableRows, physicalNames,
- defaultFamilyName, viewExpression, disableWAL, multiTenant, storeNulls, viewType,viewIndexType, viewIndexId,
- indexType, baseColumnCount, rowKeyOrderOptimizable, transactionProvider, updateCacheFrequency,
- indexDisableTimestamp, isNamespaceMapped, autoPartitionSeqName, isAppendOnlySchema, storageScheme,
- qualifierEncodingScheme, encodedCQCounter, useStatsForParallelization, null);
- }
-
- private PTableImpl(PTable table, boolean rowKeyOrderOptimizable, PIndexState state, long timeStamp,
- long sequenceNumber, Collection<PColumn> columns, PName defaultFamily, PTableType type,
- int baseTableColumnCount, PName schemaName, PName tableName, String viewStatement,
- long updateCacheFrequency, PName tenantId, List<PTable> indexes) throws SQLException {
- init(tenantId, schemaName, tableName, type, state, timeStamp, sequenceNumber, table.getPKName(),
- table.getBucketNum(), columns, table.getParentSchemaName(), table.getParentTableName(), indexes,
- table.isImmutableRows(), table.getPhysicalNames(), defaultFamily, viewStatement, table.isWALDisabled(),
- table.isMultiTenant(), table.getStoreNulls(), table.getViewType(), table.getViewIndexType(), table.getViewIndexId(),
- table.getIndexType(), baseTableColumnCount, rowKeyOrderOptimizable, table.getTransactionProvider(),
- updateCacheFrequency, table.getIndexDisableTimestamp(), table.isNamespaceMapped(),
- table.getAutoPartitionSeqName(), table.isAppendOnlySchema(), table.getImmutableStorageScheme(),
- table.getEncodingScheme(), table.getEncodedCQCounter(), table.useStatsForParallelization(), null);
- }
-
- private PTableImpl(PName tenantId, PName schemaName, PName tableName, PTableType type, PIndexState state,
- long timeStamp, long sequenceNumber, PName pkName, Integer bucketNum, Collection<PColumn> columns,
- PName parentSchemaName, PName parentTableName, List<PTable> indexes, boolean isImmutableRows,
- List<PName> physicalNames, PName defaultFamilyName, String viewExpression, boolean disableWAL, boolean multiTenant,
- boolean storeNulls, ViewType viewType, PDataType viewIndexType, Long viewIndexId, IndexType indexType,
- int baseColumnCount, boolean rowKeyOrderOptimizable, TransactionFactory.Provider transactionProvider, long updateCacheFrequency,
- long indexDisableTimestamp, boolean isNamespaceMapped, String autoPartitionSeqName, boolean isAppendOnlySchema, ImmutableStorageScheme storageScheme,
- QualifierEncodingScheme qualifierEncodingScheme, EncodedCQCounter encodedCQCounter,
- Boolean useStatsForParallelization, Collection<PColumn> excludedColumns)
- throws SQLException {
- init(tenantId, schemaName, tableName, type, state, timeStamp, sequenceNumber, pkName, bucketNum, columns,
- parentSchemaName, parentTableName, indexes, isImmutableRows, physicalNames, defaultFamilyName,
- viewExpression, disableWAL, multiTenant, storeNulls, viewType, viewIndexType, viewIndexId, indexType, baseColumnCount, rowKeyOrderOptimizable,
- transactionProvider, updateCacheFrequency, indexDisableTimestamp, isNamespaceMapped, autoPartitionSeqName, isAppendOnlySchema, storageScheme,
- qualifierEncodingScheme, encodedCQCounter, useStatsForParallelization, excludedColumns);
- }
-
+
@Override
public long getUpdateCacheFrequency() {
return updateCacheFrequency;
@@ -434,208 +843,14 @@ public class PTableImpl implements PTable {
return viewType;
}
-
@Override
public int getEstimatedSize() {
return estimatedSize;
}
- private void init(PName tenantId, PName schemaName, PName tableName, PTableType type, PIndexState state, long timeStamp, long sequenceNumber,
- PName pkName, Integer bucketNum, Collection<PColumn> columns, PName parentSchemaName, PName parentTableName,
- List<PTable> indexes, boolean isImmutableRows, List<PName> physicalNames, PName defaultFamilyName, String viewExpression, boolean disableWAL,
- boolean multiTenant, boolean storeNulls, ViewType viewType,PDataType viewIndexType, Long viewIndexId,
- IndexType indexType , int baseColumnCount, boolean rowKeyOrderOptimizable, TransactionFactory.Provider transactionProvider, long updateCacheFrequency, long indexDisableTimestamp,
- boolean isNamespaceMapped, String autoPartitionSeqName, boolean isAppendOnlySchema, ImmutableStorageScheme storageScheme, QualifierEncodingScheme qualifierEncodingScheme,
- EncodedCQCounter encodedCQCounter, Boolean useStatsForParallelization, Collection<PColumn> excludedColumns) throws SQLException {
- Preconditions.checkNotNull(schemaName);
- Preconditions.checkArgument(tenantId==null || tenantId.getBytes().length > 0); // tenantId should be null or not empty
- int estimatedSize = SizedUtil.OBJECT_SIZE * 2 + 23 * SizedUtil.POINTER_SIZE + 4 * SizedUtil.INT_SIZE + 2 * SizedUtil.LONG_SIZE + 2 * SizedUtil.INT_OBJECT_SIZE +
- PNameFactory.getEstimatedSize(tenantId) +
- PNameFactory.getEstimatedSize(schemaName) +
- PNameFactory.getEstimatedSize(tableName) +
- PNameFactory.getEstimatedSize(pkName) +
- PNameFactory.getEstimatedSize(parentTableName) +
- PNameFactory.getEstimatedSize(defaultFamilyName);
- this.tenantId = tenantId;
- this.schemaName = schemaName;
- this.tableName = tableName;
- this.name = PNameFactory.newName(SchemaUtil.getTableName(schemaName.getString(), tableName.getString()));
- this.key = new PTableKey(tenantId, name.getString());
- this.type = type;
- this.state = state;
- this.timeStamp = timeStamp;
- this.indexDisableTimestamp = indexDisableTimestamp;
- this.sequenceNumber = sequenceNumber;
- this.pkName = pkName;
- this.isImmutableRows = isImmutableRows;
- this.defaultFamilyName = defaultFamilyName;
- this.viewStatement = viewExpression;
- this.disableWAL = disableWAL;
- this.multiTenant = multiTenant;
- this.storeNulls = storeNulls;
- this.viewType = viewType;
- this.viewIndexType = viewIndexType;
- this.viewIndexId = viewIndexId;
- this.indexType = indexType;
- this.transactionProvider = transactionProvider;
- this.rowKeyOrderOptimizable = rowKeyOrderOptimizable;
- this.updateCacheFrequency = updateCacheFrequency;
- this.isNamespaceMapped = isNamespaceMapped;
- this.autoPartitionSeqName = autoPartitionSeqName;
- this.isAppendOnlySchema = isAppendOnlySchema;
- // null check for backward compatibility and sanity. If any of the two below is null, then it means the table is a non-encoded table.
- this.immutableStorageScheme = storageScheme == null ? ImmutableStorageScheme.ONE_CELL_PER_COLUMN : storageScheme;
- this.qualifierEncodingScheme = qualifierEncodingScheme == null ? QualifierEncodingScheme.NON_ENCODED_QUALIFIERS : qualifierEncodingScheme;
- List<PColumn> pkColumns;
- PColumn[] allColumns;
-
- this.columnsByName = ArrayListMultimap.create(columns.size(), 1);
- this.kvColumnsByQualifiers = Maps.newHashMapWithExpectedSize(columns.size());
- int numPKColumns = 0;
- if (bucketNum != null) {
- // Add salt column to allColumns and pkColumns, but don't add to
- // columnsByName, since it should not be addressable via name.
- allColumns = new PColumn[columns.size()+1];
- allColumns[SALTING_COLUMN.getPosition()] = SALTING_COLUMN;
- pkColumns = Lists.newArrayListWithExpectedSize(columns.size()+1);
- ++numPKColumns;
- } else {
- allColumns = new PColumn[columns.size()];
- pkColumns = Lists.newArrayListWithExpectedSize(columns.size());
- }
- // Must do this as with the new method of storing diffs, we just care about ordinal position
- // relative order and not the true ordinal value itself.
- List<PColumn> sortedColumns = Lists.newArrayList(columns);
- Collections.sort(sortedColumns, new Comparator<PColumn>() {
- @Override
- public int compare(PColumn o1, PColumn o2) {
- return Integer.valueOf(o1.getPosition()).compareTo(o2.getPosition());
- }
- });
-
- int position = 0;
- if (bucketNum != null) {
- position = 1;
- }
- for (PColumn column : sortedColumns) {
- allColumns[position] = column;
- position++;
- PName familyName = column.getFamilyName();
- if (familyName == null) {
- ++numPKColumns;
- }
- String columnName = column.getName().getString();
- if (columnsByName.put(columnName, column)) {
- int count = 0;
- for (PColumn dupColumn : columnsByName.get(columnName)) {
- if (Objects.equal(familyName, dupColumn.getFamilyName())) {
- count++;
- if (count > 1) {
- throw new ColumnAlreadyExistsException(schemaName.getString(), name.getString(), columnName);
- }
- }
- }
- }
- byte[] cq = column.getColumnQualifierBytes();
- String cf = column.getFamilyName() != null ? column.getFamilyName().getString() : null;
- if (cf != null && cq != null) {
- KVColumnFamilyQualifier info = new KVColumnFamilyQualifier(cf, cq);
- if (kvColumnsByQualifiers.get(info) != null) {
- throw new ColumnAlreadyExistsException(schemaName.getString(),
- name.getString(), columnName);
- }
- kvColumnsByQualifiers.put(info, column);
- }
- }
- estimatedSize += SizedUtil.sizeOfMap(allColumns.length, SizedUtil.POINTER_SIZE, SizedUtil.sizeOfArrayList(1)); // for multi-map
-
- this.bucketNum = bucketNum;
- this.allColumns = ImmutableList.copyOf(allColumns);
- estimatedSize += SizedUtil.sizeOfMap(numPKColumns) + SizedUtil.sizeOfMap(allColumns.length);
-
- RowKeySchemaBuilder builder = new RowKeySchemaBuilder(numPKColumns);
- // Two pass so that column order in column families matches overall column order
- // and to ensure that column family order is constant
- int maxExpectedSize = allColumns.length - numPKColumns;
- // Maintain iteration order so that column families are ordered as they are listed
- Map<PName, List<PColumn>> familyMap = Maps.newLinkedHashMap();
- PColumn rowTimestampCol = null;
- for (PColumn column : allColumns) {
- PName familyName = column.getFamilyName();
- if (familyName == null) {
- hasColumnsRequiringUpgrade |=
- ( column.getSortOrder() == SortOrder.DESC
- && (!column.getDataType().isFixedWidth()
- || column.getDataType() == PChar.INSTANCE
- || column.getDataType() == PFloat.INSTANCE
- || column.getDataType() == PDouble.INSTANCE
- || column.getDataType() == PBinary.INSTANCE) )
- || (column.getSortOrder() == SortOrder.ASC && column.getDataType() == PBinary.INSTANCE && column.getMaxLength() != null && column.getMaxLength() > 1);
- pkColumns.add(column);
- if (column.isRowTimestamp()) {
- rowTimestampCol = column;
- }
- }
- if (familyName == null) {
- estimatedSize += column.getEstimatedSize(); // PK columns
- builder.addField(column, column.isNullable(), column.getSortOrder());
- } else {
- List<PColumn> columnsInFamily = familyMap.get(familyName);
- if (columnsInFamily == null) {
- columnsInFamily = Lists.newArrayListWithExpectedSize(maxExpectedSize);
- familyMap.put(familyName, columnsInFamily);
- }
- columnsInFamily.add(column);
- }
- }
- this.pkColumns = ImmutableList.copyOf(pkColumns);
- if (rowTimestampCol != null) {
- this.rowTimestampColPos = this.pkColumns.indexOf(rowTimestampCol);
- } else {
- this.rowTimestampColPos = -1;
- }
-
- builder.rowKeyOrderOptimizable(this.rowKeyOrderOptimizable()); // after hasDescVarLengthColumns is calculated
- this.rowKeySchema = builder.build();
- estimatedSize += rowKeySchema.getEstimatedSize();
- Iterator<Map.Entry<PName,List<PColumn>>> iterator = familyMap.entrySet().iterator();
- PColumnFamily[] families = new PColumnFamily[familyMap.size()];
- ImmutableMap.Builder<String, PColumnFamily> familyByString = ImmutableMap.builder();
- ImmutableSortedMap.Builder<byte[], PColumnFamily> familyByBytes = ImmutableSortedMap
- .orderedBy(Bytes.BYTES_COMPARATOR);
- for (int i = 0; i < families.length; i++) {
- Map.Entry<PName,List<PColumn>> entry = iterator.next();
- PColumnFamily family = new PColumnFamilyImpl(entry.getKey(), entry.getValue());
- families[i] = family;
- familyByString.put(family.getName().getString(), family);
- familyByBytes.put(family.getName().getBytes(), family);
- estimatedSize += family.getEstimatedSize();
- }
- this.families = ImmutableList.copyOf(families);
- this.familyByBytes = familyByBytes.build();
- this.familyByString = familyByString.build();
- estimatedSize += SizedUtil.sizeOfArrayList(families.length);
- estimatedSize += SizedUtil.sizeOfMap(families.length) * 2;
- this.indexes = indexes == null ? Collections.<PTable>emptyList() : indexes;
- for (PTable index : this.indexes) {
- estimatedSize += index.getEstimatedSize();
- }
-
- this.parentSchemaName = parentSchemaName;
- this.parentTableName = parentTableName;
- this.parentName = parentTableName == null ? null : PNameFactory.newName(SchemaUtil.getTableName(
- parentSchemaName!=null ? parentSchemaName.getString() : null, parentTableName.getString()));
- estimatedSize += PNameFactory.getEstimatedSize(this.parentName);
-
- this.physicalNames = physicalNames == null ? ImmutableList.<PName>of() : ImmutableList.copyOf(physicalNames);
- for (PName name : this.physicalNames) {
- estimatedSize += name.getEstimatedSize();
- }
- this.estimatedSize = estimatedSize;
- this.baseColumnCount = baseColumnCount;
- this.encodedCQCounter = encodedCQCounter;
- this.useStatsForParallelization = useStatsForParallelization;
- this.excludedColumns = excludedColumns == null ? ImmutableList.<PColumn>of() : ImmutableList.copyOf(excludedColumns);
+ public static void checkTenantId(PName tenantId) {
+ // tenantId should be null or not empty
+ Preconditions.checkArgument(tenantId == null || tenantId.getBytes().length > 0);
}
@Override
@@ -1340,9 +1555,9 @@ public class PTableImpl implements PTable {
if (table.hasIsNamespaceMapped()) {
isNamespaceMapped = table.getIsNamespaceMapped();
}
- String autoParititonSeqName = null;
+ String autoPartitionSeqName = null;
if (table.hasAutoParititonSeqName()) {
- autoParititonSeqName = table.getAutoParititonSeqName();
+ autoPartitionSeqName = table.getAutoParititonSeqName();
}
boolean isAppendOnlySchema = false;
if (table.hasIsAppendOnlySchema()) {
@@ -1358,7 +1573,7 @@ public class PTableImpl implements PTable {
if (table.hasEncodingScheme()) {
qualifierEncodingScheme = QualifierEncodingScheme.fromSerializedValue(table.getEncodingScheme().toByteArray()[0]);
}
- EncodedCQCounter encodedColumnQualifierCounter = null;
+ EncodedCQCounter encodedColumnQualifierCounter;
if ((!EncodedColumnsUtil.usesEncodedColumnNames(qualifierEncodingScheme) || tableType == PTableType.VIEW)) {
encodedColumnQualifierCounter = PTable.EncodedCQCounter.NULL_COUNTER;
}
@@ -1375,14 +1590,50 @@ public class PTableImpl implements PTable {
useStatsForParallelization = table.getUseStatsForParallelization();
}
try {
- PTableImpl result = new PTableImpl();
- result.init(tenantId, schemaName, tableName, tableType, indexState, timeStamp, sequenceNumber, pkName,
- (bucketNum == NO_SALTING) ? null : bucketNum, columns, parentSchemaName, parentTableName, indexes,
- isImmutableRows, physicalNames, defaultFamilyName, viewStatement, disableWAL,
- multiTenant, storeNulls, viewType, viewIndexType, viewIndexId, indexType, baseColumnCount, rowKeyOrderOptimizable,
- transactionProvider, updateCacheFrequency, indexDisableTimestamp, isNamespaceMapped, autoParititonSeqName,
- isAppendOnlySchema, storageScheme, qualifierEncodingScheme, encodedColumnQualifierCounter, useStatsForParallelization, null);
- return result;
+ return new PTableImpl.Builder()
+ .setType(tableType)
+ .setState(indexState)
+ .setTimeStamp(timeStamp)
+ .setIndexDisableTimestamp(indexDisableTimestamp)
+ .setSequenceNumber(sequenceNumber)
+ .setImmutableRows(isImmutableRows)
+ .setViewStatement(viewStatement)
+ .setDisableWAL(disableWAL)
+ .setMultiTenant(multiTenant)
+ .setStoreNulls(storeNulls)
+ .setViewType(viewType)
+ .setViewIndexType(viewIndexType)
+ .setViewIndexId(viewIndexId)
+ .setIndexType(indexType)
+ .setTransactionProvider(transactionProvider)
+ .setUpdateCacheFrequency(updateCacheFrequency)
+ .setNamespaceMapped(isNamespaceMapped)
+ .setAutoPartitionSeqName(autoPartitionSeqName)
+ .setAppendOnlySchema(isAppendOnlySchema)
+ // null check for backward compatibility and sanity. If any of the two below is null,
+ // then it means the table is a non-encoded table.
+ .setImmutableStorageScheme(storageScheme == null ?
+ ImmutableStorageScheme.ONE_CELL_PER_COLUMN : storageScheme)
+ .setQualifierEncodingScheme(qualifierEncodingScheme == null ?
+ QualifierEncodingScheme.NON_ENCODED_QUALIFIERS : qualifierEncodingScheme)
+ .setBaseColumnCount(baseColumnCount)
+ .setEncodedCQCounter(encodedColumnQualifierCounter)
+ .setUseStatsForParallelization(useStatsForParallelization)
+ .setExcludedColumns(ImmutableList.<PColumn>of())
+ .setTenantId(tenantId)
+ .setSchemaName(schemaName)
+ .setTableName(tableName)
+ .setPkName(pkName)
+ .setDefaultFamilyName(defaultFamilyName)
+ .setRowKeyOrderOptimizable(rowKeyOrderOptimizable)
+ .setBucketNum((bucketNum == NO_SALTING) ? null : bucketNum)
+ .setIndexes(indexes == null ? Collections.<PTable>emptyList() : indexes)
+ .setParentSchemaName(parentSchemaName)
+ .setParentTableName(parentTableName)
+ .setPhysicalNames(physicalNames == null ?
+ ImmutableList.<PName>of() : ImmutableList.copyOf(physicalNames))
+ .setColumns(columns)
+ .build();
} catch (SQLException e) {
throw new RuntimeException(e); // Impossible
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6083ae5/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java
index 37cae22..b40c0b8 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/TableRef.java
@@ -17,6 +17,8 @@
*/
package org.apache.phoenix.schema;
+import java.sql.SQLException;
+import java.util.Collections;
import java.util.Objects;
import org.apache.phoenix.compile.TupleProjectionCompiler;
@@ -28,7 +30,7 @@ import org.apache.phoenix.util.SchemaUtil;
public class TableRef {
- public static final TableRef EMPTY_TABLE_REF = new TableRef(new PTableImpl());
+ public static final TableRef EMPTY_TABLE_REF = createEmptyTableRef();
private PTable table;
private long upperBoundTimeStamp;
@@ -37,6 +39,19 @@ public class TableRef {
private final boolean hasDynamicCols;
private final long currentTime;
+ private static TableRef createEmptyTableRef() {
+ try {
+ return new TableRef(new PTableImpl.Builder()
+ .setIndexes(Collections.<PTable>emptyList())
+ .setPhysicalNames(Collections.<PName>emptyList())
+ .setRowKeySchema(RowKeySchema.EMPTY_SCHEMA)
+ .build());
+ } catch (SQLException e) {
+ // Should never happen
+ return null;
+ }
+ }
+
public TableRef(TableRef tableRef) {
this(tableRef.alias, tableRef.table, tableRef.upperBoundTimeStamp, tableRef.lowerBoundTimeStamp, tableRef.hasDynamicCols);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6083ae5/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java b/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java
index 26caa0d..34f82d2 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/execute/CorrelatePlanTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.phoenix.execute;
+import static org.apache.phoenix.query.QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT;
import static org.apache.phoenix.query.QueryConstants.VALUE_COLUMN_FAMILY;
import static org.apache.phoenix.util.PhoenixRuntime.CONNECTIONLESS;
import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL;
@@ -30,6 +31,7 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import com.google.common.collect.ImmutableList;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Scan;
@@ -258,11 +260,31 @@ public class CorrelatePlanTest {
i, expr.getSortOrder(), null, null, false, name, false, false, colName.getBytes(), HConstants.LATEST_TIMESTAMP));
}
try {
- PTable pTable = PTableImpl.makePTable(null, PName.EMPTY_NAME, PName.EMPTY_NAME,
- PTableType.SUBQUERY, null, MetaDataProtocol.MIN_TABLE_TIMESTAMP, PTable.INITIAL_SEQ_NUM,
- null, null, columns, null, null, Collections.<PTable>emptyList(),
- false, Collections.<PName>emptyList(), null, null, false, false, false, null, null,
- null, null, true, null, 0, 0L, Boolean.FALSE, null, false, ImmutableStorageScheme.ONE_CELL_PER_COLUMN, QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, EncodedCQCounter.NULL_COUNTER, true);
+ PTable pTable = new PTableImpl.Builder()
+ .setType(PTableType.SUBQUERY)
+ .setTimeStamp(MetaDataProtocol.MIN_TABLE_TIMESTAMP)
+ .setIndexDisableTimestamp(0L)
+ .setSequenceNumber(PTable.INITIAL_SEQ_NUM)
+ .setImmutableRows(false)
+ .setDisableWAL(false)
+ .setMultiTenant(false)
+ .setStoreNulls(false)
+ .setUpdateCacheFrequency(0)
+ .setNamespaceMapped(Boolean.FALSE)
+ .setAppendOnlySchema(false)
+ .setImmutableStorageScheme(ImmutableStorageScheme.ONE_CELL_PER_COLUMN)
+ .setQualifierEncodingScheme(QualifierEncodingScheme.NON_ENCODED_QUALIFIERS)
+ .setBaseColumnCount(BASE_TABLE_BASE_COLUMN_COUNT)
+ .setEncodedCQCounter(EncodedCQCounter.NULL_COUNTER)
+ .setUseStatsForParallelization(true)
+ .setExcludedColumns(ImmutableList.<PColumn>of())
+ .setSchemaName(PName.EMPTY_NAME)
+ .setTableName(PName.EMPTY_NAME)
+ .setRowKeyOrderOptimizable(true)
+ .setIndexes(Collections.<PTable>emptyList())
+ .setPhysicalNames(ImmutableList.<PName>of())
+ .setColumns(columns)
+ .build();
TableRef sourceTable = new TableRef(pTable);
List<ColumnRef> sourceColumnRefs = Lists.<ColumnRef> newArrayList();
for (PColumn column : sourceTable.getTable().getColumns()) {
http://git-wip-us.apache.org/repos/asf/phoenix/blob/d6083ae5/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java b/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java
index f400d0b..110bd26 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/execute/LiteralResultIteratorPlanTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.phoenix.execute;
+import static org.apache.phoenix.query.QueryConstants.BASE_TABLE_BASE_COLUMN_COUNT;
import static org.apache.phoenix.query.QueryConstants.VALUE_COLUMN_FAMILY;
import static org.apache.phoenix.util.PhoenixRuntime.CONNECTIONLESS;
import static org.apache.phoenix.util.PhoenixRuntime.JDBC_PROTOCOL;
@@ -30,6 +31,7 @@ import java.sql.SQLException;
import java.util.Collections;
import java.util.List;
+import com.google.common.collect.ImmutableList;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.KeyValue;
import org.apache.hadoop.hbase.client.Scan;
@@ -181,12 +183,33 @@ public class LiteralResultIteratorPlanTest {
HConstants.LATEST_TIMESTAMP));
}
try {
- PTable pTable = PTableImpl.makePTable(null, PName.EMPTY_NAME, PName.EMPTY_NAME, PTableType.SUBQUERY, null,
- MetaDataProtocol.MIN_TABLE_TIMESTAMP, PTable.INITIAL_SEQ_NUM, null, null, columns, null, null,
- Collections.<PTable> emptyList(), false, Collections.<PName> emptyList(), null, null, false, false,
- false, null, null, null, null, true, null, 0, 0L, false, null, false, ImmutableStorageScheme.ONE_CELL_PER_COLUMN, QualifierEncodingScheme.NON_ENCODED_QUALIFIERS, EncodedCQCounter.NULL_COUNTER, true);
+ PTable pTable = new PTableImpl.Builder()
+ .setType(PTableType.SUBQUERY)
+ .setTimeStamp(MetaDataProtocol.MIN_TABLE_TIMESTAMP)
+ .setIndexDisableTimestamp(0L)
+ .setSequenceNumber(PTable.INITIAL_SEQ_NUM)
+ .setImmutableRows(false)
+ .setDisableWAL(false)
+ .setMultiTenant(false)
+ .setStoreNulls(false)
+ .setUpdateCacheFrequency(0)
+ .setNamespaceMapped(false)
+ .setAppendOnlySchema(false)
+ .setImmutableStorageScheme(ImmutableStorageScheme.ONE_CELL_PER_COLUMN)
+ .setQualifierEncodingScheme(QualifierEncodingScheme.NON_ENCODED_QUALIFIERS)
+ .setBaseColumnCount(BASE_TABLE_BASE_COLUMN_COUNT)
+ .setEncodedCQCounter(EncodedCQCounter.NULL_COUNTER)
+ .setUseStatsForParallelization(true)
+ .setExcludedColumns(ImmutableList.<PColumn>of())
+ .setSchemaName(PName.EMPTY_NAME)
+ .setTableName(PName.EMPTY_NAME)
+ .setRowKeyOrderOptimizable(true)
+ .setIndexes(Collections.<PTable>emptyList())
+ .setPhysicalNames(ImmutableList.<PName>of())
+ .setColumns(columns)
+ .build();
TableRef sourceTable = new TableRef(pTable);
- List<ColumnRef> sourceColumnRefs = Lists.<ColumnRef> newArrayList();
+ List<ColumnRef> sourceColumnRefs = Lists.newArrayList();
for (PColumn column : sourceTable.getTable().getColumns()) {
sourceColumnRefs.add(new ColumnRef(sourceTable, column.getPosition()));
}
[20/22] phoenix git commit: PHOENIX-5013 Increase timeout for Tephra
discovery service
Posted by ja...@apache.org.
PHOENIX-5013 Increase timeout for Tephra discovery service
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/169270da
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/169270da
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/169270da
Branch: refs/heads/omid2
Commit: 169270daa894628f3dacf2957ef6cfc8a99d2f6a
Parents: d1efd66
Author: James Taylor <ja...@apache.org>
Authored: Sat Nov 10 11:07:02 2018 -0800
Committer: James Taylor <ja...@apache.org>
Committed: Sat Nov 10 11:07:02 2018 -0800
----------------------------------------------------------------------
.../apache/phoenix/query/QueryServicesTestImpl.java | 6 +++---
.../test/java/org/apache/phoenix/util/TestUtil.java | 14 ++++++++++++++
2 files changed, 17 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/169270da/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java b/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
index 841abb6..49fb8e8 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/query/QueryServicesTestImpl.java
@@ -25,8 +25,8 @@ import org.apache.hadoop.hbase.regionserver.wal.IndexedWALEditCodec;
import org.apache.phoenix.transaction.OmidTransactionProvider;
import org.apache.phoenix.util.PhoenixRuntime;
import org.apache.phoenix.util.ReadOnlyProps;
+import org.apache.phoenix.util.TestUtil;
import org.apache.tephra.TxConstants;
-import org.apache.twill.internal.utils.Networks;
/**
@@ -129,12 +129,12 @@ public final class QueryServicesTestImpl extends BaseQueryServicesImpl {
.set(TxConstants.Manager.CFG_DO_PERSIST, false)
.set(TxConstants.Service.CFG_DATA_TX_CLIENT_RETRY_STRATEGY, "n-times")
.set(TxConstants.Service.CFG_DATA_TX_CLIENT_ATTEMPTS, 1)
- .set(TxConstants.Service.CFG_DATA_TX_BIND_PORT, Networks.getRandomPort())
+ .set(TxConstants.Service.CFG_DATA_TX_BIND_PORT, TestUtil.getRandomPort())
.set(TxConstants.Manager.CFG_TX_SNAPSHOT_DIR, Files.createTempDir().getAbsolutePath())
.set(TxConstants.Manager.CFG_TX_TIMEOUT, DEFAULT_TXN_TIMEOUT_SECONDS)
.set(TxConstants.Manager.CFG_TX_SNAPSHOT_INTERVAL, 5L)
// setup default test configs for Omid
- .set(OmidTransactionProvider.OMID_TSO_PORT, Networks.getRandomPort())
+ .set(OmidTransactionProvider.OMID_TSO_PORT, TestUtil.getRandomPort())
;
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/169270da/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
index f0a26b9..f3faa0c 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/TestUtil.java
@@ -36,6 +36,7 @@ import static org.junit.Assert.fail;
import java.io.File;
import java.io.IOException;
import java.math.BigDecimal;
+import java.net.ServerSocket;
import java.sql.Connection;
import java.sql.Date;
import java.sql.DriverManager;
@@ -1105,4 +1106,17 @@ public class TestUtil {
}
return filteredData;
}
+
+ /**
+ * Find a random free port in localhost for binding.
+ * @return A port number or -1 for failure.
+ */
+ public static int getRandomPort() {
+ try (ServerSocket socket = new ServerSocket(0)) {
+ socket.setReuseAddress(true);
+ return socket.getLocalPort();
+ } catch (IOException e) {
+ return -1;
+ }
+ }
}
[18/22] phoenix git commit: Revert "PHOENIX-4971 Drop index will
execute successfully using Incorrect name of parent tables"
Posted by ja...@apache.org.
Revert "PHOENIX-4971 Drop index will execute successfully using Incorrect name of parent tables"
This reverts commit 527098ed621d62ba18fc19099ec67d8b29e027f4.
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/fd7ae940
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/fd7ae940
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/fd7ae940
Branch: refs/heads/omid2
Commit: fd7ae94065ae33f1bb20bca60b7756595f8b10b9
Parents: d6083ae
Author: Thomas D'Silva <td...@apache.org>
Authored: Wed Nov 7 11:10:05 2018 -0800
Committer: Thomas D'Silva <td...@apache.org>
Committed: Wed Nov 7 11:10:05 2018 -0800
----------------------------------------------------------------------
.../phoenix/end2end/index/DropMetadataIT.java | 23 +-------------------
.../phoenix/exception/SQLExceptionCode.java | 2 --
.../apache/phoenix/schema/MetaDataClient.java | 15 -------------
3 files changed, 1 insertion(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/fd7ae940/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java
index 3c670c5..b92ed8d 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java
@@ -18,12 +18,9 @@
package org.apache.phoenix.end2end.index;
import static org.apache.phoenix.util.TestUtil.HBASE_NATIVE_SCHEMA_NAME;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
import java.sql.Connection;
import java.sql.DriverManager;
-import java.sql.SQLException;
import java.util.Properties;
import org.apache.hadoop.hbase.HColumnDescriptor;
@@ -32,7 +29,6 @@ import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.end2end.ParallelStatsDisabledIT;
-import org.apache.phoenix.exception.SQLExceptionCode;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.util.PropertiesUtil;
@@ -60,24 +56,7 @@ public class DropMetadataIT extends ParallelStatsDisabledIT {
String url = QueryUtil.getConnectionUrl(props, config, PRINCIPAL);
return DriverManager.getConnection(url, props);
}
-
- @Test
- public void testDropIndexTableHasSameNameWithDataTable() {
- String tableName = generateUniqueName();
- String indexName = "IDX_" + tableName;
- try (Connection conn = DriverManager.getConnection(getUrl())) {
- String createTable = "CREATE TABLE " + tableName + " (id varchar not null primary key, col integer)";
- conn.createStatement().execute(createTable);
- String createIndex = "CREATE INDEX " + indexName + " on " + tableName + "(col)";
- conn.createStatement().execute(createIndex);
- String dropIndex = "DROP INDEX " + indexName + " on " + indexName;
- conn.createStatement().execute(dropIndex);
- fail("should not execute successfully");
- } catch (SQLException e) {
- assertTrue(SQLExceptionCode.PARENT_TABLE_NOT_FOUND.getErrorCode() == e.getErrorCode());
- }
- }
-
+
@Test
public void testDropViewKeepsHTable() throws Exception {
Connection conn = getConnection();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/fd7ae940/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
index 5bffed5..d557714 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
@@ -185,8 +185,6 @@ public enum SQLExceptionCode {
INVALID_REPLAY_AT(533, "42910", "Value of REPLAY_AT cannot be less than zero."),
UNEQUAL_SCN_AND_BUILD_INDEX_AT(534, "42911", "If both specified, values of CURRENT_SCN and BUILD_INDEX_AT must be equal."),
ONLY_INDEX_UPDATABLE_AT_SCN(535, "42912", "Only an index may be updated when the BUILD_INDEX_AT property is specified"),
- PARENT_TABLE_NOT_FOUND(536, "42913", "Can't drop the index because the parent table in the DROP statement is incorrect."),
-
/**
* HBase and Phoenix specific implementation defined sub-classes.
* Column family related exceptions.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/fd7ae940/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index aae9fc4..3b68247 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -23,7 +23,6 @@ import static org.apache.hadoop.hbase.HColumnDescriptor.TTL;
import static org.apache.phoenix.coprocessor.BaseScannerRegionObserver.ANALYZE_TABLE;
import static org.apache.phoenix.coprocessor.BaseScannerRegionObserver.RUN_UPDATE_STATS_ASYNC_ATTRIB;
import static org.apache.phoenix.exception.SQLExceptionCode.INSUFFICIENT_MULTI_TENANT_COLUMNS;
-import static org.apache.phoenix.exception.SQLExceptionCode.PARENT_TABLE_NOT_FOUND;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.APPEND_ONLY_SCHEMA;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ARG_POSITION;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ARRAY_SIZE;
@@ -3125,20 +3124,6 @@ public class MetaDataClient {
}
MutationState dropTable(String schemaName, String tableName, String parentTableName, PTableType tableType,
boolean ifExists, boolean cascade, boolean skipAddingParentColumns) throws SQLException {
- // Checking the parent table whether exists
- String fullTableName = SchemaUtil.getTableName(schemaName, tableName);
- try {
- PTable ptable = connection.getTable(new PTableKey(connection.getTenantId(), fullTableName));
- if (parentTableName != null &&!parentTableName.equals(ptable.getParentTableName().getString())) {
- throw new SQLExceptionInfo.Builder(PARENT_TABLE_NOT_FOUND)
- .setSchemaName(schemaName).setTableName(tableName).build().buildException();
- }
- } catch (TableNotFoundException e) {
- if (!ifExists) {
- throw e;
- }
- }
-
connection.rollback();
boolean wasAutoCommit = connection.getAutoCommit();
PName tenantId = connection.getTenantId();
[03/22] phoenix git commit: PHOENIX-4971 Drop index will execute
successfully using Incorrect name of parent tables
Posted by ja...@apache.org.
PHOENIX-4971 Drop index will execute successfully using Incorrect name of parent tables
Signed-off-by: Geoffrey Jacoby <gj...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/527098ed
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/527098ed
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/527098ed
Branch: refs/heads/omid2
Commit: 527098ed621d62ba18fc19099ec67d8b29e027f4
Parents: 3903ad7
Author: Jaanai <cl...@gmail.com>
Authored: Sun Oct 28 19:10:51 2018 +0800
Committer: Geoffrey Jacoby <gj...@apache.org>
Committed: Sun Oct 28 15:28:39 2018 -0700
----------------------------------------------------------------------
.../phoenix/end2end/index/DropMetadataIT.java | 23 +++++++++++++++++++-
.../phoenix/exception/SQLExceptionCode.java | 2 ++
.../apache/phoenix/schema/MetaDataClient.java | 15 +++++++++++++
3 files changed, 39 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/527098ed/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java
index b92ed8d..3c670c5 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/DropMetadataIT.java
@@ -18,9 +18,12 @@
package org.apache.phoenix.end2end.index;
import static org.apache.phoenix.util.TestUtil.HBASE_NATIVE_SCHEMA_NAME;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import java.sql.Connection;
import java.sql.DriverManager;
+import java.sql.SQLException;
import java.util.Properties;
import org.apache.hadoop.hbase.HColumnDescriptor;
@@ -29,6 +32,7 @@ import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.phoenix.end2end.ParallelStatsDisabledIT;
+import org.apache.phoenix.exception.SQLExceptionCode;
import org.apache.phoenix.jdbc.PhoenixConnection;
import org.apache.phoenix.query.QueryServices;
import org.apache.phoenix.util.PropertiesUtil;
@@ -56,7 +60,24 @@ public class DropMetadataIT extends ParallelStatsDisabledIT {
String url = QueryUtil.getConnectionUrl(props, config, PRINCIPAL);
return DriverManager.getConnection(url, props);
}
-
+
+ @Test
+ public void testDropIndexTableHasSameNameWithDataTable() {
+ String tableName = generateUniqueName();
+ String indexName = "IDX_" + tableName;
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ String createTable = "CREATE TABLE " + tableName + " (id varchar not null primary key, col integer)";
+ conn.createStatement().execute(createTable);
+ String createIndex = "CREATE INDEX " + indexName + " on " + tableName + "(col)";
+ conn.createStatement().execute(createIndex);
+ String dropIndex = "DROP INDEX " + indexName + " on " + indexName;
+ conn.createStatement().execute(dropIndex);
+ fail("should not execute successfully");
+ } catch (SQLException e) {
+ assertTrue(SQLExceptionCode.PARENT_TABLE_NOT_FOUND.getErrorCode() == e.getErrorCode());
+ }
+ }
+
@Test
public void testDropViewKeepsHTable() throws Exception {
Connection conn = getConnection();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/527098ed/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
index d84857d..6696521 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/exception/SQLExceptionCode.java
@@ -184,6 +184,8 @@ public enum SQLExceptionCode {
INVALID_REPLAY_AT(533, "42910", "Value of REPLAY_AT cannot be less than zero."),
UNEQUAL_SCN_AND_BUILD_INDEX_AT(534, "42911", "If both specified, values of CURRENT_SCN and BUILD_INDEX_AT must be equal."),
ONLY_INDEX_UPDATABLE_AT_SCN(535, "42912", "Only an index may be updated when the BUILD_INDEX_AT property is specified"),
+ PARENT_TABLE_NOT_FOUND(536, "42913", "Can't drop the index because the parent table in the DROP statement is incorrect."),
+
/**
* HBase and Phoenix specific implementation defined sub-classes.
* Column family related exceptions.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/527098ed/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
index 66e8baa..6563c40 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataClient.java
@@ -23,6 +23,7 @@ import static org.apache.hadoop.hbase.HColumnDescriptor.TTL;
import static org.apache.phoenix.coprocessor.BaseScannerRegionObserver.ANALYZE_TABLE;
import static org.apache.phoenix.coprocessor.BaseScannerRegionObserver.RUN_UPDATE_STATS_ASYNC_ATTRIB;
import static org.apache.phoenix.exception.SQLExceptionCode.INSUFFICIENT_MULTI_TENANT_COLUMNS;
+import static org.apache.phoenix.exception.SQLExceptionCode.PARENT_TABLE_NOT_FOUND;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.APPEND_ONLY_SCHEMA;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ARG_POSITION;
import static org.apache.phoenix.jdbc.PhoenixDatabaseMetaData.ARRAY_SIZE;
@@ -3036,6 +3037,20 @@ public class MetaDataClient {
}
MutationState dropTable(String schemaName, String tableName, String parentTableName, PTableType tableType,
boolean ifExists, boolean cascade, boolean skipAddingParentColumns) throws SQLException {
+ // Checking the parent table whether exists
+ String fullTableName = SchemaUtil.getTableName(schemaName, tableName);
+ try {
+ PTable ptable = connection.getTable(new PTableKey(connection.getTenantId(), fullTableName));
+ if (parentTableName != null &&!parentTableName.equals(ptable.getParentTableName().getString())) {
+ throw new SQLExceptionInfo.Builder(PARENT_TABLE_NOT_FOUND)
+ .setSchemaName(schemaName).setTableName(tableName).build().buildException();
+ }
+ } catch (TableNotFoundException e) {
+ if (!ifExists) {
+ throw e;
+ }
+ }
+
connection.rollback();
boolean wasAutoCommit = connection.getAutoCommit();
PName tenantId = connection.getTenantId();
[08/22] phoenix git commit: PHOENIX-4989 Include disruptor jar in
shaded dependency
Posted by ja...@apache.org.
PHOENIX-4989 Include disruptor jar in shaded dependency
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo
Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/c3ed4f84
Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/c3ed4f84
Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/c3ed4f84
Branch: refs/heads/omid2
Commit: c3ed4f84444f80ea9825f1b6f1db0933ceddcd22
Parents: fc35cce
Author: Aman Poonia <am...@gmail.com>
Authored: Tue Oct 30 13:57:52 2018 -0700
Committer: Vincent Poon <vi...@apache.org>
Committed: Thu Nov 1 17:21:15 2018 -0700
----------------------------------------------------------------------
phoenix-server/pom.xml | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/phoenix/blob/c3ed4f84/phoenix-server/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-server/pom.xml b/phoenix-server/pom.xml
index f5ba7f7..339bf8a 100644
--- a/phoenix-server/pom.xml
+++ b/phoenix-server/pom.xml
@@ -139,6 +139,7 @@
<include>com.ibm.icu:icu4j</include>
<include>com.ibm.icu:icu4j-charset</include>
<include>com.ibm.icu:icu4j-localespi</include>
+ <include>com.lmax:disruptor</include>
</includes>
<excludes>
<exclude>org.apache.phoenix:phoenix-server</exclude>
[11/22] phoenix git commit: PHOENIX-4981 Add tests for ORDER BY,
GROUP BY and salted tables using phoenix-spark
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/SaltedTableIT.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/SaltedTableIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/SaltedTableIT.java
index c9168f1..69c9869 100644
--- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/SaltedTableIT.java
+++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/salted/SaltedTableIT.java
@@ -37,104 +37,18 @@ import org.apache.phoenix.util.QueryUtil;
import org.apache.phoenix.util.SchemaUtil;
import org.junit.Test;
-
/**
* Tests for table with transparent salting.
*/
-public class SaltedTableIT extends ParallelStatsDisabledIT {
-
- private static String getUniqueTableName() {
- return SchemaUtil.getTableName(generateUniqueName(), generateUniqueName());
- }
-
- private static String initTableValues(byte[][] splits) throws Exception {
- String tableName = getUniqueTableName();
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = DriverManager.getConnection(getUrl(), props);
-
- // Rows we inserted:
- // 1ab123abc111
- // 1abc456abc111
- // 1de123abc111
- // 2abc123def222
- // 3abc123ghi333
- // 4abc123jkl444
- try {
- // Upsert with no column specifies.
- ensureTableCreated(getUrl(), tableName, TABLE_WITH_SALTING, splits, null, null);
- String query = "UPSERT INTO " + tableName + " VALUES(?,?,?,?,?)";
- PreparedStatement stmt = conn.prepareStatement(query);
- stmt.setInt(1, 1);
- stmt.setString(2, "ab");
- stmt.setString(3, "123");
- stmt.setString(4, "abc");
- stmt.setInt(5, 111);
- stmt.execute();
- conn.commit();
-
- stmt.setInt(1, 1);
- stmt.setString(2, "abc");
- stmt.setString(3, "456");
- stmt.setString(4, "abc");
- stmt.setInt(5, 111);
- stmt.execute();
- conn.commit();
-
- // Test upsert when statement explicitly specifies the columns to upsert into.
- query = "UPSERT INTO " + tableName +
- " (a_integer, a_string, a_id, b_string, b_integer) " +
- " VALUES(?,?,?,?,?)";
- stmt = conn.prepareStatement(query);
-
- stmt.setInt(1, 1);
- stmt.setString(2, "de");
- stmt.setString(3, "123");
- stmt.setString(4, "abc");
- stmt.setInt(5, 111);
- stmt.execute();
- conn.commit();
-
- stmt.setInt(1, 2);
- stmt.setString(2, "abc");
- stmt.setString(3, "123");
- stmt.setString(4, "def");
- stmt.setInt(5, 222);
- stmt.execute();
- conn.commit();
-
- // Test upsert when order of column is shuffled.
- query = "UPSERT INTO " + tableName +
- " (a_string, a_integer, a_id, b_string, b_integer) " +
- " VALUES(?,?,?,?,?)";
- stmt = conn.prepareStatement(query);
- stmt.setString(1, "abc");
- stmt.setInt(2, 3);
- stmt.setString(3, "123");
- stmt.setString(4, "ghi");
- stmt.setInt(5, 333);
- stmt.execute();
- conn.commit();
-
- stmt.setString(1, "abc");
- stmt.setInt(2, 4);
- stmt.setString(3, "123");
- stmt.setString(4, "jkl");
- stmt.setInt(5, 444);
- stmt.execute();
- conn.commit();
- } finally {
- conn.close();
- }
- return tableName;
- }
+public class SaltedTableIT extends BaseSaltedTableIT {
@Test
public void testTableWithInvalidBucketNumber() throws Exception {
Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
Connection conn = DriverManager.getConnection(getUrl(), props);
try {
- String query = "create table " + getUniqueTableName() + " (a_integer integer not null CONSTRAINT pk PRIMARY KEY (a_integer)) SALT_BUCKETS = 257";
+ String query = "create table " + generateUniqueName() + " (a_integer integer not null CONSTRAINT pk PRIMARY KEY (a_integer)) SALT_BUCKETS = 257";
PreparedStatement stmt = conn.prepareStatement(query);
stmt.execute();
fail("Should have caught exception");
@@ -148,370 +62,12 @@ public class SaltedTableIT extends ParallelStatsDisabledIT {
@Test
public void testTableWithSplit() throws Exception {
try {
- createTestTable(getUrl(), "create table " + getUniqueTableName() + " (a_integer integer not null primary key) SALT_BUCKETS = 4",
+ createTestTable(getUrl(), "create table " + generateUniqueName() + " (a_integer integer not null primary key) SALT_BUCKETS = 4",
new byte[][] {{1}, {2,3}, {2,5}, {3}}, null);
fail("Should have caught exception");
} catch (SQLException e) {
assertTrue(e.getMessage(), e.getMessage().contains("ERROR 1022 (42Y81): Should not specify split points on salted table with default row key order."));
}
}
-
- @Test
- public void testSelectValueNoWhereClause() throws Exception {
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = DriverManager.getConnection(getUrl(), props);
- try {
- String tableName = initTableValues(null);
-
- String query = "SELECT * FROM " + tableName;
- PreparedStatement statement = conn.prepareStatement(query);
- ResultSet rs = statement.executeQuery();
-
- assertTrue(rs.next());
- assertEquals(1, rs.getInt(1));
- assertEquals("ab", rs.getString(2));
- assertEquals("123", rs.getString(3));
- assertEquals("abc", rs.getString(4));
- assertEquals(111, rs.getInt(5));
-
- assertTrue(rs.next());
- assertEquals(1, rs.getInt(1));
- assertEquals("abc", rs.getString(2));
- assertEquals("456", rs.getString(3));
- assertEquals("abc", rs.getString(4));
- assertEquals(111, rs.getInt(5));
-
- assertTrue(rs.next());
- assertEquals(1, rs.getInt(1));
- assertEquals("de", rs.getString(2));
- assertEquals("123", rs.getString(3));
- assertEquals("abc", rs.getString(4));
- assertEquals(111, rs.getInt(5));
-
- assertTrue(rs.next());
- assertEquals(2, rs.getInt(1));
- assertEquals("abc", rs.getString(2));
- assertEquals("123", rs.getString(3));
- assertEquals("def", rs.getString(4));
- assertEquals(222, rs.getInt(5));
-
- assertTrue(rs.next());
- assertEquals(3, rs.getInt(1));
- assertEquals("abc", rs.getString(2));
- assertEquals("123", rs.getString(3));
- assertEquals("ghi", rs.getString(4));
- assertEquals(333, rs.getInt(5));
-
- assertTrue(rs.next());
- assertEquals(4, rs.getInt(1));
- assertEquals("abc", rs.getString(2));
- assertEquals("123", rs.getString(3));
- assertEquals("jkl", rs.getString(4));
- assertEquals(444, rs.getInt(5));
-
- assertFalse(rs.next());
- } finally {
- conn.close();
- }
- }
-
- @Test
- public void testSelectValueWithFullyQualifiedWhereClause() throws Exception {
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = DriverManager.getConnection(getUrl(), props);
- try {
- String tableName = initTableValues(null);
- String query;
- PreparedStatement stmt;
- ResultSet rs;
-
- // Variable length slot with bounded ranges.
- query = "SELECT * FROM " + tableName +
- " WHERE a_integer = 1 AND a_string >= 'ab' AND a_string < 'de' AND a_id = '123'";
- stmt = conn.prepareStatement(query);
- rs = stmt.executeQuery();
- assertTrue(rs.next());
- assertEquals(1, rs.getInt(1));
- assertEquals("ab", rs.getString(2));
- assertEquals("123", rs.getString(3));
- assertEquals("abc", rs.getString(4));
- assertEquals(111, rs.getInt(5));
- assertFalse(rs.next());
-
- // all single slots with one value.
- query = "SELECT * FROM " + tableName +
- " WHERE a_integer = 1 AND a_string = 'ab' AND a_id = '123'";
- stmt = conn.prepareStatement(query);
-
- rs = stmt.executeQuery();
- assertTrue(rs.next());
- assertEquals(1, rs.getInt(1));
- assertEquals("ab", rs.getString(2));
- assertEquals("123", rs.getString(3));
- assertEquals("abc", rs.getString(4));
- assertEquals(111, rs.getInt(5));
- assertFalse(rs.next());
-
- // all single slots with multiple values.
- query = "SELECT * FROM " + tableName +
- " WHERE a_integer in (2, 4) AND a_string = 'abc' AND a_id = '123'";
- stmt = conn.prepareStatement(query);
- rs = stmt.executeQuery();
-
- assertTrue(rs.next());
- assertEquals(2, rs.getInt(1));
- assertEquals("abc", rs.getString(2));
- assertEquals("123", rs.getString(3));
- assertEquals("def", rs.getString(4));
- assertEquals(222, rs.getInt(5));
-
- assertTrue(rs.next());
- assertEquals(4, rs.getInt(1));
- assertEquals("abc", rs.getString(2));
- assertEquals("123", rs.getString(3));
- assertEquals("jkl", rs.getString(4));
- assertEquals(444, rs.getInt(5));
- assertFalse(rs.next());
-
- query = "SELECT a_integer, a_string FROM " + tableName +
- " WHERE a_integer in (1,2,3,4) AND a_string in ('a', 'abc', 'de') AND a_id = '123'";
- stmt = conn.prepareStatement(query);
- rs = stmt.executeQuery();
-
- assertTrue(rs.next());
- assertEquals(1, rs.getInt(1));
- assertEquals("de", rs.getString(2));
-
- assertTrue(rs.next());
- assertEquals(2, rs.getInt(1));
- assertEquals("abc", rs.getString(2));
-
- assertTrue(rs.next());
- assertEquals(3, rs.getInt(1));
- assertEquals("abc", rs.getString(2));
-
- assertTrue(rs.next());
- assertEquals(4, rs.getInt(1));
- assertEquals("abc", rs.getString(2));
- assertFalse(rs.next());
-
- // fixed length slot with bounded ranges.
- query = "SELECT a_string, a_id FROM " + tableName +
- " WHERE a_integer > 1 AND a_integer < 4 AND a_string = 'abc' AND a_id = '123'";
- stmt = conn.prepareStatement(query);
- rs = stmt.executeQuery();
- assertTrue(rs.next());
- assertEquals("abc", rs.getString(1));
- assertEquals("123", rs.getString(2));
-
- assertTrue(rs.next());
- assertEquals("abc", rs.getString(1));
- assertEquals("123", rs.getString(2));
- assertFalse(rs.next());
-
- // fixed length slot with unbound ranges.
- query = "SELECT b_string, b_integer FROM " + tableName +
- " WHERE a_integer > 1 AND a_string = 'abc' AND a_id = '123'";
- stmt = conn.prepareStatement(query);
- rs = stmt.executeQuery();
- assertTrue(rs.next());
- assertEquals("def", rs.getString(1));
- assertEquals(222, rs.getInt(2));
-
- assertTrue(rs.next());
- assertEquals("ghi", rs.getString(1));
- assertEquals(333, rs.getInt(2));
-
- assertTrue(rs.next());
- assertEquals("jkl", rs.getString(1));
- assertEquals(444, rs.getInt(2));
- assertFalse(rs.next());
-
- // Variable length slot with unbounded ranges.
- query = "SELECT * FROM " + tableName +
- " WHERE a_integer = 1 AND a_string > 'ab' AND a_id = '123'";
- stmt = conn.prepareStatement(query);
- rs = stmt.executeQuery();
- assertTrue(rs.next());
- assertEquals(1, rs.getInt(1));
- assertEquals("de", rs.getString(2));
- assertEquals("123", rs.getString(3));
- assertEquals("abc", rs.getString(4));
- assertEquals(111, rs.getInt(5));
- assertFalse(rs.next());
-
- } finally {
- conn.close();
- }
- }
-
- @Test
- public void testSelectValueWithNotFullyQualifiedWhereClause() throws Exception {
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = DriverManager.getConnection(getUrl(), props);
- try {
- String tableName = initTableValues(null);
-
- // Where without fully qualified key, point query.
- String query = "SELECT * FROM " + tableName + " WHERE a_integer = ? AND a_string = ?";
- PreparedStatement stmt = conn.prepareStatement(query);
-
- stmt.setInt(1, 1);
- stmt.setString(2, "abc");
- ResultSet rs = stmt.executeQuery();
- assertTrue(rs.next());
- assertEquals(1, rs.getInt(1));
- assertEquals("abc", rs.getString(2));
- assertEquals("456", rs.getString(3));
- assertEquals("abc", rs.getString(4));
- assertEquals(111, rs.getInt(5));
- assertFalse(rs.next());
-
- // Where without fully qualified key, range query.
- query = "SELECT * FROM " + tableName + " WHERE a_integer >= 2";
- stmt = conn.prepareStatement(query);
- rs = stmt.executeQuery();
- assertTrue(rs.next());
- assertEquals(2, rs.getInt(1));
- assertEquals("abc", rs.getString(2));
- assertEquals("123", rs.getString(3));
- assertEquals("def", rs.getString(4));
- assertEquals(222, rs.getInt(5));
-
- assertTrue(rs.next());
- assertEquals(3, rs.getInt(1));
- assertEquals("abc", rs.getString(2));
- assertEquals("123", rs.getString(3));
- assertEquals("ghi", rs.getString(4));
- assertEquals(333, rs.getInt(5));
-
- assertTrue(rs.next());
- assertEquals(4, rs.getInt(1));
- assertEquals("abc", rs.getString(2));
- assertEquals("123", rs.getString(3));
- assertEquals("jkl", rs.getString(4));
- assertEquals(444, rs.getInt(5));
- assertFalse(rs.next());
-
- // With point query.
- query = "SELECT a_string FROM " + tableName + " WHERE a_string = ?";
- stmt = conn.prepareStatement(query);
- stmt.setString(1, "de");
- rs = stmt.executeQuery();
- assertTrue(rs.next());
- assertEquals("de", rs.getString(1));
- assertFalse(rs.next());
-
- query = "SELECT a_id FROM " + tableName + " WHERE a_id = ?";
- stmt = conn.prepareStatement(query);
- stmt.setString(1, "456");
- rs = stmt.executeQuery();
- assertTrue(rs.next());
- assertEquals("456", rs.getString(1));
- assertFalse(rs.next());
- } finally {
- conn.close();
- }
- }
-
- @Test
- public void testSelectWithGroupBy() throws Exception {
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = DriverManager.getConnection(getUrl(), props);
- try {
- String tableName = initTableValues(null);
-
- String query = "SELECT a_integer FROM " + tableName + " GROUP BY a_integer";
- PreparedStatement stmt = conn.prepareStatement(query);
- ResultSet rs = stmt.executeQuery();
- int count = 0;
- while (rs.next()) {
- count++;
- }
- assertEquals("Group by does not return the right count.", count, 4);
- } finally {
- conn.close();
- }
- }
- @Test
- public void testLimitScan() throws Exception {
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = DriverManager.getConnection(getUrl(), props);
- try {
- String tableName = initTableValues(null);
-
- String query = "SELECT a_integer FROM " + tableName + " WHERE a_string='abc' LIMIT 1";
- PreparedStatement stmt = conn.prepareStatement(query);
- ResultSet rs = stmt.executeQuery();
- assertTrue(rs.next());
- assertEquals(1, rs.getInt(1));
- assertFalse(rs.next());
- } finally {
- conn.close();
- }
- }
-
- @Test
- public void testSelectWithOrderByRowKey() throws Exception {
- Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
- Connection conn = DriverManager.getConnection(getUrl(), props);
- try {
- String tableName = initTableValues(null);
-
- String query = "SELECT * FROM " + tableName + " ORDER BY a_integer, a_string, a_id";
- PreparedStatement statement = conn.prepareStatement(query);
- ResultSet explainPlan = statement.executeQuery("EXPLAIN " + query);
- // Confirm that ORDER BY in row key order will be optimized out for salted table
- assertEquals("CLIENT PARALLEL 4-WAY FULL SCAN OVER " + tableName + "\n" +
- "CLIENT MERGE SORT", QueryUtil.getExplainPlan(explainPlan));
- ResultSet rs = statement.executeQuery();
-
- assertTrue(rs.next());
- assertEquals(1, rs.getInt(1));
- assertEquals("ab", rs.getString(2));
- assertEquals("123", rs.getString(3));
- assertEquals("abc", rs.getString(4));
- assertEquals(111, rs.getInt(5));
-
- assertTrue(rs.next());
- assertEquals(1, rs.getInt(1));
- assertEquals("abc", rs.getString(2));
- assertEquals("456", rs.getString(3));
- assertEquals("abc", rs.getString(4));
- assertEquals(111, rs.getInt(5));
-
- assertTrue(rs.next());
- assertEquals(1, rs.getInt(1));
- assertEquals("de", rs.getString(2));
- assertEquals("123", rs.getString(3));
- assertEquals("abc", rs.getString(4));
- assertEquals(111, rs.getInt(5));
-
- assertTrue(rs.next());
- assertEquals(2, rs.getInt(1));
- assertEquals("abc", rs.getString(2));
- assertEquals("123", rs.getString(3));
- assertEquals("def", rs.getString(4));
- assertEquals(222, rs.getInt(5));
-
- assertTrue(rs.next());
- assertEquals(3, rs.getInt(1));
- assertEquals("abc", rs.getString(2));
- assertEquals("123", rs.getString(3));
- assertEquals("ghi", rs.getString(4));
- assertEquals(333, rs.getInt(5));
-
- assertTrue(rs.next());
- assertEquals(4, rs.getInt(1));
- assertEquals("abc", rs.getString(2));
- assertEquals("123", rs.getString(3));
- assertEquals("jkl", rs.getString(4));
- assertEquals(444, rs.getInt(5));
-
- assertFalse(rs.next());
- } finally {
- conn.close();
- }
- }
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-core/src/main/java/org/apache/phoenix/util/QueryBuilder.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/QueryBuilder.java b/phoenix-core/src/main/java/org/apache/phoenix/util/QueryBuilder.java
new file mode 100644
index 0000000..afce0dd
--- /dev/null
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/QueryBuilder.java
@@ -0,0 +1,211 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.util;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import org.apache.commons.collections.CollectionUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.phoenix.parse.HintNode;
+
+import java.util.Collections;
+import java.util.List;
+
+import static org.apache.phoenix.util.SchemaUtil.getEscapedFullColumnName;
+
+public class QueryBuilder {
+
+ private String fullTableName;
+ // regular columns that are in the select clause
+ private List<String> selectColumns = Collections.emptyList();
+
+ // columns that are required for expressions in the select clause
+ private List<String> selectExpressionColumns = Collections.emptyList();
+ // expression string in the select clause (for eg COL1 || COL2)
+ private String selectExpression;
+ private String whereClause;
+ private String orderByClause;
+ private String groupByClause;
+ private String havingClause;
+ private HintNode.Hint hint;
+ private boolean escapeCols;
+ private boolean distinct;
+ private int limit;
+
+ public String getFullTableName() {
+ return fullTableName;
+ }
+
+ /**
+ * @return column names required to evaluate this select statement
+ */
+ public List<String> getRequiredColumns() {
+ List<String> allColumns = Lists.newArrayList(selectColumns);
+ if (!CollectionUtils.isEmpty(selectExpressionColumns)) {
+ allColumns.addAll(selectExpressionColumns);
+ }
+ return allColumns;
+ }
+
+ public String getWhereClause() {
+ return whereClause;
+ }
+
+ public HintNode.Hint getHint() {
+ return hint;
+ }
+
+ public String getOrderByClause() {
+ return orderByClause;
+ }
+
+ public String getGroupByClause() {
+ return groupByClause;
+ }
+
+ public QueryBuilder setOrderByClause(String orderByClause) {
+ this.orderByClause = orderByClause;
+ return this;
+ }
+
+ public QueryBuilder setFullTableName(String fullTableName) {
+ this.fullTableName = fullTableName;
+ return this;
+ }
+
+ public QueryBuilder setSelectColumns(List<String> columns) {
+ this.selectColumns = columns;
+ return this;
+ }
+
+ public QueryBuilder setWhereClause(String whereClause) {
+ this.whereClause = whereClause;
+ return this;
+ }
+
+ public QueryBuilder setHint(HintNode.Hint hint) {
+ this.hint = hint;
+ return this;
+ }
+
+ public QueryBuilder setEscapeCols(boolean escapeCols) {
+ this.escapeCols = escapeCols;
+ return this;
+ }
+
+ public QueryBuilder setGroupByClause(String groupByClause) {
+ this.groupByClause = groupByClause;
+ return this;
+ }
+
+ public QueryBuilder setHavingClause(String havingClause) {
+ this.havingClause = havingClause;
+ return this;
+ }
+
+ public List<String> getSelectExpressionColumns() {
+ return selectExpressionColumns;
+ }
+
+ public QueryBuilder setSelectExpressionColumns(List<String> selectExpressionColumns) {
+ this.selectExpressionColumns = selectExpressionColumns;
+ return this;
+ }
+
+ public String getSelectExpression() {
+ return selectExpression;
+ }
+
+ public QueryBuilder setSelectExpression(String selectExpression) {
+ this.selectExpression = selectExpression;
+ return this;
+ }
+
+ public QueryBuilder setDistinct(boolean distinct) {
+ this.distinct = distinct;
+ return this;
+ }
+
+ public QueryBuilder setLimit(int limit) {
+ this.limit = limit;
+ return this;
+ }
+
+ public String build() {
+ Preconditions.checkNotNull(fullTableName, "Table name cannot be null");
+ if (CollectionUtils.isEmpty(selectColumns) && StringUtils.isBlank(selectExpression)) {
+ throw new IllegalArgumentException("At least one column or select expression must be provided");
+ }
+ StringBuilder query = new StringBuilder();
+ query.append("SELECT ");
+
+ if (distinct) {
+ query.append(" DISTINCT ");
+ }
+
+ if (hint != null) {
+ final HintNode node = new HintNode(hint.name());
+ String hintStr = node.toString();
+ query.append(hintStr);
+ }
+
+ StringBuilder selectClauseBuilder = new StringBuilder();
+ if (StringUtils.isNotBlank(selectExpression)) {
+ if (selectClauseBuilder.length()!=0) {
+ selectClauseBuilder.append(" , ");
+ }
+ selectClauseBuilder.append(selectExpression);
+ }
+
+ boolean first = true;
+ for (String col : selectColumns) {
+ if (StringUtils.isNotBlank(col)) {
+ if ((first && selectClauseBuilder.length()!=0) || !first) {
+ selectClauseBuilder.append(" , ");
+ }
+ String fullColumnName = col;
+ if (escapeCols) {
+ fullColumnName = getEscapedFullColumnName(col);
+ }
+ selectClauseBuilder.append(fullColumnName);
+ first = false;
+ }
+ }
+
+ query.append(selectClauseBuilder);
+ query.append(" FROM ");
+ query.append(fullTableName);
+ if (StringUtils.isNotBlank(whereClause)) {
+ query.append(" WHERE (").append(whereClause).append(")");
+ }
+ if (StringUtils.isNotBlank(groupByClause)) {
+ query.append(" GROUP BY ").append(groupByClause);
+ }
+ if (StringUtils.isNotBlank(havingClause)) {
+ query.append(" HAVING ").append(havingClause);
+ }
+ if (StringUtils.isNotBlank(orderByClause)) {
+ query.append(" ORDER BY ").append(orderByClause);
+ }
+ if (limit > 0) {
+ query.append(" LIMIT ").append(limit);
+ }
+ return query.toString();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-core/src/main/java/org/apache/phoenix/util/QueryUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/QueryUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/QueryUtil.java
index 94cbfea..4501158 100644
--- a/phoenix-core/src/main/java/org/apache/phoenix/util/QueryUtil.java
+++ b/phoenix-core/src/main/java/org/apache/phoenix/util/QueryUtil.java
@@ -217,45 +217,15 @@ public final class QueryUtil {
*
* @param fullTableName name of the table for which the select statement needs to be created.
* @param columns list of columns to be projected in the select statement.
- * @param conditions The condition clause to be added to the WHERE condition
+ * @param whereClause The condition clause to be added to the WHERE condition
* @param hint hint to use
* @param escapeCols whether to escape the projected columns
* @return Select Query
*/
public static String constructSelectStatement(String fullTableName, List<String> columns,
- final String conditions, Hint hint, boolean escapeCols) {
- Preconditions.checkNotNull(fullTableName, "Table name cannot be null");
- if (columns == null || columns.isEmpty()) {
- throw new IllegalArgumentException("At least one column must be provided");
- }
- StringBuilder query = new StringBuilder();
- query.append("SELECT ");
-
- String hintStr = "";
- if (hint != null) {
- final HintNode node = new HintNode(hint.name());
- hintStr = node.toString();
- }
- query.append(hintStr);
-
- for (String col : columns) {
- if (col != null) {
- String fullColumnName = col;
- if (escapeCols) {
- fullColumnName = getEscapedFullColumnName(col);
- }
- query.append(fullColumnName);
- query.append(",");
- }
- }
- // Remove the trailing comma
- query.setLength(query.length() - 1);
- query.append(" FROM ");
- query.append(fullTableName);
- if (conditions != null && conditions.length() > 0) {
- query.append(" WHERE (").append(conditions).append(")");
- }
- return query.toString();
+ final String whereClause, Hint hint, boolean escapeCols) {
+ return new QueryBuilder().setFullTableName(fullTableName).setSelectColumns(columns)
+ .setWhereClause(whereClause).setHint(hint).setEscapeCols(escapeCols).build();
}
/**
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/index/IndexScrutinyTableOutputTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/index/IndexScrutinyTableOutputTest.java b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/index/IndexScrutinyTableOutputTest.java
index c6bb739..a904bca 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/index/IndexScrutinyTableOutputTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/index/IndexScrutinyTableOutputTest.java
@@ -47,7 +47,7 @@ public class IndexScrutinyTableOutputTest extends BaseIndexTest {
String sqlStr =
IndexScrutinyTableOutput.getSqlQueryAllInvalidRows(conn, columnNames,
SCRUTINY_TIME_MILLIS);
- assertEquals("SELECT \"SOURCE_TABLE\",\"TARGET_TABLE\",\"SCRUTINY_EXECUTE_TIME\",\"SOURCE_ROW_PK_HASH\",\"SOURCE_TS\",\"TARGET_TS\",\"HAS_TARGET_ROW\",\"ID\",\"PK_PART2\",\"NAME\",\"ZIP\",\":ID\",\":PK_PART2\",\"0:NAME\",\"0:ZIP\" FROM PHOENIX_INDEX_SCRUTINY(\"ID\" INTEGER,\"PK_PART2\" TINYINT,\"NAME\" VARCHAR,\"ZIP\" BIGINT,\":ID\" INTEGER,\":PK_PART2\" TINYINT,\"0:NAME\" VARCHAR,\"0:ZIP\" BIGINT) WHERE (\"SOURCE_TABLE\",\"TARGET_TABLE\",\"SCRUTINY_EXECUTE_TIME\") IN (('TEST_SCHEMA.TEST_INDEX_COLUMN_NAMES_UTIL','TEST_SCHEMA.TEST_ICN_INDEX',1502908914193))",
+ assertEquals("SELECT \"SOURCE_TABLE\" , \"TARGET_TABLE\" , \"SCRUTINY_EXECUTE_TIME\" , \"SOURCE_ROW_PK_HASH\" , \"SOURCE_TS\" , \"TARGET_TS\" , \"HAS_TARGET_ROW\" , \"ID\" , \"PK_PART2\" , \"NAME\" , \"ZIP\" , \":ID\" , \":PK_PART2\" , \"0:NAME\" , \"0:ZIP\" FROM PHOENIX_INDEX_SCRUTINY(\"ID\" INTEGER,\"PK_PART2\" TINYINT,\"NAME\" VARCHAR,\"ZIP\" BIGINT,\":ID\" INTEGER,\":PK_PART2\" TINYINT,\"0:NAME\" VARCHAR,\"0:ZIP\" BIGINT) WHERE (\"SOURCE_TABLE\",\"TARGET_TABLE\",\"SCRUTINY_EXECUTE_TIME\") IN (('TEST_SCHEMA.TEST_INDEX_COLUMN_NAMES_UTIL','TEST_SCHEMA.TEST_ICN_INDEX',1502908914193))",
sqlStr);
}
@@ -58,7 +58,7 @@ public class IndexScrutinyTableOutputTest extends BaseIndexTest {
String query =
IndexScrutinyTableOutput.getSqlQueryMissingTargetRows(conn, columnNames,
SCRUTINY_TIME_MILLIS);
- assertEquals("SELECT \"SOURCE_TABLE\",\"TARGET_TABLE\",\"SCRUTINY_EXECUTE_TIME\",\"SOURCE_ROW_PK_HASH\",\"SOURCE_TS\",\"TARGET_TS\",\"HAS_TARGET_ROW\",\"ID\",\"PK_PART2\",\"NAME\",\"ZIP\",\":ID\",\":PK_PART2\",\"0:NAME\",\"0:ZIP\" FROM PHOENIX_INDEX_SCRUTINY(\"ID\" INTEGER,\"PK_PART2\" TINYINT,\"NAME\" VARCHAR,\"ZIP\" BIGINT,\":ID\" INTEGER,\":PK_PART2\" TINYINT,\"0:NAME\" VARCHAR,\"0:ZIP\" BIGINT) WHERE (\"SOURCE_TABLE\",\"TARGET_TABLE\",\"SCRUTINY_EXECUTE_TIME\", \"HAS_TARGET_ROW\") IN (('TEST_SCHEMA.TEST_INDEX_COLUMN_NAMES_UTIL','TEST_SCHEMA.TEST_ICN_INDEX',1502908914193,false))",
+ assertEquals("SELECT \"SOURCE_TABLE\" , \"TARGET_TABLE\" , \"SCRUTINY_EXECUTE_TIME\" , \"SOURCE_ROW_PK_HASH\" , \"SOURCE_TS\" , \"TARGET_TS\" , \"HAS_TARGET_ROW\" , \"ID\" , \"PK_PART2\" , \"NAME\" , \"ZIP\" , \":ID\" , \":PK_PART2\" , \"0:NAME\" , \"0:ZIP\" FROM PHOENIX_INDEX_SCRUTINY(\"ID\" INTEGER,\"PK_PART2\" TINYINT,\"NAME\" VARCHAR,\"ZIP\" BIGINT,\":ID\" INTEGER,\":PK_PART2\" TINYINT,\"0:NAME\" VARCHAR,\"0:ZIP\" BIGINT) WHERE (\"SOURCE_TABLE\",\"TARGET_TABLE\",\"SCRUTINY_EXECUTE_TIME\", \"HAS_TARGET_ROW\") IN (('TEST_SCHEMA.TEST_INDEX_COLUMN_NAMES_UTIL','TEST_SCHEMA.TEST_ICN_INDEX',1502908914193,false))",
query);
}
@@ -69,7 +69,7 @@ public class IndexScrutinyTableOutputTest extends BaseIndexTest {
String query =
IndexScrutinyTableOutput.getSqlQueryBadCoveredColVal(conn, columnNames,
SCRUTINY_TIME_MILLIS);
- assertEquals("SELECT \"SOURCE_TABLE\",\"TARGET_TABLE\",\"SCRUTINY_EXECUTE_TIME\",\"SOURCE_ROW_PK_HASH\",\"SOURCE_TS\",\"TARGET_TS\",\"HAS_TARGET_ROW\",\"ID\",\"PK_PART2\",\"NAME\",\"ZIP\",\":ID\",\":PK_PART2\",\"0:NAME\",\"0:ZIP\" FROM PHOENIX_INDEX_SCRUTINY(\"ID\" INTEGER,\"PK_PART2\" TINYINT,\"NAME\" VARCHAR,\"ZIP\" BIGINT,\":ID\" INTEGER,\":PK_PART2\" TINYINT,\"0:NAME\" VARCHAR,\"0:ZIP\" BIGINT) WHERE (\"SOURCE_TABLE\",\"TARGET_TABLE\",\"SCRUTINY_EXECUTE_TIME\", \"HAS_TARGET_ROW\") IN (('TEST_SCHEMA.TEST_INDEX_COLUMN_NAMES_UTIL','TEST_SCHEMA.TEST_ICN_INDEX',1502908914193,true))",
+ assertEquals("SELECT \"SOURCE_TABLE\" , \"TARGET_TABLE\" , \"SCRUTINY_EXECUTE_TIME\" , \"SOURCE_ROW_PK_HASH\" , \"SOURCE_TS\" , \"TARGET_TS\" , \"HAS_TARGET_ROW\" , \"ID\" , \"PK_PART2\" , \"NAME\" , \"ZIP\" , \":ID\" , \":PK_PART2\" , \"0:NAME\" , \"0:ZIP\" FROM PHOENIX_INDEX_SCRUTINY(\"ID\" INTEGER,\"PK_PART2\" TINYINT,\"NAME\" VARCHAR,\"ZIP\" BIGINT,\":ID\" INTEGER,\":PK_PART2\" TINYINT,\"0:NAME\" VARCHAR,\"0:ZIP\" BIGINT) WHERE (\"SOURCE_TABLE\",\"TARGET_TABLE\",\"SCRUTINY_EXECUTE_TIME\", \"HAS_TARGET_ROW\") IN (('TEST_SCHEMA.TEST_INDEX_COLUMN_NAMES_UTIL','TEST_SCHEMA.TEST_ICN_INDEX',1502908914193,true))",
query);
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtilTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtilTest.java b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtilTest.java
index f864dd5..0c4c004 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtilTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/mapreduce/util/PhoenixConfigurationUtilTest.java
@@ -145,7 +145,7 @@ public class PhoenixConfigurationUtilTest extends BaseConnectionlessQueryTest {
configuration.set(HConstants.ZOOKEEPER_QUORUM, getUrl());
PhoenixConfigurationUtil.setInputTableName(configuration, tableName);
final String selectStatement = PhoenixConfigurationUtil.getSelectStatement(configuration);
- final String expectedSelectStatement = "SELECT \"A_STRING\",\"A_BINARY\",\"0\".\"COL1\" FROM " + tableName ;
+ final String expectedSelectStatement = "SELECT \"A_STRING\" , \"A_BINARY\" , \"0\".\"COL1\" FROM " + tableName ;
assertEquals(expectedSelectStatement, selectStatement);
} finally {
conn.close();
@@ -167,7 +167,7 @@ public class PhoenixConfigurationUtilTest extends BaseConnectionlessQueryTest {
configuration.set(HConstants.ZOOKEEPER_QUORUM, getUrl());
PhoenixConfigurationUtil.setInputTableName(configuration, fullTableName);
final String selectStatement = PhoenixConfigurationUtil.getSelectStatement(configuration);
- final String expectedSelectStatement = "SELECT \"A_STRING\",\"A_BINARY\",\"0\".\"COL1\" FROM " + fullTableName;
+ final String expectedSelectStatement = "SELECT \"A_STRING\" , \"A_BINARY\" , \"0\".\"COL1\" FROM " + fullTableName;
assertEquals(expectedSelectStatement, selectStatement);
} finally {
conn.close();
@@ -209,7 +209,7 @@ public class PhoenixConfigurationUtilTest extends BaseConnectionlessQueryTest {
PhoenixConfigurationUtil.setSchemaType(configuration, SchemaType.QUERY);
PhoenixConfigurationUtil.setInputTableName(configuration, tableName);
final String selectStatement = PhoenixConfigurationUtil.getSelectStatement(configuration);
- final String expectedSelectStatement = "SELECT \"ID\",\"0\".\"VCARRAY\" FROM " + tableName ;
+ final String expectedSelectStatement = "SELECT \"ID\" , \"0\".\"VCARRAY\" FROM " + tableName ;
assertEquals(expectedSelectStatement, selectStatement);
} finally {
conn.close();
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-core/src/test/java/org/apache/phoenix/util/QueryUtilTest.java
----------------------------------------------------------------------
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/util/QueryUtilTest.java b/phoenix-core/src/test/java/org/apache/phoenix/util/QueryUtilTest.java
index 2d094f6..8ee8f97 100644
--- a/phoenix-core/src/test/java/org/apache/phoenix/util/QueryUtilTest.java
+++ b/phoenix-core/src/test/java/org/apache/phoenix/util/QueryUtilTest.java
@@ -66,14 +66,14 @@ public class QueryUtilTest {
@Test
public void testConstructSelectStatement() {
assertEquals(
- "SELECT \"ID\",\"NAME\" FROM MYTAB",
+ "SELECT \"ID\" , \"NAME\" FROM MYTAB",
QueryUtil.constructSelectStatement("MYTAB", ImmutableList.of(ID_COLUMN,NAME_COLUMN),null));
}
@Test
public void testConstructSelectStatementWithSchema() {
assertEquals(
- "SELECT \"ID\",\"NAME\" FROM A.MYTAB",
+ "SELECT \"ID\" , \"NAME\" FROM A.MYTAB",
QueryUtil.constructSelectStatement("A.MYTAB", ImmutableList.of(ID_COLUMN,NAME_COLUMN),null));
}
@@ -83,7 +83,7 @@ public class QueryUtilTest {
final String schemaName = SchemaUtil.getEscapedArgument("a");
final String fullTableName = SchemaUtil.getTableName(schemaName, tableName);
assertEquals(
- "SELECT \"ID\",\"NAME\" FROM \"a\".MYTAB",
+ "SELECT \"ID\" , \"NAME\" FROM \"a\".MYTAB",
QueryUtil.constructSelectStatement(fullTableName, ImmutableList.of(ID_COLUMN,NAME_COLUMN),null));
}
@@ -93,14 +93,14 @@ public class QueryUtilTest {
final String schemaName = SchemaUtil.getEscapedArgument("a");
final String fullTableName = SchemaUtil.getTableName(schemaName, tableName);
assertEquals(
- "SELECT \"ID\",\"NAME\" FROM \"a\".\"mytab\"",
+ "SELECT \"ID\" , \"NAME\" FROM \"a\".\"mytab\"",
QueryUtil.constructSelectStatement(fullTableName, ImmutableList.of(ID_COLUMN,NAME_COLUMN),null));
}
@Test
public void testConstructSelectWithHint() {
assertEquals(
- "SELECT /*+ NO_INDEX */ \"col1\",\"col2\" FROM MYTAB WHERE (\"col2\"=? and \"col3\" is null)",
+ "SELECT /*+ NO_INDEX */ \"col1\" , \"col2\" FROM MYTAB WHERE (\"col2\"=? and \"col3\" is null)",
QueryUtil.constructSelectStatement("MYTAB", Lists.newArrayList("col1", "col2"),
"\"col2\"=? and \"col3\" is null", Hint.NO_INDEX, true));
}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-spark/pom.xml
----------------------------------------------------------------------
diff --git a/phoenix-spark/pom.xml b/phoenix-spark/pom.xml
index 602dfb1..d3979f4 100644
--- a/phoenix-spark/pom.xml
+++ b/phoenix-spark/pom.xml
@@ -488,6 +488,14 @@
<testResources><testResource><directory>src/it/resources</directory></testResource></testResources>
<plugins>
<plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-failsafe-plugin</artifactId>
+ </plugin>
+ <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
</plugin>
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-spark/src/it/java/org/apache/phoenix/spark/AggregateIT.java
----------------------------------------------------------------------
diff --git a/phoenix-spark/src/it/java/org/apache/phoenix/spark/AggregateIT.java b/phoenix-spark/src/it/java/org/apache/phoenix/spark/AggregateIT.java
new file mode 100644
index 0000000..e4b96a3
--- /dev/null
+++ b/phoenix-spark/src/it/java/org/apache/phoenix/spark/AggregateIT.java
@@ -0,0 +1,91 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.spark;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+import org.apache.phoenix.end2end.BaseAggregateIT;
+import org.apache.phoenix.util.QueryBuilder;
+
+public class AggregateIT extends BaseAggregateIT {
+
+ @Override
+ protected ResultSet executeQueryThrowsException(Connection conn, QueryBuilder queryBuilder,
+ String expectedPhoenixExceptionMsg, String expectedSparkExceptionMsg) {
+ ResultSet rs = null;
+ try {
+ rs = executeQuery(conn, queryBuilder);
+ fail();
+ }
+ catch(Exception e) {
+ assertTrue(e.getMessage().contains(expectedSparkExceptionMsg));
+ }
+ return rs;
+ }
+
+ @Override
+ protected ResultSet executeQuery(Connection conn, QueryBuilder queryBuilder) throws SQLException {
+ return SparkUtil.executeQuery(conn, queryBuilder, getUrl(), config);
+ }
+
+ @Override
+ protected void testCountNullInNonEmptyKeyValueCF(int columnEncodedBytes) throws Exception {
+ try (Connection conn = DriverManager.getConnection(getUrl())) {
+ //Type is INT
+ String intTableName=generateUniqueName();
+ String sql="create table " + intTableName + " (mykey integer not null primary key, A.COLA integer, B.COLB integer) "
+ + "IMMUTABLE_ROWS=true, IMMUTABLE_STORAGE_SCHEME = ONE_CELL_PER_COLUMN, COLUMN_ENCODED_BYTES = " + columnEncodedBytes + ", DISABLE_WAL=true";
+
+ conn.createStatement().execute(sql);
+ conn.createStatement().execute("UPSERT INTO "+intTableName+" VALUES (1,1)");
+ conn.createStatement().execute("UPSERT INTO "+intTableName+" VALUES (2,1)");
+ conn.createStatement().execute("UPSERT INTO "+intTableName+" VALUES (3,1,2)");
+ conn.createStatement().execute("UPSERT INTO "+intTableName+" VALUES (4,1)");
+ conn.createStatement().execute("UPSERT INTO "+intTableName+" VALUES (5,1)");
+ conn.commit();
+
+ sql="select count(*) from "+intTableName;
+ QueryBuilder queryBuilder = new QueryBuilder()
+ .setSelectExpression("COUNT(*)")
+ .setFullTableName(intTableName);
+ ResultSet rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals(5, rs.getLong(1));
+
+ sql="select count(*) from "+intTableName + " where b.colb is not null";
+ queryBuilder.setWhereClause("`B.COLB` IS NOT NULL");
+ rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals(1, rs.getLong(1));
+
+ sql="select count(*) from "+intTableName + " where b.colb is null";
+ queryBuilder.setWhereClause("`B.COLB` IS NULL");
+ rs = executeQuery(conn, queryBuilder);
+ assertTrue(rs.next());
+ assertEquals(4, rs.getLong(1));
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-spark/src/it/java/org/apache/phoenix/spark/OrderByIT.java
----------------------------------------------------------------------
diff --git a/phoenix-spark/src/it/java/org/apache/phoenix/spark/OrderByIT.java b/phoenix-spark/src/it/java/org/apache/phoenix/spark/OrderByIT.java
new file mode 100644
index 0000000..bdffaf5
--- /dev/null
+++ b/phoenix-spark/src/it/java/org/apache/phoenix/spark/OrderByIT.java
@@ -0,0 +1,460 @@
+package org.apache.phoenix.spark;
+
+import static org.apache.phoenix.util.TestUtil.TEST_PROPERTIES;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.sql.Connection;
+import java.sql.Date;
+import java.sql.DriverManager;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.phoenix.end2end.BaseOrderByIT;
+import org.apache.phoenix.util.PropertiesUtil;
+import org.apache.phoenix.util.QueryBuilder;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SQLContext;
+import org.junit.Ignore;
+import org.junit.Test;
+
+import com.google.common.collect.Lists;
+
+import scala.Option;
+import scala.collection.JavaConverters;
+
+public class OrderByIT extends BaseOrderByIT {
+
+ @Override
+ protected ResultSet executeQueryThrowsException(Connection conn, QueryBuilder queryBuilder,
+ String expectedPhoenixExceptionMsg, String expectedSparkExceptionMsg) {
+ ResultSet rs = null;
+ try {
+ rs = executeQuery(conn, queryBuilder);
+ fail();
+ }
+ catch(Exception e) {
+ assertTrue(e.getMessage().contains(expectedSparkExceptionMsg));
+ }
+ return rs;
+ }
+
+ @Override
+ protected ResultSet executeQuery(Connection conn, QueryBuilder queryBuilder) throws SQLException {
+ return SparkUtil.executeQuery(conn, queryBuilder, getUrl(), config);
+ }
+
+ @Test
+ public void testOrderByWithJoin() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ conn.setAutoCommit(false);
+ String tableName1 = generateUniqueName();
+ String ddl = "CREATE TABLE " + tableName1 +
+ " (a_string varchar not null, cf1.a integer, cf1.b varchar, col1 integer, cf2.c varchar, cf2.d integer " +
+ " CONSTRAINT pk PRIMARY KEY (a_string))\n";
+ createTestTable(getUrl(), ddl);
+ String dml = "UPSERT INTO " + tableName1 + " VALUES(?,?,?,?,?,?)";
+ PreparedStatement stmt = conn.prepareStatement(dml);
+ stmt.setString(1, "a");
+ stmt.setInt(2, 40);
+ stmt.setString(3, "aa");
+ stmt.setInt(4, 10);
+ stmt.setString(5, "bb");
+ stmt.setInt(6, 20);
+ stmt.execute();
+ stmt.setString(1, "c");
+ stmt.setInt(2, 30);
+ stmt.setString(3, "cc");
+ stmt.setInt(4, 50);
+ stmt.setString(5, "dd");
+ stmt.setInt(6, 60);
+ stmt.execute();
+ stmt.setString(1, "b");
+ stmt.setInt(2, 40);
+ stmt.setString(3, "bb");
+ stmt.setInt(4, 5);
+ stmt.setString(5, "aa");
+ stmt.setInt(6, 80);
+ stmt.execute();
+ conn.commit();
+
+ String tableName2 = generateUniqueName();
+ ddl = "CREATE TABLE " + tableName2 +
+ " (a_string varchar not null, col1 integer" +
+ " CONSTRAINT pk PRIMARY KEY (a_string))\n";
+ createTestTable(getUrl(), ddl);
+
+ dml = "UPSERT INTO " + tableName2 + " VALUES(?, ?)";
+ stmt = conn.prepareStatement(dml);
+ stmt.setString(1, "a");
+ stmt.setInt(2, 40);
+ stmt.execute();
+ stmt.setString(1, "b");
+ stmt.setInt(2, 20);
+ stmt.execute();
+ stmt.setString(1, "c");
+ stmt.setInt(2, 30);
+ stmt.execute();
+ conn.commit();
+
+ // create two PhoenixRDDs using the table names and columns that are required for the JOIN query
+ List<String> table1Columns = Lists.newArrayList("A_STRING", "CF1.A", "CF1.B", "COL1", "CF2.C", "CF2.D");
+ SQLContext sqlContext = SparkUtil.getSqlContext();
+ Dataset phoenixDataSet =
+ new PhoenixRDD(SparkUtil.getSparkContext(), tableName1,
+ JavaConverters.collectionAsScalaIterableConverter(table1Columns)
+ .asScala().toSeq(),
+ Option.apply((String) null), Option.apply(getUrl()), config, false,
+ null).toDataFrame(sqlContext);
+ phoenixDataSet.registerTempTable(tableName1);
+ List<String> table2Columns = Lists.newArrayList("A_STRING", "COL1");
+ phoenixDataSet =
+ new PhoenixRDD(SparkUtil.getSparkContext(), tableName2,
+ JavaConverters.collectionAsScalaIterableConverter(table2Columns)
+ .asScala().toSeq(),
+ Option.apply((String) null), Option.apply(getUrl()), config, false,
+ null).toDataFrame(sqlContext);
+ phoenixDataSet.registerTempTable(tableName2);
+
+ String query =
+ "SELECT T1.* FROM " + tableName1 + " T1 JOIN " + tableName2
+ + " T2 ON T1.A_STRING = T2.A_STRING ORDER BY T1.`CF1.B`";
+ Dataset<Row> dataset =
+ sqlContext.sql(query);
+ List<Row> rows = dataset.collectAsList();
+ ResultSet rs = new SparkResultSet(rows, dataset.columns());
+
+ assertTrue(rs.next());
+ assertEquals("a",rs.getString(1));
+ assertEquals(40,rs.getInt(2));
+ assertEquals("aa",rs.getString(3));
+ assertEquals(10,rs.getInt(4));
+ assertEquals("bb",rs.getString(5));
+ assertEquals(20,rs.getInt(6));
+ assertTrue(rs.next());
+ assertEquals("b",rs.getString(1));
+ assertEquals(40,rs.getInt(2));
+ assertEquals("bb",rs.getString(3));
+ assertEquals(5,rs.getInt(4));
+ assertEquals("aa",rs.getString(5));
+ assertEquals(80,rs.getInt(6));
+ assertTrue(rs.next());
+ assertEquals("c",rs.getString(1));
+ assertEquals(30,rs.getInt(2));
+ assertEquals("cc",rs.getString(3));
+ assertEquals(50,rs.getInt(4));
+ assertEquals("dd",rs.getString(5));
+ assertEquals(60,rs.getInt(6));
+ assertFalse(rs.next());
+
+ query =
+ "select t1.a_string, t2.col1 from " + tableName1 + " t1 join " + tableName2
+ + " t2 on t1.a_string = t2.a_string order by t2.col1";
+ dataset = sqlContext.sql(query);
+ rows = dataset.collectAsList();
+ rs = new SparkResultSet(rows, dataset.columns());
+ assertTrue(rs.next());
+ assertEquals("b",rs.getString(1));
+ assertEquals(20,rs.getInt(2));
+ assertTrue(rs.next());
+ assertEquals("c",rs.getString(1));
+ assertEquals(30,rs.getInt(2));
+ assertTrue(rs.next());
+ assertEquals("a",rs.getString(1));
+ assertEquals(40,rs.getInt(2));
+ assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testOrderByWithUnionAll() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)){
+ conn.setAutoCommit(false);
+ String tableName1 = generateUniqueName();
+ String ddl = "CREATE TABLE " + tableName1 +
+ " (a_string varchar not null, cf1.a integer, cf1.b varchar, col1 integer, cf2.c varchar, cf2.d integer " +
+ " CONSTRAINT pk PRIMARY KEY (a_string))\n";
+ createTestTable(getUrl(), ddl);
+ String dml = "UPSERT INTO " + tableName1 + " VALUES(?,?,?,?,?,?)";
+ PreparedStatement stmt = conn.prepareStatement(dml);
+ stmt.setString(1, "a");
+ stmt.setInt(2, 40);
+ stmt.setString(3, "aa");
+ stmt.setInt(4, 10);
+ stmt.setString(5, "bb");
+ stmt.setInt(6, 20);
+ stmt.execute();
+ stmt.setString(1, "c");
+ stmt.setInt(2, 30);
+ stmt.setString(3, "cc");
+ stmt.setInt(4, 50);
+ stmt.setString(5, "dd");
+ stmt.setInt(6, 60);
+ stmt.execute();
+ stmt.setString(1, "b");
+ stmt.setInt(2, 40);
+ stmt.setString(3, "bb");
+ stmt.setInt(4, 5);
+ stmt.setString(5, "aa");
+ stmt.setInt(6, 80);
+ stmt.execute();
+ conn.commit();
+
+ String tableName2 = generateUniqueName();
+ ddl = "CREATE TABLE " + tableName2 +
+ " (a_string varchar not null, col1 integer" +
+ " CONSTRAINT pk PRIMARY KEY (a_string))\n";
+ createTestTable(getUrl(), ddl);
+
+ dml = "UPSERT INTO " + tableName2 + " VALUES(?, ?)";
+ stmt = conn.prepareStatement(dml);
+ stmt.setString(1, "aa");
+ stmt.setInt(2, 40);
+ stmt.execute();
+ stmt.setString(1, "bb");
+ stmt.setInt(2, 10);
+ stmt.execute();
+ stmt.setString(1, "cc");
+ stmt.setInt(2, 30);
+ stmt.execute();
+ conn.commit();
+
+
+ List<String> table1Columns = Lists.newArrayList("A_STRING", "CF1.A", "CF1.B", "COL1", "CF2.C", "CF2.D");
+ SQLContext sqlContext = SparkUtil.getSqlContext();
+ Dataset phoenixDataSet =
+ new PhoenixRDD(SparkUtil.getSparkContext(), tableName1,
+ JavaConverters.collectionAsScalaIterableConverter(table1Columns)
+ .asScala().toSeq(),
+ Option.apply((String) null), Option.apply(getUrl()), config, false,
+ null).toDataFrame(sqlContext);
+ phoenixDataSet.registerTempTable(tableName1);
+ List<String> table2Columns = Lists.newArrayList("A_STRING", "COL1");
+ phoenixDataSet =
+ new PhoenixRDD(SparkUtil.getSparkContext(), tableName2,
+ JavaConverters.collectionAsScalaIterableConverter(table2Columns)
+ .asScala().toSeq(),
+ Option.apply((String) null), Option.apply(getUrl()), config, false,
+ null).toDataFrame(sqlContext);
+ phoenixDataSet.registerTempTable(tableName2);
+
+ String query =
+ "select a_string, `cf2.d` from " + tableName1 + " union all select * from "
+ + tableName2 + " order by `cf2.d`";
+ Dataset<Row> dataset =
+ sqlContext.sql(query);
+ List<Row> rows = dataset.collectAsList();
+ ResultSet rs = new SparkResultSet(rows, dataset.columns());
+ assertTrue(rs.next());
+ assertEquals("bb",rs.getString(1));
+ assertEquals(10,rs.getInt(2));
+ assertTrue(rs.next());
+ assertEquals("a",rs.getString(1));
+ assertEquals(20,rs.getInt(2));
+ assertTrue(rs.next());
+ assertEquals("cc",rs.getString(1));
+ assertEquals(30,rs.getInt(2));
+ assertTrue(rs.next());
+ assertEquals("aa",rs.getString(1));
+ assertEquals(40,rs.getInt(2));
+ assertTrue(rs.next());
+ assertEquals("c",rs.getString(1));
+ assertEquals(60,rs.getInt(2));
+ assertTrue(rs.next());
+ assertEquals("b",rs.getString(1));
+ assertEquals(80,rs.getInt(2));
+ assertFalse(rs.next());
+ }
+ }
+
+ @Test
+ public void testOrderByWithExpression() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ Connection conn = DriverManager.getConnection(getUrl(), props);
+ conn.setAutoCommit(false);
+
+ try {
+ String tableName = generateUniqueName();
+ String ddl = "CREATE TABLE " + tableName +
+ " (a_string varchar not null, col1 integer, col2 integer, col3 timestamp, col4 varchar" +
+ " CONSTRAINT pk PRIMARY KEY (a_string))\n";
+ createTestTable(getUrl(), ddl);
+
+ Date date = new Date(System.currentTimeMillis());
+ String dml = "UPSERT INTO " + tableName + " VALUES(?, ?, ?, ?, ?)";
+ PreparedStatement stmt = conn.prepareStatement(dml);
+ stmt.setString(1, "a");
+ stmt.setInt(2, 40);
+ stmt.setInt(3, 20);
+ stmt.setDate(4, new Date(date.getTime()));
+ stmt.setString(5, "xxyy");
+ stmt.execute();
+ stmt.setString(1, "b");
+ stmt.setInt(2, 50);
+ stmt.setInt(3, 30);
+ stmt.setDate(4, new Date(date.getTime()-500));
+ stmt.setString(5, "yyzz");
+ stmt.execute();
+ stmt.setString(1, "c");
+ stmt.setInt(2, 60);
+ stmt.setInt(3, 20);
+ stmt.setDate(4, new Date(date.getTime()-300));
+ stmt.setString(5, "ddee");
+ stmt.execute();
+ conn.commit();
+
+ SQLContext sqlContext = SparkUtil.getSqlContext();
+ Dataset phoenixDataSet =
+ new PhoenixRDD(SparkUtil.getSparkContext(), tableName,
+ JavaConverters
+ .collectionAsScalaIterableConverter(
+ Lists.newArrayList("col1", "col2", "col4"))
+ .asScala().toSeq(),
+ Option.apply((String) null), Option.apply(getUrl()), config, false,
+ null).toDataFrame(sqlContext);
+
+ phoenixDataSet.registerTempTable(tableName);
+ Dataset<Row> dataset =
+ sqlContext.sql("SELECT col1+col2, col4, a_string FROM " + tableName
+ + " ORDER BY col1+col2, col4");
+ List<Row> rows = dataset.collectAsList();
+ ResultSet rs = new SparkResultSet(rows, dataset.columns());
+ assertTrue(rs.next());
+ assertEquals("a", rs.getString(3));
+ assertTrue(rs.next());
+ assertEquals("c", rs.getString(3));
+ assertTrue(rs.next());
+ assertEquals("b", rs.getString(3));
+ assertFalse(rs.next());
+ } catch (SQLException e) {
+ } finally {
+ conn.close();
+ }
+ }
+
+ @Test
+ public void testColumnFamily() throws Exception {
+ Properties props = PropertiesUtil.deepCopy(TEST_PROPERTIES);
+ try (Connection conn = DriverManager.getConnection(getUrl(), props)) {
+ conn.setAutoCommit(false);
+ String tableName = generateUniqueName();
+ String ddl = "CREATE TABLE " + tableName +
+ " (a_string varchar not null, cf1.a integer, cf1.b varchar, col1 integer, cf2.c varchar, cf2.d integer, col2 integer" +
+ " CONSTRAINT pk PRIMARY KEY (a_string))\n";
+ createTestTable(getUrl(), ddl);
+ String dml = "UPSERT INTO " + tableName + " VALUES(?,?,?,?,?,?,?)";
+ PreparedStatement stmt = conn.prepareStatement(dml);
+ stmt.setString(1, "a");
+ stmt.setInt(2, 40);
+ stmt.setString(3, "aa");
+ stmt.setInt(4, 10);
+ stmt.setString(5, "bb");
+ stmt.setInt(6, 20);
+ stmt.setInt(7, 1);
+ stmt.execute();
+ stmt.setString(1, "c");
+ stmt.setInt(2, 30);
+ stmt.setString(3, "cc");
+ stmt.setInt(4, 50);
+ stmt.setString(5, "dd");
+ stmt.setInt(6, 60);
+ stmt.setInt(7, 3);
+ stmt.execute();
+ stmt.setString(1, "b");
+ stmt.setInt(2, 40);
+ stmt.setString(3, "bb");
+ stmt.setInt(4, 5);
+ stmt.setString(5, "aa");
+ stmt.setInt(6, 80);
+ stmt.setInt(7, 2);
+ stmt.execute();
+ conn.commit();
+
+
+ List<String> columns =
+ Lists.newArrayList("A_STRING", "CF1.A", "CF1.B", "COL1", "CF2.C", "CF2.D",
+ "COL2");
+
+ SQLContext sqlContext = SparkUtil.getSqlContext();
+ Dataset phoenixDataSet =
+ new PhoenixRDD(SparkUtil.getSparkContext(), tableName,
+ JavaConverters.collectionAsScalaIterableConverter(columns).asScala()
+ .toSeq(),
+ Option.apply((String) null), Option.apply(url), config, false, null)
+ .toDataFrame(sqlContext);
+
+ phoenixDataSet.registerTempTable(tableName);
+ Dataset<Row> dataset =
+ sqlContext.sql("SELECT A_STRING, `CF1.A`, `CF1.B`, COL1, `CF2.C`, `CF2.D`, COL2 from "
+ + tableName + " ORDER BY `CF1.A`,`CF2.C`");
+ List<Row> rows = dataset.collectAsList();
+ ResultSet rs = new SparkResultSet(rows, dataset.columns());
+
+ assertTrue(rs.next());
+ assertEquals("c",rs.getString(1));
+ assertEquals(30,rs.getInt(2));
+ assertEquals("cc",rs.getString(3));
+ assertEquals(50,rs.getInt(4));
+ assertEquals("dd",rs.getString(5));
+ assertEquals(60,rs.getInt(6));
+ assertEquals(3,rs.getInt(7));
+ assertTrue(rs.next());
+ assertEquals("b",rs.getString(1));
+ assertEquals(40,rs.getInt(2));
+ assertEquals("bb",rs.getString(3));
+ assertEquals(5,rs.getInt(4));
+ assertEquals("aa",rs.getString(5));
+ assertEquals(80,rs.getInt(6));
+ assertEquals(2,rs.getInt(7));
+ assertTrue(rs.next());
+ assertEquals("a",rs.getString(1));
+ assertEquals(40,rs.getInt(2));
+ assertEquals("aa",rs.getString(3));
+ assertEquals(10,rs.getInt(4));
+ assertEquals("bb",rs.getString(5));
+ assertEquals(20,rs.getInt(6));
+ assertEquals(1,rs.getInt(7));
+ assertFalse(rs.next());
+
+ dataset =
+ sqlContext.sql("SELECT A_STRING, `CF1.A`, `CF1.B`, COL1, `CF2.C`, `CF2.D`, COL2 from "
+ + tableName + " ORDER BY COL2");
+ rows = dataset.collectAsList();
+ rs = new SparkResultSet(rows, dataset.columns());
+
+ assertTrue(rs.next());
+ assertEquals("a",rs.getString(1));
+ assertEquals(40,rs.getInt(2));
+ assertEquals("aa",rs.getString(3));
+ assertEquals(10,rs.getInt(4));
+ assertEquals("bb",rs.getString(5));
+ assertEquals(20,rs.getInt(6));
+ assertEquals(1,rs.getInt(7));
+ assertTrue(rs.next());
+ assertEquals("b",rs.getString(1));
+ assertEquals(40,rs.getInt(2));
+ assertEquals("bb",rs.getString(3));
+ assertEquals(5,rs.getInt(4));
+ assertEquals("aa",rs.getString(5));
+ assertEquals(80,rs.getInt(6));
+ assertEquals(2,rs.getInt(7));
+ assertTrue(rs.next());
+ assertEquals("c",rs.getString(1));
+ assertEquals(30,rs.getInt(2));
+ assertEquals("cc",rs.getString(3));
+ assertEquals(50,rs.getInt(4));
+ assertEquals("dd",rs.getString(5));
+ assertEquals(60,rs.getInt(6));
+ assertEquals(3,rs.getInt(7));
+ assertFalse(rs.next());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-spark/src/it/java/org/apache/phoenix/spark/SaltedTableIT.java
----------------------------------------------------------------------
diff --git a/phoenix-spark/src/it/java/org/apache/phoenix/spark/SaltedTableIT.java b/phoenix-spark/src/it/java/org/apache/phoenix/spark/SaltedTableIT.java
new file mode 100644
index 0000000..d72acbd
--- /dev/null
+++ b/phoenix-spark/src/it/java/org/apache/phoenix/spark/SaltedTableIT.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.spark;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+
+import org.apache.phoenix.end2end.salted.BaseSaltedTableIT;
+import org.apache.phoenix.util.QueryBuilder;
+
+public class SaltedTableIT extends BaseSaltedTableIT {
+
+ @Override
+ protected ResultSet executeQueryThrowsException(Connection conn, QueryBuilder queryBuilder,
+ String expectedPhoenixExceptionMsg, String expectedSparkExceptionMsg) {
+ ResultSet rs = null;
+ try {
+ rs = executeQuery(conn, queryBuilder);
+ fail();
+ }
+ catch(Exception e) {
+ assertTrue(e.getMessage().contains(expectedSparkExceptionMsg));
+ }
+ return rs;
+ }
+
+ @Override
+ protected ResultSet executeQuery(Connection conn, QueryBuilder queryBuilder) throws SQLException {
+ return SparkUtil.executeQuery(conn, queryBuilder, getUrl(), config);
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-spark/src/it/java/org/apache/phoenix/spark/SparkUtil.java
----------------------------------------------------------------------
diff --git a/phoenix-spark/src/it/java/org/apache/phoenix/spark/SparkUtil.java b/phoenix-spark/src/it/java/org/apache/phoenix/spark/SparkUtil.java
new file mode 100644
index 0000000..6285209
--- /dev/null
+++ b/phoenix-spark/src/it/java/org/apache/phoenix/spark/SparkUtil.java
@@ -0,0 +1,87 @@
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.spark;
+
+import com.google.common.base.Joiner;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.phoenix.jdbc.PhoenixConnection;
+import org.apache.phoenix.query.QueryServices;
+import org.apache.phoenix.util.QueryBuilder;
+import org.apache.spark.SparkContext;
+import org.apache.spark.sql.Dataset;
+import org.apache.spark.sql.Row;
+import org.apache.spark.sql.SQLContext;
+import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.execution.SparkPlan;
+import scala.Option;
+import scala.collection.JavaConverters;
+
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.List;
+
+public class SparkUtil {
+
+ public static final String APP_NAME = "Java Spark Tests";
+ public static final String NUM_EXECUTORS = "local[2]";
+ public static final String UI_SHOW_CONSOLE_PROGRESS = "spark.ui.showConsoleProgress";
+
+ public static SparkContext getSparkContext() {
+ return SparkSession.builder().appName(APP_NAME).master(NUM_EXECUTORS)
+ .config(UI_SHOW_CONSOLE_PROGRESS, false).getOrCreate().sparkContext();
+ }
+
+ public static SQLContext getSqlContext() {
+ return SparkSession.builder().appName(APP_NAME).master(NUM_EXECUTORS)
+ .config(UI_SHOW_CONSOLE_PROGRESS, false).getOrCreate().sqlContext();
+ }
+
+ public static ResultSet executeQuery(Connection conn, QueryBuilder queryBuilder, String url, Configuration config)
+ throws SQLException {
+ SQLContext sqlContext = SparkUtil.getSqlContext();
+
+ boolean forceRowKeyOrder =
+ conn.unwrap(PhoenixConnection.class).getQueryServices().getProps()
+ .getBoolean(QueryServices.FORCE_ROW_KEY_ORDER_ATTRIB, false);
+ // if we are forcing row key order we have to add an ORDER BY
+ // here we assume that the required columns are in the primary key column order
+ String prevOrderBy = queryBuilder.getOrderByClause();
+ if (forceRowKeyOrder && (queryBuilder.getOrderByClause()==null || queryBuilder.getOrderByClause().isEmpty())) {
+ queryBuilder.setOrderByClause(Joiner.on(", ").join(queryBuilder.getRequiredColumns()));
+ }
+
+ // create PhoenixRDD using the table name and columns that are required by the query
+ // since we don't set the predicate filtering is done after rows are returned from spark
+ Dataset phoenixDataSet =
+ new PhoenixRDD(SparkUtil.getSparkContext(), queryBuilder.getFullTableName(),
+ JavaConverters.collectionAsScalaIterableConverter(queryBuilder.getRequiredColumns()).asScala()
+ .toSeq(),
+ Option.apply((String) null), Option.apply(url), config, false,
+ null).toDataFrame(sqlContext);
+
+ phoenixDataSet.registerTempTable(queryBuilder.getFullTableName());
+ Dataset<Row> dataset = sqlContext.sql(queryBuilder.build());
+ SparkPlan plan = dataset.queryExecution().executedPlan();
+ List<Row> rows = dataset.collectAsList();
+ queryBuilder.setOrderByClause(prevOrderBy);
+ ResultSet rs = new SparkResultSet(rows, dataset.columns());
+ return rs;
+ }
+}
http://git-wip-us.apache.org/repos/asf/phoenix/blob/9bfaf183/phoenix-spark/src/it/scala/org/apache/phoenix/spark/PhoenixSparkIT.scala
----------------------------------------------------------------------
diff --git a/phoenix-spark/src/it/scala/org/apache/phoenix/spark/PhoenixSparkIT.scala b/phoenix-spark/src/it/scala/org/apache/phoenix/spark/PhoenixSparkIT.scala
index 4e11acc..d1e38fa 100644
--- a/phoenix-spark/src/it/scala/org/apache/phoenix/spark/PhoenixSparkIT.scala
+++ b/phoenix-spark/src/it/scala/org/apache/phoenix/spark/PhoenixSparkIT.scala
@@ -23,6 +23,7 @@ import org.joda.time.DateTime
import org.apache.spark.{SparkConf, SparkContext}
import scala.collection.mutable.ListBuffer
import org.apache.hadoop.conf.Configuration
+
/**
* Note: If running directly from an IDE, these are the recommended VM parameters:
* -Xmx1536m -XX:MaxPermSize=512m -XX:ReservedCodeCacheSize=512m
@@ -287,11 +288,11 @@ class PhoenixSparkIT extends AbstractPhoenixSparkIT {
val plan = res.queryExecution.sparkPlan
// filters should be pushed into phoenix relation
- assert(plan.toString.contains("PushedFilters: [IsNotNull(COL1), IsNotNull(ID), " +
- "EqualTo(COL1,test_row_1), EqualTo(ID,1)]"))
+ assert(plan.toString.contains("PushedFilters: [*IsNotNull(COL1), *IsNotNull(ID), " +
+ "*EqualTo(COL1,test_row_1), *EqualTo(ID,1)]"))
// spark should run the filters on the rows returned by Phoenix
- assert(!plan.toString.contains("Filter (((isnotnull(COL1#8) && isnotnull(ID#7L)) " +
- "&& (COL1#8 = test_row_1)) && (ID#7L = 1))"))
+ assert(!plan.toString.matches(".*Filter (((isnotnull(COL1.*) && isnotnull(ID.*)) "
+ + " && (COL1.* = test_row_1)) && (ID.* = 1)).*"))
}
test("Can persist a dataframe using 'DataFrame.saveToPhoenix'") {